From bd263b2ed7d3803aeb45260576ce3a26af7f12a0 Mon Sep 17 00:00:00 2001 From: Ethan Dalool Date: Mon, 28 Nov 2016 20:16:16 -0800 Subject: [PATCH] checkpoint --- README.md | 4 + helpers.py | 31 ++++++ requirements.txt | 3 + static/common.css | 32 ++++++ static/common.js | 83 ++++++++++++++ static/favicon.png | Bin 0 -> 2498 bytes static/favicon.svg | 138 +++++++++++++++++++++++ templates/channel.html | 224 +++++++++++++++++++++++++++++++++++++ templates/channels.html | 96 ++++++++++++++++ templates/header.html | 6 + templates/root.html | 28 +++++ ycdl.py | 239 ++++++++++++++++++++++++++++++++++++++++ ycdl_easy.py | 11 ++ ycdl_launch.py | 29 +++++ ycdl_site.py | 195 ++++++++++++++++++++++++++++++++ ytapi.py | 81 ++++++++++++++ ytqueue.py | 23 ++++ 17 files changed, 1223 insertions(+) create mode 100644 README.md create mode 100644 helpers.py create mode 100644 requirements.txt create mode 100644 static/common.css create mode 100644 static/common.js create mode 100644 static/favicon.png create mode 100644 static/favicon.svg create mode 100644 templates/channel.html create mode 100644 templates/channels.html create mode 100644 templates/header.html create mode 100644 templates/root.html create mode 100644 ycdl.py create mode 100644 ycdl_easy.py create mode 100644 ycdl_launch.py create mode 100644 ycdl_site.py create mode 100644 ytapi.py create mode 100644 ytqueue.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..4109cf1 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +Youtube Channel Downloader +========================== + +You are responsible for your own `bot.py` file, containing a variable `YOUTUBE_KEY`. \ No newline at end of file diff --git a/helpers.py b/helpers.py new file mode 100644 index 0000000..19d7ab2 --- /dev/null +++ b/helpers.py @@ -0,0 +1,31 @@ +def chunk_sequence(sequence, chunk_length, allow_incomplete=True): + ''' + Given a sequence, divide it into sequences of length `chunk_length`. + + :param allow_incomplete: If True, allow the final chunk to be shorter if the + given sequence is not an exact multiple of `chunk_length`. + If False, the incomplete chunk will be discarded. + ''' + (complete, leftover) = divmod(len(sequence), chunk_length) + if not allow_incomplete: + leftover = 0 + + chunk_count = complete + min(leftover, 1) + + chunks = [] + for x in range(chunk_count): + left = chunk_length * x + right = left + chunk_length + chunks.append(sequence[left:right]) + + return chunks + +def truthystring(s): + if isinstance(s, (bool, int)) or s is None: + return s + s = s.lower() + if s in {'1', 'true', 't', 'yes', 'y', 'on'}: + return True + if s in {'null', 'none'}: + return None + return False diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..82b6d25 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +flask +gevent +https://github.com/voussoir/else/raw/master/_voussoirkit/voussoirkit.zip diff --git a/static/common.css b/static/common.css new file mode 100644 index 0000000..c25663e --- /dev/null +++ b/static/common.css @@ -0,0 +1,32 @@ +body +{ + display: flex; + flex-direction: column; + background-color:#fff; + margin: 8px; +} +#header +{ + display: flex; + flex-direction: row; + justify-content: center; + align-content: center; + margin-bottom: 4px; +} +.header_element +{ + display: flex; + justify-content: center; + flex: 1; + background-color: rgba(0, 0, 0, 0.1); +} +.header_element:hover +{ + background-color: rgba(0, 0, 0, 0.2); +} +#content_body +{ + flex: 0 0 auto; + display: flex; + flex-direction: row; +} diff --git a/static/common.js b/static/common.js new file mode 100644 index 0000000..a4797c8 --- /dev/null +++ b/static/common.js @@ -0,0 +1,83 @@ +function post_example(key, value, callback) +{ + var url = "/postexample"; + data = new FormData(); + data.append(key, value); + return post(url, data, callback); +} + +function null_callback() +{ + return; +} + +function post(url, data, callback) +{ + var request = new XMLHttpRequest(); + request.answer = null; + request.onreadystatechange = function() + { + if (request.readyState == 4) + { + var text = request.responseText; + if (callback != null) + { + console.log(text); + callback(JSON.parse(text)); + } + } + }; + var asynchronous = true; + request.open("POST", url, asynchronous); + request.send(data); +} + +function bind_box_to_button(box, button) +{ + box.onkeydown=function() + { + if (event.keyCode == 13) + { + button.click(); + } + }; +} +function entry_with_history_hook(box, button) +{ + //console.log(event.keyCode); + if (box.entry_history === undefined) + {box.entry_history = [];} + if (box.entry_history_pos === undefined) + {box.entry_history_pos = -1;} + if (event.keyCode == 13) + { + /* Enter */ + box.entry_history.push(box.value); + button.click(); + box.value = ""; + } + else if (event.keyCode == 38) + { + + /* Up arrow */ + if (box.entry_history.length == 0) + {return} + if (box.entry_history_pos == -1) + { + box.entry_history_pos = box.entry_history.length - 1; + } + else if (box.entry_history_pos > 0) + { + box.entry_history_pos -= 1; + } + box.value = box.entry_history[box.entry_history_pos]; + } + else if (event.keyCode == 27) + { + box.value = ""; + } + else + { + box.entry_history_pos = -1; + } +} diff --git a/static/favicon.png b/static/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..35cdc36757e8e0529fa405fb292e3f04d09e96af GIT binary patch literal 2498 zcmV;z2|f0SP)Bf6v|7g|sXnrD$wYR(5t)87;MH)7Xl^ z)PAsOtWEj>Q(vl;#6m&iOU2a2wDF}NsQ96wR0OoxG_9DbX&Vzu$3zZR$0s8>UO?mkD99&? z#4RXK0iSoY<5#XicsX}MMSjuNm3mIV4(|OFA_ZWewKXmc4O>yJc5$o(KKtCa?8l+J zi`cidb$4ex9XB9!00t6?n?%_R%=U9Eif}KkqMw#=0iDTY)6B8l7J;QfGkYy& zyn<5?&Pj-DKQn8_?N{#Jox=yd0q9I7n`VvWb|~E7Yux3QCex?W7;mi@+e`RpcLp69 znc#v7FRi~ZJiGZOzaYfCNCg=f&23Vd5+5bAH@38J;f`0lOjTguJ2x_V;DBkVSK2mn z-~P<0>{f*zc-kjpJn^pYhQ z#_j>;hRN*pUIBr|_Yv->&l9U@rPI!20J`GwnX0m)9>VLk4IJAuL@ru&puN4h$WI2K zQDvDj@q0dy@gQ7bB~=4sFb%>(_&jGMr*{=AQ(Hi|&F8|vd)_HL>NYS2+Cz94pB3&b zwpjrE3ldi;a%t5|zih^8P~-zMK29xAfoW7frzvv%R2dvp;j`6 z(C)dH1Mc&BtcFN|*5KFK*zGY@+Ke4TB)I?O3aEwfsk$U$z}$KZuj@95Bz&?wH>MM> zlj|ou>TST!UUkBYXpR9hq9nK~Za;w$bj+we1I;mYak8ovP^EC^wV^Y62u=V?g)`-( z8p?qh4}LuvB%8+YGvR57YBd( zaPU2^W=@n;%vMsxm&H+p7t2T1t|R{14jRut-{^E$G+uClu|0QukVEU9HYdq)zg&bD z%W7%L0^MgM%ggG)0pmkM^etbB=6wHVv3IFM zqjA=KGJBCb`{5oAKd~mNK3@zx_9WR4I(!pcxnIt@qil~(W0$oklR2hnXDt$(0)tz( zadOA2p~d;(#A|PGWb2Fmncc-N!*w4mNv8)b08x2!l8h+a$qoGcQAW~T5oDS%lIfxM z;YZB2!wWAo$Xif+Cx~pX<%>uA>^XYwUQRw+{ZYRB@GxC>uOL4>Tod8Nx`@3U3IVvR zC)J_oKDR+Il>H2h_V#h`z6UE)aTuu5``~KMc4s^jTm+x@fBj(B9&r9IpltOL6xe`c zC4KtV-#Pl5=lx{5;_$}bGW3_fPK+Ngvj^csnRIIEpP?peqryoH7X~(^SV z-u2U3fz$83KXJdWq3}ZegiM!B#rDYn6wcm07g@NPInC$kx%UCajvVpRJc-<~<8;XH&p>*GU1bpJTmLz0e#adOmg9t2iODs3-M;!nYqhl-cHUbe4_bO1N=}*y8PNxJk-k2!aovbWA5DGm#l(Vq16zP=>McCb zb;>&pBDY`Lm+JL&+<>Y+n|tq~#4Vz%1?K%fZtGr465cPJ#CT#XVOp!*QnEW?eP zmK^UqQvoB<5P9 zx)_UvmRgtCstI5Y6W<=rD0DjmoKiRengR~kNT-!!*OJ5O!Fu}lUk#>P%>uj~DF6Tf M07*qoM6N<$g3+eXYXATM literal 0 HcmV?d00001 diff --git a/static/favicon.svg b/static/favicon.svg new file mode 100644 index 0000000..36b21c9 --- /dev/null +++ b/static/favicon.svg @@ -0,0 +1,138 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/channel.html b/templates/channel.html new file mode 100644 index 0000000..52783fa --- /dev/null +++ b/templates/channel.html @@ -0,0 +1,224 @@ + + + + {% import "header.html" as header %} + {{channel['name']}} + + + + + + + + + +{{header.make_header()}} +
+ + + All + Pending + Ignored + Downloaded + {{videos|length}} items + {% for video in videos %} +
+ {{video['_published_str']}} - {{video['title']}} +
+ {% if video['download'] == "downloaded" %} + + + {% elif video['download'] == "ignored" %} + + + {% else %} + + + + {% endif %} +
+
+ {% endfor %} +
+ + + + + diff --git a/templates/channels.html b/templates/channels.html new file mode 100644 index 0000000..db3050f --- /dev/null +++ b/templates/channels.html @@ -0,0 +1,96 @@ + + + + {% import "header.html" as header %} + Channels + + + + + + + + + +{{header.make_header()}} +
+ + +
+ + +
+ {% for channel in channels %} + {% if channel['has_pending'] %} +
+ {% else %} +
+ {% endif %} + {{channel['name']}} +
+ {% endfor %} +
+ + + + + diff --git a/templates/header.html b/templates/header.html new file mode 100644 index 0000000..fa3564a --- /dev/null +++ b/templates/header.html @@ -0,0 +1,6 @@ +{% macro make_header() %} + +{% endmacro %} \ No newline at end of file diff --git a/templates/root.html b/templates/root.html new file mode 100644 index 0000000..bb65094 --- /dev/null +++ b/templates/root.html @@ -0,0 +1,28 @@ + + + + {% import "header.html" as header %} + YCDL + + + + + + + + + Manage channels + + + + + diff --git a/ycdl.py b/ycdl.py new file mode 100644 index 0000000..cb5a8c2 --- /dev/null +++ b/ycdl.py @@ -0,0 +1,239 @@ +import logging +import os +import sqlite3 + +import ytapi + +# AVAILABLE FORMATTERS: +# url, id +# Note that if the channel has a value in the `directory` column, the bot will +# chdir there before executing. +YOUTUBE_DL_COMMAND = 'touch C:\\Incoming\\ytqueue\\{id}.ytqueue' + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) +logging.getLogger('googleapiclient.discovery').setLevel(logging.WARNING) +logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARNING) + +SQL_CHANNEL_COLUMNS = [ + 'id', + 'name', + 'directory', +] + +SQL_VIDEO_COLUMNS = [ + 'id', + 'published', + 'author_id', + 'title', + 'description', + 'thumbnail', + 'download', +] + +SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)} +SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)} + +DATABASE_VERSION = 1 +DB_INIT = ''' +PRAGMA count_changes = OFF; +PRAGMA cache_size = 10000; +PRAGMA user_version = {user_version}; +CREATE TABLE IF NOT EXISTS channels( + id TEXT, + name TEXT, + directory TEXT COLLATE NOCASE +); +CREATE TABLE IF NOT EXISTS videos( + id TEXT, + published INT, + author_id TEXT, + title TEXT, + description TEXT, + thumbnail TEXT, + download TEXT +); + +CREATE INDEX IF NOT EXISTS index_channel_id on channels(id); +CREATE INDEX IF NOT EXISTS index_video_id on videos(id); +CREATE INDEX IF NOT EXISTS index_video_published on videos(published); +CREATE INDEX IF NOT EXISTS index_video_download on videos(download); +'''.format(user_version=DATABASE_VERSION) + +DEFAULT_DBNAME = 'ycdl.db' + +ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}' + +def verify_is_abspath(path): + ''' + TO DO: Determine whether this is actually correct. + ''' + if os.path.abspath(path) != path: + raise ValueError('Not an abspath') + +class YCDL: + def __init__(self, youtube, database_filename=None): + self.youtube = youtube + if database_filename is None: + database_filename = DEFAULT_DBNAME + + existing_database = os.path.exists(database_filename) + self.sql = sqlite3.connect(database_filename) + self.cur = self.sql.cursor() + + if existing_database: + self.cur.execute('PRAGMA user_version') + existing_version = self.cur.fetchone()[0] + if existing_version != DATABASE_VERSION: + message = ERROR_DATABASE_OUTOFDATE + message = message.format(current=existing_version, new=DATABASE_VERSION) + print(message) + raise SystemExit + + statements = DB_INIT.split(';') + for statement in statements: + self.cur.execute(statement) + + def add_channel( + self, + channel_id, + commit=False, + download_directory=None, + get_videos=True, + name=None, + ): + if self.get_channel(channel_id) is not None: + return + + if name is None: + name = self.youtube.get_user_name(channel_id) + + data = [None] * len(SQL_CHANNEL) + data[SQL_CHANNEL['id']] = channel_id + data[SQL_CHANNEL['name']] = name + if download_directory is not None: + verify_is_abspath(download_directory) + data[SQL_CHANNEL['directory']] = download_directory + + self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data) + if get_videos: + self.refresh_channel(channel_id, commit=False) + if commit: + self.sql.commit() + + def channel_has_pending(self, channel_id): + query = 'SELECT * FROM videos WHERE author_id == ? AND download == "pending"' + self.cur.execute(query, [channel_id]) + return self.cur.fetchone() is not None + + def channel_directory(self, channel_id): + self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id]) + fetch = self.cur.fetchone() + if fetch is None: + return None + return fetch[SQL_CHANNEL['directory']] + + def download_video(self, video, force=False): + if not isinstance(video, ytapi.Video): + video = self.youtube.get_video(video) + + self.add_channel(video.author_id, get_videos=False, commit=False) + status = self.insert_video(video, commit=True) + + if status['row'][SQL_VIDEO['download']] != 'pending' and not force: + print('That video does not need to be downloaded.') + return + + download_directory = self.channel_directory(video.author_id) + download_directory = download_directory or os.getcwd() + + current_directory = os.getcwd() + os.makedirs(download_directory, exist_ok=True) + os.chdir(download_directory) + url = 'https://www.youtube.com/watch?v={id}'.format(id=video.id) + command = YOUTUBE_DL_COMMAND.format(url=url, id=video.id) + os.system(command) + os.chdir(current_directory) + + self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video.id]) + self.sql.commit() + + def get_channel(self, channel_id): + self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id]) + fetch = self.cur.fetchone() + if not fetch: + return None + fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL} + return fetch + + def get_channels(self): + self.cur.execute('SELECT * FROM channels') + channels = self.cur.fetchall() + channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels] + channels.sort(key=lambda x: x['name'].lower()) + return channels + + def get_videos(self, channel_id=None): + if channel_id is not None: + self.cur.execute('SELECT * FROM videos WHERE author_id == ?', [channel_id]) + else: + self.cur.execute('SELECT * FROM videos ') + videos = self.cur.fetchall() + if not videos: + return [] + videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos] + videos.sort(key=lambda x: x['published'], reverse=True) + return videos + + def insert_video(self, video, commit=True): + if not isinstance(video, ytapi.Video): + video = self.youtube.get_video(video) + + self.add_channel(video.author_id, get_videos=False, commit=False) + self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id]) + fetch = self.cur.fetchone() + if fetch is not None: + return {'new': False, 'row': fetch} + + data = [None] * len(SQL_VIDEO) + data[SQL_VIDEO['id']] = video.id + data[SQL_VIDEO['published']] = video.published + data[SQL_VIDEO['author_id']] = video.author_id + data[SQL_VIDEO['title']] = video.title + data[SQL_VIDEO['description']] = video.description + data[SQL_VIDEO['thumbnail']] = video.thumbnail['url'] + data[SQL_VIDEO['download']] = 'pending' + + self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?)', data) + if commit: + self.sql.commit() + return {'new': True, 'row': data} + + def mark_video_state(self, video_id, state, commit=True): + ''' + Mark the video as ignored, pending, or downloaded. + ''' + if state not in ['ignored', 'pending', 'downloaded']: + raise ValueError(state) + self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id]) + if self.cur.fetchone() is None: + raise KeyError(video_id) + self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id]) + if commit: + self.sql.commit() + + def refresh_all_channels(self, force=False, commit=True): + for channel in self.get_channels(): + self.refresh_channel(channel['id'], force=force, commit=commit) + if commit: + self.sql.commit() + + def refresh_channel(self, channel_id, force=False, commit=True): + video_generator = self.youtube.get_user_videos(uid=channel_id) + log.debug('Refreshing channel: %s', channel_id) + for video in video_generator: + status = self.insert_video(video, commit=False) + if not force and not status['new']: + break + if commit: + self.sql.commit() diff --git a/ycdl_easy.py b/ycdl_easy.py new file mode 100644 index 0000000..f610a47 --- /dev/null +++ b/ycdl_easy.py @@ -0,0 +1,11 @@ +''' +Run `python -i ycdl_easy.py to get an interpreter +session with these variables preloaded. +''' + +import bot +import ycdl +import ytapi + +youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY) +youtube = ycdl.YCDL(youtube_core) diff --git a/ycdl_launch.py b/ycdl_launch.py new file mode 100644 index 0000000..6ce64f6 --- /dev/null +++ b/ycdl_launch.py @@ -0,0 +1,29 @@ +import gevent.monkey +gevent.monkey.patch_all() + +import gevent.pywsgi +import gevent.wsgi +import sys +import ycdl_site + +if len(sys.argv) == 2: + port = int(sys.argv[1]) +else: + port = 5000 + +if port == 443: + http = gevent.pywsgi.WSGIServer( + listener=('', port), + application=ycdl_site.site, + keyfile='https\\flasksite.key', + certfile='https\\flasksite.crt', + ) +else: + http = gevent.pywsgi.WSGIServer( + listener=('', port), + application=ycdl_site.site, + ) + + +print('Starting server') +http.serve_forever() diff --git a/ycdl_site.py b/ycdl_site.py new file mode 100644 index 0000000..b13b10a --- /dev/null +++ b/ycdl_site.py @@ -0,0 +1,195 @@ +''' +Do not execute this file directly. +Use ycdl_launch.py to start the server with gevent. +''' +import datetime +import flask +from flask import request +import json +import mimetypes +import os + +import bot +import helpers +import ycdl +import ytapi + +youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY) +youtube = ycdl.YCDL(youtube_core) + +site = flask.Flask(__name__) +site.config.update( + SEND_FILE_MAX_AGE_DEFAULT=180, + TEMPLATES_AUTO_RELOAD=True, +) +site.jinja_env.add_extension('jinja2.ext.do') +site.debug = True + +#################################################################################################### +#################################################################################################### +#################################################################################################### +#################################################################################################### + +def make_json_response(j, *args, **kwargs): + dumped = json.dumps(j) + response = flask.Response(dumped, *args, **kwargs) + response.headers['Content-Type'] = 'application/json;charset=utf-8' + return response + +def send_file(filepath): + ''' + Range-enabled file sending. + ''' + try: + file_size = os.path.getsize(filepath) + except FileNotFoundError: + flask.abort(404) + + outgoing_headers = {} + mimetype = mimetypes.guess_type(filepath)[0] + if mimetype is not None: + if 'text/' in mimetype: + mimetype += '; charset=utf-8' + outgoing_headers['Content-Type'] = mimetype + + if 'range' in request.headers: + desired_range = request.headers['range'].lower() + desired_range = desired_range.split('bytes=')[-1] + + int_helper = lambda x: int(x) if x.isdigit() else None + if '-' in desired_range: + (desired_min, desired_max) = desired_range.split('-') + range_min = int_helper(desired_min) + range_max = int_helper(desired_max) + else: + range_min = int_helper(desired_range) + + if range_min is None: + range_min = 0 + if range_max is None: + range_max = file_size + + # because ranges are 0-indexed + range_max = min(range_max, file_size - 1) + range_min = max(range_min, 0) + + range_header = 'bytes {min}-{max}/{outof}'.format( + min=range_min, + max=range_max, + outof=file_size, + ) + outgoing_headers['Content-Range'] = range_header + status = 206 + else: + range_max = file_size - 1 + range_min = 0 + status = 200 + + outgoing_headers['Accept-Ranges'] = 'bytes' + outgoing_headers['Content-Length'] = (range_max - range_min) + 1 + + if request.method == 'HEAD': + outgoing_data = bytes() + else: + outgoing_data = helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max) + + response = flask.Response( + outgoing_data, + status=status, + headers=outgoing_headers, + ) + return response + +#################################################################################################### +#################################################################################################### +#################################################################################################### +#################################################################################################### + +@site.route('/') +def root(): + return flask.render_template('root.html') + +@site.route('/favicon.ico') +@site.route('/favicon.png') +def favicon(): + filename = os.path.join('static', 'favicon.png') + return flask.send_file(filename) + +@site.route('/channels') +def get_channels(): + channels = youtube.get_channels() + for channel in channels: + channel['has_pending'] = youtube.channel_has_pending(channel['id']) + return flask.render_template('channels.html', channels=channels) + +@site.route('/channel/') +@site.route('/channel//') +def get_channel(channel_id, download_filter=None): + channel = youtube.get_channel(channel_id) + if channel is None: + flask.abort(404) + videos = youtube.get_videos(channel_id=channel_id) + if download_filter is not None: + videos = [video for video in videos if video['download'] == download_filter] + for video in videos: + published = video['published'] + published = datetime.datetime.utcfromtimestamp(published) + published = published.strftime('%Y %m %d') + video['_published_str'] = published + return flask.render_template('channel.html', channel=channel, videos=videos) + +@site.route('/static/') +def get_static(filename): + filename = filename.replace('\\', os.sep) + filename = filename.replace('/', os.sep) + filename = os.path.join('static', filename) + return flask.send_file(filename) + +@site.route('/mark_video_state', methods=['POST']) +def post_mark_video_state(): + if 'video_id' not in request.form or 'state' not in request.form: + flask.abort(400) + video_id = request.form['video_id'] + state = request.form['state'] + try: + youtube.mark_video_state(video_id, state) + except KeyError: + flask.abort(404) + except ValueError: + flask.abort(400) + return make_json_response({'video_id': video_id, 'state': state}) + +@site.route('/refresh_all_channels', methods=['POST']) +def post_refresh_all_channels(): + force = request.form.get('force', False) + force = helpers.truthystring(force) + youtube.refresh_all_channels(force=force) + return make_json_response({}) + +@site.route('/refresh_channel', methods=['POST']) +def post_refresh_channel(): + if 'channel_id' not in request.form: + flask.abort(400) + channel_id = request.form['channel_id'] + channel_id = channel_id.strip() + if not channel_id: + flask.abort(400) + force = request.form.get('force', False) + force = helpers.truthystring(force) + youtube.refresh_channel(channel_id, force=force) + return make_json_response({}) + +@site.route('/start_download', methods=['POST']) +def post_start_download(): + if 'video_id' not in request.form: + flask.abort(400) + video_id = request.form['video_id'] + video_info = youtube_core.get_video([video_id]) + if video_info == []: + flask.abort(404) + for video in video_info: + youtube.download_video(video) + return make_json_response({'video_id': video_id, 'state': 'downloaded'}) + +if __name__ == '__main__': + pass diff --git a/ytapi.py b/ytapi.py new file mode 100644 index 0000000..2598226 --- /dev/null +++ b/ytapi.py @@ -0,0 +1,81 @@ +import apiclient.discovery +import datetime + +import helpers + +class Video: + def __init__(self, snippet): + self.id = snippet['id'] + + snippet = snippet['snippet'] + self.title = snippet['title'] or '[untitled]' + self.description = snippet['description'] + self.author_id = snippet['channelId'] + self.author_name = snippet['channelTitle'] + # Something like '2016-10-01T21:00:01' + self.published_string = snippet['publishedAt'] + published = snippet['publishedAt'] + published = published.split('.')[0] + published = datetime.datetime.strptime(published, '%Y-%m-%dT%H:%M:%S') + self.published = published.timestamp() + + thumbnails = snippet['thumbnails'] + best_thumbnail = max(thumbnails, key=lambda x: thumbnails[x]['width'] * thumbnails[x]['height']) + self.thumbnail = thumbnails[best_thumbnail] + + +class Youtube: + def __init__(self, key): + youtube = apiclient.discovery.build( + developerKey=key, + serviceName='youtube', + version='v3', + ) + self.youtube = youtube + + def get_user_name(self, uid): + user = self.youtube.channels().list(part='snippet', id=uid).execute() + return user['items'][0]['snippet']['title'] + + def get_user_videos(self, username=None, uid=None): + if username: + user = self.youtube.channels().list(part='contentDetails', forUsername=username).execute() + else: + user = self.youtube.channels().list(part='contentDetails', id=uid).execute() + upload_playlist = user['items'][0]['contentDetails']['relatedPlaylists']['uploads'] + page_token = None + while True: + items = self.youtube.playlistItems().list( + maxResults=50, + pageToken=page_token, + part='contentDetails', + playlistId=upload_playlist, + ).execute() + page_token = items.get('nextPageToken', None) + new = [item['contentDetails']['videoId'] for item in items['items']] + count = len(new) + new = self.get_video(new) + new.sort(key=lambda x: x.published, reverse=True) + yield from new + #print('Found %d more, %d total' % (count, len(videos))) + if page_token is None or count < 50: + break + + def get_video(self, video_ids): + if isinstance(video_ids, str): + singular = True + video_ids = [video_ids] + else: + singular = False + video_ids = helpers.chunk_sequence(video_ids, 50) + results = [] + for chunk in video_ids: + chunk = ','.join(chunk) + data = self.youtube.videos().list(part='snippet', id=chunk).execute() + items = data['items'] + results += items + #print('Found %d more, %d total' % (len(items), len(results))) + results = [Video(snippet) for snippet in results] + if singular and len(results) == 1: + return results[0] + return results diff --git a/ytqueue.py b/ytqueue.py new file mode 100644 index 0000000..5de38cf --- /dev/null +++ b/ytqueue.py @@ -0,0 +1,23 @@ +''' +I was having trouble making my Flask server perform the youtube-dl without +clogging up the other site activities. So instead I'll just have the server +export ytqueue files, which this script will download as a separate process. + +Rather than maintaining a text file or database of IDs to be downloaded, +I'm fine with creating each ID as a file and letting the filesystem act +as the to-do list. +''' +import os +import time + +YOUTUBE_DL = 'youtube-dlw https://www.youtube.com/watch?v={id}' + +while True: + print(time.strftime('%H:%M:%S'), 'Looking for files.') + queue = [f for f in os.listdir() if f.endswith('.ytqueue')] + for filename in queue: + yt_id = filename.split('.')[0] + command = YOUTUBE_DL.format(id=yt_id) + os.system(command) + os.remove(filename) + time.sleep(10)