+
+
+
+
+
diff --git a/YoutubeChannelDownloader/templates/channels.html b/YoutubeChannelDownloader/templates/channels.html
new file mode 100644
index 0000000..6c3a691
--- /dev/null
+++ b/YoutubeChannelDownloader/templates/channels.html
@@ -0,0 +1,67 @@
+
+
+
+ {% import "header.html" as header %}
+
+
+
+ {% for channel in channels %}
+ {% if channel['has_pending'] %}
+
+ {% else %}
+
+ {% endfor %}
+
+
+
+
+
+
diff --git a/YoutubeChannelDownloader/templates/header.html b/YoutubeChannelDownloader/templates/header.html
new file mode 100644
index 0000000..fa3564a
--- /dev/null
+++ b/YoutubeChannelDownloader/templates/header.html
@@ -0,0 +1,6 @@
+{% macro make_header() %}
+
+{% endmacro %}
\ No newline at end of file
diff --git a/YoutubeChannelDownloader/templates/root.html b/YoutubeChannelDownloader/templates/root.html
new file mode 100644
index 0000000..5dfa283
--- /dev/null
+++ b/YoutubeChannelDownloader/templates/root.html
@@ -0,0 +1,28 @@
+
+
+
+
+
+
+ {% import "header.html" as header %}
+
Flasksite
+
+
+
+
+
+
Manage channels
+
+
+
+
+
diff --git a/YoutubeChannelDownloader/ycdl.py b/YoutubeChannelDownloader/ycdl.py
new file mode 100644
index 0000000..90e893f
--- /dev/null
+++ b/YoutubeChannelDownloader/ycdl.py
@@ -0,0 +1,214 @@
+import os
+import sqlite3
+import ytapi
+
+# AVAILABLE FORMATTERS:
+# url, id
+YOUTUBE_DL_COMMAND = 'touch {id}.ytqueue'
+
+SQL_CHANNEL_COLUMNS = [
+ 'id',
+ 'name',
+ 'directory',
+]
+
+SQL_VIDEO_COLUMNS = [
+ 'id',
+ 'published',
+ 'author_id',
+ 'title',
+ 'description',
+ 'thumbnail',
+ 'download',
+]
+
+SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)}
+SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)}
+
+DATABASE_VERSION = 1
+DB_INIT = '''
+PRAGMA count_changes = OFF;
+PRAGMA cache_size = 10000;
+PRAGMA user_version = {user_version};
+CREATE TABLE IF NOT EXISTS channels(
+ id TEXT,
+ name TEXT,
+ directory TEXT COLLATE NOCASE
+);
+CREATE TABLE IF NOT EXISTS videos(
+ id TEXT,
+ published INT,
+ author_id TEXT,
+ title TEXT,
+ description TEXT,
+ thumbnail TEXT,
+ download TEXT
+);
+
+
+CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
+CREATE INDEX IF NOT EXISTS index_video_id on videos(id);
+CREATE INDEX IF NOT EXISTS index_video_published on videos(published);
+CREATE INDEX IF NOT EXISTS index_video_download on videos(download);
+
+'''.format(user_version=DATABASE_VERSION)
+
+DEFAULT_DBNAME = 'ycdl.db'
+
+ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
+
+def verify_is_abspath(path):
+ '''
+ TO DO: Determine whether this is actually correct.
+ '''
+ if os.path.abspath(path) != path:
+ raise ValueError('Not an abspath')
+
+class YCDL:
+ def __init__(self, youtube, database_filename=None):
+ self.youtube = youtube
+ if database_filename is None:
+ database_filename = DEFAULT_DBNAME
+
+ existing_database = os.path.exists(database_filename)
+ self.sql = sqlite3.connect(database_filename)
+ self.cur = self.sql.cursor()
+
+ if existing_database:
+ self.cur.execute('PRAGMA user_version')
+ existing_version = self.cur.fetchone()[0]
+ if existing_version != DATABASE_VERSION:
+ message = ERROR_DATABASE_OUTOFDATE
+ message = message.format(current=existing_version, new=DATABASE_VERSION)
+ print(message)
+ raise SystemExit
+
+ statements = DB_INIT.split(';')
+ for statement in statements:
+ self.cur.execute(statement)
+
+ def add_channel(self, channel_id, name=None, download_directory=None, get_videos=True, commit=False):
+ if self.get_channel(channel_id) is not None:
+ return
+
+ if name is None:
+ name = self.youtube.get_user_name(channel_id)
+
+ data = [None] * len(SQL_CHANNEL)
+ data[SQL_CHANNEL['id']] = channel_id
+ data[SQL_CHANNEL['name']] = name
+ if download_directory is not None:
+ verify_is_abspath(download_directory)
+ data[SQL_CHANNEL['directory']] = download_directory
+
+ self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data)
+ if get_videos:
+ self.refresh_channel(channel_id, commit=False)
+ if commit:
+ self.sql.commit()
+
+ def channel_has_pending(self, channel_id):
+ self.cur.execute('SELECT * FROM videos WHERE author_id == ? AND download == "pending"', [channel_id])
+ return self.cur.fetchone() is not None
+
+ def channel_directory(self, channel_id):
+ self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
+ fetch = self.cur.fetchone()
+ if fetch is None:
+ return None
+ return fetch[SQL_CHANNEL['directory']]
+
+ def download_video(self, video, force=False):
+ if not isinstance(video, ytapi.Video):
+ video = self.youtube.get_video(video)
+
+ self.add_channel(video.author_id, get_videos=False, commit=False)
+ status = self.insert_video(video, commit=True)
+
+ if status['row'][SQL_VIDEO['download']] != 'pending' and not force:
+ print('That video does not need to be downloaded.')
+ return
+
+ download_directory = self.channel_directory(video.author_id)
+ download_directory = download_directory or os.getcwd()
+
+ current_directory = os.getcwd()
+ os.makedirs(download_directory, exist_ok=True)
+ os.chdir(download_directory)
+ url = 'https://www.youtube.com/watch?v={id}'.format(id=video.id)
+ command = YOUTUBE_DL_COMMAND.format(url=url, id=video.id)
+ os.system(command)
+ os.chdir(current_directory)
+
+ self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video.id])
+ self.sql.commit()
+
+ def get_channel(self, channel_id):
+ self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
+ fetch = self.cur.fetchone()
+ if not fetch:
+ return None
+ fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL}
+ return fetch
+
+ def get_channels(self):
+ self.cur.execute('SELECT * FROM channels')
+ channels = self.cur.fetchall()
+ channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels]
+ channels.sort(key=lambda x: x['name'].lower())
+ return channels
+
+ def get_videos_by_channel(self, channel_id):
+ self.cur.execute('SELECT * FROM videos WHERE author_id == ?', [channel_id])
+ videos = self.cur.fetchall()
+ if not videos:
+ return []
+ videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
+ videos.sort(key=lambda x: x['published'], reverse=True)
+ return videos
+
+ def mark_video_state(self, video_id, state, commit=True):
+ '''
+ Mark the video as ignored, pending, or downloaded.
+ '''
+ if state not in ['ignored', 'pending', 'downloaded']:
+ raise ValueError(state)
+ self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
+ if self.cur.fetchone() is None:
+ raise KeyError(video_id)
+ self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
+ if commit:
+ self.sql.commit()
+
+ def refresh_channel(self, channel_id, force=True, commit=True):
+ video_generator = self.youtube.get_user_videos(uid=channel_id)
+ for video in video_generator:
+ status = self.insert_video(video, commit=False)
+ if not force and not status['new']:
+ break
+ if commit:
+ self.sql.commit()
+
+ def insert_video(self, video, commit=True):
+ if not isinstance(video, ytapi.Video):
+ video = self.youtube.get_video(video)
+
+ self.add_channel(video.author_id, get_videos=False, commit=False)
+ self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id])
+ fetch = self.cur.fetchone()
+ if fetch is not None:
+ return {'new': False, 'row': fetch}
+
+ data = [None] * len(SQL_VIDEO)
+ data[SQL_VIDEO['id']] = video.id
+ data[SQL_VIDEO['published']] = video.published
+ data[SQL_VIDEO['author_id']] = video.author_id
+ data[SQL_VIDEO['title']] = video.title
+ data[SQL_VIDEO['description']] = video.description
+ data[SQL_VIDEO['thumbnail']] = video.thumbnail['url']
+ data[SQL_VIDEO['download']] = 'pending'
+
+ self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?)', data)
+ if commit:
+ self.sql.commit()
+ return {'new': True, 'row': data}
diff --git a/YoutubeChannelDownloader/ycdl_easy.py b/YoutubeChannelDownloader/ycdl_easy.py
new file mode 100644
index 0000000..c78e8ab
--- /dev/null
+++ b/YoutubeChannelDownloader/ycdl_easy.py
@@ -0,0 +1,6 @@
+import ytapi
+import ycdl
+import bot
+
+youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
+youtube = ycdl.YCDL(youtube_core)
diff --git a/YoutubeChannelDownloader/ycdl_launch.py b/YoutubeChannelDownloader/ycdl_launch.py
new file mode 100644
index 0000000..8bca03b
--- /dev/null
+++ b/YoutubeChannelDownloader/ycdl_launch.py
@@ -0,0 +1,29 @@
+import gevent.monkey
+gevent.monkey.patch_all()
+
+import ycdl_site
+import gevent.pywsgi
+import gevent.wsgi
+import sys
+
+if len(sys.argv) == 2:
+ port = int(sys.argv[1])
+else:
+ port = 5000
+
+if port == 443:
+ http = gevent.pywsgi.WSGIServer(
+ listener=('', port),
+ application=ycdl_site.site,
+ keyfile='https\\flasksite.key',
+ certfile='https\\flasksite.crt',
+ )
+else:
+ http = gevent.pywsgi.WSGIServer(
+ listener=('', port),
+ application=ycdl_site.site,
+ )
+
+
+print('Starting server')
+http.serve_forever()
diff --git a/YoutubeChannelDownloader/ycdl_site.py b/YoutubeChannelDownloader/ycdl_site.py
new file mode 100644
index 0000000..523cf23
--- /dev/null
+++ b/YoutubeChannelDownloader/ycdl_site.py
@@ -0,0 +1,213 @@
+import flask
+from flask import request
+import json
+import mimetypes
+import os
+import sqlite3
+import threading
+import time
+
+import ytapi
+import ycdl
+import bot
+
+youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
+youtube = ycdl.YCDL(youtube_core)
+
+site = flask.Flask(__name__)
+site.config.update(
+ SEND_FILE_MAX_AGE_DEFAULT=180,
+ TEMPLATES_AUTO_RELOAD=True,
+)
+site.jinja_env.add_extension('jinja2.ext.do')
+site.debug = True
+
+download_queue = set()
+
+####################################################################################################
+####################################################################################################
+####################################################################################################
+####################################################################################################
+
+#def handle_download_queue():
+# while True:
+# if len(download_queue) > 0:
+# item = download_queue.pop()
+# youtube.download_video(item)
+# time.sleep(2)
+#
+#DOWNLOAD_QUEUE_THREAD = threading.Thread(target=handle_download_queue)
+#DOWNLOAD_QUEUE_THREAD.daemon = True
+#DOWNLOAD_QUEUE_THREAD.start()
+
+def make_json_response(j, *args, **kwargs):
+ dumped = json.dumps(j)
+ response = flask.Response(dumped, *args, **kwargs)
+ response.headers['Content-Type'] = 'application/json;charset=utf-8'
+ return response
+
+def send_file(filepath):
+ '''
+ Range-enabled file sending.
+ '''
+ try:
+ file_size = os.path.getsize(filepath)
+ except FileNotFoundError:
+ flask.abort(404)
+
+ outgoing_headers = {}
+ mimetype = mimetypes.guess_type(filepath)[0]
+ if mimetype is not None:
+ if 'text/' in mimetype:
+ mimetype += '; charset=utf-8'
+ outgoing_headers['Content-Type'] = mimetype
+
+ if 'range' in request.headers:
+ desired_range = request.headers['range'].lower()
+ desired_range = desired_range.split('bytes=')[-1]
+
+ int_helper = lambda x: int(x) if x.isdigit() else None
+ if '-' in desired_range:
+ (desired_min, desired_max) = desired_range.split('-')
+ range_min = int_helper(desired_min)
+ range_max = int_helper(desired_max)
+ else:
+ range_min = int_helper(desired_range)
+
+ if range_min is None:
+ range_min = 0
+ if range_max is None:
+ range_max = file_size
+
+ # because ranges are 0-indexed
+ range_max = min(range_max, file_size - 1)
+ range_min = max(range_min, 0)
+
+ range_header = 'bytes {min}-{max}/{outof}'.format(
+ min=range_min,
+ max=range_max,
+ outof=file_size,
+ )
+ outgoing_headers['Content-Range'] = range_header
+ status = 206
+ else:
+ range_max = file_size - 1
+ range_min = 0
+ status = 200
+
+ outgoing_headers['Accept-Ranges'] = 'bytes'
+ outgoing_headers['Content-Length'] = (range_max - range_min) + 1
+
+ if request.method == 'HEAD':
+ outgoing_data = bytes()
+ else:
+ outgoing_data = helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max)
+
+ response = flask.Response(
+ outgoing_data,
+ status=status,
+ headers=outgoing_headers,
+ )
+ return response
+
+def truthystring(s):
+ if isinstance(s, (bool, int)) or s is None:
+ return s
+ s = s.lower()
+ if s in {'1', 'true', 't', 'yes', 'y', 'on'}:
+ return True
+ if s in {'null', 'none'}:
+ return None
+ return False
+
+
+####################################################################################################
+####################################################################################################
+####################################################################################################
+####################################################################################################
+
+@site.route('/')
+def root():
+ return flask.render_template('root.html')
+
+@site.route('/channels')
+def get_channels():
+ channels = youtube.get_channels()
+ for channel in channels:
+ channel['has_pending'] = youtube.channel_has_pending(channel['id'])
+ return flask.render_template('channels.html', channels=channels)
+
+@site.route('/channel/
')
+@site.route('/channel//')
+def get_channel(channel_id, download_filter=None):
+ channel = youtube.get_channel(channel_id)
+ if channel is None:
+ flask.abort(404)
+ videos = youtube.get_videos_by_channel(channel_id)
+ if download_filter is not None:
+ videos = [video for video in videos if video['download'] == download_filter]
+ return flask.render_template('channel.html', channel=channel, videos=videos)
+
+@site.route('/favicon.ico')
+@site.route('/favicon.png')
+def favicon():
+ filename = os.path.join('static', 'favicon.png')
+ return flask.send_file(filename)
+
+@site.route('/static/')
+def get_static(filename):
+ filename = filename.replace('\\', os.sep)
+ filename = filename.replace('/', os.sep)
+ filename = os.path.join('static', filename)
+ return flask.send_file(filename)
+
+@site.route('/mark_video_state', methods=['POST'])
+def post_mark_video_state():
+ if 'video_id' not in request.form or 'state' not in request.form:
+ flask.abort(400)
+ video_id = request.form['video_id']
+ state = request.form['state']
+ try:
+ youtube.mark_video_state(video_id, state)
+ except KeyError:
+ flask.abort(404)
+ except ValueError:
+ flask.abort(400)
+ return make_json_response({})
+
+@site.route('/refresh_channel', methods=['POST'])
+def post_refresh_channel():
+ if 'channel_id' not in request.form:
+ flask.abort(400)
+ channel_id = request.form['channel_id']
+ force = request.form.get('force', False)
+ force = truthystring(force)
+ print('Refresh channel', channel_id)
+ youtube.refresh_channel(channel_id, force=force)
+ return make_json_response({})
+
+@site.route('/refresh_all_channels', methods=['POST'])
+def post_refresh_all_channels():
+ force = request.form.get('force', False)
+ force = truthystring(force)
+ for channel in youtube.get_channels():
+ print('Refresh channel', channel['id'])
+ youtube.refresh_channel(channel['id'], force=force)
+ return make_json_response({})
+
+@site.route('/start_download', methods=['POST'])
+def post_start_download():
+ if 'video_id' not in request.form:
+ flask.abort(400)
+ video_id = request.form['video_id']
+ video_info = youtube_core.get_video([video_id])
+ if video_info == []:
+ flask.abort(404)
+ for video in video_info:
+ #download_queue.add(video)
+ youtube.download_video(video)
+ #print(video)
+ return make_json_response({})
+
+if __name__ == '__main__':
+ pass
diff --git a/YoutubeChannelDownloader/ytapi.py b/YoutubeChannelDownloader/ytapi.py
new file mode 100644
index 0000000..a6c3237
--- /dev/null
+++ b/YoutubeChannelDownloader/ytapi.py
@@ -0,0 +1,102 @@
+import apiclient.discovery
+import datetime
+import sqlite3
+
+class Video:
+ def __init__(self, snippet):
+ self.id = snippet['id']
+
+ snippet = snippet['snippet']
+ self.title = snippet['title'] or '[untitled]'
+ self.description = snippet['description']
+ self.author_id = snippet['channelId']
+ self.author_name = snippet['channelTitle']
+ # Something like '2016-10-01T21:00:01'
+ self.published_string = snippet['publishedAt']
+ published = snippet['publishedAt']
+ published = published.split('.')[0]
+ published = datetime.datetime.strptime(published, '%Y-%m-%dT%H:%M:%S')
+ self.published = published.timestamp()
+
+ thumbnails = snippet['thumbnails']
+ best_thumbnail = max(thumbnails, key=lambda x: thumbnails[x]['width'] * thumbnails[x]['height'])
+ self.thumbnail = thumbnails[best_thumbnail]
+
+
+class Youtube:
+ def __init__(self, key):
+ youtube = apiclient.discovery.build(
+ developerKey=key,
+ serviceName='youtube',
+ version='v3',
+ )
+ self.youtube = youtube
+
+ def get_user_name(self, uid):
+ user = self.youtube.channels().list(part='snippet', id=uid).execute()
+ return user['items'][0]['snippet']['title']
+
+ def get_user_videos(self, username=None, uid=None):
+ if username:
+ user = self.youtube.channels().list(part='contentDetails', forUsername=username).execute()
+ else:
+ user = self.youtube.channels().list(part='contentDetails', id=uid).execute()
+ upload_playlist = user['items'][0]['contentDetails']['relatedPlaylists']['uploads']
+ page_token = None
+ while True:
+ items = self.youtube.playlistItems().list(
+ maxResults=50,
+ pageToken=page_token,
+ part='contentDetails',
+ playlistId=upload_playlist,
+ ).execute()
+ page_token = items.get('nextPageToken', None)
+ new = [item['contentDetails']['videoId'] for item in items['items']]
+ count = len(new)
+ new = self.get_video(new)
+ new.sort(key=lambda x: x.published, reverse=True)
+ yield from new
+ #print('Found %d more, %d total' % (count, len(videos)))
+ if page_token is None or count < 50:
+ break
+
+ def get_video(self, video_ids):
+ if isinstance(video_ids, str):
+ singular = True
+ video_ids = [video_ids]
+ else:
+ singular = False
+ video_ids = chunk_sequence(video_ids, 50)
+ results = []
+ for chunk in video_ids:
+ chunk = ','.join(chunk)
+ data = self.youtube.videos().list(part='snippet', id=chunk).execute()
+ items = data['items']
+ results += items
+ #print('Found %d more, %d total' % (len(items), len(results)))
+ results = [Video(snippet) for snippet in results]
+ if singular and len(results) == 1:
+ return results[0]
+ return results
+
+
+def chunk_sequence(sequence, chunk_length, allow_incomplete=True):
+ """Given a sequence, divide it into sequences of length `chunk_length`.
+
+ :param allow_incomplete: If True, allow the final chunk to be shorter if the
+ given sequence is not an exact multiple of `chunk_length`.
+ If False, the incomplete chunk will be discarded.
+ """
+ (complete, leftover) = divmod(len(sequence), chunk_length)
+ if not allow_incomplete:
+ leftover = 0
+
+ chunk_count = complete + min(leftover, 1)
+
+ chunks = []
+ for x in range(chunk_count):
+ left = chunk_length * x
+ right = left + chunk_length
+ chunks.append(sequence[left:right])
+
+ return chunks
diff --git a/YoutubeChannelDownloader/ytqueue.py b/YoutubeChannelDownloader/ytqueue.py
new file mode 100644
index 0000000..067f2a4
--- /dev/null
+++ b/YoutubeChannelDownloader/ytqueue.py
@@ -0,0 +1,20 @@
+'''
+I was having trouble making my Flask server perform the youtube-dl without
+slowing down and clogging up the other site activities. So instead I'll just
+have the server export ytqueue files, which this script will download
+as a separate process.
+'''
+import os
+import time
+
+YOUTUBE_DL = 'youtube-dlw https://www.youtube.com/watch?v={id}'
+
+while True:
+ print(time.strftime('%H:%M:%S'), 'Looking for files.')
+ queue = [f for f in os.listdir() if f.endswith('.ytqueue')]
+ for filename in queue:
+ yt_id = filename.split('.')[0]
+ command = YOUTUBE_DL.format(id=yt_id)
+ os.system(command)
+ os.remove(filename)
+ time.sleep(10)
\ No newline at end of file