voussoir/else
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

else

Browse source
This commit is contained in:
Ethan Dalool 2016-11-21 23:40:01 -08:00
parent cea8ef6007
commit 4e1880f9a0
15 changed files with 973 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
Templates/flask/templates/root.html
View file

@ -1,5 +1,10 @@
<!DOCTYPE html5>
<html>
<head>
{% import "header.html" as header %}
<title>Flasksite</title>
<meta charset="UTF-8">
<link rel="stylesheet" href="/static/common.css">
<style>
body, a
@ -10,12 +15,6 @@ body, a
align-items: center;
}
</style>
<head>
{% import "header.html" as header %}
<title>Flasksite</title>
<meta charset="UTF-8">
<link rel="stylesheet" href="/static/common.css">
</head>
<body>

23
Templates/flask/templates/template.html Normal file
View file

@ -0,0 +1,23 @@
<!DOCTYPE html5>
<html>
<head>
{% import "header.html" as header %}
<title>Flasksite</title>
<meta charset="UTF-8">
<link rel="stylesheet" href="/static/common.css">
<style>
</style>
</head>
<body>
<div id="content_body">
<p>test</p>
</div>
</body>
</html>
<script type="text/javascript">
</script>

32
YoutubeChannelDownloader/static/common.css Normal file
View file

@ -0,0 +1,32 @@
body
{
display: flex;
flex-direction: column;
background-color:#fff;
margin: 8px;
}
#header
{
display: flex;
flex-direction: row;
justify-content: center;
align-content: center;
margin-bottom: 4px;
}
.header_element
{
display: flex;
justify-content: center;
flex: 1;
background-color: rgba(0, 0, 0, 0.1);
}
.header_element:hover
{
background-color: rgba(0, 0, 0, 0.2);
}
#content_body
{
flex: 0 0 auto;
display: flex;
flex-direction: row;
}

83
YoutubeChannelDownloader/static/common.js Normal file
View file

@ -0,0 +1,83 @@
function post_example(key, value, callback)
{
var url = "/postexample";
data = new FormData();
data.append(key, value);
return post(url, data, callback);
}
function null_callback()
{
return;
}
function post(url, data, callback)
{
var request = new XMLHttpRequest();
request.answer = null;
request.onreadystatechange = function()
{
if (request.readyState == 4)
{
var text = request.responseText;
if (callback != null)
{
console.log(text);
callback(JSON.parse(text));
}
}
};
var asynchronous = true;
request.open("POST", url, asynchronous);
request.send(data);
}
function bind_box_to_button(box, button)
{
box.onkeydown=function()
{
if (event.keyCode == 13)
{
button.click();
}
};
}
function entry_with_history_hook(box, button)
{
//console.log(event.keyCode);
if (box.entry_history === undefined)
{box.entry_history = [];}
if (box.entry_history_pos === undefined)
{box.entry_history_pos = -1;}
if (event.keyCode == 13)
{
/* Enter */
box.entry_history.push(box.value);
button.click();
box.value = "";
}
else if (event.keyCode == 38)
{
/* Up arrow */
if (box.entry_history.length == 0)
{return}
if (box.entry_history_pos == -1)
{
box.entry_history_pos = box.entry_history.length - 1;
}
else if (box.entry_history_pos > 0)
{
box.entry_history_pos -= 1;
}
box.value = box.entry_history[box.entry_history_pos];
}
else if (event.keyCode == 27)
{
box.value = "";
}
else
{
box.entry_history_pos = -1;
}
}

BIN
YoutubeChannelDownloader/static/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 448 B

145
YoutubeChannelDownloader/templates/channel.html Normal file
View file

@ -0,0 +1,145 @@
<!DOCTYPE html5>
<html>
<head>
{% import "header.html" as header %}
<title>{{channel['name']}}</title>
<meta charset="UTF-8">
<link rel="stylesheet" href="/static/common.css">
<script src="/static/common.js"></script>
<style>
#content_body
{
display: flex;
flex-direction: column;
}
.video_card_downloaded,
.video_card_ignored,
.video_card_pending
{
position: relative;
margin: 8px;
padding: 10px;
border-radius: 4px;
border: 1px solid #000;
}
.video_card_pending
{
background-color: #ffffaa;
}
.video_card_ignored
{
background-color: #ffc886;
}
.video_card_downloaded
{
background-color: #aaffaa;
}
.action_toolbox
{
float: right;
display: inline-flex;
flex-direction: column;
position: relative;
}
.video_action_dropdown
{
z-index: 1;
background-color: #fff;
padding: 4px;
border: 1px solid #000;
position: absolute;
top: 100%;
right: 0;
display: none;
flex-direction: column;
}
.refresh_button
{
width: 10%;
}
</style>
</head>
<body>
{{header.make_header()}}
<div id="content_body">
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', false, null_callback)">Refresh new videos</button>
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', true, null_callback)">Refresh everything</button>
<span><a href="/channel/{{channel['id']}}">All</a></span>
<span><a href="/channel/{{channel['id']}}/pending">Pending</a></span>
<span><a href="/channel/{{channel['id']}}/ignored">Ignored</a></span>
<span><a href="/channel/{{channel['id']}}/downloaded">Downloaded</a></span>
{% for video in videos %}
{% if video['download'] == "downloaded" %}
<div class="video_card_downloaded">
{% elif video['download'] == "ignored" %}
<div class="video_card_ignored">
{% else %}
<div class="video_card_pending">
{% endif %}
<a href="https://www.youtube.com/watch?v={{video['id']}}">{{video['title']}}</a>
<div class="action_toolbox">
<button class="video_action_dropdown_toggle" onclick="toggle_dropdown(this.nextElementSibling)">Actions</button>
<div class="video_action_dropdown">
{% if video['download'] == "downloaded" %}
<button class="video_action_ignore" onclick="mark_video_state('{{video['id']}}', 'pending', null_callback); toggle_dropdown(this.parentElement);">Revert to Pending</button>
{% elif video['download'] == "ignored" %}
<button class="video_action_ignore" onclick="mark_video_state('{{video['id']}}', 'pending', null_callback); toggle_dropdown(this.parentElement);">Revert to Pending</button>
{% else %}
<button class="video_action_download" onclick="start_download('{{video['id']}}', null_callback); toggle_dropdown(this.parentElement);">Download</button>
<button class="video_action_ignore" onclick="mark_video_state('{{video['id']}}', 'ignored', null_callback); toggle_dropdown(this.parentElement);">Ignore</button>
{% endif %}
</div>
</div>
</div>
{% endfor %}
</div>
</body>
</html>
<script type="text/javascript">
function refresh_channel(channel_id, force, callback)
{
var url = "/refresh_channel";
data = new FormData();
data.append("channel_id", channel_id);
data.append("force", force)
return post(url, data, callback);
}
function mark_video_state(video_id, state, callback)
{
var url = "/mark_video_state";
data = new FormData();
data.append("video_id", video_id);
data.append("state", state);
return post(url, data, callback);
}
function start_download(video_id, callback)
{
var url = "/start_download";
data = new FormData();
data.append("video_id", video_id);
return post(url, data, callback);
}
function toggle_dropdown(dropdown)
{
if (dropdown.style.display != "inline-flex")
{
dropdown.style.display = "inline-flex";
}
else
{
dropdown.style.display = "none";
}
}
</script>

67
YoutubeChannelDownloader/templates/channels.html Normal file
View file

@ -0,0 +1,67 @@
<!DOCTYPE html5>
<html>
<head>
{% import "header.html" as header %}
<title>Channels</title>
<meta charset="UTF-8">
<link rel="stylesheet" href="/static/common.css">
<script src="/static/common.js"></script>
<style>
#content_body
{
display: flex;
flex-direction: column;
}
.channel_card_downloaded,
.channel_card_pending
{
margin: 8px;
padding: 10px;
border-radius: 4px;
border: 1px solid #000;
}
.channel_card_pending
{
background-color: #ffffaa;
}
.channel_card_downloaded
{
background-color: #aaffaa;
}
.refresh_button
{
width: 10%;
}
</style>
</head>
<body>
{{header.make_header()}}
<div id="content_body">
<button class="refresh_button" onclick="refresh_all_channels(false)">Refresh new videos</button>
<button class="refresh_button" onclick="refresh_all_channels(true)">Refresh everything</button>
{% for channel in channels %}
{% if channel['has_pending'] %}
<div class="channel_card_pending">
{% else %}
<div class="channel_card_downloaded">
{% endif %}
<a href="/channel/{{channel['id']}}">{{channel['name']}}</a>
</div>
{% endfor %}
</div>
</body>
</html>
<script type="text/javascript">
function refresh_all_channels(force)
{
var url = "/refresh_all_channels";
data = new FormData();
data.append("force", force)
return post(url, data, null_callback);
}
</script>

6
YoutubeChannelDownloader/templates/header.html Normal file
View file

@ -0,0 +1,6 @@
{% macro make_header() %}
<div id="header">
<a class="header_element" href="/">Home</a>
<a class="header_element" href="/channels">Channels</a>
</div>
{% endmacro %}

28
YoutubeChannelDownloader/templates/root.html Normal file
View file

@ -0,0 +1,28 @@
<!DOCTYPE html5>
<html>
<style>
body, a
{
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
}
</style>
<head>
{% import "header.html" as header %}
<title>Flasksite</title>
<meta charset="UTF-8">
<link rel="stylesheet" href="/static/common.css">
</head>
<body>
<a href='/channels'>Manage channels</a>
</body>
</html>
<script type="text/javascript">
</script>

214
YoutubeChannelDownloader/ycdl.py Normal file
View file

@ -0,0 +1,214 @@
import os
import sqlite3
import ytapi
# AVAILABLE FORMATTERS:
# url, id
YOUTUBE_DL_COMMAND = 'touch {id}.ytqueue'
SQL_CHANNEL_COLUMNS = [
'id',
'name',
'directory',
]
SQL_VIDEO_COLUMNS = [
'id',
'published',
'author_id',
'title',
'description',
'thumbnail',
'download',
]
SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)}
SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)}
DATABASE_VERSION = 1
DB_INIT = '''
PRAGMA count_changes = OFF;
PRAGMA cache_size = 10000;
PRAGMA user_version = {user_version};
CREATE TABLE IF NOT EXISTS channels(
id TEXT,
name TEXT,
directory TEXT COLLATE NOCASE
);
CREATE TABLE IF NOT EXISTS videos(
id TEXT,
published INT,
author_id TEXT,
title TEXT,
description TEXT,
thumbnail TEXT,
download TEXT
);
CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
CREATE INDEX IF NOT EXISTS index_video_id on videos(id);
CREATE INDEX IF NOT EXISTS index_video_published on videos(published);
CREATE INDEX IF NOT EXISTS index_video_download on videos(download);
'''.format(user_version=DATABASE_VERSION)
DEFAULT_DBNAME = 'ycdl.db'
ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
def verify_is_abspath(path):
'''
TO DO: Determine whether this is actually correct.
'''
if os.path.abspath(path) != path:
raise ValueError('Not an abspath')
class YCDL:
def __init__(self, youtube, database_filename=None):
self.youtube = youtube
if database_filename is None:
database_filename = DEFAULT_DBNAME
existing_database = os.path.exists(database_filename)
self.sql = sqlite3.connect(database_filename)
self.cur = self.sql.cursor()
if existing_database:
self.cur.execute('PRAGMA user_version')
existing_version = self.cur.fetchone()[0]
if existing_version != DATABASE_VERSION:
message = ERROR_DATABASE_OUTOFDATE
message = message.format(current=existing_version, new=DATABASE_VERSION)
print(message)
raise SystemExit
statements = DB_INIT.split(';')
for statement in statements:
self.cur.execute(statement)
def add_channel(self, channel_id, name=None, download_directory=None, get_videos=True, commit=False):
if self.get_channel(channel_id) is not None:
return
if name is None:
name = self.youtube.get_user_name(channel_id)
data = [None] * len(SQL_CHANNEL)
data[SQL_CHANNEL['id']] = channel_id
data[SQL_CHANNEL['name']] = name
if download_directory is not None:
verify_is_abspath(download_directory)
data[SQL_CHANNEL['directory']] = download_directory
self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data)
if get_videos:
self.refresh_channel(channel_id, commit=False)
if commit:
self.sql.commit()
def channel_has_pending(self, channel_id):
self.cur.execute('SELECT * FROM videos WHERE author_id == ? AND download == "pending"', [channel_id])
return self.cur.fetchone() is not None
def channel_directory(self, channel_id):
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
fetch = self.cur.fetchone()
if fetch is None:
return None
return fetch[SQL_CHANNEL['directory']]
def download_video(self, video, force=False):
if not isinstance(video, ytapi.Video):
video = self.youtube.get_video(video)
self.add_channel(video.author_id, get_videos=False, commit=False)
status = self.insert_video(video, commit=True)
if status['row'][SQL_VIDEO['download']] != 'pending' and not force:
print('That video does not need to be downloaded.')
return
download_directory = self.channel_directory(video.author_id)
download_directory = download_directory or os.getcwd()
current_directory = os.getcwd()
os.makedirs(download_directory, exist_ok=True)
os.chdir(download_directory)
url = 'https://www.youtube.com/watch?v={id}'.format(id=video.id)
command = YOUTUBE_DL_COMMAND.format(url=url, id=video.id)
os.system(command)
os.chdir(current_directory)
self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video.id])
self.sql.commit()
def get_channel(self, channel_id):
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
fetch = self.cur.fetchone()
if not fetch:
return None
fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL}
return fetch
def get_channels(self):
self.cur.execute('SELECT * FROM channels')
channels = self.cur.fetchall()
channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels]
channels.sort(key=lambda x: x['name'].lower())
return channels
def get_videos_by_channel(self, channel_id):
self.cur.execute('SELECT * FROM videos WHERE author_id == ?', [channel_id])
videos = self.cur.fetchall()
if not videos:
return []
videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
videos.sort(key=lambda x: x['published'], reverse=True)
return videos
def mark_video_state(self, video_id, state, commit=True):
'''
Mark the video as ignored, pending, or downloaded.
'''
if state not in ['ignored', 'pending', 'downloaded']:
raise ValueError(state)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
if self.cur.fetchone() is None:
raise KeyError(video_id)
self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
if commit:
self.sql.commit()
def refresh_channel(self, channel_id, force=True, commit=True):
video_generator = self.youtube.get_user_videos(uid=channel_id)
for video in video_generator:
status = self.insert_video(video, commit=False)
if not force and not status['new']:
break
if commit:
self.sql.commit()
def insert_video(self, video, commit=True):
if not isinstance(video, ytapi.Video):
video = self.youtube.get_video(video)
self.add_channel(video.author_id, get_videos=False, commit=False)
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id])
fetch = self.cur.fetchone()
if fetch is not None:
return {'new': False, 'row': fetch}
data = [None] * len(SQL_VIDEO)
data[SQL_VIDEO['id']] = video.id
data[SQL_VIDEO['published']] = video.published
data[SQL_VIDEO['author_id']] = video.author_id
data[SQL_VIDEO['title']] = video.title
data[SQL_VIDEO['description']] = video.description
data[SQL_VIDEO['thumbnail']] = video.thumbnail['url']
data[SQL_VIDEO['download']] = 'pending'
self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?)', data)
if commit:
self.sql.commit()
return {'new': True, 'row': data}

6
YoutubeChannelDownloader/ycdl_easy.py Normal file
View file

@ -0,0 +1,6 @@
import ytapi
import ycdl
import bot
youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
youtube = ycdl.YCDL(youtube_core)

29
YoutubeChannelDownloader/ycdl_launch.py Normal file
View file

@ -0,0 +1,29 @@
import gevent.monkey
gevent.monkey.patch_all()
import ycdl_site
import gevent.pywsgi
import gevent.wsgi
import sys
if len(sys.argv) == 2:
port = int(sys.argv[1])
else:
port = 5000
if port == 443:
http = gevent.pywsgi.WSGIServer(
listener=('', port),
application=ycdl_site.site,
keyfile='https\\flasksite.key',
certfile='https\\flasksite.crt',
)
else:
http = gevent.pywsgi.WSGIServer(
listener=('', port),
application=ycdl_site.site,
)
print('Starting server')
http.serve_forever()

213
YoutubeChannelDownloader/ycdl_site.py Normal file
View file

@ -0,0 +1,213 @@
import flask
from flask import request
import json
import mimetypes
import os
import sqlite3
import threading
import time
import ytapi
import ycdl
import bot
youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
youtube = ycdl.YCDL(youtube_core)
site = flask.Flask(__name__)
site.config.update(
SEND_FILE_MAX_AGE_DEFAULT=180,
TEMPLATES_AUTO_RELOAD=True,
)
site.jinja_env.add_extension('jinja2.ext.do')
site.debug = True
download_queue = set()
####################################################################################################
####################################################################################################
####################################################################################################
####################################################################################################
#def handle_download_queue():
# while True:
# if len(download_queue) > 0:
# item = download_queue.pop()
# youtube.download_video(item)
# time.sleep(2)
#
#DOWNLOAD_QUEUE_THREAD = threading.Thread(target=handle_download_queue)
#DOWNLOAD_QUEUE_THREAD.daemon = True
#DOWNLOAD_QUEUE_THREAD.start()
def make_json_response(j, *args, **kwargs):
dumped = json.dumps(j)
response = flask.Response(dumped, *args, **kwargs)
response.headers['Content-Type'] = 'application/json;charset=utf-8'
return response
def send_file(filepath):
'''
Range-enabled file sending.
'''
try:
file_size = os.path.getsize(filepath)
except FileNotFoundError:
flask.abort(404)
outgoing_headers = {}
mimetype = mimetypes.guess_type(filepath)[0]
if mimetype is not None:
if 'text/' in mimetype:
mimetype += '; charset=utf-8'
outgoing_headers['Content-Type'] = mimetype
if 'range' in request.headers:
desired_range = request.headers['range'].lower()
desired_range = desired_range.split('bytes=')[-1]
int_helper = lambda x: int(x) if x.isdigit() else None
if '-' in desired_range:
(desired_min, desired_max) = desired_range.split('-')
range_min = int_helper(desired_min)
range_max = int_helper(desired_max)
else:
range_min = int_helper(desired_range)
if range_min is None:
range_min = 0
if range_max is None:
range_max = file_size
# because ranges are 0-indexed
range_max = min(range_max, file_size - 1)
range_min = max(range_min, 0)
range_header = 'bytes {min}-{max}/{outof}'.format(
min=range_min,
max=range_max,
outof=file_size,
)
outgoing_headers['Content-Range'] = range_header
status = 206
else:
range_max = file_size - 1
range_min = 0
status = 200
outgoing_headers['Accept-Ranges'] = 'bytes'
outgoing_headers['Content-Length'] = (range_max - range_min) + 1
if request.method == 'HEAD':
outgoing_data = bytes()
else:
outgoing_data = helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max)
response = flask.Response(
outgoing_data,
status=status,
headers=outgoing_headers,
)
return response
def truthystring(s):
if isinstance(s, (bool, int)) or s is None:
return s
s = s.lower()
if s in {'1', 'true', 't', 'yes', 'y', 'on'}:
return True
if s in {'null', 'none'}:
return None
return False
####################################################################################################
####################################################################################################
####################################################################################################
####################################################################################################
@site.route('/')
def root():
return flask.render_template('root.html')
@site.route('/channels')
def get_channels():
channels = youtube.get_channels()
for channel in channels:
channel['has_pending'] = youtube.channel_has_pending(channel['id'])
return flask.render_template('channels.html', channels=channels)
@site.route('/channel/<channel_id>')
@site.route('/channel/<channel_id>/<download_filter>')
def get_channel(channel_id, download_filter=None):
channel = youtube.get_channel(channel_id)
if channel is None:
flask.abort(404)
videos = youtube.get_videos_by_channel(channel_id)
if download_filter is not None:
videos = [video for video in videos if video['download'] == download_filter]
return flask.render_template('channel.html', channel=channel, videos=videos)
@site.route('/favicon.ico')
@site.route('/favicon.png')
def favicon():
filename = os.path.join('static', 'favicon.png')
return flask.send_file(filename)
@site.route('/static/<filename>')
def get_static(filename):
filename = filename.replace('\\', os.sep)
filename = filename.replace('/', os.sep)
filename = os.path.join('static', filename)
return flask.send_file(filename)
@site.route('/mark_video_state', methods=['POST'])
def post_mark_video_state():
if 'video_id' not in request.form or 'state' not in request.form:
flask.abort(400)
video_id = request.form['video_id']
state = request.form['state']
try:
youtube.mark_video_state(video_id, state)
except KeyError:
flask.abort(404)
except ValueError:
flask.abort(400)
return make_json_response({})
@site.route('/refresh_channel', methods=['POST'])
def post_refresh_channel():
if 'channel_id' not in request.form:
flask.abort(400)
channel_id = request.form['channel_id']
force = request.form.get('force', False)
force = truthystring(force)
print('Refresh channel', channel_id)
youtube.refresh_channel(channel_id, force=force)
return make_json_response({})
@site.route('/refresh_all_channels', methods=['POST'])
def post_refresh_all_channels():
force = request.form.get('force', False)
force = truthystring(force)
for channel in youtube.get_channels():
print('Refresh channel', channel['id'])
youtube.refresh_channel(channel['id'], force=force)
return make_json_response({})
@site.route('/start_download', methods=['POST'])
def post_start_download():
if 'video_id' not in request.form:
flask.abort(400)
video_id = request.form['video_id']
video_info = youtube_core.get_video([video_id])
if video_info == []:
flask.abort(404)
for video in video_info:
#download_queue.add(video)
youtube.download_video(video)
#print(video)
return make_json_response({})
if __name__ == '__main__':
pass

102
YoutubeChannelDownloader/ytapi.py Normal file
View file

@ -0,0 +1,102 @@
import apiclient.discovery
import datetime
import sqlite3
class Video:
def __init__(self, snippet):
self.id = snippet['id']
snippet = snippet['snippet']
self.title = snippet['title'] or '[untitled]'
self.description = snippet['description']
self.author_id = snippet['channelId']
self.author_name = snippet['channelTitle']
# Something like '2016-10-01T21:00:01'
self.published_string = snippet['publishedAt']
published = snippet['publishedAt']
published = published.split('.')[0]
published = datetime.datetime.strptime(published, '%Y-%m-%dT%H:%M:%S')
self.published = published.timestamp()
thumbnails = snippet['thumbnails']
best_thumbnail = max(thumbnails, key=lambda x: thumbnails[x]['width'] * thumbnails[x]['height'])
self.thumbnail = thumbnails[best_thumbnail]
class Youtube:
def __init__(self, key):
youtube = apiclient.discovery.build(
developerKey=key,
serviceName='youtube',
version='v3',
)
self.youtube = youtube
def get_user_name(self, uid):
user = self.youtube.channels().list(part='snippet', id=uid).execute()
return user['items'][0]['snippet']['title']
def get_user_videos(self, username=None, uid=None):
if username:
user = self.youtube.channels().list(part='contentDetails', forUsername=username).execute()
else:
user = self.youtube.channels().list(part='contentDetails', id=uid).execute()
upload_playlist = user['items'][0]['contentDetails']['relatedPlaylists']['uploads']
page_token = None
while True:
items = self.youtube.playlistItems().list(
maxResults=50,
pageToken=page_token,
part='contentDetails',
playlistId=upload_playlist,
).execute()
page_token = items.get('nextPageToken', None)
new = [item['contentDetails']['videoId'] for item in items['items']]
count = len(new)
new = self.get_video(new)
new.sort(key=lambda x: x.published, reverse=True)
yield from new
#print('Found %d more, %d total' % (count, len(videos)))
if page_token is None or count < 50:
break
def get_video(self, video_ids):
if isinstance(video_ids, str):
singular = True
video_ids = [video_ids]
else:
singular = False
video_ids = chunk_sequence(video_ids, 50)
results = []
for chunk in video_ids:
chunk = ','.join(chunk)
data = self.youtube.videos().list(part='snippet', id=chunk).execute()
items = data['items']
results += items
#print('Found %d more, %d total' % (len(items), len(results)))
results = [Video(snippet) for snippet in results]
if singular and len(results) == 1:
return results[0]
return results
def chunk_sequence(sequence, chunk_length, allow_incomplete=True):
"""Given a sequence, divide it into sequences of length `chunk_length`.
:param allow_incomplete: If True, allow the final chunk to be shorter if the
given sequence is not an exact multiple of `chunk_length`.
If False, the incomplete chunk will be discarded.
"""
(complete, leftover) = divmod(len(sequence), chunk_length)
if not allow_incomplete:
leftover = 0
chunk_count = complete + min(leftover, 1)
chunks = []
for x in range(chunk_count):
left = chunk_length * x
right = left + chunk_length
chunks.append(sequence[left:right])
return chunks

20
YoutubeChannelDownloader/ytqueue.py Normal file
View file

@ -0,0 +1,20 @@
'''
I was having trouble making my Flask server perform the youtube-dl without
slowing down and clogging up the other site activities. So instead I'll just
have the server export ytqueue files, which this script will download
as a separate process.
'''
import os
import time
YOUTUBE_DL = 'youtube-dlw https://www.youtube.com/watch?v={id}'
while True:
print(time.strftime('%H:%M:%S'), 'Looking for files.')
queue = [f for f in os.listdir() if f.endswith('.ytqueue')]
for filename in queue:
yt_id = filename.split('.')[0]
command = YOUTUBE_DL.format(id=yt_id)
os.system(command)
os.remove(filename)
time.sleep(10)