else
This commit is contained in:
Ethan Dalool
parent
cea8ef6007
commit
4e1880f9a0
15 changed files with 973 additions and 6 deletions
|
|
@ -1,5 +1,10 @@
|
||||||
<!DOCTYPE html5>
|
<!DOCTYPE html5>
|
||||||
<html>
|
<html>
|
||||||
|
<head>
|
||||||
|
{% import "header.html" as header %}
|
||||||
|
<title>Flasksite</title>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<link rel="stylesheet" href="/static/common.css">
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
body, a
|
body, a
|
||||||
|
|
@ -10,12 +15,6 @@ body, a
|
||||||
align-items: center;
|
align-items: center;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|
||||||
<head>
|
|
||||||
{% import "header.html" as header %}
|
|
||||||
<title>Flasksite</title>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<link rel="stylesheet" href="/static/common.css">
|
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
|
|
|
||||||
23
Templates/flask/templates/template.html
Normal file
23
Templates/flask/templates/template.html
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
<!DOCTYPE html5>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
{% import "header.html" as header %}
|
||||||
|
<title>Flasksite</title>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<link rel="stylesheet" href="/static/common.css">
|
||||||
|
|
||||||
|
<style>
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<div id="content_body">
|
||||||
|
<p>test</p>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
</script>
|
||||||
32
YoutubeChannelDownloader/static/common.css
Normal file
32
YoutubeChannelDownloader/static/common.css
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
body
|
||||||
|
{
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
background-color:#fff;
|
||||||
|
margin: 8px;
|
||||||
|
}
|
||||||
|
#header
|
||||||
|
{
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
justify-content: center;
|
||||||
|
align-content: center;
|
||||||
|
margin-bottom: 4px;
|
||||||
|
}
|
||||||
|
.header_element
|
||||||
|
{
|
||||||
|
display: flex;
|
||||||
|
justify-content: center;
|
||||||
|
flex: 1;
|
||||||
|
background-color: rgba(0, 0, 0, 0.1);
|
||||||
|
}
|
||||||
|
.header_element:hover
|
||||||
|
{
|
||||||
|
background-color: rgba(0, 0, 0, 0.2);
|
||||||
|
}
|
||||||
|
#content_body
|
||||||
|
{
|
||||||
|
flex: 0 0 auto;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
}
|
||||||
83
YoutubeChannelDownloader/static/common.js
Normal file
83
YoutubeChannelDownloader/static/common.js
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
function post_example(key, value, callback)
|
||||||
|
{
|
||||||
|
var url = "/postexample";
|
||||||
|
data = new FormData();
|
||||||
|
data.append(key, value);
|
||||||
|
return post(url, data, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
function null_callback()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
function post(url, data, callback)
|
||||||
|
{
|
||||||
|
var request = new XMLHttpRequest();
|
||||||
|
request.answer = null;
|
||||||
|
request.onreadystatechange = function()
|
||||||
|
{
|
||||||
|
if (request.readyState == 4)
|
||||||
|
{
|
||||||
|
var text = request.responseText;
|
||||||
|
if (callback != null)
|
||||||
|
{
|
||||||
|
console.log(text);
|
||||||
|
callback(JSON.parse(text));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
var asynchronous = true;
|
||||||
|
request.open("POST", url, asynchronous);
|
||||||
|
request.send(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bind_box_to_button(box, button)
|
||||||
|
{
|
||||||
|
box.onkeydown=function()
|
||||||
|
{
|
||||||
|
if (event.keyCode == 13)
|
||||||
|
{
|
||||||
|
button.click();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function entry_with_history_hook(box, button)
|
||||||
|
{
|
||||||
|
//console.log(event.keyCode);
|
||||||
|
if (box.entry_history === undefined)
|
||||||
|
{box.entry_history = [];}
|
||||||
|
if (box.entry_history_pos === undefined)
|
||||||
|
{box.entry_history_pos = -1;}
|
||||||
|
if (event.keyCode == 13)
|
||||||
|
{
|
||||||
|
/* Enter */
|
||||||
|
box.entry_history.push(box.value);
|
||||||
|
button.click();
|
||||||
|
box.value = "";
|
||||||
|
}
|
||||||
|
else if (event.keyCode == 38)
|
||||||
|
{
|
||||||
|
|
||||||
|
/* Up arrow */
|
||||||
|
if (box.entry_history.length == 0)
|
||||||
|
{return}
|
||||||
|
if (box.entry_history_pos == -1)
|
||||||
|
{
|
||||||
|
box.entry_history_pos = box.entry_history.length - 1;
|
||||||
|
}
|
||||||
|
else if (box.entry_history_pos > 0)
|
||||||
|
{
|
||||||
|
box.entry_history_pos -= 1;
|
||||||
|
}
|
||||||
|
box.value = box.entry_history[box.entry_history_pos];
|
||||||
|
}
|
||||||
|
else if (event.keyCode == 27)
|
||||||
|
{
|
||||||
|
box.value = "";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
box.entry_history_pos = -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
BIN
YoutubeChannelDownloader/static/favicon.png
Normal file
BIN
YoutubeChannelDownloader/static/favicon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 448 B |
145
YoutubeChannelDownloader/templates/channel.html
Normal file
145
YoutubeChannelDownloader/templates/channel.html
Normal file
|
|
@ -0,0 +1,145 @@
|
||||||
|
<!DOCTYPE html5>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
{% import "header.html" as header %}
|
||||||
|
<title>{{channel['name']}}</title>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<link rel="stylesheet" href="/static/common.css">
|
||||||
|
<script src="/static/common.js"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
#content_body
|
||||||
|
{
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
.video_card_downloaded,
|
||||||
|
.video_card_ignored,
|
||||||
|
.video_card_pending
|
||||||
|
{
|
||||||
|
position: relative;
|
||||||
|
margin: 8px;
|
||||||
|
padding: 10px;
|
||||||
|
border-radius: 4px;
|
||||||
|
border: 1px solid #000;
|
||||||
|
}
|
||||||
|
.video_card_pending
|
||||||
|
{
|
||||||
|
background-color: #ffffaa;
|
||||||
|
}
|
||||||
|
.video_card_ignored
|
||||||
|
{
|
||||||
|
background-color: #ffc886;
|
||||||
|
}
|
||||||
|
.video_card_downloaded
|
||||||
|
{
|
||||||
|
background-color: #aaffaa;
|
||||||
|
}
|
||||||
|
|
||||||
|
.action_toolbox
|
||||||
|
{
|
||||||
|
float: right;
|
||||||
|
display: inline-flex;
|
||||||
|
flex-direction: column;
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
.video_action_dropdown
|
||||||
|
{
|
||||||
|
z-index: 1;
|
||||||
|
background-color: #fff;
|
||||||
|
padding: 4px;
|
||||||
|
border: 1px solid #000;
|
||||||
|
position: absolute;
|
||||||
|
top: 100%;
|
||||||
|
right: 0;
|
||||||
|
display: none;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
.refresh_button
|
||||||
|
{
|
||||||
|
width: 10%;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
|
||||||
|
<body>
|
||||||
|
{{header.make_header()}}
|
||||||
|
<div id="content_body">
|
||||||
|
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', false, null_callback)">Refresh new videos</button>
|
||||||
|
<button class="refresh_button" onclick="refresh_channel('{{channel['id']}}', true, null_callback)">Refresh everything</button>
|
||||||
|
<span><a href="/channel/{{channel['id']}}">All</a></span>
|
||||||
|
<span><a href="/channel/{{channel['id']}}/pending">Pending</a></span>
|
||||||
|
<span><a href="/channel/{{channel['id']}}/ignored">Ignored</a></span>
|
||||||
|
<span><a href="/channel/{{channel['id']}}/downloaded">Downloaded</a></span>
|
||||||
|
{% for video in videos %}
|
||||||
|
|
||||||
|
{% if video['download'] == "downloaded" %}
|
||||||
|
<div class="video_card_downloaded">
|
||||||
|
{% elif video['download'] == "ignored" %}
|
||||||
|
<div class="video_card_ignored">
|
||||||
|
{% else %}
|
||||||
|
<div class="video_card_pending">
|
||||||
|
{% endif %}
|
||||||
|
<a href="https://www.youtube.com/watch?v={{video['id']}}">{{video['title']}}</a>
|
||||||
|
<div class="action_toolbox">
|
||||||
|
<button class="video_action_dropdown_toggle" onclick="toggle_dropdown(this.nextElementSibling)">Actions</button>
|
||||||
|
<div class="video_action_dropdown">
|
||||||
|
{% if video['download'] == "downloaded" %}
|
||||||
|
<button class="video_action_ignore" onclick="mark_video_state('{{video['id']}}', 'pending', null_callback); toggle_dropdown(this.parentElement);">Revert to Pending</button>
|
||||||
|
{% elif video['download'] == "ignored" %}
|
||||||
|
<button class="video_action_ignore" onclick="mark_video_state('{{video['id']}}', 'pending', null_callback); toggle_dropdown(this.parentElement);">Revert to Pending</button>
|
||||||
|
{% else %}
|
||||||
|
<button class="video_action_download" onclick="start_download('{{video['id']}}', null_callback); toggle_dropdown(this.parentElement);">Download</button>
|
||||||
|
<button class="video_action_ignore" onclick="mark_video_state('{{video['id']}}', 'ignored', null_callback); toggle_dropdown(this.parentElement);">Ignore</button>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
function refresh_channel(channel_id, force, callback)
|
||||||
|
{
|
||||||
|
var url = "/refresh_channel";
|
||||||
|
data = new FormData();
|
||||||
|
data.append("channel_id", channel_id);
|
||||||
|
data.append("force", force)
|
||||||
|
return post(url, data, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
function mark_video_state(video_id, state, callback)
|
||||||
|
{
|
||||||
|
var url = "/mark_video_state";
|
||||||
|
data = new FormData();
|
||||||
|
data.append("video_id", video_id);
|
||||||
|
data.append("state", state);
|
||||||
|
return post(url, data, callback);
|
||||||
|
}
|
||||||
|
function start_download(video_id, callback)
|
||||||
|
{
|
||||||
|
var url = "/start_download";
|
||||||
|
data = new FormData();
|
||||||
|
data.append("video_id", video_id);
|
||||||
|
return post(url, data, callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function toggle_dropdown(dropdown)
|
||||||
|
{
|
||||||
|
if (dropdown.style.display != "inline-flex")
|
||||||
|
{
|
||||||
|
dropdown.style.display = "inline-flex";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
dropdown.style.display = "none";
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
</script>
|
||||||
67
YoutubeChannelDownloader/templates/channels.html
Normal file
67
YoutubeChannelDownloader/templates/channels.html
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
<!DOCTYPE html5>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
{% import "header.html" as header %}
|
||||||
|
<title>Channels</title>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<link rel="stylesheet" href="/static/common.css">
|
||||||
|
<script src="/static/common.js"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
#content_body
|
||||||
|
{
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
.channel_card_downloaded,
|
||||||
|
.channel_card_pending
|
||||||
|
{
|
||||||
|
margin: 8px;
|
||||||
|
padding: 10px;
|
||||||
|
border-radius: 4px;
|
||||||
|
border: 1px solid #000;
|
||||||
|
}
|
||||||
|
.channel_card_pending
|
||||||
|
{
|
||||||
|
background-color: #ffffaa;
|
||||||
|
}
|
||||||
|
.channel_card_downloaded
|
||||||
|
{
|
||||||
|
background-color: #aaffaa;
|
||||||
|
}
|
||||||
|
.refresh_button
|
||||||
|
{
|
||||||
|
width: 10%;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
|
||||||
|
<body>
|
||||||
|
{{header.make_header()}}
|
||||||
|
<div id="content_body">
|
||||||
|
<button class="refresh_button" onclick="refresh_all_channels(false)">Refresh new videos</button>
|
||||||
|
<button class="refresh_button" onclick="refresh_all_channels(true)">Refresh everything</button>
|
||||||
|
{% for channel in channels %}
|
||||||
|
{% if channel['has_pending'] %}
|
||||||
|
<div class="channel_card_pending">
|
||||||
|
{% else %}
|
||||||
|
<div class="channel_card_downloaded">
|
||||||
|
{% endif %}
|
||||||
|
<a href="/channel/{{channel['id']}}">{{channel['name']}}</a>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
function refresh_all_channels(force)
|
||||||
|
{
|
||||||
|
var url = "/refresh_all_channels";
|
||||||
|
data = new FormData();
|
||||||
|
data.append("force", force)
|
||||||
|
return post(url, data, null_callback);
|
||||||
|
}
|
||||||
|
</script>
|
||||||
6
YoutubeChannelDownloader/templates/header.html
Normal file
6
YoutubeChannelDownloader/templates/header.html
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
{% macro make_header() %}
|
||||||
|
<div id="header">
|
||||||
|
<a class="header_element" href="/">Home</a>
|
||||||
|
<a class="header_element" href="/channels">Channels</a>
|
||||||
|
</div>
|
||||||
|
{% endmacro %}
|
||||||
28
YoutubeChannelDownloader/templates/root.html
Normal file
28
YoutubeChannelDownloader/templates/root.html
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
<!DOCTYPE html5>
|
||||||
|
<html>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
body, a
|
||||||
|
{
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
justify-content: center;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
{% import "header.html" as header %}
|
||||||
|
<title>Flasksite</title>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<link rel="stylesheet" href="/static/common.css">
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<a href='/channels'>Manage channels</a>
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
</script>
|
||||||
214
YoutubeChannelDownloader/ycdl.py
Normal file
214
YoutubeChannelDownloader/ycdl.py
Normal file
|
|
@ -0,0 +1,214 @@
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import ytapi
|
||||||
|
|
||||||
|
# AVAILABLE FORMATTERS:
|
||||||
|
# url, id
|
||||||
|
YOUTUBE_DL_COMMAND = 'touch {id}.ytqueue'
|
||||||
|
|
||||||
|
SQL_CHANNEL_COLUMNS = [
|
||||||
|
'id',
|
||||||
|
'name',
|
||||||
|
'directory',
|
||||||
|
]
|
||||||
|
|
||||||
|
SQL_VIDEO_COLUMNS = [
|
||||||
|
'id',
|
||||||
|
'published',
|
||||||
|
'author_id',
|
||||||
|
'title',
|
||||||
|
'description',
|
||||||
|
'thumbnail',
|
||||||
|
'download',
|
||||||
|
]
|
||||||
|
|
||||||
|
SQL_CHANNEL = {key:index for (index, key) in enumerate(SQL_CHANNEL_COLUMNS)}
|
||||||
|
SQL_VIDEO = {key:index for (index, key) in enumerate(SQL_VIDEO_COLUMNS)}
|
||||||
|
|
||||||
|
DATABASE_VERSION = 1
|
||||||
|
DB_INIT = '''
|
||||||
|
PRAGMA count_changes = OFF;
|
||||||
|
PRAGMA cache_size = 10000;
|
||||||
|
PRAGMA user_version = {user_version};
|
||||||
|
CREATE TABLE IF NOT EXISTS channels(
|
||||||
|
id TEXT,
|
||||||
|
name TEXT,
|
||||||
|
directory TEXT COLLATE NOCASE
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS videos(
|
||||||
|
id TEXT,
|
||||||
|
published INT,
|
||||||
|
author_id TEXT,
|
||||||
|
title TEXT,
|
||||||
|
description TEXT,
|
||||||
|
thumbnail TEXT,
|
||||||
|
download TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS index_channel_id on channels(id);
|
||||||
|
CREATE INDEX IF NOT EXISTS index_video_id on videos(id);
|
||||||
|
CREATE INDEX IF NOT EXISTS index_video_published on videos(published);
|
||||||
|
CREATE INDEX IF NOT EXISTS index_video_download on videos(download);
|
||||||
|
|
||||||
|
'''.format(user_version=DATABASE_VERSION)
|
||||||
|
|
||||||
|
DEFAULT_DBNAME = 'ycdl.db'
|
||||||
|
|
||||||
|
ERROR_DATABASE_OUTOFDATE = 'Database is out-of-date. {current} should be {new}'
|
||||||
|
|
||||||
|
def verify_is_abspath(path):
|
||||||
|
'''
|
||||||
|
TO DO: Determine whether this is actually correct.
|
||||||
|
'''
|
||||||
|
if os.path.abspath(path) != path:
|
||||||
|
raise ValueError('Not an abspath')
|
||||||
|
|
||||||
|
class YCDL:
|
||||||
|
def __init__(self, youtube, database_filename=None):
|
||||||
|
self.youtube = youtube
|
||||||
|
if database_filename is None:
|
||||||
|
database_filename = DEFAULT_DBNAME
|
||||||
|
|
||||||
|
existing_database = os.path.exists(database_filename)
|
||||||
|
self.sql = sqlite3.connect(database_filename)
|
||||||
|
self.cur = self.sql.cursor()
|
||||||
|
|
||||||
|
if existing_database:
|
||||||
|
self.cur.execute('PRAGMA user_version')
|
||||||
|
existing_version = self.cur.fetchone()[0]
|
||||||
|
if existing_version != DATABASE_VERSION:
|
||||||
|
message = ERROR_DATABASE_OUTOFDATE
|
||||||
|
message = message.format(current=existing_version, new=DATABASE_VERSION)
|
||||||
|
print(message)
|
||||||
|
raise SystemExit
|
||||||
|
|
||||||
|
statements = DB_INIT.split(';')
|
||||||
|
for statement in statements:
|
||||||
|
self.cur.execute(statement)
|
||||||
|
|
||||||
|
def add_channel(self, channel_id, name=None, download_directory=None, get_videos=True, commit=False):
|
||||||
|
if self.get_channel(channel_id) is not None:
|
||||||
|
return
|
||||||
|
|
||||||
|
if name is None:
|
||||||
|
name = self.youtube.get_user_name(channel_id)
|
||||||
|
|
||||||
|
data = [None] * len(SQL_CHANNEL)
|
||||||
|
data[SQL_CHANNEL['id']] = channel_id
|
||||||
|
data[SQL_CHANNEL['name']] = name
|
||||||
|
if download_directory is not None:
|
||||||
|
verify_is_abspath(download_directory)
|
||||||
|
data[SQL_CHANNEL['directory']] = download_directory
|
||||||
|
|
||||||
|
self.cur.execute('INSERT INTO channels VALUES(?, ?, ?)', data)
|
||||||
|
if get_videos:
|
||||||
|
self.refresh_channel(channel_id, commit=False)
|
||||||
|
if commit:
|
||||||
|
self.sql.commit()
|
||||||
|
|
||||||
|
def channel_has_pending(self, channel_id):
|
||||||
|
self.cur.execute('SELECT * FROM videos WHERE author_id == ? AND download == "pending"', [channel_id])
|
||||||
|
return self.cur.fetchone() is not None
|
||||||
|
|
||||||
|
def channel_directory(self, channel_id):
|
||||||
|
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
|
||||||
|
fetch = self.cur.fetchone()
|
||||||
|
if fetch is None:
|
||||||
|
return None
|
||||||
|
return fetch[SQL_CHANNEL['directory']]
|
||||||
|
|
||||||
|
def download_video(self, video, force=False):
|
||||||
|
if not isinstance(video, ytapi.Video):
|
||||||
|
video = self.youtube.get_video(video)
|
||||||
|
|
||||||
|
self.add_channel(video.author_id, get_videos=False, commit=False)
|
||||||
|
status = self.insert_video(video, commit=True)
|
||||||
|
|
||||||
|
if status['row'][SQL_VIDEO['download']] != 'pending' and not force:
|
||||||
|
print('That video does not need to be downloaded.')
|
||||||
|
return
|
||||||
|
|
||||||
|
download_directory = self.channel_directory(video.author_id)
|
||||||
|
download_directory = download_directory or os.getcwd()
|
||||||
|
|
||||||
|
current_directory = os.getcwd()
|
||||||
|
os.makedirs(download_directory, exist_ok=True)
|
||||||
|
os.chdir(download_directory)
|
||||||
|
url = 'https://www.youtube.com/watch?v={id}'.format(id=video.id)
|
||||||
|
command = YOUTUBE_DL_COMMAND.format(url=url, id=video.id)
|
||||||
|
os.system(command)
|
||||||
|
os.chdir(current_directory)
|
||||||
|
|
||||||
|
self.cur.execute('UPDATE videos SET download = "downloaded" WHERE id == ?', [video.id])
|
||||||
|
self.sql.commit()
|
||||||
|
|
||||||
|
def get_channel(self, channel_id):
|
||||||
|
self.cur.execute('SELECT * FROM channels WHERE id == ?', [channel_id])
|
||||||
|
fetch = self.cur.fetchone()
|
||||||
|
if not fetch:
|
||||||
|
return None
|
||||||
|
fetch = {key: fetch[SQL_CHANNEL[key]] for key in SQL_CHANNEL}
|
||||||
|
return fetch
|
||||||
|
|
||||||
|
def get_channels(self):
|
||||||
|
self.cur.execute('SELECT * FROM channels')
|
||||||
|
channels = self.cur.fetchall()
|
||||||
|
channels = [{key: channel[SQL_CHANNEL[key]] for key in SQL_CHANNEL} for channel in channels]
|
||||||
|
channels.sort(key=lambda x: x['name'].lower())
|
||||||
|
return channels
|
||||||
|
|
||||||
|
def get_videos_by_channel(self, channel_id):
|
||||||
|
self.cur.execute('SELECT * FROM videos WHERE author_id == ?', [channel_id])
|
||||||
|
videos = self.cur.fetchall()
|
||||||
|
if not videos:
|
||||||
|
return []
|
||||||
|
videos = [{key: video[SQL_VIDEO[key]] for key in SQL_VIDEO} for video in videos]
|
||||||
|
videos.sort(key=lambda x: x['published'], reverse=True)
|
||||||
|
return videos
|
||||||
|
|
||||||
|
def mark_video_state(self, video_id, state, commit=True):
|
||||||
|
'''
|
||||||
|
Mark the video as ignored, pending, or downloaded.
|
||||||
|
'''
|
||||||
|
if state not in ['ignored', 'pending', 'downloaded']:
|
||||||
|
raise ValueError(state)
|
||||||
|
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video_id])
|
||||||
|
if self.cur.fetchone() is None:
|
||||||
|
raise KeyError(video_id)
|
||||||
|
self.cur.execute('UPDATE videos SET download = ? WHERE id == ?', [state, video_id])
|
||||||
|
if commit:
|
||||||
|
self.sql.commit()
|
||||||
|
|
||||||
|
def refresh_channel(self, channel_id, force=True, commit=True):
|
||||||
|
video_generator = self.youtube.get_user_videos(uid=channel_id)
|
||||||
|
for video in video_generator:
|
||||||
|
status = self.insert_video(video, commit=False)
|
||||||
|
if not force and not status['new']:
|
||||||
|
break
|
||||||
|
if commit:
|
||||||
|
self.sql.commit()
|
||||||
|
|
||||||
|
def insert_video(self, video, commit=True):
|
||||||
|
if not isinstance(video, ytapi.Video):
|
||||||
|
video = self.youtube.get_video(video)
|
||||||
|
|
||||||
|
self.add_channel(video.author_id, get_videos=False, commit=False)
|
||||||
|
self.cur.execute('SELECT * FROM videos WHERE id == ?', [video.id])
|
||||||
|
fetch = self.cur.fetchone()
|
||||||
|
if fetch is not None:
|
||||||
|
return {'new': False, 'row': fetch}
|
||||||
|
|
||||||
|
data = [None] * len(SQL_VIDEO)
|
||||||
|
data[SQL_VIDEO['id']] = video.id
|
||||||
|
data[SQL_VIDEO['published']] = video.published
|
||||||
|
data[SQL_VIDEO['author_id']] = video.author_id
|
||||||
|
data[SQL_VIDEO['title']] = video.title
|
||||||
|
data[SQL_VIDEO['description']] = video.description
|
||||||
|
data[SQL_VIDEO['thumbnail']] = video.thumbnail['url']
|
||||||
|
data[SQL_VIDEO['download']] = 'pending'
|
||||||
|
|
||||||
|
self.cur.execute('INSERT INTO videos VALUES(?, ?, ?, ?, ?, ?, ?)', data)
|
||||||
|
if commit:
|
||||||
|
self.sql.commit()
|
||||||
|
return {'new': True, 'row': data}
|
||||||
6
YoutubeChannelDownloader/ycdl_easy.py
Normal file
6
YoutubeChannelDownloader/ycdl_easy.py
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
import ytapi
|
||||||
|
import ycdl
|
||||||
|
import bot
|
||||||
|
|
||||||
|
youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
|
||||||
|
youtube = ycdl.YCDL(youtube_core)
|
||||||
29
YoutubeChannelDownloader/ycdl_launch.py
Normal file
29
YoutubeChannelDownloader/ycdl_launch.py
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
import gevent.monkey
|
||||||
|
gevent.monkey.patch_all()
|
||||||
|
|
||||||
|
import ycdl_site
|
||||||
|
import gevent.pywsgi
|
||||||
|
import gevent.wsgi
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if len(sys.argv) == 2:
|
||||||
|
port = int(sys.argv[1])
|
||||||
|
else:
|
||||||
|
port = 5000
|
||||||
|
|
||||||
|
if port == 443:
|
||||||
|
http = gevent.pywsgi.WSGIServer(
|
||||||
|
listener=('', port),
|
||||||
|
application=ycdl_site.site,
|
||||||
|
keyfile='https\\flasksite.key',
|
||||||
|
certfile='https\\flasksite.crt',
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
http = gevent.pywsgi.WSGIServer(
|
||||||
|
listener=('', port),
|
||||||
|
application=ycdl_site.site,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
print('Starting server')
|
||||||
|
http.serve_forever()
|
||||||
213
YoutubeChannelDownloader/ycdl_site.py
Normal file
213
YoutubeChannelDownloader/ycdl_site.py
Normal file
|
|
@ -0,0 +1,213 @@
|
||||||
|
import flask
|
||||||
|
from flask import request
|
||||||
|
import json
|
||||||
|
import mimetypes
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
import ytapi
|
||||||
|
import ycdl
|
||||||
|
import bot
|
||||||
|
|
||||||
|
youtube_core = ytapi.Youtube(bot.YOUTUBE_KEY)
|
||||||
|
youtube = ycdl.YCDL(youtube_core)
|
||||||
|
|
||||||
|
site = flask.Flask(__name__)
|
||||||
|
site.config.update(
|
||||||
|
SEND_FILE_MAX_AGE_DEFAULT=180,
|
||||||
|
TEMPLATES_AUTO_RELOAD=True,
|
||||||
|
)
|
||||||
|
site.jinja_env.add_extension('jinja2.ext.do')
|
||||||
|
site.debug = True
|
||||||
|
|
||||||
|
download_queue = set()
|
||||||
|
|
||||||
|
####################################################################################################
|
||||||
|
####################################################################################################
|
||||||
|
####################################################################################################
|
||||||
|
####################################################################################################
|
||||||
|
|
||||||
|
#def handle_download_queue():
|
||||||
|
# while True:
|
||||||
|
# if len(download_queue) > 0:
|
||||||
|
# item = download_queue.pop()
|
||||||
|
# youtube.download_video(item)
|
||||||
|
# time.sleep(2)
|
||||||
|
#
|
||||||
|
#DOWNLOAD_QUEUE_THREAD = threading.Thread(target=handle_download_queue)
|
||||||
|
#DOWNLOAD_QUEUE_THREAD.daemon = True
|
||||||
|
#DOWNLOAD_QUEUE_THREAD.start()
|
||||||
|
|
||||||
|
def make_json_response(j, *args, **kwargs):
|
||||||
|
dumped = json.dumps(j)
|
||||||
|
response = flask.Response(dumped, *args, **kwargs)
|
||||||
|
response.headers['Content-Type'] = 'application/json;charset=utf-8'
|
||||||
|
return response
|
||||||
|
|
||||||
|
def send_file(filepath):
|
||||||
|
'''
|
||||||
|
Range-enabled file sending.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
file_size = os.path.getsize(filepath)
|
||||||
|
except FileNotFoundError:
|
||||||
|
flask.abort(404)
|
||||||
|
|
||||||
|
outgoing_headers = {}
|
||||||
|
mimetype = mimetypes.guess_type(filepath)[0]
|
||||||
|
if mimetype is not None:
|
||||||
|
if 'text/' in mimetype:
|
||||||
|
mimetype += '; charset=utf-8'
|
||||||
|
outgoing_headers['Content-Type'] = mimetype
|
||||||
|
|
||||||
|
if 'range' in request.headers:
|
||||||
|
desired_range = request.headers['range'].lower()
|
||||||
|
desired_range = desired_range.split('bytes=')[-1]
|
||||||
|
|
||||||
|
int_helper = lambda x: int(x) if x.isdigit() else None
|
||||||
|
if '-' in desired_range:
|
||||||
|
(desired_min, desired_max) = desired_range.split('-')
|
||||||
|
range_min = int_helper(desired_min)
|
||||||
|
range_max = int_helper(desired_max)
|
||||||
|
else:
|
||||||
|
range_min = int_helper(desired_range)
|
||||||
|
|
||||||
|
if range_min is None:
|
||||||
|
range_min = 0
|
||||||
|
if range_max is None:
|
||||||
|
range_max = file_size
|
||||||
|
|
||||||
|
# because ranges are 0-indexed
|
||||||
|
range_max = min(range_max, file_size - 1)
|
||||||
|
range_min = max(range_min, 0)
|
||||||
|
|
||||||
|
range_header = 'bytes {min}-{max}/{outof}'.format(
|
||||||
|
min=range_min,
|
||||||
|
max=range_max,
|
||||||
|
outof=file_size,
|
||||||
|
)
|
||||||
|
outgoing_headers['Content-Range'] = range_header
|
||||||
|
status = 206
|
||||||
|
else:
|
||||||
|
range_max = file_size - 1
|
||||||
|
range_min = 0
|
||||||
|
status = 200
|
||||||
|
|
||||||
|
outgoing_headers['Accept-Ranges'] = 'bytes'
|
||||||
|
outgoing_headers['Content-Length'] = (range_max - range_min) + 1
|
||||||
|
|
||||||
|
if request.method == 'HEAD':
|
||||||
|
outgoing_data = bytes()
|
||||||
|
else:
|
||||||
|
outgoing_data = helpers.read_filebytes(filepath, range_min=range_min, range_max=range_max)
|
||||||
|
|
||||||
|
response = flask.Response(
|
||||||
|
outgoing_data,
|
||||||
|
status=status,
|
||||||
|
headers=outgoing_headers,
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def truthystring(s):
|
||||||
|
if isinstance(s, (bool, int)) or s is None:
|
||||||
|
return s
|
||||||
|
s = s.lower()
|
||||||
|
if s in {'1', 'true', 't', 'yes', 'y', 'on'}:
|
||||||
|
return True
|
||||||
|
if s in {'null', 'none'}:
|
||||||
|
return None
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
####################################################################################################
|
||||||
|
####################################################################################################
|
||||||
|
####################################################################################################
|
||||||
|
####################################################################################################
|
||||||
|
|
||||||
|
@site.route('/')
|
||||||
|
def root():
|
||||||
|
return flask.render_template('root.html')
|
||||||
|
|
||||||
|
@site.route('/channels')
|
||||||
|
def get_channels():
|
||||||
|
channels = youtube.get_channels()
|
||||||
|
for channel in channels:
|
||||||
|
channel['has_pending'] = youtube.channel_has_pending(channel['id'])
|
||||||
|
return flask.render_template('channels.html', channels=channels)
|
||||||
|
|
||||||
|
@site.route('/channel/<channel_id>')
|
||||||
|
@site.route('/channel/<channel_id>/<download_filter>')
|
||||||
|
def get_channel(channel_id, download_filter=None):
|
||||||
|
channel = youtube.get_channel(channel_id)
|
||||||
|
if channel is None:
|
||||||
|
flask.abort(404)
|
||||||
|
videos = youtube.get_videos_by_channel(channel_id)
|
||||||
|
if download_filter is not None:
|
||||||
|
videos = [video for video in videos if video['download'] == download_filter]
|
||||||
|
return flask.render_template('channel.html', channel=channel, videos=videos)
|
||||||
|
|
||||||
|
@site.route('/favicon.ico')
|
||||||
|
@site.route('/favicon.png')
|
||||||
|
def favicon():
|
||||||
|
filename = os.path.join('static', 'favicon.png')
|
||||||
|
return flask.send_file(filename)
|
||||||
|
|
||||||
|
@site.route('/static/<filename>')
|
||||||
|
def get_static(filename):
|
||||||
|
filename = filename.replace('\\', os.sep)
|
||||||
|
filename = filename.replace('/', os.sep)
|
||||||
|
filename = os.path.join('static', filename)
|
||||||
|
return flask.send_file(filename)
|
||||||
|
|
||||||
|
@site.route('/mark_video_state', methods=['POST'])
|
||||||
|
def post_mark_video_state():
|
||||||
|
if 'video_id' not in request.form or 'state' not in request.form:
|
||||||
|
flask.abort(400)
|
||||||
|
video_id = request.form['video_id']
|
||||||
|
state = request.form['state']
|
||||||
|
try:
|
||||||
|
youtube.mark_video_state(video_id, state)
|
||||||
|
except KeyError:
|
||||||
|
flask.abort(404)
|
||||||
|
except ValueError:
|
||||||
|
flask.abort(400)
|
||||||
|
return make_json_response({})
|
||||||
|
|
||||||
|
@site.route('/refresh_channel', methods=['POST'])
|
||||||
|
def post_refresh_channel():
|
||||||
|
if 'channel_id' not in request.form:
|
||||||
|
flask.abort(400)
|
||||||
|
channel_id = request.form['channel_id']
|
||||||
|
force = request.form.get('force', False)
|
||||||
|
force = truthystring(force)
|
||||||
|
print('Refresh channel', channel_id)
|
||||||
|
youtube.refresh_channel(channel_id, force=force)
|
||||||
|
return make_json_response({})
|
||||||
|
|
||||||
|
@site.route('/refresh_all_channels', methods=['POST'])
|
||||||
|
def post_refresh_all_channels():
|
||||||
|
force = request.form.get('force', False)
|
||||||
|
force = truthystring(force)
|
||||||
|
for channel in youtube.get_channels():
|
||||||
|
print('Refresh channel', channel['id'])
|
||||||
|
youtube.refresh_channel(channel['id'], force=force)
|
||||||
|
return make_json_response({})
|
||||||
|
|
||||||
|
@site.route('/start_download', methods=['POST'])
|
||||||
|
def post_start_download():
|
||||||
|
if 'video_id' not in request.form:
|
||||||
|
flask.abort(400)
|
||||||
|
video_id = request.form['video_id']
|
||||||
|
video_info = youtube_core.get_video([video_id])
|
||||||
|
if video_info == []:
|
||||||
|
flask.abort(404)
|
||||||
|
for video in video_info:
|
||||||
|
#download_queue.add(video)
|
||||||
|
youtube.download_video(video)
|
||||||
|
#print(video)
|
||||||
|
return make_json_response({})
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
pass
|
||||||
102
YoutubeChannelDownloader/ytapi.py
Normal file
102
YoutubeChannelDownloader/ytapi.py
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
import apiclient.discovery
|
||||||
|
import datetime
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
class Video:
|
||||||
|
def __init__(self, snippet):
|
||||||
|
self.id = snippet['id']
|
||||||
|
|
||||||
|
snippet = snippet['snippet']
|
||||||
|
self.title = snippet['title'] or '[untitled]'
|
||||||
|
self.description = snippet['description']
|
||||||
|
self.author_id = snippet['channelId']
|
||||||
|
self.author_name = snippet['channelTitle']
|
||||||
|
# Something like '2016-10-01T21:00:01'
|
||||||
|
self.published_string = snippet['publishedAt']
|
||||||
|
published = snippet['publishedAt']
|
||||||
|
published = published.split('.')[0]
|
||||||
|
published = datetime.datetime.strptime(published, '%Y-%m-%dT%H:%M:%S')
|
||||||
|
self.published = published.timestamp()
|
||||||
|
|
||||||
|
thumbnails = snippet['thumbnails']
|
||||||
|
best_thumbnail = max(thumbnails, key=lambda x: thumbnails[x]['width'] * thumbnails[x]['height'])
|
||||||
|
self.thumbnail = thumbnails[best_thumbnail]
|
||||||
|
|
||||||
|
|
||||||
|
class Youtube:
|
||||||
|
def __init__(self, key):
|
||||||
|
youtube = apiclient.discovery.build(
|
||||||
|
developerKey=key,
|
||||||
|
serviceName='youtube',
|
||||||
|
version='v3',
|
||||||
|
)
|
||||||
|
self.youtube = youtube
|
||||||
|
|
||||||
|
def get_user_name(self, uid):
|
||||||
|
user = self.youtube.channels().list(part='snippet', id=uid).execute()
|
||||||
|
return user['items'][0]['snippet']['title']
|
||||||
|
|
||||||
|
def get_user_videos(self, username=None, uid=None):
|
||||||
|
if username:
|
||||||
|
user = self.youtube.channels().list(part='contentDetails', forUsername=username).execute()
|
||||||
|
else:
|
||||||
|
user = self.youtube.channels().list(part='contentDetails', id=uid).execute()
|
||||||
|
upload_playlist = user['items'][0]['contentDetails']['relatedPlaylists']['uploads']
|
||||||
|
page_token = None
|
||||||
|
while True:
|
||||||
|
items = self.youtube.playlistItems().list(
|
||||||
|
maxResults=50,
|
||||||
|
pageToken=page_token,
|
||||||
|
part='contentDetails',
|
||||||
|
playlistId=upload_playlist,
|
||||||
|
).execute()
|
||||||
|
page_token = items.get('nextPageToken', None)
|
||||||
|
new = [item['contentDetails']['videoId'] for item in items['items']]
|
||||||
|
count = len(new)
|
||||||
|
new = self.get_video(new)
|
||||||
|
new.sort(key=lambda x: x.published, reverse=True)
|
||||||
|
yield from new
|
||||||
|
#print('Found %d more, %d total' % (count, len(videos)))
|
||||||
|
if page_token is None or count < 50:
|
||||||
|
break
|
||||||
|
|
||||||
|
def get_video(self, video_ids):
|
||||||
|
if isinstance(video_ids, str):
|
||||||
|
singular = True
|
||||||
|
video_ids = [video_ids]
|
||||||
|
else:
|
||||||
|
singular = False
|
||||||
|
video_ids = chunk_sequence(video_ids, 50)
|
||||||
|
results = []
|
||||||
|
for chunk in video_ids:
|
||||||
|
chunk = ','.join(chunk)
|
||||||
|
data = self.youtube.videos().list(part='snippet', id=chunk).execute()
|
||||||
|
items = data['items']
|
||||||
|
results += items
|
||||||
|
#print('Found %d more, %d total' % (len(items), len(results)))
|
||||||
|
results = [Video(snippet) for snippet in results]
|
||||||
|
if singular and len(results) == 1:
|
||||||
|
return results[0]
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def chunk_sequence(sequence, chunk_length, allow_incomplete=True):
|
||||||
|
"""Given a sequence, divide it into sequences of length `chunk_length`.
|
||||||
|
|
||||||
|
:param allow_incomplete: If True, allow the final chunk to be shorter if the
|
||||||
|
given sequence is not an exact multiple of `chunk_length`.
|
||||||
|
If False, the incomplete chunk will be discarded.
|
||||||
|
"""
|
||||||
|
(complete, leftover) = divmod(len(sequence), chunk_length)
|
||||||
|
if not allow_incomplete:
|
||||||
|
leftover = 0
|
||||||
|
|
||||||
|
chunk_count = complete + min(leftover, 1)
|
||||||
|
|
||||||
|
chunks = []
|
||||||
|
for x in range(chunk_count):
|
||||||
|
left = chunk_length * x
|
||||||
|
right = left + chunk_length
|
||||||
|
chunks.append(sequence[left:right])
|
||||||
|
|
||||||
|
return chunks
|
||||||
20
YoutubeChannelDownloader/ytqueue.py
Normal file
20
YoutubeChannelDownloader/ytqueue.py
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
'''
|
||||||
|
I was having trouble making my Flask server perform the youtube-dl without
|
||||||
|
slowing down and clogging up the other site activities. So instead I'll just
|
||||||
|
have the server export ytqueue files, which this script will download
|
||||||
|
as a separate process.
|
||||||
|
'''
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
YOUTUBE_DL = 'youtube-dlw https://www.youtube.com/watch?v={id}'
|
||||||
|
|
||||||
|
while True:
|
||||||
|
print(time.strftime('%H:%M:%S'), 'Looking for files.')
|
||||||
|
queue = [f for f in os.listdir() if f.endswith('.ytqueue')]
|
||||||
|
for filename in queue:
|
||||||
|
yt_id = filename.split('.')[0]
|
||||||
|
command = YOUTUBE_DL.format(id=yt_id)
|
||||||
|
os.system(command)
|
||||||
|
os.remove(filename)
|
||||||
|
time.sleep(10)
|
||||||
Loading…
Reference in a new issue