From 3c55ee395f2fbfb03210b8b40d0a9946796d68d1 Mon Sep 17 00:00:00 2001 From: Herwin Date: Mon, 28 Feb 2022 03:25:15 +0100 Subject: [PATCH] Finished dockerfile, fixed logging error in threads, Improved readmes, Updated dependencies Update Dockerfile, config.json, and 4 more files... --- Dockerfile | 34 +++++++++++++++++----------------- config.json | 13 ++++++------- readme.md | 31 ++++++++++++++++++++++++++++--- requirements.txt | 2 +- yaa/workers/youtube/live.py | 5 +++-- yaa/workers/youtube/uploads.py | 7 ++++--- 6 files changed, 59 insertions(+), 33 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2a76989..fa3d5e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,38 @@ +# This should be the lightest image. FROM python:alpine -# Installing dependencies +# Reading the build arguments. +# * BUID -> Build UID +# * BGID -> Build GID +ARG BUID=1000 +ARG BGID=1000 + +# Installing dependencies. # * 'g++', 'make' -> Compiling some dependencies. # * 'py3-lxml' -> Pre-compiled module, allows you to ignore compilation errors. - -# Used to compile lxml for streamlink (Not used for the moment) -# RUN apk add --no-cache libxslt-dev libxml2-dev - -# Used for compiling streamlink and its dependencies RUN apk update && apk upgrade && \ apk add --no-cache g++ make py3-lxml ffmpeg -# Installing required Python packages -RUN pip install --upgrade streamlink==2.3.0 yt-dlp==2021.12.01 +# Previously used to compile lxml for streamlink. (Not used for the moment) +# RUN apk add --no-cache libxslt-dev libxml2-dev + +# Installing required Python packages. +ADD --chown=BUID:BGID ./requirements.txt /app/requirements.txt +RUN pip install --upgrade -r /app/requirements.txt -# Removing packages that are no longer needed and were used for compiling streamlink and its dependencies +# Removing packages that are no longer needed and were used for compiling streamlink and its dependencies. # * Some are left however: libgcc, libstdc++, libgomp and gmp # * Reduces the HDD usage from 198 MiB to 127 MiB for these packages RUN apk del g++ make -# Reading the build arguments -# * BUID -> Build UID -# * BGID -> Build GID -ARG BUID=1000 -ARG BGID=1000 - # Copying the app's files to the container ADD --chown=BUID:BGID . /app -# Running the application +# Running the application. # * Thanks to jdpus for the "-u" flag, it fixes the output (https://stackoverflow.com/a/29745541/4135541) WORKDIR /app USER $BUID:$BGID CMD ["python", "-u", "/app/app.py"] -# Used for debugging +# Used for debugging. #CMD ["tail", "-f", "/dev/null"] diff --git a/config.json b/config.json index 471b987..113844a 100644 --- a/config.json +++ b/config.json @@ -9,7 +9,6 @@ }, "youtube": { "output_subdir": "./youtube", - "general_prefix": "yt-", "delay_ms_metadata_download": 30000, "logging_level_worker": 10, "logging_level_thread": 10, @@ -19,10 +18,10 @@ "channel_id": "RogueInternetMan", "name": "Rogue, Internet Man", "output_subdir": "./rogue_internet_man", - "live_subdir": "./uploads", - "upload_subdir": "./livestreams", + "live_subdir": "./livestreams", + "upload_subdir": "./uploads", "check_live": true, - "check_upload": true, + "check_upload": false, "interval_ms_live": 90000, "interval_ms_upload": 3700000, "quality_live": "best", @@ -38,10 +37,10 @@ "channel_id": "propainkey", "name": "propainkey", "output_subdir": "./propainkey", - "live_subdir": "./uploads", - "upload_subdir": "./livestreams", + "live_subdir": "./livestreams", + "upload_subdir": "./uploads", "check_live": false, - "check_upload": false, + "check_upload": true, "interval_ms_live": 120000, "interval_ms_upload": 5100000, "quality_live": "best", diff --git a/readme.md b/readme.md index 9c4e77f..9a22c89 100644 --- a/readme.md +++ b/readme.md @@ -26,11 +26,14 @@ The application isn't designed to be used by another one as a module. * Configurable delays, actions, locations per channel. * Planned for v1.0.0 * Native support for cookies for *yt-dlp* and maybe *streamlink*. + * Better support to prevent command injection. (Will block some features if used) +* Planned for later + * Using TOML for the config file only Python 3.11 is released and stable. ## Requirements -* [Python](https://www.python.org/) v3.9 -* [Streamlink](https://streamlink.github.io/) v3.1.1 -* [yt-dlp](https://github.com/yt-dlp/yt-dlp) v2021.09.02 +* [Python](https://www.python.org/) >= v3.9, < v3.11 (Preferably) +* [Streamlink](https://streamlink.github.io/) >= v3.1.1, < v4.\*.\* +* [yt-dlp](https://github.com/yt-dlp/yt-dlp) >= v2022.02.04 The application may work just fine with older or newer versions of these pieces of software, but they will not be supported.
All requirements, except for *Python*, can be installed via *pip* or manually. @@ -98,6 +101,28 @@ appropriate [environment variables](#environment-variables) tells the applicatio Please refer to [config.md](config.md) for more information on the config file and its fields. +## Build Arguments *(Docker)* + + + + + + + + + + + + + + + + + + + +
VariableTypeRemarkDefault
BUIDIntegerUser ID under which the app runs and who owns the app's directory.
May not work on Windows and NTFS volumes !		1000
BGIDIntegerGroup ID under which the app runs and who owns the app's directory.
May not work on Windows and NTFS volumes !		1000
+ ## Environment Variables Any environment variable that is not set will have the effect of its default value. diff --git a/requirements.txt b/requirements.txt index 9c52535..e4791bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ streamlink>=3.1.1,<4.0.0 -yt-dlp==2021.12.01 +yt-dlp>=2022.02.04 diff --git a/yaa/workers/youtube/live.py b/yaa/workers/youtube/live.py index 8c88773..6f1502e 100644 --- a/yaa/workers/youtube/live.py +++ b/yaa/workers/youtube/live.py @@ -40,8 +40,6 @@ def __thread_yt_live(worker: YouTubeWorker, **args): :param args: Raw arguments passed by workers. (Not used) :return: Nothing. (See the description for more info) """ - # Makes non-debug logs possible to debug from a bird's eye view. - worker.logger_thread.info("Running 'YouTube Live' thread for '{}' !".format(worker.channel.channel_config.name)) # Preparing the logger for the 1st time if needed. if worker.logger_thread is None: @@ -50,6 +48,9 @@ def __thread_yt_live(worker: YouTubeWorker, **args): worker.channel.config.youtube.logging_level_thread ) + # Makes non-debug logs possible to debug from a bird's eye view. + worker.logger_thread.info("Running 'YouTube Live' thread for '{}' !".format(worker.channel.channel_config.name)) + # Preparing the base filename with no extension file_base_name: str = "{}-live-{}".format( worker.channel.channel_config.internal_id, diff --git a/yaa/workers/youtube/uploads.py b/yaa/workers/youtube/uploads.py index d62c73d..50c9a99 100644 --- a/yaa/workers/youtube/uploads.py +++ b/yaa/workers/youtube/uploads.py @@ -39,9 +39,6 @@ def __thread_yt_upload(worker: YouTubeWorker, **args): :param args: Raw arguments passed by workers. (Not used) :return: Nothing. (See the description for more info) """ - # Makes non-debug logs possible to debug from a bird's eye view. - worker.logger_thread.info("Running 'YouTube Uploads' thread for '{}' !".format(worker.channel.channel_config.name)) - # Preparing the logger for the 1st time if needed. if worker.logger_thread is None: worker.logger_thread = get_logger( @@ -49,6 +46,9 @@ def __thread_yt_upload(worker: YouTubeWorker, **args): worker.channel.config.youtube.logging_level_thread ) + # Makes non-debug logs possible to debug from a bird's eye view. + worker.logger_thread.info("Running 'YouTube Uploads' thread for '{}' !".format(worker.channel.channel_config.name)) + # Preparing the command command: str = "yt-dlp --no-warnings --newline --no-progress --dateafter now-{}days {}{}-f {} {}" \ "https://www.youtube.com/c/{}" @@ -91,6 +91,7 @@ def __thread_yt_upload(worker: YouTubeWorker, **args): worker.last_return_code = process.returncode + # TODO: Figure out a safe way to re-encode these lines for 0.8.0 ! lines = process.stdout.readlines() process.stdout.flush() for line in lines: