Feature innertube client fix

generate_release.py

@@ -92,7 +92,7 @@ def wine_run(command_parts):
 # confused with working directory. I'm calling it the same thing so it will
 # have that name when extracted from the final release zip archive)
 log('Making copy of youtube-local files')
-check(os.system('git archive --format tar master | 7z x -si -ttar -oyoutube-local'))
+check(os.system('git archive --format tar HEAD | 7z x -si -ttar -oyoutube-local'))
 
 if len(os.listdir('./youtube-local')) == 0:
     raise Exception('Failed to copy youtube-local files')
@@ -217,10 +217,10 @@ def wine_run(command_parts):
 log('Removing pyc files')   # Have to do this because get-pip and some packages don't respect --no-compile
 remove_files_with_extensions(r'./python', ['.pyc'])
 
-log('Removing dist-info and __pycache__')
+log('Removing __pycache__')
 for root, dirs, files in os.walk(r'./python'):
     for dir in dirs:
-        if dir == '__pycache__' or dir.endswith('.dist-info'):
+        if dir == '__pycache__':
             shutil.rmtree(os.path.join(root, dir))
 
 

requirements.txt

@@ -6,3 +6,5 @@ urllib3>=1.24.1
 defusedxml>=0.5.0
 cachetools>=4.0.0
 stem>=1.8.0
+fake-useragent>=2.0.0
+dukpy>=0.4.0

server.py

@@ -20,7 +20,8 @@
 import re
 import sys
 import time
-
+import os
+import json
 
 
 
@@ -50,11 +51,49 @@ def parse_range(range_header, content_length):
         end_byte = int(end)
     return start_byte, end_byte
 
+innertube_client_id = settings.innertube_client_id
+innertube_client = util.innertube_client
+client = innertube_client[innertube_client_id]
 def proxy_site(env, start_response, video=False):
+    client_params = util.INNERTUBE_CLIENTS[client]
+    client_context = client_params['INNERTUBE_CONTEXT']
+    client_ua = client_context.get('userAgent') or util.mobile_user_agent
     send_headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)',
+        'User-Agent': client_ua,
         'Accept': '*/*',
+        'Accept-language': 'en-US',
+        'X-YouTube-Client-Name': client_params['INNERTUBE_CONTEXT_CLIENT_NAME'],
+        'X-YouTube-Client-Version': client_context['client']['clientVersion'],
     }
+    visitor_data_file = os.path.join(settings.data_dir,'visitorData.txt')
+    visitor_data = None
+    if not settings.use_po_token:
+        if settings.use_visitor_data:
+            if os.path.exists(visitor_data_file):
+                try:
+                    with open(visitor_data_file, "r") as file:
+                        visitor_data = file.read()
+                        file.close()
+                except OSError:
+                    print('An OS error prevents accessing visitorData.txt file')
+        else:
+            po_token_cache = os.path.join(settings.data_dir, 'po_token_cache.txt')
+            if os.path.exists(po_token_cache):
+                try:
+                    with open(po_token_cache, "r") as file:
+                        po_token_dict = json.loads(file.read())
+                        visitor_data = po_token_dict.get('visitorData') or None
+                        file.close()
+                except OSError:
+                    print('An OS error prevents accessing po_token_cache.txt')
+
+    google_domains = [ 'youtube.com', 'youtube-nocookie.com', 'youtu.be', 'googlevideo.com', 'ytimg.com', 'ggpht.com', 'googleapis.com' ]
+    for domain in google_domains:
+        if env['SERVER_NAME'].endswith(domain):
+            if visitor_data != None:
+                send_headers['X-Goog-Visitor-Id'] = visitor_data
+                break
+
     current_range_start = 0
     range_end = None
     if 'HTTP_RANGE' in env:
@@ -92,6 +131,8 @@ def proxy_site(env, start_response, video=False):
         except AttributeError:
             pass
         if video:
+            send_headers = (list(send_headers)
+                            + [('origin', 'https://www.youtube.com')])
             response_headers = (list(response_headers)
                                 +[('Access-Control-Allow-Origin', '*')])
 

settings.py

@@ -235,6 +235,56 @@
         'category': 'interface',
     }),
 
+    ('use_video_download', {
+        'type': int,
+        'default': 0,
+        'comment': '',
+        'options': [
+            (0, 'Disabled'),
+            (1, 'Enabled'),
+        ],
+        'category': 'interface',
+        'comment': '''If enabled, you may incur legal issues with RIAA. Disabled by default.
+More info: https://torrentfreak.com/riaa-thwarts-youts-attempt-to-declare-youtube-ripping-legal-221002/
+Archive: https://archive.ph/OZQbN''',
+    }),
+
+    ('innertube_client_id', {
+        'type': int,
+        'default': 6,
+        'label': 'innertube client',
+        'comment': '''innertube client to access YouTube API, i.e. fetching player data
+        Available clients: android, android-test-suite, ios, tv_embedded, web, web_creator, mweb
+        ''',
+        'options': [
+            (0, 'android'),
+            (1, 'android-test-suite'),
+            (2, 'ios'),
+            (3, 'tv_embedded'),
+            (4, 'web'),
+            (5, 'web_creator'),
+            (6, 'mweb'),
+        ],
+        'category': 'network',
+    }),
+
+    ('use_visitor_data', {
+        'label': 'Use visitor data',
+        'type': bool,
+        'default': False,
+        'comment': '''Add X-Goog-Visitor-Id header to outgoing http requests''',
+        'category': 'network',
+        }),
+
+    ('use_po_token', {
+        'label': 'Use po_token',
+        'type': bool,
+        'default': False,
+        'comment': '''Use visitor_data and po_token using po-token-generator''',
+        'category': 'network',
+        }),
+
+
     ('proxy_images', {
         'label': 'Route images',
         'type': bool,
@@ -309,6 +359,16 @@
         'comment': '',
     }),
 
+    ('allow_age_restricted_content', {
+        'type': bool,
+        'label': 'Allow playback of age restricted videos',
+        'default': False,
+        'comment': '''Allow playback of age restricted videos.
+If set to True, the system will try reloading the video player when encountering age restricted content.''',
+        'category': 'playback',
+    }),
+
+
     ('gather_googlevideo_domains', {
         'type': bool,
         'default': False,

youtube/channel.py

@@ -30,6 +30,7 @@
     ('X-YouTube-Client-Name', '2'),
     ('X-YouTube-Client-Version', '2.20180830'),
 ) + util.mobile_ua
+headers_client = util.client_xhr_headers
 real_cookie = (('Cookie', 'VISITOR_INFO1_LIVE=8XihrAcN1l4'),)
 generic_cookie = (('Cookie', 'VISITOR_INFO1_LIVE=ST1Ti53r4fU'),)
 
@@ -283,7 +284,7 @@ def get_number_of_videos_channel(channel_id):
     url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&pbj=1'
 
     try:
-        response = util.fetch_url(url, headers_mobile,
+        response = util.fetch_url(url, headers_client,
             debug_name='number_of_videos', report_text='Got number of videos')
     except urllib.error.HTTPError as e:
         traceback.print_exc()

youtube/comments.py

@@ -59,7 +59,7 @@ def request_comments(ctoken, replies=False):
     })
 
     content = util.fetch_url(
-        url, headers=util.mobile_xhr_headers + util.json_header, data=data,
+        url, headers=util.client_xhr_headers + util.json_header, data=data,
         report_text='Retrieved comments', debug_name='request_comments')
     content = content.decode('utf-8')
 

youtube/proto_debug.py

@@ -1,4 +1,5 @@
 # TODO: clean this file up more and heavily refactor
+from fake_useragent import UserAgent
 
 ''' Helper functions for reverse engineering protobuf.
 
@@ -571,15 +572,15 @@ def pp(obj, indent=1):
     print(_pp(obj, indent))
 
 
-desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0'
+desktop_user_agent = UserAgent(os='Windows').firefox
 desktop_headers = (
     ('Accept', '*/*'),
     ('Accept-Language', 'en-US,en;q=0.5'),
     ('X-YouTube-Client-Name', '1'),
     ('X-YouTube-Client-Version', '2.20180830'),
 ) + (('User-Agent', desktop_user_agent),)
 
-mobile_user_agent = 'Mozilla/5.0 (Linux; Android 7.0; Redmi Note 4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36'
+mobile_user_agent = UserAgent(os='Android').chrome
 mobile_headers = (
     ('Accept', '*/*'),
     ('Accept-Language', 'en-US,en;q=0.5'),

youtube/search.py

@@ -42,11 +42,14 @@ def page_number_to_sp_parameter(page, autocorrect, sort, filters):
     result = proto.uint(1, sort) + filters_enc + autocorrect + proto.uint(9, offset) + proto.string(61, b'')
     return base64.urlsafe_b64encode(result).decode('ascii')
 
+desktop_user_agent = util.desktop_user_agent
+mobile_user_agent = util.mobile_user_agent
+
 def get_search_json(query, page, autocorrect, sort, filters):
     url = "https://www.youtube.com/results?search_query=" + urllib.parse.quote_plus(query)
     headers = {
         'Host': 'www.youtube.com',
-        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)',
+        'User-Agent': desktop_user_agent,
         'Accept': '*/*',
         'Accept-Language': 'en-US,en;q=0.5',
         'X-YouTube-Client-Name': '1',

youtube/templates/watch.html

@@ -528,6 +528,7 @@ <h2 class="title">{{ title }}</h2>
 
         <span class="direct-link"><a href="https://youtu.be/{{ video_id }}">Direct Link</a></span>
 
+        {% if settings.use_video_download != 0 %}
         <details class="download-dropdown">
             <summary class="download-dropdown-label">Download</summary>
             <ul class="download-dropdown-content">
@@ -556,7 +557,7 @@ <h2 class="title">{{ title }}</h2>
                 {% endfor %}
             </ul>
         </details>
-
+	{% endif %}
 
         <span class="description">{{ common_elements.text_runs(description)|escape|urlize|timestamps|safe }}</span>
         <div class="music-list">