fix(yt-dl): validate audio url and add retry logic

Contents/Code/default_prefs.py

@@ -17,8 +17,9 @@
     int_update_themes_interval='60',
     int_update_database_cache_interval='60',
     int_plexapi_plexapi_timeout='180',
-    int_plexapi_upload_retries_max='3',
+    int_plexapi_upload_retries_max='6',
     int_plexapi_upload_threads='3',
+    int_youtube_retries_max='8',
     str_youtube_cookies='',
     enum_webapp_locale='en',
     str_webapp_http_host='0.0.0.0',

Contents/Code/plex_api_helper.py

@@ -406,7 +406,7 @@
     ...
     """
     count = 0
-    while count <= int(Prefs['int_plexapi_upload_retries_max']):
+    while count <= max(0, int(Prefs['int_plexapi_upload_retries_max'])):
         try:
             if filepath:
                 if method == item.uploadTheme:

Contents/Code/youtube_dl_helper.py

@@ -4,7 +4,10 @@
 import logging
 import json
 import os
+import re
 import tempfile
+import time
+import urlparse
 
 # plex debugging
 try:
@@ -16,6 +19,7 @@
     from plexhints.prefs_kit import Prefs  # prefs kit
 
 # imports from Libraries\Shared
+import requests
 from typing import Optional
 import youtube_dl
 
@@ -26,7 +30,38 @@
 plugin_logger = logging.getLogger(plugin_identifier)
 
 
-def nsbool(value):
+def build_fallback_playback_url(playback_url):
+    # type: (str) -> Optional[str]
+    """
+    Build a fallback URL for a YouTube video.
+
+    Parameters
+    ----------
+    playback_url : str
+        The playback URL of the YouTube audio/video format.
+
+    Returns
+    -------
+    Optional[str]
+        The fallback URL of the playback URL.
+    """
+    query = youtube_dl.utils.parse_qs(url=playback_url)
+    mn = query.get('mn', [''])[0]
+    fvip = query.get('fvip', [''])[0]
+
+    mn = youtube_dl.utils.str_or_none(mn, '').split(',')
+    if len(mn) > 1 and mn[1] and fvip:
+        fmt_url_parsed = urlparse.urlparse(playback_url)
+        new_netloc = re.sub(
+            r'\d+', fvip, fmt_url_parsed.netloc.split('---')[0]) + '---' + mn[1] + '.googlevideo.com'
+
+        return youtube_dl.utils.update_url_query(
+            url=fmt_url_parsed._replace(netloc=new_netloc).geturl(),
+            query={'fallback_count': '1'},
+        )
+
+
+def ns_bool(value):
     # type: (bool) -> str
     """
     Format a boolean value for a Netscape cookie jar file.
@@ -69,8 +104,9 @@
     cookie_jar_file.write('# Netscape HTTP Cookie File\n')
 
     youtube_dl_params = dict(
-        cookiefile=cookie_jar_file.name,
+        cookiefile=cookie_jar_file.name if Prefs['str_youtube_cookies'] else None,
         logger=plugin_logger,
+        noplaylist=True,
         socket_timeout=10,
         youtube_include_dash_manifest=False,
     )
@@ -83,9 +119,9 @@
                 expiry = int(cookie.get('expiry', 0))
                 values = [
                     cookie['domain'],
-                    nsbool(include_subdomain),
+                    ns_bool(include_subdomain),
                     cookie['path'],
-                    nsbool(cookie['secure']),
+                    ns_bool(cookie['secure']),
                     str(expiry),
                     cookie['name'],
                     cookie['value']
@@ -100,38 +136,42 @@
     try:
         ydl = youtube_dl.YoutubeDL(params=youtube_dl_params)
 
-        with ydl:
-            try:
-                result = ydl.extract_info(
-                    url=url,
-                    download=False  # We just want to extract the info
-                )
-            except Exception as exc:
-                if isinstance(exc, youtube_dl.utils.ExtractorError) and exc.expected:
-                    Log.Info('YDL returned YT error while downloading {}: {}'.format(url, exc))
-                else:
-                    Log.Exception('YDL returned an unexpected error while downloading {}: {}'.format(url, exc))
-                return None
-
-            if 'entries' in result:
-                # Can be a playlist or a list of videos
-                video_data = result['entries'][0]
-            else:
-                # Just a video
-                video_data = result
-
-        selected = {
-            'opus': {
-                'size': 0,
-                'audio_url': None
-            },
-            'mp4a': {
-                'size': 0,
-                'audio_url': None
-            },
-        }
-        if video_data:
-            for fmt in video_data['formats']:  # loop through formats, select largest audio size for better quality
+        audio_url = None
+
+        count = 0
+        while count <= max(0, int(Prefs['int_youtube_retries_max'])):
+            sleep_time = 2 ** count
+            time.sleep(sleep_time)
+            with ydl:
+                try:
+                    result = ydl.extract_info(
+                        url=url,
+                        download=False  # We just want to extract the info
+                    )
+                except Exception as exc:
+                    if isinstance(exc, youtube_dl.utils.ExtractorError) and exc.expected:
+                        Log.Info('YDL returned YT error while downloading {}: {}'.format(url, exc))
+                    else:
+                        Log.Exception('YDL returned an unexpected error while downloading {}: {}'.format(url, exc))
+                    count += 1
+                    continue
+
+                # If a playlist was provided, select the first video
+                video_data = result['entries'][0] if 'entries' in result else result if result else {}
+
+            selected = {
+                'opus': {
+                    'size': 0,
+                    'audio_url': None
+                },
+                'mp4a': {
+                    'size': 0,
+                    'audio_url': None
+                },
+            }
+
+            # loop through formats, select the largest audio size for better quality
+            for fmt in video_data.get('formats', []):
                 if 'audio only' in fmt['format']:
                     if 'opus' == fmt['acodec']:
                         temp_codec = 'opus'
@@ -145,20 +185,43 @@
                         selected[temp_codec]['size'] = filesize
                         selected[temp_codec]['audio_url'] = fmt['url']
 
-        audio_url = None
+            if 0 < selected['opus']['size'] > selected['mp4a']['size']:
+                audio_url = selected['opus']['audio_url']
+            elif 0 < selected['mp4a']['size'] > selected['opus']['size']:
+                audio_url = selected['mp4a']['audio_url']
 
-        if 0 < selected['opus']['size'] > selected['mp4a']['size']:
-            audio_url = selected['opus']['audio_url']
-        elif 0 < selected['mp4a']['size'] > selected['opus']['size']:
-            audio_url = selected['mp4a']['audio_url']
+            if audio_url and Prefs['bool_prefer_mp4a_codec']:  # mp4a codec is preferred
+                if selected['mp4a']['audio_url']:  # mp4a codec is available
+                    audio_url = selected['mp4a']['audio_url']
+                elif selected['opus']['audio_url']:  # fallback to opus :(
+                    audio_url = selected['opus']['audio_url']
 
-        if audio_url and Prefs['bool_prefer_mp4a_codec']:  # mp4a codec is preferred
-            if selected['mp4a']['audio_url']:  # mp4a codec is available
-                audio_url = selected['mp4a']['audio_url']
-            elif selected['opus']['audio_url']:  # fallback to opus :(
-                audio_url = selected['opus']['audio_url']
+            if audio_url:
+                validate = requests.get(url=audio_url, stream=True)
+                if validate.status_code == 200:
+                    return audio_url
+                else:
+                    Log.Warn('Failed to validate audio URL for video {}'.format(url))
+
+                    # build a fallback URL
+                    fallback_url = build_fallback_playback_url(playback_url=audio_url)
+                    if fallback_url:
+                        audio_url = fallback_url
+                        Log.Warn('Trying fallback URL for video {}'.format(url))
+                        validate = requests.get(url=audio_url, stream=True)
+                        if validate.status_code == 200:
+                            return audio_url
+                        else:
+                            Log.Warn('Failed to validate fallback URL for video {}'.format(url))
+                            audio_url = None
+                    else:
+                        Log.Warn('Failed to build fallback URL for video {}'.format(url))
+                        audio_url = None
+
+                    count += 1
+                    continue
 
-        return audio_url  # return None or url found
+            return audio_url  # return None or url found
     finally:
         try:
             os.remove(cookie_jar_file.name)

Contents/DefaultPrefs.json

@@ -125,6 +125,13 @@
 		"default": "3",
 		"secure": "false"
 	},
+	{
+		"id": "int_youtube_retries_max",
+		"type": "text",
+		"label": "int_youtube_retries_max",
+		"default": "8",
+		"secure": "false"
+	},
 	{
 		"id": "str_youtube_cookies",
 		"type": "text",

Contents/Strings/en.json

@@ -15,8 +15,9 @@
   "int_update_themes_interval": "Interval for automatic update task, in minutes (min: 15)",
   "int_update_database_cache_interval": "Interval for database cache update task, in minutes (min: 15)",
   "int_plexapi_plexapi_timeout": "PlexAPI Timeout, in seconds (min: 1)",
-  "int_plexapi_upload_retries_max": "Max Retries, integer (min: 0)",
+  "int_plexapi_upload_retries_max": "Max Retries (PlexAPI uploads), integer (min: 0)",
   "int_plexapi_upload_threads": "Multiprocessing Threads, integer (min: 1)",
+  "int_youtube_retries_max": "Max Retries (YouTube), integer (min: 0)",
   "str_youtube_cookies": "YouTube Cookies (JSON format)",
   "enum_webapp_locale": "Web UI Locale",
   "str_webapp_http_host": "Web UI Host Address (requires Plex Media Server restart)",

docs/source/about/usage.rst

@@ -228,8 +228,8 @@ Default
 Minimum
    ``1``
 
-Max Retries
-^^^^^^^^^^^
+Max Retries (PlexAPI uploads)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Description
    The number of times to retry uploading theme audio to the Plex server. The time between retries will increase
@@ -242,6 +242,20 @@ Default
 Minimum
    ``0``
 
+Max Retries (YouTube)
+^^^^^^^^^^^^^^^^^^^^^
+
+Description
+   The number of times to retry getting an audio url from YouTube. The time between retries will increase
+   exponentially. The time between is calculated as ``2 ^ retry_number``. For example, the first retry will occur
+   after 2 seconds, the second retry will occur after 4 seconds, and the third retry will occur after 8 seconds.
+
+Default
+   ``8``
+
+Minimum
+   ``0``
+
 Multiprocessing Threads
 ^^^^^^^^^^^^^^^^^^^^^^^
 

tests/conftest.py

@@ -59,7 +59,7 @@ def wait_for_themes(section):
     timer = 0
     with_themes = 0
     total = len(section.all())
-    while timer < 180 and with_themes < total:
+    while timer < 600 and with_themes < total:
         with_themes = 0
         try:
             for item in section.all():

tests/unit/test_migration_helper.py

@@ -54,7 +54,7 @@ def test_validate_migration_key(migration_helper_fixture, key, raise_exception,
 @pytest.mark.parametrize('key, expected', [
     (migration_helper_object.LOCKED_THEMES, None),
     (migration_helper_object.LOCKED_COLLECTION_FIELDS, None),
-    pytest.param('invalid', None, marks=pytest.mark.xfail(raises=AttributeError)),
+    pytest.param('invalid', None, marks=pytest.mark.xfail(raises=AttributeError, reason="Cannot migrate in CI")),
 ])
 def test_get_migration_status(migration_helper_fixture, migration_status_file, key, expected):
     migration_status = migration_helper_fixture.get_migration_status(key=key)
@@ -64,7 +64,7 @@ def test_get_migration_status(migration_helper_fixture, migration_status_file, k
 @pytest.mark.parametrize('key', [
     migration_helper_object.LOCKED_THEMES,
     migration_helper_object.LOCKED_COLLECTION_FIELDS,
-    pytest.param('invalid', marks=pytest.mark.xfail(raises=AttributeError)),
+    pytest.param('invalid', marks=pytest.mark.xfail(raises=AttributeError, reason="Cannot migrate in CI")),
 ])
 def test_set_migration_status(migration_helper_fixture, migration_status_file, key):
     # perform the test twice, to load an existing migration file