More improvements to HLS/DASH external downloader code

* Fix error when there is no `protocol` in `info_dict`
* Move HLS byte range detection to `Aria2cFD` so that the download will fall back to the native downloader instead of ffmpeg
* Fix bug with getting no fragments in DASH
* Convert `check_results` in `can_download` to a generator

yt_dlp/YoutubeDL.py

@@ -2437,7 +2437,8 @@ def correct_ext(filename):
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
-                if get_suitable_downloader(info_dict, self.params).__name__ == 'HlsFD':
+                if ('protocol' in info_dict
+                        and get_suitable_downloader(info_dict, self.params).__name__ == 'HlsFD'):
                     if fixup_policy == 'warn':
                         self.report_warning('%s: malformed AAC bitstream detected.' % (
                             info_dict['id']))

yt_dlp/downloader/common.py

@@ -326,6 +326,12 @@ def report_unable_to_resume(self):
         """Report it was impossible to resume download."""
         self.to_screen('[download] Unable to resume')
 
+    @staticmethod
+    def supports_manifest(manifest):
+        """ Whether the downloader can download the fragments from the manifest.
+        Redefine in subclasses if needed. """
+        pass
+
     def download(self, filename, info_dict, subtitle=False):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise

yt_dlp/downloader/dash.py

@@ -12,7 +12,8 @@
 
 class DashSegmentsFD(FragmentFD):
     """
-    Download segments in a DASH manifest
+    Download segments in a DASH manifest. External downloaders can take over
+    the fragment downloads by supporting the 'frag_urls' protocol
     """
 
     FD_NAME = 'dashsegments'
@@ -37,7 +38,7 @@ def real_download(self, filename, info_dict):
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        fragments = []
+        fragments_to_download = []
         frag_index = 0
         for i, fragment in enumerate(fragments):
             frag_index += 1
@@ -49,7 +50,7 @@ def real_download(self, filename, info_dict):
                 fragment_url = urljoin(fragment_base_url, fragment['path'])
 
             if real_downloader:
-                fragments.append({
+                fragments_to_download.append({
                     'url': fragment_url,
                 })
                 continue
@@ -92,7 +93,7 @@ def real_download(self, filename, info_dict):
 
         if real_downloader:
             info_copy = info_dict.copy()
-            info_copy['fragments'] = fragments
+            info_copy['fragments'] = fragments_to_download
             fd = real_downloader(self.ydl, self.params)
             # TODO: Make progress updates work without hooking twice
             # for ph in self._progress_hooks:

yt_dlp/downloader/external.py

@@ -125,7 +125,7 @@ def _call_downloader(self, tmpfilename, info_dict):
         if 'fragments' in info_dict:
             file_list = []
             dest, _ = sanitize_open(tmpfilename, 'wb')
-            for [i, fragment] in enumerate(info_dict['fragments']):
+            for i, fragment in enumerate(info_dict['fragments']):
                 file = '%s_%s.frag' % (tmpfilename, i)
                 decrypt_info = fragment.get('decrypt_info')
                 src, _ = sanitize_open(file, 'rb')
@@ -242,6 +242,15 @@ class Aria2cFD(ExternalFD):
     AVAILABLE_OPT = '-v'
     SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'frag_urls')
 
+    @staticmethod
+    def supports_manifest(manifest):
+        UNSUPPORTED_FEATURES = [
+            r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [1]
+            # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
+        ]
+        check_results = (not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
+        return all(check_results)
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-c']
         dn = os.path.dirname(tmpfilename)
@@ -264,7 +273,7 @@ def _make_cmd(self, tmpfilename, info_dict):
             cmd += ['--uri-selector', 'inorder', '--download-result=hide']
             url_list_file = '%s.frag.urls' % tmpfilename
             url_list = []
-            for [i, fragment] in enumerate(info_dict['fragments']):
+            for i, fragment in enumerate(info_dict['fragments']):
                 tmpsegmentname = '%s_%s.frag' % (os.path.basename(tmpfilename), i)
                 url_list.append('%s\n\tout=%s' % (fragment['url'], tmpsegmentname))
             stream, _ = sanitize_open(url_list_file, 'wb')

yt_dlp/downloader/hls.py

@@ -24,12 +24,16 @@
 
 
 class HlsFD(FragmentFD):
-    """ A limited implementation that does not require ffmpeg """
+    """
+    Download segments in a m3u8 manifest. External downloaders can take over
+    the fragment downloads by supporting the 'frag_urls' protocol and
+    re-defining 'supports_manifest' function
+    """
 
     FD_NAME = 'hlsnative'
 
     @staticmethod
-    def can_download(manifest, info_dict, allow_unplayable_formats=False, real_downloader=None, with_crypto=can_decrypt_frag):
+    def can_download(manifest, info_dict, allow_unplayable_formats=False, with_crypto=can_decrypt_frag):
         UNSUPPORTED_FEATURES = [
             # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
 
@@ -53,16 +57,15 @@ def can_download(manifest, info_dict, allow_unplayable_formats=False, real_downl
             UNSUPPORTED_FEATURES += [
                 r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
             ]
-        if real_downloader:
-            UNSUPPORTED_FEATURES += [
-                r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
-            ]
-        check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
+
+        def check_results():
+            yield not info_dict.get('is_live')
             is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
-        check_results.append(with_crypto or not is_aes128_enc)
-        check_results.append(not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest))
-        check_results.append(not info_dict.get('is_live'))
-        return all(check_results)
+            yield with_crypto or not is_aes128_enc
+            yield not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest)
+            for feature in UNSUPPORTED_FEATURES:
+                yield not re.search(feature, manifest)
+        return all(check_results())
 
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
@@ -72,9 +75,7 @@ def real_download(self, filename, info_dict):
         man_url = urlh.geturl()
         s = urlh.read().decode('utf-8', 'ignore')
 
-        real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
-
-        if not self.can_download(s, info_dict, self.params.get('allow_unplayable_formats'), real_downloader):
+        if not self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')):
             if info_dict.get('extra_param_to_segment_url') or info_dict.get('_decryption_key_url'):
                 self.report_error('pycryptodome not found. Please install it.')
                 return False
@@ -89,6 +90,10 @@ def real_download(self, filename, info_dict):
             #     fd.add_progress_hook(ph)
             return fd.real_download(filename, info_dict)
 
+        real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
+        if real_downloader and not real_downloader.supports_manifest(s):
+            real_downloader = None
+
         def is_ad_fragment_start(s):
             return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
                     or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad'))