From da8e2912b165005f76779a115a071cd6132ceedf Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Thu, 23 Feb 2023 04:18:45 +0100 Subject: [PATCH 0001/1178] [utils] `Popen`: Shim undocumented `text_mode` property Fixes #6317 Authored by: Grub4K --- yt_dlp/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 994239897c..4fe718bf07 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -879,6 +879,7 @@ def __init__(self, *args, env=None, text=False, **kwargs): env = os.environ.copy() self._fix_pyinstaller_ld_path(env) + self.__text_mode = kwargs.get('encoding') or kwargs.get('errors') or text or kwargs.get('universal_newlines') if text is True: kwargs['universal_newlines'] = True # For 3.6 compatibility kwargs.setdefault('encoding', 'utf-8') @@ -900,7 +901,7 @@ def kill(self, *, timeout=0): @classmethod def run(cls, *args, timeout=None, **kwargs): with cls(*args, **kwargs) as proc: - default = '' if proc.text_mode else b'' + default = '' if proc.__text_mode else b'' stdout, stderr = proc.communicate_or_kill(timeout=timeout) return stdout or default, stderr or default, proc.returncode From cc09083636ce21e58ff74f45eac2dbda507462b0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 24 Feb 2023 10:39:43 +0530 Subject: [PATCH 0002/1178] [utils] `LenientJSONDecoder`: Parse unclosed objects --- yt_dlp/utils.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 4fe718bf07..9ff096433b 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -593,21 +593,43 @@ def clean_html(html): class LenientJSONDecoder(json.JSONDecoder): - def __init__(self, *args, transform_source=None, ignore_extra=False, **kwargs): + # TODO: Write tests + def __init__(self, *args, transform_source=None, ignore_extra=False, close_objects=0, **kwargs): self.transform_source, self.ignore_extra = transform_source, ignore_extra + self._close_attempts = 2 * close_objects super().__init__(*args, **kwargs) + @staticmethod + def _close_object(err): + doc = err.doc[:err.pos] + # We need to add comma first to get the correct error message + if err.msg.startswith('Expecting \',\''): + return doc + ',' + elif not doc.endswith(','): + return + + if err.msg.startswith('Expecting property name'): + return doc[:-1] + '}' + elif err.msg.startswith('Expecting value'): + return doc[:-1] + ']' + def decode(self, s): if self.transform_source: s = self.transform_source(s) - try: - if self.ignore_extra: - return self.raw_decode(s.lstrip())[0] - return super().decode(s) - except json.JSONDecodeError as e: - if e.pos is not None: + for attempt in range(self._close_attempts + 1): + try: + if self.ignore_extra: + return self.raw_decode(s.lstrip())[0] + return super().decode(s) + except json.JSONDecodeError as e: + if e.pos is None: + raise + elif attempt < self._close_attempts: + s = self._close_object(e) + if s is not None: + continue raise type(e)(f'{e.msg} in {s[e.pos-10:e.pos+10]!r}', s, e.pos) - raise + assert False, 'Too many attempts to decode JSON' def sanitize_open(filename, open_mode): From 43a3eaf96393b712d60cbcf5c6cb1e90ed7f42f5 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 26 Feb 2023 10:16:30 +0530 Subject: [PATCH 0003/1178] [extractor] Fix DRM detection in m3u8 Fixes https://github.com/ytdl-org/youtube-dl/issues/31693#issuecomment-1445202857 --- yt_dlp/extractor/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index ebacc87bc0..86bef173f5 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -2063,6 +2063,7 @@ def extract_media(x_media_line): 'protocol': entry_protocol, 'preference': preference, 'quality': quality, + 'has_drm': has_drm, 'vcodec': 'none' if media_type == 'AUDIO' else None, } for idx in _extract_m3u8_playlist_indices(manifest_url)) @@ -2122,6 +2123,7 @@ def build_stream_name(): 'protocol': entry_protocol, 'preference': preference, 'quality': quality, + 'has_drm': has_drm, } resolution = last_stream_inf.get('RESOLUTION') if resolution: From 8e9fe43cd393e69fa49b3d842aa3180c1d105b8f Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 26 Feb 2023 10:27:04 +0530 Subject: [PATCH 0004/1178] [extractor/generic] Handle basic-auth when checking redirects Closes #6352 --- yt_dlp/extractor/generic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index 55e55d5248..d76ef3e31c 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -15,6 +15,7 @@ UnsupportedError, determine_ext, dict_get, + extract_basic_auth, format_field, int_or_none, is_html, @@ -2372,9 +2373,8 @@ def _real_extract(self, url): **smuggled_data.get('http_headers', {}) }) new_url = full_response.geturl() - if new_url == urllib.parse.urlparse(url)._replace(scheme='https').geturl(): - url = new_url - elif url != new_url: + url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl() + if new_url != extract_basic_auth(url)[0]: self.report_following_redirect(new_url) if force_videoid: new_url = smuggle_url(new_url, {'force_videoid': force_videoid}) From 4d248e29d20d983ededab0b03d4fe69dff9eb4ed Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 28 Feb 2023 23:09:20 +0530 Subject: [PATCH 0005/1178] [extractor/GoogleDrive] Fix some audio Only those with source url, but no confirmation page --- yt_dlp/extractor/googledrive.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py index e027ea7c4d..9e2ccde005 100644 --- a/yt_dlp/extractor/googledrive.py +++ b/yt_dlp/extractor/googledrive.py @@ -3,8 +3,8 @@ from .common import InfoExtractor from ..compat import compat_parse_qs from ..utils import ( - determine_ext, ExtractorError, + determine_ext, get_element_by_class, int_or_none, lowercase_escape, @@ -163,15 +163,13 @@ def _real_extract(self, url): video_id = self._match_id(url) video_info = compat_parse_qs(self._download_webpage( 'https://drive.google.com/get_video_info', - video_id, query={'docid': video_id})) + video_id, 'Downloading video webpage', query={'docid': video_id})) def get_value(key): return try_get(video_info, lambda x: x[key][0]) reason = get_value('reason') title = get_value('title') - if not title and reason: - raise ExtractorError(reason, expected=True) formats = [] fmt_stream_map = (get_value('fmt_stream_map') or '').split(',') @@ -216,6 +214,11 @@ def request_source_file(source_url, kind): urlh = request_source_file(source_url, 'source') if urlh: def add_source_format(urlh): + nonlocal title + if not title: + title = self._search_regex( + r'\bfilename="([^"]+)"', urlh.headers.get('Content-Disposition'), + 'title', default=None) formats.append({ # Use redirect URLs as download URLs in order to calculate # correct cookies in _calc_cookies. @@ -251,7 +254,10 @@ def add_source_format(urlh): or 'unable to extract confirmation code') if not formats and reason: - self.raise_no_formats(reason, expected=True) + if title: + self.raise_no_formats(reason, expected=True) + else: + raise ExtractorError(reason, expected=True) hl = get_value('hl') subtitles_id = None From 5038f6d713303e0967d002216e7a88652401c22a Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 28 Feb 2023 23:03:44 +0530 Subject: [PATCH 0006/1178] [extractor/youtube] Construct dash formats with `range` query Closes #6369 --- yt_dlp/extractor/youtube.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index be82bc6899..0227a1f83c 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -3776,10 +3776,19 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l if no_video: dct['abr'] = tbr if no_audio or no_video: - dct['downloader_options'] = { - # Youtube throttles chunks >~10M - 'http_chunk_size': 10485760, - } + CHUNK_SIZE = 10 << 20 + dct.update({ + 'request_data': b'x', + 'protocol': 'http_dash_segments', + 'fragments': [{ + 'url': update_url_query(dct['url'], { + 'range': f'{range_start}-{min(range_start + CHUNK_SIZE - 1, dct["filesize"])}' + }) + } for range_start in range(0, dct['filesize'], CHUNK_SIZE)] + } if dct['filesize'] else { + 'downloader_options': {'http_chunk_size': CHUNK_SIZE} # No longer useful? + }) + if dct.get('ext'): dct['container'] = dct['ext'] + '_dash' From b059188383eee4fa336ef728dda3ff4bb7335625 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 28 Feb 2023 22:32:20 +0530 Subject: [PATCH 0007/1178] [plugins] Don't look in `.egg` directories Closes #6306 --- yt_dlp/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/plugins.py b/yt_dlp/plugins.py index 6eecdb4d0c..6422c7a51d 100644 --- a/yt_dlp/plugins.py +++ b/yt_dlp/plugins.py @@ -88,7 +88,7 @@ def _get_package_paths(*root_paths, containing_folder='plugins'): candidate = path / parts if candidate.is_dir(): yield candidate - elif path.suffix in ('.zip', '.egg', '.whl'): + elif path.suffix in ('.zip', '.egg', '.whl') and path.is_file(): if parts in dirs_in_zip(path): yield candidate From 65f6e807804d2af5e00f2aecd72bfc43af19324a Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 28 Feb 2023 23:10:54 +0530 Subject: [PATCH 0008/1178] [dependencies] Simplify `Cryptodome` Closes #6292, closes #6272, closes #6338 --- test/test_aes.py | 4 +-- yt_dlp/__pyinstaller/hook-yt_dlp.py | 28 +--------------- yt_dlp/aes.py | 6 ++-- yt_dlp/compat/_legacy.py | 2 +- yt_dlp/compat/compat_utils.py | 2 +- yt_dlp/dependencies/Cryptodome.py | 50 ++++++++++++++++++----------- yt_dlp/dependencies/__init__.py | 2 +- yt_dlp/downloader/hls.py | 2 +- yt_dlp/extractor/bilibili.py | 6 ++-- yt_dlp/extractor/ivi.py | 8 ++--- yt_dlp/extractor/wrestleuniverse.py | 6 ++-- 11 files changed, 52 insertions(+), 64 deletions(-) diff --git a/test/test_aes.py b/test/test_aes.py index 18f15fecb6..a26abfd7d0 100644 --- a/test/test_aes.py +++ b/test/test_aes.py @@ -48,7 +48,7 @@ def test_cbc_decrypt(self): data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd' decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv)) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) - if Cryptodome: + if Cryptodome.AES: decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv)) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) @@ -78,7 +78,7 @@ def test_gcm_decrypt(self): decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify( bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12])) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) - if Cryptodome: + if Cryptodome.AES: decrypted = aes_gcm_decrypt_and_verify_bytes( data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12])) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) diff --git a/yt_dlp/__pyinstaller/hook-yt_dlp.py b/yt_dlp/__pyinstaller/hook-yt_dlp.py index 057cfef2f9..63dcdffe02 100644 --- a/yt_dlp/__pyinstaller/hook-yt_dlp.py +++ b/yt_dlp/__pyinstaller/hook-yt_dlp.py @@ -1,30 +1,8 @@ -import ast -import os import sys -from pathlib import Path from PyInstaller.utils.hooks import collect_submodules -def find_attribute_accesses(node, name, path=()): - if isinstance(node, ast.Attribute): - path = [*path, node.attr] - if isinstance(node.value, ast.Name) and node.value.id == name: - yield path[::-1] - for child in ast.iter_child_nodes(node): - yield from find_attribute_accesses(child, name, path) - - -def collect_used_submodules(name, level): - for dirpath, _, filenames in os.walk(Path(__file__).parent.parent): - for filename in filenames: - if not filename.endswith('.py'): - continue - with open(Path(dirpath) / filename, encoding='utf8') as f: - for submodule in find_attribute_accesses(ast.parse(f.read()), name): - yield '.'.join(submodule[:level]) - - def pycryptodome_module(): try: import Cryptodome # noqa: F401 @@ -41,12 +19,8 @@ def pycryptodome_module(): def get_hidden_imports(): yield 'yt_dlp.compat._legacy' + yield pycryptodome_module() yield from collect_submodules('websockets') - - crypto = pycryptodome_module() - for sm in set(collect_used_submodules('Cryptodome', 2)): - yield f'{crypto}.{sm}' - # These are auto-detected, but explicitly add them just in case yield from ('mutagen', 'brotli', 'certifi') diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py index deff0a2b3d..b3a383cd9c 100644 --- a/yt_dlp/aes.py +++ b/yt_dlp/aes.py @@ -5,14 +5,14 @@ from .dependencies import Cryptodome from .utils import bytes_to_intlist, intlist_to_bytes -if Cryptodome: +if Cryptodome.AES: def aes_cbc_decrypt_bytes(data, key, iv): """ Decrypt bytes with AES-CBC using pycryptodome """ - return Cryptodome.Cipher.AES.new(key, Cryptodome.Cipher.AES.MODE_CBC, iv).decrypt(data) + return Cryptodome.AES.new(key, Cryptodome.AES.MODE_CBC, iv).decrypt(data) def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce): """ Decrypt bytes with AES-GCM using pycryptodome """ - return Cryptodome.Cipher.AES.new(key, Cryptodome.Cipher.AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) + return Cryptodome.AES.new(key, Cryptodome.AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) else: def aes_cbc_decrypt_bytes(data, key, iv): diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py index 84d749209e..83bf869a80 100644 --- a/yt_dlp/compat/_legacy.py +++ b/yt_dlp/compat/_legacy.py @@ -32,9 +32,9 @@ from . import compat_expanduser, compat_HTMLParseError, compat_realpath from .compat_utils import passthrough_module -from ..dependencies import Cryptodome_AES as compat_pycrypto_AES # noqa: F401 from ..dependencies import brotli as compat_brotli # noqa: F401 from ..dependencies import websockets as compat_websockets # noqa: F401 +from ..dependencies.Cryptodome import AES as compat_pycrypto_AES # noqa: F401 passthrough_module(__name__, '...utils', ('WINDOWS_VT_MODE', 'windows_enable_vt_mode')) diff --git a/yt_dlp/compat/compat_utils.py b/yt_dlp/compat/compat_utils.py index 8956b3bf1f..3ca46d270c 100644 --- a/yt_dlp/compat/compat_utils.py +++ b/yt_dlp/compat/compat_utils.py @@ -48,7 +48,7 @@ def passthrough_module(parent, child, allowed_attributes=(..., ), *, callback=la """Passthrough parent module into a child module, creating the parent if necessary""" def __getattr__(attr): if _is_package(parent): - with contextlib.suppress(ImportError): + with contextlib.suppress(ModuleNotFoundError): return importlib.import_module(f'.{attr}', parent.__name__) ret = from_child(attr) diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py index 2adc513740..a50bce4d4f 100644 --- a/yt_dlp/dependencies/Cryptodome.py +++ b/yt_dlp/dependencies/Cryptodome.py @@ -1,8 +1,5 @@ import types -from ..compat import functools -from ..compat.compat_utils import passthrough_module - try: import Cryptodome as _parent except ImportError: @@ -12,19 +9,36 @@ _parent = types.ModuleType('no_Cryptodome') __bool__ = lambda: False -passthrough_module(__name__, _parent, (..., '__version__')) -del passthrough_module - - -@property -@functools.cache -def _yt_dlp__identifier(): - if _parent.__name__ == 'Crypto': +__version__ = '' +AES = PKCS1_v1_5 = Blowfish = PKCS1_OAEP = SHA1 = CMAC = RSA = None +try: + if _parent.__name__ == 'Cryptodome': + from Cryptodome import __version__ + from Cryptodome.Cipher import AES + from Cryptodome.Cipher import PKCS1_v1_5 + from Cryptodome.Cipher import Blowfish + from Cryptodome.Cipher import PKCS1_OAEP + from Cryptodome.Hash import SHA1 + from Cryptodome.Hash import CMAC + from Cryptodome.PublicKey import RSA + elif _parent.__name__ == 'Crypto': + from Crypto import __version__ from Crypto.Cipher import AES - try: - # In pycrypto, mode defaults to ECB. See: - # https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode - AES.new(b'abcdefghijklmnop') - except TypeError: - return 'pycrypto' - return _parent.__name__ + from Crypto.Cipher import PKCS1_v1_5 + from Crypto.Cipher import Blowfish + from Crypto.Cipher import PKCS1_OAEP + from Crypto.Hash import SHA1 + from Crypto.Hash import CMAC + from Crypto.PublicKey import RSA +except ImportError: + __version__ = f'broken {__version__}'.strip() + + +_yt_dlp__identifier = _parent.__name__ +if AES and _yt_dlp__identifier == 'Crypto': + try: + # In pycrypto, mode defaults to ECB. See: + # https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode + AES.new(b'abcdefghijklmnop') + except TypeError: + _yt_dlp__identifier = 'pycrypto' diff --git a/yt_dlp/dependencies/__init__.py b/yt_dlp/dependencies/__init__.py index c2214e6dba..6e7d29c5ca 100644 --- a/yt_dlp/dependencies/__init__.py +++ b/yt_dlp/dependencies/__init__.py @@ -73,7 +73,7 @@ # Deprecated -Cryptodome_AES = Cryptodome.Cipher.AES if Cryptodome else None +Cryptodome_AES = Cryptodome.AES __all__ = [ diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py index 29d6f62411..f2868dc52b 100644 --- a/yt_dlp/downloader/hls.py +++ b/yt_dlp/downloader/hls.py @@ -70,7 +70,7 @@ def real_download(self, filename, info_dict): can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None if can_download: has_ffmpeg = FFmpegFD.available() - no_crypto = not Cryptodome and '#EXT-X-KEY:METHOD=AES-128' in s + no_crypto = not Cryptodome.AES and '#EXT-X-KEY:METHOD=AES-128' in s if no_crypto and has_ffmpeg: can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available' elif no_crypto: diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py index f4180633ab..2252840b3a 100644 --- a/yt_dlp/extractor/bilibili.py +++ b/yt_dlp/extractor/bilibili.py @@ -894,15 +894,15 @@ def _parse_video_metadata(self, video_data): } def _perform_login(self, username, password): - if not Cryptodome: + if not Cryptodome.RSA: raise ExtractorError('pycryptodomex not found. Please install', expected=True) key_data = self._download_json( 'https://passport.bilibili.tv/x/intl/passport-login/web/key?lang=en-US', None, note='Downloading login key', errnote='Unable to download login key')['data'] - public_key = Cryptodome.PublicKey.RSA.importKey(key_data['key']) - password_hash = Cryptodome.Cipher.PKCS1_v1_5.new(public_key).encrypt((key_data['hash'] + password).encode('utf-8')) + public_key = Cryptodome.RSA.importKey(key_data['key']) + password_hash = Cryptodome.PKCS1_v1_5.new(public_key).encrypt((key_data['hash'] + password).encode('utf-8')) login_post = self._download_json( 'https://passport.bilibili.tv/x/intl/passport-login/web/login/password?lang=en-US', None, data=urlencode_postdata({ 'username': username, diff --git a/yt_dlp/extractor/ivi.py b/yt_dlp/extractor/ivi.py index 96220bea9c..fa5ceec95b 100644 --- a/yt_dlp/extractor/ivi.py +++ b/yt_dlp/extractor/ivi.py @@ -91,7 +91,7 @@ def _real_extract(self, url): for site in (353, 183): content_data = (data % site).encode() if site == 353: - if not Cryptodome: + if not Cryptodome.CMAC: continue timestamp = (self._download_json( @@ -105,8 +105,8 @@ def _real_extract(self, url): query = { 'ts': timestamp, - 'sign': Cryptodome.Hash.CMAC.new(self._LIGHT_KEY, timestamp.encode() + content_data, - Cryptodome.Cipher.Blowfish).hexdigest(), + 'sign': Cryptodome.CMAC.new(self._LIGHT_KEY, timestamp.encode() + content_data, + Cryptodome.Blowfish).hexdigest(), } else: query = {} @@ -126,7 +126,7 @@ def _real_extract(self, url): extractor_msg = 'Video %s does not exist' elif site == 353: continue - elif not Cryptodome: + elif not Cryptodome.CMAC: raise ExtractorError('pycryptodomex not found. Please install', expected=True) elif message: extractor_msg += ': ' + message diff --git a/yt_dlp/extractor/wrestleuniverse.py b/yt_dlp/extractor/wrestleuniverse.py index 78e7c83abc..5c6dec2c40 100644 --- a/yt_dlp/extractor/wrestleuniverse.py +++ b/yt_dlp/extractor/wrestleuniverse.py @@ -50,10 +50,10 @@ def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={ data=data, headers=headers, query=query, fatal=fatal) def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True): - if not Cryptodome: + if not Cryptodome.RSA: raise ExtractorError('pycryptodomex not found. Please install', expected=True) - private_key = Cryptodome.PublicKey.RSA.generate(2048) - cipher = Cryptodome.Cipher.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.Hash.SHA1) + private_key = Cryptodome.RSA.generate(2048) + cipher = Cryptodome.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.SHA1) def decrypt(data): if not data: From f34804b2f920f62a6e893a14a9e2a2144b14dd23 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 28 Feb 2023 23:34:43 +0530 Subject: [PATCH 0009/1178] [extractor/youtube] Fix 5038f6d713303e0967d002216e7a88652401c22a * [fragment] Fix `request_data` * [youtube] Don't use POST for now. It may be easier to break in future Authored by: bashonly, coletdjnz --- yt_dlp/downloader/fragment.py | 3 ++- yt_dlp/extractor/common.py | 1 + yt_dlp/extractor/youtube.py | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index 039cb14927..377f138b76 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -466,7 +466,8 @@ def error_callback(err, count, retries): for retry in RetryManager(self.params.get('fragment_retries'), error_callback): try: ctx['fragment_count'] = fragment.get('fragment_count') - if not self._download_fragment(ctx, fragment['url'], info_dict, headers): + if not self._download_fragment( + ctx, fragment['url'], info_dict, headers, info_dict.get('request_data')): return except (urllib.error.HTTPError, http.client.IncompleteRead) as err: retry.error = err diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 86bef173f5..98efe0e9da 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -132,6 +132,7 @@ class InfoExtractor: is parsed from a string (in case of fragmented media) for MSS - URL of the ISM manifest. + * request_data Data to send in POST request to the URL * manifest_url The URL of the manifest file in case of fragmented media: diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 0227a1f83c..f5ffce7750 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -3778,7 +3778,6 @@ def _extract_formats_and_subtitles(self, streaming_data, video_id, player_url, l if no_audio or no_video: CHUNK_SIZE = 10 << 20 dct.update({ - 'request_data': b'x', 'protocol': 'http_dash_segments', 'fragments': [{ 'url': update_url_query(dct['url'], { From 31e183557fcd1b937582f9429f29207c1261f501 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 28 Feb 2023 23:50:34 +0530 Subject: [PATCH 0010/1178] [extractor/youtube] Extract channel `view_count` when `/about` tab is passed --- yt_dlp/extractor/youtube.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index f5ffce7750..d1696349aa 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -4905,6 +4905,10 @@ def _get_uncropped(url): info['view_count'] = self._get_count(playlist_stats, 1) if info['view_count'] is None: # 0 is allowed info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText') + if info['view_count'] is None: + info['view_count'] = self._get_count(data, ( + 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer', + 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText')) info['playlist_count'] = self._get_count(playlist_stats, 0) if info['playlist_count'] is None: # 0 is allowed @@ -6124,6 +6128,23 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor): } }], 'params': {'extract_flat': True}, + }, { + 'url': 'https://www.youtube.com/@3blue1brown/about', + 'info_dict': { + 'id': 'UCYO_jab_esuFRV4b17AJtAw', + 'tags': ['Mathematics'], + 'title': '3Blue1Brown - About', + 'uploader_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw', + 'channel_follower_count': int, + 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw', + 'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw', + 'channel': '3Blue1Brown', + 'uploader': '3Blue1Brown', + 'view_count': int, + 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw', + 'description': 'md5:e1384e8a133307dd10edee76e875d62f', + }, + 'playlist_count': 0, }] @classmethod From 5b28cef72db3b531680d89c121631c73ae05354f Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 28 Feb 2023 23:31:02 +0530 Subject: [PATCH 0011/1178] [cleanup] Misc --- .github/ISSUE_TEMPLATE/1_broken_site.yml | 2 + .../ISSUE_TEMPLATE/2_site_support_request.yml | 2 + .../ISSUE_TEMPLATE/3_site_feature_request.yml | 2 + .github/ISSUE_TEMPLATE/4_bug_report.yml | 2 + .github/ISSUE_TEMPLATE/5_feature_request.yml | 2 + .github/ISSUE_TEMPLATE/6_question.yml | 2 + CONTRIBUTING.md | 2 +- Changelog.md | 4 +- README.md | 1 + devscripts/make_issue_template.py | 2 + supportedsites.md | 366 +++++++++--------- yt_dlp/YoutubeDL.py | 2 +- yt_dlp/dependencies/Cryptodome.py | 18 +- yt_dlp/downloader/fragment.py | 2 +- yt_dlp/extractor/common.py | 2 +- yt_dlp/extractor/youtube.py | 4 +- 16 files changed, 212 insertions(+), 203 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml index e1103fb848..48e8890c52 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.yml +++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml @@ -50,6 +50,8 @@ body: options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true + - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" + required: false - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml index 90d7294ac0..d43d62f033 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml @@ -62,6 +62,8 @@ body: options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true + - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" + required: false - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml index 5b59852c70..352b472420 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml @@ -58,6 +58,8 @@ body: options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true + - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" + required: false - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml index bd4695f878..7588b8ed84 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml @@ -43,6 +43,8 @@ body: options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true + - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" + required: false - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml index 8c7f315e9e..fdda50b7bd 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml @@ -40,6 +40,8 @@ body: label: Provide verbose output that clearly demonstrates the problem options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) + - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" + required: false - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below - type: textarea id: log diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml index 4a13446286..56ce74654d 100644 --- a/.github/ISSUE_TEMPLATE/6_question.yml +++ b/.github/ISSUE_TEMPLATE/6_question.yml @@ -46,6 +46,8 @@ body: label: Provide verbose output that clearly demonstrates the problem options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) + - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" + required: false - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below - type: textarea id: log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 551db674e2..ae2c454239 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,7 +127,7 @@ ### Are you willing to share account details if needed? ### Is the website primarily used for piracy? -We follow [youtube-dl's policy](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) to not support services that is primarily used for infringing copyright. Additionally, it has been decided to not to support porn sites that specialize in deep fake. We also cannot support any service that serves only [DRM protected content](https://en.wikipedia.org/wiki/Digital_rights_management). +We follow [youtube-dl's policy](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) to not support services that is primarily used for infringing copyright. Additionally, it has been decided to not to support porn sites that specialize in fakes. We also cannot support any service that serves only [DRM protected content](https://en.wikipedia.org/wiki/Digital_rights_management). diff --git a/Changelog.md b/Changelog.md index 8d3ac089ce..24bc8a2e27 100644 --- a/Changelog.md +++ b/Changelog.md @@ -50,8 +50,8 @@ ### 2023.02.17 * [extractor/txxx] Add extractors by [chio0hai](https://github.com/chio0hai) * [extractor/vocaroo] Add extractor by [SuperSonicHub1](https://github.com/SuperSonicHub1), [qbnu](https://github.com/qbnu) * [extractor/wrestleuniverse] Add extractors by [Grub4K](https://github.com/Grub4K), [bashonly](https://github.com/bashonly) -* [extractor/yappy] Add extractor by [HobbyistDev](https://github.com/HobbyistDev) -* **[extractor/youtube] Fix `uploader_id` extraction** by [bashonly](https://github.com/bashonly) +* [extractor/yappy] Add extractor by [HobbyistDev](https://github.com/HobbyistDev), [dirkf](https://github.com/dirkf) +* [extractor/youtube] **Fix `uploader_id` extraction** by [bashonly](https://github.com/bashonly) * [extractor/youtube] Add hyperpipe instances by [Generator](https://github.com/Generator) * [extractor/youtube] Handle `consent.youtube` * [extractor/youtube] Support `/live/` URL diff --git a/README.md b/README.md index 9b91775bc7..3d3db933ac 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ ### Differences in default behavior Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc: +* yt-dlp supports only [Python 3.7+](## "Windows 7"), and *may* remove support for more versions as they [become EOL](https://devguide.python.org/versions/#python-release-cycle); while [youtube-dl still supports Python 2.6+ and 3.2+](https://github.com/ytdl-org/youtube-dl/issues/30568#issue-1118238743) * The options `--auto-number` (`-A`), `--title` (`-t`) and `--literal` (`-l`), no longer work. See [removed options](#Removed) for details * `avconv` is not supported as an alternative to `ffmpeg` * yt-dlp stores config files in slightly different locations to youtube-dl. See [CONFIGURATION](#configuration) for a list of correct locations diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 1ee00f2b89..39b95c8da6 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -24,6 +24,8 @@ options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true + - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" + required: false - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below required: true - type: textarea diff --git a/supportedsites.md b/supportedsites.md index b545ec540d..d7ac6dce5e 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -28,14 +28,14 @@ # Supported sites - **abcnews:video** - **abcotvs**: ABC Owned Television Stations - **abcotvs:clips** - - **AbemaTV**: [abematv] + - **AbemaTV**: [*abematv*](## "netrc machine") - **AbemaTVTitle** - **AcademicEarth:Course** - **acast** - **acast:channel** - **AcFunBangumi** - **AcFunVideo** - - **ADN**: [animationdigitalnetwork] Animation Digital Network + - **ADN**: [*animationdigitalnetwork*](## "netrc machine") Animation Digital Network - **AdobeConnect** - **adobetv** - **adobetv:channel** @@ -47,8 +47,8 @@ # Supported sites - **aenetworks:collection** - **aenetworks:show** - **AeonCo** - - **afreecatv**: [afreecatv] afreecatv.com - - **afreecatv:live**: [afreecatv] afreecatv.com + - **afreecatv**: [*afreecatv*](## "netrc machine") afreecatv.com + - **afreecatv:live**: [*afreecatv*](## "netrc machine") afreecatv.com - **afreecatv:user** - **AirMozilla** - **AirTV** @@ -59,8 +59,8 @@ # Supported sites - **AlphaPorno** - **Alsace20TV** - **Alsace20TVEmbed** - - **Alura**: [alura] - - **AluraCourse**: [aluracourse] + - **Alura**: [*alura*](## "netrc machine") + - **AluraCourse**: [*aluracourse*](## "netrc machine") - **Amara** - **AmazonMiniTV** - **amazonminitv:season**: Amazon MiniTV Season, "minitv:season:" prefix @@ -100,7 +100,7 @@ # Supported sites - **ArteTVPlaylist** - **AsianCrush** - **AsianCrushPlaylist** - - **AtresPlayer**: [atresplayer] + - **AtresPlayer**: [*atresplayer*](## "netrc machine") - **AtScaleConfEvent** - **ATTTechChannel** - **ATVAt** @@ -128,15 +128,15 @@ # Supported sites - **Bandcamp:user** - **Bandcamp:weekly** - **BannedVideo** - - **bbc**: [bbc] BBC - - **bbc.co.uk**: [bbc] BBC iPlayer + - **bbc**: [*bbc*](## "netrc machine") BBC + - **bbc.co.uk**: [*bbc*](## "netrc machine") BBC iPlayer - **bbc.co.uk:article**: BBC articles - **bbc.co.uk:​iplayer:episodes** - **bbc.co.uk:​iplayer:group** - **bbc.co.uk:playlist** - - **BBVTV**: [bbvtv] - - **BBVTVLive**: [bbvtv] - - **BBVTVRecordings**: [bbvtv] + - **BBVTV**: [*bbvtv*](## "netrc machine") + - **BBVTVLive**: [*bbvtv*](## "netrc machine") + - **BBVTVRecordings**: [*bbvtv*](## "netrc machine") - **BeatBumpPlaylist** - **BeatBumpVideo** - **Beatport** @@ -165,8 +165,8 @@ # Supported sites - **BilibiliSpaceAudio** - **BilibiliSpacePlaylist** - **BilibiliSpaceVideo** - - **BiliIntl**: [biliintl] - - **biliIntl:series**: [biliintl] + - **BiliIntl**: [*biliintl*](## "netrc machine") + - **biliIntl:series**: [*biliintl*](## "netrc machine") - **BiliLive** - **BioBioChileTV** - **Biography** @@ -232,7 +232,7 @@ # Supported sites - **cbssports:embed** - **CCMA** - **CCTV**: 央视网 - - **CDA**: [cdapl] + - **CDA**: [*cdapl*](## "netrc machine") - **Cellebrite** - **CeskaTelevize** - **CGTN** @@ -286,8 +286,8 @@ # Supported sites - **CrooksAndLiars** - **CrowdBunker** - **CrowdBunkerChannel** - - **crunchyroll**: [crunchyroll] - - **crunchyroll:playlist**: [crunchyroll] + - **crunchyroll**: [*crunchyroll*](## "netrc machine") + - **crunchyroll:playlist**: [*crunchyroll*](## "netrc machine") - **CSpan**: C-SPAN - **CSpanCongress** - **CtsNews**: 華視新聞 @@ -295,18 +295,18 @@ # Supported sites - **CTVNews** - **cu.ntv.co.jp**: Nippon Television Network - **CultureUnplugged** - - **curiositystream**: [curiositystream] - - **curiositystream:collections**: [curiositystream] - - **curiositystream:series**: [curiositystream] + - **curiositystream**: [*curiositystream*](## "netrc machine") + - **curiositystream:collections**: [*curiositystream*](## "netrc machine") + - **curiositystream:series**: [*curiositystream*](## "netrc machine") - **CWTV** - - **Cybrary**: [cybrary] - - **CybraryCourse**: [cybrary] + - **Cybrary**: [*cybrary*](## "netrc machine") + - **CybraryCourse**: [*cybrary*](## "netrc machine") - **Daftsex** - **DagelijkseKost**: dagelijksekost.een.be - **DailyMail** - - **dailymotion**: [dailymotion] - - **dailymotion:playlist**: [dailymotion] - - **dailymotion:user**: [dailymotion] + - **dailymotion**: [*dailymotion*](## "netrc machine") + - **dailymotion:playlist**: [*dailymotion*](## "netrc machine") + - **dailymotion:user**: [*dailymotion*](## "netrc machine") - **DailyWire** - **DailyWirePodcast** - **damtomo:record** @@ -328,7 +328,7 @@ # Supported sites - **DeuxMNews** - **DHM**: Filmarchiv - Deutsches Historisches Museum - **Digg** - - **DigitalConcertHall**: [digitalconcerthall] DigitalConcertHall extractor + - **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor - **DigitallySpeaking** - **Digiteka** - **Discovery** @@ -351,7 +351,7 @@ # Supported sites - **DRBonanza** - **Drooble** - **Dropbox** - - **Dropout**: [dropout] + - **Dropout**: [*dropout*](## "netrc machine") - **DropoutSeason** - **DrTuber** - **drtv** @@ -373,9 +373,9 @@ # Supported sites - **egghead:lesson**: egghead.io lesson - **ehftv** - **eHow** - - **EinsUndEinsTV**: [1und1tv] - - **EinsUndEinsTVLive**: [1und1tv] - - **EinsUndEinsTVRecordings**: [1und1tv] + - **EinsUndEinsTV**: [*1und1tv*](## "netrc machine") + - **EinsUndEinsTVLive**: [*1und1tv*](## "netrc machine") + - **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine") - **Einthusan** - **eitb.tv** - **EllenTube** @@ -390,7 +390,7 @@ # Supported sites - **EpiconSeries** - **Epoch** - **Eporner** - - **EroProfile**: [eroprofile] + - **EroProfile**: [*eroprofile*](## "netrc machine") - **EroProfile:album** - **ertflix**: ERTFLIX videos - **ertflix:codename**: ERTFLIX videos by codename @@ -405,20 +405,20 @@ # Supported sites - **EuropeanTour** - **Eurosport** - **EUScreen** - - **EWETV**: [ewetv] - - **EWETVLive**: [ewetv] - - **EWETVRecordings**: [ewetv] + - **EWETV**: [*ewetv*](## "netrc machine") + - **EWETVLive**: [*ewetv*](## "netrc machine") + - **EWETVRecordings**: [*ewetv*](## "netrc machine") - **ExpoTV** - **Expressen** - **ExtremeTube** - **EyedoTV** - - **facebook**: [facebook] + - **facebook**: [*facebook*](## "netrc machine") - **facebook:reel** - **FacebookPluginsVideo** - - **fancode:live**: [fancode] - - **fancode:vod**: [fancode] + - **fancode:live**: [*fancode*](## "netrc machine") + - **fancode:vod**: [*fancode*](## "netrc machine") - **faz.net** - - **fc2**: [fc2] + - **fc2**: [*fc2*](## "netrc machine") - **fc2:embed** - **fc2:live** - **Fczenit** @@ -452,20 +452,20 @@ # Supported sites - **freespeech.org** - **freetv:series** - **FreeTvMovies** - - **FrontendMasters**: [frontendmasters] - - **FrontendMastersCourse**: [frontendmasters] - - **FrontendMastersLesson**: [frontendmasters] + - **FrontendMasters**: [*frontendmasters*](## "netrc machine") + - **FrontendMastersCourse**: [*frontendmasters*](## "netrc machine") + - **FrontendMastersLesson**: [*frontendmasters*](## "netrc machine") - **FujiTVFODPlus7** - - **Funimation**: [funimation] - - **funimation:page**: [funimation] - - **funimation:show**: [funimation] + - **Funimation**: [*funimation*](## "netrc machine") + - **funimation:page**: [*funimation*](## "netrc machine") + - **funimation:show**: [*funimation*](## "netrc machine") - **Funk** - **Fusion** - **Fux** - **FuyinTV** - **Gab** - **GabTV** - - **Gaia**: [gaia] + - **Gaia**: [*gaia*](## "netrc machine") - **GameInformer** - **GameJolt** - **GameJoltCommunity** @@ -477,9 +477,9 @@ # Supported sites - **GameStar** - **Gaskrank** - **Gazeta** - - **GDCVault**: [gdcvault] + - **GDCVault**: [*gdcvault*](## "netrc machine") - **GediDigital** - - **gem.cbc.ca**: [cbcgem] + - **gem.cbc.ca**: [*cbcgem*](## "netrc machine") - **gem.cbc.ca:live** - **gem.cbc.ca:playlist** - **Genius** @@ -489,11 +489,11 @@ # Supported sites - **Gfycat** - **GiantBomb** - **Giga** - - **GlattvisionTV**: [glattvisiontv] - - **GlattvisionTVLive**: [glattvisiontv] - - **GlattvisionTVRecordings**: [glattvisiontv] + - **GlattvisionTV**: [*glattvisiontv*](## "netrc machine") + - **GlattvisionTVLive**: [*glattvisiontv*](## "netrc machine") + - **GlattvisionTVRecordings**: [*glattvisiontv*](## "netrc machine") - **Glide**: Glide mobile video messages (glide.me) - - **Globo**: [globo] + - **Globo**: [*globo*](## "netrc machine") - **GloboArticle** - **glomex**: Glomex videos - **glomex:embed**: Glomex embedded videos @@ -507,7 +507,7 @@ # Supported sites - **google:​podcasts:feed** - **GoogleDrive** - **GoogleDrive:Folder** - - **GoPlay**: [goplay] + - **GoPlay**: [*goplay*](## "netrc machine") - **GoPro** - **Goshgay** - **GoToStage** @@ -527,7 +527,7 @@ # Supported sites - **hgtv.com:show** - **HGTVDe** - **HGTVUsa** - - **HiDive**: [hidive] + - **HiDive**: [*hidive*](## "netrc machine") - **HistoricFilms** - **history:player** - **history:topic**: History.com Topic @@ -544,8 +544,8 @@ # Supported sites - **Howcast** - **HowStuffWorks** - **hrfernsehen** - - **HRTi**: [hrti] - - **HRTiPlaylist**: [hrti] + - **HRTi**: [*hrti*](## "netrc machine") + - **HRTiPlaylist**: [*hrti*](## "netrc machine") - **HSEProduct** - **HSEShow** - **html5** @@ -575,19 +575,19 @@ # Supported sites - **Inc** - **IndavideoEmbed** - **InfoQ** - - **Instagram**: [instagram] - - **instagram:story**: [instagram] - - **instagram:tag**: [instagram] Instagram hashtag search URLs - - **instagram:user**: [instagram] Instagram user profile + - **Instagram**: [*instagram*](## "netrc machine") + - **instagram:story**: [*instagram*](## "netrc machine") + - **instagram:tag**: [*instagram*](## "netrc machine") Instagram hashtag search URLs + - **instagram:user**: [*instagram*](## "netrc machine") Instagram user profile - **InstagramIOS**: IOS instagram:// URL - **Internazionale** - **InternetVideoArchive** - **InvestigationDiscovery** - - **IPrima**: [iprima] + - **IPrima**: [*iprima*](## "netrc machine") - **IPrimaCNN** - **iq.com**: International version of iQiyi - **iq.com:album** - - **iqiyi**: [iqiyi] 爱奇艺 + - **iqiyi**: [*iqiyi*](## "netrc machine") 爱奇艺 - **IslamChannel** - **IslamChannelSeries** - **IsraelNationalNews** @@ -660,9 +660,9 @@ # Supported sites - **LcpPlay** - **Le**: 乐视网 - **Lecture2Go** - - **Lecturio**: [lecturio] - - **LecturioCourse**: [lecturio] - - **LecturioDeCourse**: [lecturio] + - **Lecturio**: [*lecturio*](## "netrc machine") + - **LecturioCourse**: [*lecturio*](## "netrc machine") + - **LecturioDeCourse**: [*lecturio*](## "netrc machine") - **LEGO** - **Lemonde** - **Lenta** @@ -678,10 +678,10 @@ # Supported sites - **limelight:channel_list** - **LineLive** - **LineLiveChannel** - - **LinkedIn**: [linkedin] - - **linkedin:learning**: [linkedin] - - **linkedin:​learning:course**: [linkedin] - - **LinuxAcademy**: [linuxacademy] + - **LinkedIn**: [*linkedin*](## "netrc machine") + - **linkedin:learning**: [*linkedin*](## "netrc machine") + - **linkedin:​learning:course**: [*linkedin*](## "netrc machine") + - **LinuxAcademy**: [*linuxacademy*](## "netrc machine") - **Liputan6** - **ListenNotes** - **LiTV** @@ -696,8 +696,8 @@ # Supported sites - **LoveHomePorn** - **LRTStream** - **LRTVOD** - - **lynda**: [lynda] lynda.com videos - - **lynda:course**: [lynda] lynda.com online courses + - **lynda**: [*lynda*](## "netrc machine") lynda.com videos + - **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses - **m6** - **MagentaMusik360** - **mailru**: Видео@Mail.Ru @@ -767,13 +767,13 @@ # Supported sites - **mixcloud:user** - **MLB** - **MLBArticle** - - **MLBTV**: [mlb] + - **MLBTV**: [*mlb*](## "netrc machine") - **MLBVideo** - **MLSSoccer** - **Mnet** - - **MNetTV**: [mnettv] - - **MNetTVLive**: [mnettv] - - **MNetTVRecordings**: [mnettv] + - **MNetTV**: [*mnettv*](## "netrc machine") + - **MNetTVLive**: [*mnettv*](## "netrc machine") + - **MNetTVRecordings**: [*mnettv*](## "netrc machine") - **MochaVideo** - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net - **Mofosex** @@ -852,9 +852,9 @@ # Supported sites - **ndr:embed** - **ndr:​embed:base** - **NDTV** - - **Nebula**: [watchnebula] - - **nebula:channel**: [watchnebula] - - **nebula:subscriptions**: [watchnebula] + - **Nebula**: [*watchnebula*](## "netrc machine") + - **nebula:channel**: [*watchnebula*](## "netrc machine") + - **nebula:subscriptions**: [*watchnebula*](## "netrc machine") - **NerdCubedFeed** - **netease:album**: 网易云音乐 - 专辑 - **netease:djradio**: 网易云音乐 - 电台 @@ -863,9 +863,9 @@ # Supported sites - **netease:program**: 网易云音乐 - 电台节目 - **netease:singer**: 网易云音乐 - 歌手 - **netease:song**: 网易云音乐 - - **NetPlusTV**: [netplus] - - **NetPlusTVLive**: [netplus] - - **NetPlusTVRecordings**: [netplus] + - **NetPlusTV**: [*netplus*](## "netrc machine") + - **NetPlusTVLive**: [*netplus*](## "netrc machine") + - **NetPlusTVRecordings**: [*netplus*](## "netrc machine") - **Netverse** - **NetversePlaylist** - **NetverseSearch**: "netsearch:" prefix @@ -898,7 +898,7 @@ # Supported sites - **nickelodeon:br** - **nickelodeonru** - **nicknight** - - **niconico**: [niconico] ニコニコ動画 + - **niconico**: [*niconico*](## "netrc machine") ニコニコ動画 - **niconico:history**: NicoNico user history or likes. Requires cookies. - **niconico:playlist** - **niconico:series** @@ -911,7 +911,7 @@ # Supported sites - **Nitter** - **njoy**: N-JOY - **njoy:embed** - - **NJPWWorld**: [njpwworld] 新日本プロレスワールド + - **NJPWWorld**: [*njpwworld*](## "netrc machine") 新日本プロレスワールド - **NobelPrize** - **NoicePodcast** - **NonkTube** @@ -980,11 +980,11 @@ # Supported sites - **orf:iptv**: iptv.ORF.at - **orf:radio** - **orf:tvthek**: ORF TVthek - - **OsnatelTV**: [osnateltv] - - **OsnatelTVLive**: [osnateltv] - - **OsnatelTVRecordings**: [osnateltv] + - **OsnatelTV**: [*osnateltv*](## "netrc machine") + - **OsnatelTVLive**: [*osnateltv*](## "netrc machine") + - **OsnatelTVRecordings**: [*osnateltv*](## "netrc machine") - **OutsideTV** - - **PacktPub**: [packtpub] + - **PacktPub**: [*packtpub*](## "netrc machine") - **PacktPubCourse** - **PalcoMP3:artist** - **PalcoMP3:song** @@ -1007,7 +1007,7 @@ # Supported sites - **peer.tv** - **PeerTube** - **PeerTube:Playlist** - - **peloton**: [peloton] + - **peloton**: [*peloton*](## "netrc machine") - **peloton:live**: Peloton Live - **People** - **PerformGroup** @@ -1016,7 +1016,7 @@ # Supported sites - **PhilharmonieDeParis**: Philharmonie de Paris - **phoenix.de** - **Photobucket** - - **Piapro**: [piapro] + - **Piapro**: [*piapro*](## "netrc machine") - **Picarto** - **PicartoVod** - **Piksel** @@ -1027,11 +1027,11 @@ # Supported sites - **pixiv:​sketch:user** - **Pladform** - **PlanetMarathi** - - **Platzi**: [platzi] - - **PlatziCourse**: [platzi] + - **Platzi**: [*platzi*](## "netrc machine") + - **PlatziCourse**: [*platzi*](## "netrc machine") - **play.fm** - **player.sky.it** - - **PlayPlusTV**: [playplustv] + - **PlayPlusTV**: [*playplustv*](## "netrc machine") - **PlayStuff** - **PlaysTV** - **PlaySuisse** @@ -1039,7 +1039,7 @@ # Supported sites - **Playvid** - **PlayVids** - **Playwire** - - **pluralsight**: [pluralsight] + - **pluralsight**: [*pluralsight*](## "netrc machine") - **pluralsight:course** - **PlutoTV** - **PodbayFM** @@ -1048,8 +1048,8 @@ # Supported sites - **podomatic** - **Pokemon** - **PokemonWatch** - - **PokerGo**: [pokergo] - - **PokerGoCollection**: [pokergo] + - **PokerGo**: [*pokergo*](## "netrc machine") + - **PokerGoCollection**: [*pokergo*](## "netrc machine") - **PolsatGo** - **PolskieRadio** - **polskieradio:audition** @@ -1066,11 +1066,11 @@ # Supported sites - **Pornez** - **PornFlip** - **PornHd** - - **PornHub**: [pornhub] PornHub and Thumbzilla - - **PornHubPagedVideoList**: [pornhub] - - **PornHubPlaylist**: [pornhub] - - **PornHubUser**: [pornhub] - - **PornHubUserVideosUpload**: [pornhub] + - **PornHub**: [*pornhub*](## "netrc machine") PornHub and Thumbzilla + - **PornHubPagedVideoList**: [*pornhub*](## "netrc machine") + - **PornHubPlaylist**: [*pornhub*](## "netrc machine") + - **PornHubUser**: [*pornhub*](## "netrc machine") + - **PornHubUserVideosUpload**: [*pornhub*](## "netrc machine") - **Pornotube** - **PornoVoisines** - **PornoXO** @@ -1098,9 +1098,9 @@ # Supported sites - **qqmusic:playlist**: QQ音乐 - 歌单 - **qqmusic:singer**: QQ音乐 - 歌手 - **qqmusic:toplist**: QQ音乐 - 排行榜 - - **QuantumTV**: [quantumtv] - - **QuantumTVLive**: [quantumtv] - - **QuantumTVRecordings**: [quantumtv] + - **QuantumTV**: [*quantumtv*](## "netrc machine") + - **QuantumTVLive**: [*quantumtv*](## "netrc machine") + - **QuantumTVRecordings**: [*quantumtv*](## "netrc machine") - **Qub** - **R7** - **R7Article** @@ -1157,16 +1157,16 @@ # Supported sites - **RICE** - **RMCDecouverte** - **RockstarGames** - - **Rokfin**: [rokfin] + - **Rokfin**: [*rokfin*](## "netrc machine") - **rokfin:channel**: Rokfin Channels - **rokfin:search**: Rokfin Search; "rkfnsearch:" prefix - **rokfin:stack**: Rokfin Stacks - - **RoosterTeeth**: [roosterteeth] - - **RoosterTeethSeries**: [roosterteeth] + - **RoosterTeeth**: [*roosterteeth*](## "netrc machine") + - **RoosterTeethSeries**: [*roosterteeth*](## "netrc machine") - **RottenTomatoes** - **Rozhlas** - **RozhlasVltava** - - **RTBF**: [rtbf] + - **RTBF**: [*rtbf*](## "netrc machine") - **RTDocumentry** - **RTDocumentryPlaylist** - **rte**: Raidió Teilifís Éireann TV @@ -1208,16 +1208,16 @@ # Supported sites - **Ruutu** - **Ruv** - **ruv.is:spila** - - **safari**: [safari] safaribooksonline.com online video - - **safari:api**: [safari] - - **safari:course**: [safari] safaribooksonline.com online courses + - **safari**: [*safari*](## "netrc machine") safaribooksonline.com online video + - **safari:api**: [*safari*](## "netrc machine") + - **safari:course**: [*safari*](## "netrc machine") safaribooksonline.com online courses - **Saitosan** - - **SAKTV**: [saktv] - - **SAKTVLive**: [saktv] - - **SAKTVRecordings**: [saktv] - - **SaltTV**: [salttv] - - **SaltTVLive**: [salttv] - - **SaltTVRecordings**: [salttv] + - **SAKTV**: [*saktv*](## "netrc machine") + - **SAKTVLive**: [*saktv*](## "netrc machine") + - **SAKTVRecordings**: [*saktv*](## "netrc machine") + - **SaltTV**: [*salttv*](## "netrc machine") + - **SaltTVLive**: [*salttv*](## "netrc machine") + - **SaltTVRecordings**: [*salttv*](## "netrc machine") - **SampleFocus** - **Sangiin**: 参議院インターネット審議中継 (archive) - **Sapo**: SAPO Vídeos @@ -1233,8 +1233,8 @@ # Supported sites - **ScrippsNetworks** - **scrippsnetworks:watch** - **Scrolller** - - **SCTE**: [scte] - - **SCTECourse**: [scte] + - **SCTE**: [*scte*](## "netrc machine") + - **SCTECourse**: [*scte*](## "netrc machine") - **Seeker** - **SenateGov** - **SenateISVP** @@ -1243,7 +1243,7 @@ # Supported sites - **Sexu** - **SeznamZpravy** - **SeznamZpravyArticle** - - **Shahid**: [shahid] + - **Shahid**: [*shahid*](## "netrc machine") - **ShahidShow** - **Shared**: shared.sx - **ShareVideosEmbed** @@ -1273,16 +1273,16 @@ # Supported sites - **Smotrim** - **Snotr** - **Sohu** - - **SonyLIV**: [sonyliv] + - **SonyLIV**: [*sonyliv*](## "netrc machine") - **SonyLIVSeries** - - **soundcloud**: [soundcloud] - - **soundcloud:playlist**: [soundcloud] - - **soundcloud:related**: [soundcloud] - - **soundcloud:search**: [soundcloud] Soundcloud search; "scsearch:" prefix - - **soundcloud:set**: [soundcloud] - - **soundcloud:trackstation**: [soundcloud] - - **soundcloud:user**: [soundcloud] - - **soundcloud:​user:permalink**: [soundcloud] + - **soundcloud**: [*soundcloud*](## "netrc machine") + - **soundcloud:playlist**: [*soundcloud*](## "netrc machine") + - **soundcloud:related**: [*soundcloud*](## "netrc machine") + - **soundcloud:search**: [*soundcloud*](## "netrc machine") Soundcloud search; "scsearch:" prefix + - **soundcloud:set**: [*soundcloud*](## "netrc machine") + - **soundcloud:trackstation**: [*soundcloud*](## "netrc machine") + - **soundcloud:user**: [*soundcloud*](## "netrc machine") + - **soundcloud:​user:permalink**: [*soundcloud*](## "netrc machine") - **SoundcloudEmbed** - **soundgasm** - **soundgasm:profile** @@ -1349,13 +1349,13 @@ # Supported sites - **Tass** - **TBS** - **TDSLifeway** - - **Teachable**: [teachable] - - **TeachableCourse**: [teachable] + - **Teachable**: [*teachable*](## "netrc machine") + - **TeachableCourse**: [*teachable*](## "netrc machine") - **teachertube**: teachertube.com videos - **teachertube:​user:collection**: teachertube.com user and collection videos - **TeachingChannel** - **Teamcoco** - - **TeamTreeHouse**: [teamtreehouse] + - **TeamTreeHouse**: [*teamtreehouse*](## "netrc machine") - **TechTalks** - **techtv.mit.edu** - **TedEmbed** @@ -1378,8 +1378,8 @@ # Supported sites - **TeleTask** - **Telewebion** - **Tempo** - - **TennisTV**: [tennistv] - - **TenPlay**: [10play] + - **TennisTV**: [*tennistv*](## "netrc machine") + - **TenPlay**: [*10play*](## "netrc machine") - **TF1** - **TFO** - **TheHoleTv** @@ -1417,13 +1417,13 @@ # Supported sites - **tokfm:audition** - **tokfm:podcast** - **ToonGoggles** - - **tou.tv**: [toutv] + - **tou.tv**: [*toutv*](## "netrc machine") - **Toypics**: Toypics video - **ToypicsUser**: Toypics user profile - **TrailerAddict**: (**Currently broken**) - **TravelChannel** - - **Triller**: [triller] - - **TrillerUser**: [triller] + - **Triller**: [*triller*](## "netrc machine") + - **TrillerUser**: [*triller*](## "netrc machine") - **Trilulilu** - **Trovo** - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix @@ -1435,11 +1435,11 @@ # Supported sites - **Truth** - **TruTV** - **Tube8** - - **TubeTuGraz**: [tubetugraz] tube.tugraz.at - - **TubeTuGrazSeries**: [tubetugraz] - - **TubiTv**: [tubitv] + - **TubeTuGraz**: [*tubetugraz*](## "netrc machine") tube.tugraz.at + - **TubeTuGrazSeries**: [*tubetugraz*](## "netrc machine") + - **TubiTv**: [*tubitv*](## "netrc machine") - **TubiTvShow** - - **Tumblr**: [tumblr] + - **Tumblr**: [*tumblr*](## "netrc machine") - **tunein:clip** - **tunein:program** - **tunein:station** @@ -1489,13 +1489,13 @@ # Supported sites - **TwitCasting** - **TwitCastingLive** - **TwitCastingUser** - - **twitch:clips**: [twitch] - - **twitch:stream**: [twitch] - - **twitch:vod**: [twitch] - - **TwitchCollection**: [twitch] - - **TwitchVideos**: [twitch] - - **TwitchVideosClips**: [twitch] - - **TwitchVideosCollections**: [twitch] + - **twitch:clips**: [*twitch*](## "netrc machine") + - **twitch:stream**: [*twitch*](## "netrc machine") + - **twitch:vod**: [*twitch*](## "netrc machine") + - **TwitchCollection**: [*twitch*](## "netrc machine") + - **TwitchVideos**: [*twitch*](## "netrc machine") + - **TwitchVideosClips**: [*twitch*](## "netrc machine") + - **TwitchVideosCollections**: [*twitch*](## "netrc machine") - **twitter** - **twitter:amplify** - **twitter:broadcast** @@ -1503,11 +1503,11 @@ # Supported sites - **twitter:shortener** - **twitter:spaces** - **Txxx** - - **udemy**: [udemy] - - **udemy:course**: [udemy] + - **udemy**: [*udemy*](## "netrc machine") + - **udemy:course**: [*udemy*](## "netrc machine") - **UDNEmbed**: 聯合影音 - - **UFCArabia**: [ufcarabia] - - **UFCTV**: [ufctv] + - **UFCArabia**: [*ufcarabia*](## "netrc machine") + - **UFCTV**: [*ufctv*](## "netrc machine") - **ukcolumn** - **UKTVPlay** - **umg:de**: Universal Music Deutschland @@ -1537,7 +1537,7 @@ # Supported sites - **VevoPlaylist** - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet - **vh1.com** - - **vhx:embed**: [vimeo] + - **vhx:embed**: [*vimeo*](## "netrc machine") - **Viafree** - **vice** - **vice:article** @@ -1560,25 +1560,25 @@ # Supported sites - **videomore:season** - **videomore:video** - **VideoPress** - - **Vidio**: [vidio] - - **VidioLive**: [vidio] - - **VidioPremier**: [vidio] + - **Vidio**: [*vidio*](## "netrc machine") + - **VidioLive**: [*vidio*](## "netrc machine") + - **VidioPremier**: [*vidio*](## "netrc machine") - **VidLii** - **viewlift** - **viewlift:embed** - **Viidea** - - **viki**: [viki] - - **viki:channel**: [viki] - - **vimeo**: [vimeo] - - **vimeo:album**: [vimeo] - - **vimeo:channel**: [vimeo] - - **vimeo:group**: [vimeo] - - **vimeo:likes**: [vimeo] Vimeo user likes - - **vimeo:ondemand**: [vimeo] - - **vimeo:pro**: [vimeo] - - **vimeo:review**: [vimeo] Review pages on vimeo - - **vimeo:user**: [vimeo] - - **vimeo:watchlater**: [vimeo] Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication) + - **viki**: [*viki*](## "netrc machine") + - **viki:channel**: [*viki*](## "netrc machine") + - **vimeo**: [*vimeo*](## "netrc machine") + - **vimeo:album**: [*vimeo*](## "netrc machine") + - **vimeo:channel**: [*vimeo*](## "netrc machine") + - **vimeo:group**: [*vimeo*](## "netrc machine") + - **vimeo:likes**: [*vimeo*](## "netrc machine") Vimeo user likes + - **vimeo:ondemand**: [*vimeo*](## "netrc machine") + - **vimeo:pro**: [*vimeo*](## "netrc machine") + - **vimeo:review**: [*vimeo*](## "netrc machine") Review pages on vimeo + - **vimeo:user**: [*vimeo*](## "netrc machine") + - **vimeo:watchlater**: [*vimeo*](## "netrc machine") Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication) - **Vimm:recording** - **Vimm:stream** - **ViMP** @@ -1588,13 +1588,13 @@ # Supported sites - **vine:user** - **Viqeo** - **Viu** - - **viu:ott**: [viu] + - **viu:ott**: [*viu*](## "netrc machine") - **viu:playlist** - **ViuOTTIndonesia** - **Vivo**: vivo.sx - - **vk**: [vk] VK - - **vk:uservideos**: [vk] VK - User's Videos - - **vk:wallpost**: [vk] + - **vk**: [*vk*](## "netrc machine") VK + - **vk:uservideos**: [*vk*](## "netrc machine") VK - User's Videos + - **vk:wallpost**: [*vk*](## "netrc machine") - **vm.tiktok** - **Vocaroo** - **Vodlocker** @@ -1613,14 +1613,14 @@ # Supported sites - **vqq:video** - **Vrak** - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza - - **VrtNU**: [vrtnu] VrtNU.be - - **vrv**: [vrv] + - **VrtNU**: [*vrtnu*](## "netrc machine") VrtNU.be + - **vrv**: [*vrv*](## "netrc machine") - **vrv:series** - **VShare** - **VTM** - - **VTXTV**: [vtxtv] - - **VTXTVLive**: [vtxtv] - - **VTXTVRecordings**: [vtxtv] + - **VTXTV**: [*vtxtv*](## "netrc machine") + - **VTXTVLive**: [*vtxtv*](## "netrc machine") + - **VTXTVRecordings**: [*vtxtv*](## "netrc machine") - **VuClip** - **Vupload** - **VVVVID** @@ -1629,9 +1629,9 @@ # Supported sites - **Vzaar** - **Wakanim** - **Walla** - - **WalyTV**: [walytv] - - **WalyTVLive**: [walytv] - - **WalyTVRecordings**: [walytv] + - **WalyTV**: [*walytv*](## "netrc machine") + - **WalyTVLive**: [*walytv*](## "netrc machine") + - **WalyTVRecordings**: [*walytv*](## "netrc machine") - **wasdtv:clip** - **wasdtv:record** - **wasdtv:stream** @@ -1743,13 +1743,13 @@ # Supported sites - **YoutubeLivestreamEmbed**: YouTube livestream embeds - **YoutubeYtBe**: youtu.be - **Zapiks** - - **Zattoo**: [zattoo] - - **ZattooLive**: [zattoo] - - **ZattooMovies**: [zattoo] - - **ZattooRecordings**: [zattoo] + - **Zattoo**: [*zattoo*](## "netrc machine") + - **ZattooLive**: [*zattoo*](## "netrc machine") + - **ZattooMovies**: [*zattoo*](## "netrc machine") + - **ZattooRecordings**: [*zattoo*](## "netrc machine") - **ZDF** - **ZDFChannel** - - **Zee5**: [zee5] + - **Zee5**: [*zee5*](## "netrc machine") - **zee5:series** - **ZeeNews** - **ZenYandex** diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index d6c5ce769f..00846cd7e0 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -614,7 +614,7 @@ def __init__(self, params=None, auto_init=True): '\n You will no longer receive updates on this version') if current_version < MIN_SUPPORTED: msg = 'Python version %d.%d is no longer supported' - self.deprecation_warning( + self.deprecated_feature( f'{msg}! Please update to Python %d.%d or above' % (*current_version, *MIN_RECOMMENDED)) if self.params.get('allow_unplayable_formats'): diff --git a/yt_dlp/dependencies/Cryptodome.py b/yt_dlp/dependencies/Cryptodome.py index a50bce4d4f..74ab6575ce 100644 --- a/yt_dlp/dependencies/Cryptodome.py +++ b/yt_dlp/dependencies/Cryptodome.py @@ -14,22 +14,14 @@ try: if _parent.__name__ == 'Cryptodome': from Cryptodome import __version__ - from Cryptodome.Cipher import AES - from Cryptodome.Cipher import PKCS1_v1_5 - from Cryptodome.Cipher import Blowfish - from Cryptodome.Cipher import PKCS1_OAEP - from Cryptodome.Hash import SHA1 - from Cryptodome.Hash import CMAC + from Cryptodome.Cipher import AES, PKCS1_OAEP, Blowfish, PKCS1_v1_5 + from Cryptodome.Hash import CMAC, SHA1 from Cryptodome.PublicKey import RSA elif _parent.__name__ == 'Crypto': from Crypto import __version__ - from Crypto.Cipher import AES - from Crypto.Cipher import PKCS1_v1_5 - from Crypto.Cipher import Blowfish - from Crypto.Cipher import PKCS1_OAEP - from Crypto.Hash import SHA1 - from Crypto.Hash import CMAC - from Crypto.PublicKey import RSA + from Crypto.Cipher import AES, PKCS1_OAEP, Blowfish, PKCS1_v1_5 # noqa: F401 + from Crypto.Hash import CMAC, SHA1 # noqa: F401 + from Crypto.PublicKey import RSA # noqa: F401 except ImportError: __version__ = f'broken {__version__}'.strip() diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index 377f138b76..3dc638f523 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -497,7 +497,7 @@ def _download_fragment(fragment): download_fragment(fragment, ctx_copy) return fragment, fragment['frag_index'], ctx_copy.get('fragment_filename_sanitized') - self.report_warning('The download speed shown is only of one thread. This is a known issue and patches are welcome') + self.report_warning('The download speed shown is only of one thread. This is a known issue') with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool: try: for fragment, frag_index, frag_filename in pool.map(_download_fragment, fragments): diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 98efe0e9da..8ad63b4118 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -3527,7 +3527,7 @@ def description(cls, *, markdown=True, search_examples=None): desc = '' if cls._NETRC_MACHINE: if markdown: - desc += f' [{cls._NETRC_MACHINE}]' + desc += f' [*{cls._NETRC_MACHINE}*](## "netrc machine")' else: desc += f' [{cls._NETRC_MACHINE}]' if cls.IE_DESC is False: diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index d1696349aa..44e9322937 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -956,7 +956,7 @@ def _extract_response(self, item_id, query, note='Downloading API JSON', headers @staticmethod def is_music_url(url): - return re.match(r'https?://music\.youtube\.com/', url) is not None + return re.match(r'(https?://)?music\.youtube\.com/', url) is not None def _extract_video(self, renderer): video_id = renderer.get('videoId') @@ -6211,6 +6211,8 @@ def _real_extract(self, url, smuggled_data): original_tab_id, display_id = tab[1:], f'{item_id}{tab}' if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts: url = f'{pre}/videos{post}' + if smuggled_data.get('is_music_url'): + self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}') # Handle both video/playlist URLs qs = parse_qs(url) From 7f51861b1820c37b157a239b1fe30628d907c034 Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Wed, 1 Mar 2023 07:56:53 +0000 Subject: [PATCH 0012/1178] [extractor/youtube] Detect and break on looping comments (#6301) Fixes https://github.com/yt-dlp/yt-dlp/issues/6290 Authored by: coletdjnz --- yt_dlp/extractor/youtube.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 44e9322937..b02e0153af 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -3341,6 +3341,13 @@ def extract_thread(contents): comment = self._extract_comment(comment_renderer, parent) if not comment: continue + # Sometimes YouTube may break and give us infinite looping comments. + # See: https://github.com/yt-dlp/yt-dlp/issues/6290 + if comment['id'] in tracker['seen_comment_ids']: + self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.') + yield + else: + tracker['seen_comment_ids'].add(comment['id']) tracker['running_total'] += 1 tracker['total_reply_comments' if parent else 'total_parent_comments'] += 1 @@ -3365,7 +3372,8 @@ def extract_thread(contents): est_total=0, current_page_thread=0, total_parent_comments=0, - total_reply_comments=0) + total_reply_comments=0, + seen_comment_ids=set()) # TODO: Deprecated # YouTube comments have a max depth of 2 From b38cae49e6f4849c8ee2a774bdc3c1c647ae5f0e Mon Sep 17 00:00:00 2001 From: bashonly Date: Wed, 1 Mar 2023 06:38:02 -0600 Subject: [PATCH 0013/1178] [extractor/generic] Detect manifest links via extension Authored by: bashonly --- yt_dlp/extractor/generic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index d76ef3e31c..49aa5a1f5c 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -2393,14 +2393,15 @@ def _real_extract(self, url): self.report_detected('direct video link') headers = smuggled_data.get('http_headers', {}) format_id = str(m.group('format_id')) + ext = determine_ext(url) subtitles = {} - if format_id.endswith('mpegurl'): + if format_id.endswith('mpegurl') or ext == 'm3u8': formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers) info_dict.update(self._fragment_query(url)) - elif format_id.endswith('mpd') or format_id.endswith('dash+xml'): + elif format_id.endswith('mpd') or format_id.endswith('dash+xml') or ext == 'mpd': formats, subtitles = self._extract_mpd_formats_and_subtitles(url, video_id, headers=headers) info_dict.update(self._fragment_query(url)) - elif format_id == 'f4m': + elif format_id == 'f4m' or ext == 'f4m': formats = self._extract_f4m_formats(url, video_id, headers=headers) else: formats = [{ From 9fddc12ab022a31754e0eaa358fc4e1dfa974587 Mon Sep 17 00:00:00 2001 From: std-move <26625259+std-move@users.noreply.github.com> Date: Thu, 2 Mar 2023 19:33:33 +0100 Subject: [PATCH 0014/1178] [extractor/iprima] Fix extractor (#6291) Authored by: std-move Closes #6187 --- yt_dlp/extractor/iprima.py | 41 +++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/yt_dlp/extractor/iprima.py b/yt_dlp/extractor/iprima.py index 181820542c..e58e9c2ee1 100644 --- a/yt_dlp/extractor/iprima.py +++ b/yt_dlp/extractor/iprima.py @@ -7,7 +7,8 @@ js_to_json, urlencode_postdata, ExtractorError, - parse_qs + parse_qs, + traverse_obj ) @@ -15,8 +16,7 @@ class IPrimaIE(InfoExtractor): _VALID_URL = r'https?://(?!cnn)(?:[^/]+)\.iprima\.cz/(?:[^/]+/)*(?P[^/?#&]+)' _GEO_BYPASS = False _NETRC_MACHINE = 'iprima' - _LOGIN_URL = 'https://auth.iprima.cz/oauth2/login' - _TOKEN_URL = 'https://auth.iprima.cz/oauth2/token' + _AUTH_ROOT = 'https://auth.iprima.cz' access_token = None _TESTS = [{ @@ -67,7 +67,7 @@ def _perform_login(self, username, password): return login_page = self._download_webpage( - self._LOGIN_URL, None, note='Downloading login page', + f'{self._AUTH_ROOT}/oauth2/login', None, note='Downloading login page', errnote='Downloading login page failed') login_form = self._hidden_inputs(login_page) @@ -76,11 +76,20 @@ def _perform_login(self, username, password): '_email': username, '_password': password}) - _, login_handle = self._download_webpage_handle( - self._LOGIN_URL, None, data=urlencode_postdata(login_form), + profile_select_html, login_handle = self._download_webpage_handle( + f'{self._AUTH_ROOT}/oauth2/login', None, data=urlencode_postdata(login_form), note='Logging in') - code = parse_qs(login_handle.geturl()).get('code')[0] + # a profile may need to be selected first, even when there is only a single one + if '/profile-select' in login_handle.geturl(): + profile_id = self._search_regex( + r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id') + + login_handle = self._request_webpage( + f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None, + query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile') + + code = traverse_obj(login_handle.geturl(), ({parse_qs}, 'code', 0)) if not code: raise ExtractorError('Login failed', expected=True) @@ -89,10 +98,10 @@ def _perform_login(self, username, password): 'client_id': 'prima_sso', 'grant_type': 'authorization_code', 'code': code, - 'redirect_uri': 'https://auth.iprima.cz/sso/auth-check'} + 'redirect_uri': f'{self._AUTH_ROOT}/sso/auth-check'} token_data = self._download_json( - self._TOKEN_URL, None, + f'{self._AUTH_ROOT}/oauth2/token', None, note='Downloading token', errnote='Downloading token failed', data=urlencode_postdata(token_request_data)) @@ -115,14 +124,22 @@ def _real_extract(self, url): webpage = self._download_webpage(url, video_id) - title = self._html_search_meta( + title = self._html_extract_title(webpage) or self._html_search_meta( ['og:title', 'twitter:title'], webpage, 'title', default=None) video_id = self._search_regex(( r'productId\s*=\s*([\'"])(?Pp\d+)\1', - r'pproduct_id\s*=\s*([\'"])(?Pp\d+)\1'), - webpage, 'real id', group='id') + r'pproduct_id\s*=\s*([\'"])(?Pp\d+)\1', + ), webpage, 'real id', group='id', default=None) + + if not video_id: + nuxt_data = self._search_nuxt_data(webpage, video_id, traverse='data') + video_id = traverse_obj( + nuxt_data, (..., 'content', 'additionals', 'videoPlayId', {str}), get_all=False) + + if not video_id: + self.raise_no_formats('Unable to extract video ID from webpage') metadata = self._download_json( f'https://api.play-backend.iprima.cz/api/v1//products/id-{video_id}/play', From 77d6d136468d0c23c8e79bc937898747804f585a Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 3 Mar 2023 03:34:56 -0600 Subject: [PATCH 0015/1178] [extractor/ntvru] Extract HLS and DASH formats (#6403) Closes #5915 Authored by: bashonly --- yt_dlp/extractor/ntvru.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/yt_dlp/extractor/ntvru.py b/yt_dlp/extractor/ntvru.py index 8d5877daa0..91b7724eb4 100644 --- a/yt_dlp/extractor/ntvru.py +++ b/yt_dlp/extractor/ntvru.py @@ -21,6 +21,7 @@ class NTVRuIE(InfoExtractor): 'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины', 'thumbnail': r're:^http://.*\.jpg', 'duration': 136, + 'view_count': int, }, }, { 'url': 'http://www.ntv.ru/video/novosti/750370/', @@ -32,6 +33,7 @@ class NTVRuIE(InfoExtractor): 'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход', 'thumbnail': r're:^http://.*\.jpg', 'duration': 172, + 'view_count': int, }, }, { 'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416', @@ -43,6 +45,7 @@ class NTVRuIE(InfoExtractor): 'description': '«Сегодня». 21 марта 2014 года. 16:00', 'thumbnail': r're:^http://.*\.jpg', 'duration': 1496, + 'view_count': int, }, }, { 'url': 'https://www.ntv.ru/kino/Koma_film/m70281/o336036/video/', @@ -54,6 +57,7 @@ class NTVRuIE(InfoExtractor): 'description': 'Остросюжетный фильм «Кома»', 'thumbnail': r're:^http://.*\.jpg', 'duration': 5592, + 'view_count': int, }, }, { 'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/', @@ -65,6 +69,7 @@ class NTVRuIE(InfoExtractor): 'description': '«Дело врачей»: «Деревце жизни»', 'thumbnail': r're:^http://.*\.jpg', 'duration': 2590, + 'view_count': int, }, }, { # Schemeless file URL @@ -115,6 +120,14 @@ def _real_extract(self, url): 'url': file_, 'filesize': int_or_none(xpath_text(video, './%ssize' % format_id)), }) + hls_manifest = xpath_text(video, './playback/hls') + if hls_manifest: + formats.extend(self._extract_m3u8_formats( + hls_manifest, video_id, m3u8_id='hls', fatal=False)) + dash_manifest = xpath_text(video, './playback/dash') + if dash_manifest: + formats.extend(self._extract_mpd_formats( + dash_manifest, video_id, mpd_id='dash', fatal=False)) return { 'id': xpath_text(video, './id'), From 2d5a8c5db2bd4ff1c2e45e00cd890a10f8ffca9e Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 3 Mar 2023 03:37:23 -0600 Subject: [PATCH 0016/1178] [extractor/mediastream] Improve WinSports support (#6401) Closes #6360 Authored by: bashonly --- yt_dlp/extractor/mediastream.py | 41 +++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/yt_dlp/extractor/mediastream.py b/yt_dlp/extractor/mediastream.py index 4d39495276..e8d427a319 100644 --- a/yt_dlp/extractor/mediastream.py +++ b/yt_dlp/extractor/mediastream.py @@ -1,7 +1,13 @@ import re from .common import InfoExtractor -from ..utils import clean_html, get_element_html_by_class +from ..utils import ( + remove_end, + str_or_none, + strip_or_none, + traverse_obj, + urljoin, +) class MediaStreamIE(InfoExtractor): @@ -117,39 +123,56 @@ def _real_extract(self, url): class WinSportsVideoIE(InfoExtractor): - _VALID_URL = r'https?://www\.winsports\.co/videos/(?P[\w-]+)-(?P\d+)' + _VALID_URL = r'https?://www\.winsports\.co/videos/(?P[\w-]+)' _TESTS = [{ 'url': 'https://www.winsports.co/videos/siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco-60536', 'info_dict': { 'id': '62dc8357162c4b0821fcfb3c', - 'display_id': 'siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco', + 'display_id': 'siempre-castellanos-gran-atajada-del-portero-cardenal-para-evitar-la-caida-de-su-arco-60536', 'title': '¡Siempre Castellanos! Gran atajada del portero \'cardenal\' para evitar la caída de su arco', 'description': 'md5:eb811b2b2882bdc59431732c06b905f2', 'thumbnail': r're:^https?://[^?#]+62dc8357162c4b0821fcfb3c', 'ext': 'mp4', }, + 'params': {'skip_download': 'm3u8'}, }, { 'url': 'https://www.winsports.co/videos/observa-aqui-los-goles-del-empate-entre-tolima-y-nacional-60548', 'info_dict': { 'id': '62dcb875ef12a5526790b552', - 'display_id': 'observa-aqui-los-goles-del-empate-entre-tolima-y-nacional', + 'display_id': 'observa-aqui-los-goles-del-empate-entre-tolima-y-nacional-60548', 'title': 'Observa aquí los goles del empate entre Tolima y Nacional', 'description': 'md5:b19402ba6e46558b93fd24b873eea9c9', 'thumbnail': r're:^https?://[^?#]+62dcb875ef12a5526790b552', 'ext': 'mp4', }, + 'params': {'skip_download': 'm3u8'}, + }, { + 'url': 'https://www.winsports.co/videos/equidad-vuelve-defender-su-arco-de-remates-de-junior', + 'info_dict': { + 'id': '63fa7eca72f1741ad3a4d515', + 'display_id': 'equidad-vuelve-defender-su-arco-de-remates-de-junior', + 'title': '⚽ Equidad vuelve a defender su arco de remates de Junior', + 'description': 'Remate de Sierra', + 'thumbnail': r're:^https?://[^?#]+63fa7eca72f1741ad3a4d515', + 'ext': 'mp4', + }, + 'params': {'skip_download': 'm3u8'}, }] def _real_extract(self, url): - display_id, video_id = self._match_valid_url(url).group('display_id', 'id') + display_id = self._match_id(url) webpage = self._download_webpage(url, display_id) - + json_ld = self._search_json_ld(webpage, display_id, expected_type='VideoObject', default={}) media_setting_json = self._search_json( r']+data-drupal-selector="drupal-settings-json">', webpage, 'drupal-setting-json', display_id) - mediastream_id = media_setting_json['settings']['mediastream_formatter'][video_id]['mediastream_id'] + mediastream_id = traverse_obj( + media_setting_json, ('settings', 'mediastream_formatter', ..., 'mediastream_id', {str_or_none}), + get_all=False) or json_ld.get('url') + if not mediastream_id: + self.raise_no_formats('No MediaStream embed found in webpage') return self.url_result( - f'https://mdstrm.com/embed/{mediastream_id}', MediaStreamIE, video_id, url_transparent=True, - display_id=display_id, video_title=clean_html(get_element_html_by_class('title-news', webpage))) + urljoin('https://mdstrm.com/embed/', mediastream_id), MediaStreamIE, display_id, url_transparent=True, + display_id=display_id, video_title=strip_or_none(remove_end(json_ld.get('title'), '| Win Sports'))) From 40d77d89027cd0e0ce31d22aec81db3e1d433900 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 3 Mar 2023 03:42:54 -0600 Subject: [PATCH 0017/1178] [extractor/yle_areena] Extract non-Kaltura videos (#6402) Closes #6066 Authored by: bashonly --- yt_dlp/extractor/yle_areena.py | 37 ++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/yt_dlp/extractor/yle_areena.py b/yt_dlp/extractor/yle_areena.py index 98d3b1949a..c5b45f0cb6 100644 --- a/yt_dlp/extractor/yle_areena.py +++ b/yt_dlp/extractor/yle_areena.py @@ -61,7 +61,22 @@ class YleAreenaIE(InfoExtractor): 'age_limit': 0, 'webpage_url': 'https://areena.yle.fi/1-2158940' } - } + }, + { + 'url': 'https://areena.yle.fi/1-64829589', + 'info_dict': { + 'id': '1-64829589', + 'ext': 'mp4', + 'title': 'HKO & Mälkki & Tanner', + 'description': 'md5:b4f1b1af2c6569b33f75179a86eea156', + 'series': 'Helsingin kaupunginorkesterin konsertteja', + 'thumbnail': r're:^https?://.+\.jpg$', + 'release_date': '20230120', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, ] def _real_extract(self, url): @@ -91,12 +106,22 @@ def _real_extract(self, url): 'name': sub.get('kind'), }) + kaltura_id = traverse_obj(video_data, ('data', 'ongoing_ondemand', 'kaltura', 'id'), expected_type=str) + if kaltura_id: + info_dict = { + '_type': 'url_transparent', + 'url': smuggle_url(f'kaltura:1955031:{kaltura_id}', {'source_url': url}), + 'ie_key': KalturaIE.ie_key(), + } + else: + info_dict = { + 'id': video_id, + 'formats': self._extract_m3u8_formats( + video_data['data']['ongoing_ondemand']['manifest_url'], video_id, 'mp4', m3u8_id='hls'), + } + return { - '_type': 'url_transparent', - 'url': smuggle_url( - f'kaltura:1955031:{video_data["data"]["ongoing_ondemand"]["kaltura"]["id"]}', - {'source_url': url}), - 'ie_key': KalturaIE.ie_key(), + **info_dict, 'title': (traverse_obj(video_data, ('data', 'ongoing_ondemand', 'title', 'fin'), expected_type=str) or episode or info.get('title')), 'description': description, From 9acf1ee25f7ad3920ede574a9de95b8c18626af4 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 3 Mar 2023 16:48:54 +0530 Subject: [PATCH 0018/1178] [jsinterp] Handle `Date` at epoch 0 Closes #6400 --- test/test_youtube_signature.py | 4 ++++ yt_dlp/jsinterp.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 3203538bb8..336e80291f 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -66,6 +66,10 @@ ] _NSIG_TESTS = [ + ( + 'https://www.youtube.com/s/player/7862ca1f/player_ias.vflset/en_US/base.js', + 'X_LCxVDjAavgE5t', 'yxJ1dM6iz5ogUg', + ), ( 'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js', 'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w', diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py index c2d056aa19..31ab204d75 100644 --- a/yt_dlp/jsinterp.py +++ b/yt_dlp/jsinterp.py @@ -355,11 +355,11 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): obj = expr[4:] if obj.startswith('Date('): left, right = self._separate_at_paren(obj[4:]) - expr = unified_timestamp( + date = unified_timestamp( self.interpret_expression(left, local_vars, allow_recursion), False) - if not expr: + if date is None: raise self.Exception(f'Failed to parse date {left!r}', expr) - expr = self._dump(int(expr * 1000), local_vars) + right + expr = self._dump(int(date * 1000), local_vars) + right else: raise self.Exception(f'Unsupported object {obj}', expr) From d400e261cf029a3f20d364113b14de973be75404 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Fri, 3 Mar 2023 22:31:41 +0530 Subject: [PATCH 0019/1178] [devscripts] Script to generate changelog (#6220) Authored by: Grub4K --- README.md | 7 +- devscripts/changelog_override.json | 1 + devscripts/changelog_override.schema.json | 96 +++++ devscripts/make_changelog.py | 491 ++++++++++++++++++++++ 4 files changed, 593 insertions(+), 2 deletions(-) create mode 100644 devscripts/changelog_override.json create mode 100644 devscripts/changelog_override.schema.json create mode 100644 devscripts/make_changelog.py diff --git a/README.md b/README.md index 3d3db933ac..ddd71eeeb2 100644 --- a/README.md +++ b/README.md @@ -311,10 +311,13 @@ ### Standalone Py2Exe Builds (Windows) ### Related scripts -* **`devscripts/update-version.py [revision]`** - Update the version number based on current date -* **`devscripts/set-variant.py variant [-M update_message]`** - Set the build variant of the executable +* **`devscripts/update-version.py`** - Update the version number based on current date. +* **`devscripts/set-variant.py`** - Set the build variant of the executable. +* **`devscripts/make_changelog.py`** - Create a markdown changelog using short commit messages and update `CONTRIBUTORS` file. * **`devscripts/make_lazy_extractors.py`** - Create lazy extractors. Running this before building the binaries (any variant) will improve their startup performance. Set the environment variable `YTDLP_NO_LAZY_EXTRACTORS=1` if you wish to forcefully disable lazy extractor loading. +Note: See their `--help` for more info. + You can also fork the project on GitHub and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a full release # USAGE AND OPTIONS diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/devscripts/changelog_override.json @@ -0,0 +1 @@ +{} diff --git a/devscripts/changelog_override.schema.json b/devscripts/changelog_override.schema.json new file mode 100644 index 0000000000..9bd747b701 --- /dev/null +++ b/devscripts/changelog_override.schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft/2020-12/schema", + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "action": { + "enum": [ + "add" + ] + }, + "when": { + "type": "string", + "pattern": "^([0-9a-f]{40}|\\d{4}\\.\\d{2}\\.\\d{2})$" + }, + "hash": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "short": { + "type": "string" + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "action", + "short" + ] + }, + { + "type": "object", + "properties": { + "action": { + "enum": [ + "remove" + ] + }, + "when": { + "type": "string", + "pattern": "^([0-9a-f]{40}|\\d{4}\\.\\d{2}\\.\\d{2})$" + }, + "hash": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": [ + "action", + "hash" + ] + }, + { + "type": "object", + "properties": { + "action": { + "enum": [ + "change" + ] + }, + "when": { + "type": "string", + "pattern": "^([0-9a-f]{40}|\\d{4}\\.\\d{2}\\.\\d{2})$" + }, + "hash": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "short": { + "type": "string" + }, + "authors": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "action", + "hash", + "short", + "authors" + ] + } + ] + } +} diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py new file mode 100644 index 0000000000..b66181b53d --- /dev/null +++ b/devscripts/make_changelog.py @@ -0,0 +1,491 @@ +from __future__ import annotations + +import enum +import itertools +import json +import logging +import re +import subprocess +import sys +from collections import defaultdict +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +BASE_URL = 'https://github.com' +LOCATION_PATH = Path(__file__).parent + +logger = logging.getLogger(__name__) + + +class CommitGroup(enum.Enum): + UPSTREAM = None + PRIORITY = 'Important' + CORE = 'Core' + EXTRACTOR = 'Extractor' + DOWNLOADER = 'Downloader' + POSTPROCESSOR = 'Postprocessor' + MISC = 'Misc.' + + @classmethod + @lru_cache + def commit_lookup(cls): + return { + name: group + for group, names in { + cls.PRIORITY: {''}, + cls.UPSTREAM: {'upstream'}, + cls.CORE: { + 'aes', + 'cache', + 'compat_utils', + 'compat', + 'cookies', + 'core', + 'dependencies', + 'jsinterp', + 'outtmpl', + 'plugins', + 'update', + 'utils', + }, + cls.MISC: { + 'build', + 'cleanup', + 'devscripts', + 'docs', + 'misc', + 'test', + }, + cls.EXTRACTOR: {'extractor', 'extractors'}, + cls.DOWNLOADER: {'downloader'}, + cls.POSTPROCESSOR: {'postprocessor'}, + }.items() + for name in names + } + + @classmethod + def get(cls, value): + result = cls.commit_lookup().get(value) + if result: + logger.debug(f'Mapped {value!r} => {result.name}') + return result + + +@dataclass +class Commit: + hash: str | None + short: str + authors: list[str] + + def __str__(self): + result = f'{self.short!r}' + + if self.hash: + result += f' ({self.hash[:7]})' + + if self.authors: + authors = ', '.join(self.authors) + result += f' by {authors}' + + return result + + +@dataclass +class CommitInfo: + details: str | None + sub_details: tuple[str, ...] + message: str + issues: list[str] + commit: Commit + fixes: list[Commit] + + def key(self): + return ((self.details or '').lower(), self.sub_details, self.message) + + +class Changelog: + MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE) + + def __init__(self, groups, repo): + self._groups = groups + self._repo = repo + + def __str__(self): + return '\n'.join(self._format_groups(self._groups)).replace('\t', ' ') + + def _format_groups(self, groups): + for item in CommitGroup: + group = groups[item] + if group: + yield self.format_module(item.value, group) + + def format_module(self, name, group): + result = f'\n#### {name} changes\n' if name else '\n' + return result + '\n'.join(self._format_group(group)) + + def _format_group(self, group): + sorted_group = sorted(group, key=CommitInfo.key) + detail_groups = itertools.groupby(sorted_group, lambda item: (item.details or '').lower()) + for details, items in detail_groups: + if not details: + indent = '' + else: + yield f'- {details}' + indent = '\t' + + if details == 'cleanup': + items, cleanup_misc_items = self._filter_cleanup_misc_items(items) + + sub_detail_groups = itertools.groupby(items, lambda item: item.sub_details) + for sub_details, entries in sub_detail_groups: + if not sub_details: + for entry in entries: + yield f'{indent}- {self.format_single_change(entry)}' + continue + + prefix = f'{indent}- {", ".join(sub_details)}' + entries = list(entries) + if len(entries) == 1: + yield f'{prefix}: {self.format_single_change(entries[0])}' + continue + + yield prefix + for entry in entries: + yield f'{indent}\t- {self.format_single_change(entry)}' + + if details == 'cleanup' and cleanup_misc_items: + yield from self._format_cleanup_misc_sub_group(cleanup_misc_items) + + def _filter_cleanup_misc_items(self, items): + cleanup_misc_items = defaultdict(list) + non_misc_items = [] + for item in items: + if self.MISC_RE.search(item.message): + cleanup_misc_items[tuple(item.commit.authors)].append(item) + else: + non_misc_items.append(item) + + return non_misc_items, cleanup_misc_items + + def _format_cleanup_misc_sub_group(self, group): + prefix = '\t- Miscellaneous' + if len(group) == 1: + yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}' + return + + yield prefix + for message in self._format_cleanup_misc_items(group): + yield f'\t\t- {message}' + + def _format_cleanup_misc_items(self, group): + for authors, infos in group.items(): + message = ', '.join( + self._format_message_link(None, info.commit.hash) + for info in sorted(infos, key=lambda item: item.commit.hash or '')) + yield f'{message} by {self._format_authors(authors)}' + + def format_single_change(self, info): + message = self._format_message_link(info.message, info.commit.hash) + if info.issues: + message = f'{message} ({self._format_issues(info.issues)})' + + if info.commit.authors: + message = f'{message} by {self._format_authors(info.commit.authors)}' + + if info.fixes: + fix_message = ', '.join(f'{self._format_message_link(None, fix.hash)}' for fix in info.fixes) + + authors = sorted({author for fix in info.fixes for author in fix.authors}, key=str.casefold) + if authors != info.commit.authors: + fix_message = f'{fix_message} by {self._format_authors(authors)}' + + message = f'{message} (With fixes in {fix_message})' + + return message + + def _format_message_link(self, message, hash): + assert message or hash, 'Improperly defined commit message or override' + message = message if message else hash[:7] + return f'[{message}]({self.repo_url}/commit/{hash})' if hash else message + + def _format_issues(self, issues): + return ', '.join(f'[#{issue}]({self.repo_url}/issues/{issue})' for issue in issues) + + @staticmethod + def _format_authors(authors): + return ', '.join(f'[{author}]({BASE_URL}/{author})' for author in authors) + + @property + def repo_url(self): + return f'{BASE_URL}/{self._repo}' + + +class CommitRange: + COMMAND = 'git' + COMMIT_SEPARATOR = '-----' + + AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE) + MESSAGE_RE = re.compile(r''' + (?:\[ + (?P[^\]\/:,]+) + (?:/(?P
[^\]:,]+))? + (?:[:,](?P[^\]]+))? + \]\ )? + (?:`?(?P[^:`]+)`?: )? + (?P.+?) + (?:\ \((?P\#\d+(?:,\ \#\d+)*)\))? + ''', re.VERBOSE | re.DOTALL) + EXTRACTOR_INDICATOR_RE = re.compile(r'(?:Fix|Add)\s+Extractors?', re.IGNORECASE) + FIXES_RE = re.compile(r'(?i:Fix(?:es)?(?:\s+for)?|Revert)\s+([\da-f]{40})') + UPSTREAM_MERGE_RE = re.compile(r'Update to ytdl-commit-([\da-f]+)') + + def __init__(self, start, end, default_author=None) -> None: + self._start = start + self._end = end + self._commits, self._fixes = self._get_commits_and_fixes(default_author) + self._commits_added = [] + + @classmethod + def from_single(cls, commitish='HEAD', default_author=None): + start_commitish = cls.get_prev_tag(commitish) + end_commitish = cls.get_next_tag(commitish) + if start_commitish == end_commitish: + start_commitish = cls.get_prev_tag(f'{commitish}~') + logger.info(f'Determined range from {commitish!r}: {start_commitish}..{end_commitish}') + return cls(start_commitish, end_commitish, default_author) + + @classmethod + def get_prev_tag(cls, commitish): + command = [cls.COMMAND, 'describe', '--tags', '--abbrev=0', '--exclude=*[^0-9.]*', commitish] + return subprocess.check_output(command, text=True).strip() + + @classmethod + def get_next_tag(cls, commitish): + result = subprocess.run( + [cls.COMMAND, 'describe', '--contains', '--abbrev=0', commitish], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + if result.returncode: + return 'HEAD' + + return result.stdout.partition('~')[0].strip() + + def __iter__(self): + return iter(itertools.chain(self._commits.values(), self._commits_added)) + + def __len__(self): + return len(self._commits) + len(self._commits_added) + + def __contains__(self, commit): + if isinstance(commit, Commit): + if not commit.hash: + return False + commit = commit.hash + + return commit in self._commits + + def _is_ancestor(self, commitish): + return bool(subprocess.call( + [self.COMMAND, 'merge-base', '--is-ancestor', commitish, self._start])) + + def _get_commits_and_fixes(self, default_author): + result = subprocess.check_output([ + self.COMMAND, 'log', f'--format=%H%n%s%n%b%n{self.COMMIT_SEPARATOR}', + f'{self._start}..{self._end}'], text=True) + + commits = {} + fixes = defaultdict(list) + lines = iter(result.splitlines(False)) + for line in lines: + commit_hash = line + short = next(lines) + skip = short.startswith('Release ') or short == '[version] update' + + authors = [default_author] if default_author else [] + for line in iter(lambda: next(lines), self.COMMIT_SEPARATOR): + match = self.AUTHOR_INDICATOR_RE.match(line) + if match: + authors = sorted(map(str.strip, line[match.end():].split(',')), key=str.casefold) + + commit = Commit(commit_hash, short, authors) + if skip: + logger.debug(f'Skipped commit: {commit}') + continue + + fix_match = self.FIXES_RE.search(commit.short) + if fix_match: + commitish = fix_match.group(1) + fixes[commitish].append(commit) + + commits[commit.hash] = commit + + for commitish, fix_commits in fixes.items(): + if commitish in commits: + hashes = ', '.join(commit.hash[:7] for commit in fix_commits) + logger.info(f'Found fix(es) for {commitish[:7]}: {hashes}') + for fix_commit in fix_commits: + del commits[fix_commit.hash] + else: + logger.debug(f'Commit with fixes not in changes: {commitish[:7]}') + + return commits, fixes + + def apply_overrides(self, overrides): + for override in overrides: + when = override.get('when') + if when and when not in self and when != self._start: + logger.debug(f'Ignored {when!r}, not in commits {self._start!r}') + continue + + override_hash = override.get('hash') + if override['action'] == 'add': + commit = Commit(override.get('hash'), override['short'], override.get('authors') or []) + logger.info(f'ADD {commit}') + self._commits_added.append(commit) + + elif override['action'] == 'remove': + if override_hash in self._commits: + logger.info(f'REMOVE {self._commits[override_hash]}') + del self._commits[override_hash] + + elif override['action'] == 'change': + if override_hash not in self._commits: + continue + commit = Commit(override_hash, override['short'], override['authors']) + logger.info(f'CHANGE {self._commits[commit.hash]} -> {commit}') + self._commits[commit.hash] = commit + + self._commits = {key: value for key, value in reversed(self._commits.items())} + + def groups(self): + groups = defaultdict(list) + for commit in self: + upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short) + if upstream_re: + commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}' + + match = self.MESSAGE_RE.fullmatch(commit.short) + if not match: + logger.error(f'Error parsing short commit message: {commit.short!r}') + continue + + prefix, details, sub_details, sub_details_alt, message, issues = match.groups() + group = None + if prefix: + if prefix == 'priority': + prefix, _, details = (details or '').partition('/') + logger.debug(f'Priority: {message!r}') + group = CommitGroup.PRIORITY + + if not details and prefix: + if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'): + logger.debug(f'Replaced details with {prefix!r}') + details = prefix or None + + if details == 'common': + details = None + + if details: + details = details.strip() + + else: + group = CommitGroup.CORE + + sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.lower().replace(':', ',') + sub_details = tuple(filter(None, map(str.strip, sub_details.split(',')))) + + issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else [] + + if not group: + group = CommitGroup.get(prefix.lower()) + if not group: + if self.EXTRACTOR_INDICATOR_RE.search(commit.short): + group = CommitGroup.EXTRACTOR + else: + group = CommitGroup.POSTPROCESSOR + logger.warning(f'Failed to map {commit.short!r}, selected {group.name}') + + commit_info = CommitInfo( + details, sub_details, message.strip(), + issues, commit, self._fixes[commit.hash]) + logger.debug(f'Resolved {commit.short!r} to {commit_info!r}') + groups[group].append(commit_info) + + return groups + + +def get_new_contributors(contributors_path, commits): + contributors = set() + if contributors_path.exists(): + with contributors_path.open() as file: + for line in filter(None, map(str.strip, file)): + author, _, _ = line.partition(' (') + authors = author.split('/') + contributors.update(map(str.casefold, authors)) + + new_contributors = set() + for commit in commits: + for author in commit.authors: + author_folded = author.casefold() + if author_folded not in contributors: + contributors.add(author_folded) + new_contributors.add(author) + + return sorted(new_contributors, key=str.casefold) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser( + description='Create a changelog markdown from a git commit range') + parser.add_argument( + 'commitish', default='HEAD', nargs='?', + help='The commitish to create the range from (default: %(default)s)') + parser.add_argument( + '-v', '--verbosity', action='count', default=0, + help='increase verbosity (can be used twice)') + parser.add_argument( + '-c', '--contributors', action='store_true', + help='update CONTRIBUTORS file (default: %(default)s)') + parser.add_argument( + '--contributors-path', type=Path, default=LOCATION_PATH.parent / 'CONTRIBUTORS', + help='path to the CONTRIBUTORS file') + parser.add_argument( + '--no-override', action='store_true', + help='skip override json in commit generation (default: %(default)s)') + parser.add_argument( + '--override-path', type=Path, default=LOCATION_PATH / 'changelog_override.json', + help='path to the changelog_override.json file') + parser.add_argument( + '--default-author', default='pukkandan', + help='the author to use without a author indicator (default: %(default)s)') + parser.add_argument( + '--repo', default='yt-dlp/yt-dlp', + help='the github repository to use for the operations (default: %(default)s)') + args = parser.parse_args() + + logging.basicConfig( + datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}', + level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr) + + commits = CommitRange.from_single(args.commitish, args.default_author) + + if not args.no_override: + if args.override_path.exists(): + with args.override_path.open() as file: + overrides = json.load(file) + commits.apply_overrides(overrides) + else: + logger.warning(f'File {args.override_path.as_posix()} does not exist') + + logger.info(f'Loaded {len(commits)} commits') + + new_contributors = get_new_contributors(args.contributors_path, commits) + if new_contributors: + if args.contributors: + with args.contributors_path.open('a') as file: + file.writelines(f'{contributor}\n' for contributor in new_contributors) + logger.info(f'New contributors: {", ".join(new_contributors)}') + + print(Changelog(commits.groups(), args.repo)) From 29cb20bd563c02671b31dd840139e93dd37150a1 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Fri, 3 Mar 2023 22:33:12 +0530 Subject: [PATCH 0020/1178] [build] Automated builds and nightly releases (#6220) Closes #1839 Authored by: Grub4K, bashonly Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com> --- .github/workflows/build.yml | 541 ++++++++++++-------------- .github/workflows/publish.yml | 80 ++++ .github/workflows/release-nightly.yml | 49 +++ .github/workflows/release.yml | 125 ++++++ Changelog.md | 8 +- README.md | 8 +- devscripts/make_readme.py | 22 +- devscripts/update-version.py | 46 ++- yt_dlp/YoutubeDL.py | 6 +- 9 files changed, 552 insertions(+), 333 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/release-nightly.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6041376a4d..2183903ea4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,393 +1,338 @@ -name: Build -on: workflow_dispatch +name: Build Artifacts +on: + workflow_call: + inputs: + version: + required: true + type: string + channel: + required: false + default: stable + type: string + unix: + default: true + type: boolean + linux_arm: + default: true + type: boolean + macos: + default: true + type: boolean + macos_legacy: + default: true + type: boolean + windows: + default: true + type: boolean + windows32: + default: true + type: boolean + meta_files: + default: true + type: boolean + + workflow_dispatch: + inputs: + version: + description: Version tag (YYYY.MM.DD[.REV]) + required: true + type: string + channel: + description: Update channel (stable/nightly) + required: true + default: stable + type: string + unix: + description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip + default: true + type: boolean + linux_arm: + description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l + default: true + type: boolean + macos: + description: yt-dlp_macos, yt-dlp_macos.zip + default: true + type: boolean + macos_legacy: + description: yt-dlp_macos_legacy + default: true + type: boolean + windows: + description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip + default: true + type: boolean + windows32: + description: yt-dlp_x86.exe + default: true + type: boolean + meta_files: + description: SHA2-256SUMS, SHA2-512SUMS, _update_spec + default: true + type: boolean + permissions: contents: read jobs: - prepare: - permissions: - contents: write # for push_release + unix: + if: inputs.unix runs-on: ubuntu-latest - outputs: - version_suffix: ${{ steps.version_suffix.outputs.version_suffix }} - ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }} - head_sha: ${{ steps.push_release.outputs.head_sha }} steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Set version suffix - id: version_suffix - env: - PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }} - if: "env.PUSH_VERSION_COMMIT == ''" - run: echo "version_suffix=$(date -u +"%H%M%S")" >> "$GITHUB_OUTPUT" - - name: Bump version - id: bump_version - run: | - python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }} - make issuetemplates - - - name: Push to release - id: push_release - run: | - git config --global user.name github-actions - git config --global user.email github-actions@example.com - git add -u - git commit -m "[version] update" -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl" - git push origin --force ${{ github.event.ref }}:release - echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Update master - env: - PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }} - if: "env.PUSH_VERSION_COMMIT != ''" - run: git push origin ${{ github.event.ref }} - - - build_unix: - needs: prepare - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - uses: conda-incubator/setup-miniconda@v2 - with: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: conda-incubator/setup-miniconda@v2 + with: miniforge-variant: Mambaforge use-mamba: true channels: conda-forge auto-update-conda: true - activate-environment: '' + activate-environment: "" auto-activate-base: false - - name: Install Requirements - run: | + - name: Install Requirements + run: | sudo apt-get -y install zip pandoc man sed - python -m pip install -U pip setuptools wheel twine + python -m pip install -U pip setuptools wheel python -m pip install -U Pyinstaller -r requirements.txt reqs=$(mktemp) echo -e 'python=3.10.*\npyinstaller' >$reqs sed 's/^brotli.*/brotli-python/' >$reqs mamba create -n build --file $reqs - - name: Prepare - run: | - python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} + - name: Prepare + run: | + python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} python devscripts/make_lazy_extractors.py - - name: Build Unix platform-independent binary - run: | + - name: Build Unix platform-independent binary + run: | make all tar - - name: Build Unix standalone binary - shell: bash -l {0} - run: | + - name: Build Unix standalone binary + shell: bash -l {0} + run: | unset LD_LIBRARY_PATH # Harmful; set by setup-python conda activate build python pyinst.py --onedir (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .) python pyinst.py + mv ./dist/yt-dlp_linux ./yt-dlp_linux + mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - path: | - yt-dlp - yt-dlp.tar.gz - dist/yt-dlp_linux - dist/yt-dlp_linux.zip + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + yt-dlp + yt-dlp.tar.gz + yt-dlp_linux + yt-dlp_linux.zip - - name: Build and publish on PyPi - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - if: "env.TWINE_PASSWORD != ''" - run: | - rm -rf dist/* - python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update" - python setup.py sdist bdist_wheel - twine upload dist/* - - - name: Install SSH private key for Homebrew - env: - BREW_TOKEN: ${{ secrets.BREW_TOKEN }} - if: "env.BREW_TOKEN != ''" - uses: yt-dlp/ssh-agent@v0.5.3 - with: - ssh-private-key: ${{ env.BREW_TOKEN }} - - name: Update Homebrew Formulae - env: - BREW_TOKEN: ${{ secrets.BREW_TOKEN }} - if: "env.BREW_TOKEN != ''" - run: | - git clone git@github.com:yt-dlp/homebrew-taps taps/ - python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.ytdlp_version }}" - git -C taps/ config user.name github-actions - git -C taps/ config user.email github-actions@example.com - git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.ytdlp_version }}' - git -C taps/ push - - - build_linux_arm: + linux_arm: + if: inputs.linux_arm permissions: - packages: write # for Creating cache + contents: read + packages: write # for creating cache runs-on: ubuntu-latest - needs: prepare strategy: matrix: architecture: - - armv7 - - aarch64 + - armv7 + - aarch64 steps: - - uses: actions/checkout@v3 - with: - path: ./repo - - name: Virtualized Install, Prepare & Build - uses: yt-dlp/run-on-arch-action@v2 - with: - githubToken: ${{ github.token }} # To cache image - arch: ${{ matrix.architecture }} - distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS - dockerRunArgs: --volume "${PWD}/repo:/repo" - install: | # Installing Python 3.10 from the Deadsnakes repo raises errors - apt update - apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip - python3.8 -m pip install -U pip setuptools wheel - # Cannot access requirements.txt from the repo directory at this stage - python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi + - uses: actions/checkout@v3 + with: + path: ./repo + - name: Virtualized Install, Prepare & Build + uses: yt-dlp/run-on-arch-action@v2 + with: + # Ref: https://github.com/uraimo/run-on-arch-action/issues/55 + env: | + GITHUB_WORKFLOW: build + githubToken: ${{ github.token }} # To cache image + arch: ${{ matrix.architecture }} + distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS + dockerRunArgs: --volume "${PWD}/repo:/repo" + install: | # Installing Python 3.10 from the Deadsnakes repo raises errors + apt update + apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip + python3.8 -m pip install -U pip setuptools wheel + # Cannot access requirements.txt from the repo directory at this stage + python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi - run: | - cd repo - python3.8 -m pip install -U Pyinstaller -r requirements.txt # Cached version may be out of date - python3.8 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} - python3.8 devscripts/make_lazy_extractors.py - python3.8 pyinst.py + run: | + cd repo + python3.8 -m pip install -U Pyinstaller -r requirements.txt # Cached version may be out of date + python3.8 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} + python3.8 devscripts/make_lazy_extractors.py + python3.8 pyinst.py - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - path: | # run-on-arch-action designates armv7l as armv7 - repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | # run-on-arch-action designates armv7l as armv7 + repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} - - build_macos: + macos: + if: inputs.macos runs-on: macos-11 - needs: prepare steps: - - uses: actions/checkout@v3 - # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used - - name: Install Requirements - run: | + - uses: actions/checkout@v3 + # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used + - name: Install Requirements + run: | brew install coreutils /usr/bin/python3 -m pip install -U --user pip Pyinstaller -r requirements.txt - - name: Prepare - run: | - /usr/bin/python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} + - name: Prepare + run: | + /usr/bin/python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} /usr/bin/python3 devscripts/make_lazy_extractors.py - - name: Build - run: | + - name: Build + run: | /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .) /usr/bin/python3 pyinst.py --target-architecture universal2 - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - path: | - dist/yt-dlp_macos - dist/yt-dlp_macos.zip + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_macos + dist/yt-dlp_macos.zip - - build_macos_legacy: + macos_legacy: + if: inputs.macos_legacy runs-on: macos-latest - needs: prepare steps: - - uses: actions/checkout@v3 - - name: Install Python - # We need the official Python, because the GA ones only support newer macOS versions - env: - PYTHON_VERSION: 3.10.5 - MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools - run: | + - uses: actions/checkout@v3 + - name: Install Python + # We need the official Python, because the GA ones only support newer macOS versions + env: + PYTHON_VERSION: 3.10.5 + MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools + run: | # Hack to get the latest patch version. Uncomment if needed #brew install python@3.10 #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 ) curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg" sudo installer -pkg python.pkg -target / python3 --version - - name: Install Requirements - run: | + - name: Install Requirements + run: | brew install coreutils python3 -m pip install -U --user pip Pyinstaller -r requirements.txt - - name: Prepare - run: | - python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} + - name: Prepare + run: | + python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} python3 devscripts/make_lazy_extractors.py - - name: Build - run: | + - name: Build + run: | python3 pyinst.py mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - path: | - dist/yt-dlp_macos_legacy + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_macos_legacy - - build_windows: + windows: + if: inputs.windows runs-on: windows-latest - needs: prepare steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: # 3.8 is used for Win7 support - python-version: '3.8' - - name: Install Requirements - run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: # 3.8 is used for Win7 support + python-version: "3.8" + - name: Install Requirements + run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds python -m pip install -U pip setuptools wheel py2exe pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt - - name: Prepare - run: | - python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} + - name: Prepare + run: | + python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} python devscripts/make_lazy_extractors.py - - name: Build - run: | + - name: Build + run: | python setup.py py2exe Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe python pyinst.py python pyinst.py --onedir Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - path: | - dist/yt-dlp.exe - dist/yt-dlp_min.exe - dist/yt-dlp_win.zip + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp.exe + dist/yt-dlp_min.exe + dist/yt-dlp_win.zip - - build_windows32: + windows32: + if: inputs.windows32 runs-on: windows-latest - needs: prepare steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390 - python-version: '3.7' - architecture: 'x86' - - name: Install Requirements - run: | + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390 + python-version: "3.7" + architecture: "x86" + - name: Install Requirements + run: | python -m pip install -U pip setuptools wheel pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt - - name: Prepare - run: | - python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }} + - name: Prepare + run: | + python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} python devscripts/make_lazy_extractors.py - - name: Build - run: | + - name: Build + run: | python pyinst.py - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - path: | - dist/yt-dlp_x86.exe + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_x86.exe - - publish_release: - permissions: - contents: write # for action-gh-release + meta_files: + if: inputs.meta_files && always() + needs: + - unix + - linux_arm + - macos + - macos_legacy + - windows + - windows32 runs-on: ubuntu-latest - needs: [prepare, build_unix, build_linux_arm, build_windows, build_windows32, build_macos, build_macos_legacy] - steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v3 - - name: Get Changelog - run: | - changelog=$(grep -oPz '(?s)(?<=### ${{ needs.prepare.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)' Changelog.md) || true - echo "changelog<> $GITHUB_ENV - echo "$changelog" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - name: Make Update spec - run: | - echo "# This file is used for regulating self-update" >> _update_spec - echo "lock 2022.07.18 .+ Python 3.6" >> _update_spec - - name: Make SHA2-SUMS files - run: | - sha256sum artifact/yt-dlp | awk '{print $1 " yt-dlp"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp.tar.gz | awk '{print $1 " yt-dlp.tar.gz"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp.exe | awk '{print $1 " yt-dlp.exe"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_win.zip | awk '{print $1 " yt-dlp_win.zip"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_min.exe | awk '{print $1 " yt-dlp_min.exe"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_x86.exe | awk '{print $1 " yt-dlp_x86.exe"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_macos | awk '{print $1 " yt-dlp_macos"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_macos.zip | awk '{print $1 " yt-dlp_macos.zip"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_macos_legacy | awk '{print $1 " yt-dlp_macos_legacy"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_linux_armv7l | awk '{print $1 " yt-dlp_linux_armv7l"}' >> SHA2-256SUMS - sha256sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 " yt-dlp_linux_aarch64"}' >> SHA2-256SUMS - sha256sum artifact/dist/yt-dlp_linux | awk '{print $1 " yt-dlp_linux"}' >> SHA2-256SUMS - sha256sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 " yt-dlp_linux.zip"}' >> SHA2-256SUMS - sha512sum artifact/yt-dlp | awk '{print $1 " yt-dlp"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp.tar.gz | awk '{print $1 " yt-dlp.tar.gz"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp.exe | awk '{print $1 " yt-dlp.exe"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_win.zip | awk '{print $1 " yt-dlp_win.zip"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_min.exe | awk '{print $1 " yt-dlp_min.exe"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_x86.exe | awk '{print $1 " yt-dlp_x86.exe"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_macos | awk '{print $1 " yt-dlp_macos"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_macos.zip | awk '{print $1 " yt-dlp_macos.zip"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_macos_legacy | awk '{print $1 " yt-dlp_macos_legacy"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_linux_armv7l | awk '{print $1 " yt-dlp_linux_armv7l"}' >> SHA2-512SUMS - sha512sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 " yt-dlp_linux_aarch64"}' >> SHA2-512SUMS - sha512sum artifact/dist/yt-dlp_linux | awk '{print $1 " yt-dlp_linux"}' >> SHA2-512SUMS - sha512sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 " yt-dlp_linux.zip"}' >> SHA2-512SUMS + - name: Make SHA2-SUMS files + run: | + cd ./artifact/ + sha256sum * > ../SHA2-256SUMS + sha512sum * > ../SHA2-512SUMS - - name: Publish Release - uses: yt-dlp/action-gh-release@v1 - with: - tag_name: ${{ needs.prepare.outputs.ytdlp_version }} - name: yt-dlp ${{ needs.prepare.outputs.ytdlp_version }} - target_commitish: ${{ needs.prepare.outputs.head_sha }} - body: | - #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README + - name: Make Update spec + run: | + cat >> _update_spec << EOF + # This file is used for regulating self-update + lock 2022.08.18.36 .+ Python 3.6 + EOF - --- -

Changelog

 -

- - ${{ env.changelog }} - -

-
- files: | - SHA2-256SUMS - SHA2-512SUMS - artifact/yt-dlp - artifact/yt-dlp.tar.gz - artifact/yt-dlp.exe - artifact/yt-dlp_win.zip - artifact/yt-dlp_min.exe - artifact/yt-dlp_x86.exe - artifact/yt-dlp_macos - artifact/yt-dlp_macos.zip - artifact/yt-dlp_macos_legacy - artifact/yt-dlp_linux_armv7l - artifact/yt-dlp_linux_aarch64 - artifact/dist/yt-dlp_linux - artifact/dist/yt-dlp_linux.zip - _update_spec + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + SHA*SUMS* + _update_spec diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..42e66a29cb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,80 @@ +name: Publish +on: + workflow_call: + inputs: + nightly: + default: false + required: false + type: boolean + version: + required: true + type: string + target_commitish: + required: true + type: string + secrets: + ARCHIVE_REPO_TOKEN: + required: false + +permissions: + contents: write + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Generate release notes + run: | + cat >> ./RELEASE_NOTES << EOF + #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files) + --- +

Changelog

 + $(python ./devscripts/make_changelog.py -vv) +
+ EOF + echo "**This is an automated nightly pre-release build**" >> ./PRERELEASE_NOTES + cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES + echo "Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}" >> ./ARCHIVE_NOTES + cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES + + - name: Archive nightly release + env: + GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }} + GH_REPO: ${{ vars.ARCHIVE_REPO }} + if: | + inputs.nightly && env.GH_TOKEN != '' && env.GH_REPO != '' + run: | + gh release create \ + --notes-file ARCHIVE_NOTES \ + --title "Build ${{ inputs.version }}" \ + ${{ inputs.version }} \ + artifact/* + + - name: Prune old nightly release + if: inputs.nightly + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release delete --yes --cleanup-tag "nightly" || true + git tag --delete "nightly" || true + sleep 5 # Enough time to cover deletion race condition + + - name: Publish release${{ inputs.nightly && ' (nightly)' || '' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create \ + --notes-file ${{ inputs.nightly && 'PRE' || '' }}RELEASE_NOTES \ + --target ${{ inputs.target_commitish }} \ + --title "yt-dlp ${{ inputs.nightly && 'nightly ' || '' }}${{ inputs.version }}" \ + ${{ inputs.nightly && '--prerelease "nightly"' || inputs.version }} \ + artifact/* diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml new file mode 100644 index 0000000000..ec079b8d05 --- /dev/null +++ b/.github/workflows/release-nightly.yml @@ -0,0 +1,49 @@ +name: Release (nightly) +on: + push: + branches: + - master + paths: + - "**.py" + - "!yt_dlp/version.py" +concurrency: + group: release-nightly + cancel-in-progress: true +permissions: + contents: read + +jobs: + prepare: + if: vars.BUILD_NIGHTLY != '' + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get_version.outputs.version }} + + steps: + - uses: actions/checkout@v3 + - name: Get version + id: get_version + run: | + python devscripts/update-version.py "$(date -u +"%H%M%S")" | grep -Po "version=\d+(\.\d+){3}" >> "$GITHUB_OUTPUT" + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + channel: nightly + permissions: + contents: read + packages: write # For package cache + + publish: + needs: [prepare, build] + uses: ./.github/workflows/publish.yml + secrets: + ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }} + permissions: + contents: write + with: + nightly: true + version: ${{ needs.prepare.outputs.version }} + target_commitish: ${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..c97cd1f4a8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,125 @@ +name: Release +on: workflow_dispatch +permissions: + contents: read + +jobs: + prepare: + permissions: + contents: write + runs-on: ubuntu-latest + outputs: + version: ${{ steps.update_version.outputs.version }} + head_sha: ${{ steps.push_release.outputs.head_sha }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Update version + id: update_version + run: | + python devscripts/update-version.py ${{ vars.PUSH_VERSION_COMMIT == '' && '"$(date -u +"%H%M%S")"' || '' }} | \ + grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT" + + - name: Update documentation + run: | + make doc + sed '/### /Q' Changelog.md >> ./CHANGELOG + echo '### ${{ steps.update_version.outputs.version }}' >> ./CHANGELOG + python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG + echo >> ./CHANGELOG + grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG + cat ./CHANGELOG > Changelog.md + + - name: Push to release + id: push_release + run: | + git config --global user.name github-actions + git config --global user.email github-actions@example.com + git add -u + git commit -m "Release ${{ steps.update_version.outputs.version }}" \ + -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl" + git push origin --force ${{ github.event.ref }}:release + echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Update master + if: vars.PUSH_VERSION_COMMIT != '' + run: git push origin ${{ github.event.ref }} + + publish_pypi_homebrew: + needs: prepare + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Requirements + run: | + python -m pip install -U pip setuptools wheel twine + python -m pip install -U -r requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py ${{ needs.prepare.outputs.version }} + python devscripts/make_lazy_extractors.py + + - name: Build and publish on PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + if: env.TWINE_PASSWORD != '' + run: | + rm -rf dist/* + python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update" + python setup.py sdist bdist_wheel + twine upload dist/* + + - name: Checkout Homebrew repository + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' + uses: actions/checkout@v3 + with: + repository: yt-dlp/homebrew-taps + path: taps + ssh-key: ${{ secrets.BREW_TOKEN }} + + - name: Update Homebrew Formulae + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' + run: | + python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}" + git -C taps/ config user.name github-actions + git -C taps/ config user.email github-actions@example.com + git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}' + git -C taps/ push + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + permissions: + contents: read + packages: write # For package cache + + publish: + needs: [prepare, build] + uses: ./.github/workflows/publish.yml + permissions: + contents: write + with: + version: ${{ needs.prepare.outputs.version }} + target_commitish: ${{ needs.prepare.outputs.head_sha }} diff --git a/Changelog.md b/Changelog.md index 24bc8a2e27..60bd99f722 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,13 +1,7 @@ # Changelog ### 2023.02.17 diff --git a/README.md b/README.md index ddd71eeeb2..e6e95b1472 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,8 @@ ### Related scripts Note: See their `--help` for more info. -You can also fork the project on GitHub and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a full release +### Forking the project +If you fork the project on GitHub, you can run your fork's [build workflow](.github/workflows/build.yml) to automatically build the selected version(s) as artifacts. Alternatively, you can run the [release workflow](.github/workflows/release.yml) or enable the [nightly workflow](.github/workflows/release-nightly.yml) to create full (pre-)releases. # USAGE AND OPTIONS @@ -460,9 +461,8 @@ ## Video Selection: --date DATE Download only videos uploaded on this date. The date can be "YYYYMMDD" or in the format [now|today|yesterday][-N[day|week|month|year]]. - E.g. "--date today-2weeks" downloads - only videos uploaded on the same day two - weeks ago + E.g. "--date today-2weeks" downloads only + videos uploaded on the same day two weeks ago --datebefore DATE Download only videos uploaded on or before this date. The date formats accepted is the same as --date diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index fad993a199..2270b31d3b 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -45,33 +45,43 @@ def apply_patch(text, patch): delim = f'\n{" " * switch_col_width}' PATCHES = ( - ( # Standardize update message + ( # Standardize `--update` message r'(?m)^( -U, --update\s+).+(\n \s.+)*$', r'\1Update this program to the latest version', ), - ( # Headings + ( # Headings r'(?m)^ (\w.+\n)( (?=\w))?', r'## \1' ), - ( # Do not split URLs + ( # Fixup `--date` formatting + rf'(?m)( --date DATE.+({delim}[^\[]+)*)\[.+({delim}.+)*$', + (rf'\1[now|today|yesterday][-N[day|week|month|year]].{delim}' + f'E.g. "--date today-2weeks" downloads only{delim}' + 'videos uploaded on the same day two weeks ago'), + ), + ( # Do not split URLs rf'({delim[:-1]})? (?P