[ie/10play] Rework extractor (#14417)

Closes #14276
Authored by: seproDev, Sipherdrakon

Co-authored-by: Sipherdrakon <64430430+Sipherdrakon@users.noreply.github.com>

yt_dlp/__init__.py

@@ -155,7 +155,7 @@ def set_default_compat(compat_name, opt_name, default=True, remove_compat=True):
     if 'format-sort' in opts.compat_opts:
         opts.format_sort.extend(FormatSorter.ytdl_default)
     elif 'prefer-vp9-sort' in opts.compat_opts:
-        opts.format_sort.extend(FormatSorter._prefer_vp9_sort)
+        FormatSorter.default = FormatSorter._prefer_vp9_sort
 
     if 'mtime-by-default' in opts.compat_opts:
         if opts.updatetime is None:

yt_dlp/extractor/_extractors.py

@@ -337,6 +337,7 @@
     CBCGemIE,
     CBCGemLiveIE,
     CBCGemPlaylistIE,
+    CBCListenIE,
     CBCPlayerIE,
     CBCPlayerPlaylistIE,
 )

yt_dlp/extractor/cbc.py

@@ -31,7 +31,7 @@
 
 class CBCIE(InfoExtractor):
     IE_NAME = 'cbc.ca'
-    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/)(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/|listen/|i/caffeine/syndicate/)(?:[^/?#]+/)+(?P<id>[^/?#]+)'
     _TESTS = [{
         # with mediaId
         'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
@@ -112,10 +112,6 @@ class CBCIE(InfoExtractor):
         'playlist_mincount': 6,
     }]
 
-    @classmethod
-    def suitable(cls, url):
-        return False if CBCPlayerIE.suitable(url) else super().suitable(url)
-
     def _extract_player_init(self, player_init, display_id):
         player_info = self._parse_json(player_init, display_id, js_to_json)
         media_id = player_info.get('mediaId')
@@ -913,3 +909,63 @@ def _real_extract(self, url):
                 'thumbnail': ('images', 'card', 'url'),
             }),
         }
+
+
+class CBCListenIE(InfoExtractor):
+    IE_NAME = 'cbc.ca:listen'
+    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/listen/(?:cbc-podcasts|live-radio)/[\w-]+/[\w-]+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.cbc.ca/listen/cbc-podcasts/1353-the-naked-emperor/episode/16142603-introducing-understood-who-broke-the-internet',
+        'info_dict': {
+            'id': '16142603',
+            'title': 'Introducing Understood: Who Broke the Internet?',
+            'ext': 'mp3',
+            'description': 'md5:c605117500084e43f08a950adc6a708c',
+            'duration': 229,
+            'timestamp': 1745812800,
+            'release_timestamp': 1745827200,
+            'release_date': '20250428',
+            'upload_date': '20250428',
+        },
+    }, {
+        'url': 'https://www.cbc.ca/listen/live-radio/1-64-the-house/clip/16170773-should-canada-suck-stand-donald-trump',
+        'info_dict': {
+            'id': '16170773',
+            'title': 'Should Canada suck up or stand up to Donald Trump?',
+            'ext': 'mp3',
+            'description': 'md5:7385194f1cdda8df27ba3764b35e7976',
+            'duration': 3159,
+            'timestamp': 1758340800,
+            'release_timestamp': 1758254400,
+            'release_date': '20250919',
+            'upload_date': '20250920',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        response = self._download_json(
+            f'https://www.cbc.ca/listen/api/v1/clips/{video_id}', video_id, fatal=False)
+        data = traverse_obj(response, ('data', {dict}))
+        if not data:
+            self.report_warning('API failed to return data. Falling back to webpage parsing')
+            webpage = self._download_webpage(url, video_id)
+            preloaded_state = self._search_json(
+                r'window\.__PRELOADED_STATE__\s*=', webpage, 'preloaded state',
+                video_id, transform_source=js_to_json)
+            data = traverse_obj(preloaded_state, (
+                ('podcastDetailData', 'showDetailData'), ..., 'episodes',
+                lambda _, v: str(v['clipID']) == video_id, any, {require('episode data')}))
+
+        return {
+            'id': video_id,
+            **traverse_obj(data, {
+                'url': (('src', 'url'), {url_or_none}, any),
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'release_timestamp': ('releasedAt', {int_or_none(scale=1000)}),
+                'timestamp': ('airdate', {int_or_none(scale=1000)}),
+                'duration': ('duration', {int_or_none}),
+            }),
+        }

yt_dlp/extractor/dropout.py

@@ -18,15 +18,15 @@
 
 
 class DropoutIE(InfoExtractor):
-    _LOGIN_URL = 'https://www.dropout.tv/login'
+    _LOGIN_URL = 'https://watch.dropout.tv/login'
     _NETRC_MACHINE = 'dropout'
 
-    _VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?:[^/]+/)*videos/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:watch\.)?dropout\.tv/(?:[^/?#]+/)*videos/(?P<id>[^/?#]+)/?(?:[?#]|$)'
     _TESTS = [
         {
-            'url': 'https://www.dropout.tv/game-changer/season:2/videos/yes-or-no',
+            'url': 'https://watch.dropout.tv/game-changer/season:2/videos/yes-or-no',
             'note': 'Episode in a series',
-            'md5': '5e000fdfd8d8fa46ff40456f1c2af04a',
+            'md5': '4b76963f904f8bc4ba22dcf0e66ada06',
             'info_dict': {
                 'id': '738153',
                 'display_id': 'yes-or-no',
@@ -45,35 +45,35 @@ class DropoutIE(InfoExtractor):
                 'uploader_url': 'https://vimeo.com/user80538407',
                 'uploader': 'OTT Videos',
             },
-            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest', 'Failed to parse XML: not well-formed'],
         },
         {
-            'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1/videos/episode-1',
+            'url': 'https://watch.dropout.tv/tablepop-presents-megadungeon-live/season:1/videos/enter-through-the-gift-shop',
             'note': 'Episode in a series (missing release_date)',
-            'md5': '712caf7c191f1c47c8f1879520c2fa5c',
+            'md5': 'b08fb03050585ea25cd7ee092db9134c',
             'info_dict': {
-                'id': '320562',
-                'display_id': 'episode-1',
+                'id': '624270',
+                'display_id': 'enter-through-the-gift-shop',
                 'ext': 'mp4',
-                'title': 'The Beginning Begins',
-                'description': 'The cast introduces their PCs, including a neurotic elf, a goblin PI, and a corn-worshipping cleric.',
-                'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/4421ed0d-f630-4c88-9004-5251b2b8adfa.jpg',
-                'series': 'Dimension 20: Fantasy High',
+                'title': 'Enter Through the Gift Shop',
+                'description': 'A new adventuring party explores a gift shop and runs into a friendly orc -- and some angry goblins.',
+                'thumbnail': 'https://vhx.imgix.net/chuncensoredstaging/assets/a1d876c3-3dee-4cd0-87c6-27a851b1d0ec.jpg',
+                'series': 'TablePop Presents: MEGADUNGEON LIVE!',
                 'season_number': 1,
                 'season': 'Season 1',
                 'episode_number': 1,
-                'episode': 'The Beginning Begins',
-                'duration': 6838,
+                'episode': 'Enter Through the Gift Shop',
+                'duration': 7101,
                 'uploader_id': 'user80538407',
                 'uploader_url': 'https://vimeo.com/user80538407',
                 'uploader': 'OTT Videos',
             },
-            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest', 'Failed to parse XML: not well-formed'],
         },
         {
-            'url': 'https://www.dropout.tv/videos/misfits-magic-holiday-special',
+            'url': 'https://watch.dropout.tv/videos/misfits-magic-holiday-special',
             'note': 'Episode not in a series',
-            'md5': 'c30fa18999c5880d156339f13c953a26',
+            'md5': '1e6428f7756b02c93b573d39ddd789fe',
             'info_dict': {
                 'id': '1915774',
                 'display_id': 'misfits-magic-holiday-special',
@@ -87,7 +87,7 @@ class DropoutIE(InfoExtractor):
                 'uploader_url': 'https://vimeo.com/user80538407',
                 'uploader': 'OTT Videos',
             },
-            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest'],
+            'expected_warnings': ['Ignoring subtitle tracks found in the HLS manifest', 'Failed to parse XML: not well-formed'],
         },
     ]
 
@@ -125,7 +125,7 @@ def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = None
-        if self._get_cookies('https://www.dropout.tv').get('_session'):
+        if self._get_cookies('https://watch.dropout.tv').get('_session'):
             webpage = self._download_webpage(url, display_id)
         if not webpage or '<div id="watch-unauthorized"' in webpage:
             login_err = self._login(display_id)
@@ -148,7 +148,7 @@ def _real_extract(self, url):
         return {
             '_type': 'url_transparent',
             'ie_key': VHXEmbedIE.ie_key(),
-            'url': VHXEmbedIE._smuggle_referrer(embed_url, 'https://www.dropout.tv'),
+            'url': VHXEmbedIE._smuggle_referrer(embed_url, 'https://watch.dropout.tv'),
             'id': self._search_regex(r'embed\.vhx\.tv/videos/(.+?)\?', embed_url, 'id'),
             'display_id': display_id,
             'title': title,
@@ -167,10 +167,10 @@ def _real_extract(self, url):
 
 class DropoutSeasonIE(InfoExtractor):
     _PAGE_SIZE = 24
-    _VALID_URL = r'https?://(?:www\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:(?P<season>[0-9]+)/?$)'
+    _VALID_URL = r'https?://(?:watch\.)?dropout\.tv/(?P<id>[^\/$&?#]+)(?:/?$|/season:(?P<season>[0-9]+)/?$)'
     _TESTS = [
         {
-            'url': 'https://www.dropout.tv/dimension-20-fantasy-high/season:1',
+            'url': 'https://watch.dropout.tv/dimension-20-fantasy-high/season:1',
             'note': 'Multi-season series with the season in the url',
             'playlist_count': 24,
             'info_dict': {
@@ -179,7 +179,7 @@ class DropoutSeasonIE(InfoExtractor):
             },
         },
         {
-            'url': 'https://www.dropout.tv/dimension-20-fantasy-high',
+            'url': 'https://watch.dropout.tv/dimension-20-fantasy-high',
             'note': 'Multi-season series with the season not in the url',
             'playlist_count': 24,
             'info_dict': {
@@ -188,7 +188,7 @@ class DropoutSeasonIE(InfoExtractor):
             },
         },
         {
-            'url': 'https://www.dropout.tv/dimension-20-shriek-week',
+            'url': 'https://watch.dropout.tv/dimension-20-shriek-week',
             'note': 'Single-season series',
             'playlist_count': 4,
             'info_dict': {
@@ -197,7 +197,7 @@ class DropoutSeasonIE(InfoExtractor):
             },
         },
         {
-            'url': 'https://www.dropout.tv/breaking-news-no-laugh-newsroom/season:3',
+            'url': 'https://watch.dropout.tv/breaking-news-no-laugh-newsroom/season:3',
             'note': 'Multi-season series with season in the url that requires pagination',
             'playlist_count': 25,
             'info_dict': {

yt_dlp/extractor/soundcloud.py

@@ -438,7 +438,7 @@ class SoundcloudIE(SoundcloudBaseIE):
                             (?P<title>[\w\d-]+)
                             (?:/(?P<token>(?!(?:albums|sets|recommended))[^?]+?))?
                             (?:[?].*)?$)
-                       |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
+                       |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?:soundcloud%3Atracks%3A)?(?P<track_id>\d+)
                           (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
                     )
                     '''
@@ -692,6 +692,9 @@ class SoundcloudIE(SoundcloudBaseIE):
         # Go+ (account with active subscription needed)
         'url': 'https://soundcloud.com/taylorswiftofficial/look-what-you-made-me-do',
         'only_matching': True,
+    }, {
+        'url': 'https://api.soundcloud.com/tracks/soundcloud%3Atracks%3A1083788353',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/tenplay.py

@@ -1,12 +1,20 @@
+import base64
+import datetime as dt
 import itertools
+import json
+import re
+import time
 
 from .common import InfoExtractor
-from ..networking import HEADRequest
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
+    encode_data_uri,
+    filter_dict,
     int_or_none,
-    update_url_query,
+    jwt_decode_hs256,
     url_or_none,
+    urlencode_postdata,
     urljoin,
 )
 from ..utils.traversal import traverse_obj
@@ -100,31 +108,148 @@ class TenPlayIE(InfoExtractor):
         'R': 18,
         'X': 18,
     }
+    _TOKEN_CACHE_KEY = 'token_data'
+    _SEGMENT_BITRATE_RE = r'(?m)-(?:300|150|75|55)0000-(\d+(?:-[\da-f]+)?)\.ts$'
+
+    _refresh_token = None
+    _access_token = None
+
+    @staticmethod
+    def _filter_ads_from_m3u8(m3u8_doc):
+        out = []
+        for line in m3u8_doc.splitlines():
+            if line.startswith('https://redirector.googlevideo.com/'):
+                out.pop()
+                continue
+            out.append(line)
+
+        return '\n'.join(out)
+
+    @staticmethod
+    def _generate_xnetwork_ten_auth_token():
+        ts = dt.datetime.now(dt.timezone.utc).strftime('%Y%m%d%H%M%S')
+        return base64.b64encode(ts.encode()).decode()
+
+    @staticmethod
+    def _is_jwt_expired(token):
+        return jwt_decode_hs256(token)['exp'] - time.time() < 300
+
+    def _refresh_access_token(self):
+        try:
+            refresh_data = self._download_json(
+                'https://10.com.au/api/token/refresh', None, 'Refreshing access token',
+                headers={
+                    'Content-Type': 'application/json',
+                }, data=json.dumps({
+                    'accessToken': self._access_token,
+                    'refreshToken': self._refresh_token,
+                }).encode())
+        except ExtractorError as e:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
+                self._refresh_token = self._access_token = None
+                self.cache.store(self._NETRC_MACHINE, self._TOKEN_CACHE_KEY, [None, None])
+                self.report_warning('Refresh token has been invalidated; retrying with credentials')
+                self._perform_login(*self._get_login_info())
+                return
+            raise
+        self._access_token = refresh_data['accessToken']
+        self._refresh_token = refresh_data['refreshToken']
+        self.cache.store(self._NETRC_MACHINE, self._TOKEN_CACHE_KEY, [self._refresh_token, self._access_token])
+
+    def _perform_login(self, username, password):
+        if not self._refresh_token:
+            self._refresh_token, self._access_token = self.cache.load(
+                self._NETRC_MACHINE, self._TOKEN_CACHE_KEY, default=[None, None])
+        if self._refresh_token and self._access_token:
+            self.write_debug('Using cached refresh token')
+            return
+
+        try:
+            auth_data = self._download_json(
+                'https://10.com.au/api/user/auth', None, 'Logging in',
+                headers={
+                    'Content-Type': 'application/json',
+                    'X-Network-Ten-Auth': self._generate_xnetwork_ten_auth_token(),
+                    'Referer': 'https://10.com.au/',
+                }, data=json.dumps({
+                    'email': username,
+                    'password': password,
+                }).encode())
+        except ExtractorError as e:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 400:
+                raise ExtractorError('Invalid username/password', expected=True)
+            raise
+
+        self._refresh_token = auth_data['jwt']['refreshToken']
+        self._access_token = auth_data['jwt']['accessToken']
+        self.cache.store(self._NETRC_MACHINE, self._TOKEN_CACHE_KEY, [self._refresh_token, self._access_token])
+
+    def _call_playback_api(self, content_id):
+        if self._access_token and self._is_jwt_expired(self._access_token):
+            self._refresh_access_token()
+        for is_retry in (False, True):
+            try:
+                return self._download_json_handle(
+                    f'https://10.com.au/api/v1/videos/playback/{content_id}/', content_id,
+                    note='Downloading video JSON', query={'platform': 'samsung'},
+                    headers=filter_dict({
+                        'TP-AcceptFeature': 'v1/fw;v1/drm',
+                        'Authorization': f'Bearer {self._access_token}' if self._access_token else None,
+                    }))
+            except ExtractorError as e:
+                if not is_retry and isinstance(e.cause, HTTPError) and e.cause.status == 403:
+                    if self._access_token:
+                        self.to_screen('Access token has expired; refreshing')
+                        self._refresh_access_token()
+                        continue
+                    elif not self._get_login_info()[0]:
+                        self.raise_login_required('Login required to access this video', method='password')
+                raise
 
     def _real_extract(self, url):
         content_id = self._match_id(url)
         data = self._download_json(
-            'https://10.com.au/api/v1/videos/' + content_id, content_id)
+            f'https://10.com.au/api/v1/videos/{content_id}', content_id)
 
-        video_data = self._download_json(
-            f'https://vod.ten.com.au/api/videos/bcquery?command=find_videos_by_id&video_id={data["altId"]}',
-            content_id, 'Downloading video JSON')
-        # Dash URL 404s, changing the m3u8 format works
-        m3u8_url = self._request_webpage(
-            HEADRequest(update_url_query(video_data['items'][0]['dashManifestUrl'], {
-                'manifest': 'm3u',
-            })),
-            content_id, 'Checking stream URL').url
-        if '10play-not-in-oz' in m3u8_url:
-            self.raise_geo_restricted(countries=['AU'])
-        if '10play_unsupported' in m3u8_url:
-            raise ExtractorError('Unable to extract stream')
-        # Attempt to get a higher quality stream
-        formats = self._extract_m3u8_formats(
-            m3u8_url.replace(',150,75,55,0000', ',500,300,150,75,55,0000'),
-            content_id, 'mp4', fatal=False)
-        if not formats:
-            formats = self._extract_m3u8_formats(m3u8_url, content_id, 'mp4')
+        video_data, urlh = self._call_playback_api(content_id)
+        content_source_id = video_data['dai']['contentSourceId']
+        video_id = video_data['dai']['videoId']
+        auth_token = urlh.get_header('x-dai-auth')
+        if not auth_token:
+            raise ExtractorError('Failed to get DAI auth token')
+
+        dai_data = self._download_json(
+            f'https://pubads.g.doubleclick.net/ondemand/hls/content/{content_source_id}/vid/{video_id}/streams',
+            content_id, note='Downloading DAI JSON',
+            data=urlencode_postdata({'auth-token': auth_token}))
+
+        # Ignore subs to avoid ad break cleanup
+        formats, _ = self._extract_m3u8_formats_and_subtitles(
+            dai_data['stream_manifest'], content_id, 'mp4')
+
+        already_have_1080p = False
+        for fmt in formats:
+            m3u8_doc = self._download_webpage(
+                fmt['url'], content_id, note='Downloading m3u8 information')
+            m3u8_doc = self._filter_ads_from_m3u8(m3u8_doc)
+            fmt['hls_media_playlist_data'] = m3u8_doc
+            if fmt.get('height') == 1080:
+                already_have_1080p = True
+
+        # Attempt format upgrade
+        if not already_have_1080p and m3u8_doc and re.search(self._SEGMENT_BITRATE_RE, m3u8_doc):
+            m3u8_doc = re.sub(self._SEGMENT_BITRATE_RE, r'-5000000-\1.ts', m3u8_doc)
+            m3u8_doc = re.sub(r'-(?:300|150|75|55)0000\.key"', r'-5000000.key"', m3u8_doc)
+            formats.append({
+                'format_id': 'upgrade-attempt-1080p',
+                'url': encode_data_uri(m3u8_doc.encode(), 'application/x-mpegurl'),
+                'hls_media_playlist_data': m3u8_doc,
+                'width': 1920,
+                'height': 1080,
+                'ext': 'mp4',
+                'protocol': 'm3u8_native',
+                '__needs_testing': True,
+            })
 
         return {
             'id': content_id,

yt_dlp/extractor/youtube/_video.py

@@ -4110,7 +4110,9 @@ def is_bad_format(fmt):
                 else 'video'),
             'release_timestamp': live_start_time,
             '_format_sort_fields': (  # source_preference is lower for potentially damaged formats
-                'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec', 'channels', 'acodec', 'lang', 'proto'),
+                'quality', 'res', 'fps', 'hdr:12', 'source',
+                'vcodec:vp9.2' if 'prefer-vp9-sort' in self.get_param('compat_opts', []) else 'vcodec',
+                'channels', 'acodec', 'lang', 'proto'),
         }
 
         def get_lang_code(track):