[test] Skip flaky tests if source unchanged (#14970)

Authored by: bashonly, Grub4K
Co-authored-by: bashonly <bashonly@protonmail.com>

.github/workflows/core.yml

@@ -56,6 +56,8 @@ jobs:
           python-version: pypy-3.11
     steps:
     - uses: actions/checkout@v5
+      with:
+        fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -65,6 +67,25 @@ jobs:
     - name: Run tests
       timeout-minutes: 15
       continue-on-error: False
+      env:
+        source: ${{ (github.event_name == 'push' && github.event.before) || 'origin/master' }}
+        target: ${{ (github.event_name == 'push' && github.event.after) || 'HEAD' }}
+      shell: bash
       run: |
+        flags=()
+        # Check if a networking file is involved
+        patterns="\
+        ^yt_dlp/networking/
+        ^yt_dlp/utils/networking\.py$
+        ^test/test_http_proxy\.py$
+        ^test/test_networking\.py$
+        ^test/test_networking_utils\.py$
+        ^test/test_socks\.py$
+        ^test/test_websockets\.py$
+        ^pyproject\.toml$
+        "
+        if git diff --name-only "${source}" "${target}" | grep -Ef <(printf '%s' "${patterns}"); then
+          flags+=(--flaky)
+        fi
         python3 -m yt_dlp -v || true  # Print debug head
-        python3 ./devscripts/run_tests.py --pytest-args '--reruns 2 --reruns-delay 3.0' core
+        python3 -m devscripts.run_tests "${flags[@]}" --pytest-args '--reruns 2 --reruns-delay 3.0' core

devscripts/run_tests.py

@@ -17,6 +17,18 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
     parser.add_argument(
         'test', help='an extractor test, test path, or one of "core" or "download"', nargs='*')
+    parser.add_argument(
+        '--flaky',
+        action='store_true',
+        default=None,
+        help='Allow running flaky tests. (default: run, unless in CI)',
+    )
+    parser.add_argument(
+        '--no-flaky',
+        action='store_false',
+        dest='flaky',
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument(
         '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
     parser.add_argument(
@@ -24,10 +36,11 @@ def parse_args():
     return parser.parse_args()
 
 
-def run_tests(*tests, pattern=None, ci=False):
+def run_tests(*tests, pattern=None, ci=False, flaky: bool | None = None):
     # XXX: hatch uses `tests` if no arguments are passed
     run_core = 'core' in tests or 'tests' in tests or (not pattern and not tests)
     run_download = 'download' in tests
+    run_flaky = flaky or (flaky is None and not ci)
 
     pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '')
     arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)]
@@ -44,6 +57,8 @@ def run_tests(*tests, pattern=None, ci=False):
             test if '/' in test
             else f'test/test_download.py::TestDownload::test_{fix_test_name(test)}'
             for test in tests)
+    if not run_flaky:
+        arguments.append('--disallow-flaky')
 
     print(f'Running {arguments}', flush=True)
     try:
@@ -72,6 +87,11 @@ def run_tests(*tests, pattern=None, ci=False):
         args = parse_args()
 
         os.chdir(Path(__file__).parent.parent)
-        sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI'))))
+        sys.exit(run_tests(
+            *args.test,
+            pattern=args.k,
+            ci=bool(os.getenv('CI')),
+            flaky=args.flaky,
+        ))
     except KeyboardInterrupt:
         pass

test/conftest.py

@@ -52,6 +52,33 @@ def skip_handlers_if(request, handler):
             pytest.skip(marker.args[1] if len(marker.args) > 1 else '')
 
 
+@pytest.fixture(autouse=True)
+def handler_flaky(request, handler):
+    """Mark a certain handler as being flaky.
+
+    This will skip the test if pytest does not get run using `--allow-flaky`
+
+    usage:
+        pytest.mark.handler_flaky('my_handler', os.name != 'nt', reason='reason')
+    """
+    for marker in request.node.iter_markers(handler_flaky.__name__):
+        if (
+            marker.args[0] == handler.RH_KEY
+            and (not marker.args[1:] or any(marker.args[1:]))
+            and request.config.getoption('disallow_flaky')
+        ):
+            reason = marker.kwargs.get('reason')
+            pytest.skip(f'flaky: {reason}' if reason else 'flaky')
+
+
+def pytest_addoption(parser, pluginmanager):
+    parser.addoption(
+        '--disallow-flaky',
+        action='store_true',
+        help='disallow flaky tests from running.',
+    )
+
+
 def pytest_configure(config):
     config.addinivalue_line(
         'markers', 'skip_handler(handler): skip test for the given handler',
@@ -62,3 +89,6 @@ def pytest_configure(config):
     config.addinivalue_line(
         'markers', 'skip_handlers_if(handler): skip test for handlers when the condition is true',
     )
+    config.addinivalue_line(
+        'markers', 'handler_flaky(handler): mark handler as flaky if condition is true',
+    )

test/test_http_proxy.py

@@ -247,6 +247,7 @@ def ctx(request):
 
 @pytest.mark.parametrize(
     'handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 @pytest.mark.parametrize('ctx', ['http'], indirect=True)  # pure http proxy can only support http
 class TestHTTPProxy:
     def test_http_no_auth(self, handler, ctx):
@@ -315,6 +316,7 @@ def test_http_with_idn(self, handler, ctx):
         ('Requests', 'https'),
         ('CurlCFFI', 'https'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestHTTPConnectProxy:
     def test_http_connect_no_auth(self, handler, ctx):
         with ctx.http_server(HTTPConnectProxyHandler) as server_address:

test/test_networking.py

@@ -312,6 +312,7 @@ def setup_class(cls):
 
 
 @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
 class TestHTTPRequestHandler(TestRequestHandlerBase):
 
     def test_verify_cert(self, handler):
@@ -756,6 +757,7 @@ def test_partial_read_then_full_read(self, handler):
 
 
 @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestClientCertificate:
     @classmethod
     def setup_class(cls):
@@ -1060,6 +1062,7 @@ def test_http_response_auto_close(self, handler):
 
 
 @pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', os.name == 'nt', reason='segfaults')
 class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
 
     @pytest.mark.parametrize('params,extensions', [

test/test_socks.py

@@ -295,6 +295,7 @@ def ctx(request):
         ('Websockets', 'ws'),
         ('CurlCFFI', 'http'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestSocks4Proxy:
     def test_socks4_no_auth(self, handler, ctx):
         with handler() as rh:
@@ -370,6 +371,7 @@ def test_timeout(self, handler, ctx):
         ('Websockets', 'ws'),
         ('CurlCFFI', 'http'),
     ], indirect=True)
+@pytest.mark.handler_flaky('CurlCFFI', reason='segfaults')
 class TestSocks5Proxy:
 
     def test_socks5_no_auth(self, handler, ctx):

test/test_websockets.py

@@ -38,6 +38,13 @@
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+pytestmark = pytest.mark.handler_flaky(
+    'Websockets',
+    os.name != 'nt' and sys.implementation.name == 'pypy',
+    reason='segfaults',
+)
+
+
 def websocket_handler(websocket):
     for message in websocket:
         if isinstance(message, bytes):

yt_dlp/cookies.py

@@ -557,7 +557,7 @@ def decrypt(self, encrypted_value):
 
 
 def _extract_safari_cookies(profile, logger):
-    if sys.platform != 'darwin':
+    if sys.platform not in ('darwin', 'ios'):
         raise ValueError(f'unsupported platform: {sys.platform}')
 
     if profile:

yt_dlp/extractor/_extractors.py

@@ -640,7 +640,10 @@
     FilmOnIE,
 )
 from .filmweb import FilmwebIE
-from .firsttv import FirstTVIE
+from .firsttv import (
+    FirstTVIE,
+    FirstTVLiveIE,
+)
 from .fivetv import FiveTVIE
 from .flextv import FlexTVIE
 from .flickr import FlickrIE

yt_dlp/extractor/firsttv.py

@@ -10,7 +10,7 @@
     unified_strdate,
     url_or_none,
 )
-from ..utils.traversal import traverse_obj
+from ..utils.traversal import require, traverse_obj
 
 
 class FirstTVIE(InfoExtractor):
@@ -129,3 +129,36 @@ def _real_extract(self, url):
         return self.playlist_result(
             self._entries(items), display_id, self._og_search_title(webpage, default=None),
             thumbnail=self._og_search_thumbnail(webpage, default=None))
+
+
+class FirstTVLiveIE(InfoExtractor):
+    IE_NAME = '1tv:live'
+    IE_DESC = 'Первый канал (прямой эфир)'
+    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/live'
+
+    _TESTS = [{
+        'url': 'https://www.1tv.ru/live',
+        'info_dict': {
+            'id': 'live',
+            'ext': 'mp4',
+            'title': r're:ПЕРВЫЙ КАНАЛ ПРЯМОЙ ЭФИР СМОТРЕТЬ ОНЛАЙН \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'live_status': 'is_live',
+        },
+        'params': {'skip_download': 'livestream'},
+    }]
+
+    def _real_extract(self, url):
+        display_id = 'live'
+        webpage = self._download_webpage(url, display_id, fatal=False)
+
+        streams_list = self._download_json('https://stream.1tv.ru/api/playlist/1tvch-v1_as_array.json', display_id)
+        mpd_url = traverse_obj(streams_list, ('mpd', ..., {url_or_none}, any, {require('mpd url')}))
+        # FFmpeg needs to be passed -re to not seek past live window. This is handled by core
+        formats, _ = self._extract_mpd_formats_and_subtitles(mpd_url, display_id, mpd_id='dash')
+
+        return {
+            'id': display_id,
+            'title': self._html_extract_title(webpage),
+            'formats': formats,
+            'is_live': True,
+        }