diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abab5c2..eb30da3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,13 +32,23 @@ jobs: python -m pip install flit flit install -s - - name: Test with pytest + - name: Test with pytest (non-web) + if: ${{ matrix.python-version != '3.14' }} run: > pytest -v -ra --doctest-modules --cov=pyabc2 --cov-report xml --cov-report term-missing + -m "not web" -n auto + - name: Test with pytest (full, including web) + if: ${{ matrix.python-version == '3.14' }} + run: > + pytest -v -ra + --doctest-modules + --cov=pyabc2 --cov-report xml --cov-report term-missing + -n auto --dist loadgroup + - name: Check type annotations run: | mypy --non-interactive . diff --git a/docs/changes.md b/docs/changes.md index 95d405f..898d77f 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -1,5 +1,9 @@ # Release notes +## v0.1.4 (unreleased) + +* Updates for Eskin and Bill Black ({pull}`112`) + ## v0.1.3 (2026-04-17) * Improve styling of abcjs containers, pandas dataframes, and ipywidgets diff --git a/docs/examples/sources.ipynb b/docs/examples/sources.ipynb index 5fbb208..483b806 100644 --- a/docs/examples/sources.ipynb +++ b/docs/examples/sources.ipynb @@ -402,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "Tune(df.query(\"group == 'jigs'\").iloc[0].abc)" + "Tune(df.query(\"group == 'Jig A-G'\").iloc[0].abc)" ] }, { diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 04cf74c..856e80a 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -8,60 +8,105 @@ * `requests `__ """ +from __future__ import annotations + +import functools import logging import re from pathlib import Path +from typing import TYPE_CHECKING from pyabc2._util import get_logger as _get_logger +if TYPE_CHECKING: # pragma: no cover + import requests + logger = _get_logger(__name__) HERE = Path(__file__).parent SAVE_TO = HERE / "_bill-black" TXT_FNS = [ - "a-tunes-1.txt", - "b-tunes-1.txt", - "c-tunes-1.txt", - "d-tunes-1.txt", - "e-tunes-1.txt", - "f-tunes-1.txt", - "g-tunes-1.txt", - "h-tunes-1.txt", - "i-tunes-1.txt", - "j-tunes-1.txt", - "k-tunes-1.txt", - "l-tunes-1.txt", - "m-tunes-1.txt", - "n-tunes-1.txt", - "o-tunes-1.txt", - "pq-tunes-1.txt", - "r-tunes-1.txt", - "s-tunes-2.rtf", - "t-tunes-1.txt", - "uv-tunes-1.txt", - "wz-tunes-1.txt", + "a-tunes.txt", + "b-tunes.txt", + "c-tunes.txt", + "d-tunes.txt", + "e-tunes.txt", + "f-tunes.txt", + "g-tunes.txt", + "h-tunes.txt", + "i-tunes.txt", + "j-tunes.txt", + "k-tunes.txt", + "L-tunes.txt", + "m-tunes.txt", + "n-tunes.txt", + "o-tunes.txt", + "pq-tunes.txt", + "r-tunes.txt", + "s-tunes.txt", + "t-tunes.txt", + "uv-tunes.txt", + "wxyz-tunes.txt", ] +@functools.lru_cache(1) +def _get_session() -> requests.Session: + return _build_session() + + +def _build_session() -> requests.Session: + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=5, + backoff_factor=0.5, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[403, 429, 500, 502, 503, 504], + # Bill Black seems to sporadically return 403 (forbidden) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) + + return session + + def download() -> None: - """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/text/ + """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. """ + import threading import zipfile from concurrent.futures import ThreadPoolExecutor - import requests + thread_local = threading.local() + + def get_worker_session() -> requests.Session: + # One Session per worker thread to avoid cross-thread Session sharing. + try: + return thread_local.session + except AttributeError: + session = _build_session() + thread_local.session = session + return session def download_one(url): - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + session = get_worker_session() + r = session.get(url, timeout=5) r.raise_for_status() return r.text with ThreadPoolExecutor(max_workers=4) as executor: futures = [] for fn in TXT_FNS: - url = f"https://www.capeirish.com/ittl/alltunes/text/{fn}" + url = f"https://www.capeirish.com/ittl/alltunes/alltunes-text/{fn}" futures.append(executor.submit(download_one, url)) SAVE_TO.mkdir(exist_ok=True) diff --git a/pyabc2/sources/bill_black_tunefolders.py b/pyabc2/sources/bill_black_tunefolders.py index 7a37944..f60dd79 100644 --- a/pyabc2/sources/bill_black_tunefolders.py +++ b/pyabc2/sources/bill_black_tunefolders.py @@ -4,8 +4,8 @@ https://www.capeirish.com/ittl/ As of the 2025-06-14 update, the "tunefolders" method is deprecated. -Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/html/), -while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/text/), +Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/alltunes-html/), +while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/alltunes-text/), both split up alphabetically by tune name. Requires: @@ -19,6 +19,7 @@ from pathlib import Path from pyabc2._util import get_logger as _get_logger +from pyabc2.sources.bill_black import _get_session logger = _get_logger(__name__) @@ -209,7 +210,7 @@ def get_collection(key: str) -> Collection: def download(key: str | Iterable[str] | None = None) -> None: import gzip - import requests + session = _get_session() SAVE_TO.mkdir(exist_ok=True) @@ -224,7 +225,7 @@ def download(key: str | Iterable[str] | None = None) -> None: for url in collection.abc_urls: p = collection.url_to_file(url) logger.info(f"Downloading {url} to {p.relative_to(HERE).as_posix()}") - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() # Extract filename from URL and append .gz diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index c2fe38b..94414e7 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -7,8 +7,12 @@ * `requests `__ """ +from __future__ import annotations + +import functools import json import re +from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING, Literal, NamedTuple from urllib.parse import parse_qs, urlsplit @@ -19,6 +23,7 @@ if TYPE_CHECKING: # pragma: no cover import pandas + import requests logger = _get_logger(__name__) @@ -30,14 +35,15 @@ _CCE_SD = "https://michaeleskin.com/cce_sd" _TUNEBOOK_KEY_TO_URL = { # https://michaeleskin.com/tunebooks.html#websites_irish - "kss": f"{_TBWS}/king_street_sessions_tunebook_17Jan2025.html", - "oflaherty_2025": f"{_TBWS}/oflahertys_2025_retreat_tunes_final.html", - "carp": f"{_TBWS}/carp_celtic_jam_tunebook_17Jan2025.html", + "kss": f"{_TBWS}/king-street-session-tunebook-25jun2026.html", + "oflaherty_2025": f"{_TBWS}/oflaherty-2025-retreat-tunes-played-slowly.html", + "oflaherty_2026": f"{_TBWS}/oflaherty-2026-retreat-tunes-played-slowly.html", + "carp": f"{_TBWS}/carp_celtic_jam_tunebook_29jun2026.html", "hardy_2024": f"{_TBWS}/paul_hardy_2024_8feb2025.html", - "hardy_2025": f"{_TBWS}/paul_hardy_2025_12aug2025.html", - "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_17Jan2025.html", + "hardy_2025": f"{_TBWS}/paul_hardy_2025_29jun2026.html", + "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_29jun2026.html", "cce_san_diego_jan2025": f"{_CCE_SD}/cce_san_diego_tunes_31jan2025.html", - "cce_san_diego_nov2025": f"{_CCE_SD}/cce_san_diego_tunes_10nov2025.html", + "cce_san_diego_jun2026": f"{_TBWS}/comhaltas-san-diego-tunebook-24jun2026.html", # https://michaeleskin.com/tunebooks.html#websites_18th_century_collections "playford1": f"{_TBWS}/playford_1_partington_17jan2025.html", "playford2": f"{_TBWS}/playford_2_partington_17jan2025.html", @@ -50,7 +56,7 @@ # Definitive versions _TUNEBOOK_ALIAS = { - "cce_san_diego": "cce_san_diego_nov2025", + "cce_san_diego": "cce_san_diego_jun2026", } for _alias, _target in _TUNEBOOK_ALIAS.items(): _TUNEBOOK_KEY_TO_URL[_alias] = _TUNEBOOK_KEY_TO_URL[_target] @@ -58,6 +64,29 @@ _URL_NETLOCS = {"michaeleskin.com", "www.michaeleskin.com"} +@functools.lru_cache(1) +def _get_session() -> requests.Session: + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=5, + backoff_factor=0.5, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[415, 429, 500, 502, 503, 504], + # Eskin seems to sporadically return 415 (unsupported media type) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) + + return session + + def _deflate(s: str, /) -> str: """Use deflate (zlib) to compress and base64-encode `s`.""" import base64 @@ -233,22 +262,11 @@ def get_tunebook_info(key: str) -> EskinTunebookInfo: ) -def _download_data(key: str): - """Extract and save the tune data from the tunebook webpage as JSON.""" - import gzip - - import requests - - tb_info = get_tunebook_info(key) - - r = requests.get(tb_info.url, timeout=5) - r.raise_for_status() - html = r.text - +def _extract_data_from_html_2025(html: str, *, key: str): # First find the tune type options by searching for 'tunes = type;' types = sorted(set(re.findall(r"tunes = (.*?);", html))) if types: - pass + pass # pragma: no cover elif "const tunes=[" in html: # no types, just one list of tunes types = ["tunes"] else: @@ -258,13 +276,13 @@ def _download_data(key: str): all_data = {} for type_ in types: m = re.search(rf"const {type_}=\[(.*?)\];", html, flags=re.DOTALL) - if m is None: + if m is None: # pragma: no cover raise RuntimeError(f"Unable to find data for type {type_!r}") s_data = "[" + m.group(1) + "]" try: data = json.loads(s_data) - except json.JSONDecodeError as e: + except json.JSONDecodeError as e: # pragma: no cover w = 25 a = max(0, e.pos - w) b = min(len(s_data), e.pos + w) @@ -281,9 +299,118 @@ def _download_data(key: str): all_data[type_] = data + return all_data + + +def _extract_data_from_html_2026(html: str, *, key: str): + # The ABCs are in ', + html, + flags=re.DOTALL | re.MULTILINE, + ): + tune_id, abc_raw = m.groups() + + abc_lines = [] + for line in abc_raw.splitlines(): + line = line.strip() + if not line or line.startswith("%"): + continue + abc_lines.append(line) + + tunes.append({"id": tune_id, "abc": "\n".join(abc_lines)}) + + # Group tunes + all_data = defaultdict(list) + for tune in tunes: + group_id = group_assignments.get(tune["id"]) + if group_id is None: # pragma: no cover + logger.warning(f"No group assignment found for tune {tune['id']}") + continue + group_name = group_names.get(group_id, f"Unknown group {group_id}") + all_data[group_name].append(tune) + + return all_data + + +def _download_data(key: str): + """Extract and save the tune data from the tunebook webpage as JSON.""" + import gzip + + session = _get_session() + + tb_info = get_tunebook_info(key) + + r = session.get(tb_info.url, timeout=5) + r.raise_for_status() + html = r.text + + try: + data = _extract_data_from_html_2025(html, key=key) + except RuntimeError as e: + if str(e) == "Unable to detect tune types": + # We can be pretty sure it's the new format + logger.debug("Tune groups array not detected, assuming new format.") + data = _extract_data_from_html_2026(html, key=key) + else: + raise + SAVE_TO.mkdir(exist_ok=True) with gzip.open(tb_info.path, "wt") as f: - json.dump(all_data, f, indent=2) + json.dump(data, f, indent=2) def _load_data(key: str): @@ -294,7 +421,7 @@ def _load_data(key: str): return json.load(f) -def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": +def load_meta(key: str, *, redownload: bool = False) -> pandas.DataFrame: """Load the tunebook data, no parsing. Parameters @@ -318,7 +445,7 @@ def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": - Paul Hardy's Session Tunebook * - ``kss`` - King Street Sessions Tunebook - * - ``oflaherty_2025`` + * - ``oflaherty_{2025,2026}`` - O'Flaherty's Retreat Tunes * - ``playford{1,2,3}`` - Playford vols. 1--3 diff --git a/pyproject.toml b/pyproject.toml index 76624c2..f51326a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ sources = [ "numpy ==2.*", "pandas ==2.*", "requests ==2.*", + "urllib3 ==2.*", ] abcjs-widget = [ "anywidget", @@ -62,7 +63,10 @@ profile = "black" line_length = 100 [tool.pytest.ini_options] -markers = ["slow"] +markers = [ + "slow", + "web: tests that hit external websites", +] [tool.mypy] exclude = ["^venv"] diff --git a/tests/test_sources.py b/tests/test_sources.py index 425c8c4..3d97792 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -44,6 +44,8 @@ def test_example_random(): assert type(tune) is Tune +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_tune_type_file_prefix(): norbeck._maybe_download() all_fps = list(norbeck.SAVE_TO.glob("*.abc")) @@ -58,6 +60,8 @@ def test_norbeck_tune_type_file_prefix(): assert set(all_fps) == all_fps_type_set +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_unique(): # X value should be unique within tune type norbeck._maybe_download() @@ -74,6 +78,8 @@ def test_norbeck_x_unique(): assert len(set(xs)) == len(xs), "X unique" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_vals(): # Make sure they match with what file says norbeck._maybe_download() @@ -100,6 +106,8 @@ def test_norbeck_x_vals(): @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_load(): # NOTE: downloads files if not already present @@ -131,6 +139,8 @@ def test_norbeck_load(): norbeck.load("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "url,title,key,type", [ @@ -149,11 +159,14 @@ def test_the_session_load_url(url, title, key, type): assert tune.header["reference number"] == "1" +@pytest.mark.xdist_group("the_session") def test_the_session_url_check(): with pytest.raises(AssertionError): the_session.load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_load_archive_threaded(): # NOTE: downloads file if not already present with pytest.warns(UserWarning, match=r"The Session tune\(s\) failed to load"): @@ -163,12 +176,15 @@ def test_the_session_load_archive_threaded(): assert tunes1 == tunes2 +@pytest.mark.xdist_group("the_session") def test_the_session_download_invalid(): with pytest.raises(ValueError): _ = the_session.download("asdf") @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "which", ["aliases", "events", "recordings", "sessions", "sets", "tune_popularity", "tunes"] ) @@ -203,6 +219,7 @@ def test_the_session_load_meta(which): # in df3, `pd.Float64Dtype()` +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_invalid(): with pytest.raises(ValueError): _ = the_session.load_meta("asdf") @@ -211,6 +228,7 @@ def test_the_session_load_meta_invalid(): _ = the_session.load_meta("sessions", format="asdf") +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_doc_consistency(): s_options = ", ".join(repr(x) for x in sorted(the_session._META_ALLOWED)) expected_line = f"which : {{{s_options}}}" @@ -246,12 +264,16 @@ def test_int_downcast(): assert s3.dtype == expected_dtype_ext +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize("netloc", sorted(the_session._URL_NETLOCS)) def test_load_url_the_session(netloc): tune = load_url(f"https://{netloc}/tunes/10000") assert tune.title == "Brian Quinn's" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") @pytest.mark.parametrize("netloc", sorted(norbeck._URL_NETLOCS)) def test_load_url_norbeck(netloc): import requests @@ -265,6 +287,7 @@ def test_load_url_norbeck(netloc): assert tune.title == "For The Love Of Music" +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("netloc", sorted(eskin._URL_NETLOCS)) @pytest.mark.parametrize("param", list(ESKIN_COMPRESSED_ABC_DATA)) def test_load_url_eskin(netloc, param): @@ -283,14 +306,16 @@ def test_load_url_invalid_domain(): _ = load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") def test_eskin_tunebook_bad_url_redirects(): - import requests + session = eskin._get_session() # Bad URL (2025 -> 3025) # Redirects to the home page. # Nothing in `r.history`. `allow_redirects=False` has no impact. url = "https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov3025.html" - r = requests.head(url, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert r.status_code == 302 @@ -299,12 +324,14 @@ def test_eskin_tunebook_bad_url_redirects(): assert r.is_redirect +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_url_exist(key): - import requests + session = eskin._get_session() url = eskin._TUNEBOOK_KEY_TO_URL[key] - r = requests.head(url, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() # Bad URLs seem to just redirect to his homepage, # so we need to check the final URL @@ -316,11 +343,13 @@ def test_eskin_tunebook_url_exist(key): raise ValueError(f"{key!r} URL {url} redirects to homepage") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") def test_eskin_tunebook_url_current(): - import requests + session = eskin._get_session() url = "https://michaeleskin.com/tunebooks.html" - r = requests.get(url, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() if ( r.status_code == 302 @@ -342,11 +371,16 @@ def test_eskin_tunebook_url_current(): raise ValueError(f"Could not find link for tunebook {key!r} in tunebooks page.") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_data_load(key): df = eskin.load_meta(key) + assert len(df) >= 20 + tune_group_keys = { + # Old KSS "airs_songs", "hornpipes", "jigs", @@ -361,14 +395,53 @@ def test_eskin_tunebook_data_load(key): "slipjigs", "strathspeys", "waltzes", + # New KSS + "Long Dance", + "Barndance", + "Reel O-R", + "Reel F-K", + "Set Dance", + "Reel A-E", + "Single Jig", + "Set dance", + "Jig or march", + "Quadrille", + "Highland Schottische", + "Pipe Reel", + "Strathspey", + "Hornpipe", + "Reel S-Y", + "Reel L-N", + "March", + "Waltz", + "Slip Jig", + "Jig A-G", + "Slide", + "Other", + "Polka or Reel", + "Bourree", + "Schottische", + "Air", + "Polka or march", + "Song", + "Fling", + "Slow Air", + "Mazurka", + "Jig Q-Y", + "Polka", + "Jig H-P", } + unique_groups = set(df.group.unique()) if key in {"kss"}: - assert set(df.group.unique()) <= tune_group_keys + assert unique_groups <= tune_group_keys else: - assert df.group.unique().tolist() == ["tunes"] + assert unique_groups == {"tunes"} or all( + re.fullmatch(r"^[A-Z\-]+$", g) is not None for g in unique_groups + ) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_parsing(): # From https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov2025.html url = "https://michaeleskin.com/abctools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" @@ -387,6 +460,7 @@ def test_eskin_abc_url_parsing(): assert sum(line.startswith(r"%%") for line in abc_no_rm.splitlines()) > 0 +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_missing_param(): url = "https://michaeleskin.com/abctools/abctools.html?" with pytest.raises( @@ -396,18 +470,21 @@ def test_eskin_abc_url_missing_param(): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param(): url = "https://michaeleskin.com/abctools/abctools.html?lzw=hi" with pytest.raises(RuntimeError, match="Failed to decompress LZString data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param_def(): url = "https://michaeleskin.com/abctools/abctools.html?def=hi" with pytest.raises(RuntimeError, match="Failed to decompress deflate data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad(caplog): url = "https://michaeleski.com/deftools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" with caplog.at_level("DEBUG"): @@ -419,6 +496,8 @@ def test_eskin_abc_url_bad(caplog): ] +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("use_lzw", [True, False]) def test_eskin_abc_url_creation(use_lzw): import requests @@ -435,11 +514,13 @@ def test_eskin_abc_url_creation(use_lzw): raise ValueError(f"URL {url} redirects to homepage") +@pytest.mark.xdist_group("eskin") def test_eskin_invalid_tunebook_key(): with pytest.raises(ValueError, match="Unknown Eskin tunebook key: 'asdf'"): _ = eskin.get_tunebook_info("asdf") +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_invalid_length(): s = "eJyFjbEKwkAQRPv9iv2DQyvd7jbGK0wQJIVtktucJ4FIghZyH-abcdefg" with pytest.raises( @@ -449,64 +530,78 @@ def test_eskin_inflate_invalid_length(): _ = eskin._inflate(s) +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_pad_3(): s = "abc" assert eskin._inflate(eskin._deflate(s)) == s +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_https(): - import requests + session = bill_black._get_session() url = "http://www.capeirish.com/ittl/tunefolders/" url_https = url.replace("http://", "https://") - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" not in r.headers - r = requests.head(url_https, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url_https, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" in r.headers @pytest.mark.xfail(reason="Bill Black tunefolders are currently in flux", strict=False) +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") @pytest.mark.parametrize("key", list(bill_black_tunefolders._KEY_TO_COLLECTION)) def test_bill_black_tunefolders(key): lst = bill_black_tunefolders.load_meta(key, redownload=True) assert len(lst) > 0 +@pytest.mark.xdist_group("bill_black") def test_bill_black_tunefolders_invalid_key(): with pytest.raises(ValueError, match="Unknown collection key: 'asdf'"): _ = bill_black_tunefolders.get_collection("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_text_fns(): - import requests + session = bill_black._get_session() - url = "http://www.capeirish.com/ittl/alltunes/text/" - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + url = "http://www.capeirish.com/ittl/alltunes/alltunes-text/" + r = session.get(url, timeout=5) r.raise_for_status() - fns_web = sorted(re.findall(r'href=["\']([a-z0-9\-]+\.(?:txt|rtf))["\']', r.text)) - if "s-tunes-1.txt" in fns_web: - # We're using s-tunes-2, not both - fns_web.remove("s-tunes-1.txt") + fns_web = sorted( + re.findall(r'href=["\']([a-zA-Z0-9\-]+\.(?:txt|rtf))["\']', r.text), + key=lambda s: s.lower(), + ) assert bill_black.TXT_FNS == fns_web +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_load(): lst = bill_black.load_meta() assert len(lst) > 0 assert lst[0].startswith("X:") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_tune_collections(): df = the_session.get_tune_collections(1) # Cooley's assert not df.empty +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_set(): tunes = the_session.get_member_set(65013, 106212) assert len(tunes) == 3 @@ -516,6 +611,8 @@ def test_the_session_get_member_set(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets(): sets = the_session.get_member_sets(65013) assert len(sets) >= 1 @@ -525,6 +622,8 @@ def test_the_session_get_member_sets(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets_multipage(): sets = the_session.get_member_sets(1, pages=3, size=2, max_threads=2, orderby="oldest") assert len(sets) == 6 @@ -534,6 +633,7 @@ def test_the_session_get_member_sets_multipage(): assert d["tune_id"] == 560 +@pytest.mark.xdist_group("the_session") def test_the_session_consume_validation(): f = the_session.get_member_sets @@ -547,11 +647,15 @@ def test_the_session_consume_validation(): _ = f(1, max_threads=0) +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_consume_auto_leading_slash(): (d,) = the_session._consume("tunes/22878") assert d["name"] == "Jack Farrell's" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") @pytest.mark.parametrize("key", list(hardy._TUNEBOOK_KEY_TO_URL)) def test_hardy_load_meta(key): abcs = hardy.load_meta(key) @@ -563,6 +667,8 @@ def test_hardy_load_meta(key): assert "\n\n" not in abc, "no empty lines within a tune block" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_load_meta_remove_prefs(): # Default: no % lines key = "session" @@ -576,16 +682,20 @@ def test_hardy_load_meta_remove_prefs(): assert sum(line.lstrip().startswith("%") for abc in abcs_raw for line in abc.splitlines()) > 1 +@pytest.mark.xdist_group("hardy") def test_hardy_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.load_meta("asdf") +@pytest.mark.xdist_group("hardy") def test_hardy_download_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.download("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_annex_is_latest(): """Confirm the hardcoded 'annex' URL points to the current (non-superseded) annex file.""" import requests