From a6c1e9519e6e4ac93d9a560ae24819144101f4e2 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 10:21:53 -0500 Subject: [PATCH 01/17] Update Eskin URLs but note the new tunebook websites have the ABCs stored differently (in script tags) --- pyabc2/sources/eskin.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index c2fe38b..9c379ea 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -30,14 +30,16 @@ _CCE_SD = "https://michaeleskin.com/cce_sd" _TUNEBOOK_KEY_TO_URL = { # https://michaeleskin.com/tunebooks.html#websites_irish - "kss": f"{_TBWS}/king_street_sessions_tunebook_17Jan2025.html", - "oflaherty_2025": f"{_TBWS}/oflahertys_2025_retreat_tunes_final.html", - "carp": f"{_TBWS}/carp_celtic_jam_tunebook_17Jan2025.html", + "kss": f"{_TBWS}/king-street-session-tunebook-25jun2026.html", + "oflaherty_2025": f"{_TBWS}/oflaherty-2025-retreat-tunes-played-slowly.html", + "oflaherty_2026": f"{_TBWS}/oflaherty-2026-retreat-tunes-played-slowly.html", + "carp": f"{_TBWS}/carp_celtic_jam_tunebook_29jun2026.html", "hardy_2024": f"{_TBWS}/paul_hardy_2024_8feb2025.html", - "hardy_2025": f"{_TBWS}/paul_hardy_2025_12aug2025.html", - "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_17Jan2025.html", + "hardy_2025": f"{_TBWS}/paul_hardy_2025_29jun2026.html", + "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_29jun2026.html", "cce_san_diego_jan2025": f"{_CCE_SD}/cce_san_diego_tunes_31jan2025.html", "cce_san_diego_nov2025": f"{_CCE_SD}/cce_san_diego_tunes_10nov2025.html", + "cce_san_diego_jun2026": f"{_TBWS}/comhaltas-san-diego-tunebook-24jun2026.html", # https://michaeleskin.com/tunebooks.html#websites_18th_century_collections "playford1": f"{_TBWS}/playford_1_partington_17jan2025.html", "playford2": f"{_TBWS}/playford_2_partington_17jan2025.html", @@ -50,7 +52,7 @@ # Definitive versions _TUNEBOOK_ALIAS = { - "cce_san_diego": "cce_san_diego_nov2025", + "cce_san_diego": "cce_san_diego_jun2026", } for _alias, _target in _TUNEBOOK_ALIAS.items(): _TUNEBOOK_KEY_TO_URL[_alias] = _TUNEBOOK_KEY_TO_URL[_target] @@ -318,7 +320,7 @@ def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": - Paul Hardy's Session Tunebook * - ``kss`` - King Street Sessions Tunebook - * - ``oflaherty_2025`` + * - ``oflaherty_{2025,2026}`` - O'Flaherty's Retreat Tunes * - ``playford{1,2,3}`` - Playford vols. 1--3 From e684125c0a9fd02471594287515b46654372a18e Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 11:45:39 -0500 Subject: [PATCH 02/17] Extract tunes and groups from new Eskin tbws --- pyabc2/sources/eskin.py | 117 +++++++++++++++++++++++++++++++++++----- tests/test_sources.py | 46 +++++++++++++++- 2 files changed, 147 insertions(+), 16 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 9c379ea..44d871c 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -9,6 +9,7 @@ import json import re +from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING, Literal, NamedTuple from urllib.parse import parse_qs, urlsplit @@ -38,7 +39,6 @@ "hardy_2025": f"{_TBWS}/paul_hardy_2025_29jun2026.html", "cce_dublin_2001": f"{_CCE_SD}/cce_dublin_2001_tunebook_29jun2026.html", "cce_san_diego_jan2025": f"{_CCE_SD}/cce_san_diego_tunes_31jan2025.html", - "cce_san_diego_nov2025": f"{_CCE_SD}/cce_san_diego_tunes_10nov2025.html", "cce_san_diego_jun2026": f"{_TBWS}/comhaltas-san-diego-tunebook-24jun2026.html", # https://michaeleskin.com/tunebooks.html#websites_18th_century_collections "playford1": f"{_TBWS}/playford_1_partington_17jan2025.html", @@ -235,18 +235,7 @@ def get_tunebook_info(key: str) -> EskinTunebookInfo: ) -def _download_data(key: str): - """Extract and save the tune data from the tunebook webpage as JSON.""" - import gzip - - import requests - - tb_info = get_tunebook_info(key) - - r = requests.get(tb_info.url, timeout=5) - r.raise_for_status() - html = r.text - +def _extract_data_from_html_2025(html: str, *, key: str): # First find the tune type options by searching for 'tunes = type;' types = sorted(set(re.findall(r"tunes = (.*?);", html))) if types: @@ -283,9 +272,109 @@ def _download_data(key: str): all_data[type_] = data + return all_data + + +def _extract_data_from_html_2026(html: str, *, key: str): + # The ABCs are in ', + html, + flags=re.DOTALL | re.MULTILINE, + ): + tune_id, abc_raw = m.groups() + + abc_lines = [] + for line in abc_raw.splitlines(): + line = line.strip() + if line.startswith("%"): + continue + abc_lines.append(line) + + tunes.append({"id": tune_id, "abc": "\n".join(abc_lines)}) + + # Group tunes + all_data = defaultdict(list) + for tune in tunes: + group_id = group_assignments.get(tune["id"]) + if group_id is None: + logger.warning(f"No group assignment found for tune {tune['id']}") + continue + group_name = group_names.get(group_id, f"Unknown group {group_id}") + all_data[group_name].append(tune) + + return all_data + + +def _download_data(key: str): + """Extract and save the tune data from the tunebook webpage as JSON.""" + import gzip + + import requests + + tb_info = get_tunebook_info(key) + + r = requests.get(tb_info.url, timeout=5) + r.raise_for_status() + html = r.text + + try: + data = _extract_data_from_html_2025(html, key=key) + except RuntimeError: + data = _extract_data_from_html_2026(html, key=key) + SAVE_TO.mkdir(exist_ok=True) with gzip.open(tb_info.path, "wt") as f: - json.dump(all_data, f, indent=2) + json.dump(data, f, indent=2) def _load_data(key: str): diff --git a/tests/test_sources.py b/tests/test_sources.py index 425c8c4..f4e9e6d 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -346,7 +346,10 @@ def test_eskin_tunebook_url_current(): def test_eskin_tunebook_data_load(key): df = eskin.load_meta(key) + assert len(df) >= 20 + tune_group_keys = { + # Old KSS "airs_songs", "hornpipes", "jigs", @@ -361,12 +364,51 @@ def test_eskin_tunebook_data_load(key): "slipjigs", "strathspeys", "waltzes", + # New KSS + "Long Dance", + "Barndance", + "Reel O-R", + "Reel F-K", + "Set Dance", + "Reel A-E", + "Single Jig", + "Set dance", + "Jig or march", + "Quadrille", + "Highland Schottish", # XXX: "Highland Schottische" + "Pipe Reel", + "Strathspey", + "Hornpipe", + "Reel S-Y", + "Reel L-N", + "March", + "Waltz", + "Slip Jig", + "Jig A-G", + "Slide", + "Highland Scottische", # XXX: "Highland Schottische" + "Other", + "Polka or Reel", + "Bourree", + "Schottische", + "Air", + "Polka or march", + "Song", + "Fling", + "Slow Air", + "Mazurka", + "Jig Q-Y", + "Polka", + "Jig H-P", } + unique_groups = set(df.group.unique()) if key in {"kss"}: - assert set(df.group.unique()) <= tune_group_keys + assert unique_groups <= tune_group_keys else: - assert df.group.unique().tolist() == ["tunes"] + assert unique_groups == {"tunes"} or all( + re.fullmatch(r"^[A-Z\-]+$", g) is not None for g in unique_groups + ) def test_eskin_abc_url_parsing(): From 4b3305fc8babfb96f135216b63fb2678cbe1d186 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 11:48:51 -0500 Subject: [PATCH 03/17] Normalize "Highland Schottische" --- pyabc2/sources/eskin.py | 4 ++++ tests/test_sources.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 44d871c..f12a906 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -320,6 +320,10 @@ def _extract_data_from_html_2026(html: str, *, key: str): group_id, group_name_raw = m.groups() group_name = re.sub(r" *\([0-9]+\)$", "", unescape(group_name_raw)) logger.debug(f"Found group name: {group_id=}, {group_name=}") + if group_name in {"Highland Schottish", "Highland Scottische"}: + group_name_norm = "Highland Schottische" + logger.debug(f"Normalizing group name {group_name!r} to {group_name_norm!r}") + group_name = group_name_norm group_names[group_id] = group_name if not group_names: logger.debug("No group names found in HTML") diff --git a/tests/test_sources.py b/tests/test_sources.py index f4e9e6d..09aa4b1 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -375,7 +375,7 @@ def test_eskin_tunebook_data_load(key): "Set dance", "Jig or march", "Quadrille", - "Highland Schottish", # XXX: "Highland Schottische" + "Highland Schottische", "Pipe Reel", "Strathspey", "Hornpipe", @@ -386,7 +386,6 @@ def test_eskin_tunebook_data_load(key): "Slip Jig", "Jig A-G", "Slide", - "Highland Scottische", # XXX: "Highland Schottische" "Other", "Polka or Reel", "Bourree", From 876040a0ac5ef7d9873bf98452196bdfe13b2a0e Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:01:08 -0500 Subject: [PATCH 04/17] Update Bill Black alltunes --- pyabc2/sources/bill_black.py | 46 ++++++++++++------------ pyabc2/sources/bill_black_tunefolders.py | 4 +-- tests/test_sources.py | 10 +++--- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 04cf74c..6558e37 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -20,32 +20,32 @@ SAVE_TO = HERE / "_bill-black" TXT_FNS = [ - "a-tunes-1.txt", - "b-tunes-1.txt", - "c-tunes-1.txt", - "d-tunes-1.txt", - "e-tunes-1.txt", - "f-tunes-1.txt", - "g-tunes-1.txt", - "h-tunes-1.txt", - "i-tunes-1.txt", - "j-tunes-1.txt", - "k-tunes-1.txt", - "l-tunes-1.txt", - "m-tunes-1.txt", - "n-tunes-1.txt", - "o-tunes-1.txt", - "pq-tunes-1.txt", - "r-tunes-1.txt", - "s-tunes-2.rtf", - "t-tunes-1.txt", - "uv-tunes-1.txt", - "wz-tunes-1.txt", + "a-tunes.txt", + "b-tunes.txt", + "c-tunes.txt", + "d-tunes.txt", + "e-tunes.txt", + "f-tunes.txt", + "g-tunes.txt", + "h-tunes.txt", + "i-tunes.txt", + "j-tunes.txt", + "k-tunes.txt", + "L-tunes.txt", + "m-tunes.txt", + "n-tunes.txt", + "o-tunes.txt", + "pq-tunes.txt", + "r-tunes.txt", + "s-tunes.txt", + "t-tunes.txt", + "uv-tunes.txt", + "wxyz-tunes.txt", ] def download() -> None: - """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/text/ + """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. """ import zipfile @@ -61,7 +61,7 @@ def download_one(url): with ThreadPoolExecutor(max_workers=4) as executor: futures = [] for fn in TXT_FNS: - url = f"https://www.capeirish.com/ittl/alltunes/text/{fn}" + url = f"https://www.capeirish.com/ittl/alltunes/alltunes-text/{fn}" futures.append(executor.submit(download_one, url)) SAVE_TO.mkdir(exist_ok=True) diff --git a/pyabc2/sources/bill_black_tunefolders.py b/pyabc2/sources/bill_black_tunefolders.py index 7a37944..324eb5c 100644 --- a/pyabc2/sources/bill_black_tunefolders.py +++ b/pyabc2/sources/bill_black_tunefolders.py @@ -4,8 +4,8 @@ https://www.capeirish.com/ittl/ As of the 2025-06-14 update, the "tunefolders" method is deprecated. -Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/html/), -while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/text/), +Bill Black is now using the Eskin ABC Tools (https://www.capeirish.com/ittl/alltunes/alltunes-html/), +while also posting ABC text files (https://www.capeirish.com/ittl/alltunes/alltunes-text/), both split up alphabetically by tune name. Requires: diff --git a/tests/test_sources.py b/tests/test_sources.py index 09aa4b1..e785a0a 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -525,14 +525,14 @@ def test_bill_black_tunefolders_invalid_key(): def test_bill_black_text_fns(): import requests - url = "http://www.capeirish.com/ittl/alltunes/text/" + url = "http://www.capeirish.com/ittl/alltunes/alltunes-text/" r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() - fns_web = sorted(re.findall(r'href=["\']([a-z0-9\-]+\.(?:txt|rtf))["\']', r.text)) - if "s-tunes-1.txt" in fns_web: - # We're using s-tunes-2, not both - fns_web.remove("s-tunes-1.txt") + fns_web = sorted( + re.findall(r'href=["\']([a-zA-Z0-9\-]+\.(?:txt|rtf))["\']', r.text), + key=lambda s: s.lower(), + ) assert bill_black.TXT_FNS == fns_web From 97baaccdd5666b051e21230497a8a400218a2c01 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:21:32 -0500 Subject: [PATCH 05/17] Update KSS group query for new groups --- docs/examples/sources.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/sources.ipynb b/docs/examples/sources.ipynb index 5fbb208..483b806 100644 --- a/docs/examples/sources.ipynb +++ b/docs/examples/sources.ipynb @@ -402,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "Tune(df.query(\"group == 'jigs'\").iloc[0].abc)" + "Tune(df.query(\"group == 'Jig A-G'\").iloc[0].abc)" ] }, { From d43b7d2c60a21bcf230aec00ea2b25bb7d58ce79 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:54:14 -0500 Subject: [PATCH 06/17] Add User-Agent header for Eskin requests Getting 415 errors in the Python 3.10 CI run (though the 3.14 is fine, and my 3.13 local testing too) --- pyabc2/sources/eskin.py | 2 +- tests/test_sources.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index f12a906..6dbd3d7 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -367,7 +367,7 @@ def _download_data(key: str): tb_info = get_tunebook_info(key) - r = requests.get(tb_info.url, timeout=5) + r = requests.get(tb_info.url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() html = r.text diff --git a/tests/test_sources.py b/tests/test_sources.py index e785a0a..f502380 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -290,7 +290,7 @@ def test_eskin_tunebook_bad_url_redirects(): # Redirects to the home page. # Nothing in `r.history`. `allow_redirects=False` has no impact. url = "https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov3025.html" - r = requests.head(url, timeout=5) + r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() assert r.status_code == 302 @@ -304,7 +304,7 @@ def test_eskin_tunebook_url_exist(key): import requests url = eskin._TUNEBOOK_KEY_TO_URL[key] - r = requests.head(url, timeout=5) + r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() # Bad URLs seem to just redirect to his homepage, # so we need to check the final URL @@ -320,7 +320,7 @@ def test_eskin_tunebook_url_current(): import requests url = "https://michaeleskin.com/tunebooks.html" - r = requests.get(url, timeout=5) + r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) r.raise_for_status() if ( r.status_code == 302 From ae652b1de8c84cec38b9507e88b26ae9f889f9ac Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:57:59 -0500 Subject: [PATCH 07/17] cov --- pyabc2/sources/eskin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 6dbd3d7..b5cd3d9 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -239,7 +239,7 @@ def _extract_data_from_html_2025(html: str, *, key: str): # First find the tune type options by searching for 'tunes = type;' types = sorted(set(re.findall(r"tunes = (.*?);", html))) if types: - pass + pass # pragma: no cover elif "const tunes=[" in html: # no types, just one list of tunes types = ["tunes"] else: @@ -249,13 +249,13 @@ def _extract_data_from_html_2025(html: str, *, key: str): all_data = {} for type_ in types: m = re.search(rf"const {type_}=\[(.*?)\];", html, flags=re.DOTALL) - if m is None: + if m is None: # pragma: no cover raise RuntimeError(f"Unable to find data for type {type_!r}") s_data = "[" + m.group(1) + "]" try: data = json.loads(s_data) - except json.JSONDecodeError as e: + except json.JSONDecodeError as e: # pragma: no cover w = 25 a = max(0, e.pos - w) b = min(len(s_data), e.pos + w) @@ -302,7 +302,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): tune_name = unescape(tune_name) logger.debug(f"Found group assignment: {tune_id=}, {group_id=}, {tune_name=}") group_assignments[tune_id] = group_id - if not group_assignments: + if not group_assignments: # pragma: no cover logger.debug("No group assignments found in HTML") # Find group names @@ -325,7 +325,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): logger.debug(f"Normalizing group name {group_name!r} to {group_name_norm!r}") group_name = group_name_norm group_names[group_id] = group_name - if not group_names: + if not group_names: # pragma: no cover logger.debug("No group names found in HTML") # Find tunes @@ -350,7 +350,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): all_data = defaultdict(list) for tune in tunes: group_id = group_assignments.get(tune["id"]) - if group_id is None: + if group_id is None: # pragma: no cover logger.warning(f"No group assignment found for tune {tune['id']}") continue group_name = group_names.get(group_id, f"Unknown group {group_id}") From 9eda0da5c2bf7d76e345bfb40504e3e75fb506f1 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 12:58:46 -0500 Subject: [PATCH 08/17] changelog --- docs/changes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changes.md b/docs/changes.md index 95d405f..898d77f 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -1,5 +1,9 @@ # Release notes +## v0.1.4 (unreleased) + +* Updates for Eskin and Bill Black ({pull}`112`) + ## v0.1.3 (2026-04-17) * Improve styling of abcjs containers, pandas dataframes, and ipywidgets From 372b0435fbe727a384e45e20a5f85351bbbfc971 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 13:16:52 -0500 Subject: [PATCH 09/17] Skip blank line (e.g. leading) --- pyabc2/sources/eskin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index b5cd3d9..f90b3a8 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -340,7 +340,7 @@ def _extract_data_from_html_2026(html: str, *, key: str): abc_lines = [] for line in abc_raw.splitlines(): line = line.strip() - if line.startswith("%"): + if not line or line.startswith("%"): continue abc_lines.append(line) From 2e8264ef1a801d3c4e856a0d31030f43a76cd4e8 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 13:50:10 -0500 Subject: [PATCH 10/17] Add retries for Eskin and BB using urllib3 Retry --- pyabc2/sources/bill_black.py | 27 ++++++++++++++++++-- pyabc2/sources/bill_black_tunefolders.py | 5 ++-- pyabc2/sources/eskin.py | 32 +++++++++++++++++++++--- pyproject.toml | 1 + tests/test_sources.py | 22 ++++++++-------- 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 6558e37..5f3872d 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -8,6 +8,7 @@ * `requests `__ """ +import functools import logging import re from pathlib import Path @@ -44,6 +45,28 @@ ] +@functools.lru_cache(1) +def _get_session(): + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=10, + backoff_factor=1.0, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[403, 429, 500, 502, 503, 504], + # Bill Black seems to sporadically return 403 (forbidden) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + + return session + + def download() -> None: """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. @@ -51,10 +74,10 @@ def download() -> None: import zipfile from concurrent.futures import ThreadPoolExecutor - import requests + session = _get_session() def download_one(url): - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() return r.text diff --git a/pyabc2/sources/bill_black_tunefolders.py b/pyabc2/sources/bill_black_tunefolders.py index 324eb5c..f60dd79 100644 --- a/pyabc2/sources/bill_black_tunefolders.py +++ b/pyabc2/sources/bill_black_tunefolders.py @@ -19,6 +19,7 @@ from pathlib import Path from pyabc2._util import get_logger as _get_logger +from pyabc2.sources.bill_black import _get_session logger = _get_logger(__name__) @@ -209,7 +210,7 @@ def get_collection(key: str) -> Collection: def download(key: str | Iterable[str] | None = None) -> None: import gzip - import requests + session = _get_session() SAVE_TO.mkdir(exist_ok=True) @@ -224,7 +225,7 @@ def download(key: str | Iterable[str] | None = None) -> None: for url in collection.abc_urls: p = collection.url_to_file(url) logger.info(f"Downloading {url} to {p.relative_to(HERE).as_posix()}") - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() # Extract filename from URL and append .gz diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index f90b3a8..1065d08 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -7,6 +7,9 @@ * `requests `__ """ +from __future__ import annotations + +import functools import json import re from collections import defaultdict @@ -20,6 +23,7 @@ if TYPE_CHECKING: # pragma: no cover import pandas + import requests logger = _get_logger(__name__) @@ -60,6 +64,28 @@ _URL_NETLOCS = {"michaeleskin.com", "www.michaeleskin.com"} +@functools.lru_cache(1) +def _get_session() -> requests.Session: + import requests + from requests.adapters import HTTPAdapter + from urllib3.util import Retry + + session = requests.Session() + session.headers.update({"User-Agent": "pyabc2"}) + retries = Retry( + total=5, + backoff_factor=0.5, + backoff_jitter=0.5, + allowed_methods={"GET", "HEAD"}, + status_forcelist=[415, 429, 500, 502, 503, 504], + # Eskin seems to sporadically return 415 (unsupported media type) + # possibly to indicate a temporary server issue or throttling/anti-bot + ) + session.mount("https://", HTTPAdapter(max_retries=retries)) + + return session + + def _deflate(s: str, /) -> str: """Use deflate (zlib) to compress and base64-encode `s`.""" import base64 @@ -363,11 +389,11 @@ def _download_data(key: str): """Extract and save the tune data from the tunebook webpage as JSON.""" import gzip - import requests + session = _get_session() tb_info = get_tunebook_info(key) - r = requests.get(tb_info.url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(tb_info.url, timeout=5) r.raise_for_status() html = r.text @@ -389,7 +415,7 @@ def _load_data(key: str): return json.load(f) -def load_meta(key: str, *, redownload: bool = False) -> "pandas.DataFrame": +def load_meta(key: str, *, redownload: bool = False) -> pandas.DataFrame: """Load the tunebook data, no parsing. Parameters diff --git a/pyproject.toml b/pyproject.toml index 76624c2..eea9e44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ sources = [ "numpy ==2.*", "pandas ==2.*", "requests ==2.*", + "urllib3 ==2.*", ] abcjs-widget = [ "anywidget", diff --git a/tests/test_sources.py b/tests/test_sources.py index f502380..0b81512 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -284,13 +284,13 @@ def test_load_url_invalid_domain(): def test_eskin_tunebook_bad_url_redirects(): - import requests + session = eskin._get_session() # Bad URL (2025 -> 3025) # Redirects to the home page. # Nothing in `r.history`. `allow_redirects=False` has no impact. url = "https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov3025.html" - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert r.status_code == 302 @@ -301,10 +301,10 @@ def test_eskin_tunebook_bad_url_redirects(): @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_url_exist(key): - import requests + session = eskin._get_session() url = eskin._TUNEBOOK_KEY_TO_URL[key] - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() # Bad URLs seem to just redirect to his homepage, # so we need to check the final URL @@ -317,10 +317,10 @@ def test_eskin_tunebook_url_exist(key): def test_eskin_tunebook_url_current(): - import requests + session = eskin._get_session() url = "https://michaeleskin.com/tunebooks.html" - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() if ( r.status_code == 302 @@ -496,16 +496,16 @@ def test_eskin_inflate_pad_3(): def test_bill_black_https(): - import requests + session = bill_black._get_session() url = "http://www.capeirish.com/ittl/tunefolders/" url_https = url.replace("http://", "https://") - r = requests.head(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" not in r.headers - r = requests.head(url_https, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.head(url_https, timeout=5) r.raise_for_status() assert "Strict-Transport-Security" in r.headers @@ -523,10 +523,10 @@ def test_bill_black_tunefolders_invalid_key(): def test_bill_black_text_fns(): - import requests + session = bill_black._get_session() url = "http://www.capeirish.com/ittl/alltunes/alltunes-text/" - r = requests.get(url, headers={"User-Agent": "pyabc2"}, timeout=5) + r = session.get(url, timeout=5) r.raise_for_status() fns_web = sorted( From 2acd9d45e50864850a9a02565b46e103977a2819 Mon Sep 17 00:00:00 2001 From: Zachary Moon Date: Fri, 3 Jul 2026 14:11:29 -0500 Subject: [PATCH 11/17] Configure Eskin and BB to retry for `http://` too might as well Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pyabc2/sources/bill_black.py | 1 + pyabc2/sources/eskin.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 5f3872d..fb912a7 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -63,6 +63,7 @@ def _get_session(): # possibly to indicate a temporary server issue or throttling/anti-bot ) session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) return session diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 1065d08..91b4b42 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -82,6 +82,7 @@ def _get_session() -> requests.Session: # possibly to indicate a temporary server issue or throttling/anti-bot ) session.mount("https://", HTTPAdapter(max_retries=retries)) + session.mount("http://", HTTPAdapter(max_retries=retries)) return session From 3cc64b97b49a1c14b6075dc56ee3d926bba7ae7f Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:12:55 -0500 Subject: [PATCH 12/17] Type requests session for BB --- pyabc2/sources/bill_black.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index fb912a7..95ddb1c 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -8,13 +8,19 @@ * `requests `__ """ +from __future__ import annotations + import functools import logging import re from pathlib import Path +from typing import TYPE_CHECKING from pyabc2._util import get_logger as _get_logger +if TYPE_CHECKING: # pragma: no cover + import requests + logger = _get_logger(__name__) HERE = Path(__file__).parent @@ -46,7 +52,7 @@ @functools.lru_cache(1) -def _get_session(): +def _get_session() -> requests.Session: import requests from requests.adapters import HTTPAdapter from urllib3.util import Retry From b27e7b6cf8c17c6649aa911d1395fe2e16c4a2f8 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:21:56 -0500 Subject: [PATCH 13/17] Switch to new parser only for specific message --- pyabc2/sources/eskin.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 91b4b42..94414e7 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -400,8 +400,13 @@ def _download_data(key: str): try: data = _extract_data_from_html_2025(html, key=key) - except RuntimeError: - data = _extract_data_from_html_2026(html, key=key) + except RuntimeError as e: + if str(e) == "Unable to detect tune types": + # We can be pretty sure it's the new format + logger.debug("Tune groups array not detected, assuming new format.") + data = _extract_data_from_html_2026(html, key=key) + else: + raise SAVE_TO.mkdir(exist_ok=True) with gzip.open(tb_info.path, "wt") as f: From 6453cd34532dc64587b0e8727dced1d898da7aeb Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:32:05 -0500 Subject: [PATCH 14/17] BB one Session per thread --- pyabc2/sources/bill_black.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 95ddb1c..7049edd 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -53,6 +53,10 @@ @functools.lru_cache(1) def _get_session() -> requests.Session: + return _build_session() + + +def _build_session() -> requests.Session: import requests from requests.adapters import HTTPAdapter from urllib3.util import Retry @@ -78,12 +82,23 @@ def download() -> None: """Download the alphabetical text files from https://www.capeirish.com/ittl/alltunes/alltunes-text/ and store them in a compressed archive. """ + import threading import zipfile from concurrent.futures import ThreadPoolExecutor - session = _get_session() + thread_local = threading.local() + + def get_worker_session() -> requests.Session: + # One Session per worker thread to avoid cross-thread Session sharing. + try: + return thread_local.session + except AttributeError: + session = _build_session() + thread_local.session = session + return session def download_one(url): + session = get_worker_session() r = session.get(url, timeout=5) r.raise_for_status() return r.text From 440d3c0230885ed503f4c88a304ac3e0d7f4026a Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 14:32:58 -0500 Subject: [PATCH 15/17] Try harder with Eskin just got 415's again in a CI run --- pyabc2/sources/eskin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index 94414e7..c52ef3c 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -73,8 +73,8 @@ def _get_session() -> requests.Session: session = requests.Session() session.headers.update({"User-Agent": "pyabc2"}) retries = Retry( - total=5, - backoff_factor=0.5, + total=10, + backoff_factor=1.0, backoff_jitter=0.5, allowed_methods={"GET", "HEAD"}, status_forcelist=[415, 429, 500, 502, 503, 504], From 77446a0d51a8bad8d4928b7f7fe88cc7907f8d36 Mon Sep 17 00:00:00 2001 From: zmoon Date: Fri, 3 Jul 2026 18:31:04 -0500 Subject: [PATCH 16/17] Try less hard Eskin ended up making a CI run take 149m and still fail --- pyabc2/sources/bill_black.py | 4 ++-- pyabc2/sources/eskin.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyabc2/sources/bill_black.py b/pyabc2/sources/bill_black.py index 7049edd..856e80a 100644 --- a/pyabc2/sources/bill_black.py +++ b/pyabc2/sources/bill_black.py @@ -64,8 +64,8 @@ def _build_session() -> requests.Session: session = requests.Session() session.headers.update({"User-Agent": "pyabc2"}) retries = Retry( - total=10, - backoff_factor=1.0, + total=5, + backoff_factor=0.5, backoff_jitter=0.5, allowed_methods={"GET", "HEAD"}, status_forcelist=[403, 429, 500, 502, 503, 504], diff --git a/pyabc2/sources/eskin.py b/pyabc2/sources/eskin.py index c52ef3c..94414e7 100644 --- a/pyabc2/sources/eskin.py +++ b/pyabc2/sources/eskin.py @@ -73,8 +73,8 @@ def _get_session() -> requests.Session: session = requests.Session() session.headers.update({"User-Agent": "pyabc2"}) retries = Retry( - total=10, - backoff_factor=1.0, + total=5, + backoff_factor=0.5, backoff_jitter=0.5, allowed_methods={"GET", "HEAD"}, status_forcelist=[415, 429, 500, 502, 503, 504], From c4223a77909415a0b6ee5f2d1e11dd47597c662e Mon Sep 17 00:00:00 2001 From: zmoon Date: Sat, 4 Jul 2026 14:25:26 -0500 Subject: [PATCH 17/17] Only run web tests in one CI run and use xdist groups --- .github/workflows/ci.yml | 12 ++++++- pyproject.toml | 5 ++- tests/test_sources.py | 69 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abab5c2..eb30da3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,13 +32,23 @@ jobs: python -m pip install flit flit install -s - - name: Test with pytest + - name: Test with pytest (non-web) + if: ${{ matrix.python-version != '3.14' }} run: > pytest -v -ra --doctest-modules --cov=pyabc2 --cov-report xml --cov-report term-missing + -m "not web" -n auto + - name: Test with pytest (full, including web) + if: ${{ matrix.python-version == '3.14' }} + run: > + pytest -v -ra + --doctest-modules + --cov=pyabc2 --cov-report xml --cov-report term-missing + -n auto --dist loadgroup + - name: Check type annotations run: | mypy --non-interactive . diff --git a/pyproject.toml b/pyproject.toml index eea9e44..f51326a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,10 @@ profile = "black" line_length = 100 [tool.pytest.ini_options] -markers = ["slow"] +markers = [ + "slow", + "web: tests that hit external websites", +] [tool.mypy] exclude = ["^venv"] diff --git a/tests/test_sources.py b/tests/test_sources.py index 0b81512..3d97792 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -44,6 +44,8 @@ def test_example_random(): assert type(tune) is Tune +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_tune_type_file_prefix(): norbeck._maybe_download() all_fps = list(norbeck.SAVE_TO.glob("*.abc")) @@ -58,6 +60,8 @@ def test_norbeck_tune_type_file_prefix(): assert set(all_fps) == all_fps_type_set +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_unique(): # X value should be unique within tune type norbeck._maybe_download() @@ -74,6 +78,8 @@ def test_norbeck_x_unique(): assert len(set(xs)) == len(xs), "X unique" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_x_vals(): # Make sure they match with what file says norbeck._maybe_download() @@ -100,6 +106,8 @@ def test_norbeck_x_vals(): @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") def test_norbeck_load(): # NOTE: downloads files if not already present @@ -131,6 +139,8 @@ def test_norbeck_load(): norbeck.load("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "url,title,key,type", [ @@ -149,11 +159,14 @@ def test_the_session_load_url(url, title, key, type): assert tune.header["reference number"] == "1" +@pytest.mark.xdist_group("the_session") def test_the_session_url_check(): with pytest.raises(AssertionError): the_session.load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_load_archive_threaded(): # NOTE: downloads file if not already present with pytest.warns(UserWarning, match=r"The Session tune\(s\) failed to load"): @@ -163,12 +176,15 @@ def test_the_session_load_archive_threaded(): assert tunes1 == tunes2 +@pytest.mark.xdist_group("the_session") def test_the_session_download_invalid(): with pytest.raises(ValueError): _ = the_session.download("asdf") @pytest.mark.slow +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize( "which", ["aliases", "events", "recordings", "sessions", "sets", "tune_popularity", "tunes"] ) @@ -203,6 +219,7 @@ def test_the_session_load_meta(which): # in df3, `pd.Float64Dtype()` +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_invalid(): with pytest.raises(ValueError): _ = the_session.load_meta("asdf") @@ -211,6 +228,7 @@ def test_the_session_load_meta_invalid(): _ = the_session.load_meta("sessions", format="asdf") +@pytest.mark.xdist_group("the_session") def test_the_session_load_meta_doc_consistency(): s_options = ", ".join(repr(x) for x in sorted(the_session._META_ALLOWED)) expected_line = f"which : {{{s_options}}}" @@ -246,12 +264,16 @@ def test_int_downcast(): assert s3.dtype == expected_dtype_ext +@pytest.mark.web +@pytest.mark.xdist_group("the_session") @pytest.mark.parametrize("netloc", sorted(the_session._URL_NETLOCS)) def test_load_url_the_session(netloc): tune = load_url(f"https://{netloc}/tunes/10000") assert tune.title == "Brian Quinn's" +@pytest.mark.web +@pytest.mark.xdist_group("norbeck") @pytest.mark.parametrize("netloc", sorted(norbeck._URL_NETLOCS)) def test_load_url_norbeck(netloc): import requests @@ -265,6 +287,7 @@ def test_load_url_norbeck(netloc): assert tune.title == "For The Love Of Music" +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("netloc", sorted(eskin._URL_NETLOCS)) @pytest.mark.parametrize("param", list(ESKIN_COMPRESSED_ABC_DATA)) def test_load_url_eskin(netloc, param): @@ -283,6 +306,8 @@ def test_load_url_invalid_domain(): _ = load_url("https://www.google.com") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") def test_eskin_tunebook_bad_url_redirects(): session = eskin._get_session() @@ -299,6 +324,8 @@ def test_eskin_tunebook_bad_url_redirects(): assert r.is_redirect +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_url_exist(key): session = eskin._get_session() @@ -316,6 +343,8 @@ def test_eskin_tunebook_url_exist(key): raise ValueError(f"{key!r} URL {url} redirects to homepage") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") def test_eskin_tunebook_url_current(): session = eskin._get_session() @@ -342,6 +371,8 @@ def test_eskin_tunebook_url_current(): raise ValueError(f"Could not find link for tunebook {key!r} in tunebooks page.") +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("key", eskin._TUNEBOOK_KEY_TO_URL) def test_eskin_tunebook_data_load(key): df = eskin.load_meta(key) @@ -410,6 +441,7 @@ def test_eskin_tunebook_data_load(key): ) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_parsing(): # From https://michaeleskin.com/cce_sd/cce_san_diego_tunes_10nov2025.html url = "https://michaeleskin.com/abctools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" @@ -428,6 +460,7 @@ def test_eskin_abc_url_parsing(): assert sum(line.startswith(r"%%") for line in abc_no_rm.splitlines()) > 0 +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_missing_param(): url = "https://michaeleskin.com/abctools/abctools.html?" with pytest.raises( @@ -437,18 +470,21 @@ def test_eskin_abc_url_missing_param(): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param(): url = "https://michaeleskin.com/abctools/abctools.html?lzw=hi" with pytest.raises(RuntimeError, match="Failed to decompress LZString data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad_param_def(): url = "https://michaeleskin.com/abctools/abctools.html?def=hi" with pytest.raises(RuntimeError, match="Failed to decompress deflate data"): _ = eskin.abctools_url_to_abc(url) +@pytest.mark.xdist_group("eskin") def test_eskin_abc_url_bad(caplog): url = "https://michaeleski.com/deftools/abctools.html?lzw=BoLgjAUApFAuCWsA2BTAZgewHawAQAUBDJQhLDXMADmigGcBXAIwWXWzyJLIrAGZa8LJkw4CxUkN4CYAB0IAnWHVGcJPSjLgoAtrIyrx3KZtqwUAD1iGuk8qYAqIBwAsUuAIJMmKAJ64IABlwAHoaAFkQABYQqIgARRAAJliAXgBOAAYIACUQHJQUJGg6AHchAHNcTIA6SABpEABxaEImAGMAKzoAfToMBiwAE0M0UiYMX1pwgEkAERncWQUMCoVCHWrp+cWmQjo6ZdWtmFmF3HaXDAUho6rs053cPYOANwwkXAA2OMfzy+uQ3enx+rSGQx6xCQPVkJF8e3aAGsekghIi6BAAEQeHSYzx8XAAIU8SVwTQAorgAD6eDxNDy4TFNPGE8HEmnY3G0jzEunkgBi1MZzLJTXpRLZ1KxOLxHlJPJJZMpNI8dIZTJZko5MsVCr5go5IrF4tZQyqVKp0q5KAqttwhFJeyFGtwFTaVUIFRQQ2dOptdodrsIzuZTDdaFd7iGpMtnLx-o9FTDIcxnuTnu9vutto9pLdKbDhAjXqG7IAukA&format=noten&ssp=10&name=The_Abbey&play=1" with caplog.at_level("DEBUG"): @@ -460,6 +496,8 @@ def test_eskin_abc_url_bad(caplog): ] +@pytest.mark.web +@pytest.mark.xdist_group("eskin") @pytest.mark.parametrize("use_lzw", [True, False]) def test_eskin_abc_url_creation(use_lzw): import requests @@ -476,11 +514,13 @@ def test_eskin_abc_url_creation(use_lzw): raise ValueError(f"URL {url} redirects to homepage") +@pytest.mark.xdist_group("eskin") def test_eskin_invalid_tunebook_key(): with pytest.raises(ValueError, match="Unknown Eskin tunebook key: 'asdf'"): _ = eskin.get_tunebook_info("asdf") +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_invalid_length(): s = "eJyFjbEKwkAQRPv9iv2DQyvd7jbGK0wQJIVtktucJ4FIghZyH-abcdefg" with pytest.raises( @@ -490,11 +530,14 @@ def test_eskin_inflate_invalid_length(): _ = eskin._inflate(s) +@pytest.mark.xdist_group("eskin") def test_eskin_inflate_pad_3(): s = "abc" assert eskin._inflate(eskin._deflate(s)) == s +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_https(): session = bill_black._get_session() @@ -511,17 +554,22 @@ def test_bill_black_https(): @pytest.mark.xfail(reason="Bill Black tunefolders are currently in flux", strict=False) +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") @pytest.mark.parametrize("key", list(bill_black_tunefolders._KEY_TO_COLLECTION)) def test_bill_black_tunefolders(key): lst = bill_black_tunefolders.load_meta(key, redownload=True) assert len(lst) > 0 +@pytest.mark.xdist_group("bill_black") def test_bill_black_tunefolders_invalid_key(): with pytest.raises(ValueError, match="Unknown collection key: 'asdf'"): _ = bill_black_tunefolders.get_collection("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_text_fns(): session = bill_black._get_session() @@ -537,17 +585,23 @@ def test_bill_black_text_fns(): assert bill_black.TXT_FNS == fns_web +@pytest.mark.web +@pytest.mark.xdist_group("bill_black") def test_bill_black_load(): lst = bill_black.load_meta() assert len(lst) > 0 assert lst[0].startswith("X:") +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_tune_collections(): df = the_session.get_tune_collections(1) # Cooley's assert not df.empty +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_set(): tunes = the_session.get_member_set(65013, 106212) assert len(tunes) == 3 @@ -557,6 +611,8 @@ def test_the_session_get_member_set(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets(): sets = the_session.get_member_sets(65013) assert len(sets) >= 1 @@ -566,6 +622,8 @@ def test_the_session_get_member_sets(): assert d["setting_id"] == 31341 +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_get_member_sets_multipage(): sets = the_session.get_member_sets(1, pages=3, size=2, max_threads=2, orderby="oldest") assert len(sets) == 6 @@ -575,6 +633,7 @@ def test_the_session_get_member_sets_multipage(): assert d["tune_id"] == 560 +@pytest.mark.xdist_group("the_session") def test_the_session_consume_validation(): f = the_session.get_member_sets @@ -588,11 +647,15 @@ def test_the_session_consume_validation(): _ = f(1, max_threads=0) +@pytest.mark.web +@pytest.mark.xdist_group("the_session") def test_the_session_consume_auto_leading_slash(): (d,) = the_session._consume("tunes/22878") assert d["name"] == "Jack Farrell's" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") @pytest.mark.parametrize("key", list(hardy._TUNEBOOK_KEY_TO_URL)) def test_hardy_load_meta(key): abcs = hardy.load_meta(key) @@ -604,6 +667,8 @@ def test_hardy_load_meta(key): assert "\n\n" not in abc, "no empty lines within a tune block" +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_load_meta_remove_prefs(): # Default: no % lines key = "session" @@ -617,16 +682,20 @@ def test_hardy_load_meta_remove_prefs(): assert sum(line.lstrip().startswith("%") for abc in abcs_raw for line in abc.splitlines()) > 1 +@pytest.mark.xdist_group("hardy") def test_hardy_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.load_meta("asdf") +@pytest.mark.xdist_group("hardy") def test_hardy_download_bad_key(): with pytest.raises(ValueError, match="Unknown Hardy tunebook key"): _ = hardy.download("asdf") +@pytest.mark.web +@pytest.mark.xdist_group("hardy") def test_hardy_annex_is_latest(): """Confirm the hardcoded 'annex' URL points to the current (non-superseded) annex file.""" import requests