Graduate TDLib large-media downloads to production (main account)

docs/operator-workflows.md

@@ -116,3 +116,43 @@ Use subscriber export only with explicit user intent, private local output
 paths, and clear reporting of `visible_count`, `exported_count`, `missing`, and
 completeness caveats. Do not treat a single `get_participants` slice as a full
 export.
+
+## TDLib Large-Media Downloads (main account only)
+
+`tg download` (and `download_post()` under it) can optionally route large
+media downloads on the `main` account through TDLib instead of Telethon,
+based on a measured advantage confirmed in a live POC (+78.7% faster average
+elapsed, resumability confirmed on 3 real files — see
+`mcp/docs/superpowers/specs/2026-07-01-tdlib-large-media-download-design.md`).
+This does **not** change any other Telegram operation or account: reads,
+search, sends, and MCP tool downloads (`download_media_batch`,
+`download_dialog_media`) stay on Telethon unchanged.
+
+The capability stays off until explicitly enabled. Rollout:
+
+1. Install the optional extra: `pip install -e ".[tdlib]"` (from `mcp/`).
+2. Run the one-time interactive login for `main`:
+   ```
+   PYTHONPATH=src .venv/bin/python scripts/tdlib_login.py --phone +<your number>
+   # then, after Telegram sends a code to another active session:
+   PYTHONPATH=src .venv/bin/python scripts/tdlib_login.py --code <code>
+   # only if 2FA is enabled:
+   PYTHONPATH=src .venv/bin/python scripts/tdlib_login.py --password <password>
+   ```
+3. Set in `main`'s env (`~/.telegram-mcp/launchd.env` or `mcp/.env`):
+   ```
+   TELEGRAM_TDLIB_ENABLED=true
+   ```
+   Optional tuning: `TELEGRAM_TDLIB_SESSION_DIR` (default
+   `~/.telegram-mcp-tdlib/main`), `TELEGRAM_TDLIB_DOWNLOAD_THRESHOLD_MB`
+   (default `20`).
+4. Watch telemetry (`download_post_backend` events: `backend`,
+   `route_attempted`, `fallback_reason`) for the backend-used distribution
+   and fallback rate before considering wider rollout (other accounts, lower
+   threshold) — each of those is a separate future decision, not part of
+   this change.
+
+Every TDLib failure mode (session not authorized, network error, unsupported
+content type, lock not acquired within 5s) falls back to Telethon
+automatically using the connection already open for routing — there is no
+new failure mode a user can hit from this change.

experiments/tdlib-media-poc/.env.example

@@ -0,0 +1,3 @@
+# Reuse the same values as mcp/.env (same my.telegram.org app, safe to share).
+TELEGRAM_API_ID=
+TELEGRAM_API_HASH=

experiments/tdlib-media-poc/.gitignore

@@ -0,0 +1,6 @@
+data/
+.env
+__pycache__/
+*.pyc
+.pytest_cache/
+.venv/

experiments/tdlib-media-poc/README.md

@@ -0,0 +1,36 @@
+# TDLib media-download POC
+
+Isolated lab proof of concept required by
+[control-plane/docs/adr/2026-06-21-tdlib-is-not-default-runtime.md](../../control-plane/docs/adr/2026-06-21-tdlib-is-not-default-runtime.md).
+
+Scenario: media download latency/resumability, measured against the current
+`telegram-mcp` (Telethon) path, on the `main` account only.
+
+## Isolation guarantees
+
+- TDLib state lives at `data/tdlib/` (gitignored). It is never pointed at the
+  Telethon session tree (`~/.telegram-mcp/`).
+- Read-only Telegram operations only.
+- Not wired into `mcp/`, `plugin/`, LaunchAgents, or release gates.
+
+## Setup
+
+```bash
+cd experiments/tdlib-media-poc
+uv sync
+cp .env.example .env   # fill in TELEGRAM_API_ID / TELEGRAM_API_HASH from mcp/.env
+```
+
+## Run order
+
+1. `uv run python benchmark/build_benchmark_set.py <t.me-link> [<t.me-link> ...]`
+2. `uv run python benchmark/login_tdlib.py` (one-time, interactive — needs a live login code)
+3. `uv run python benchmark/run_telethon.py`
+4. `uv run python benchmark/run_tdlib.py`
+5. `uv run python benchmark/compare.py` → writes `data/RESULTS.md`
+
+## Tests
+
+```bash
+uv run pytest tests/ -v
+```

experiments/tdlib-media-poc/benchmark/build_benchmark_set.py

@@ -0,0 +1,55 @@
+"""Build data/benchmark_set.json from operator-supplied t.me links.
+
+Usage:
+    uv run python benchmark/build_benchmark_set.py <t.me-link> [<t.me-link> ...]
+
+For each link this shells out to the existing `tg message` CLI (read-only)
+to fetch file_size metadata, then writes the isolated benchmark set used by
+both the Telethon and TDLib benchmark runners.
+"""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from benchmark.models import BenchmarkTarget, save_benchmark_set
+from benchmark.select_targets import extract_file_size, parse_link
+
+POC_ROOT = Path(__file__).resolve().parent.parent
+TG_CLI = POC_ROOT.parent.parent / "mcp" / "bin" / "tg"
+OUTPUT_PATH = POC_ROOT / "data" / "benchmark_set.json"
+
+
+def fetch_metadata(chat: str, message_id: int) -> dict:
+    proc = subprocess.run(
+        [str(TG_CLI), "message", chat, str(message_id), "--json", "--account", "main"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return json.loads(proc.stdout)
+
+
+def main(links: list[str]) -> None:
+    targets = []
+    for index, link in enumerate(links, start=1):
+        chat, message_id = parse_link(link)
+        envelope = fetch_metadata(chat, message_id)
+        targets.append(
+            BenchmarkTarget(
+                label=f"target-{index}",
+                chat=chat,
+                message_id=message_id,
+                link=link,
+                expected_size_bytes=extract_file_size(envelope),
+            )
+        )
+    save_benchmark_set(OUTPUT_PATH, targets)
+    print(f"Wrote {len(targets)} targets to {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        raise SystemExit("usage: build_benchmark_set.py <t.me-link> [<t.me-link> ...]")
+    main(sys.argv[1:])

experiments/tdlib-media-poc/benchmark/compare.py

@@ -0,0 +1,68 @@
+"""Compare Telethon vs TDLib download benchmark results and write RESULTS.md.
+
+Live usage:
+    uv run python benchmark/compare.py
+"""
+
+from pathlib import Path
+from statistics import mean
+
+from benchmark.models import DownloadResult, load_results
+
+POC_ROOT = Path(__file__).resolve().parent.parent
+TELETHON_RESULTS_PATH = POC_ROOT / "data" / "results_telethon.json"
+TDLIB_RESULTS_PATH = POC_ROOT / "data" / "results_tdlib.json"
+REPORT_PATH = POC_ROOT / "data" / "RESULTS.md"
+
+
+def build_report(telethon_results: list[DownloadResult], tdlib_results: list[DownloadResult]) -> str:
+    lines = [
+        "# TDLib vs Telethon: media download latency/resumability POC results",
+        "",
+        "| label | backend | ok | elapsed_seconds | bytes_downloaded | resumed |",
+        "|---|---|---|---|---|---|",
+    ]
+    for result in [*telethon_results, *tdlib_results]:
+        lines.append(
+            f"| {result.label} | {result.backend} | {result.ok} | {result.elapsed_seconds:.2f} "
+            f"| {result.bytes_downloaded} | {result.resumed} |"
+        )
+
+    telethon_ok = [r for r in telethon_results if r.ok]
+    tdlib_ok = [r for r in tdlib_results if r.ok]
+    lines.append("")
+    if telethon_ok and tdlib_ok:
+        telethon_avg = mean(r.elapsed_seconds for r in telethon_ok)
+        tdlib_avg = mean(r.elapsed_seconds for r in tdlib_ok)
+        delta_pct = ((telethon_avg - tdlib_avg) / telethon_avg) * 100
+        lines.append(
+            f"Average elapsed: telethon={telethon_avg:.2f}s, tdlib={tdlib_avg:.2f}s "
+            f"({delta_pct:+.1f}% telethon vs tdlib)."
+        )
+    else:
+        lines.append("Not enough successful runs on both backends to compare averages.")
+
+    tdlib_resumed = any(r.resumed and r.ok for r in tdlib_results)
+    lines.append(f"TDLib demonstrated successful resume after interruption: {tdlib_resumed}.")
+
+    lines.append("")
+    lines.append("## ADR kill-criteria checklist (manual assessment)")
+    lines.append("- [ ] Required sharing/converting Telethon session files? (must be No)")
+    lines.append("- [ ] Auth/DB/update-loop code became the main work? (must be No)")
+    lines.append("- [ ] No clear measured advantage over telegram-mcp? (see averages above)")
+    lines.append("- [ ] Read behavior diverged from telegram-mcp in a way agents would need to understand? (must be No)")
+    lines.append("- [ ] Required new persistent daemon management before proving value? (must be No)")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    telethon_results = load_results(TELETHON_RESULTS_PATH)
+    tdlib_results = load_results(TDLIB_RESULTS_PATH)
+    report = build_report(telethon_results, tdlib_results)
+    REPORT_PATH.write_text(report)
+    print(f"Wrote {REPORT_PATH}")
+
+
+if __name__ == "__main__":
+    main()

experiments/tdlib-media-poc/benchmark/login_tdlib.py

@@ -0,0 +1,105 @@
+"""Interactive-by-invocation TDLib login for the isolated media-download POC.
+
+pytdbot's Client.start() only auto-drives the authorization state machine
+for bot-token logins (see pytdbot.Client.__handle_authorization_state_wait_phone_number,
+which no-ops unless a bot token is set). For a real user account it does
+nothing at authorizationStateWaitPhoneNumber, so this script drives the
+state machine manually across multiple invocations, since a live phone/SMS
+code can't be fed into a blocking input() call from outside a real TTY:
+
+    uv run python benchmark/login_tdlib.py --phone +15551234567
+    # (Telegram sends a login code to your other active sessions)
+    uv run python benchmark/login_tdlib.py --code 12345
+    # (only if the account has 2FA enabled)
+    uv run python benchmark/login_tdlib.py --password ...
+
+Each invocation reconnects to the same persistent session under data/tdlib/,
+fully separate from the Telethon session telegram-mcp owns.
+"""
+
+import argparse
+import asyncio
+import os
+from pathlib import Path
+
+import pytdbot
+from dotenv import load_dotenv
+
+from benchmark.tdlib_client import build_client
+
+POC_ROOT = Path(__file__).resolve().parent.parent
+DATA_DIR = POC_ROOT / "data" / "tdlib"
+
+_STATE_TO_REQUIRED_ARG = {
+    "authorizationStateWaitPhoneNumber": "phone",
+    "authorizationStateWaitCode": "code",
+    "authorizationStateWaitPassword": "password",
+}
+
+
+def required_arg_for_state(state: str) -> str | None:
+    return _STATE_TO_REQUIRED_ARG.get(state)
+
+
+def raise_if_error(result):
+    if isinstance(result, pytdbot.types.Error):
+        raise RuntimeError(f"TDLib error {result['code']}: {result['message']}")
+    return result
+
+
+async def wait_for_stable_state(client: pytdbot.Client) -> str:
+    for _ in range(20):
+        state = client.authorization_state
+        if state and state != "authorizationStateWaitTdlibParameters":
+            return state
+        await asyncio.sleep(0.5)
+    return client.authorization_state
+
+
+async def main(phone: str | None, code: str | None, password: str | None) -> None:
+    load_dotenv(POC_ROOT / ".env")
+    api_id = int(os.environ["TELEGRAM_API_ID"])
+    api_hash = os.environ["TELEGRAM_API_HASH"]
+
+    client = build_client(api_id=api_id, api_hash=api_hash, files_directory=str(DATA_DIR))
+    await client.start(wait_login=False)
+
+    state = await wait_for_stable_state(client)
+    print(f"Current authorization state: {state}")
+
+    required_arg = required_arg_for_state(state)
+    supplied = {"phone": phone, "code": code, "password": password}.get(required_arg)
+
+    if required_arg and not supplied:
+        print(
+            f"Need --{required_arg}. Re-run: "
+            f"uv run python benchmark/login_tdlib.py --{required_arg} <value>"
+        )
+        await client.stop()
+        return
+
+    if state == "authorizationStateWaitPhoneNumber":
+        raise_if_error(await client.setAuthenticationPhoneNumber(phone_number=phone))
+        print("Code requested. Check Telegram on your other devices, then run with --code.")
+    elif state == "authorizationStateWaitCode":
+        raise_if_error(await client.checkAuthenticationCode(code=code))
+        print("Code accepted.")
+    elif state == "authorizationStateWaitPassword":
+        raise_if_error(await client.checkAuthenticationPassword(password=password))
+        print("Password accepted.")
+    elif state == "authorizationStateReady":
+        me = raise_if_error(await client.getMe())
+        print(f"Already logged in: {me}")
+
+    await asyncio.sleep(1)
+    print(f"Authorization state now: {client.authorization_state}")
+    await client.stop()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--phone")
+    parser.add_argument("--code")
+    parser.add_argument("--password")
+    args = parser.parse_args()
+    asyncio.run(main(args.phone, args.code, args.password))

experiments/tdlib-media-poc/benchmark/models.py

@@ -0,0 +1,45 @@
+"""Data models for the TDLib vs Telethon media-download benchmark."""
+
+import json
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class BenchmarkTarget:
+    label: str
+    chat: str
+    message_id: int
+    link: str
+    expected_size_bytes: int | None
+
+
+@dataclass(frozen=True)
+class DownloadResult:
+    label: str
+    backend: str  # "telethon" or "tdlib"
+    ok: bool
+    elapsed_seconds: float
+    bytes_downloaded: int | None
+    resumed: bool
+    error: str | None = None
+
+
+def load_benchmark_set(path: Path) -> list[BenchmarkTarget]:
+    raw = json.loads(path.read_text())
+    return [BenchmarkTarget(**item) for item in raw]
+
+
+def save_benchmark_set(path: Path, targets: list[BenchmarkTarget]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps([asdict(t) for t in targets], indent=2))
+
+
+def load_results(path: Path) -> list[DownloadResult]:
+    raw = json.loads(path.read_text())
+    return [DownloadResult(**item) for item in raw]
+
+
+def save_results(path: Path, results: list[DownloadResult]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps([asdict(r) for r in results], indent=2))