chore: add ruff + mypy + CI tooling (and fix a pyarrow NameError)

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  push:
+    branches: [dev, main]
+  pull_request:
+    branches: [dev, main]
+
+jobs:
+  lint:
+    name: ruff (lint)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - run: pip install ruff==0.15.21
+      - run: ruff check src tests
+
+  typecheck:
+    name: mypy (advisory)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - run: pip install mypy
+      # Advisory: the codebase is partially typed. Print findings but never
+      # fail the build (green check) until type coverage improves. Tighten by
+      # dropping the `|| true` once the baseline is cleaned up.
+      - run: mypy src/gasbench --ignore-missing-imports || true

pyproject.toml

@@ -41,6 +41,13 @@ dependencies = [
     "open-clip-torch==3.3.0",
 ]
 
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "ruff==0.15.21",
+    "mypy>=1.10",
+]
+
 [project.scripts]
 gasbench = "gasbench.cli:main"
 
@@ -57,6 +64,23 @@ where = ["src"]
 line-length = 88
 target-version = ['py310']
 
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+# High-signal baseline: pyflakes (real bugs) + import/statement errors.
+# Style rules (line length, naming) are intentionally left off for now to keep
+# the diff focused; expand this list over time.
+select = ["E4", "E7", "E9", "F"]
+
+[tool.mypy]
+python_version = "3.10"
+ignore_missing_imports = true
+follow_imports = "silent"
+# Advisory only for now — the codebase is partially typed. Not enforced in CI.
+
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 markers = ["slow: marks tests as slow (network I/O)"]

src/gasbench/benchmark.py

@@ -3,7 +3,6 @@
 import json
 import time
 import uuid
-import asyncio
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, Optional
@@ -181,7 +180,7 @@ async def load_model_for_benchmark(
         benchmark_results["validation"]["input_type"] = str(input_specs[0].type)
         benchmark_results["validation"]["output_shape"] = str(output_specs[0].shape)
 
-        logger.info(f"Model loaded successfully")
+        logger.info("Model loaded successfully")
 
         # Get execution providers
         providers = session.get_providers()
@@ -237,7 +236,7 @@ async def execute_benchmark(
     if dataset_filters:
         logger.info(f"Dataset filters: {dataset_filters}")
     if modality == "image":
-        df = await run_image_benchmark(
+        await run_image_benchmark(
             session,
             input_specs,
             benchmark_results,
@@ -264,7 +263,7 @@ async def execute_benchmark(
         )
         benchmark_score = benchmark_results.get("image_results", {}).get("benchmark_score", 0.0)
     elif modality == "video":
-        df = await run_video_benchmark(
+        await run_video_benchmark(
             session,
             input_specs,
             benchmark_results,
@@ -291,7 +290,7 @@ async def execute_benchmark(
         )
         benchmark_score = benchmark_results.get("video_results", {}).get("benchmark_score", 0.0)
     elif modality == "audio":
-        df = await run_audio_benchmark(
+        await run_audio_benchmark(
             session,
             input_specs,
             benchmark_results,
@@ -375,7 +374,7 @@ def format_benchmark_summary(benchmark_results: Dict) -> str:
 
             per_dataset = results.get("per_dataset_results", {})
             if per_dataset:
-                lines.append(f"\n📋 PER-DATASET RESULTS:")
+                lines.append("\n📋 PER-DATASET RESULTS:")
                 for dataset_name, dataset_results in per_dataset.items():
                     accuracy = dataset_results.get("accuracy", 0.0)
                     total = dataset_results.get("total", 0)
@@ -394,7 +393,7 @@ def format_benchmark_summary(benchmark_results: Dict) -> str:
                     entry["samples"] += int(ds_stats.get("total", 0))
                     entry["correct"] += int(ds_stats.get("correct", 0))
                 if by_type:
-                    lines.append(f"\n🎭 ACCURACY BY MEDIA TYPE:")
+                    lines.append("\n🎭 ACCURACY BY MEDIA TYPE:")
                     for mtype, v in by_type.items():
                         samples = v["samples"]
                         correct = v["correct"]
@@ -403,7 +402,7 @@ def format_benchmark_summary(benchmark_results: Dict) -> str:
 
     validation = benchmark_results.get("validation", {})
     if validation:
-        lines.append(f"\n🔍 MODEL VALIDATION:")
+        lines.append("\n🔍 MODEL VALIDATION:")
         lines.append(f"  Input Shape: {validation.get('input_shape', 'N/A')}")
         lines.append(f"  Input Type: {validation.get('input_type', 'N/A')}")
         lines.append(f"  Output Shape: {validation.get('output_shape', 'N/A')}")

src/gasbench/benchmarks/utils/metrics.py

@@ -117,7 +117,6 @@ def compute_sn34_score(
         Returns:
             float in [0,1].
         """
-        import math
 
         # Normalize MCC: [-1, 1] -> [0, 1]
         bin_mcc = float(self.calculate_binary_mcc())

src/gasbench/benchmarks/utils/pytorch_session.py

@@ -4,9 +4,9 @@
 import torch
 import numpy as np
 from pathlib import Path
-from typing import List, Any, Optional
+from typing import List, Any
 
-from .custom_model_loader import load_custom_model, load_model_config
+from .custom_model_loader import load_custom_model
 from ...logger import get_logger
 
 logger = get_logger(__name__)

src/gasbench/benchmarks/video_bench.py

@@ -1,5 +1,4 @@
 import os
-import asyncio
 import numpy as np
 import traceback
 from typing import Dict, Optional
@@ -352,8 +351,6 @@ async def run_video_benchmark(
                     f"{dataset_config.name} ({dataset_cap} samples)"
                 )
 
-                dataset_skipped = 0
-
                 try:
                     is_gasstation = "gasstation" in dataset_config.name.lower()
                     if skip_missing:

src/gasbench/cli.py

@@ -101,15 +101,15 @@ def command_run(args):
         print(f"📦 Using custom PyTorch model from: {model_path}")
     elif model_path.suffix.lower() == ".onnx":
         # ONNX format no longer supported for competition
-        print(f"Error: ONNX format is no longer supported for competition.")
-        print(f"Please use a safetensors model directory containing:")
-        print(f"  - model_config.yaml")
-        print(f"  - model.py")
-        print(f"  - *.safetensors weights")
-        print(f"See docs/Safetensors.md for requirements.")
+        print("Error: ONNX format is no longer supported for competition.")
+        print("Please use a safetensors model directory containing:")
+        print("  - model_config.yaml")
+        print("  - model.py")
+        print("  - *.safetensors weights")
+        print("See docs/Safetensors.md for requirements.")
         return 1
     else:
-        print(f"Error: Model must be a directory with model_config.yaml, model.py, and *.safetensors")
+        print("Error: Model must be a directory with model_config.yaml, model.py, and *.safetensors")
         print(f"Got: {model_path}")
         return 1
 
@@ -182,9 +182,9 @@ def command_run(args):
             f.write(format_benchmark_summary(results))
 
         print(f"\n📊 Results saved to: {results_path}")
-        print(f"  - JSON:    results.json")
-        print(f"  - Parquet: records.parquet")
-        print(f"  - Summary: summary.txt")
+        print("  - JSON:    results.json")
+        print("  - Parquet: records.parquet")
+        print("  - Summary: summary.txt")
 
         if results.get("benchmark_completed"):
             print("\n✅ Benchmark completed successfully")
@@ -207,7 +207,6 @@ def command_verify_cache(args):
     from .dataset.cache import (
         scan_cache_directory,
         compute_cache_statistics,
-        verify_cache_against_configs,
         format_size_bytes,
     )
 
@@ -230,27 +229,27 @@ def command_verify_cache(args):
 
     by_modality, by_media_type = compute_cache_statistics(datasets)
 
-    print(f"\n📊 CACHE SUMMARY")
+    print("\n📊 CACHE SUMMARY")
     print(f"Total Datasets: {len(datasets)}")
     print(f"Total Samples:  {total_samples:,}")
     print(f"Total Size:     {format_size_bytes(total_size_bytes)}")
 
-    print(f"\n📈 BY MODALITY")
+    print("\n📈 BY MODALITY")
     for modality in sorted(by_modality.keys()):
         stats = by_modality[modality]
         print(
             f"  {modality.upper():12} {stats['count']:3} datasets, {stats['samples']:7,} samples, {format_size_bytes(stats['size']):>10}"
         )
 
-    print(f"\n🎭 BY MEDIA TYPE")
+    print("\n🎭 BY MEDIA TYPE")
     for mtype in sorted(by_media_type.keys()):
         stats = by_media_type[mtype]
         print(
             f"  {mtype:15} {stats['count']:3} datasets, {stats['samples']:7,} samples, {format_size_bytes(stats['size']):>10}"
         )
 
     if args.verbose:
-        print(f"\n📋 DETAILED LISTING")
+        print("\n📋 DETAILED LISTING")
 
         datasets_by_mod = defaultdict(list)
         for ds in datasets:
@@ -304,7 +303,7 @@ def _verify_against_config(cached_names, cached_datasets, args, cache_dir):
 
     if missing:
         has_issues = True
-        print(f"\n❌ MISSING DATASETS (in config but not in cache):")
+        print("\n❌ MISSING DATASETS (in config but not in cache):")
         by_modality = defaultdict(list)
         for ds in expected_datasets:
             if ds.name in missing:
@@ -318,7 +317,7 @@ def _verify_against_config(cached_names, cached_datasets, args, cache_dir):
 
     if extra:
         has_issues = True
-        print(f"\n⚠️  EXTRA DATASETS (in cache but not in config):")
+        print("\n⚠️  EXTRA DATASETS (in cache but not in config):")
         by_modality = defaultdict(list)
         for ds in cached_datasets:
             if ds["name"] in extra:

src/gasbench/dataset/cache.py

@@ -32,7 +32,7 @@ def check_dataset_cache(
         ):
             try:
                 with open(dataset_info_file, "r") as f:
-                    dataset_info = json.load(f)
+                    json.load(f)  # validate JSON is readable
 
                 sample_count = len(
                     [

src/gasbench/dataset/config.py

@@ -2,10 +2,8 @@
 from typing import Dict, List, Optional, Tuple
 from pathlib import Path
 import os
-import json
 import yaml
 from importlib.resources import files
-from collections import defaultdict
 import hashlib
 
 from ..logger import get_logger
@@ -404,7 +402,7 @@ def load_datasets_from_yaml(yaml_path: str) -> Dict[str, List[BenchmarkDatasetCo
         raise yaml.YAMLError(f"Failed to parse YAML file {yaml_path}: {e}")
 
     if not isinstance(data, dict):
-        raise ValueError(f"YAML file must contain a dictionary at root level")
+        raise ValueError("YAML file must contain a dictionary at root level")
 
     all_errors = []
     result = {"image": [], "video": [], "audio": []}

src/gasbench/dataset/download.py

@@ -21,6 +21,7 @@
 import soundfile as sf
 
 import pandas as pd
+import pyarrow
 import pyarrow.parquet as pq
 import requests
 import huggingface_hub as hf_hub
@@ -223,7 +224,6 @@ def download_and_extract(
                     f"No files found for {dataset.path} with format {dataset.source_format}"
                 )
 
-                src_fmt = str(getattr(dataset, "source_format", "")).lower().lstrip(".")
                 fallback_formats = [".parquet", ".zip", ".tar", ".tar.gz"]
                 for fallback_format in fallback_formats:
                     if fallback_format != dataset.source_format:
@@ -619,7 +619,6 @@ def _process_gasstation(
 
     # Track parquet files we've already processed
     if downloaded_archives is not None:
-        parquet_basenames = {os.path.basename(url) for url in parquet_urls}
         parquets_to_download = [
             url
             for url in parquet_urls
@@ -971,14 +970,18 @@ def _process_zip_or_tar(
 
             if is_zip:
                 list_entries = archive.namelist()
-                get_name = lambda e: e
+
+                def get_name(e):
+                    return e
 
                 def open_entry(e):
                     return archive.open(e)
 
             else:
                 list_entries = [m for m in archive.getmembers() if m.isreg()]
-                get_name = lambda m: m.name
+
+                def get_name(m):
+                    return m.name
 
                 def open_entry(m):
                     return archive.extractfile(m)
@@ -1507,7 +1510,7 @@ def list_modelscope_files(repo_id, extension=None):
             f"Failed to list files of type {extension} in ModelScope repo {repo_id}: {e}"
         )
         logger.error(
-            f"Make sure 'modelscope' package is installed: pip install modelscope"
+            "Make sure 'modelscope' package is installed: pip install modelscope"
         )
 
     return files

src/gasbench/dataset/iterator.py

@@ -8,7 +8,6 @@
 from pathlib import Path
 from typing import Optional, Dict
 
-from PIL import Image
 
 from ..logger import get_logger
 from .config import BenchmarkDatasetConfig

src/gasbench/dataset/utils/gasstation_utils.py

@@ -9,7 +9,7 @@
 
 import os
 import json
-from typing import List, Set, Optional, Dict, Tuple
+from typing import List, Set, Optional, Tuple
 from pathlib import Path
 from datetime import datetime, timedelta
 

src/gasbench/download_manager.py

@@ -149,7 +149,7 @@ async def _download_dataset(self, task: DatasetTask, progress: Progress):
             await asyncio.sleep(0.3)
             progress.remove_task(task_id)
 
-        except Exception as e:
+        except Exception:
             progress.update(task_id, description=f"[red]❌ {task.dataset.name[:45]}")
             await asyncio.sleep(0.3)
             progress.remove_task(task_id)