Release 0.8.0

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  push:
+    branches: [dev, main]
+  pull_request:
+    branches: [dev, main]
+
+jobs:
+  lint:
+    name: ruff (lint)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - run: pip install ruff==0.15.21
+      - run: ruff check src tests
+
+  typecheck:
+    name: mypy (advisory)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - run: pip install mypy
+      # Advisory: the codebase is partially typed. Print findings but never
+      # fail the build (green check) until type coverage improves. Tighten by
+      # dropping the `|| true` once the baseline is cleaned up.
+      - run: mypy src/gasbench --ignore-missing-imports || true

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gasbench"
-version = "0.7.5"
+version = "0.8.0"
 description = "GASBench - ML model benchmark evaluation package"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -41,6 +41,13 @@ dependencies = [
     "open-clip-torch==3.3.0",
 ]
 
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "ruff==0.15.21",
+    "mypy>=1.10",
+]
+
 [project.scripts]
 gasbench = "gasbench.cli:main"
 
@@ -57,6 +64,23 @@ where = ["src"]
 line-length = 88
 target-version = ['py310']
 
+[tool.ruff]
+line-length = 88
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+# High-signal baseline: pyflakes (real bugs) + import/statement errors.
+# Style rules (line length, naming) are intentionally left off for now to keep
+# the diff focused; expand this list over time.
+select = ["E4", "E7", "E9", "F"]
+
+[tool.mypy]
+python_version = "3.10"
+ignore_missing_imports = true
+follow_imports = "silent"
+# Advisory only for now — the codebase is partially typed. Not enforced in CI.
+
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 markers = ["slow: marks tests as slow (network I/O)"]

src/gasbench/benchmark.py

@@ -3,7 +3,6 @@
 import json
 import time
 import uuid
-import asyncio
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, Optional
@@ -181,7 +180,7 @@ async def load_model_for_benchmark(
         benchmark_results["validation"]["input_type"] = str(input_specs[0].type)
         benchmark_results["validation"]["output_shape"] = str(output_specs[0].shape)
 
-        logger.info(f"Model loaded successfully")
+        logger.info("Model loaded successfully")
 
         # Get execution providers
         providers = session.get_providers()
@@ -237,7 +236,7 @@ async def execute_benchmark(
     if dataset_filters:
         logger.info(f"Dataset filters: {dataset_filters}")
     if modality == "image":
-        df = await run_image_benchmark(
+        await run_image_benchmark(
             session,
             input_specs,
             benchmark_results,
@@ -264,7 +263,7 @@ async def execute_benchmark(
         )
         benchmark_score = benchmark_results.get("image_results", {}).get("benchmark_score", 0.0)
     elif modality == "video":
-        df = await run_video_benchmark(
+        await run_video_benchmark(
             session,
             input_specs,
             benchmark_results,
@@ -291,7 +290,7 @@ async def execute_benchmark(
         )
         benchmark_score = benchmark_results.get("video_results", {}).get("benchmark_score", 0.0)
     elif modality == "audio":
-        df = await run_audio_benchmark(
+        await run_audio_benchmark(
             session,
             input_specs,
             benchmark_results,
@@ -375,7 +374,7 @@ def format_benchmark_summary(benchmark_results: Dict) -> str:
 
             per_dataset = results.get("per_dataset_results", {})
             if per_dataset:
-                lines.append(f"\n📋 PER-DATASET RESULTS:")
+                lines.append("\n📋 PER-DATASET RESULTS:")
                 for dataset_name, dataset_results in per_dataset.items():
                     accuracy = dataset_results.get("accuracy", 0.0)
                     total = dataset_results.get("total", 0)
@@ -394,7 +393,7 @@ def format_benchmark_summary(benchmark_results: Dict) -> str:
                     entry["samples"] += int(ds_stats.get("total", 0))
                     entry["correct"] += int(ds_stats.get("correct", 0))
                 if by_type:
-                    lines.append(f"\n🎭 ACCURACY BY MEDIA TYPE:")
+                    lines.append("\n🎭 ACCURACY BY MEDIA TYPE:")
                     for mtype, v in by_type.items():
                         samples = v["samples"]
                         correct = v["correct"]
@@ -403,7 +402,7 @@ def format_benchmark_summary(benchmark_results: Dict) -> str:
 
     validation = benchmark_results.get("validation", {})
     if validation:
-        lines.append(f"\n🔍 MODEL VALIDATION:")
+        lines.append("\n🔍 MODEL VALIDATION:")
         lines.append(f"  Input Shape: {validation.get('input_shape', 'N/A')}")
         lines.append(f"  Input Type: {validation.get('input_type', 'N/A')}")
         lines.append(f"  Output Shape: {validation.get('output_shape', 'N/A')}")

src/gasbench/benchmarks/audio_bench.py

@@ -1,16 +1,19 @@
 import os
-import time
-import traceback
 import numpy as np
 from typing import Dict, Optional
 
 from ..logger import get_logger
 from ..processing.media import process_audio_sample
 from ..dataset.iterator import DatasetIterator
 
-from .utils.inference import process_model_output
 from .recording import BenchmarkRunRecorder, log_dataset_summary
-from .common import BenchmarkRunConfig, build_plan, create_tracker, finalize_run
+from .common import (
+    BenchmarkRunConfig,
+    build_plan,
+    create_tracker,
+    finalize_run,
+    run_batch_and_record,
+)
 import pandas as pd
 
 logger = get_logger(__name__)
@@ -40,49 +43,22 @@ def process_batch(
 
         # Squeeze any extra dimensions (e.g., (1, 96000) -> (96000,))
         batch_audio_np = [b.squeeze() if b.ndim > 1 else b for b in batch_audio_np]
-        
+
         batch_array = np.stack(batch_audio_np)
     except Exception as e:
         logger.error(f"Failed to stack audio batch: {e}")
-        return
-
-    start = time.time()
-    outputs = None
-    try:
-        outputs = session.run(None, {input_specs[0].name: batch_array})
-    except Exception as e:
-        logger.error(f"Inference failed: {e} (batch shape: {batch_array.shape})")
-        for i, (label, sample, sample_index, dataset_name, sample_seed) in enumerate(
-            batch_metadata
-        ):
+        for label, sample, sample_index, dataset_name, sample_seed in batch_metadata:
             tracker.add_error(
                 dataset_name=dataset_name,
                 sample_index=sample_index,
                 sample=sample,
-                error_message=f"inference-failed: {str(e)[:160]}",
+                error_message=f"stack-failed: {str(e)[:160]}",
             )
         return
-    batch_inference_time = (time.time() - start) * 1000
-    per_sample_time = batch_inference_time / len(batch_audio)
-
-    for i, (label, sample, sample_index, dataset_name, sample_seed) in enumerate(
-        batch_metadata
-    ):
-        predicted, pred_probs = process_model_output(outputs[0][i])
-
-        tracker.add_ok(
-            dataset_name=dataset_name,
-            sample_index=sample_index,
-            sample=sample,
-            label=label,
-            predicted=predicted,
-            probs=pred_probs,
-            inference_time_ms=per_sample_time,
-            batch_inference_time_ms=batch_inference_time,
-            batch_id=batch_id,
-            batch_size=len(batch_audio),
-            sample_seed=sample_seed,
-        )
+
+    run_batch_and_record(
+        session, input_specs, batch_array, batch_metadata, tracker, batch_id
+    )
 
 
 async def run_audio_benchmark(

src/gasbench/benchmarks/common.py

@@ -1,7 +1,10 @@
 import json
+import time
 from dataclasses import dataclass
 from typing import Dict, List, Optional, Tuple
 
+import numpy as np
+
 from ..config import DEFAULT_TARGET_SIZE
 from ..dataset.config import (
     get_benchmark_size,
@@ -19,9 +22,75 @@
     compute_generator_stats_from_df,
 )
 from .utils import calculate_per_source_accuracy
+from .utils.inference import process_model_output
 from ..logger import get_logger
 
 
+def stack_uniform_batch(items: List[np.ndarray]) -> np.ndarray:
+    """Stack a list of same-shaped arrays into a single batched array."""
+    first = items[0]
+    batch_array = np.empty((len(items),) + first.shape, dtype=first.dtype)
+    for i, item in enumerate(items):
+        batch_array[i] = item
+    return batch_array
+
+
+def run_batch_and_record(
+    session,
+    input_specs,
+    batch_array: np.ndarray,
+    batch_metadata,
+    tracker: BenchmarkRunRecorder,
+    batch_id: int,
+    aug_pass: bool = False,
+):
+    """Run one batch through the model and record rows in the tracker.
+
+    On inference failure every sample in the batch is recorded as an error
+    (rather than propagating and aborting the whole dataset), so a model that
+    crashes on hard samples cannot silently drop them from its score.
+    """
+    logger = get_logger(__name__)
+    if not batch_metadata:
+        return
+
+    start = time.time()
+    try:
+        outputs = session.run(None, {input_specs[0].name: batch_array})
+    except Exception as e:
+        logger.error(f"Inference failed: {e} (batch shape: {batch_array.shape})")
+        for label, sample, sample_index, dataset_name, sample_seed in batch_metadata:
+            tracker.add_error(
+                dataset_name=dataset_name,
+                sample_index=sample_index,
+                sample=sample,
+                error_message=f"inference-failed: {str(e)[:160]}",
+            )
+        return
+
+    batch_inference_time = (time.time() - start) * 1000
+    per_sample_time = batch_inference_time / len(batch_metadata)
+
+    for i, (label, sample, sample_index, dataset_name, sample_seed) in enumerate(
+        batch_metadata
+    ):
+        predicted, pred_probs = process_model_output(outputs[0][i])
+        tracker.add_ok(
+            dataset_name=dataset_name,
+            sample_index=sample_index,
+            sample=sample,
+            label=label,
+            predicted=predicted,
+            probs=pred_probs,
+            inference_time_ms=per_sample_time,
+            batch_inference_time_ms=batch_inference_time,
+            batch_id=batch_id,
+            batch_size=len(batch_metadata),
+            sample_seed=sample_seed,
+            aug_pass=aug_pass,
+        )
+
+
 @dataclass
 class BenchmarkRunConfig:
     modality: str

src/gasbench/benchmarks/image_bench.py

@@ -1,5 +1,4 @@
 import os
-import time
 import traceback
 import numpy as np
 from typing import Dict, Optional
@@ -19,9 +18,15 @@
 
 from ..dataset.iterator import DatasetIterator
 
-from .utils.inference import process_model_output
 from .recording import BenchmarkRunRecorder, log_dataset_summary, build_sample_id
-from .common import BenchmarkRunConfig, build_plan, create_tracker, finalize_run
+from .common import (
+    BenchmarkRunConfig,
+    build_plan,
+    create_tracker,
+    finalize_run,
+    stack_uniform_batch,
+    run_batch_and_record,
+)
 from .aug_cache import img_aug_cache_path, write_aug_cache
 import pandas as pd
 
@@ -219,36 +224,10 @@ def process_batch(
     """push a batch of images through the model and record rows in tracker."""
     if not batch_images:
         return
-
-    first = batch_images[0]
-    batch_array = np.empty((len(batch_images),) + first.shape, dtype=first.dtype)
-    for i, img in enumerate(batch_images):
-        batch_array[i] = img
-
-    start = time.time()
-    outputs = session.run(None, {input_specs[0].name: batch_array})
-    batch_inference_time = (time.time() - start) * 1000
-    per_sample_time = batch_inference_time / len(batch_images)
-
-    for i, (label, sample, sample_index, dataset_name, sample_seed) in enumerate(
-        batch_metadata
-    ):
-        predicted, pred_probs = process_model_output(outputs[0][i])
-
-        tracker.add_ok(
-            dataset_name=dataset_name,
-            sample_index=sample_index,
-            sample=sample,
-            label=label,
-            predicted=predicted,
-            probs=pred_probs,
-            inference_time_ms=per_sample_time,
-            batch_inference_time_ms=batch_inference_time,
-            batch_id=batch_id,
-            batch_size=len(batch_images),
-            sample_seed=sample_seed,
-            aug_pass=aug_pass,
-        )
+    batch_array = stack_uniform_batch(batch_images)
+    run_batch_and_record(
+        session, input_specs, batch_array, batch_metadata, tracker, batch_id, aug_pass
+    )
 
 
 async def run_image_benchmark(