Add resource/throughput benchmark metrics and offline CI smoke coverage

.gitignore

@@ -15,4 +15,9 @@ benchmarks/*.parquet
 bin/*
 .venv/
 .env
-*.DS_Store
+*.DS_Store
+
+# Sample benchmark output committed as the benchmarking "how to" deliverable
+!benchmarks/results/sample/
+!benchmarks/results/sample/*.json
+!benchmarks/results/sample/*.csv

benchmarks/ReadMe.md

@@ -125,6 +125,30 @@ You can benchmark vCache on your own datasets. The script supports `.csv` and `.
 
 Benchmark results are saved to the `benchmarks/results/` directory, organized by dataset, embedding model, and LLM. For each run, the output includes:
 - **JSON files** containing raw data on cache hits, misses, latency, accuracy metrics, and internal vCache statistics.
+- **CSV files** with the same per-query metrics in a flat, row-per-query table (`cache_hit`, `latency_direct`, `latency_vcache`, `cpu_percent`, `memory_mb`, `gpu_util_percent`, ...), convenient for spreadsheets or `pandas`.
 - **Plot images (`.png`, `.pdf`)** visualizing key trade-offs, such as cache hit rate vs. accuracy and latency savings.
 
-These metrics help assess the trade-offs between reliability, efficiency, and reuse across different semantic caching strategies.
+These metrics help assess the trade-offs between reliability, efficiency, and reuse across different semantic caching strategies.
+
+A sample run's output is committed at [`benchmarks/results/sample/`](results/sample/) so you can see the file format without running anything.
+
+
+### Resource & Throughput Metrics
+
+Alongside cache hit rate, accuracy, and latency, every run also records, per query:
+- `cpu_percent_list` / `memory_mb_list`: the benchmark process's CPU usage (%) and resident memory (MB), sampled via [`psutil`](https://pypi.org/project/psutil/) right after each query completes. `peak_memory_mb` is the run's maximum.
+- `gpu_util_list`: GPU utilization (%) of device 0, sampled via [`pynvml`](https://pypi.org/project/pynvml/). This is **best-effort**: it's `None` for every query unless you `pip install pynvml` and have a working NVIDIA driver — no error is raised either way.
+
+And for the run as a whole:
+- `elapsed_time_sec`: total wall-clock time for the benchmark loop.
+- `throughput_qps`: queries processed per second (`num_queries / elapsed_time_sec`).
+- `throughput_tps`: tokens processed per second, summing prompt + response tokens (counted with [`tiktoken`](https://pypi.org/project/tiktoken/)'s `cl100k_base` encoding when available, falling back to a whitespace word count otherwise).
+
+These are implemented in `benchmarks/common/resource_metrics.py` and wired into `Benchmark.update_stats` / `dump_results_to_json` / `dump_results_to_csv` in `benchmarks/benchmark.py`.
+
+
+## Continuous Integration
+
+Running the full benchmark suite on every commit isn't practical — it downloads large Hugging Face datasets and, for custom/live datasets, makes real LLM API calls. Instead, `tests/integration/test_benchmark_smoke.py` and `tests/unit/Benchmark/test_resource_metrics.py` exercise the exact same metrics pipeline (`Benchmark.run_benchmark_loop`, `update_stats`, `dump_results_to_json`/`dump_results_to_csv`, and the resource-sampling helpers) against a small synthetic, fully offline dataset using `BenchmarkInferenceEngine`/`BenchmarkEmbeddingEngine` (pre-set responses/embeddings, no network or API key required). These tests run automatically in the `test` job of `.github/workflows/ci.yml` on every commit, so regressions in the benchmarking instrumentation are caught immediately, without the cost or flakiness of a full-scale run.
+
+If you want to track real performance trends over time, periodically run `python benchmarks/benchmark.py` with a small `RUN_COMBINATIONS` entry and commit or archive the resulting JSON/CSV — see `benchmarks/results/sample/` for the expected format.

benchmarks/benchmark.py

@@ -66,6 +66,11 @@
     answers_have_same_meaning_llm,
     answers_have_same_meaning_static,
 )
+from benchmarks.common.resource_metrics import (
+    ResourceSampler,
+    count_tokens,
+    gpu_utilization_percent,
+)
 from vcache.config import VCacheConfig
 from vcache.inference_engine.strategies.benchmark import (
     BenchmarkInferenceEngine,
@@ -399,6 +404,13 @@ def stats_set_up(self):
         self.fn_list: List[int] = []
         self.latency_direct_list: List[float] = []
         self.latency_vcache_list: List[float] = []
+        self.cpu_percent_list: List[float] = []
+        self.memory_mb_list: List[float] = []
+        self.gpu_util_list: List[float] = []
+        self._resource_sampler: ResourceSampler = ResourceSampler()
+        self._total_tokens: int = 0
+        self._loop_start_time: float = None
+        self.elapsed_time_sec: float = None
         self.observations_dict: Dict[str, Dict[str, float]] = {}
         self.gammas_dict: Dict[str, float] = {}
         self.t_hats_dict: Dict[str, float] = {}
@@ -431,6 +443,7 @@ def run_benchmark_loop_custom(self, data_entries: List[Dict], max_samples: int):
             desc="Processing entries",
             disable=DISABLE_PROGRESS_BAR,
         )
+        self._loop_start_time = time.time()
 
         for idx, data_entry in enumerate(data_entries):
             if idx >= max_samples:
@@ -467,11 +480,14 @@ def run_benchmark_loop_custom(self, data_entries: List[Dict], max_samples: int):
                 nn_metadata=nn_metadata,
                 latency_direct=latency_direct,
                 latency_vcache=latency_vcache,
+                prompt=prompt,
+                response_text=cache_response or label_response,
             )
 
             pbar.update(1)
 
         pbar.close()
+        self.elapsed_time_sec = time.time() - self._loop_start_time
 
     def run_benchmark_loop(self, data_entries: List[Dict], max_samples: int):
         """Run benchmark loop for pre-computed datasets from HuggingFace.
@@ -499,6 +515,7 @@ def run_benchmark_loop(self, data_entries: List[Dict], max_samples: int):
             disable=DISABLE_PROGRESS_BAR,
         )
         logging.info(f"data_entries: {data_entries}")
+        self._loop_start_time = time.time()
 
         for idx, data_entry in enumerate(data_entries):
             if idx >= max_samples:
@@ -557,11 +574,14 @@ def run_benchmark_loop(self, data_entries: List[Dict], max_samples: int):
                 nn_metadata=nn_metadata,
                 latency_direct=latency_direct,
                 latency_vcache=latency_vcache,
+                prompt=prompt,
+                response_text=cache_response or label_response,
             )
 
             pbar.update(1)
 
         pbar.close()
+        self.elapsed_time_sec = time.time() - self._loop_start_time
 
     def test_run_benchmark(self, max_samples):
         """Main benchmark execution method that loads data and runs evaluation.
@@ -616,6 +636,7 @@ def test_run_benchmark(self, max_samples):
             return
 
         self.dump_results_to_json()
+        self.dump_results_to_csv()
         generate_individual_plots(
             self,
             font_size=PLOT_FONT_SIZE,
@@ -636,6 +657,8 @@ def update_stats(
         nn_metadata: EmbeddingMetadataObj,
         latency_direct: float,
         latency_vcache: float,
+        prompt: str = "",
+        response_text: str = "",
     ):
         """Update benchmark statistics with results from a single inference.
 
@@ -653,6 +676,8 @@ def update_stats(
             nn_metadata: Metadata object for the nearest neighbor in cache.
             latency_direct: Latency for direct inference without cache.
             latency_vcache: Latency for vCache inference including cache logic.
+            prompt: The input prompt, used to estimate token throughput.
+            response_text: The response text, used to estimate token throughput.
 
         Note:
             The method uses different correctness evaluation strategies based on
@@ -711,6 +736,11 @@ def update_stats(
         self.latency_direct_list.append(latency_direct)
         self.latency_vcache_list.append(latency_vcache)
 
+        self.cpu_percent_list.append(self._resource_sampler.cpu_percent())
+        self.memory_mb_list.append(self._resource_sampler.memory_mb())
+        self.gpu_util_list.append(gpu_utilization_percent())
+        self._total_tokens += count_tokens(prompt) + count_tokens(response_text)
+
     def get_vcache_answer(
         self,
         prompt: str,
@@ -885,6 +915,18 @@ def dump_results_to_json(self):
         self.t_primes_dict = t_primes_dict
         self.var_ts_dict = var_ts_dict
 
+        # Throughput is derived from the sum of per-query latencies (the
+        # simulated/live model latency each request actually incurred),
+        # rather than the wall-clock time of the benchmark loop. The loop's
+        # wall-clock time is dominated by harness overhead (dataset
+        # iteration, pandas/tqdm bookkeeping, the inter-iteration sleep) and
+        # does not reflect real request throughput, especially for
+        # pre-computed datasets where model latency is injected rather than
+        # actually elapsed.
+        simulated_time_sec = (
+            sum(self.latency_vcache_list) if self.latency_vcache_list else None
+        )
+
         try:
             global_observations_dict = self.vcache.vcache_policy.global_observations
             global_gamma = self.vcache.vcache_policy.bayesian.global_gamma
@@ -916,6 +958,20 @@ def dump_results_to_json(self):
             "fn_list": self.fn_list,
             "latency_direct_list": self.latency_direct_list,
             "latency_vectorq_list": self.latency_vcache_list,
+            "cpu_percent_list": self.cpu_percent_list,
+            "memory_mb_list": self.memory_mb_list,
+            "peak_memory_mb": max(self.memory_mb_list) if self.memory_mb_list else None,
+            "gpu_util_list": self.gpu_util_list,
+            "elapsed_time_sec": self.elapsed_time_sec,
+            "simulated_time_sec": simulated_time_sec,
+            "throughput_qps": (
+                len(self.cache_hit_list) / simulated_time_sec
+                if simulated_time_sec
+                else None
+            ),
+            "throughput_tps": (
+                self._total_tokens / simulated_time_sec if simulated_time_sec else None
+            ),
             "observations_dict": self.observations_dict,
             "gammas_dict": self.gammas_dict,
             "t_hats_dict": self.t_hats_dict,
@@ -933,6 +989,35 @@ def dump_results_to_json(self):
             json.dump(data, json_file, indent=4)
         logging.info(f"Results successfully dumped to {filepath}")
 
+    def dump_results_to_csv(self):
+        """Serialize per-query benchmark results to a CSV file.
+
+        Writes one row per processed query, covering cache hit/miss outcome,
+        classification labels, latency, and the resource/throughput metrics
+        (CPU, memory, GPU utilization) sampled for that query. This is a
+        row-aligned companion to `dump_results_to_json`, useful for quick
+        spreadsheet-style inspection or downstream analysis with pandas.
+        """
+        df = pd.DataFrame(
+            {
+                "cache_hit": self.cache_hit_list,
+                "cache_miss": self.cache_miss_list,
+                "tp": self.tp_list,
+                "fp": self.fp_list,
+                "tn": self.tn_list,
+                "fn": self.fn_list,
+                "latency_direct": self.latency_direct_list,
+                "latency_vcache": self.latency_vcache_list,
+                "cpu_percent": self.cpu_percent_list,
+                "memory_mb": self.memory_mb_list,
+                "gpu_util_percent": self.gpu_util_list,
+            }
+        )
+
+        filepath = self.output_folder_path + f"/results_{self.timestamp}.csv"
+        df.to_csv(filepath, index=False)
+        logging.info(f"Results successfully dumped to {filepath}")
+
 
 ########################################################################################################################
 ### Helper #############################################################################################################

benchmarks/common/resource_metrics.py

@@ -0,0 +1,102 @@
+import atexit
+import os
+from typing import Optional
+
+import psutil
+
+_gpu_unavailable: bool = False
+_gpu_handle = None
+_token_encoding = None
+_tiktoken_unavailable: bool = False
+
+
+class ResourceSampler:
+    """Samples CPU and memory usage of the current process."""
+
+    def __init__(self):
+        self._process: psutil.Process = psutil.Process(os.getpid())
+        # Priming call: psutil.cpu_percent() always returns 0.0 on the first
+        # invocation, since it measures usage since the *previous* call.
+        self._process.cpu_percent(interval=None)
+
+    def cpu_percent(self) -> float:
+        """Returns process CPU usage in percent since the last sample."""
+        return self._process.cpu_percent(interval=None)
+
+    def memory_mb(self) -> float:
+        """Returns the process's resident set size (RSS) in megabytes."""
+        return self._process.memory_info().rss / (1024 * 1024)
+
+
+def gpu_utilization_percent() -> Optional[float]:
+    """Returns the current GPU utilization in percent, if available.
+
+    This is best-effort: it returns None if `pynvml` isn't installed, no
+    NVIDIA driver is present, or querying the device otherwise fails. Once
+    unavailable, it is assumed to stay unavailable for the process lifetime
+    so we don't re-probe on every call.
+
+    NVML is initialized at most once per process (cached handle) and torn
+    down via `atexit` rather than being re-initialized on every call.
+
+    Returns:
+        The utilization of GPU device 0 in percent, or None if unavailable.
+    """
+    global _gpu_unavailable, _gpu_handle
+    if _gpu_unavailable:
+        return None
+
+    try:
+        import pynvml
+
+        if _gpu_handle is None:
+            pynvml.nvmlInit()
+            atexit.register(_shutdown_nvml)
+            _gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0)
+        return float(pynvml.nvmlDeviceGetUtilizationRates(_gpu_handle).gpu)
+    except Exception:
+        _gpu_unavailable = True
+        return None
+
+
+def _shutdown_nvml() -> None:
+    """Shuts down NVML if it was initialized by `gpu_utilization_percent`."""
+    global _gpu_handle
+    if _gpu_handle is None:
+        return
+    try:
+        import pynvml
+
+        pynvml.nvmlShutdown()
+    except Exception:
+        pass
+    _gpu_handle = None
+
+
+def count_tokens(text: str) -> int:
+    """Estimates the number of tokens in `text`.
+
+    Uses tiktoken's cl100k_base encoding when available, falling back to a
+    whitespace-based word count otherwise.
+
+    Args:
+        text: The text to count tokens for.
+
+    Returns:
+        The estimated token count. 0 for empty/None text.
+    """
+    global _token_encoding, _tiktoken_unavailable
+    if not text:
+        return 0
+
+    if not _tiktoken_unavailable and _token_encoding is None:
+        try:
+            import tiktoken
+
+            _token_encoding = tiktoken.get_encoding("cl100k_base")
+        except Exception:
+            _tiktoken_unavailable = True
+
+    if _token_encoding is not None:
+        return len(_token_encoding.encode(text))
+    return len(text.split())

benchmarks/results/sample/results_sample.csv

@@ -0,0 +1,11 @@
+cache_hit,cache_miss,tp,fp,tn,fn,latency_direct,latency_vcache,cpu_percent,memory_mb,gpu_util_percent
+0,1,0,0,1,0,1.1837,1.2047,66.5,64.9921875,
+1,0,1,0,0,0,0.965,0.0189,53.9,101.37109375,
+0,1,0,0,1,0,1.2419,1.289,66.5,101.37109375,
+1,0,1,0,0,0,1.3353,0.0135,66.5,101.375,
+0,1,0,0,1,0,1.0532,1.0744,99.7,101.375,
+0,1,0,0,1,0,0.9312,0.9714,99.7,101.375,
+1,0,1,0,0,0,0.8159,0.018,100.8,101.3828125,
+0,1,0,0,1,0,1.1899,1.2317,49.6,101.3828125,
+0,1,0,0,1,0,0.9323,0.9759,67.9,101.3828125,
+0,1,0,0,1,0,1.2857,1.306,66.5,101.3828125,