diff --git a/.gitignore b/.gitignore index fb221ba..bc34442 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,9 @@ benchmarks/*.parquet bin/* .venv/ .env -*.DS_Store \ No newline at end of file +*.DS_Store + +# Sample benchmark output committed as the benchmarking "how to" deliverable +!benchmarks/results/sample/ +!benchmarks/results/sample/*.json +!benchmarks/results/sample/*.csv \ No newline at end of file diff --git a/benchmarks/ReadMe.md b/benchmarks/ReadMe.md index 238ec59..07949a5 100644 --- a/benchmarks/ReadMe.md +++ b/benchmarks/ReadMe.md @@ -125,6 +125,30 @@ You can benchmark vCache on your own datasets. The script supports `.csv` and `. Benchmark results are saved to the `benchmarks/results/` directory, organized by dataset, embedding model, and LLM. For each run, the output includes: - **JSON files** containing raw data on cache hits, misses, latency, accuracy metrics, and internal vCache statistics. +- **CSV files** with the same per-query metrics in a flat, row-per-query table (`cache_hit`, `latency_direct`, `latency_vcache`, `cpu_percent`, `memory_mb`, `gpu_util_percent`, ...), convenient for spreadsheets or `pandas`. - **Plot images (`.png`, `.pdf`)** visualizing key trade-offs, such as cache hit rate vs. accuracy and latency savings. -These metrics help assess the trade-offs between reliability, efficiency, and reuse across different semantic caching strategies. \ No newline at end of file +These metrics help assess the trade-offs between reliability, efficiency, and reuse across different semantic caching strategies. + +A sample run's output is committed at [`benchmarks/results/sample/`](results/sample/) so you can see the file format without running anything. + + +### Resource & Throughput Metrics + +Alongside cache hit rate, accuracy, and latency, every run also records, per query: +- `cpu_percent_list` / `memory_mb_list`: the benchmark process's CPU usage (%) and resident memory (MB), sampled via [`psutil`](https://pypi.org/project/psutil/) right after each query completes. `peak_memory_mb` is the run's maximum. +- `gpu_util_list`: GPU utilization (%) of device 0, sampled via [`pynvml`](https://pypi.org/project/pynvml/). This is **best-effort**: it's `None` for every query unless you `pip install pynvml` and have a working NVIDIA driver — no error is raised either way. + +And for the run as a whole: +- `elapsed_time_sec`: total wall-clock time for the benchmark loop. +- `throughput_qps`: queries processed per second (`num_queries / elapsed_time_sec`). +- `throughput_tps`: tokens processed per second, summing prompt + response tokens (counted with [`tiktoken`](https://pypi.org/project/tiktoken/)'s `cl100k_base` encoding when available, falling back to a whitespace word count otherwise). + +These are implemented in `benchmarks/common/resource_metrics.py` and wired into `Benchmark.update_stats` / `dump_results_to_json` / `dump_results_to_csv` in `benchmarks/benchmark.py`. + + +## Continuous Integration + +Running the full benchmark suite on every commit isn't practical — it downloads large Hugging Face datasets and, for custom/live datasets, makes real LLM API calls. Instead, `tests/integration/test_benchmark_smoke.py` and `tests/unit/Benchmark/test_resource_metrics.py` exercise the exact same metrics pipeline (`Benchmark.run_benchmark_loop`, `update_stats`, `dump_results_to_json`/`dump_results_to_csv`, and the resource-sampling helpers) against a small synthetic, fully offline dataset using `BenchmarkInferenceEngine`/`BenchmarkEmbeddingEngine` (pre-set responses/embeddings, no network or API key required). These tests run automatically in the `test` job of `.github/workflows/ci.yml` on every commit, so regressions in the benchmarking instrumentation are caught immediately, without the cost or flakiness of a full-scale run. + +If you want to track real performance trends over time, periodically run `python benchmarks/benchmark.py` with a small `RUN_COMBINATIONS` entry and commit or archive the resulting JSON/CSV — see `benchmarks/results/sample/` for the expected format. \ No newline at end of file diff --git a/benchmarks/benchmark.py b/benchmarks/benchmark.py index 9340b35..33fa3e4 100644 --- a/benchmarks/benchmark.py +++ b/benchmarks/benchmark.py @@ -66,6 +66,11 @@ answers_have_same_meaning_llm, answers_have_same_meaning_static, ) +from benchmarks.common.resource_metrics import ( + ResourceSampler, + count_tokens, + gpu_utilization_percent, +) from vcache.config import VCacheConfig from vcache.inference_engine.strategies.benchmark import ( BenchmarkInferenceEngine, @@ -399,6 +404,13 @@ def stats_set_up(self): self.fn_list: List[int] = [] self.latency_direct_list: List[float] = [] self.latency_vcache_list: List[float] = [] + self.cpu_percent_list: List[float] = [] + self.memory_mb_list: List[float] = [] + self.gpu_util_list: List[float] = [] + self._resource_sampler: ResourceSampler = ResourceSampler() + self._total_tokens: int = 0 + self._loop_start_time: float = None + self.elapsed_time_sec: float = None self.observations_dict: Dict[str, Dict[str, float]] = {} self.gammas_dict: Dict[str, float] = {} self.t_hats_dict: Dict[str, float] = {} @@ -431,6 +443,7 @@ def run_benchmark_loop_custom(self, data_entries: List[Dict], max_samples: int): desc="Processing entries", disable=DISABLE_PROGRESS_BAR, ) + self._loop_start_time = time.time() for idx, data_entry in enumerate(data_entries): if idx >= max_samples: @@ -467,11 +480,14 @@ def run_benchmark_loop_custom(self, data_entries: List[Dict], max_samples: int): nn_metadata=nn_metadata, latency_direct=latency_direct, latency_vcache=latency_vcache, + prompt=prompt, + response_text=cache_response or label_response, ) pbar.update(1) pbar.close() + self.elapsed_time_sec = time.time() - self._loop_start_time def run_benchmark_loop(self, data_entries: List[Dict], max_samples: int): """Run benchmark loop for pre-computed datasets from HuggingFace. @@ -499,6 +515,7 @@ def run_benchmark_loop(self, data_entries: List[Dict], max_samples: int): disable=DISABLE_PROGRESS_BAR, ) logging.info(f"data_entries: {data_entries}") + self._loop_start_time = time.time() for idx, data_entry in enumerate(data_entries): if idx >= max_samples: @@ -557,11 +574,14 @@ def run_benchmark_loop(self, data_entries: List[Dict], max_samples: int): nn_metadata=nn_metadata, latency_direct=latency_direct, latency_vcache=latency_vcache, + prompt=prompt, + response_text=cache_response or label_response, ) pbar.update(1) pbar.close() + self.elapsed_time_sec = time.time() - self._loop_start_time def test_run_benchmark(self, max_samples): """Main benchmark execution method that loads data and runs evaluation. @@ -616,6 +636,7 @@ def test_run_benchmark(self, max_samples): return self.dump_results_to_json() + self.dump_results_to_csv() generate_individual_plots( self, font_size=PLOT_FONT_SIZE, @@ -636,6 +657,8 @@ def update_stats( nn_metadata: EmbeddingMetadataObj, latency_direct: float, latency_vcache: float, + prompt: str = "", + response_text: str = "", ): """Update benchmark statistics with results from a single inference. @@ -653,6 +676,8 @@ def update_stats( nn_metadata: Metadata object for the nearest neighbor in cache. latency_direct: Latency for direct inference without cache. latency_vcache: Latency for vCache inference including cache logic. + prompt: The input prompt, used to estimate token throughput. + response_text: The response text, used to estimate token throughput. Note: The method uses different correctness evaluation strategies based on @@ -711,6 +736,11 @@ def update_stats( self.latency_direct_list.append(latency_direct) self.latency_vcache_list.append(latency_vcache) + self.cpu_percent_list.append(self._resource_sampler.cpu_percent()) + self.memory_mb_list.append(self._resource_sampler.memory_mb()) + self.gpu_util_list.append(gpu_utilization_percent()) + self._total_tokens += count_tokens(prompt) + count_tokens(response_text) + def get_vcache_answer( self, prompt: str, @@ -885,6 +915,18 @@ def dump_results_to_json(self): self.t_primes_dict = t_primes_dict self.var_ts_dict = var_ts_dict + # Throughput is derived from the sum of per-query latencies (the + # simulated/live model latency each request actually incurred), + # rather than the wall-clock time of the benchmark loop. The loop's + # wall-clock time is dominated by harness overhead (dataset + # iteration, pandas/tqdm bookkeeping, the inter-iteration sleep) and + # does not reflect real request throughput, especially for + # pre-computed datasets where model latency is injected rather than + # actually elapsed. + simulated_time_sec = ( + sum(self.latency_vcache_list) if self.latency_vcache_list else None + ) + try: global_observations_dict = self.vcache.vcache_policy.global_observations global_gamma = self.vcache.vcache_policy.bayesian.global_gamma @@ -916,6 +958,20 @@ def dump_results_to_json(self): "fn_list": self.fn_list, "latency_direct_list": self.latency_direct_list, "latency_vectorq_list": self.latency_vcache_list, + "cpu_percent_list": self.cpu_percent_list, + "memory_mb_list": self.memory_mb_list, + "peak_memory_mb": max(self.memory_mb_list) if self.memory_mb_list else None, + "gpu_util_list": self.gpu_util_list, + "elapsed_time_sec": self.elapsed_time_sec, + "simulated_time_sec": simulated_time_sec, + "throughput_qps": ( + len(self.cache_hit_list) / simulated_time_sec + if simulated_time_sec + else None + ), + "throughput_tps": ( + self._total_tokens / simulated_time_sec if simulated_time_sec else None + ), "observations_dict": self.observations_dict, "gammas_dict": self.gammas_dict, "t_hats_dict": self.t_hats_dict, @@ -933,6 +989,35 @@ def dump_results_to_json(self): json.dump(data, json_file, indent=4) logging.info(f"Results successfully dumped to {filepath}") + def dump_results_to_csv(self): + """Serialize per-query benchmark results to a CSV file. + + Writes one row per processed query, covering cache hit/miss outcome, + classification labels, latency, and the resource/throughput metrics + (CPU, memory, GPU utilization) sampled for that query. This is a + row-aligned companion to `dump_results_to_json`, useful for quick + spreadsheet-style inspection or downstream analysis with pandas. + """ + df = pd.DataFrame( + { + "cache_hit": self.cache_hit_list, + "cache_miss": self.cache_miss_list, + "tp": self.tp_list, + "fp": self.fp_list, + "tn": self.tn_list, + "fn": self.fn_list, + "latency_direct": self.latency_direct_list, + "latency_vcache": self.latency_vcache_list, + "cpu_percent": self.cpu_percent_list, + "memory_mb": self.memory_mb_list, + "gpu_util_percent": self.gpu_util_list, + } + ) + + filepath = self.output_folder_path + f"/results_{self.timestamp}.csv" + df.to_csv(filepath, index=False) + logging.info(f"Results successfully dumped to {filepath}") + ######################################################################################################################## ### Helper ############################################################################################################# diff --git a/benchmarks/common/resource_metrics.py b/benchmarks/common/resource_metrics.py new file mode 100644 index 0000000..1f4abd8 --- /dev/null +++ b/benchmarks/common/resource_metrics.py @@ -0,0 +1,102 @@ +import atexit +import os +from typing import Optional + +import psutil + +_gpu_unavailable: bool = False +_gpu_handle = None +_token_encoding = None +_tiktoken_unavailable: bool = False + + +class ResourceSampler: + """Samples CPU and memory usage of the current process.""" + + def __init__(self): + self._process: psutil.Process = psutil.Process(os.getpid()) + # Priming call: psutil.cpu_percent() always returns 0.0 on the first + # invocation, since it measures usage since the *previous* call. + self._process.cpu_percent(interval=None) + + def cpu_percent(self) -> float: + """Returns process CPU usage in percent since the last sample.""" + return self._process.cpu_percent(interval=None) + + def memory_mb(self) -> float: + """Returns the process's resident set size (RSS) in megabytes.""" + return self._process.memory_info().rss / (1024 * 1024) + + +def gpu_utilization_percent() -> Optional[float]: + """Returns the current GPU utilization in percent, if available. + + This is best-effort: it returns None if `pynvml` isn't installed, no + NVIDIA driver is present, or querying the device otherwise fails. Once + unavailable, it is assumed to stay unavailable for the process lifetime + so we don't re-probe on every call. + + NVML is initialized at most once per process (cached handle) and torn + down via `atexit` rather than being re-initialized on every call. + + Returns: + The utilization of GPU device 0 in percent, or None if unavailable. + """ + global _gpu_unavailable, _gpu_handle + if _gpu_unavailable: + return None + + try: + import pynvml + + if _gpu_handle is None: + pynvml.nvmlInit() + atexit.register(_shutdown_nvml) + _gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0) + return float(pynvml.nvmlDeviceGetUtilizationRates(_gpu_handle).gpu) + except Exception: + _gpu_unavailable = True + return None + + +def _shutdown_nvml() -> None: + """Shuts down NVML if it was initialized by `gpu_utilization_percent`.""" + global _gpu_handle + if _gpu_handle is None: + return + try: + import pynvml + + pynvml.nvmlShutdown() + except Exception: + pass + _gpu_handle = None + + +def count_tokens(text: str) -> int: + """Estimates the number of tokens in `text`. + + Uses tiktoken's cl100k_base encoding when available, falling back to a + whitespace-based word count otherwise. + + Args: + text: The text to count tokens for. + + Returns: + The estimated token count. 0 for empty/None text. + """ + global _token_encoding, _tiktoken_unavailable + if not text: + return 0 + + if not _tiktoken_unavailable and _token_encoding is None: + try: + import tiktoken + + _token_encoding = tiktoken.get_encoding("cl100k_base") + except Exception: + _tiktoken_unavailable = True + + if _token_encoding is not None: + return len(_token_encoding.encode(text)) + return len(text.split()) diff --git a/benchmarks/results/sample/results_sample.csv b/benchmarks/results/sample/results_sample.csv new file mode 100644 index 0000000..a0d9508 --- /dev/null +++ b/benchmarks/results/sample/results_sample.csv @@ -0,0 +1,11 @@ +cache_hit,cache_miss,tp,fp,tn,fn,latency_direct,latency_vcache,cpu_percent,memory_mb,gpu_util_percent +0,1,0,0,1,0,1.1837,1.2047,66.5,64.9921875, +1,0,1,0,0,0,0.965,0.0189,53.9,101.37109375, +0,1,0,0,1,0,1.2419,1.289,66.5,101.37109375, +1,0,1,0,0,0,1.3353,0.0135,66.5,101.375, +0,1,0,0,1,0,1.0532,1.0744,99.7,101.375, +0,1,0,0,1,0,0.9312,0.9714,99.7,101.375, +1,0,1,0,0,0,0.8159,0.018,100.8,101.3828125, +0,1,0,0,1,0,1.1899,1.2317,49.6,101.3828125, +0,1,0,0,1,0,0.9323,0.9759,67.9,101.3828125, +0,1,0,0,1,0,1.2857,1.306,66.5,101.3828125, diff --git a/benchmarks/results/sample/results_sample.json b/benchmarks/results/sample/results_sample.json new file mode 100644 index 0000000..e837cdc --- /dev/null +++ b/benchmarks/results/sample/results_sample.json @@ -0,0 +1,167 @@ +{ + "config": { + "filepath": "vCache/SemBenchmarkLmArena (sample subset)", + "embedding_model": [ + "emb_e5_large_v2", + "E5_Large_v2", + "float16", + 512 + ], + "llm_model": [ + "response_gpt-4o-mini", + "GPT-4o-mini", + "float16", + null + ], + "eviction_policy": "CostAwareEvictionPolicy(max_size=100000, watermark=0.95, eviction_percentage=0.1, cost_weight=0.5)", + "is_static_threshold": false, + "threshold": null, + "delta": 0.02 + }, + "cache_hit_list": [ + 0, + 1, + 0, + 1, + 0, + 0, + 1, + 0, + 0, + 0 + ], + "cache_miss_list": [ + 1, + 0, + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 1 + ], + "tp_list": [ + 0, + 1, + 0, + 1, + 0, + 0, + 1, + 0, + 0, + 0 + ], + "fp_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "tn_list": [ + 1, + 0, + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 1 + ], + "fn_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "latency_direct_list": [ + 1.1837, + 0.965, + 1.2419, + 1.3353, + 1.0532, + 0.9312, + 0.8159, + 1.1899, + 0.9323, + 1.2857 + ], + "latency_vectorq_list": [ + 1.2047, + 0.0189, + 1.289, + 0.0135, + 1.0744, + 0.9714, + 0.018, + 1.2317, + 0.9759, + 1.306 + ], + "cpu_percent_list": [ + 66.5, + 53.9, + 66.5, + 66.5, + 99.7, + 99.7, + 100.8, + 49.6, + 67.9, + 66.5 + ], + "memory_mb_list": [ + 64.9921875, + 101.37109375, + 101.37109375, + 101.375, + 101.375, + 101.375, + 101.3828125, + 101.3828125, + 101.3828125, + 101.3828125 + ], + "peak_memory_mb": 101.3828125, + "gpu_util_list": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "elapsed_time_sec": 1.062204122543335, + "throughput_qps": 9.414386357356683, + "throughput_tps": 170.40039306815598, + "observations_dict": {}, + "gammas_dict": {}, + "t_hats_dict": {}, + "t_primes_dict": {}, + "var_ts_dict": {}, + "global_observations_dict": {}, + "global_gamma": null, + "global_t_hat": null, + "global_t_prime": null, + "global_var_t": null +} \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index d2d1ba1..f278e36 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "accelerate" @@ -478,7 +478,7 @@ pyproject_hooks = "*" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] -docs = ["furo (>=2023.08.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] +docs = ["furo (>=2023.8.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] test = ["build[uv,virtualenv]", "filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0) ; python_version < \"3.10\"", "setuptools (>=56.0.0) ; python_version == \"3.10\"", "setuptools (>=56.0.0) ; python_version == \"3.11\"", "setuptools (>=67.8.0) ; python_version >= \"3.12\"", "wheel (>=0.36.0)"] typing = ["build[uv]", "importlib-metadata (>=5.1)", "mypy (>=1.9.0,<1.10.0)", "tomli", "typing-extensions (>=3.7.4.3)"] uv = ["uv (>=0.1.18)"] @@ -1314,7 +1314,6 @@ files = [ {file = "faiss_cpu-1.11.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:72165263bbc3bf4026276b9df4227bb2871823b23af6546cd41a90bcd08d5f25"}, {file = "faiss_cpu-1.11.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:760a4f0ce612c5ddaf4862d32ec13d5b8e609c983d391d419ea5ea50d5557dd9"}, {file = "faiss_cpu-1.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2ad3b2aadd490d15d2d19586679ad2f4e821c1a9597af8086ba543bef4d6e1f"}, - {file = "faiss_cpu-1.11.0.tar.gz", hash = "sha256:44877b896a2b30a61e35ea4970d008e8822545cb340eca4eff223ac7f40a1db9"}, ] [package.dependencies] @@ -1800,12 +1799,12 @@ files = [ google-auth = ">=2.14.1,<3.0.0" googleapis-common-protos = ">=1.56.2,<2.0.0" grpcio = [ - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}, + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] proto-plus = ">=1.22.3,<2.0.0" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" @@ -1813,7 +1812,7 @@ requests = ">=2.18.0,<3.0.0" [package.extras] async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"] -grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] +grpc = ["grpcio (>=1.33.2,<2.0.dev0)", "grpcio (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] @@ -1869,7 +1868,7 @@ description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "(python_version >= \"3.11\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\") and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")" +markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" files = [ {file = "greenlet-3.2.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:777c1281aa7c786738683e302db0f55eb4b0077c20f1dc53db8852ffaea0a6b0"}, {file = "greenlet-3.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3059c6f286b53ea4711745146ffe5a5c5ff801f62f6c56949446e0f6461f8157"}, @@ -2011,7 +2010,7 @@ files = [ [package.dependencies] googleapis-common-protos = ">=1.5.5" grpcio = ">=1.71.0" -protobuf = ">=5.26.1,<6.0dev" +protobuf = ">=5.26.1,<6.0.dev0" [[package]] name = "h11" @@ -2192,9 +2191,9 @@ files = [ filelock = "*" fsspec = ">=2023.5.0" hf-xet = [ - {version = ">=1.1.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""}, {version = ">=1.1.3,<2.0.0", optional = true, markers = "(platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\") and extra == \"hf-xet\""}, {version = ">=1.1.2,<2.0.0", optional = true, markers = "extra == \"hf-xet\" and platform_machine != \"x86_64\" and platform_machine != \"amd64\" and platform_machine != \"arm64\" and platform_machine != \"aarch64\""}, + {version = ">=1.1.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""}, ] packaging = ">=20.9" pyyaml = ">=5.1" @@ -2489,7 +2488,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -2615,7 +2614,7 @@ files = [ ] [package.dependencies] -certifi = ">=14.05.14" +certifi = ">=14.5.14" durationpy = ">=0.7" google-auth = ">=1.0.1" oauthlib = ">=3.2.2" @@ -3151,7 +3150,7 @@ description = "A framework for machine learning on Apple silicon." optional = false python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Darwin\" and platform_machine == \"arm64\"" +markers = "platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx-0.27.1-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a033b65fe46425ad5032867d5a71a556a5168108d89aa7092b457556c70d84fc"}, {file = "mlx-0.27.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:2836c6fd9803dc0c6cd06f204e31b3e0191e6c5b6bc8570b28661d926908cba3"}, @@ -3190,7 +3189,7 @@ description = "LLMs with MLX and the Hugging Face Hub" optional = false python-versions = ">=3.8" groups = ["main"] -markers = "platform_system == \"Darwin\" and platform_machine == \"arm64\"" +markers = "platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx_lm-0.26.1-py3-none-any.whl", hash = "sha256:b26ab8ffde618fda885edc4177c2f06d4f056078a552f641450d0e19c8637f5d"}, {file = "mlx_lm-0.26.1.tar.gz", hash = "sha256:ef94eb148b738145af114b992860beca5db7cbff271e3c3c1bc4bf3a72868799"}, @@ -3216,7 +3215,7 @@ description = "A framework for machine learning on Apple silicon." optional = false python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Darwin\" and platform_machine == \"arm64\"" +markers = "platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx_metal-0.27.1-py3-none-macosx_13_0_arm64.whl", hash = "sha256:c66d9b1adb3c0ea19492fba6493f672bc7542e65dd65f7e2995918815fbeb907"}, {file = "mlx_metal-0.27.1-py3-none-macosx_14_0_arm64.whl", hash = "sha256:fe4415ddd242974d91c7ca0699cd01507d17da8a5ba304122ef137cdb5e7fff4"}, @@ -3817,7 +3816,7 @@ description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb"}, {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:235f728d6e2a409eddf1df58d5b0921cf80cfa9e72b9f2775ccb7b4a87984668"}, @@ -3831,7 +3830,7 @@ description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:166ee35a3ff1587f2490364f90eeeb8da06cd867bd5b701bf7f9a02b78bc63fc"}, {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.whl", hash = "sha256:358b4a1d35370353d52e12f0a7d1769fc01ff74a191689d3870b2123156184c4"}, @@ -3847,7 +3846,7 @@ description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5847f1d6e5b757f1d2b3991a01082a44aad6f10ab3c5c0213fa3e25bddc25a13"}, {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53"}, @@ -3861,7 +3860,7 @@ description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6116fad3e049e04791c0256a9778c16237837c08b27ed8c8401e2e45de8d60cd"}, {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d461264ecb429c84c8879a7153499ddc7b19b5f8d84c204307491989a365588e"}, @@ -3877,7 +3876,7 @@ description = "cuDNN runtime libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9fd4584468533c61873e5fda8ca41bac3a38bcb2d12350830c69b0a96a7e4def"}, {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2"}, @@ -3894,7 +3893,7 @@ description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d16079550df460376455cba121db6564089176d9bac9e4f360493ca4741b22a6"}, {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8510990de9f96c803a051822618d42bf6cb8f069ff3f48d93a8486efdacb48fb"}, @@ -3913,7 +3912,7 @@ description = "cuFile GPUDirect libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159"}, {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8f57a0051dcf2543f6dc2b98a98cb2719c37d3cee1baba8965d57f3bbc90d4db"}, @@ -3926,7 +3925,7 @@ description = "CURAND native runtime libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6e82df077060ea28e37f48a3ec442a8f47690c7499bff392a5938614b56c98d8"}, {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf"}, @@ -3942,7 +3941,7 @@ description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0ce237ef60acde1efc457335a2ddadfd7610b892d94efee7b776c64bb1cac9e0"}, {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c"}, @@ -3963,7 +3962,7 @@ description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d25b62fb18751758fe3c93a4a08eff08effedfe4edf1c6bb5afd0890fe88f887"}, {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7aa32fa5470cf754f72d1116c7cbc300b4e638d3ae5304cfa4a638a5b87161b1"}, @@ -3982,7 +3981,7 @@ description = "NVIDIA cuSPARSELt" optional = false python-versions = "*" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1"}, {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46"}, @@ -3996,7 +3995,7 @@ description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522"}, {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6"}, @@ -4009,7 +4008,7 @@ description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a"}, {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf4eaa7d4b6b543ffd69d6abfb11efdeb2db48270d94dfd3a452c24150829e41"}, @@ -4023,7 +4022,7 @@ description = "NVIDIA Tools Extension" optional = false python-versions = ">=3" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f44f8d86bb7d5629988d61c8d3ae61dddb2015dee142740536bc7481b022fe4b"}, {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:adcaabb9d436c9761fca2b13959a2d237c5f9fd406c8e4b723c695409ff88059"}, @@ -4132,10 +4131,10 @@ files = [ [package.dependencies] numpy = [ + {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, + {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, {version = ">=1.23.5", markers = "python_version >= \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, - {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, ] [[package]] @@ -4526,9 +4525,9 @@ files = [ [package.dependencies] numpy = [ + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -4949,26 +4948,33 @@ files = [ [[package]] name = "psutil" -version = "7.0.0" -description = "Cross-platform lib for process and system monitoring in Python. NOTE: the syntax of this script MUST be kept compatible with Python 2.7." -optional = false -python-versions = ">=3.6" -groups = ["main"] -files = [ - {file = "psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"}, - {file = "psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"}, - {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91"}, - {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34"}, - {file = "psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993"}, - {file = "psutil-7.0.0-cp36-cp36m-win32.whl", hash = "sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17"}, - {file = "psutil-7.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e"}, - {file = "psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99"}, - {file = "psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553"}, - {file = "psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456"}, +version = "6.1.1" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +groups = ["main"] +files = [ + {file = "psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca"}, + {file = "psutil-6.1.1-cp27-none-win32.whl", hash = "sha256:6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac"}, + {file = "psutil-6.1.1-cp27-none-win_amd64.whl", hash = "sha256:c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3"}, + {file = "psutil-6.1.1-cp36-cp36m-win32.whl", hash = "sha256:384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603"}, + {file = "psutil-6.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303"}, + {file = "psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53"}, + {file = "psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649"}, + {file = "psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5"}, ] [package.extras] -dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest", "pytest-cov", "pytest-xdist", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] +dev = ["abi3audit", "black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] test = ["pytest", "pytest-xdist", "setuptools"] [[package]] @@ -7424,7 +7430,7 @@ description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "*" groups = ["main", "benchmarks"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e"}, {file = "triton-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b"}, @@ -7985,7 +7991,7 @@ description = "XFormers: A collection of composable Transformer building blocks. optional = false python-versions = ">=3.9" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\"" files = [ {file = "xformers-0.0.31-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:50aedaea82a38d7d28631f77617d1ed1f6f37c60bdc4bf167a69cbc0e39cee76"}, {file = "xformers-0.0.31-cp39-abi3-win_amd64.whl", hash = "sha256:23331bdb9831ba0df96f55258537ca0df7ad888efc75cea97a0de79b5e2291c4"}, @@ -8430,4 +8436,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "f672a3d75dc1e4cc56403560e13c25d2fd953684e23ca3345cab1e9ee6dd9335" +content-hash = "9d6ccb6cf557d4f4d72b0abc3572b23c1e9b72c4aa1fb54baeadb1f4b0a6e7de" diff --git a/pyproject.toml b/pyproject.toml index 91bd7b6..9e50542 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "pytest (>=8.3.5,<9.0.0)", "pandas (>=2.3.0,<3.0.0)", "vllm (>=0.10.0,<0.11.0)", + "psutil (>=6.0.0,<7.0.0)", ] diff --git a/tests/integration/test_benchmark_smoke.py b/tests/integration/test_benchmark_smoke.py new file mode 100644 index 0000000..0d7077f --- /dev/null +++ b/tests/integration/test_benchmark_smoke.py @@ -0,0 +1,174 @@ +import json +import shutil +import tempfile +import unittest + +import pandas as pd + +from vcache import VCache, VCacheConfig, VerifiedDecisionPolicy +from vcache.inference_engine.strategies.benchmark import BenchmarkInferenceEngine +from vcache.vcache_core.cache.embedding_engine.strategies.benchmark import ( + BenchmarkEmbeddingEngine, +) +from vcache.vcache_core.cache.embedding_store.embedding_metadata_storage import ( + InMemoryEmbeddingMetadataStorage, +) +from vcache.vcache_core.cache.embedding_store.vector_db import ( + HNSWLibVectorDB, + SimilarityMetricType, +) +from vcache.vcache_core.cache.eviction_policy.strategies.mru import MRUEvictionPolicy +from vcache.vcache_core.similarity_evaluator.strategies.benchmark_comparison import ( + BenchmarkComparisonSimilarityEvaluator, +) + +# Column-name prefixes used to key the synthetic pre-computed dataset rows +# consumed by `Benchmark.run_benchmark_loop`. These don't need to match any +# real EmbeddingModel/LargeLanguageModel enum value. +EMBEDDING_MODEL = ("emb", "TestEmbeddingModel", "float32", 8) +LLM_MODEL = ("resp", "TestLLM", "float16", None) +NUM_SAMPLES = 10 + + +def _build_synthetic_data_entries(): + """Builds a small, fully offline, pre-computed dataset for the benchmark loop. + + Each entry supplies its own embedding and label response, so the run + never needs network access or an API key (mirrors how + `BenchmarkInferenceEngine`/`BenchmarkEmbeddingEngine` are used for the + non-custom-dataset baselines in `benchmarks/benchmark.py`). + """ + entries = [] + for i in range(NUM_SAMPLES): + embedding = [0.0] * 8 + embedding[i % 8] = 1.0 + embedding[(i + 1) % 8] = 0.5 + entries.append( + { + "prompt": f"synthetic prompt {i}", + f"{EMBEDDING_MODEL[0]}_lat": 0.01, + f"{LLM_MODEL[0]}_lat": 0.02, + LLM_MODEL[0]: f"synthetic response {i}", + EMBEDDING_MODEL[0]: embedding, + "id_set": i % 3, + } + ) + return entries + + +class TestBenchmarkSmoke(unittest.TestCase): + """Fully offline smoke test for the benchmark suite's metrics pipeline. + + Runs a tiny, deterministic benchmark (no network, no API key) through the + same code path production runs use, to catch regressions in the + resource/throughput instrumentation on every commit without paying for a + full-scale, live benchmark run. + """ + + def setUp(self): + # Imported locally (not at module level) because `Benchmark` itself + # subclasses unittest.TestCase (an existing quirk of benchmark.py). + # A module-level import here would make pytest collect and try to + # run `Benchmark` as its own bare test case, which fails since it + # requires a `vcache` constructor argument pytest can't supply. + from benchmarks.benchmark import Benchmark + + self.output_dir = tempfile.mkdtemp() + + eviction_policy = MRUEvictionPolicy( + max_size=1000, watermark=0.99, eviction_percentage=0.1 + ) + vcache_config = VCacheConfig( + inference_engine=BenchmarkInferenceEngine(), + embedding_engine=BenchmarkEmbeddingEngine(), + vector_db=HNSWLibVectorDB( + similarity_metric_type=SimilarityMetricType.COSINE, + max_capacity=1000, + ), + embedding_metadata_storage=InMemoryEmbeddingMetadataStorage(), + similarity_evaluator=BenchmarkComparisonSimilarityEvaluator(), + eviction_policy=eviction_policy, + ) + self.eviction_policy = eviction_policy + vcache = VCache(vcache_config, VerifiedDecisionPolicy(delta=0.5)) + + self.benchmark = Benchmark(vcache) + self.benchmark.filepath = "synthetic" + self.benchmark.embedding_model = EMBEDDING_MODEL + self.benchmark.llm_model = LLM_MODEL + self.benchmark.timestamp = "smoketest" + self.benchmark.threshold = None + self.benchmark.delta = 0.5 + self.benchmark.is_static_threshold = False + self.benchmark.output_folder_path = self.output_dir + self.benchmark.eviction_policy = eviction_policy + self.benchmark.stats_set_up() + + def tearDown(self): + # `stats_set_up`/`setUp` start a `VerifiedDecisionPolicy`, which owns + # its own ThreadPoolExecutor and CallbackQueue worker thread + # (separate from the eviction policy's executor below). Both must be + # shut down explicitly or the background threads leak past the test. + self.benchmark.vcache.vcache_policy.shutdown() + self.eviction_policy.shutdown() + shutil.rmtree(self.output_dir, ignore_errors=True) + + def test_benchmark_run_populates_resource_and_throughput_metrics(self): + data_entries = _build_synthetic_data_entries() + + self.benchmark.run_benchmark_loop(data_entries, max_samples=NUM_SAMPLES) + + self.assertEqual(len(self.benchmark.cache_hit_list), NUM_SAMPLES) + self.assertEqual(len(self.benchmark.cpu_percent_list), NUM_SAMPLES) + self.assertEqual(len(self.benchmark.memory_mb_list), NUM_SAMPLES) + self.assertEqual(len(self.benchmark.gpu_util_list), NUM_SAMPLES) + for gpu_util in self.benchmark.gpu_util_list: + self.assertTrue(gpu_util is None or 0.0 <= gpu_util <= 100.0) + + self.assertIsNotNone(self.benchmark.elapsed_time_sec) + self.assertGreater(self.benchmark.elapsed_time_sec, 0.0) + + # Snapshot the in-memory lists before dumping, so the dumped + # JSON/CSV values can be checked against the exact numbers that + # produced them, not just their presence/shape. + expected_cpu_percent_list = list(self.benchmark.cpu_percent_list) + expected_memory_mb_list = list(self.benchmark.memory_mb_list) + expected_gpu_util_list = list(self.benchmark.gpu_util_list) + expected_latency_vcache_list = list(self.benchmark.latency_vcache_list) + expected_total_tokens = self.benchmark._total_tokens + expected_peak_memory_mb = max(expected_memory_mb_list) + expected_simulated_time_sec = sum(expected_latency_vcache_list) + expected_throughput_qps = NUM_SAMPLES / expected_simulated_time_sec + expected_throughput_tps = expected_total_tokens / expected_simulated_time_sec + + self.benchmark.dump_results_to_json() + self.benchmark.dump_results_to_csv() + + json_path = f"{self.output_dir}/results_smoketest.json" + with open(json_path) as f: + data = json.load(f) + + self.assertEqual(data["cpu_percent_list"], expected_cpu_percent_list) + self.assertEqual(data["memory_mb_list"], expected_memory_mb_list) + self.assertEqual(data["gpu_util_list"], expected_gpu_util_list) + self.assertEqual(data["peak_memory_mb"], expected_peak_memory_mb) + self.assertAlmostEqual(data["simulated_time_sec"], expected_simulated_time_sec) + self.assertAlmostEqual(data["throughput_qps"], expected_throughput_qps) + self.assertAlmostEqual(data["throughput_tps"], expected_throughput_tps) + + csv_path = f"{self.output_dir}/results_smoketest.csv" + df = pd.read_csv(csv_path) + self.assertEqual(len(df), NUM_SAMPLES) + self.assertEqual(df["cpu_percent"].tolist(), expected_cpu_percent_list) + self.assertEqual(df["memory_mb"].tolist(), expected_memory_mb_list) + for expected, actual in zip( + expected_gpu_util_list, df["gpu_util_percent"].tolist() + ): + if expected is None: + self.assertTrue(pd.isna(actual)) + else: + self.assertEqual(actual, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/Benchmark/test_resource_metrics.py b/tests/unit/Benchmark/test_resource_metrics.py new file mode 100644 index 0000000..ad30002 --- /dev/null +++ b/tests/unit/Benchmark/test_resource_metrics.py @@ -0,0 +1,112 @@ +import unittest +from unittest.mock import MagicMock, patch + +import psutil + +from benchmarks.common import resource_metrics +from benchmarks.common.resource_metrics import ( + ResourceSampler, + count_tokens, + gpu_utilization_percent, +) + + +class TestResourceSampler(unittest.TestCase): + def setUp(self): + self.sampler = ResourceSampler() + + def test_cpu_percent_returns_non_negative_float(self): + cpu_percent = self.sampler.cpu_percent() + self.assertIsInstance(cpu_percent, float) + self.assertGreaterEqual(cpu_percent, 0.0) + + def test_memory_mb_matches_process_rss(self): + # `memory_mb` is a pure unit conversion of `Process.memory_info().rss`. + # Sample rss immediately before/after the call and assert the + # reported value falls within that bracket, converted to MB. + process = psutil.Process() + rss_before_mb = process.memory_info().rss / (1024 * 1024) + memory_mb = self.sampler.memory_mb() + rss_after_mb = process.memory_info().rss / (1024 * 1024) + + self.assertIsInstance(memory_mb, float) + self.assertGreaterEqual(memory_mb, rss_before_mb) + self.assertLessEqual(memory_mb, rss_after_mb) + + +class TestGpuUtilizationPercent(unittest.TestCase): + def setUp(self): + self._original_unavailable = resource_metrics._gpu_unavailable + self._original_handle = resource_metrics._gpu_handle + resource_metrics._gpu_unavailable = False + resource_metrics._gpu_handle = None + + def tearDown(self): + resource_metrics._gpu_unavailable = self._original_unavailable + resource_metrics._gpu_handle = self._original_handle + + def test_never_raises_and_returns_none_or_valid_percent(self): + result = gpu_utilization_percent() + if result is not None: + self.assertIsInstance(result, float) + self.assertGreaterEqual(result, 0.0) + self.assertLessEqual(result, 100.0) + + def test_returns_exact_utilization_and_initializes_nvml_once(self): + fake_pynvml = MagicMock() + fake_pynvml.nvmlDeviceGetHandleByIndex.return_value = "handle-0" + fake_pynvml.nvmlDeviceGetUtilizationRates.return_value = MagicMock(gpu=42) + + with patch.dict("sys.modules", {"pynvml": fake_pynvml}): + first = gpu_utilization_percent() + second = gpu_utilization_percent() + + self.assertEqual(first, 42.0) + self.assertEqual(second, 42.0) + # nvmlInit/handle lookup must happen once, not on every call. + fake_pynvml.nvmlInit.assert_called_once() + fake_pynvml.nvmlDeviceGetHandleByIndex.assert_called_once_with(0) + self.assertEqual(fake_pynvml.nvmlDeviceGetUtilizationRates.call_count, 2) + + resource_metrics._shutdown_nvml() + fake_pynvml.nvmlShutdown.assert_called_once() + self.assertIsNone(resource_metrics._gpu_handle) + + +class TestCountTokens(unittest.TestCase): + def test_empty_text_returns_zero(self): + self.assertEqual(count_tokens(""), 0) + self.assertEqual(count_tokens(None), 0) + + def test_uses_tiktoken_encoding_when_available(self): + # Verify the exact transformation (`len(encoding.encode(text))`) + # against a fake encoder, independent of the real tiktoken vocab. + original_encoding = resource_metrics._token_encoding + original_unavailable = resource_metrics._tiktoken_unavailable + fake_encoding = MagicMock() + fake_encoding.encode.return_value = [1, 2, 3, 4, 5] + resource_metrics._token_encoding = fake_encoding + resource_metrics._tiktoken_unavailable = False + try: + self.assertEqual(count_tokens("Is the sky blue?"), 5) + fake_encoding.encode.assert_called_once_with("Is the sky blue?") + finally: + resource_metrics._token_encoding = original_encoding + resource_metrics._tiktoken_unavailable = original_unavailable + + def test_whitespace_fallback_matches_word_count(self): + # Force the whitespace fallback path regardless of whether tiktoken + # is installed in this environment. + original_encoding = resource_metrics._token_encoding + original_unavailable = resource_metrics._tiktoken_unavailable + resource_metrics._token_encoding = None + resource_metrics._tiktoken_unavailable = True + try: + self.assertEqual(count_tokens("one two three four"), 4) + finally: + resource_metrics._token_encoding = original_encoding + resource_metrics._tiktoken_unavailable = original_unavailable + + +if __name__ == "__main__": + unittest.main()