From eb5fd93c6f5bc263496bf9a7b4213b59e3ff1383 Mon Sep 17 00:00:00 2001 From: safaa Date: Thu, 16 Jul 2026 16:15:24 +0300 Subject: [PATCH 1/3] Add cost-aware cache eviction policy Protects cached responses that were expensive to generate (e.g. slow LLM calls) from eviction, blending generation cost with staleness instead of relying on recency alone like LRU. --- .../test_eviction_policy.py | 44 ++++++ .../test_vcache_policy.py | 8 +- vcache/__init__.py | 2 + vcache/vcache_core/cache/cache.py | 8 +- .../embedding_metadata_obj.py | 7 + .../cache/embedding_store/embedding_store.py | 11 +- .../cache/eviction_policy/README.md | 22 ++- .../cache/eviction_policy/__init__.py | 4 + .../eviction_policy/strategies/__init__.py | 4 + .../eviction_policy/strategies/cost_aware.py | 128 ++++++++++++++++++ vcache/vcache_policy/strategies/verified.py | 28 +++- 11 files changed, 255 insertions(+), 11 deletions(-) create mode 100644 vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py diff --git a/tests/unit/EvictionPolicyStrategy/test_eviction_policy.py b/tests/unit/EvictionPolicyStrategy/test_eviction_policy.py index 52fadc4..c369291 100644 --- a/tests/unit/EvictionPolicyStrategy/test_eviction_policy.py +++ b/tests/unit/EvictionPolicyStrategy/test_eviction_policy.py @@ -3,6 +3,9 @@ from datetime import datetime, timezone from unittest.mock import MagicMock +from vcache.vcache_core.cache.eviction_policy.strategies.cost_aware import ( + CostAwareEvictionPolicy, +) from vcache.vcache_core.cache.eviction_policy.strategies.fifo import ( FIFOEvictionPolicy, ) @@ -33,6 +36,7 @@ def setUp(self): mock_meta.embedding_id = i mock_meta.created_at = datetime.now(timezone.utc) mock_meta.last_accessed = mock_meta.created_at + mock_meta.cost = None self.metadata.append(mock_meta) time.sleep(0.01) # Ensure timestamps are distinct @@ -138,6 +142,46 @@ def test_scu_fallback_eviction(self): self.assertEqual(len(victims), self.num_to_evict) self.assertEqual(sorted(victims), [2, 3]) + def test_cost_aware_matches_lru_with_no_cost_data(self): + """With no cost data, CostAwareEvictionPolicy should behave like LRU.""" + policy = CostAwareEvictionPolicy(self.max_size, 0.9, self.eviction_percentage) + + # Simulate access to items 0 and 1, making them the most recently used + policy.update_eviction_metadata(self.metadata[0]) + time.sleep(0.01) + policy.update_eviction_metadata(self.metadata[1]) + + victims = policy.select_victims(self.metadata) + + # Same result as plain LRU: items 2 and 3 are the least recently used + self.assertEqual(len(victims), self.num_to_evict) + self.assertEqual(sorted(victims), [2, 3]) + + def test_cost_aware_protects_expensive_items(self): + """CostAwareEvictionPolicy should protect stale but expensive items.""" + policy = CostAwareEvictionPolicy( + self.max_size, 0.9, self.eviction_percentage, cost_weight=0.9 + ) + + # Simulate access to items 0 and 1, making them the most recently used. + # This leaves items 2, 3, and 4 as the stalest, in that order (item 2 + # is the most stale, matching the plain-LRU test which evicts [2, 3]). + policy.update_eviction_metadata(self.metadata[0]) + time.sleep(0.01) + policy.update_eviction_metadata(self.metadata[1]) + + for meta in self.metadata: + meta.cost = 0.0 + # Item 2 was very expensive to generate, so it should be protected + # from eviction in favor of the next stalest (cheap) item, item 4. + self.metadata[2].cost = 100.0 + + victims = policy.select_victims(self.metadata) + + self.assertEqual(len(victims), self.num_to_evict) + self.assertNotIn(2, victims) + self.assertEqual(sorted(victims), [3, 4]) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/VCachePolicyStrategy/test_vcache_policy.py b/tests/unit/VCachePolicyStrategy/test_vcache_policy.py index 48e3be5..04bc09e 100644 --- a/tests/unit/VCachePolicyStrategy/test_vcache_policy.py +++ b/tests/unit/VCachePolicyStrategy/test_vcache_policy.py @@ -1,6 +1,6 @@ import unittest from concurrent.futures import ThreadPoolExecutor, as_completed -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch from vcache.config import VCacheConfig from vcache.vcache_core.cache.embedding_store.embedding_metadata_storage.embedding_metadata_obj import ( @@ -23,7 +23,7 @@ def setUp(self): self.mock_metadata_store = {} self.next_embedding_id = 0 - def add_to_cache(prompt, response, id_set=None): + def add_to_cache(prompt, response, id_set=None, cost=None): self.next_embedding_id += 1 # Simulate adding metadata mock_meta = MagicMock(spec=EmbeddingMetadataObj) @@ -75,7 +75,7 @@ def test_empty_cache_is_miss(self): self.assertFalse(is_hit) self.assertEqual(response, "new response") self.mock_cache.add.assert_called_once_with( - prompt="prompt", response="new response", id_set=1 + prompt="prompt", response="new response", id_set=1, cost=ANY ) @patch("vcache.vcache_policy.strategies.verified._Algorithm.select_action") @@ -208,7 +208,7 @@ def test_race_condition_read_side_graceful_failure(self): self.assertEqual(response, "fallback response") # It should have been treated as a cache miss, so add is called self.mock_cache.add.assert_called_once_with( - prompt="prompt", response="fallback response", id_set=1 + prompt="prompt", response="fallback response", id_set=1, cost=ANY ) def test_race_condition_update_vs_eviction(self): diff --git a/vcache/__init__.py b/vcache/__init__.py index 10e2261..a9d3281 100644 --- a/vcache/__init__.py +++ b/vcache/__init__.py @@ -50,6 +50,7 @@ # Concrete Eviction policies from .vcache_core.cache.eviction_policy import ( + CostAwareEvictionPolicy, FIFOEvictionPolicy, LRUEvictionPolicy, MRUEvictionPolicy, @@ -125,6 +126,7 @@ "FIFOEvictionPolicy", "NoEvictionPolicy", "SCUEvictionPolicy", + "CostAwareEvictionPolicy", # Concrete Embedding metadata storage "InMemoryEmbeddingMetadataStorage", "LangchainMetadataStorage", diff --git a/vcache/vcache_core/cache/cache.py b/vcache/vcache_core/cache/cache.py index e22b75b..34d0fdc 100644 --- a/vcache/vcache_core/cache/cache.py +++ b/vcache/vcache_core/cache/cache.py @@ -30,7 +30,9 @@ def __init__( self.embedding_engine: EmbeddingEngine = embedding_engine self.eviction_policy: EvictionPolicy = eviction_policy - def add(self, prompt: str, response: str, id_set: int) -> int: + def add( + self, prompt: str, response: str, id_set: int, cost: float = None + ) -> int: """Computes and adds an embedding to the vector database and metadata store. Note: @@ -44,12 +46,14 @@ def add(self, prompt: str, response: str, id_set: int) -> int: id_set (int): The set identifier for the embedding. This is used in the benchmark to identify if the nearest neighbor is from the same set (if the cached response is correct or incorrect). + cost (float): The cost (e.g. LLM inference latency in seconds) incurred + to generate the response. Used by cost-aware eviction policies. Returns: int: The ID of the newly added embedding. """ embedding = self.embedding_engine.get_embedding(prompt) - return self.embedding_store.add_embedding(embedding, response, id_set) + return self.embedding_store.add_embedding(embedding, response, id_set, cost) def remove(self, embedding_id: int) -> int: """Removes an embedding and its metadata from the cache. diff --git a/vcache/vcache_core/cache/embedding_store/embedding_metadata_storage/embedding_metadata_obj.py b/vcache/vcache_core/cache/embedding_store/embedding_metadata_storage/embedding_metadata_obj.py index 894e661..bc3fe96 100644 --- a/vcache/vcache_core/cache/embedding_store/embedding_metadata_storage/embedding_metadata_obj.py +++ b/vcache/vcache_core/cache/embedding_store/embedding_metadata_storage/embedding_metadata_obj.py @@ -13,6 +13,7 @@ def __init__( response: str, last_accessed: Optional[datetime] = None, id_set: int = -1, + cost: Optional[float] = None, ): """Initializes the embedding metadata object. @@ -25,6 +26,11 @@ def __init__( id_set (int): The set identifier for the embedding. This is used in the benchmark to identify if the nearest neighbor is from the same set (if the cached response is correct or incorrect). + cost (Optional[float]): The cost (e.g. LLM inference latency in + seconds) that was incurred to generate the cached response. Used + by cost-aware eviction policies to avoid evicting expensive-to + -regenerate responses. If unknown, treated as free by such + policies. """ #### Core metadata ################################################### @@ -49,6 +55,7 @@ def __init__( ) self.created_at: datetime = datetime.now(timezone.utc) self.usage_count: int = 0 + self.cost: Optional[float] = cost def __eq__(self, other: object) -> bool: """Checks for equality with another EmbeddingMetadataObj. diff --git a/vcache/vcache_core/cache/embedding_store/embedding_store.py b/vcache/vcache_core/cache/embedding_store/embedding_store.py index f4bfcf5..2708a67 100644 --- a/vcache/vcache_core/cache/embedding_store/embedding_store.py +++ b/vcache/vcache_core/cache/embedding_store/embedding_store.py @@ -32,7 +32,13 @@ def __init__( self._add_lock = threading.Lock() self._remove_lock = threading.Lock() - def add_embedding(self, embedding: List[float], response: str, id_set: int) -> int: + def add_embedding( + self, + embedding: List[float], + response: str, + id_set: int, + cost: float = None, + ) -> int: """Adds an embedding and its metadata to the store. This operation is thread-safe. @@ -43,6 +49,8 @@ def add_embedding(self, embedding: List[float], response: str, id_set: int) -> i id_set (int): The set identifier for the embedding. This is used in the benchmark to identify if the nearest neighbor is from the same set (if the cached response is correct or incorrect). + cost (float): The cost (e.g. LLM inference latency in seconds) incurred + to generate the response. Used by cost-aware eviction policies. Returns: int: The ID of the added embedding. @@ -53,6 +61,7 @@ def add_embedding(self, embedding: List[float], response: str, id_set: int) -> i embedding_id=embedding_id, response=response, id_set=id_set, + cost=cost, ) self.embedding_metadata_storage.add_metadata( embedding_id=embedding_id, metadata=metadata diff --git a/vcache/vcache_core/cache/eviction_policy/README.md b/vcache/vcache_core/cache/eviction_policy/README.md index f04755e..8fdc138 100644 --- a/vcache/vcache_core/cache/eviction_policy/README.md +++ b/vcache/vcache_core/cache/eviction_policy/README.md @@ -1,6 +1,6 @@ # Cache Eviction Strategies -A cache eviction policy is a crucial component of `vCache` that determines which items to remove when the cache reaches its capacity. The choice of policy significantly impacts cache performance, as it dictates how the system prioritizes the stored information. `vCache` supports both traditional heuristics (FIFO, LRU, MRU) and a novel, statistically-driven strategy (SCU). +A cache eviction policy is a crucial component of `vCache` that determines which items to remove when the cache reaches its capacity. The choice of policy significantly impacts cache performance, as it dictates how the system prioritizes the stored information. `vCache` supports traditional heuristics (FIFO, LRU, MRU), a cost-aware heuristic (Cost-Aware), and a novel, statistically-driven strategy (SCU). ## First-In, First-Out (FIFO) @@ -21,6 +21,26 @@ Conversely, the MRU policy evicts the items that have been accessed most recentl +## Cost-Aware Eviction + +Traditional recency-based policies like LRU treat every cached item as equally expensive to regenerate. In practice, the cost of a cache miss can vary a lot: some responses come from a cheap, fast model call, while others come from an expensive, slow one. The `CostAwareEvictionPolicy` accounts for this by combining an item's staleness with the cost that was incurred to generate its cached response, so that expensive items are protected from eviction even if they are accessed less frequently. + +Each item added to the cache can carry a `cost` (e.g. the LLM inference latency, in seconds, that was measured when the response was generated). This is threaded through `Cache.add()` / `EmbeddingStore.add_embedding()` into the item's `EmbeddingMetadataObj`. If the cost is unknown, it is treated as free. + +### Eviction Priority + +For every item $i$, the policy computes a normalized staleness (time since `last_accessed`, relative to the most stale item in the cache) and a normalized cost (relative to the most expensive item in the cache). The eviction priority is: + +$P_i = (1 - w) \cdot \text{staleness}'_i + w \cdot (1 - \text{cost}'_i)$ + +where $w$ is the configurable `cost_weight` in $[0, 1]$. Items with the highest priority are evicted first. + +- With `cost_weight = 0`, this reduces exactly to LRU. +- With `cost_weight = 1`, only cost matters and the cheapest items are evicted first, regardless of recency. +- The default `cost_weight = 0.5` balances both signals. + +Because unknown costs default to `0.0`, a cache with no cost data behaves identically to LRU without any special-cased fallback logic. + ## Sky Confident Utility (SCU) Traditional cache eviction policies, such as LRU and FIFO, rely on temporal heuristics that are poorly suited for intelligent caching systems. These methods fail to consider the learned, per-item performance characteristics available in `vCache`. diff --git a/vcache/vcache_core/cache/eviction_policy/__init__.py b/vcache/vcache_core/cache/eviction_policy/__init__.py index f85eb59..943193d 100644 --- a/vcache/vcache_core/cache/eviction_policy/__init__.py +++ b/vcache/vcache_core/cache/eviction_policy/__init__.py @@ -1,4 +1,7 @@ from vcache.vcache_core.cache.eviction_policy.eviction_policy import EvictionPolicy +from vcache.vcache_core.cache.eviction_policy.strategies.cost_aware import ( + CostAwareEvictionPolicy, +) from vcache.vcache_core.cache.eviction_policy.strategies.fifo import FIFOEvictionPolicy from vcache.vcache_core.cache.eviction_policy.strategies.lru import LRUEvictionPolicy from vcache.vcache_core.cache.eviction_policy.strategies.mru import MRUEvictionPolicy @@ -14,4 +17,5 @@ "NoEvictionPolicy", "EvictionPolicy", "SCUEvictionPolicy", + "CostAwareEvictionPolicy", ] diff --git a/vcache/vcache_core/cache/eviction_policy/strategies/__init__.py b/vcache/vcache_core/cache/eviction_policy/strategies/__init__.py index ea37bfe..32877cd 100644 --- a/vcache/vcache_core/cache/eviction_policy/strategies/__init__.py +++ b/vcache/vcache_core/cache/eviction_policy/strategies/__init__.py @@ -1,3 +1,6 @@ +from vcache.vcache_core.cache.eviction_policy.strategies.cost_aware import ( + CostAwareEvictionPolicy, +) from vcache.vcache_core.cache.eviction_policy.strategies.fifo import FIFOEvictionPolicy from vcache.vcache_core.cache.eviction_policy.strategies.lru import LRUEvictionPolicy from vcache.vcache_core.cache.eviction_policy.strategies.mru import MRUEvictionPolicy @@ -12,4 +15,5 @@ "FIFOEvictionPolicy", "NoEvictionPolicy", "SCUEvictionPolicy", + "CostAwareEvictionPolicy", ] diff --git a/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py b/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py new file mode 100644 index 0000000..ad2f21b --- /dev/null +++ b/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py @@ -0,0 +1,128 @@ +from datetime import datetime, timezone +from typing import List, Tuple + +from vcache.vcache_core.cache.embedding_store.embedding_metadata_storage.embedding_metadata_obj import ( + EmbeddingMetadataObj, +) +from vcache.vcache_core.cache.eviction_policy.eviction_policy import EvictionPolicy + + +class CostAwareEvictionPolicy(EvictionPolicy): + _MIN_DATETIME: datetime = datetime.min.replace(tzinfo=timezone.utc) + + def __init__( + self, + max_size: int, + watermark: float = 0.95, + eviction_percentage: float = 0.1, + cost_weight: float = 0.5, + ): + """ + Implements a cost-aware eviction policy. + + This policy evicts items that are both stale (like LRU) and cheap to + regenerate, while protecting items whose cached response was expensive + to produce (e.g. a slow LLM call) even if they are accessed less + frequently. Each item's eviction priority is a weighted blend of its + normalized staleness and its normalized (inverted) generation cost: + + priority = (1 - cost_weight) * normalized_staleness + + cost_weight * (1 - normalized_cost) + + Items with the highest priority are evicted first. Items with unknown + cost (`None`) are treated as free, i.e. no more protected than under + plain LRU. With `cost_weight=0.0` this policy is equivalent to LRU. + + The eviction process is triggered when the number of items in the cache + exceeds a "high-watermark" threshold, which is a percentage of the + absolute `max_size`. Once triggered, the policy will evict a number + of items equivalent to `eviction_percentage` of the `max_size`. + + Args: + max_size: The absolute maximum number of items the cache can hold. + watermark: The percentage of `max_size` that triggers eviction. + eviction_percentage: The percentage of `max_size` to evict. + cost_weight: How strongly generation cost should be weighed against + staleness, in [0, 1]. Higher values protect expensive items more + strongly; 0 reduces this policy to plain LRU. + """ + super().__init__( + max_size=max_size, + watermark=watermark, + eviction_percentage=eviction_percentage, + ) + + if not (0 <= cost_weight <= 1.0): + cost_weight = 0.5 + self.logger.warning("Cost weight must be in [0,1]. Setting to 0.5.") + + self.cost_weight: float = cost_weight + + def update_eviction_metadata(self, metadata: EmbeddingMetadataObj) -> None: + """Updates the metadata object's last-accessed timestamp. + + Args: + metadata (EmbeddingMetadataObj): The metadata object to update. + """ + metadata.last_accessed = datetime.now(timezone.utc) + + def select_victims(self, all_metadata: List[EmbeddingMetadataObj]) -> List[int]: + """Selects victims for eviction based on staleness and generation cost. + + Args: + all_metadata (List[EmbeddingMetadataObj]): A list of all metadata + objects in the cache. + + Returns: + List[int]: A list of embedding IDs for the items to be evicted. + """ + num_to_evict: int = int(self.max_size * self.eviction_percentage) + if num_to_evict == 0 or not all_metadata: + return [] + + now: datetime = datetime.now(timezone.utc) + staleness: List[float] = [ + ( + now + - ( + meta.last_accessed + if meta.last_accessed is not None + else self._MIN_DATETIME + ) + ).total_seconds() + for meta in all_metadata + ] + costs: List[float] = [ + meta.cost if meta.cost is not None else 0.0 for meta in all_metadata + ] + + max_staleness: float = max(staleness) or 1.0 + max_cost: float = max(costs) or 1.0 + + priorities: List[Tuple[int, float]] = [] + for meta, stale, cost in zip(all_metadata, staleness, costs): + normalized_staleness: float = stale / max_staleness + normalized_cost: float = cost / max_cost + priority: float = ( + 1 - self.cost_weight + ) * normalized_staleness + self.cost_weight * (1 - normalized_cost) + priorities.append((meta.embedding_id, priority)) + + priorities.sort(key=lambda x: x[1], reverse=True) + victims: List[int] = [ + embedding_id for embedding_id, _ in priorities[:num_to_evict] + ] + return victims + + def __str__(self) -> str: + """Returns a string representation of the CostAwareEvictionPolicy. + + Returns: + str: A string representation of the instance. + """ + return ( + f"CostAwareEvictionPolicy(max_size={self.max_size}, " + f"watermark={self.watermark}, " + f"eviction_percentage={self.eviction_percentage}, " + f"cost_weight={self.cost_weight})" + ) diff --git a/vcache/vcache_policy/strategies/verified.py b/vcache/vcache_policy/strategies/verified.py index 740b6d2..ad25af7 100644 --- a/vcache/vcache_policy/strategies/verified.py +++ b/vcache/vcache_policy/strategies/verified.py @@ -199,10 +199,12 @@ def process_request( knn = self.cache.get_knn(prompt=prompt, k=1) if not knn: + start_time = time.time() response = self.inference_engine.create( prompt=prompt, system_prompt=system_prompt ) - self.cache.add(prompt=prompt, response=response, id_set=id_set) + cost = time.time() - start_time + self.cache.add(prompt=prompt, response=response, id_set=id_set, cost=cost) return False, response, EmbeddingMetadataObj(embedding_id=-1, response="") similarity_score, embedding_id = knn[0] @@ -213,10 +215,14 @@ def process_request( ) except Exception: # Cache eviction fallback + start_time = time.time() new_response: str = self.inference_engine.create( prompt=prompt, system_prompt=system_prompt ) - self.cache.add(prompt=prompt, response=new_response, id_set=id_set) + cost = time.time() - start_time + self.cache.add( + prompt=prompt, response=new_response, id_set=id_set, cost=cost + ) return ( False, new_response, @@ -231,9 +237,11 @@ def process_request( case _Action.EXPLOIT: return True, nn_metadata.response, nn_metadata case _Action.EXPLORE: + start_time = time.time() response = self.inference_engine.create( prompt=prompt, system_prompt=system_prompt ) + cost = time.time() - start_time self.__update_cache( response=response, @@ -242,6 +250,7 @@ def process_request( embedding_id=embedding_id, prompt=prompt, label_id_set=id_set, + cost=cost, ) return False, response, nn_metadata @@ -254,6 +263,7 @@ def __update_cache( embedding_id: int, prompt: str, label_id_set: int, + cost: float = None, ) -> None: """ Asynchronously validates the correctness of the cached response and updates the cache. @@ -274,6 +284,8 @@ def __update_cache( label_id_set: The set identifier for the embedding. This is used in the benchmark to identify if the nearest neighbor is from the same set (if the cached response is correct or incorrect). + cost: The cost (e.g. LLM inference latency in seconds) incurred to + generate the response. """ if self.executor is None: raise ValueError("Executor not initialized. Call setup() first.") @@ -287,6 +299,7 @@ def __update_cache( nn_metadata.response, label_id_set, nn_metadata.id_set, + cost, ) def __submit_for_background_update( @@ -298,6 +311,7 @@ def __submit_for_background_update( cached_response: str, label_id_set: int, nn_id_set: int, + cost: float = None, ): """ Submits a task to check answer similarity and queue a cache update. @@ -314,6 +328,8 @@ def __submit_for_background_update( cached_response: The response from the cached nearest neighbor. label_id_set: The id_set of the label. nn_id_set: The id_set of the nearest neighbor. + cost: The cost (e.g. LLM inference latency in seconds) incurred to + generate the new response. """ should_have_exploited = self.similarity_evaluator.answers_similar( a=new_response, b=cached_response, id_set_a=label_id_set, id_set_b=nn_id_set @@ -326,6 +342,7 @@ def __submit_for_background_update( embedding_id, prompt, label_id_set, + cost, ) ) @@ -351,6 +368,8 @@ def __perform_cache_update(self, update_args: tuple) -> None: - id_set (int): The set identifier for the embedding. This is used in the benchmark to identify if the nearest neighbor is from the same set (if the cached response is correct or incorrect). + - cost (float): The cost (e.g. LLM inference latency in + seconds) incurred to generate the new response. """ ( should_have_exploited, @@ -359,6 +378,7 @@ def __perform_cache_update(self, update_args: tuple) -> None: embedding_id, prompt, id_set, + cost, ) = update_args try: @@ -386,7 +406,9 @@ def __perform_cache_update(self, update_args: tuple) -> None: return if not should_have_exploited: - self.cache.add(prompt=prompt, response=new_response, id_set=id_set) + self.cache.add( + prompt=prompt, response=new_response, id_set=id_set, cost=cost + ) try: self.cache.update_metadata( From b1a5fcecdfb735d0553200dca37b4e565966f498 Mon Sep 17 00:00:00 2001 From: safaa Date: Fri, 17 Jul 2026 09:25:16 +0300 Subject: [PATCH 2/3] Address review feedback on cost-aware eviction policy - Extract normalization and priority calculation into helper methods (_staleness_seconds, _min_max_normalize, _compute_priority). - Switch from max-only scaling to true min-max normalization, so the freshest/cheapest item in the cache maps to 0 instead of just the most stale/expensive item mapping to 1. - Measure inference latency with time.perf_counter() instead of time.time(), since perf_counter is monotonic and immune to system clock adjustments that could otherwise produce inaccurate or negative latency values. --- .../cache/eviction_policy/README.md | 2 +- .../eviction_policy/strategies/cost_aware.py | 84 ++++++++++++++----- vcache/vcache_policy/strategies/verified.py | 12 +-- 3 files changed, 70 insertions(+), 28 deletions(-) diff --git a/vcache/vcache_core/cache/eviction_policy/README.md b/vcache/vcache_core/cache/eviction_policy/README.md index 8fdc138..685a81f 100644 --- a/vcache/vcache_core/cache/eviction_policy/README.md +++ b/vcache/vcache_core/cache/eviction_policy/README.md @@ -29,7 +29,7 @@ Each item added to the cache can carry a `cost` (e.g. the LLM inference latency, ### Eviction Priority -For every item $i$, the policy computes a normalized staleness (time since `last_accessed`, relative to the most stale item in the cache) and a normalized cost (relative to the most expensive item in the cache). The eviction priority is: +For every item $i$, the policy computes a min-max normalized staleness (time since `last_accessed`, scaled so the freshest item in the cache maps to 0 and the most stale maps to 1) and a min-max normalized cost (scaled so the cheapest item maps to 0 and the most expensive maps to 1). The eviction priority is: $P_i = (1 - w) \cdot \text{staleness}'_i + w \cdot (1 - \text{cost}'_i)$ diff --git a/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py b/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py index ad2f21b..9fba6eb 100644 --- a/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py +++ b/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py @@ -81,32 +81,21 @@ def select_victims(self, all_metadata: List[EmbeddingMetadataObj]) -> List[int]: return [] now: datetime = datetime.now(timezone.utc) - staleness: List[float] = [ - ( - now - - ( - meta.last_accessed - if meta.last_accessed is not None - else self._MIN_DATETIME - ) - ).total_seconds() - for meta in all_metadata - ] + staleness: List[float] = [self._staleness_seconds(meta, now) for meta in all_metadata] costs: List[float] = [ meta.cost if meta.cost is not None else 0.0 for meta in all_metadata ] - max_staleness: float = max(staleness) or 1.0 - max_cost: float = max(costs) or 1.0 + normalized_staleness: List[float] = self._min_max_normalize(staleness) + normalized_cost: List[float] = self._min_max_normalize(costs) - priorities: List[Tuple[int, float]] = [] - for meta, stale, cost in zip(all_metadata, staleness, costs): - normalized_staleness: float = stale / max_staleness - normalized_cost: float = cost / max_cost - priority: float = ( - 1 - self.cost_weight - ) * normalized_staleness + self.cost_weight * (1 - normalized_cost) - priorities.append((meta.embedding_id, priority)) + priorities: List[Tuple[int, float]] = [ + ( + meta.embedding_id, + self._compute_priority(stale, cost), + ) + for meta, stale, cost in zip(all_metadata, normalized_staleness, normalized_cost) + ] priorities.sort(key=lambda x: x[1], reverse=True) victims: List[int] = [ @@ -114,6 +103,59 @@ def select_victims(self, all_metadata: List[EmbeddingMetadataObj]) -> List[int]: ] return victims + def _staleness_seconds(self, metadata: EmbeddingMetadataObj, now: datetime) -> float: + """Computes how long ago an item was last accessed, in seconds. + + Args: + metadata (EmbeddingMetadataObj): The metadata object to inspect. + now (datetime): The current time to measure staleness against. + + Returns: + float: Seconds since `last_accessed`. Items that were never + accessed are treated as maximally stale. + """ + last_accessed = ( + metadata.last_accessed + if metadata.last_accessed is not None + else self._MIN_DATETIME + ) + return (now - last_accessed).total_seconds() + + @staticmethod + def _min_max_normalize(values: List[float]) -> List[float]: + """Min-max normalizes a list of values to the [0, 1] range. + + Args: + values (List[float]): The values to normalize. + + Returns: + List[float]: The normalized values, in the same order. If all + values are equal (zero range), every value normalizes to 0.0, + since there is no variation to distinguish them by. + """ + min_value: float = min(values) + max_value: float = max(values) + value_range: float = max_value - min_value + if value_range == 0: + return [0.0 for _ in values] + return [(value - min_value) / value_range for value in values] + + def _compute_priority( + self, normalized_staleness: float, normalized_cost: float + ) -> float: + """Computes an item's eviction priority from its normalized metrics. + + Args: + normalized_staleness (float): The item's staleness, normalized to [0, 1]. + normalized_cost (float): The item's generation cost, normalized to [0, 1]. + + Returns: + float: The eviction priority. Higher values are evicted first. + """ + return (1 - self.cost_weight) * normalized_staleness + self.cost_weight * ( + 1 - normalized_cost + ) + def __str__(self) -> str: """Returns a string representation of the CostAwareEvictionPolicy. diff --git a/vcache/vcache_policy/strategies/verified.py b/vcache/vcache_policy/strategies/verified.py index ad25af7..3826ad8 100644 --- a/vcache/vcache_policy/strategies/verified.py +++ b/vcache/vcache_policy/strategies/verified.py @@ -199,11 +199,11 @@ def process_request( knn = self.cache.get_knn(prompt=prompt, k=1) if not knn: - start_time = time.time() + start_time = time.perf_counter() response = self.inference_engine.create( prompt=prompt, system_prompt=system_prompt ) - cost = time.time() - start_time + cost = time.perf_counter() - start_time self.cache.add(prompt=prompt, response=response, id_set=id_set, cost=cost) return False, response, EmbeddingMetadataObj(embedding_id=-1, response="") @@ -215,11 +215,11 @@ def process_request( ) except Exception: # Cache eviction fallback - start_time = time.time() + start_time = time.perf_counter() new_response: str = self.inference_engine.create( prompt=prompt, system_prompt=system_prompt ) - cost = time.time() - start_time + cost = time.perf_counter() - start_time self.cache.add( prompt=prompt, response=new_response, id_set=id_set, cost=cost ) @@ -237,11 +237,11 @@ def process_request( case _Action.EXPLOIT: return True, nn_metadata.response, nn_metadata case _Action.EXPLORE: - start_time = time.time() + start_time = time.perf_counter() response = self.inference_engine.create( prompt=prompt, system_prompt=system_prompt ) - cost = time.time() - start_time + cost = time.perf_counter() - start_time self.__update_cache( response=response, From 090e6674e522f4ce8c4f718b458bdf881645df86 Mon Sep 17 00:00:00 2001 From: safaa Date: Fri, 17 Jul 2026 20:39:28 +0300 Subject: [PATCH 3/3] Apply ruff formatting --- vcache/vcache_core/cache/cache.py | 4 +--- .../cache/eviction_policy/strategies/cost_aware.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/vcache/vcache_core/cache/cache.py b/vcache/vcache_core/cache/cache.py index 34d0fdc..81f9943 100644 --- a/vcache/vcache_core/cache/cache.py +++ b/vcache/vcache_core/cache/cache.py @@ -30,9 +30,7 @@ def __init__( self.embedding_engine: EmbeddingEngine = embedding_engine self.eviction_policy: EvictionPolicy = eviction_policy - def add( - self, prompt: str, response: str, id_set: int, cost: float = None - ) -> int: + def add(self, prompt: str, response: str, id_set: int, cost: float = None) -> int: """Computes and adds an embedding to the vector database and metadata store. Note: diff --git a/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py b/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py index 9fba6eb..c9d69f0 100644 --- a/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py +++ b/vcache/vcache_core/cache/eviction_policy/strategies/cost_aware.py @@ -81,7 +81,9 @@ def select_victims(self, all_metadata: List[EmbeddingMetadataObj]) -> List[int]: return [] now: datetime = datetime.now(timezone.utc) - staleness: List[float] = [self._staleness_seconds(meta, now) for meta in all_metadata] + staleness: List[float] = [ + self._staleness_seconds(meta, now) for meta in all_metadata + ] costs: List[float] = [ meta.cost if meta.cost is not None else 0.0 for meta in all_metadata ] @@ -94,7 +96,9 @@ def select_victims(self, all_metadata: List[EmbeddingMetadataObj]) -> List[int]: meta.embedding_id, self._compute_priority(stale, cost), ) - for meta, stale, cost in zip(all_metadata, normalized_staleness, normalized_cost) + for meta, stale, cost in zip( + all_metadata, normalized_staleness, normalized_cost + ) ] priorities.sort(key=lambda x: x[1], reverse=True) @@ -103,7 +107,9 @@ def select_victims(self, all_metadata: List[EmbeddingMetadataObj]) -> List[int]: ] return victims - def _staleness_seconds(self, metadata: EmbeddingMetadataObj, now: datetime) -> float: + def _staleness_seconds( + self, metadata: EmbeddingMetadataObj, now: datetime + ) -> float: """Computes how long ago an item was last accessed, in seconds. Args: