Add cost-aware cache eviction policy

tests/unit/EvictionPolicyStrategy/test_eviction_policy.py

@@ -3,6 +3,9 @@
 from datetime import datetime, timezone
 from unittest.mock import MagicMock
 
+from vcache.vcache_core.cache.eviction_policy.strategies.cost_aware import (
+    CostAwareEvictionPolicy,
+)
 from vcache.vcache_core.cache.eviction_policy.strategies.fifo import (
     FIFOEvictionPolicy,
 )
@@ -33,6 +36,7 @@ def setUp(self):
             mock_meta.embedding_id = i
             mock_meta.created_at = datetime.now(timezone.utc)
             mock_meta.last_accessed = mock_meta.created_at
+            mock_meta.cost = None
             self.metadata.append(mock_meta)
             time.sleep(0.01)  # Ensure timestamps are distinct
 
@@ -138,6 +142,46 @@ def test_scu_fallback_eviction(self):
         self.assertEqual(len(victims), self.num_to_evict)
         self.assertEqual(sorted(victims), [2, 3])
 
+    def test_cost_aware_matches_lru_with_no_cost_data(self):
+        """With no cost data, CostAwareEvictionPolicy should behave like LRU."""
+        policy = CostAwareEvictionPolicy(self.max_size, 0.9, self.eviction_percentage)
+
+        # Simulate access to items 0 and 1, making them the most recently used
+        policy.update_eviction_metadata(self.metadata[0])
+        time.sleep(0.01)
+        policy.update_eviction_metadata(self.metadata[1])
+
+        victims = policy.select_victims(self.metadata)
+
+        # Same result as plain LRU: items 2 and 3 are the least recently used
+        self.assertEqual(len(victims), self.num_to_evict)
+        self.assertEqual(sorted(victims), [2, 3])
+
+    def test_cost_aware_protects_expensive_items(self):
+        """CostAwareEvictionPolicy should protect stale but expensive items."""
+        policy = CostAwareEvictionPolicy(
+            self.max_size, 0.9, self.eviction_percentage, cost_weight=0.9
+        )
+
+        # Simulate access to items 0 and 1, making them the most recently used.
+        # This leaves items 2, 3, and 4 as the stalest, in that order (item 2
+        # is the most stale, matching the plain-LRU test which evicts [2, 3]).
+        policy.update_eviction_metadata(self.metadata[0])
+        time.sleep(0.01)
+        policy.update_eviction_metadata(self.metadata[1])
+
+        for meta in self.metadata:
+            meta.cost = 0.0
+        # Item 2 was very expensive to generate, so it should be protected
+        # from eviction in favor of the next stalest (cheap) item, item 4.
+        self.metadata[2].cost = 100.0
+
+        victims = policy.select_victims(self.metadata)
+
+        self.assertEqual(len(victims), self.num_to_evict)
+        self.assertNotIn(2, victims)
+        self.assertEqual(sorted(victims), [3, 4])
+
 
 if __name__ == "__main__":
     unittest.main()

tests/unit/VCachePolicyStrategy/test_vcache_policy.py

@@ -1,6 +1,6 @@
 import unittest
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from unittest.mock import MagicMock, patch
+from unittest.mock import ANY, MagicMock, patch
 
 from vcache.config import VCacheConfig
 from vcache.vcache_core.cache.embedding_store.embedding_metadata_storage.embedding_metadata_obj import (
@@ -23,7 +23,7 @@ def setUp(self):
         self.mock_metadata_store = {}
         self.next_embedding_id = 0
 
-        def add_to_cache(prompt, response, id_set=None):
+        def add_to_cache(prompt, response, id_set=None, cost=None):
             self.next_embedding_id += 1
             # Simulate adding metadata
             mock_meta = MagicMock(spec=EmbeddingMetadataObj)
@@ -75,7 +75,7 @@ def test_empty_cache_is_miss(self):
         self.assertFalse(is_hit)
         self.assertEqual(response, "new response")
         self.mock_cache.add.assert_called_once_with(
-            prompt="prompt", response="new response", id_set=1
+            prompt="prompt", response="new response", id_set=1, cost=ANY
         )
 
     @patch("vcache.vcache_policy.strategies.verified._Algorithm.select_action")
@@ -208,7 +208,7 @@ def test_race_condition_read_side_graceful_failure(self):
         self.assertEqual(response, "fallback response")
         # It should have been treated as a cache miss, so add is called
         self.mock_cache.add.assert_called_once_with(
-            prompt="prompt", response="fallback response", id_set=1
+            prompt="prompt", response="fallback response", id_set=1, cost=ANY
         )
 
     def test_race_condition_update_vs_eviction(self):

vcache/__init__.py

@@ -50,6 +50,7 @@
 
 # Concrete Eviction policies
 from .vcache_core.cache.eviction_policy import (
+    CostAwareEvictionPolicy,
     FIFOEvictionPolicy,
     LRUEvictionPolicy,
     MRUEvictionPolicy,
@@ -125,6 +126,7 @@
     "FIFOEvictionPolicy",
     "NoEvictionPolicy",
     "SCUEvictionPolicy",
+    "CostAwareEvictionPolicy",
     # Concrete Embedding metadata storage
     "InMemoryEmbeddingMetadataStorage",
     "LangchainMetadataStorage",

vcache/vcache_core/cache/cache.py

@@ -30,7 +30,7 @@ def __init__(
         self.embedding_engine: EmbeddingEngine = embedding_engine
         self.eviction_policy: EvictionPolicy = eviction_policy
 
-    def add(self, prompt: str, response: str, id_set: int) -> int:
+    def add(self, prompt: str, response: str, id_set: int, cost: float = None) -> int:
         """Computes and adds an embedding to the vector database and metadata store.
 
         Note:
@@ -44,12 +44,14 @@ def add(self, prompt: str, response: str, id_set: int) -> int:
             id_set (int): The set identifier for the embedding. This is used in the
                 benchmark to identify if the nearest neighbor is from the same set
                 (if the cached response is correct or incorrect).
+            cost (float): The cost (e.g. LLM inference latency in seconds) incurred
+                to generate the response. Used by cost-aware eviction policies.
 
         Returns:
             int: The ID of the newly added embedding.
         """
         embedding = self.embedding_engine.get_embedding(prompt)
-        return self.embedding_store.add_embedding(embedding, response, id_set)
+        return self.embedding_store.add_embedding(embedding, response, id_set, cost)
 
     def remove(self, embedding_id: int) -> int:
         """Removes an embedding and its metadata from the cache.

vcache/vcache_core/cache/embedding_store/embedding_metadata_storage/embedding_metadata_obj.py

@@ -13,6 +13,7 @@ def __init__(
         response: str,
         last_accessed: Optional[datetime] = None,
         id_set: int = -1,
+        cost: Optional[float] = None,
     ):
         """Initializes the embedding metadata object.
 
@@ -25,6 +26,11 @@ def __init__(
             id_set (int): The set identifier for the embedding. This is used in the
                 benchmark to identify if the nearest neighbor is from the same set
                 (if the cached response is correct or incorrect).
+            cost (Optional[float]): The cost (e.g. LLM inference latency in
+                seconds) that was incurred to generate the cached response. Used
+                by cost-aware eviction policies to avoid evicting expensive-to
+                -regenerate responses. If unknown, treated as free by such
+                policies.
         """
 
         #### Core metadata ###################################################
@@ -49,6 +55,7 @@ def __init__(
         )
         self.created_at: datetime = datetime.now(timezone.utc)
         self.usage_count: int = 0
+        self.cost: Optional[float] = cost
 
     def __eq__(self, other: object) -> bool:
         """Checks for equality with another EmbeddingMetadataObj.

vcache/vcache_core/cache/embedding_store/embedding_store.py

@@ -32,7 +32,13 @@ def __init__(
         self._add_lock = threading.Lock()
         self._remove_lock = threading.Lock()
 
-    def add_embedding(self, embedding: List[float], response: str, id_set: int) -> int:
+    def add_embedding(
+        self,
+        embedding: List[float],
+        response: str,
+        id_set: int,
+        cost: float = None,
+    ) -> int:
         """Adds an embedding and its metadata to the store.
 
         This operation is thread-safe.
@@ -43,6 +49,8 @@ def add_embedding(self, embedding: List[float], response: str, id_set: int) -> i
             id_set (int): The set identifier for the embedding. This is used in the
                 benchmark to identify if the nearest neighbor is from the same set
                 (if the cached response is correct or incorrect).
+            cost (float): The cost (e.g. LLM inference latency in seconds) incurred
+                to generate the response. Used by cost-aware eviction policies.
 
         Returns:
             int: The ID of the added embedding.
@@ -53,6 +61,7 @@ def add_embedding(self, embedding: List[float], response: str, id_set: int) -> i
                 embedding_id=embedding_id,
                 response=response,
                 id_set=id_set,
+                cost=cost,
             )
             self.embedding_metadata_storage.add_metadata(
                 embedding_id=embedding_id, metadata=metadata

vcache/vcache_core/cache/eviction_policy/README.md

@@ -1,6 +1,6 @@
 # Cache Eviction Strategies
 
-A cache eviction policy is a crucial component of `vCache` that determines which items to remove when the cache reaches its capacity. The choice of policy significantly impacts cache performance, as it dictates how the system prioritizes the stored information. `vCache` supports both traditional heuristics (FIFO, LRU, MRU) and a novel, statistically-driven strategy (SCU).
+A cache eviction policy is a crucial component of `vCache` that determines which items to remove when the cache reaches its capacity. The choice of policy significantly impacts cache performance, as it dictates how the system prioritizes the stored information. `vCache` supports traditional heuristics (FIFO, LRU, MRU), a cost-aware heuristic (Cost-Aware), and a novel, statistically-driven strategy (SCU).
 
 
 ## First-In, First-Out (FIFO)
@@ -21,6 +21,26 @@ Conversely, the MRU policy evicts the items that have been accessed most recentl
 
 
 
+## Cost-Aware Eviction
+
+Traditional recency-based policies like LRU treat every cached item as equally expensive to regenerate. In practice, the cost of a cache miss can vary a lot: some responses come from a cheap, fast model call, while others come from an expensive, slow one. The `CostAwareEvictionPolicy` accounts for this by combining an item's staleness with the cost that was incurred to generate its cached response, so that expensive items are protected from eviction even if they are accessed less frequently.
+
+Each item added to the cache can carry a `cost` (e.g. the LLM inference latency, in seconds, that was measured when the response was generated). This is threaded through `Cache.add()` / `EmbeddingStore.add_embedding()` into the item's `EmbeddingMetadataObj`. If the cost is unknown, it is treated as free.
+
+### Eviction Priority
+
+For every item $i$, the policy computes a min-max normalized staleness (time since `last_accessed`, scaled so the freshest item in the cache maps to 0 and the most stale maps to 1) and a min-max normalized cost (scaled so the cheapest item maps to 0 and the most expensive maps to 1). The eviction priority is:
+
+$P_i = (1 - w) \cdot \text{staleness}'_i + w \cdot (1 - \text{cost}'_i)$
+
+where $w$ is the configurable `cost_weight` in $[0, 1]$. Items with the highest priority are evicted first.
+
+- With `cost_weight = 0`, this reduces exactly to LRU.
+- With `cost_weight = 1`, only cost matters and the cheapest items are evicted first, regardless of recency.
+- The default `cost_weight = 0.5` balances both signals.
+
+Because unknown costs default to `0.0`, a cache with no cost data behaves identically to LRU without any special-cased fallback logic.
+
 ## Sky Confident Utility (SCU)
 
 Traditional cache eviction policies, such as LRU and FIFO, rely on temporal heuristics that are poorly suited for intelligent caching systems. These methods fail to consider the learned, per-item performance characteristics available in `vCache`.

vcache/vcache_core/cache/eviction_policy/__init__.py

@@ -1,4 +1,7 @@
 from vcache.vcache_core.cache.eviction_policy.eviction_policy import EvictionPolicy
+from vcache.vcache_core.cache.eviction_policy.strategies.cost_aware import (
+    CostAwareEvictionPolicy,
+)
 from vcache.vcache_core.cache.eviction_policy.strategies.fifo import FIFOEvictionPolicy
 from vcache.vcache_core.cache.eviction_policy.strategies.lru import LRUEvictionPolicy
 from vcache.vcache_core.cache.eviction_policy.strategies.mru import MRUEvictionPolicy
@@ -14,4 +17,5 @@
     "NoEvictionPolicy",
     "EvictionPolicy",
     "SCUEvictionPolicy",
+    "CostAwareEvictionPolicy",
 ]

vcache/vcache_core/cache/eviction_policy/strategies/__init__.py

@@ -1,3 +1,6 @@
+from vcache.vcache_core.cache.eviction_policy.strategies.cost_aware import (
+    CostAwareEvictionPolicy,
+)
 from vcache.vcache_core.cache.eviction_policy.strategies.fifo import FIFOEvictionPolicy
 from vcache.vcache_core.cache.eviction_policy.strategies.lru import LRUEvictionPolicy
 from vcache.vcache_core.cache.eviction_policy.strategies.mru import MRUEvictionPolicy
@@ -12,4 +15,5 @@
     "FIFOEvictionPolicy",
     "NoEvictionPolicy",
     "SCUEvictionPolicy",
+    "CostAwareEvictionPolicy",
 ]