From 82adf28cf01ecf6b77b9d8f9a0f23feec15a99e7 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Thu, 23 Jul 2026 20:16:20 +0530 Subject: [PATCH 01/10] docs(examples): fix Compress/PromptCompress return signatures in README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The examples showed 'out, err := tok.Compress(...)' but the actual API returns (string, Stats) and (string, PromptStats) respectively β€” no error return. Fix the examples to match the real signatures. --- examples/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/README.md b/examples/README.md index afcd12adc..8f6121c70 100644 --- a/examples/README.md +++ b/examples/README.md @@ -21,14 +21,15 @@ import "github.com/GrayCodeAI/tok" in := "Hey, could you please help me figure out why this React component keeps " + "re-rendering every time the props change?" -out, err := tok.Compress(in, tok.Aggressive) +out, stats := tok.Compress(in, tok.Aggressive) // out: "React component re-renders on prop change. Why?" +// stats: compression ratio, token savings, etc. ``` `PromptCompress` lets you pick an intensity level directly: ```go -out, err := tok.PromptCompress(in, tok.IntensityFull) +out, stats := tok.PromptCompress(in, tok.IntensityFull) // intensities: tok.IntensityLite, tok.IntensityFull, tok.IntensityUltra ``` From a553769f6c4d89c5c3d3f6896b38a2798225a67e Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Fri, 24 Jul 2026 04:24:28 +0530 Subject: [PATCH 02/10] feat: add EstimateCost(text, model) and fix doc examples tok.EstimateCost was referenced across README, QUICK_START, DEPLOYMENT, AGENT_INTEGRATION, TROUBLESHOOTING, PERFORMANCE, and examples/README but did not exist. Adds the missing function (estimates USD cost of text as input tokens via EstimateTokensForModel + GetModelPricing) with tests, and fixes the doc examples to match the actual API: - Compress/PromptCompress return (string, Stats)/(string, PromptStats), not (string, error): corrected out, err := to out, _ := / out, stats := - PromptCompress single-value assignments now use out, _ := - EstimateCost now takes text (string) and returns float64 (single value) - Fixed 3-value Compress misuses in architecture.md --- README.md | 2 +- cost.go | 14 ++++++++++++++ cost_test.go | 32 ++++++++++++++++++++++++++++++++ docs/AGENT_INTEGRATION.md | 5 ++--- docs/DEPLOYMENT.md | 2 +- docs/PERFORMANCE.md | 17 +++++++---------- docs/QUICK_START.md | 21 ++++++++++----------- docs/SECURITY.md | 2 +- docs/TROUBLESHOOTING.md | 9 ++++----- docs/architecture.md | 4 ++-- examples/README.md | 11 +++++------ 11 files changed, 79 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index c7faa21d9..58424aa93 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ import "github.com/GrayCodeAI/tok" ### Compress a prompt ```go -out := tok.PromptCompress("Please implement authentication", tok.IntensityUltra) +out, _ := tok.PromptCompress("Please implement authentication", tok.IntensityUltra) // β†’ "Implement auth." ``` diff --git a/cost.go b/cost.go index 987d668e2..3aa806d41 100644 --- a/cost.go +++ b/cost.go @@ -129,6 +129,20 @@ func EstimateCostSavings(stats Stats, model string) float64 { return float64(stats.TokensSaved) / 1000 * pricing.InputPricePer1K } +// EstimateCost estimates the USD cost of processing text with a model, +// counting the text as input tokens. It is a convenience wrapper around +// EstimateTokensForModel and GetModelPricing. +// +// Returns 0 if the model is unknown (no pricing registered). +func EstimateCost(text string, model string) float64 { + pricing, ok := GetModelPricing(model) + if !ok { + return 0 + } + tokens := EstimateTokensForModel(text, model) + return float64(tokens) / 1000 * pricing.InputPricePer1K +} + // FormatCostSavings returns a human-readable cost savings string. // Returns "$X.XX saved" if the model is known, or empty string if unknown. func FormatCostSavings(stats Stats, model string) string { diff --git a/cost_test.go b/cost_test.go index d55223e1b..a91b8b726 100644 --- a/cost_test.go +++ b/cost_test.go @@ -345,6 +345,38 @@ func TestListModels_NoDuplicates(t *testing.T) { } } +func TestEstimateCost(t *testing.T) { + // gpt-4o input pricing is $0.0025 per 1K tokens. + pricing, ok := GetModelPricing("gpt-4o") + if !ok { + t.Fatal("expected gpt-4o pricing to be registered") + } + + text := "Hello, world! This is a test of the EstimateCost function." + tokens := EstimateTokensForModel(text, "gpt-4o") + want := float64(tokens) / 1000 * pricing.InputPricePer1K + + got := EstimateCost(text, "gpt-4o") + if got != want { + t.Errorf("EstimateCost = %v, want %v", got, want) + } + if got == 0 { + t.Error("expected non-zero cost for gpt-4o") + } +} + +func TestEstimateCost_UnknownModel(t *testing.T) { + if got := EstimateCost("some text", "totally-fake-model-xyz"); got != 0 { + t.Errorf("EstimateCost for unknown model = %v, want 0", got) + } +} + +func TestEstimateCost_EmptyText(t *testing.T) { + if got := EstimateCost("", "gpt-4o"); got != 0 { + t.Errorf("EstimateCost for empty text = %v, want 0", got) + } +} + func contains(s, substr string) bool { return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsSubstring(s, substr)) } diff --git a/docs/AGENT_INTEGRATION.md b/docs/AGENT_INTEGRATION.md index 260a873f1..72b527b86 100644 --- a/docs/AGENT_INTEGRATION.md +++ b/docs/AGENT_INTEGRATION.md @@ -58,15 +58,14 @@ tok.Compress(text, tok.WithMode(tok.ModeAggressive)) ### Compress a prompt ```go -out := tok.PromptCompress(prompt, tok.IntensityFull) +out, _ := tok.PromptCompress(prompt, tok.IntensityFull) // IntensityLite | IntensityFull | IntensityUltra ``` ### Estimate tokens and cost ```go -n := tok.EstimateTokensForModel(text, "gpt-4o") -cost := tok.EstimateCost(n, "gpt-4o") +cost := tok.EstimateCost(text, "gpt-4o") ``` ### Redact secrets before sending context to a model diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index b4afffb49..fd58ce69d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -38,7 +38,7 @@ import "github.com/GrayCodeAI/tok" ```go // Prompt compression (Lite / Full / Ultra intensities). -out := tok.PromptCompress("Please implement authentication", tok.IntensityUltra) +out, _ := tok.PromptCompress("Please implement authentication", tok.IntensityUltra) // Full output pipeline with options. out, _ := tok.Compress(verboseOutput, tok.Aggressive) diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 90b1805c9..f71ac3659 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -106,10 +106,10 @@ Tune the speed/compression trade-off through options passed to `tok.Compress`, o | Aggressive | `tok.Compress(text, tok.Aggressive)` | all | Strictest reduction | ```go -out, err := tok.Compress(input, tok.Aggressive) +out, _ := tok.Compress(input, tok.Aggressive) // Prompt-oriented helper with an intensity level: -out := tok.PromptCompress(input, tok.IntensityFull) +out, _ := tok.PromptCompress(input, tok.IntensityFull) ``` Additional compression options: @@ -137,7 +137,7 @@ rules, err := tok.LoadFilterRules("rules/custom.toml") if err != nil { log.Fatal(err) } -compressed, err := tok.Compress(text, tok.WithCustomFilters(rules)) +compressed, _ := tok.Compress(text, tok.WithCustomFilters(rules)) ``` Profiles (presets of filter behavior) load via `tok.LoadProfile`. @@ -147,11 +147,8 @@ Profiles (presets of filter behavior) load via `tok.LoadProfile`. ## Estimation, Cost & Tracking ```go -// Token estimate for a specific model (fast path for short strings). -n := tok.EstimateTokensForModel(text, "gpt-4o") - -// Estimated cost for a given token count / model. -cost := tok.EstimateCost(tokens, "gpt-4o") +// Estimated cost for a given model. +cost := tok.EstimateCost(text, "gpt-4o") // Track usage over a context's lifetime. tracker := tok.NewTracker(ctx) @@ -207,7 +204,7 @@ go test -bench=BenchmarkPipeline -benchmem Use a lighter intensity and let budget gates short-circuit work: ```go -out := tok.PromptCompress(text, tok.IntensityLite) +out, _ := tok.PromptCompress(text, tok.IntensityLite) ``` ### Optimize for Memory @@ -219,7 +216,7 @@ The pipeline uses pooled buffers and streams large inputs internally; prefer str Use the strongest settings when token budget matters most: ```go -out := tok.PromptCompress(text, tok.IntensityUltra) +out, _ := tok.PromptCompress(text, tok.IntensityUltra) // or out, _ := tok.Compress(text, tok.Aggressive) ``` diff --git a/docs/QUICK_START.md b/docs/QUICK_START.md index cd386839c..4ab2cb401 100644 --- a/docs/QUICK_START.md +++ b/docs/QUICK_START.md @@ -21,7 +21,7 @@ import "github.com/GrayCodeAI/tok" ## Compress in One Call ```go -out, err := tok.Compress(text) +out, _ := tok.Compress(text) ``` `Compress` uses an adaptive system that selects the right compression depth based @@ -42,31 +42,30 @@ If you want explicit control, pass options to `Compress`: ```go // Aggressive mode: deepest available compression path -out, err := tok.Compress(text, tok.Aggressive) +out, _ := tok.Compress(text, tok.Aggressive) // Code-aware compression: preserve structure for a given language -out, err := tok.Compress(src, tok.WithCodeAware("go")) +out, _ := tok.Compress(src, tok.WithCodeAware("go")) // Perplexity-guided pruning at a target retention ratio -out, err := tok.Compress(text, tok.WithPerplexityGuided(scorer, 0.5)) +out, _ := tok.Compress(text, tok.WithPerplexityGuided(scorer, 0.5)) // Custom TOML filter rules (see "Custom Filters" below) -out, err := tok.Compress(text, tok.WithCustomFilters(rules)) +out, _ := tok.Compress(text, tok.WithCustomFilters(rules)) ``` For prompt-style input there is a dedicated entry point with intensity levels: ```go -out, err := tok.PromptCompress(prompt, tok.IntensityLite) -out, err = tok.PromptCompress(prompt, tok.IntensityFull) -out, err = tok.PromptCompress(prompt, tok.IntensityUltra) +out, _ := tok.PromptCompress(prompt, tok.IntensityLite) +out, _ = tok.PromptCompress(prompt, tok.IntensityFull) +out, _ = tok.PromptCompress(prompt, tok.IntensityUltra) ``` ## Token & Cost Estimation ```go -tokens, err := tok.EstimateTokensForModel(text, "gpt-4o") -cost, err := tok.EstimateCost(text, "gpt-4o") +cost := tok.EstimateCost(text, "gpt-4o") ``` ## Redaction & Extraction Helpers @@ -98,7 +97,7 @@ rules, err := tok.LoadFilterRules("filters.toml") if err != nil { // handle error } -out, err := tok.Compress(text, tok.WithCustomFilters(rules)) +out, _ := tok.Compress(text, tok.WithCustomFilters(rules)) ``` You can also load a named profile: diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 61042061e..9a2c1cf7c 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -47,7 +47,7 @@ rules, err := tok.LoadFilterRules("filters.toml") if err != nil { log.Fatal(err) } -out, err := tok.Compress(text, tok.WithCustomFilters(rules)) +out, _ := tok.Compress(text, tok.WithCustomFilters(rules)) ``` Treat filter rule files as part of your trusted configuration. Set restrictive diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 41e455062..4d0a084db 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -73,7 +73,7 @@ import "github.com/GrayCodeAI/tok" ```go // Prompt compression at higher intensity -out := tok.PromptCompress(text, tok.IntensityFull) // or tok.IntensityUltra +out, _ := tok.PromptCompress(text, tok.IntensityFull) // or tok.IntensityUltra // Generic compression with aggressive mode out, _ := tok.Compress(text, tok.Aggressive) @@ -89,7 +89,7 @@ Available levels: `tok.IntensityLite`, `tok.IntensityFull`, `tok.IntensityUltra` 1. Drop to a lighter intensity: ```go - out := tok.PromptCompress(text, tok.IntensityLite) + out, _ := tok.PromptCompress(text, tok.IntensityLite) ``` 2. For source code, use the code-aware option so language structure is preserved: @@ -108,8 +108,7 @@ Use truncation and estimation helpers rather than guessing: ```go truncated := tok.SmartTruncate(text, maxTokens) -n := tok.EstimateTokensForModel(truncated, "gpt-4o") -cost := tok.EstimateCost(n, "gpt-4o") +cost := tok.EstimateCost(truncated, "gpt-4o") ``` ### Sensitive data appears in compressed output @@ -146,7 +145,7 @@ inputs that don't need it. 2. Use a lighter intensity for hot paths: ```go - out := tok.PromptCompress(text, tok.IntensityLite) + out, _ := tok.PromptCompress(text, tok.IntensityLite) ``` 3. For perplexity-guided compression, tune the keep ratio (lower = more diff --git a/docs/architecture.md b/docs/architecture.md index 622171c02..ae80f2b07 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -46,7 +46,7 @@ tok/ ```go // πŸ—œοΈ One-shot compression -compressed, stats, err := tok.Compress(text, +compressed, stats := tok.Compress(text, tok.WithTier(tok.TierCode), tok.WithBudget(4000), tok.WithQuery("implement OAuth flow"), @@ -54,7 +54,7 @@ compressed, stats, err := tok.Compress(text, // πŸ”„ Reusable compressor (caches tokenizer state) c := tok.NewCompressor(tok.Aggressive) -compressed, stats, err := c.Compress(text) +compressed, stats := c.Compress(text) // πŸ“Š Token estimation approx := tok.EstimateTokens(text) // fast, Β±5% diff --git a/examples/README.md b/examples/README.md index 8f6121c70..fda84294d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -36,8 +36,7 @@ out, stats := tok.PromptCompress(in, tok.IntensityFull) ### Estimate tokens and cost ```go -n := tok.EstimateTokensForModel(out, "claude-3-5-sonnet") -cost := tok.EstimateCost(n, "claude-3-5-sonnet") +cost := tok.EstimateCost(out, "claude-3-5-sonnet") ``` ## Advanced Examples @@ -53,24 +52,24 @@ if err != nil { log.Fatal(err) } -out, err := tok.Compress(in, tok.WithCustomFilters(rules)) +out, _ := tok.Compress(in, tok.WithCustomFilters(rules)) ``` ### Code-aware and perplexity-guided compression ```go // Preserve code structure for a given language. -out, err := tok.Compress(src, tok.WithCodeAware("go")) +out, _ := tok.Compress(src, tok.WithCodeAware("go")) // Drop the lowest-information tokens using your own scorer. -out, err = tok.Compress(in, tok.WithPerplexityGuided(scorer, 0.5)) +out, _ = tok.Compress(in, tok.WithPerplexityGuided(scorer, 0.5)) ``` ### Profiles ```go profile, err := tok.LoadProfile("profile.toml") -out, err := tok.Compress(in, profile.Options()...) +out, _ := tok.Compress(in, profile.Options()...) ``` ### Secret detection and sensitive files From 9863a9db4f60dc1dd68af4a1c239a4be134a8471 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Fri, 24 Jul 2026 10:54:06 +0530 Subject: [PATCH 03/10] fix(tok): sort ListModels output and assert ordering in test ListModels documented a sorted list but iterated the pricing map directly, yielding random order per run. Add sort.Strings and a sort.StringsAreSorted assertion so the contract is enforced. --- cost.go | 2 ++ cost_test.go | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/cost.go b/cost.go index 3aa806d41..8871d9b41 100644 --- a/cost.go +++ b/cost.go @@ -2,6 +2,7 @@ package tok import ( "fmt" + "sort" "strings" "sync" ) @@ -166,5 +167,6 @@ func ListModels() []string { for name := range modelPricingRegistry { models = append(models, name) } + sort.Strings(models) return models } diff --git a/cost_test.go b/cost_test.go index a91b8b726..b6f0713f8 100644 --- a/cost_test.go +++ b/cost_test.go @@ -1,6 +1,7 @@ package tok import ( + "sort" "testing" ) @@ -331,6 +332,11 @@ func TestListModels(t *testing.T) { t.Errorf("ListModels() missing expected model %q", m) } } + + // ListModels must return models in sorted order (deterministic). + if !sort.StringsAreSorted(models) { + t.Errorf("ListModels() not sorted: %v", models) + } } func TestListModels_NoDuplicates(t *testing.T) { From 8ff2a350c813167d7aebdb3c93f7080d85c0e2de Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 25 Jul 2026 12:31:15 +0530 Subject: [PATCH 04/10] feat: add runtime graph module - Add runtimegraph/ package with runtime analysis - Update ratelimit.go, ratelimit_test.go, go.mod - Update README --- README.md | 33 +++++- go.mod | 3 + ratelimit.go | 40 ++++++- ratelimit_test.go | 41 +++++++ runtimegraph/projection.go | 191 ++++++++++++++++++++++++++++++++ runtimegraph/projection_test.go | 57 ++++++++++ 6 files changed, 360 insertions(+), 5 deletions(-) create mode 100644 runtimegraph/projection.go create mode 100644 runtimegraph/projection_test.go diff --git a/README.md b/README.md index 58424aa93..0f44d993e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ tok is a **library**, not a CLI. It exposes token-efficiency primitives as a cle - **Output filtering** β€” a 31-layer pipeline (entropy pruning, perplexity filtering, AST-aware compression, H2O heavy-hitter, …) that strips noise from command output before it re-enters an LLM context. - **Token estimation** β€” `tok.EstimateTokens*` and `tok.EstimateCost` give model-aware token counts and pricing. - **Secrets scanning** β€” `tok.SecretDetector` and `tok.IsSensitiveFilename` catch credentials before they leak into prompts. -- **Rate limiting & tracking** β€” persistent SQLite-backed gain tracking (`tok.NewTracker`). +- **Usage limits & tracking** β€” thread-safe token/cost windows (`tok.NewUsageTracker`) plus persistent SQLite-backed gain tracking (`tok.NewTracker`). +- **Runtime graph projection** β€” privacy-safe compression, usage, budget, and redaction summaries (`runtimegraph.Build`). It is consumed directly as a Go module, and it powers the `tok` commands inside [Hawk](https://github.com/GrayCodeAI/hawk) (`hawk tok ...`), which imports it as a library. @@ -78,6 +79,28 @@ findings := d.DetectSecrets(text) redacted := d.RedactSecrets(text) ``` +### Track usage and enforce a budget + +```go +usage := tok.NewUsageTracker() +usage.SetLimits(tok.UsageLimits{ + HourlyTokens: 100_000, + SessionTokens: 500_000, + CostUSD: 10, +}) + +if allowed, _ := usage.CanProceed(); allowed { + usage.Record(promptTokens+completionTokens, requestCostUSD, provider, model) +} + +allowed, reason := usage.CanProceed() +snapshot := usage.GetUsage() +``` + +`SetLimits` and `GetLimits` are safe to use while requests are being recorded. +Callers own request deduplication and decide whether a failed decision blocks +the current request or the next one. + --- ## Library API Highlights @@ -88,6 +111,7 @@ redacted := d.RedactSecrets(text) - **`tok.SmartTruncate(content, maxLines, lang)`** β€” code truncation that preserves function signatures and **always reports the exact drop count** (`kept + dropped == total`). - **`tok.ExtractJSON / ExtractJSONArray / ExtractAllJSON`** β€” brace-balanced JSON extraction from LLM output with surrounding prose, markdown fences, and unterminated objects. - **`tok.NewTracker(ctx)`** β€” persistent gain tracker (SQLite + WAL, 90-day retention, pure-Go via `modernc.org/sqlite`). `Aggregate`, `Recent`, `Prune` queries. +- **`tok.NewUsageTracker()`** β€” in-memory hourly, daily, session, and cost accounting with atomic limit snapshots and threshold decisions. - **`tok.EstimateTokensFast / WithEncoding / ForModel`** β€” model-aware token estimation. - **`filter.CompressWithRetry`** β€” validate-fix-retry loop: caller supplies a `Validator` and `AdjustFunc`; the loop escalates mode/intensity and retries up to N times. - **`filter.NewTOMLFilter` / `LoadTOMLFilterFile`** β€” full 8-stage TOML pipeline as a pluggable `Filter`. @@ -168,6 +192,7 @@ Profile the hot paths with `./scripts/profile.sh [compress|tokens|filter|secrets ``` tok β”œβ”€β”€ tok.go, *.go Public library API (top-level package) +β”œβ”€β”€ runtimegraph/ Privacy-safe operations/policy/quality projection β”œβ”€β”€ internal/ β”‚ β”œβ”€β”€ compress/ Input compression engine (6 modes) β”‚ β”œβ”€β”€ filter/ Output pipeline (31 layers) @@ -181,6 +206,12 @@ tok Pure-Go, zero CGO, no runtime dependencies. +`runtimegraph.Build` projects completed compression statistics, usage +snapshots, budget decisions, and redaction summaries into `tok.graph/v1`. +Input text, model names, budget reasons, secret types, and secret values are +digest-only. Tok remains the source of truth for compression, limits, and +redaction; graph consumers receive immutable summaries. + --- ## Contributing diff --git a/go.mod b/go.mod index 97071ae8a..5d3066f95 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,15 @@ go 1.26.5 require ( github.com/BurntSushi/toml v1.6.0 + github.com/GrayCodeAI/hawk-core-contracts v0.1.8 github.com/spf13/viper v1.21.0 github.com/tiktoken-go/tokenizer v0.8.0 golang.org/x/sys v0.45.0 modernc.org/sqlite v1.51.0 ) +replace github.com/GrayCodeAI/hawk-core-contracts => ../hawk-core-contracts + require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2/v2 v2.1.0 // indirect diff --git a/ratelimit.go b/ratelimit.go index db9c3e0ce..6b37141a4 100644 --- a/ratelimit.go +++ b/ratelimit.go @@ -39,6 +39,14 @@ type UsageSummary struct { DailyPct float64 } +// UsageLimits is a concurrency-safe snapshot of UsageTracker limits. +type UsageLimits struct { + DailyTokens int + HourlyTokens int + SessionTokens int + CostUSD float64 +} + // UsageTracker tracks API usage across sessions and prevents surprise bills. type UsageTracker struct { DailyLimit int @@ -70,6 +78,30 @@ func NewUsageTracker() *UsageTracker { } } +// SetLimits atomically updates all token and cost ceilings. Zero disables the +// corresponding limit. Negative values are clamped to zero. +func (u *UsageTracker) SetLimits(limits UsageLimits) { + u.mu.Lock() + defer u.mu.Unlock() + u.DailyLimit = max(limits.DailyTokens, 0) + u.HourlyLimit = max(limits.HourlyTokens, 0) + u.SessionLimit = max(limits.SessionTokens, 0) + u.CostLimitUSD = math.Max(limits.CostUSD, 0) + u.checkThresholdsLocked() +} + +// GetLimits returns an atomic snapshot of the current ceilings. +func (u *UsageTracker) GetLimits() UsageLimits { + u.mu.Lock() + defer u.mu.Unlock() + return UsageLimits{ + DailyTokens: u.DailyLimit, + HourlyTokens: u.HourlyLimit, + SessionTokens: u.SessionLimit, + CostUSD: u.CostLimitUSD, + } +} + // Record adds a usage entry and checks thresholds. func (u *UsageTracker) Record(tokens int, costUSD float64, provider, model string) { u.mu.Lock() @@ -99,21 +131,21 @@ func (u *UsageTracker) CanProceed() (bool, string) { u.pruneOldLocked() hourlyTokens := u.hourlyTokensLocked() - if hourlyTokens >= u.HourlyLimit { + if u.HourlyLimit > 0 && hourlyTokens >= u.HourlyLimit { return false, fmt.Sprintf("hourly token limit reached (%d/%d)", hourlyTokens, u.HourlyLimit) } dailyTokens := u.dailyTokensLocked() - if dailyTokens >= u.DailyLimit { + if u.DailyLimit > 0 && dailyTokens >= u.DailyLimit { return false, fmt.Sprintf("daily token limit reached (%d/%d)", dailyTokens, u.DailyLimit) } - if u.sessionUsage >= u.SessionLimit { + if u.SessionLimit > 0 && u.sessionUsage >= u.SessionLimit { return false, fmt.Sprintf("session token limit reached (%d/%d)", u.sessionUsage, u.SessionLimit) } dailyCost := u.dailyCostLocked() - if dailyCost >= u.CostLimitUSD { + if u.CostLimitUSD > 0 && dailyCost >= u.CostLimitUSD { return false, fmt.Sprintf("daily cost limit reached ($%.2f/$%.2f)", dailyCost, u.CostLimitUSD) } diff --git a/ratelimit_test.go b/ratelimit_test.go index 09783541e..2797e7c2c 100644 --- a/ratelimit_test.go +++ b/ratelimit_test.go @@ -26,6 +26,47 @@ func TestRecordIncreasesCounters(t *testing.T) { } } +func TestUsageLimitsAtomicConfiguration(t *testing.T) { + tracker := NewUsageTracker() + tracker.SetLimits(UsageLimits{ + DailyTokens: 400, + HourlyTokens: 300, + SessionTokens: 200, + CostUSD: 1.5, + }) + + got := tracker.GetLimits() + if got != (UsageLimits{ + DailyTokens: 400, + HourlyTokens: 300, + SessionTokens: 200, + CostUSD: 1.5, + }) { + t.Fatalf("GetLimits() = %+v", got) + } + + tracker.SetLimits(UsageLimits{ + DailyTokens: -1, + HourlyTokens: -1, + SessionTokens: -1, + CostUSD: -1, + }) + if got := tracker.GetLimits(); got != (UsageLimits{}) { + t.Fatalf("negative limits were not clamped: %+v", got) + } +} + +func TestUsageLimitsZeroDisablesCeilings(t *testing.T) { + tracker := NewUsageTracker() + tracker.SetLimits(UsageLimits{}) + tracker.Record(2_000_000, 100, "provider", "model") + + allowed, reason := tracker.CanProceed() + if !allowed { + t.Fatalf("zero limits should disable every ceiling, got reason %q", reason) + } +} + func TestCanProceedReturnsFalseWhenHourlyLimitHit(t *testing.T) { tracker := NewUsageTracker() tracker.HourlyLimit = 1000 diff --git a/runtimegraph/projection.go b/runtimegraph/projection.go new file mode 100644 index 000000000..e4bb7ff3f --- /dev/null +++ b/runtimegraph/projection.go @@ -0,0 +1,191 @@ +// Package runtimegraph projects Tok compression, usage, budget, and redaction +// summaries into the portable hawk-eco graph contract. +package runtimegraph + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + graphcontracts "github.com/GrayCodeAI/hawk-core-contracts/graph" + "github.com/GrayCodeAI/tok" +) + +const SchemaVersion = "tok.graph/v1" + +type BudgetDecision struct { + Allowed bool + Reason string + HourlyLimit int + DailyLimit int + SessionLimit int + CostLimitUSD float64 +} + +type RedactionSummary struct { + MatchCount int + VerifiedCount int + Types map[string]int +} + +type Input struct { + Compression *tok.Stats + Usage *tok.UsageSummary + Budget *BudgetDecision + Redaction *RedactionSummary + Source string + ObservedAt time.Time + Scope graphcontracts.Scope + CorrelationID string + ProducerVersion string +} + +type Export struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt time.Time `json:"generated_at"` + Scope graphcontracts.Scope `json:"scope,omitempty"` + Nodes []graphcontracts.Node `json:"nodes"` + Edges []graphcontracts.Edge `json:"edges"` + Events []graphcontracts.Event `json:"events"` +} + +func Build(input Input) (*Export, error) { + if input.Compression == nil && input.Usage == nil && input.Budget == nil && input.Redaction == nil { + return nil, errors.New("runtimegraph: at least one summary is required") + } + at := input.ObservedAt.UTC() + if at.IsZero() { + at = time.Now().UTC() + } + sourceDigest := digest(input.Source) + export := &Export{ + SchemaVersion: SchemaVersion, GeneratedAt: at, Scope: input.Scope, + Nodes: []graphcontracts.Node{}, Edges: []graphcontracts.Edge{}, Events: []graphcontracts.Event{}, + } + var usageRef graphcontracts.Ref + if input.Compression != nil { + attrs := map[string]string{ + "entity": "compression", "source_digest": sourceDigest, + "original_tokens": strconv.Itoa(input.Compression.OriginalTokens), + "final_tokens": strconv.Itoa(input.Compression.FinalTokens), + "tokens_saved": strconv.Itoa(input.Compression.TokensSaved), + "reduction_percent": formatFloat(input.Compression.ReductionPercent), + "cost_savings_usd": formatFloat(input.Compression.CostSavings), + "model_digest": digest(input.Compression.Model), + "layer_count": strconv.Itoa(len(input.Compression.Layers)), + } + if _, err := addNode(export, graphcontracts.NodeOperations, "compression", attrs, input, at, sourceDigest); err != nil { + return nil, err + } + } + if input.Usage != nil { + attrs := map[string]string{ + "entity": "usage", "hourly_tokens": strconv.Itoa(input.Usage.HourlyTokens), + "hourly_remaining": strconv.Itoa(input.Usage.HourlyRemaining), + "daily_tokens": strconv.Itoa(input.Usage.DailyTokens), + "daily_remaining": strconv.Itoa(input.Usage.DailyRemaining), + "session_tokens": strconv.Itoa(input.Usage.SessionTokens), + "session_remaining": strconv.Itoa(input.Usage.SessionRemaining), + "daily_cost_usd": formatFloat(input.Usage.DailyCostUSD), + "cost_remaining_usd": formatFloat(input.Usage.CostRemaining), + "hourly_percent": formatFloat(input.Usage.HourlyPct), + "daily_percent": formatFloat(input.Usage.DailyPct), + } + ref, err := addNode(export, graphcontracts.NodeOperations, "usage", attrs, input, at, sourceDigest) + if err != nil { + return nil, err + } + usageRef = ref + } + if input.Budget != nil { + attrs := map[string]string{ + "entity": "budget_decision", "allowed": strconv.FormatBool(input.Budget.Allowed), + "reason_digest": digest(input.Budget.Reason), + "hourly_limit": strconv.Itoa(input.Budget.HourlyLimit), + "daily_limit": strconv.Itoa(input.Budget.DailyLimit), + "session_limit": strconv.Itoa(input.Budget.SessionLimit), + "cost_limit_usd": formatFloat(input.Budget.CostLimitUSD), + } + ref, err := addNode(export, graphcontracts.NodePolicy, "budget", attrs, input, at, sourceDigest) + if err != nil { + return nil, err + } + if usageRef.ID != "" { + if err := addEdge(export, ref, usageRef, graphcontracts.EdgeDependsOn, input, at); err != nil { + return nil, err + } + } + } + if input.Redaction != nil { + typeNames := make([]string, 0, len(input.Redaction.Types)) + for name := range input.Redaction.Types { + typeNames = append(typeNames, name) + } + sort.Strings(typeNames) + attrs := map[string]string{ + "entity": "redaction", "source_digest": sourceDigest, + "match_count": strconv.Itoa(max(input.Redaction.MatchCount, 0)), + "verified_count": strconv.Itoa(max(input.Redaction.VerifiedCount, 0)), + "type_count": strconv.Itoa(len(typeNames)), + "types_digest": digest(strings.Join(typeNames, "\x00")), + } + if _, err := addNode(export, graphcontracts.NodeQuality, "redaction", attrs, input, at, sourceDigest); err != nil { + return nil, err + } + } + sort.Slice(export.Nodes, func(i, j int) bool { return export.Nodes[i].ID < export.Nodes[j].ID }) + sort.Slice(export.Edges, func(i, j int) bool { return export.Edges[i].ID < export.Edges[j].ID }) + sort.Slice(export.Events, func(i, j int) bool { return export.Events[i].ID < export.Events[j].ID }) + return export, nil +} + +func addNode(export *Export, kind graphcontracts.NodeKind, entity string, attrs map[string]string, input Input, at time.Time, sourceDigest string) (graphcontracts.Ref, error) { + id := "tok/" + entity + "/" + digest(sourceDigest, at.Format(time.RFC3339Nano)) + ref := graphcontracts.Ref{Kind: kind, ID: id} + provenance := graphcontracts.Provenance{ + Producer: "tok", Version: strings.TrimSpace(input.ProducerVersion), SourceID: sourceDigest, + Evidence: []graphcontracts.ArtifactRef{{URI: "tok://" + entity + "/" + digest(sourceDigest)}}, + } + node := graphcontracts.Node{ID: id, Kind: kind, Scope: input.Scope, CreatedAt: at, Provenance: provenance, Attributes: attrs} + if err := node.Validate(); err != nil { + return graphcontracts.Ref{}, fmt.Errorf("runtimegraph: %s node: %w", entity, err) + } + event := graphcontracts.Event{ + ID: "tok/observed/" + digest(id, at.Format(time.RFC3339Nano)), Type: graphcontracts.EventObserved, + Subject: ref, Scope: input.Scope, OccurredAt: at, CorrelationID: strings.TrimSpace(input.CorrelationID), + IdempotencyKey: digest(id, at.Format(time.RFC3339Nano)), Provenance: provenance, + } + export.Nodes = append(export.Nodes, node) + export.Events = append(export.Events, event) + return ref, nil +} + +func addEdge(export *Export, from, to graphcontracts.Ref, kind graphcontracts.EdgeKind, input Input, at time.Time) error { + edge := graphcontracts.Edge{ + ID: "tok/edge/" + digest(from.ID, to.ID, string(kind)), Kind: kind, From: from, To: to, + Scope: input.Scope, CreatedAt: at, + Provenance: graphcontracts.Provenance{Producer: "tok", Version: strings.TrimSpace(input.ProducerVersion), SourceID: digest(input.Source)}, + } + if err := edge.Validate(); err != nil { + return fmt.Errorf("runtimegraph: edge: %w", err) + } + export.Edges = append(export.Edges, edge) + return nil +} + +func formatFloat(value float64) string { return strconv.FormatFloat(value, 'f', -1, 64) } + +func digest(parts ...string) string { + hash := sha256.New() + for _, part := range parts { + _, _ = hash.Write([]byte(strconv.Itoa(len(part)))) + _, _ = hash.Write([]byte{':'}) + _, _ = hash.Write([]byte(part)) + } + return hex.EncodeToString(hash.Sum(nil)) +} diff --git a/runtimegraph/projection_test.go b/runtimegraph/projection_test.go new file mode 100644 index 000000000..a33da642b --- /dev/null +++ b/runtimegraph/projection_test.go @@ -0,0 +1,57 @@ +package runtimegraph_test + +import ( + "encoding/json" + "strings" + "testing" + "time" + + graphcontracts "github.com/GrayCodeAI/hawk-core-contracts/graph" + "github.com/GrayCodeAI/tok" + "github.com/GrayCodeAI/tok/runtimegraph" +) + +func TestBuildMixedPrivacySafeProjection(t *testing.T) { + t.Parallel() + at := time.Date(2026, 7, 25, 14, 0, 0, 0, time.UTC) + stats := tok.Stats{OriginalTokens: 100, FinalTokens: 40, TokensSaved: 60, Model: "private-model"} + usage := tok.UsageSummary{HourlyTokens: 10, SessionTokens: 20, DailyCostUSD: .5} + export, err := runtimegraph.Build(runtimegraph.Input{ + Compression: &stats, Usage: &usage, + Budget: &runtimegraph.BudgetDecision{Allowed: false, Reason: "private budget reason", HourlyLimit: 10}, + Redaction: &runtimegraph.RedactionSummary{MatchCount: 2, Types: map[string]int{"OpenAI API Key": 2}}, + Source: "private source with sk-secret", ObservedAt: at, + Scope: graphcontracts.Scope{RepositoryID: "repo"}, CorrelationID: "session-1", + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + if len(export.Nodes) != 4 || len(export.Edges) != 1 || len(export.Events) != 4 { + t.Fatalf("unexpected sizes: nodes=%d edges=%d events=%d", len(export.Nodes), len(export.Edges), len(export.Events)) + } + payload, _ := json.Marshal(export) + for _, secret := range []string{"private source with sk-secret", "private budget reason", "private-model", "OpenAI API Key"} { + if strings.Contains(string(payload), secret) { + t.Fatalf("projection leaked %q", secret) + } + } + kinds := map[graphcontracts.NodeKind]bool{} + for _, node := range export.Nodes { + kinds[node.Kind] = true + if err := node.Validate(); err != nil { + t.Fatalf("invalid node: %v", err) + } + } + for _, want := range []graphcontracts.NodeKind{graphcontracts.NodeOperations, graphcontracts.NodePolicy, graphcontracts.NodeQuality} { + if !kinds[want] { + t.Fatalf("missing node kind %q", want) + } + } +} + +func TestBuildRequiresSummary(t *testing.T) { + t.Parallel() + if _, err := runtimegraph.Build(runtimegraph.Input{}); err == nil { + t.Fatal("expected empty input error") + } +} From 875d5af1d21689a3ee5c7020887e67d355e8f405 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 25 Jul 2026 13:06:40 +0530 Subject: [PATCH 05/10] feat: add runtime performance graph - Add RuntimeGraph implementation - Track runtime performance metrics across components - Support issue ranking and portable GraphSpec conversion --- runtimegraph/runtime_graph.go | 160 ++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 runtimegraph/runtime_graph.go diff --git a/runtimegraph/runtime_graph.go b/runtimegraph/runtime_graph.go new file mode 100644 index 000000000..4be715a7d --- /dev/null +++ b/runtimegraph/runtime_graph.go @@ -0,0 +1,160 @@ +// Package runtimegraph provides runtime performance graph analysis for tok. +package runtimegraph + +import ( + "fmt" + "sort" + "sync" + "time" + + graphcontracts "github.com/GrayCodeAI/hawk-core-contracts/graph" +) + +// RuntimeMetric represents a runtime performance metric. +type RuntimeMetric struct { + Name string `json:"name"` + Value float64 `json:"value"` + Threshold float64 `json:"threshold"` + Status string `json:"status"` // "pass", "warn", "fail" + UpdatedAt time.Time `json:"updated_at"` +} + +// RuntimeNode represents a runtime component with performance metrics. +type RuntimeNode struct { + ID string `json:"id"` + Type string `json:"type"` // "model", "endpoint", "pipeline", "service" + Name string `json:"name"` + Metrics []RuntimeMetric `json:"metrics"` + Children []string `json:"children,omitempty"` + Attrs map[string]interface{} `json:"attrs,omitempty"` +} + +// RuntimeEdge represents a runtime relationship between components. +type RuntimeEdge struct { + From string `json:"from"` + To string `json:"to"` + Kind string `json:"kind"` // "calls", "produces", "consumes", "depends_on" + Weight float64 `json:"weight"` +} + +// RuntimeGraph represents a runtime performance graph. +type RuntimeGraph struct { + mu sync.RWMutex + nodes map[string]*RuntimeNode + edges []RuntimeEdge +} + +// NewRuntimeGraph creates a new runtime graph. +func NewRuntimeGraph() *RuntimeGraph { + return &RuntimeGraph{ + nodes: make(map[string]*RuntimeNode), + } +} + +// AddNode adds a runtime node to the graph. +func (g *RuntimeGraph) AddNode(node *RuntimeNode) { + g.mu.Lock() + defer g.mu.Unlock() + g.nodes[node.ID] = node +} + +// AddEdge adds a runtime edge to the graph. +func (g *RuntimeGraph) AddEdge(edge RuntimeEdge) { + g.mu.Lock() + defer g.mu.Unlock() + g.edges = append(g.edges, edge) +} + +// GetNode retrieves a node by ID. +func (g *RuntimeGraph) GetNode(id string) (*RuntimeNode, bool) { + g.mu.RLock() + defer g.mu.RUnlock() + node, ok := g.nodes[id] + return node, ok +} + +// GetNodes returns all nodes. +func (g *RuntimeGraph) GetNodes() []*RuntimeNode { + g.mu.RLock() + defer g.mu.RUnlock() + result := make([]*RuntimeNode, 0, len(g.nodes)) + for _, node := range g.nodes { + result = append(result, node) + } + return result +} + +// GetEdges returns all edges. +func (g *RuntimeGraph) GetEdges() []RuntimeEdge { + g.mu.RLock() + defer g.mu.RUnlock() + return g.edges +} + +// GetFailedMetrics returns all metrics that failed their threshold. +func (g *RuntimeGraph) GetFailedMetrics() []RuntimeMetric { + g.mu.RLock() + defer g.mu.RUnlock() + result := []RuntimeMetric{} + for _, node := range g.nodes { + for _, metric := range node.Metrics { + if metric.Status == "fail" { + result = append(result, metric) + } + } + } + return result +} + +// GetTopIssues returns the top N issues by severity. +func (g *RuntimeGraph) GetTopIssues(n int) []RuntimeMetric { + failed := g.GetFailedMetrics() + sort.Slice(failed, func(i, j int) bool { + return failed[i].Value > failed[j].Value + }) + if len(failed) > n { + failed = failed[:n] + } + return failed +} + +// ToGraphSpec converts the runtime graph to a portable graph spec. +func (g *RuntimeGraph) ToGraphSpec() *graphcontracts.GraphSpec { + g.mu.RLock() + defer g.mu.RUnlock() + + nodes := make([]graphcontracts.NodeSpec, 0, len(g.nodes)) + for id, node := range g.nodes { + config := map[string]string{ + "name": node.Name, + "type": node.Type, + "metrics": fmt.Sprintf("%d", len(node.Metrics)), + } + for _, m := range node.Metrics { + config["metric_"+m.Name] = fmt.Sprintf("%.2f", m.Value) + } + + nodes = append(nodes, graphcontracts.NodeSpec{ + ID: id, + Type: graphcontracts.NodeTypeExecution, + Name: node.Name, + Config: config, + }) + } + + edges := make([]graphcontracts.EdgeSpec, 0, len(g.edges)) + for _, edge := range g.edges { + edges = append(edges, graphcontracts.EdgeSpec{ + From: edge.From, + To: edge.To, + Weight: edge.Weight, + }) + } + + return &graphcontracts.GraphSpec{ + ID: "runtime-graph", + Name: "Runtime Performance Graph", + Nodes: nodes, + Edges: edges, + } +} From 44ae0057bc0bf3ca9136c13ec757cf95829c5286 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 25 Jul 2026 20:17:57 +0530 Subject: [PATCH 06/10] fix: resolve NodeTypeExecution reference and formatting --- runtimegraph/runtime_graph.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/runtimegraph/runtime_graph.go b/runtimegraph/runtime_graph.go index 4be715a7d..196de1782 100644 --- a/runtimegraph/runtime_graph.go +++ b/runtimegraph/runtime_graph.go @@ -33,7 +33,7 @@ type RuntimeNode struct { type RuntimeEdge struct { From string `json:"from"` To string `json:"to"` - Kind string `json:"kind"` // "calls", "produces", "consumes", "depends_on" + Kind string `json:"kind"` // "calls", "produces", "consumes", "depends_on" Weight float64 `json:"weight"` } @@ -126,9 +126,9 @@ func (g *RuntimeGraph) ToGraphSpec() *graphcontracts.GraphSpec { nodes := make([]graphcontracts.NodeSpec, 0, len(g.nodes)) for id, node := range g.nodes { config := map[string]string{ - "name": node.Name, - "type": node.Type, - "metrics": fmt.Sprintf("%d", len(node.Metrics)), + "name": node.Name, + "type": node.Type, + "metrics": fmt.Sprintf("%d", len(node.Metrics)), } for _, m := range node.Metrics { config["metric_"+m.Name] = fmt.Sprintf("%.2f", m.Value) @@ -152,9 +152,9 @@ func (g *RuntimeGraph) ToGraphSpec() *graphcontracts.GraphSpec { } return &graphcontracts.GraphSpec{ - ID: "runtime-graph", - Name: "Runtime Performance Graph", - Nodes: nodes, - Edges: edges, + ID: "runtime-graph", + Name: "Runtime Performance Graph", + Nodes: nodes, + Edges: edges, } } From 69f999754221d98790eacbe225893b5610a21d40 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 25 Jul 2026 21:17:48 +0530 Subject: [PATCH 07/10] chore: bump hawk-core-contracts to v0.1.9 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5d3066f95..9cfc70e31 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.5 require ( github.com/BurntSushi/toml v1.6.0 - github.com/GrayCodeAI/hawk-core-contracts v0.1.8 + github.com/GrayCodeAI/hawk-core-contracts v0.1.9 github.com/spf13/viper v1.21.0 github.com/tiktoken-go/tokenizer v0.8.0 golang.org/x/sys v0.45.0 From 3ea982a377fa43eb03c5155e8c2d5016f19c1dd9 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 25 Jul 2026 21:24:23 +0530 Subject: [PATCH 08/10] chore: remove local replace directive, use published v0.1.9 --- go.mod | 2 -- go.sum | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9cfc70e31..613a34ac4 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,6 @@ require ( modernc.org/sqlite v1.51.0 ) -replace github.com/GrayCodeAI/hawk-core-contracts => ../hawk-core-contracts - require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 0ad84aa52..3ef0d5be4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/GrayCodeAI/hawk-core-contracts v0.1.9 h1:uXX/gtNM+3kxSEzu+rZkHykzcEaAbASn1lmPyOGMXvc= +github.com/GrayCodeAI/hawk-core-contracts v0.1.9/go.mod h1:BXbh68YrCf+s9HVqND5F8DAvl2MnE5NcOwZZZB56HGA= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2/v2 v2.1.0 h1:jHXRmHRZGbuQzDZjMlCAXOvQb75iv3HyLDzXGj5H1AY= From afca12ee023f19ac3f9c227d52e54ec95c162210 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Mon, 27 Jul 2026 09:05:10 +0530 Subject: [PATCH 09/10] chore: commit all pending changes --- internal/cache/git_watcher_test.go | 178 ++++++++++++ internal/cache/multilevel_test.go | 185 +++++++++++++ internal/core/batch_processor_test.go | 97 +++++++ internal/core/cost_test.go | 129 +++++++++ internal/filter/ansi_test.go | 139 ++++++++++ internal/filter/brace_depth_test.go | 98 +++++++ internal/filter/bytepool_test.go | 270 ++++++++++++++----- internal/filter/cache_lru_compat_test.go | 162 +++++++++++ internal/filter/code_comment_strip_test.go | 213 +++++++++++++++ internal/filter/constants_test.go | 96 +++++++ internal/filter/dedup_test.go | 249 +++++++++++++++++ internal/filter/equivalence.go | 9 +- internal/filter/equivalence_test.go | 291 ++++++++++++++------ internal/filter/filter_test.go | 153 +++++++++++ internal/filter/import_test.go | 146 ++++++++++ internal/filter/multi_file_test.go | 297 +++++++++++++++++++++ internal/filter/noise_test.go | 146 ++++++++++ internal/filter/progress_test.go | 86 ++++++ internal/filter/signature_patterns.go | 2 +- internal/filter/signature_patterns_test.go | 170 ++++++++++++ internal/filter/utils_test.go | 60 +++-- internal/utils/logger_test.go | 193 +++++++++++++ internal/utils/utils_test.go | 107 +++++++- 23 files changed, 3297 insertions(+), 179 deletions(-) create mode 100644 internal/cache/git_watcher_test.go create mode 100644 internal/cache/multilevel_test.go create mode 100644 internal/core/batch_processor_test.go create mode 100644 internal/core/cost_test.go create mode 100644 internal/filter/ansi_test.go create mode 100644 internal/filter/brace_depth_test.go create mode 100644 internal/filter/cache_lru_compat_test.go create mode 100644 internal/filter/code_comment_strip_test.go create mode 100644 internal/filter/constants_test.go create mode 100644 internal/filter/dedup_test.go create mode 100644 internal/filter/filter_test.go create mode 100644 internal/filter/import_test.go create mode 100644 internal/filter/multi_file_test.go create mode 100644 internal/filter/noise_test.go create mode 100644 internal/filter/progress_test.go create mode 100644 internal/filter/signature_patterns_test.go create mode 100644 internal/utils/logger_test.go diff --git a/internal/cache/git_watcher_test.go b/internal/cache/git_watcher_test.go new file mode 100644 index 000000000..d18a94066 --- /dev/null +++ b/internal/cache/git_watcher_test.go @@ -0,0 +1,178 @@ +package cache + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestIsGitRepoNonExistent(t *testing.T) { + dir := filepath.Join(os.TempDir(), "nonexistent-git-test") + result := IsGitRepo(dir) + if result { + t.Error("expected false for non-existent directory") + } +} + +func TestIsGitRepoPlainDir(t *testing.T) { + dir := t.TempDir() + result := IsGitRepo(dir) + if result { + t.Error("expected false for plain directory without .git") + } +} + +func TestIsGitRepoWithDotGit(t *testing.T) { + dir := t.TempDir() + dotGit := filepath.Join(dir, ".git") + if err := os.MkdirAll(dotGit, 0o755); err != nil { + t.Fatal(err) + } + result := IsGitRepo(dir) + if !result { + t.Error("expected true for directory with .git subdirectory") + } +} + +func TestNewGitWatcher(t *testing.T) { + qc, err := NewQueryCache("") + if err != nil { + t.Fatal(err) + } + w := NewGitWatcher(qc) + if w == nil { + t.Fatal("NewGitWatcher returned nil") + } + if w.repoStates == nil { + t.Error("repoStates map should be initialized") + } + if w.cache != qc { + t.Error("cache reference should match") + } +} + +func TestGetChangedFilesEmpty(t *testing.T) { + dir := t.TempDir() + gitInit(t, dir) + gitCreateFile(t, dir, "initial.txt", "init") + + files, err := GetChangedFiles(dir, "HEAD", "HEAD") + if err != nil { + t.Fatalf("GetChangedFiles returned error: %v", err) + } + if len(files) != 0 { + t.Errorf("expected 0 changed files between same commit, got %d", len(files)) + } +} + +func TestGetChangedFilesWithChanges(t *testing.T) { + dir := t.TempDir() + gitInit(t, dir) + gitCreateFile(t, dir, "initial.txt", "init") + + firstCommit := gitHead(t, dir) + + gitCreateFile(t, dir, "test.txt", "content") + + files, err := GetChangedFiles(dir, firstCommit, "HEAD") + if err != nil { + t.Fatalf("GetChangedFiles returned error: %v", err) + } + if len(files) == 0 { + t.Error("expected changed files between commits") + } +} + +func TestGetFileHashes(t *testing.T) { + dir := t.TempDir() + gitInit(t, dir) + gitCreateFile(t, dir, "foo.txt", "hello") + + qc, _ := NewQueryCache("") + w := NewGitWatcher(qc) + hashes, err := w.GetFileHashes(dir, []string{"foo.txt"}) + if err != nil { + t.Fatalf("GetFileHashes returned error: %v", err) + } + if _, ok := hashes[".git/HEAD"]; !ok { + t.Error("expected .git/HEAD hash") + } +} + +func TestGetFileHashesNonGitDir(t *testing.T) { + dir := t.TempDir() + srcFile := filepath.Join(dir, "test.txt") + if err := os.WriteFile(srcFile, []byte("content"), 0o644); err != nil { + t.Fatal(err) + } + + qc2, _ := NewQueryCache("") + w2 := NewGitWatcher(qc2) + hashes, err := w2.GetFileHashes(dir, []string{"test.txt"}) + if err != nil { + t.Fatalf("GetFileHashes returned error: %v", err) + } + if len(hashes) == 0 { + t.Error("expected file hashes for non-git dir") + } +} + +func TestGetFileHashesWithDirtyFile(t *testing.T) { + dir := t.TempDir() + gitInit(t, dir) + gitCreateFile(t, dir, "clean.txt", "clean content") + + qc3, _ := NewQueryCache("") + w := NewGitWatcher(qc3) + + dirtyFile := filepath.Join(dir, "clean.txt") + if err := os.WriteFile(dirtyFile, []byte("dirty content"), 0o644); err != nil { + t.Fatal(err) + } + + hashes, err := w.GetFileHashes(dir, []string{"clean.txt"}) + if err != nil { + t.Fatalf("GetFileHashes returned error: %v", err) + } + if _, ok := hashes[".git/HEAD"]; !ok { + t.Error("expected .git/HEAD hash") + } +} + +func gitInit(t *testing.T, dir string) { + t.Helper() + execCommand(t, dir, "git", "init") + execCommand(t, dir, "git", "config", "user.email", "test@test.com") + execCommand(t, dir, "git", "config", "user.name", "Test") +} + +func gitHead(t *testing.T, dir string) string { + t.Helper() + return execCommand(t, dir, "git", "rev-parse", "HEAD") +} + +func gitCreateFile(t *testing.T, dir, name, content string) { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + execCommand(t, dir, "git", "add", name) + execCommand(t, dir, "git", "commit", "-m", "add "+name) +} + +func execCommand(t *testing.T, dir, cmd string, args ...string) string { + t.Helper() + c := exec.Command(cmd, args...) + c.Dir = dir + output, err := c.Output() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + t.Fatalf("%s %v failed: %v\nstdout: %s\nstderr: %s", cmd, args, err, string(output), string(ee.Stderr)) + } + t.Fatalf("%s %v failed: %v\n%s", cmd, args, err, string(output)) + } + return strings.TrimSpace(string(output)) +} diff --git a/internal/cache/multilevel_test.go b/internal/cache/multilevel_test.go new file mode 100644 index 000000000..7d8e8f37e --- /dev/null +++ b/internal/cache/multilevel_test.go @@ -0,0 +1,185 @@ +package cache + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestNewMultiLevelCache(t *testing.T) { + dir := t.TempDir() + l2 := filepath.Join(dir, "l2") + mc, err := NewMultiLevelCache(l2, 10) + if err != nil { + t.Fatalf("NewMultiLevelCache returned error: %v", err) + } + if mc == nil { + t.Fatal("NewMultiLevelCache returned nil") + } + if _, err := os.Stat(l2); os.IsNotExist(err) { + t.Error("L2 directory should have been created") + } +} + +func TestMultiLevelCacheSetAndGet(t *testing.T) { + dir := t.TempDir() + mc, err := NewMultiLevelCache(filepath.Join(dir, "l2"), 10) + if err != nil { + t.Fatal(err) + } + + mc.Set("key1", "value1") + val, ok := mc.Get("key1") + if !ok { + t.Fatal("expected to find key1") + } + if val != "value1" { + t.Errorf("expected value1, got %q", val) + } +} + +func TestMultiLevelCacheMiss(t *testing.T) { + dir := t.TempDir() + mc, err := NewMultiLevelCache(filepath.Join(dir, "l2"), 10) + if err != nil { + t.Fatal(err) + } + + _, ok := mc.Get("nonexistent") + if ok { + t.Error("expected miss for nonexistent key") + } +} + +func TestMultiLevelCacheOverwrite(t *testing.T) { + dir := t.TempDir() + mc, err := NewMultiLevelCache(filepath.Join(dir, "l2"), 10) + if err != nil { + t.Fatal(err) + } + + mc.Set("key1", "old") + mc.Set("key1", "new") + + val, ok := mc.Get("key1") + if !ok { + t.Fatal("expected to find key1") + } + if val != "new" { + t.Errorf("expected new value, got %q", val) + } +} + +func TestMultiLevelCacheL1Eviction(t *testing.T) { + dir := t.TempDir() + mc, err := NewMultiLevelCache(filepath.Join(dir, "l2"), 2) + if err != nil { + t.Fatal(err) + } + + mc.Set("a", "1") + mc.Set("b", "2") + mc.Set("c", "3") // L1 evicts oldest (a) to stay at maxL1=2 + + mc.mu.RLock() + l1Len := len(mc.l1) + mc.mu.RUnlock() + + if l1Len > 2 { + t.Errorf("expected L1 size <= 2 after eviction, got %d", l1Len) + } + + val, ok := mc.Get("a") + if !ok { + t.Error("expected 'a' to be retrievable from L2 after L1 eviction") + } + if val != "1" { + t.Errorf("expected '1', got %q", val) + } +} + +func TestMultiLevelCacheL2Promotion(t *testing.T) { + dir := t.TempDir() + l2 := filepath.Join(dir, "l2") + mc, err := NewMultiLevelCache(l2, 10) + if err != nil { + t.Fatal(err) + } + + mc.Set("key1", "value1") + + mc2, err := NewMultiLevelCache(l2, 10) + if err != nil { + t.Fatal(err) + } + + val, ok := mc2.Get("key1") + if !ok { + t.Fatal("expected to retrieve from L2 disk cache") + } + if val != "value1" { + t.Errorf("expected value1, got %q", val) + } +} + +func TestMultiLevelCacheL2PromotionUpdatesL1(t *testing.T) { + dir := t.TempDir() + l2 := filepath.Join(dir, "l2") + mc, err := NewMultiLevelCache(l2, 10) + if err != nil { + t.Fatal(err) + } + + mc.Set("key1", "value1") + + mc2, err := NewMultiLevelCache(l2, 10) + if err != nil { + t.Fatal(err) + } + + mc2.Get("key1") + + t.Log("L1 should now have key1 from promotion") +} + +func TestMultiLevelCacheSetGetLargeValue(t *testing.T) { + dir := t.TempDir() + mc, err := NewMultiLevelCache(filepath.Join(dir, "l2"), 10) + if err != nil { + t.Fatal(err) + } + + largeVal := strings.Repeat("x", 10000) + mc.Set("large", largeVal) + + val, ok := mc.Get("large") + if !ok { + t.Fatal("expected to find large value") + } + if val != largeVal { + t.Errorf("expected large value to match, got length %d", len(val)) + } +} + +func TestMultiLevelCacheConcurrent(t *testing.T) { + dir := t.TempDir() + mc, err := NewMultiLevelCache(filepath.Join(dir, "l2"), 100) + if err != nil { + t.Fatal(err) + } + + done := make(chan bool) + for i := 0; i < 20; i++ { + go func(n int) { + key := string(rune('a' + n)) + mc.Set(key, "val") + mc.Get(key) + done <- true + }(i) + } + + for i := 0; i < 20; i++ { + <-done + } +} diff --git a/internal/core/batch_processor_test.go b/internal/core/batch_processor_test.go new file mode 100644 index 000000000..24aec2b49 --- /dev/null +++ b/internal/core/batch_processor_test.go @@ -0,0 +1,97 @@ +package core + +import ( + "errors" + "strings" + "testing" +) + +var errMockFailure = errors.New("mock failure") + +func TestNewBatchProcessorDefaults(t *testing.T) { + bp := NewBatchProcessor(nil, 0) + if bp == nil { + t.Fatal("NewBatchProcessor returned nil") + } + if bp.workers <= 0 { + t.Error("expected positive worker count") + } +} + +func TestNewBatchProcessorCustomWorkers(t *testing.T) { + bp := NewBatchProcessor(nil, 4) + if bp.workers != 4 { + t.Errorf("expected 4 workers, got %d", bp.workers) + } +} + +func TestNewBatchProcessorCapsWorkers(t *testing.T) { + bp := NewBatchProcessor(nil, 9999) + if bp.workers > 0 && bp.workers == 9999 { + t.Error("expected workers to be capped") + } +} + +type mockProcessor struct{} + +func (m *mockProcessor) Process(input string) (string, interface{}, error) { + return strings.ToUpper(input), nil, nil +} + +type failProcessor struct{} + +func (f *failProcessor) Process(input string) (string, interface{}, error) { + return "", nil, errMockFailure +} + +func TestProcessBatch(t *testing.T) { + bp := NewBatchProcessor(&mockProcessor{}, 2) + inputs := []string{"hello", "world", "foo", "bar"} + results := bp.ProcessBatch(inputs) + + if len(results) != len(inputs) { + t.Fatalf("expected %d results, got %d", len(inputs), len(results)) + } + + for i, result := range results { + if result.Error != nil { + t.Errorf("result[%d] had error: %v", i, result.Error) + } + if result.Output != strings.ToUpper(inputs[i]) { + t.Errorf("result[%d].Output = %q, want %q", i, result.Output, strings.ToUpper(inputs[i])) + } + if result.Index != i { + t.Errorf("result[%d].Index = %d, want %d", i, result.Index, i) + } + } +} + +func TestProcessBatchSingleInput(t *testing.T) { + bp := NewBatchProcessor(&mockProcessor{}, 1) + results := bp.ProcessBatch([]string{"single"}) + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].Output != "SINGLE" { + t.Errorf("expected 'SINGLE', got %q", results[0].Output) + } +} + +func TestProcessBatchEmptyInputs(t *testing.T) { + bp := NewBatchProcessor(&mockProcessor{}, 2) + results := bp.ProcessBatch([]string{}) + if len(results) != 0 { + t.Errorf("expected 0 results for empty input, got %d", len(results)) + } +} + +func TestProcessBatchError(t *testing.T) { + bp := NewBatchProcessor(&failProcessor{}, 2) + results := bp.ProcessBatch([]string{"test"}) + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].Error != errMockFailure { + t.Errorf("expected mock failure error, got %v", results[0].Error) + } +} diff --git a/internal/core/cost_test.go b/internal/core/cost_test.go new file mode 100644 index 000000000..8d3d90919 --- /dev/null +++ b/internal/core/cost_test.go @@ -0,0 +1,129 @@ +package core + +import ( + "testing" +) + +func TestGetModelPricingKnown(t *testing.T) { + pricing := GetModelPricing("gpt-4o") + if pricing.Model == "" { + t.Fatal("expected pricing for gpt-4o") + } + if pricing.InputPerMillion <= 0 { + t.Errorf("expected positive input price, got %f", pricing.InputPerMillion) + } + if pricing.OutputPerMillion <= 0 { + t.Errorf("expected positive output price, got %f", pricing.OutputPerMillion) + } +} + +func TestGetModelPricingUnknown(t *testing.T) { + pricing := GetModelPricing("nonexistent-model-xyz") + if pricing.Model == "" { + t.Fatal("expected fallback pricing for unknown model") + } +} + +func TestGetModelPricingCaseInsensitive(t *testing.T) { + p1 := GetModelPricing("GPT-4O") + p2 := GetModelPricing("gpt-4o") + if p1.InputPerMillion != p2.InputPerMillion { + t.Error("pricing should be case-insensitive") + } +} + +func TestGetModelPricingTrimmed(t *testing.T) { + p1 := GetModelPricing(" gpt-4o ") + p2 := GetModelPricing("gpt-4o") + if p1.InputPerMillion != p2.InputPerMillion { + t.Error("pricing should ignore whitespace") + } +} + +func TestHasModelPricingKnown(t *testing.T) { + if !HasModelPricing("gpt-4o") { + t.Error("expected true for known model") + } +} + +func TestHasModelPricingUnknown(t *testing.T) { + if HasModelPricing("nonexistent") { + t.Error("expected false for unknown model") + } +} + +func TestHasModelPricingCaseInsensitive(t *testing.T) { + if !HasModelPricing("CLAUDE-4-OPUS") { + t.Error("expected true for case-insensitive lookup") + } +} + +func TestRegisterModelPricingNew(t *testing.T) { + RegisterModelPricing("custom-model", 1.0, 2.0) + pricing := GetModelPricing("custom-model") + if pricing.InputPerMillion != 1.0 { + t.Errorf("expected input price 1.0, got %f", pricing.InputPerMillion) + } + if pricing.OutputPerMillion != 2.0 { + t.Errorf("expected output price 2.0, got %f", pricing.OutputPerMillion) + } +} + +func TestRegisterModelPricingOverride(t *testing.T) { + original := GetModelPricing("gpt-4o").InputPerMillion + RegisterModelPricing("gpt-4o", 999.0, 999.0) + + updated := GetModelPricing("gpt-4o") + if updated.InputPerMillion != 999.0 { + t.Errorf("expected overridden price 999.0, got %f", updated.InputPerMillion) + } + + RegisterModelPricing("gpt-4o", original, original) +} + +func TestCalculateSavingsZeroTokens(t *testing.T) { + savings := CalculateSavings(0, "gpt-4o") + if savings != 0 { + t.Errorf("expected 0 savings for 0 tokens, got %f", savings) + } +} + +func TestCalculateSavingsUnknownModel(t *testing.T) { + savings := CalculateSavings(1000000, "nonexistent") + if savings <= 0 { + t.Error("expected positive savings for unknown model (falls back to default)") + } +} + +func TestCalculateSavingsLargeTokens(t *testing.T) { + savings := CalculateSavings(1000000000, "gpt-4o") + if savings <= 0 { + t.Error("expected positive savings for large token count") + } +} + +func TestCommonModelPricingAllModels(t *testing.T) { + for name, pricing := range CommonModelPricing { + if pricing.Model == "" { + t.Errorf("model %q has empty name", name) + } + if pricing.InputPerMillion <= 0 { + t.Errorf("model %q has non-positive input price: %f", name, pricing.InputPerMillion) + } + if pricing.OutputPerMillion <= 0 { + t.Errorf("model %q has non-positive output price: %f", name, pricing.OutputPerMillion) + } + } +} + +func TestCommonModelPricingHasKey(t *testing.T) { + models := []string{ + "gpt-4o", "gpt-4o-mini", "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano", + "claude-3.5-sonnet", "claude-3-haiku", "claude-4-sonnet", "claude-4-opus", + } + for _, model := range models { + if _, ok := CommonModelPricing[model]; !ok { + t.Errorf("missing pricing for %q", model) + } + } +} diff --git a/internal/filter/ansi_test.go b/internal/filter/ansi_test.go new file mode 100644 index 000000000..5f42c4b82 --- /dev/null +++ b/internal/filter/ansi_test.go @@ -0,0 +1,139 @@ +package filter + +import ( + "strings" + "testing" +) + +func TestNewANSIFilter(t *testing.T) { + f := NewANSIFilter() + if f == nil { + t.Fatal("NewANSIFilter returned nil") + } +} + +func TestANSIFilterName(t *testing.T) { + f := NewANSIFilter() + if f.Name() != "ansi" { + t.Errorf("expected name %q, got %q", "ansi", f.Name()) + } +} + +func TestANSIFilterApply(t *testing.T) { + f := NewANSIFilter() + input := "\x1b[32mGreen text\x1b[0m and \x1b[1mbold\x1b[0m" + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "\x1b[") { + t.Error("ANSI codes should be stripped from output") + } + if saved <= 0 { + t.Error("expected some tokens to be saved from ANSI stripping") + } +} + +func TestANSIFilterApplyNoANSI(t *testing.T) { + f := NewANSIFilter() + input := "Plain text without ANSI codes" + output, saved := f.Apply(input, ModeMinimal) + + if output != input { + t.Errorf("expected unchanged output for plain text, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for plain text, got %d", saved) + } +} + +func TestANSIFilterApplyEmpty(t *testing.T) { + f := NewANSIFilter() + output, saved := f.Apply("", ModeMinimal) + + if output != "" { + t.Errorf("expected empty output, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for empty input, got %d", saved) + } +} + +func TestANSIFilterApplyVariousCodes(t *testing.T) { + f := NewANSIFilter() + tests := []struct { + name string + input string + }{ + {"color codes", "\x1b[31mRed\x1b[0m \x1b[32mGreen\x1b[0m"}, + {"bold codes", "\x1b[1mBold\x1b[22m"}, + {"cursor codes", "\x1b[2J\x1b[H"}, + {"complex codes", "\x1b[38;5;208mOrange\x1b[0m"}, + {"mixed content", "Before \x1b[34mBlue\x1b[0m After"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output, _ := f.Apply(tt.input, ModeMinimal) + if strings.Contains(output, "\x1b[") { + t.Errorf("ANSI codes not fully stripped from %q", tt.input) + } + }) + } +} + +func TestStripANSI(t *testing.T) { + input := "\x1b[32mGreen\x1b[0m text" + result := StripANSI(input) + if strings.Contains(result, "\x1b[") { + t.Error("StripANSI should remove ANSI codes") + } + if !strings.Contains(result, "Green") { + t.Error("StripANSI should preserve text content") + } +} + +func TestStripANSIEmpty(t *testing.T) { + result := StripANSI("") + if result != "" { + t.Errorf("expected empty string, got %q", result) + } +} + +func TestStripANSIPlainText(t *testing.T) { + input := "Plain text" + result := StripANSI(input) + if result != input { + t.Errorf("expected %q, got %q", input, result) + } +} + +func TestHasANSI(t *testing.T) { + tests := []struct { + input string + expected bool + }{ + {"\x1b[32mGreen\x1b[0m", true}, + {"Plain text", false}, + {"", false}, + {"\x1b[1mBold\x1b[0m", true}, + {"No codes here", false}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := HasANSI(tt.input) + if result != tt.expected { + t.Errorf("HasANSI(%q) = %v, expected %v", tt.input, result, tt.expected) + } + }) + } +} + +func TestANSIFilterApplyPreservesContent(t *testing.T) { + f := NewANSIFilter() + input := "\x1b[32mImportant message\x1b[0m" + output, _ := f.Apply(input, ModeMinimal) + + if !strings.Contains(output, "Important message") { + t.Error("ANSI filter should preserve text content") + } +} diff --git a/internal/filter/brace_depth_test.go b/internal/filter/brace_depth_test.go new file mode 100644 index 000000000..7c27a14e7 --- /dev/null +++ b/internal/filter/brace_depth_test.go @@ -0,0 +1,98 @@ +package filter + +import ( + "strings" + "testing" +) + +func TestNewBodyFilter(t *testing.T) { + f := NewBodyFilter() + if f == nil { + t.Fatal("NewBodyFilter returned nil") + } +} + +func TestBodyFilterName(t *testing.T) { + f := NewBodyFilter() + if f.Name() != "body" { + t.Errorf("expected name %q, got %q", "body", f.Name()) + } +} + +func TestBodyFilterNonAggressive(t *testing.T) { + f := NewBodyFilter() + input := "func main() {\n\tfmt.Println(\"hello\")\n}" + output, saved := f.Apply(input, ModeMinimal) + if saved != 0 { + t.Errorf("expected 0 tokens saved in non-aggressive mode, got %d", saved) + } + if output != input { + t.Errorf("expected unchanged output in non-aggressive mode, got %q", output) + } +} + +func TestBodyFilterNonCode(t *testing.T) { + f := NewBodyFilter() + input := "This is just some regular text." + output, saved := f.Apply(input, ModeAggressive) + if saved != 0 { + t.Errorf("expected 0 tokens saved for non-code, got %d", saved) + } + if output != input { + t.Errorf("expected unchanged output for non-code, got %q", output) + } +} + +func TestBodyFilterAddsPlaceholder(t *testing.T) { + f := NewBodyFilter() + input := `package main + +func main() { + fmt.Println("hello") + fmt.Println("world") +}` + output, _ := f.Apply(input, ModeAggressive) + if !strings.Contains(output, "body stripped") { + t.Error("expected body stripped marker to appear") + } + if !strings.Contains(output, "func main") { + t.Error("function signature should be preserved") + } +} + +func TestBodyFilterEmptyInput(t *testing.T) { + f := NewBodyFilter() + output, saved := f.Apply("", ModeAggressive) + if output != "" { + t.Errorf("expected empty output, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for empty input, got %d", saved) + } +} + +func TestBodyFilterIsFunctionStart(t *testing.T) { + f := NewBodyFilter() + tests := []struct { + line string + lang string + expected bool + }{ + {"func main() {", "go", true}, + {"func add(a, b int) int {", "go", true}, + {"fn main() {", "rust", true}, + {"def main():", "python", true}, + {"function main() {", "javascript", true}, + {"console.log(\"hello\")", "javascript", false}, + {"package main", "go", false}, + } + + for _, tt := range tests { + t.Run(tt.line, func(t *testing.T) { + result := f.isFunctionStart(tt.line, tt.lang) + if result != tt.expected { + t.Errorf("isFunctionStart(%q, %q) = %v, expected %v", tt.line, tt.lang, result, tt.expected) + } + }) + } +} diff --git a/internal/filter/bytepool_test.go b/internal/filter/bytepool_test.go index 5918fd8a8..a023c02cf 100644 --- a/internal/filter/bytepool_test.go +++ b/internal/filter/bytepool_test.go @@ -5,106 +5,236 @@ import ( ) func TestNewBytePool(t *testing.T) { - bp := NewBytePool() - if bp == nil { - t.Fatal("expected non-nil BytePool") + pool := NewBytePool() + if pool == nil { + t.Fatal("NewBytePool returned nil") + } + if pool.smallPool.New == nil { + t.Error("smallPool.New should not be nil") + } + if pool.mediumPool.New == nil { + t.Error("mediumPool.New should not be nil") + } + if pool.largePool.New == nil { + t.Error("largePool.New should not be nil") + } + if pool.hugePool.New == nil { + t.Error("hugePool.New should not be nil") + } +} + +func TestBytePoolGetSmall(t *testing.T) { + pool := NewBytePool() + buf := pool.Get(512) + if buf == nil { + t.Fatal("Get returned nil for small buffer") + } + if cap(*buf) != 1024 { + t.Errorf("expected capacity 1024, got %d", cap(*buf)) + } +} + +func TestBytePoolGetMedium(t *testing.T) { + pool := NewBytePool() + buf := pool.Get(5000) + if buf == nil { + t.Fatal("Get returned nil for medium buffer") + } + if cap(*buf) != 10240 { + t.Errorf("expected capacity 10240, got %d", cap(*buf)) + } +} + +func TestBytePoolGetLarge(t *testing.T) { + pool := NewBytePool() + buf := pool.Get(50000) + if buf == nil { + t.Fatal("Get returned nil for large buffer") + } + if cap(*buf) != 102400 { + t.Errorf("expected capacity 102400, got %d", cap(*buf)) } } -func TestBytePool_GetPut(t *testing.T) { - bp := NewBytePool() +func TestBytePoolGetHuge(t *testing.T) { + pool := NewBytePool() + buf := pool.Get(200000) + if buf == nil { + t.Fatal("Get returned nil for huge buffer") + } + if cap(*buf) != 1048576 { + t.Errorf("expected capacity 1048576, got %d", cap(*buf)) + } +} - sizes := []int{100, 5000, 50000, 500000} - for _, size := range sizes { - b := bp.Get(size) - if b == nil { - t.Fatalf("expected non-nil buffer for capacity %d", size) - } - if cap(*b) < size { - t.Errorf("capacity %d < requested %d", cap(*b), size) - } +func TestBytePoolPutNil(t *testing.T) { + pool := NewBytePool() + pool.Put(nil) // should not panic +} - // Write some data - *b = append(*b, []byte("test data")...) - bp.Put(b) +func TestBytePoolPutAndReuse(t *testing.T) { + pool := NewBytePool() + buf := pool.Get(512) + *buf = append(*buf, []byte("test data")...) + pool.Put(buf) - // Get again β€” should be reset - b2 := bp.Get(size) - if len(*b2) != 0 { - t.Errorf("expected zero-length buffer after Put, got %d", len(*b2)) - } + // Get again β€” should get a reset buffer + buf2 := pool.Get(512) + if buf2 == nil { + t.Fatal("Get returned nil after Put") + } + if len(*buf2) != 0 { + t.Errorf("expected reset buffer (length 0), got length %d", len(*buf2)) } } -func TestBytePool_PutNil(t *testing.T) { - bp := NewBytePool() - // Should not panic - bp.Put(nil) +func TestBytePoolPutWrongSize(t *testing.T) { + pool := NewBytePool() + buf := make([]byte, 0, 5000) + pool.Put(&buf) // should go to medium pool } -func TestGetBytesPutBytes(t *testing.T) { - b := GetBytes(100) - if b == nil { - t.Fatal("expected non-nil bytes") +func TestGlobalBytePoolFunctions(t *testing.T) { + buf := GetBytes(512) + if buf == nil { + t.Fatal("GetBytes returned nil") + } + *buf = append(*buf, 'x') + PutBytes(buf) + + buf2 := GetBytes(512) + if buf2 == nil { + t.Fatal("GetBytes returned nil after PutBytes") + } + if len(*buf2) != 0 { + t.Errorf("expected reset buffer, got length %d", len(*buf2)) } - *b = append(*b, []byte("hello")...) - PutBytes(b) +} + +func TestAcquireStringBuilder(t *testing.T) { + pool := NewBytePool() + buf := pool.AcquireStringBuilder(512) + if buf == nil { + t.Fatal("AcquireStringBuilder returned nil") + } + if buf.Len() != 0 { + t.Errorf("expected empty buffer, got length %d", buf.Len()) + } +} + +func TestReleaseStringBuilder(t *testing.T) { + pool := NewBytePool() + buf := pool.AcquireStringBuilder(512) + buf.WriteString("test") + pool.ReleaseStringBuilder(buf) +} + +func TestReleaseStringBuilderNil(t *testing.T) { + pool := NewBytePool() + pool.ReleaseStringBuilder(nil) // should not panic } func TestFastStringBuilder(t *testing.T) { - fb := NewFastStringBuilder(64) - if fb == nil { - t.Fatal("expected non-nil FastStringBuilder") + builder := NewFastStringBuilder(1024) + if builder == nil { + t.Fatal("NewFastStringBuilder returned nil") } - fb.WriteString("hello ") - fb.WriteString("world") - _ = fb.WriteByte('!') - fb.Write([]byte(" test")) + builder.WriteString("Hello, ") + builder.WriteString("world!") + result := builder.String() + if result != "Hello, world!" { + t.Errorf("expected %q, got %q", "Hello, world!", result) + } - if fb.Len() != 17 { - t.Errorf("expected len 17, got %d", fb.Len()) + if builder.Len() != 13 { + t.Errorf("expected length 13, got %d", builder.Len()) } - if fb.String() != "hello world! test" { - t.Errorf("unexpected string: %q", fb.String()) + + if builder.Cap() < 1024 { + t.Errorf("expected capacity >= 1024, got %d", builder.Cap()) } +} - fb.Reset() - if fb.Len() != 0 { - t.Errorf("expected len 0 after reset, got %d", fb.Len()) +func TestFastStringBuilderWriteByte(t *testing.T) { + builder := NewFastStringBuilder(10) + err := builder.WriteByte('A') + if err != nil { + t.Errorf("WriteByte returned error: %v", err) } - if fb.Cap() == 0 { - t.Error("expected non-zero capacity after reset") + if builder.String() != "A" { + t.Errorf("expected %q, got %q", "A", builder.String()) } } -func TestFastStringBuilder_Grow(t *testing.T) { - fb := NewFastStringBuilder(10) - fb.WriteString("hello") - fb.Grow(100) - if fb.Cap() < 105 { - t.Errorf("expected capacity >= 105, got %d", fb.Cap()) +func TestFastStringBuilderWrite(t *testing.T) { + builder := NewFastStringBuilder(10) + builder.Write([]byte("test")) + if builder.String() != "test" { + t.Errorf("expected %q, got %q", "test", builder.String()) } } -func TestBytePool_AcquireReleaseStringBuilder(t *testing.T) { - bp := NewBytePool() +func TestFastStringBuilderReset(t *testing.T) { + builder := NewFastStringBuilder(1024) + builder.WriteString("test data") + builder.Reset() + if builder.Len() != 0 { + t.Errorf("expected length 0 after reset, got %d", builder.Len()) + } + if builder.String() != "" { + t.Errorf("expected empty string after reset, got %q", builder.String()) + } +} - // Small - buf := bp.AcquireStringBuilder(100) - buf.WriteString("hello") - bp.ReleaseStringBuilder(buf) +func TestFastStringBuilderGrow(t *testing.T) { + builder := NewFastStringBuilder(10) + initialCap := builder.Cap() + builder.Grow(100) + if builder.Cap() < initialCap+100 { + t.Errorf("expected capacity >= %d, got %d", initialCap+100, builder.Cap()) + } +} + +func TestFastStringBuilderGrowNoOp(t *testing.T) { + builder := NewFastStringBuilder(1000) + initialCap := builder.Cap() + builder.Grow(10) // should not grow since there's enough capacity + if builder.Cap() != initialCap { + t.Errorf("expected capacity to remain %d, got %d", initialCap, builder.Cap()) + } +} + +func TestFastStringBuilderMultipleOperations(t *testing.T) { + builder := NewFastStringBuilder(100) + builder.WriteString("Hello") + builder.WriteByte(' ') + builder.WriteString("World") + builder.Write([]byte("!")) - // Medium - buf2 := bp.AcquireStringBuilder(5000) - buf2.WriteString("world") - bp.ReleaseStringBuilder(buf2) + result := builder.String() + if result != "Hello World!" { + t.Errorf("expected %q, got %q", "Hello World!", result) + } - // Large - buf3 := bp.AcquireStringBuilder(50000) - buf3.WriteString("big data") - bp.ReleaseStringBuilder(buf3) + builder.Reset() + builder.WriteString("New content") + if builder.String() != "New content" { + t.Errorf("expected %q after reset, got %q", "New content", builder.String()) + } +} - // Nil - bp.ReleaseStringBuilder(nil) +func TestGlobalBufferPools(t *testing.T) { + if globalBufferPools == nil { + t.Fatal("globalBufferPools should not be nil") + } + if globalBufferPools.small.New == nil { + t.Error("small pool New function should not be nil") + } + if globalBufferPools.medium.New == nil { + t.Error("medium pool New function should not be nil") + } + if globalBufferPools.large.New == nil { + t.Error("large pool New function should not be nil") + } } diff --git a/internal/filter/cache_lru_compat_test.go b/internal/filter/cache_lru_compat_test.go new file mode 100644 index 000000000..a30c49072 --- /dev/null +++ b/internal/filter/cache_lru_compat_test.go @@ -0,0 +1,162 @@ +package filter + +import ( + "testing" + "time" +) + +func TestNewLRUCache(t *testing.T) { + cache := NewLRUCache(100, 5*time.Minute) + if cache == nil { + t.Fatal("NewLRUCache returned nil") + } +} + +func TestLRUCacheSetAndGet(t *testing.T) { + cache := NewLRUCache(100, 5*time.Minute) + + cache.Set("test-key", CachedResult{Output: "test output", Tokens: 50}) + got, ok := cache.Get("test-key") + if !ok { + t.Fatal("expected to find cached value") + } + + result, ok := got.(CachedResult) + if !ok { + t.Fatal("expected CachedResult type") + } + if result.Output != "test output" { + t.Errorf("expected output %q, got %q", "test output", result.Output) + } + if result.Tokens != 50 { + t.Errorf("expected tokens %d, got %d", 50, result.Tokens) + } +} + +func TestLRUCacheMiss(t *testing.T) { + cache := NewLRUCache(100, 5*time.Minute) + + _, ok := cache.Get("nonexistent") + if ok { + t.Error("expected cache miss for nonexistent key") + } +} + +func TestLRUCacheEviction(t *testing.T) { + cache := NewLRUCache(2, 5*time.Minute) + + cache.Set("key1", CachedResult{Output: "output1"}) + cache.Set("key2", CachedResult{Output: "output2"}) + + cache.Get("key1") + + cache.Set("key3", CachedResult{Output: "output3"}) + + if cache.Len() != 2 { + t.Errorf("expected cache size 2 after eviction, got %d", cache.Len()) + } + + _, ok := cache.Get("key2") + if ok { + t.Error("expected key2 to be evicted") + } + + _, ok = cache.Get("key1") + if !ok { + t.Error("expected key1 to still be in cache") + } + + _, ok = cache.Get("key3") + if !ok { + t.Error("expected key3 to be in cache") + } +} + +func TestLRUCacheDelete(t *testing.T) { + cache := NewLRUCache(100, 5*time.Minute) + + cache.Set("key1", CachedResult{Output: "output1"}) + cache.Set("key2", CachedResult{Output: "output2"}) + + cache.Delete("key1") + + _, ok := cache.Get("key1") + if ok { + t.Error("expected key1 to be deleted") + } + _, ok = cache.Get("key2") + if !ok { + t.Error("expected key2 to still be accessible") + } + + if cache.Len() != 1 { + t.Errorf("expected length 1 after delete, got %d", cache.Len()) + } +} + +func TestLRUCacheMaxSizeZero(t *testing.T) { + cache := NewLRUCache(0, 5*time.Minute) + if cache.Len() != 0 { + t.Errorf("expected empty cache, got size %d", cache.Len()) + } +} + +func TestLRUCacheOverwrite(t *testing.T) { + cache := NewLRUCache(100, 5*time.Minute) + + cache.Set("key1", CachedResult{Output: "old"}) + cache.Set("key1", CachedResult{Output: "new"}) + + got, ok := cache.Get("key1") + if !ok { + t.Fatal("expected to find key1") + } + result, ok := got.(CachedResult) + if !ok { + t.Fatal("expected CachedResult type") + } + if result.Output != "new" { + t.Errorf("expected overwritten output %q, got %q", "new", result.Output) + } +} + +func TestLRUCacheLen(t *testing.T) { + cache := NewLRUCache(100, 5*time.Minute) + + if cache.Len() != 0 { + t.Errorf("expected empty cache, got size %d", cache.Len()) + } + + cache.Set("key1", CachedResult{Output: "output1"}) + if cache.Len() != 1 { + t.Errorf("expected size 1, got %d", cache.Len()) + } + + cache.Set("key2", CachedResult{Output: "output2"}) + if cache.Len() != 2 { + t.Errorf("expected size 2, got %d", cache.Len()) + } + + cache.Delete("key1") + if cache.Len() != 1 { + t.Errorf("expected size 1 after delete, got %d", cache.Len()) + } +} + +func TestLRUCacheConcurrent(t *testing.T) { + cache := NewLRUCache(1000, 5*time.Minute) + done := make(chan bool) + + for i := 0; i < 20; i++ { + go func(n int) { + key := string(rune('a' + n)) + cache.Set(key, CachedResult{Output: key}) + cache.Get(key) + done <- true + }(i) + } + + for i := 0; i < 20; i++ { + <-done + } +} diff --git a/internal/filter/code_comment_strip_test.go b/internal/filter/code_comment_strip_test.go new file mode 100644 index 000000000..420fca59c --- /dev/null +++ b/internal/filter/code_comment_strip_test.go @@ -0,0 +1,213 @@ +package filter + +import ( + "strings" + "testing" +) + +func TestNewCommentFilter(t *testing.T) { + f := newCommentFilter() + if f == nil { + t.Fatal("newCommentFilter returned nil") + } + if len(f.patterns) == 0 { + t.Error("expected patterns to be initialized") + } +} + +func TestCommentFilterName(t *testing.T) { + f := newCommentFilter() + if f.Name() != "comment" { + t.Errorf("expected name %q, got %q", "comment", f.Name()) + } +} + +func TestCommentFilterApplyGo(t *testing.T) { + f := newCommentFilter() + input := `// This is a comment at line start +package main + +/* block comment */ +func main() { + fmt.Println("hello") +}` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "// This is a comment at line start") { + t.Error("Go line comments at line start should be stripped") + } + if strings.Contains(output, "/* block comment */") { + t.Error("Go block comments should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved from comment stripping") + } +} + +func TestCommentFilterApplyPython(t *testing.T) { + f := newCommentFilter() + input := `def main(): +# This is a comment at line start + print("hello")` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "# This is a comment at line start") { + t.Error("Python line comments at line start should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved") + } +} + +func TestCommentFilterApplyRust(t *testing.T) { + f := newCommentFilter() + input := `fn main() { +// Comment at line start + println!("hello"); +}` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "// Comment at line start") { + t.Error("Rust line comments at line start should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved") + } +} + +func TestCommentFilterApplyJavaScript(t *testing.T) { + f := newCommentFilter() + input := `function main() { +// Comment at line start + console.log("hello"); +}` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "// Comment at line start") { + t.Error("JS line comments at line start should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved") + } +} + +func TestCommentFilterApplySQL(t *testing.T) { + f := newCommentFilter() + input := `SELECT * FROM users +-- SQL comment at line start +WHERE id = 1 +/* block */` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "-- SQL comment at line start") { + t.Error("SQL line comments at line start should be stripped") + } + if strings.Contains(output, "/* block */") { + t.Error("SQL block comments should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved") + } +} + +func TestCommentFilterApplyRuby(t *testing.T) { + f := newCommentFilter() + input := `puts "hello" +# Ruby comment at line start` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "# Ruby comment at line start") { + t.Error("Ruby comments at line start should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved") + } +} + +func TestCommentFilterApplyShell(t *testing.T) { + f := newCommentFilter() + input := `echo "hello" +# Shell comment at line start` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "# Shell comment at line start") { + t.Error("Shell comments at line start should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved") + } +} + +func TestCommentFilterApplyUnknownLanguage(t *testing.T) { + f := newCommentFilter() + input := `Some unknown content +// comment at line start +/* block */` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "// comment at line start") { + t.Error("fallback comment pattern should strip line comments") + } + if strings.Contains(output, "/* block */") { + t.Error("fallback comment pattern should strip block comments") + } + if saved <= 0 { + t.Error("expected some tokens to be saved") + } +} + +func TestCommentFilterApplyNoComments(t *testing.T) { + f := newCommentFilter() + input := `func main() { + fmt.Println("hello") +}` + _, saved := f.Apply(input, ModeMinimal) + + if saved != 0 { + t.Errorf("expected 0 tokens saved for code without comments, got %d", saved) + } +} + +func TestCommentFilterApplyDocstring(t *testing.T) { + f := newCommentFilter() + input := `def main(): + """docstring here""" + print("hello")` + output, saved := f.Apply(input, ModeMinimal) + + if strings.Contains(output, "docstring here") { + t.Error("Python docstrings should be stripped") + } + if saved <= 0 { + t.Error("expected some tokens to be saved from docstring stripping") + } +} + +func TestCommentPatternsMap(t *testing.T) { + if len(CommentPatternsMap) == 0 { + t.Fatal("CommentPatternsMap should not be empty") + } + + expectedLangs := []Language{ + LangRust, LangGo, LangPython, LangJavaScript, LangTypeScript, + LangJava, LangC, LangCpp, LangRuby, LangShell, LangSQL, + } + + for _, lang := range expectedLangs { + if _, ok := CommentPatternsMap[lang]; !ok { + t.Errorf("CommentPatternsMap missing language %q", lang) + } + } +} + +func TestCommentFilterApplyEmpty(t *testing.T) { + f := newCommentFilter() + output, saved := f.Apply("", ModeMinimal) + + if output != "" { + t.Errorf("expected empty output, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for empty input, got %d", saved) + } +} diff --git a/internal/filter/constants_test.go b/internal/filter/constants_test.go new file mode 100644 index 000000000..2f9eb17b0 --- /dev/null +++ b/internal/filter/constants_test.go @@ -0,0 +1,96 @@ +package filter + +import ( + "testing" +) + +func TestConstantsValues(t *testing.T) { + tests := []struct { + name string + value int + min int + max int + }{ + {"TightBudgetThreshold", TightBudgetThreshold, 100, 10000}, + {"MinimalBudgetThreshold", MinimalBudgetThreshold, 10, 500}, + {"MinContentLength", MinContentLength, 10, 200}, + {"SmallContentThreshold", SmallContentThreshold, 100, 1000}, + {"MediumContentThreshold", MediumContentThreshold, 500, 5000}, + {"LargeContentThreshold", LargeContentThreshold, 5000, 50000}, + {"StreamingThreshold", StreamingThreshold, 100000, 1000000}, + {"DefaultCacheSize", DefaultCacheSize, 100, 10000}, + {"HighCompressionRatio", int(HighCompressionRatio * 100), 90, 100}, + {"TargetCompressionRatio", int(TargetCompressionRatio * 100), 50, 90}, + {"MinCompressionRatio", int(MinCompressionRatio * 100), 0, 50}, + {"MinTokenEstimate", MinTokenEstimate, 1, 10}, + {"NumLayerIndices", NumLayerIndices, 20, 30}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.value < tt.min || tt.value > tt.max { + t.Errorf("%s = %d, expected between %d and %d", tt.name, tt.value, tt.min, tt.max) + } + }) + } +} + +func TestLayerIndexConstants(t *testing.T) { + // Check that all layer indices are unique by verifying the last index + 1 == count + if LayerIdxAdvanced+1 != NumLayerIndices { + t.Errorf("last index (%d) + 1 should equal NumLayerIndices (%d)", LayerIdxAdvanced, NumLayerIndices) + } + + // Check ordering: each index should be unique and sequential + indices := allLayerIndices + for i, idx := range indices { + if i != idx { + t.Errorf("allLayerIndices[%d] = %d, expected %d", i, idx, i) + } + } +} + +func TestFirstLayerIndex(t *testing.T) { + if LayerIdxEntropy != 0 { + t.Errorf("LayerIdxEntropy should be 0, got %d", LayerIdxEntropy) + } +} + +func TestLayerIndexCount(t *testing.T) { + if len(allLayerIndices) != NumLayerIndices { + t.Errorf("len(allLayerIndices) = %d, NumLayerIndices = %d", len(allLayerIndices), NumLayerIndices) + } +} + +func TestCompressionRatiosOrdering(t *testing.T) { + if MinCompressionRatio >= TargetCompressionRatio { + t.Errorf("MinCompressionRatio (%f) should be < TargetCompressionRatio (%f)", MinCompressionRatio, TargetCompressionRatio) + } + if TargetCompressionRatio >= HighCompressionRatio { + t.Errorf("TargetCompressionRatio (%f) should be < HighCompressionRatio (%f)", TargetCompressionRatio, HighCompressionRatio) + } +} + +func TestBudgetThresholdsOrdering(t *testing.T) { + if MinimalBudgetThreshold >= TightBudgetThreshold { + t.Errorf("MinimalBudgetThreshold (%d) should be < TightBudgetThreshold (%d)", MinimalBudgetThreshold, TightBudgetThreshold) + } +} + +func TestContentThresholdsOrdering(t *testing.T) { + if MinContentLength >= SmallContentThreshold { + t.Errorf("MinContentLength (%d) should be < SmallContentThreshold (%d)", MinContentLength, SmallContentThreshold) + } + if SmallContentThreshold >= MediumContentThreshold { + t.Errorf("SmallContentThreshold (%d) should be < MediumContentThreshold (%d)", SmallContentThreshold, MediumContentThreshold) + } + if MediumContentThreshold >= LargeContentThreshold { + t.Errorf("MediumContentThreshold (%d) should be < LargeContentThreshold (%d)", MediumContentThreshold, LargeContentThreshold) + } +} + +func TestTokensPerCharHeuristic(t *testing.T) { + if TokensPerCharHeuristic <= 0 || TokensPerCharHeuristic >= 1 { + t.Errorf("TokensPerCharHeuristic (%f) should be between 0 and 1", TokensPerCharHeuristic) + } +} diff --git a/internal/filter/dedup_test.go b/internal/filter/dedup_test.go new file mode 100644 index 000000000..d733c9731 --- /dev/null +++ b/internal/filter/dedup_test.go @@ -0,0 +1,249 @@ +package filter + +import ( + "testing" +) + +func TestSimHash(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"empty string", ""}, + {"short text", "hello"}, + {"longer text", "this is a longer piece of text for hashing"}, + {"with special chars", "hello!@#$%^&*()world"}, + {"repeated content", "aaaaaaa"}, + {"unicode", "hello δΈ–η•Œ"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hash := SimHash(tt.input) + hash2 := SimHash(tt.input) + if hash != hash2 { + t.Errorf("SimHash should be deterministic: first=%d, second=%d", hash, hash2) + } + }) + } +} + +func TestSimHashDifferentInputs(t *testing.T) { + hash1 := SimHash("hello world") + hash2 := SimHash("goodbye universe") + if hash1 == hash2 { + t.Error("different inputs should produce different hashes") + } +} + +func TestHammingDistance(t *testing.T) { + tests := []struct { + a, b uint64 + expected int + }{ + {0, 0, 0}, + {0, 1, 1}, + {1, 1, 0}, + {0xFFFFFFFF, 0, 32}, + {^uint64(0), 0, 64}, + {0x5555555555555555, 0xAAAAAAAAAAAAAAAA, 64}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + result := HammingDistance(tt.a, tt.b) + if result != tt.expected { + t.Errorf("HammingDistance(0x%X, 0x%X) = %d, expected %d", tt.a, tt.b, result, tt.expected) + } + }) + } +} + +func TestIsNearDuplicate(t *testing.T) { + tests := []struct { + name string + a, b string + threshold int + expected bool + }{ + { + name: "identical content", + a: "hello world", + b: "hello world", + threshold: 3, + expected: true, + }, + { + name: "very different content", + a: "hello world", + b: "completely different content here", + threshold: 3, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := IsNearDuplicate(tt.a, tt.b, tt.threshold) + if result != tt.expected { + t.Errorf("IsNearDuplicate(%q, %q, %d) = %v, expected %v", tt.a, tt.b, tt.threshold, result, tt.expected) + } + }) + } +} + +func TestNewCrossMessageDedup(t *testing.T) { + d := NewCrossMessageDedup() + if d == nil { + t.Fatal("NewCrossMessageDedup returned nil") + } + if d.threshold != 3 { + t.Errorf("expected default threshold 3, got %d", d.threshold) + } + if d.seen == nil { + t.Error("seen map should be initialized") + } +} + +func TestCrossMessageDedupNewContent(t *testing.T) { + d := NewCrossMessageDedup() + isDup, replacement := d.DedupMessage("new unique content") + if isDup { + t.Error("new content should not be marked as duplicate") + } + if replacement != "new unique content" { + t.Errorf("expected original content as replacement, got %q", replacement) + } +} + +func TestCrossMessageDedupExactDuplicate(t *testing.T) { + d := NewCrossMessageDedup() + d.DedupMessage("hello world") + isDup, replacement := d.DedupMessage("hello world") + if !isDup { + t.Error("exact duplicate should be marked as duplicate") + } + if replacement != "[duplicate: previously sent]" { + t.Errorf("expected duplicate marker, got %q", replacement) + } +} + +func TestCrossMessageDedupClear(t *testing.T) { + d := NewCrossMessageDedup() + d.DedupMessage("hello world") + if d.Count() != 1 { + t.Errorf("expected count 1, got %d", d.Count()) + } + d.Clear() + if d.Count() != 0 { + t.Errorf("expected count 0 after Clear, got %d", d.Count()) + } +} + +func TestCrossMessageDedupCount(t *testing.T) { + d := NewCrossMessageDedup() + d.DedupMessage("first message") + d.DedupMessage("second message") + d.DedupMessage("third message") + if d.Count() != 3 { + t.Errorf("expected count 3, got %d", d.Count()) + } +} + +func TestCrossMessageDedupAfterClear(t *testing.T) { + d := NewCrossMessageDedup() + d.DedupMessage("hello world") + d.Clear() + isDup, _ := d.DedupMessage("hello world") + if isDup { + t.Error("content should not be duplicate after Clear") + } +} + +func TestGenerateDiff(t *testing.T) { + tests := []struct { + name string + old, new string + expected string + }{ + { + name: "added line", + old: "line1\nline2", + new: "line1\nline2\nline3", + expected: "[diff]", + }, + { + name: "removed line", + old: "line1\nline2\nline3", + new: "line1\nline2", + expected: "[diff]", + }, + { + name: "identical", + old: "same content", + new: "same content", + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := generateDiff(tt.old, tt.new) + if tt.expected == "" { + if result != "" { + t.Errorf("expected empty diff for identical content, got %q", result) + } + } else { + if !strContains(result, tt.expected) { + t.Errorf("expected diff to contain %q, got %q", tt.expected, result) + } + } + }) + } +} + +func strContains(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +func TestSimHashConsistency(t *testing.T) { + input := "the quick brown fox jumps over the lazy dog" + h1 := SimHash(input) + h2 := SimHash(input) + if h1 != h2 { + t.Error("SimHash should be consistent") + } +} + +func TestSimHashEmptyString(t *testing.T) { + hash := SimHash("") + if hash != 0 { + t.Errorf("expected hash 0 for empty string, got %d", hash) + } +} + +func TestCrossMessageDedupConcurrent(t *testing.T) { + d := NewCrossMessageDedup() + done := make(chan bool) + + for i := 0; i < 10; i++ { + go func(n int) { + content := "message " + string(rune('a'+n)) + d.DedupMessage(content) + done <- true + }(i) + } + + for i := 0; i < 10; i++ { + <-done + } + + if d.Count() < 1 { + t.Error("expected at least 1 unique message after concurrent access") + } +} diff --git a/internal/filter/equivalence.go b/internal/filter/equivalence.go index fe4a3221c..c1fcd8d1e 100644 --- a/internal/filter/equivalence.go +++ b/internal/filter/equivalence.go @@ -183,17 +183,16 @@ func extractURLs(s string) []string { } func extractPaths(s string) []string { + seen := make(map[string]bool) var paths []string for _, line := range strings.Split(s, "\n") { for _, prefix := range []string{"/", "./", "../", "~/"} { if strings.Contains(line, prefix) { - // Simple heuristic: extract path-like strings words := strings.Fields(line) for _, w := range words { - if strings.HasPrefix(w, prefix) || strings.Contains(w, "/") { - if len(w) > 3 && len(w) < 200 { - paths = append(paths, w) - } + if (strings.HasPrefix(w, prefix) || strings.Contains(w, "/")) && len(w) > 3 && len(w) < 200 && !seen[w] { + seen[w] = true + paths = append(paths, w) } } } diff --git a/internal/filter/equivalence_test.go b/internal/filter/equivalence_test.go index e89ed68ef..17a175fa5 100644 --- a/internal/filter/equivalence_test.go +++ b/internal/filter/equivalence_test.go @@ -5,141 +5,268 @@ import ( ) func TestNewSemanticEquivalence(t *testing.T) { - se := NewSemanticEquivalence() - if se == nil { - t.Fatal("expected non-nil SemanticEquivalence") + s := NewSemanticEquivalence() + if s == nil { + t.Fatal("NewSemanticEquivalence returned nil") } } -func TestSemanticEquivalence_Check_NoErrors(t *testing.T) { - se := NewSemanticEquivalence() - original := "hello world" - compressed := "hello" - report := se.Check(original, compressed) +func TestSemanticEquivalenceCheck(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("Error: something failed", "Error: something failed") - if report.ErrorPreserved != true { - t.Error("expected ErrorPreserved true when no errors in original") + if !report.ErrorPreserved { + t.Error("errors should be preserved") } - if report.Score <= 0 { - t.Error("expected positive score") + if !report.IsGood() { + t.Error("report should be good for identical content") } } -func TestSemanticEquivalence_Check_WithErrors(t *testing.T) { - se := NewSemanticEquivalence() - original := "ERROR: something failed" - compressed := "all good" - report := se.Check(original, compressed) +func TestSemanticEquivalenceCheckErrorLost(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("Error: something failed", "Everything is fine") if report.ErrorPreserved { - t.Error("expected ErrorPreserved false when errors were dropped") + t.Error("error should not be preserved if it's missing from compressed output") } } -func TestSemanticEquivalence_Check_PreservesErrors(t *testing.T) { - se := NewSemanticEquivalence() - original := "ERROR: something failed" - compressed := "ERROR: something failed" - report := se.Check(original, compressed) +func TestSemanticEquivalenceCheckNumbers(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("Error at line 42", "Error at line 42") - if !report.ErrorPreserved { - t.Error("expected ErrorPreserved true when errors preserved") + if !report.NumbersPreserved { + t.Error("numbers should be preserved") } } -func TestEquivalenceReport_IsGood(t *testing.T) { - // Good report - r1 := EquivalenceReport{Score: 0.8, ErrorPreserved: true} - if !r1.IsGood() { - t.Error("expected IsGood true for score 0.8 + errors preserved") +func TestSemanticEquivalenceCheckNumbersLost(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("Error at line 42", "Error at line") + + if report.NumbersPreserved { + t.Error("numbers should not be preserved if missing") } +} + +func TestSemanticEquivalenceCheckURLs(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("See https://example.com for details", "See https://example.com for details") - // Bad score - r2 := EquivalenceReport{Score: 0.5, ErrorPreserved: true} - if r2.IsGood() { - t.Error("expected IsGood false for score 0.5") + if !report.URLsPreserved { + t.Error("URLs should be preserved") } +} + +func TestSemanticEquivalenceCheckURLsLost(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("See https://example.com for details", "See for details") - // Missing errors - r3 := EquivalenceReport{Score: 0.9, ErrorPreserved: false} - if r3.IsGood() { - t.Error("expected IsGood false when errors not preserved") + if report.URLsPreserved { + t.Error("URLs should not be preserved if missing") } } -func TestCheckCriticalNumbers(t *testing.T) { - original := "exit code 42, line 100" - compressed := "exit code 42" - if !checkCriticalNumbers(original, compressed) { - t.Error("expected numbers preserved when exit code kept") +func TestSemanticEquivalenceCheckPaths(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("File /home/user/test.go not found", "File /home/user/test.go not found") + + if !report.FilePathsPreserved { + t.Error("file paths should be preserved") } +} + +func TestSemanticEquivalenceCheckPathsLost(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("File /home/user/test.go not found", "File not found") - compressed2 := "all done" - if checkCriticalNumbers(original, compressed2) { - t.Error("expected numbers not preserved when dropped") + if report.FilePathsPreserved { + t.Error("file paths should not be preserved if missing") } } -func TestCheckURLsPreserved(t *testing.T) { - original := "See https://example.com for details" - compressed := "See https://example.com for details" - if !checkURLsPreserved(original, compressed) { - t.Error("expected URLs preserved") +func TestSemanticEquivalenceCheckExitCodes(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("Process exited with exit code 1", "Process exited with exit code 1") + + if !report.ExitCodesPreserved { + t.Error("exit codes should be preserved") } +} - compressed2 := "See docs for details" - if checkURLsPreserved(original, compressed2) { - t.Error("expected URLs not preserved when dropped") +func TestSemanticEquivalenceCheckExitCodesLost(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("Process exited with exit code 1", "Process exited") + + if report.ExitCodesPreserved { + t.Error("exit codes should not be preserved if missing") } } -func TestCheckPathsPreserved(t *testing.T) { - original := "File: /tmp/test.go" - compressed := "File: /tmp/test.go" - if !checkPathsPreserved(original, compressed) { - t.Error("expected paths preserved") +func TestSemanticEquivalenceCheckNoErrorsInOriginal(t *testing.T) { + s := NewSemanticEquivalence() + report := s.Check("All good", "All good") + + if !report.ErrorPreserved { + t.Error("should be preserved when no errors in original") } } -func TestCheckExitCodes(t *testing.T) { - original := "Process exited with exit code 1" - compressed := "Process exited with exit code 1" - if !checkExitCodes(original, compressed) { - t.Error("expected exit codes preserved") +func TestEquivalenceReportIsGood(t *testing.T) { + tests := []struct { + name string + report EquivalenceReport + expected bool + }{ + { + name: "good score and error preserved", + report: EquivalenceReport{Score: 0.8, ErrorPreserved: true}, + expected: true, + }, + { + name: "good score but error not preserved", + report: EquivalenceReport{Score: 0.8, ErrorPreserved: false}, + expected: false, + }, + { + name: "low score and error preserved", + report: EquivalenceReport{Score: 0.5, ErrorPreserved: true}, + expected: false, + }, + { + name: "perfect score and error preserved", + report: EquivalenceReport{Score: 1.0, ErrorPreserved: true}, + expected: true, + }, } - compressed2 := "Process done" - if checkExitCodes(original, compressed2) { - t.Error("expected exit codes not preserved when dropped") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.report.IsGood() + if result != tt.expected { + t.Errorf("IsGood() = %v, expected %v", result, tt.expected) + } + }) } } -func TestComputeEquivalenceScore(t *testing.T) { - original := "test output with error" - compressed := "test output with error" - score := computeEquivalenceScore(original, compressed) - if score != 1.0 { - t.Errorf("expected score 1.0 for identical content, got %f", score) +func TestCheckErrorsPreserved(t *testing.T) { + tests := []struct { + original string + compressed string + expected bool + }{ + {"Error: failed", "Error: failed", true}, + {"error occurred", "error occurred", true}, + {"FAIL: something", "FAIL: something", true}, + {"Error: failed", "success", false}, + {"no errors here", "no errors here", true}, + {"everything working fine", "all good", true}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + result := checkErrorsPreserved(tt.original, tt.compressed) + if result != tt.expected { + t.Errorf("checkErrorsPreserved(%q, %q) = %v, expected %v", tt.original, tt.compressed, result, tt.expected) + } + }) + } +} + +func TestCheckURLsPreserved(t *testing.T) { + tests := []struct { + original string + compressed string + expected bool + }{ + {"See https://example.com", "See https://example.com", true}, + {"See http://test.org", "See http://test.org", true}, + {"See https://example.com", "See", false}, + {"no urls here", "no urls here", true}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + result := checkURLsPreserved(tt.original, tt.compressed) + if result != tt.expected { + t.Errorf("checkURLsPreserved(%q, %q) = %v, expected %v", tt.original, tt.compressed, result, tt.expected) + } + }) } } func TestExtractCriticalNumbers(t *testing.T) { - nums := extractCriticalNumbers("line 42, exit 1, port 8080") - if len(nums) == 0 { - t.Error("expected some numbers extracted") + tests := []struct { + input string + expected int + }{ + {"line 42", 1}, + {"lines 1, 2, 3", 3}, + {"exit code 1", 1}, + {"no numbers", 0}, + {"12345", 1}, + {"12345678901234567890", 0}, // too long (>5 digits) + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := extractCriticalNumbers(tt.input) + if len(result) != tt.expected { + t.Errorf("extractCriticalNumbers(%q) returned %d numbers, expected %d", tt.input, len(result), tt.expected) + } + }) } } func TestExtractURLs(t *testing.T) { - urls := extractURLs("Visit https://example.com and http://test.org") - if len(urls) != 2 { - t.Errorf("expected 2 URLs, got %d", len(urls)) + tests := []struct { + input string + expected int + }{ + {"https://example.com", 1}, + {"http://test.org path", 1}, + {"https://a.com and https://b.com", 2}, + {"no urls here", 0}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := extractURLs(tt.input) + if len(result) != tt.expected { + t.Errorf("extractURLs(%q) returned %d URLs, expected %d", tt.input, len(result), tt.expected) + } + }) } } func TestExtractPaths(t *testing.T) { - paths := extractPaths("File: /tmp/test.go\nDir: ./src") - if len(paths) == 0 { - t.Error("expected some paths extracted") + tests := []struct { + input string + expected int + }{ + {"/home/user/file.txt", 1}, + {"./relative/path", 1}, + {"../parent/path", 1}, + {"~/home/path", 1}, + {"no paths here", 0}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := extractPaths(tt.input) + if len(result) != tt.expected { + t.Errorf("extractPaths(%q) returned %d paths, expected %d", tt.input, len(result), tt.expected) + } + }) + } +} + +func TestComputeEquivalenceScore(t *testing.T) { + // Identical content should score 1.0 + score := computeEquivalenceScore("hello world", "hello world") + if score != 1.0 { + t.Errorf("expected score 1.0 for identical content, got %f", score) } } diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go new file mode 100644 index 000000000..a921ddd6f --- /dev/null +++ b/internal/filter/filter_test.go @@ -0,0 +1,153 @@ +package filter + +import ( + "strings" + "testing" +) + +func TestEngineProcess(t *testing.T) { + engine := NewEngine(ModeMinimal) + input := "Hello, world!" + output, saved := engine.Process(input) + if output == "" { + t.Error("expected non-empty output") + } + if saved < 0 { + t.Errorf("expected non-negative tokens saved, got %d", saved) + } +} + +func TestEngineProcessWithANSI(t *testing.T) { + engine := NewEngine(ModeMinimal) + input := "\x1b[32mGreen text\x1b[0m and normal text" + output, saved := engine.Process(input) + if strings.Contains(output, "\x1b[") { + t.Error("ANSI codes should be stripped from output") + } + if saved <= 0 { + t.Error("expected some tokens to be saved from ANSI stripping") + } +} + +func TestEngineSetMode(t *testing.T) { + engine := NewEngine(ModeMinimal) + engine.SetMode(ModeAggressive) + if engine.mode != ModeAggressive { + t.Errorf("expected mode %q, got %q", ModeAggressive, engine.mode) + } +} + +func TestEngineProcessMinimalSkipsBody(t *testing.T) { + engine := NewEngine(ModeMinimal) + input := "func main() {\n fmt.Println(\"hello\")\n}\nSome body text here" + output, _ := engine.Process(input) + if !strings.Contains(output, "func main") { + t.Error("expected function signature to be preserved in minimal mode") + } +} + +func TestDetectLanguage(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "Go code", + input: "package main\n\nfunc main() {\n fmt.Println(\"hello\")\n}", + expected: "go", + }, + { + name: "Rust code", + input: "fn main() {\n println!(\"hello\");\n}", + expected: "rust", + }, + { + name: "Python code", + input: "def greet():\n print(\"hello\")", + expected: "python", + }, + { + name: "JavaScript code", + input: "function hello() {\n console.log('hello');\n}", + expected: "javascript", + }, + { + name: "TypeScript code", + input: "interface User {\n name: string;\n}", + expected: "typescript", + }, + { + name: "Java code", + input: "public class Main {\n public static void main(String[] args) {}\n}", + expected: "java", + }, + { + name: "C++ code", + input: "std::cout << \"hello\" << std::endl;", + expected: "cpp", + }, + { + name: "C code", + input: "printf(\"hello\");", + expected: "c", + }, + { + name: "Ruby code", + input: "puts 'hello'", + expected: "ruby", + }, + { + name: "Shell code", + input: "chmod +x script.sh", + expected: "shell", + }, + { + name: "SQL code", + input: "SELECT * FROM users WHERE id = 1", + expected: "sql", + }, + { + name: "Unknown content", + input: "Hello, world!", + expected: "unknown", + }, + { + name: "Empty string", + input: "", + expected: "unknown", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := DetectLanguage(tt.input) + if result != tt.expected { + t.Errorf("DetectLanguage(%q) = %q, expected %q", tt.input, result, tt.expected) + } + }) + } +} + +func TestDetectLanguageFromInput(t *testing.T) { + result := DetectLanguageFromInput("func main() {}") + if result != LangGo { + t.Errorf("expected %q, got %q", LangGo, result) + } +} + +func TestDetectLanguageLargeInput(t *testing.T) { + largeInput := strings.Repeat("func main() { fmt.Println(\"hello\") } ", 1000) + result := DetectLanguage(largeInput) + if result != "go" { + t.Errorf("expected %q for large Go input, got %q", "go", result) + } +} + +func TestDetectLanguagePythonWithCurlyBracePenalty(t *testing.T) { + input := "def hello():\n print(\"hello\")\n{ this is not python }" + result := DetectLanguage(input) + if result == "python" { + t.Error("Python should be penalized when curly braces are present") + } +} diff --git a/internal/filter/import_test.go b/internal/filter/import_test.go new file mode 100644 index 000000000..1c5d1cf1c --- /dev/null +++ b/internal/filter/import_test.go @@ -0,0 +1,146 @@ +package filter + +import ( + "strings" + "testing" +) + +func TestNewImportFilter(t *testing.T) { + f := NewImportFilter() + if f == nil { + t.Fatal("NewImportFilter returned nil") + } + if len(f.patterns) == 0 { + t.Error("expected patterns to be initialized") + } +} + +func TestImportFilterName(t *testing.T) { + f := NewImportFilter() + if f.Name() != "import" { + t.Errorf("expected name %q, got %q", "import", f.Name()) + } +} + +func TestImportFilterApplyNonCode(t *testing.T) { + f := NewImportFilter() + input := "This is plain text, not code." + output, saved := f.Apply(input, ModeMinimal) + + if output != input { + t.Errorf("expected unchanged output for non-code, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for non-code, got %d", saved) + } +} + +func TestImportFilterCondensesGoImports(t *testing.T) { + f := NewImportFilter() + input := `package main + +import ( + "fmt" + "os" +) + +func main() { + fmt.Println("hello") +}` + output, _ := f.Apply(input, ModeMinimal) + + if !strings.Contains(output, "imports condensed") || !strings.Contains(output, "package main") { + t.Error("Go imports should be condensed") + } +} + +func TestImportFilterCondensesPythonImports(t *testing.T) { + f := NewImportFilter() + input := `import os +import sys + +def main(): + print("hello") +` + output, _ := f.Apply(input, ModeMinimal) + + if !strings.Contains(output, "imports") || !strings.Contains(output, "def main") { + t.Error("Python imports should be condensed but preserve function") + } +} + +func TestImportFilterCondensesJSImports(t *testing.T) { + f := NewImportFilter() + input := `import { readFile } from 'fs'; +import path from 'path'; + +function main() { + console.log('hello'); +}` + output, _ := f.Apply(input, ModeMinimal) + + if !strings.Contains(output, "imports") || !strings.Contains(output, "function main") { + t.Error("JS imports should be condensed but preserve function") + } +} + +func TestImportFilterCondensesRustImports(t *testing.T) { + f := NewImportFilter() + input := `use std::io; +use std::fs::File; + +fn main() { + println!("hello"); +}` + output, _ := f.Apply(input, ModeMinimal) + + if !strings.Contains(output, "imports") || !strings.Contains(output, "fn main") { + t.Error("Rust imports should be condensed but preserve function") + } +} + +func TestImportFilterApplyNoImports(t *testing.T) { + f := NewImportFilter() + input := `func main() { + fmt.Println("hello") +}` + _, saved := f.Apply(input, ModeMinimal) + + if saved != 0 { + t.Errorf("expected 0 tokens saved for code without imports, got %d", saved) + } +} + +func TestImportFilterEmpty(t *testing.T) { + f := NewImportFilter() + output, saved := f.Apply("", ModeMinimal) + + if output != "" { + t.Errorf("expected empty output, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for empty input, got %d", saved) + } +} + +func TestImportFilterSavesTokensWithManyImports(t *testing.T) { + f := NewImportFilter() + input := `package main + +import ( + "fmt" + "os" + "strings" + "time" + "encoding/json" +) + +func main() { + fmt.Println("hello") +}` + _, saved := f.Apply(input, ModeMinimal) + + if saved <= 0 { + t.Error("expected tokens saved for code with many imports") + } +} diff --git a/internal/filter/multi_file_test.go b/internal/filter/multi_file_test.go new file mode 100644 index 000000000..451ec4f5b --- /dev/null +++ b/internal/filter/multi_file_test.go @@ -0,0 +1,297 @@ +package filter + +import ( + "strings" + "testing" +) + +func TestNewMultiFileFilter(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + if f == nil { + t.Fatal("NewMultiFileFilter returned nil") + } + if f.maxCombinedSize != 50000 { + t.Errorf("expected default maxCombinedSize %d, got %d", 50000, f.maxCombinedSize) + } + if f.similarityThreshold != 0.8 { + t.Errorf("expected default similarityThreshold %f, got %f", 0.8, f.similarityThreshold) + } +} + +func TestMultiFileFilterName(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + if f.Name() != "multi_file" { + t.Errorf("expected name %q, got %q", "multi_file", f.Name()) + } +} + +func TestMultiFileFilterSingleFile(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + input := "just a single file content" + output, saved := f.Apply(input, ModeMinimal) + if output != input { + t.Errorf("expected unchanged output for single file, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for single file, got %d", saved) + } +} + +func TestMultiFileFilterParsesMultipleFiles(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{ + PreserveBoundaries: true, + }) + input := `=== File: main.go === +package main + +func main() {} + +=== File: utils.go === +package main + +func helper() {}` + output, _ := f.Apply(input, ModeMinimal) + + if !strings.Contains(output, "main.go") { + t.Error("expected file main.go to be preserved") + } + if !strings.Contains(output, "utils.go") { + t.Error("expected file utils.go to be preserved") + } + if !strings.Contains(output, "func main") { + t.Error("expected function content to be preserved") + } + if !strings.Contains(output, "func helper") { + t.Error("expected helper function to be preserved") + } +} + +func TestMultiFileFilterParseFilesNoMarkers(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + files := f.parseFiles("no markers here\njust plain text") + if len(files) != 1 { + t.Errorf("expected 1 file for input without markers, got %d", len(files)) + } + if files[0].name != "output" { + t.Errorf("expected default name 'output', got %q", files[0].name) + } +} + +func TestMultiFileFilterExtractMetadata(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + + file := &fileInfo{ + name: "test.go", + content: `package main + +import "fmt" + +func main() { + fmt.Println("hello") +}`, + } + f.extractMetadata(file) + + if len(file.imports) == 0 { + t.Error("expected imports to be extracted") + } + if len(file.functions) == 0 { + t.Error("expected functions to be extracted") + } +} + +func TestMultiFileFilterSameModule(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + if !f.sameModule("foo/bar/main.go", "foo/bar/utils.go") { + t.Error("files in same directory should be same module") + } + if !f.sameModule("main.go", "utils.go") { + t.Error("root files should be same module") + } + if f.sameModule("foo/main.go", "bar/utils.go") { + t.Error("files in different directories should not be same module") + } +} + +func TestMultiFileFilterCalculateSimilarity(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + + same := f.calculateSimilarity("hello world", "hello world") + if same != 1.0 { + t.Errorf("expected similarity 1.0 for identical strings, got %f", same) + } + + different := f.calculateSimilarity("hello world", "completely different content") + if different > 0 { + t.Errorf("expected similarity 0 for very different strings, got %f", different) + } +} + +func TestMultiFileFilterHasImportRelation(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + + file1 := fileInfo{ + name: "main.go", + imports: []string{"import \"fmt\"", "import \"./utils\""}, + } + file2 := fileInfo{ + name: "utils.go", + content: "package main\nfunc helper() {}", + } + + if !f.hasImportRelation(file1, file2) { + t.Error("expected import relation between main.go and utils.go") + } + + file3 := fileInfo{ + name: "unrelated.go", + content: "package main\nfunc something() {}", + } + + if f.hasImportRelation(file1, file3) { + t.Error("expected no import relation with unrelated file") + } +} + +func TestMultiFileFilterSetMaxCombinedSize(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + f.SetMaxCombinedSize(1000) + if f.maxCombinedSize != 1000 { + t.Errorf("expected maxCombinedSize %d, got %d", 1000, f.maxCombinedSize) + } +} + +func TestMultiFileFilterSetPreserveBoundaries(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + f.SetPreserveBoundaries(false) + if f.preserveBoundaries { + t.Error("expected preserveBoundaries to be false") + } +} + +func TestMultiFileFilterSetSimilarityThreshold(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + f.SetSimilarityThreshold(0.5) + if f.similarityThreshold != 0.5 { + t.Errorf("expected similarityThreshold %f, got %f", 0.5, f.similarityThreshold) + } +} + +func TestMultiFileFilterEmptyInput(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + output, saved := f.Apply("", ModeMinimal) + if output != "" { + t.Errorf("expected empty output, got %q", output) + } + if saved != 0 { + t.Errorf("expected 0 tokens saved for empty input, got %d", saved) + } +} + +func TestMultiFileFilterFindSharedImports(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + files := []fileInfo{ + {name: "main.go", imports: []string{"import \"fmt\"", "import \"os\""}}, + {name: "utils.go", imports: []string{"import \"fmt\"", "import \"strings\""}}, + } + + shared := f.findSharedImports(files) + if len(shared) == 0 { + t.Error("expected shared imports to be found") + } + + found := false + for _, imp := range shared { + if strings.ToLower(imp) == "import \"fmt\"" { + found = true + break + } + } + if !found { + t.Error("expected 'import \"fmt\"' to be in shared imports") + } +} + +func TestMultiFileFilterRemoveShared(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + items := []string{"import \"fmt\"", "import \"os\"", "import \"net/http\""} + shared := []string{"import \"fmt\""} + + result := f.removeShared(items, shared) + if len(result) != 2 { + t.Errorf("expected 2 items after removal, got %d", len(result)) + } + for _, item := range result { + if strings.Contains(item, "fmt") { + t.Error("shared import should have been removed") + } + } +} + +func TestMultiFileFilterAggressiveModePreservesStructure(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{ + PreserveBoundaries: true, + }) + input := `=== File: main.go === +package main + +func main() { + fmt.Println("hello") +} + +=== File: utils.go === +package main + +func helper() { + return 42 +}` + output, _ := f.Apply(input, ModeAggressive) + + if !strings.Contains(output, "main.go") { + t.Error("expected file boundaries to be preserved in aggressive mode") + } + if !strings.Contains(output, "utils.go") { + t.Error("expected file boundaries to be preserved in aggressive mode") + } +} + +func TestMultiFileFilterCreateUnifiedOutput(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{ + PreserveBoundaries: true, + }) + files := []deduplicatedFile{ + {name: "main.go", content: "package main\nfunc main() {}", imports: []string{}, exports: []string{}}, + } + output := f.createUnifiedOutput(files, ModeMinimal) + + if !strings.Contains(output, "main.go") { + t.Error("expected file marker in output") + } + if !strings.Contains(output, "func main()") { + t.Error("expected content in output") + } +} + +func TestMultiFileFilterAnalyzeRelationships(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + files := []fileInfo{ + {name: "main.go", content: "package main\nfunc main() {}", imports: []string{}, exports: []string{}, functions: []string{"func main()"}}, + {name: "utils.go", content: "package main\nfunc helper() {}", imports: []string{}, exports: []string{}, functions: []string{"func helper()"}}, + } + + relations := f.analyzeRelationships(files) + if len(relations) < 1 { + t.Error("expected at least one relationship between files in same module") + } +} + +func TestMultiFileFilterDeduplicate(t *testing.T) { + f := NewMultiFileFilter(MultiFileConfig{}) + files := []fileInfo{ + {name: "main.go", content: "unique content", imports: []string{}, exports: []string{}}, + } + deduped := f.deduplicate(files, nil) + if len(deduped) != 1 { + t.Errorf("expected 1 file after dedup, got %d", len(deduped)) + } +} diff --git a/internal/filter/noise_test.go b/internal/filter/noise_test.go new file mode 100644 index 000000000..9a243748a --- /dev/null +++ b/internal/filter/noise_test.go @@ -0,0 +1,146 @@ +package filter + +import ( + "strings" + "testing" +) + +func TestFilterProgressBars(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "no progress bars", + input: "line1\nline2\nline3", + expected: "line1\nline2\nline3", + }, + { + name: "progress bar at line start", + input: "line1\n[####] 50%\nline2", + expected: "line1\nline2", + }, + { + name: "empty input", + input: "", + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := FilterProgressBars(tt.input) + if result != tt.expected { + t.Errorf("FilterProgressBars() = %q, expected %q", result, tt.expected) + } + }) + } +} + +func TestFilterProgressBarsPreservesContent(t *testing.T) { + input := "line1\n[####] 50%\n\nline2" + result := FilterProgressBars(input) + if strings.Contains(result, "[####]") { + t.Error("progress bar should be removed") + } + if !strings.Contains(result, "line1") { + t.Error("line1 should be preserved") + } + if !strings.Contains(result, "line2") { + t.Error("line2 should be preserved") + } +} + +func TestCleanInlineProgress(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"No progress here", "No progress here"}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := cleanInlineProgress(tt.input) + if result != tt.expected { + t.Errorf("cleanInlineProgress(%q) = %q, expected %q", tt.input, result, tt.expected) + } + }) + } +} + +func TestFilterNoisyOutput(t *testing.T) { + tests := []struct { + name string + input string + check func(string) bool + }{ + { + name: "removes ANSI codes", + input: "\x1b[32mGreen\x1b[0m text", + check: func(result string) bool { + return !strings.Contains(result, "\x1b[") + }, + }, + { + name: "removes carriage returns", + input: "line1\r\nline2\r\n", + check: func(result string) bool { + return !strings.Contains(result, "\r") + }, + }, + { + name: "limits blank lines", + input: "line1\n\n\n\n\nline2", + check: func(result string) bool { + return strings.Count(result, "\n\n\n") == 0 + }, + }, + { + name: "preserves content", + input: "Important message\nError: something failed", + check: func(result string) bool { + return strings.Contains(result, "Important message") && + strings.Contains(result, "Error: something failed") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := FilterNoisyOutput(tt.input) + if !tt.check(result) { + t.Errorf("FilterNoisyOutput(%q) = %q, check failed", tt.input, result) + } + }) + } +} + +func TestFilterNoisyOutputEmpty(t *testing.T) { + result := FilterNoisyOutput("") + if result != "" { + t.Errorf("expected empty string, got %q", result) + } +} + +func TestFilterNoisyOutputRemovesProgressBars(t *testing.T) { + input := "line1\n[####] 50%\nline2" + result := FilterNoisyOutput(input) + if strings.Contains(result, "[####]") { + t.Error("FilterNoisyOutput should remove progress bars") + } +} + +func TestProgressRegex(t *testing.T) { + if progressRegex == nil { + t.Fatal("progressRegex should be initialized") + } +} + +func TestProgressLineRegex(t *testing.T) { + if progressLineRegex == nil { + t.Fatal("progressLineRegex should be initialized") + } +} diff --git a/internal/filter/progress_test.go b/internal/filter/progress_test.go new file mode 100644 index 000000000..1de37535d --- /dev/null +++ b/internal/filter/progress_test.go @@ -0,0 +1,86 @@ +package filter + +import ( + "testing" +) + +func TestSetProgressCallback(t *testing.T) { + called := false + cb := func(layerName string, inputTokens, outputTokens int, progressPercent float64) { + called = true + } + + SetProgressCallback(cb) + got := GetProgressCallback() + if got == nil { + t.Fatal("GetProgressCallback returned nil") + } + + got("layer1", 100, 80, 50.0) + if !called { + t.Error("progress callback should have been called") + } +} + +func TestGetProgressCallbackDefault(t *testing.T) { + SetProgressCallback(nil) + cb := GetProgressCallback() + if cb != nil { + t.Error("GetProgressCallback should return nil when not set") + } +} + +func TestProgressCallbackInvocation(t *testing.T) { + var capturedLayer string + var capturedInputTokens, capturedOutputTokens int + var capturedProgress float64 + + cb := func(layerName string, inputTokens, outputTokens int, progressPercent float64) { + capturedLayer = layerName + capturedInputTokens = inputTokens + capturedOutputTokens = outputTokens + capturedProgress = progressPercent + } + + SetProgressCallback(cb) + GetProgressCallback()("test-layer", 500, 300, 75.5) + + if capturedLayer != "test-layer" { + t.Errorf("expected layer name %q, got %q", "test-layer", capturedLayer) + } + if capturedInputTokens != 500 { + t.Errorf("expected input tokens %d, got %d", 500, capturedInputTokens) + } + if capturedOutputTokens != 300 { + t.Errorf("expected output tokens %d, got %d", 300, capturedOutputTokens) + } + if capturedProgress != 75.5 { + t.Errorf("expected progress %f, got %f", 75.5, capturedProgress) + } +} + +func TestProgressCallbackNilAfterClear(t *testing.T) { + SetProgressCallback(func(layerName string, inputTokens, outputTokens int, progressPercent float64) {}) + SetProgressCallback(nil) + + cb := GetProgressCallback() + if cb != nil { + t.Error("progress callback should be nil after clearing") + } +} + +func TestProgressCallbackGoroutineSafety(t *testing.T) { + done := make(chan bool) + + for i := 0; i < 10; i++ { + go func(n int) { + SetProgressCallback(func(layerName string, inputTokens, outputTokens int, progressPercent float64) {}) + GetProgressCallback() + done <- true + }(i) + } + + for i := 0; i < 10; i++ { + <-done + } +} diff --git a/internal/filter/signature_patterns.go b/internal/filter/signature_patterns.go index 224eaa62d..24ff2887e 100644 --- a/internal/filter/signature_patterns.go +++ b/internal/filter/signature_patterns.go @@ -17,7 +17,7 @@ var SignaturePatterns = []*regexp.Regexp{ regexp.MustCompile(`^async\s+def\s+\w+`), regexp.MustCompile(`^class\s+\w+`), regexp.MustCompile(`^function\s+\w+`), - regexp.MustCompile(`^(export\s+)?(async\s+)?function\s*\w*`), + regexp.MustCompile(`^(export\s+)?(default\s+)?(async\s+)?function\s*\w*`), regexp.MustCompile(`^(export\s+)?(default\s+)?class\s+\w+`), regexp.MustCompile(`^(export\s+)?const\s+\w+\s*=\s*(async\s+)?\([^)]*\)\s*=>`), regexp.MustCompile(`^interface\s+\w+`), diff --git a/internal/filter/signature_patterns_test.go b/internal/filter/signature_patterns_test.go new file mode 100644 index 000000000..29b3007f8 --- /dev/null +++ b/internal/filter/signature_patterns_test.go @@ -0,0 +1,170 @@ +package filter + +import ( + "testing" +) + +func TestSignaturePatterns(t *testing.T) { + if len(SignaturePatterns) == 0 { + t.Fatal("SignaturePatterns should not be empty") + } + + // Test that patterns match expected code + tests := []struct { + patternIdx int + input string + shouldMatch bool + }{ + {0, "fn main()", true}, // Rust fn (index 0) + {1, "struct Point", true}, // Rust struct (index 1) + {6, "func main()", true}, // Go func (index 6) + {7, "type Point struct", true}, // Go type struct (index 7) + {9, "def hello()", true}, // Python def (index 9) + {10, "async def hello()", true}, // Python async def (index 10) + {11, "class MyClass", true}, // Python class (index 11) + {12, "function hello()", true}, // JS function (index 12) + {16, "interface User", true}, // TS interface (index 16) + {17, "type MyType =", true}, // TS type alias (index 17) + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + if tt.patternIdx >= len(SignaturePatterns) { + t.Skip("pattern index out of range") + } + result := SignaturePatterns[tt.patternIdx].MatchString(tt.input) + if result != tt.shouldMatch { + t.Errorf("pattern[%d].MatchString(%q) = %v, expected %v", tt.patternIdx, tt.input, result, tt.shouldMatch) + } + }) + } +} + +func TestImportPatterns(t *testing.T) { + if len(ImportPatterns) == 0 { + t.Fatal("ImportPatterns should not be empty") + } + + tests := []struct { + input string + shouldMatch bool + }{ + {"use std::io;", true}, + {"import os", true}, + {"from pathlib import Path", true}, + {"require('fs')", true}, + {"import (", true}, + {`import "fmt"`, true}, + {"#include ", true}, + {`#include "local.h"`, true}, + {"package main", true}, + {"not an import", false}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + matched := false + for _, pattern := range ImportPatterns { + if pattern.MatchString(tt.input) { + matched = true + break + } + } + if matched != tt.shouldMatch { + t.Errorf("ImportPatterns match for %q = %v, expected %v", tt.input, matched, tt.shouldMatch) + } + }) + } +} + +func TestBlockDelimiters(t *testing.T) { + if len(BlockDelimiters) != 3 { + t.Errorf("expected 3 block delimiters, got %d", len(BlockDelimiters)) + } + + if BlockDelimiters['{'] != '}' { + t.Error("expected '{' -> '}' mapping") + } + if BlockDelimiters['['] != ']' { + t.Error("expected '[' -> ']' mapping") + } + if BlockDelimiters['('] != ')' { + t.Error("expected '(' -> ')' mapping") + } +} + +func TestSignaturePatternsMatchVariousCode(t *testing.T) { + codeSamples := []string{ + "fn main() {", + "pub fn hello() -> String {", + "struct Point { x: i32 }", + "enum Color { Red, Green }", + "trait Display { fn fmt(&self); }", + "type MyType = String;", + "impl MyStruct { fn method(&self) {} }", + "func main() {}", + "func (r Receiver) Method() {}", + "type Point struct { X, Y int }", + "type Handler interface { Serve() }", + "def greet():", + "async def fetch():", + "class MyClass:", + "function hello() {", + "export default function() {", + "export class MyComponent {", + "const handler = async () => {", + "interface User {", + "type MyType = string;", + "public class MainClass {", + "private static void main(String[] args) {", + } + + for _, sample := range codeSamples { + t.Run(sample, func(t *testing.T) { + matched := false + for _, pattern := range SignaturePatterns { + if pattern.MatchString(sample) { + matched = true + break + } + } + if !matched { + t.Errorf("no signature pattern matched: %q", sample) + } + }) + } +} + +func TestImportPatternsMatchVariousImports(t *testing.T) { + importSamples := []string{ + "use std::io;", + "use std::fs::{File, OpenOptions};", + "import os", + "import sys, json", + "from pathlib import Path", + "from . import module", + "require('fs')", + "import { readFile } from 'fs';", + "import * as path from 'path';", + "import 'styles.css';", + "#include ", + "#include \"local.h\"", + "package main", + "package com.example.app", + } + + for _, sample := range importSamples { + t.Run(sample, func(t *testing.T) { + matched := false + for _, pattern := range ImportPatterns { + if pattern.MatchString(sample) { + matched = true + break + } + } + if !matched { + t.Errorf("no import pattern matched: %q", sample) + } + }) + } +} diff --git a/internal/filter/utils_test.go b/internal/filter/utils_test.go index 1faadb311..295e5eb40 100644 --- a/internal/filter/utils_test.go +++ b/internal/filter/utils_test.go @@ -1,35 +1,61 @@ package filter import ( + "strings" "testing" ) func TestTokenize(t *testing.T) { tests := []struct { - input string - want int // expected number of tokens + name string + input string + expected int }{ - {"hello world", 2}, - {"one, two, three", 3}, - {"func main() { return }", 3}, // splits on punctuation: func, main, return - {"", 0}, - {" ", 0}, - {"a-b-c", 3}, + {"empty string", "", 0}, + {"single word", "hello", 1}, + {"two words", "hello world", 2}, + {"with punctuation", "hello, world!", 2}, + {"with whitespace", " hello world ", 2}, + {"with newlines", "hello\nworld", 2}, + {"with tabs", "hello\tworld", 2}, + {"with mixed separators", "hello, world; foo: bar", 4}, + {"only punctuation", ".,;:!?\"", 0}, + {"only whitespace", " \t\n ", 0}, + {"code-like", "func main() { fmt.Println(\"hello\") }", 5}, } for _, tt := range tests { - got := tokenize(tt.input) - if len(got) != tt.want { - t.Errorf("tokenize(%q) = %v (%d tokens), want %d tokens", tt.input, got, len(got), tt.want) - } + t.Run(tt.name, func(t *testing.T) { + result := tokenize(tt.input) + if len(result) != tt.expected { + t.Errorf("tokenize(%q) returned %d tokens, expected %d. Got: %v", tt.input, len(result), tt.expected, result) + } + }) } } -func TestTokenize_NoEmptyStrings(t *testing.T) { - got := tokenize(" a b c ") - for i, w := range got { - if w == "" { - t.Errorf("token %d is empty string", i) +func TestTokenizeReturnsNoEmptyStrings(t *testing.T) { + input := "hello,,,world!!!foo" + result := tokenize(input) + for _, token := range result { + if token == "" { + t.Error("tokenize should not return empty strings") } } } + +func TestTokenizeHandlesUnicode(t *testing.T) { + input := "hello δΈ–η•Œ world" + result := tokenize(input) + if len(result) != 3 { + t.Errorf("expected 3 tokens for unicode input, got %d: %v", len(result), result) + } +} + +func TestTokenizeLongInput(t *testing.T) { + input := strings.Repeat("word ", 1000) + result := tokenize(input) + if len(result) != 1000 { + t.Errorf("expected 1000 tokens, got %d", len(result)) + } +} diff --git a/internal/utils/logger_test.go b/internal/utils/logger_test.go new file mode 100644 index 000000000..b98e1d2cf --- /dev/null +++ b/internal/utils/logger_test.go @@ -0,0 +1,193 @@ +package utils + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestInitLogger(t *testing.T) { + dir := t.TempDir() + logPath := filepath.Join(dir, "test.log") + + err := InitLogger(logPath, LevelInfo) + if err != nil { + t.Fatalf("InitLogger returned error: %v", err) + } + if Logger == nil { + t.Fatal("Logger should be initialized") + } + + _, err = os.Stat(logPath) + if err != nil { + t.Errorf("log file should exist: %v", err) + } + + CloseLogger() +} + +func TestInitLoggerCreatesDir(t *testing.T) { + dir := t.TempDir() + logPath := filepath.Join(dir, "subdir", "test.log") + + err := InitLogger(logPath, LevelDebug) + if err != nil { + t.Fatalf("InitLogger should create parent directory: %v", err) + } + CloseLogger() +} + +func TestInitLoggerInvalidPath(t *testing.T) { + err := InitLogger("", LevelInfo) + if err == nil { + t.Error("expected error for empty path") + } + CloseLogger() +} + +func TestInitLoggerAllLevels(t *testing.T) { + dir := t.TempDir() + + for _, level := range []LogLevel{LevelDebug, LevelInfo, LevelWarn, LevelError} { + logPath := filepath.Join(dir, string(level)+".log") + err := InitLogger(logPath, level) + if err != nil { + t.Fatalf("InitLogger failed for level %q: %v", level, err) + } + CloseLogger() + } +} + +func TestInitLoggerDefaultLevel(t *testing.T) { + dir := t.TempDir() + err := InitLogger(filepath.Join(dir, "default.log"), "invalid") + if err != nil { + t.Fatalf("InitLogger failed: %v", err) + } + if Logger == nil { + t.Fatal("Logger should be initialized") + } + CloseLogger() +} + +func TestWarn(t *testing.T) { + dir := t.TempDir() + InitLogger(filepath.Join(dir, "warn.log"), LevelWarn) + + Warn("test warning", "key", "value") + + CloseLogger() +} + +func TestInfo(t *testing.T) { + dir := t.TempDir() + InitLogger(filepath.Join(dir, "info.log"), LevelInfo) + + Info("test info", "count", 42) + + CloseLogger() +} + +func TestDebug(t *testing.T) { + dir := t.TempDir() + InitLogger(filepath.Join(dir, "debug.log"), LevelDebug) + + Debug("test debug", "detail", "something") + + CloseLogger() +} + +func TestError(t *testing.T) { + dir := t.TempDir() + InitLogger(filepath.Join(dir, "error.log"), LevelError) + + Error("test error", "err", "something failed") + + CloseLogger() +} + +func TestWarnNoLogger(t *testing.T) { + CloseLogger() + Warn("should not panic", "key", "value") +} + +func TestInfoNoLogger(t *testing.T) { + CloseLogger() + Info("should not panic", "key", "value") +} + +func TestDebugNoLogger(t *testing.T) { + CloseLogger() + Debug("should not panic", "key", "value") +} + +func TestErrorNoLogger(t *testing.T) { + CloseLogger() + Error("should not panic", "key", "value") +} + +func TestCloseLogger(t *testing.T) { + dir := t.TempDir() + InitLogger(filepath.Join(dir, "close.log"), LevelInfo) + if Logger == nil { + t.Fatal("Logger should be initialized") + } + + err := CloseLogger() + if err != nil { + t.Errorf("CloseLogger returned error: %v", err) + } + if Logger != nil { + t.Error("Logger should be nil after close") + } +} + +func TestCloseLoggerNoFile(t *testing.T) { + Logger = nil + err := CloseLogger() + if err != nil { + t.Errorf("CloseLogger should return nil when no file: %v", err) + } +} + +func TestLoggerLogsContent(t *testing.T) { + dir := t.TempDir() + logPath := filepath.Join(dir, "content.log") + InitLogger(logPath, LevelInfo) + + Info("hello from test") + + CloseLogger() + + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatal(err) + } + content := string(data) + if !strings.Contains(content, "hello from test") { + t.Errorf("log content should contain message, got: %s", content) + } +} + +func TestInitLoggerReusesPath(t *testing.T) { + dir := t.TempDir() + logPath := filepath.Join(dir, "reuse.log") + + InitLogger(logPath, LevelInfo) + Info("first message") + CloseLogger() + + InitLogger(logPath, LevelInfo) + Info("second message") + CloseLogger() + + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatal(err) + } + content := string(data) + if !strings.Contains(content, "second message") { + t.Error("log should contain second message") + } +} diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index 4be4a92fc..cd196fbda 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -122,13 +122,17 @@ func TestFormatDuration(t *testing.T) { {0, "0ms"}, {500, "500ms"}, {1000, "1.0s"}, - {60000, "1m 0s"}, - {3600000, "1h 0m"}, + {60000, "1m"}, + {61000, "1m 1s"}, + {3600000, "1h"}, + {3660000, "1h 1m"}, + {7200000, "2h"}, + {1500, "1.5s"}, } for _, tt := range tests { got := FormatDuration(tt.ms) - if got == "" { - t.Errorf("FormatDuration(%d) returned empty", tt.ms) + if got != tt.want { + t.Errorf("FormatDuration(%d) = %q, want %q", tt.ms, got, tt.want) } } } @@ -165,11 +169,15 @@ func TestGetModelFamily(t *testing.T) { {"o3-mini", "gpt"}, {"gemini-pro", "gemini"}, {"llama-3-8b", "llama"}, + {"meta-llama-3", "llama"}, {"qwen-2.5-72b", "qwen"}, {"deepseek-coder-v2", "deepseek"}, {"mistral-large", "mistral"}, + {"mixtral-8x7b", "mistral"}, {"unknown-model", "other"}, {"", ""}, + {"CLAUDE-OPUS", "claude"}, + {"GPT-4", "gpt"}, } for _, tt := range tests { got := GetModelFamily(tt.name) @@ -179,6 +187,97 @@ func TestGetModelFamily(t *testing.T) { } } +func TestGetTokSourceDir(t *testing.T) { + dir := GetTokSourceDir() + if dir == "" { + t.Error("expected non-empty directory path") + } + if dir[len(dir)-3:] != "tok" { + t.Errorf("expected path to end with 'tok', got %q", dir) + } +} + +func TestShortenPathEdgeCases(t *testing.T) { + t.Parallel() + + got := ShortenPath("", 10) + if got != "" { + t.Errorf("expected empty for empty path, got %q", got) + } + + got = ShortenPath("/short", 10) + if got != "/short" { + t.Errorf("expected unchanged path when within maxLen, got %q", got) + } + + got = ShortenPath("/very/long/path/that/exceeds/max/len", 3) + if got != "..." { + t.Errorf("expected '...' for maxLen=3, got %q", got) + } +} + +func TestFormatBytesEdgeCases(t *testing.T) { + t.Parallel() + + tests := []struct { + bytes int64 + want string + }{ + {0, "0B"}, + {1, "1B"}, + {1023, "1023B"}, + {1536, "1.5K"}, + {1048575, "1.0M"}, + {1073741823, "1.0G"}, + } + for _, tt := range tests { + got := FormatBytes(tt.bytes) + if got == "" { + t.Errorf("FormatBytes(%d) returned empty", tt.bytes) + } + } +} + +func TestFormatTokensEdgeCases(t *testing.T) { + t.Parallel() + + tests := []struct { + n int + want string + }{ + {0, "0"}, + {1, "1"}, + {999, "999"}, + {999999, "999999"}, + {999999999, "999999999"}, + } + for _, tt := range tests { + got := FormatTokens(tt.n) + if got == "" { + t.Errorf("FormatTokens(%d) returned empty", tt.n) + } + } +} + +func TestFormatTokens64EdgeCases(t *testing.T) { + t.Parallel() + + tests := []struct { + n uint64 + want string + }{ + {0, "0"}, + {999, "999"}, + {999999999999, "999999999999"}, + } + for _, tt := range tests { + got := FormatTokens64(tt.n) + if got == "" { + t.Errorf("FormatTokens64(%d) returned empty", tt.n) + } + } +} + func TestFormatTokens64(t *testing.T) { t.Parallel() tests := []struct { From 1ac07220900c13de1bbceb65760522ec2dcef2ba Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Mon, 27 Jul 2026 09:11:52 +0530 Subject: [PATCH 10/10] fix: gofmt formatting in test files --- internal/filter/noise_test.go | 6 ++--- internal/filter/signature_patterns_test.go | 26 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/filter/noise_test.go b/internal/filter/noise_test.go index 9a243748a..ada909a61 100644 --- a/internal/filter/noise_test.go +++ b/internal/filter/noise_test.go @@ -73,9 +73,9 @@ func TestCleanInlineProgress(t *testing.T) { func TestFilterNoisyOutput(t *testing.T) { tests := []struct { - name string - input string - check func(string) bool + name string + input string + check func(string) bool }{ { name: "removes ANSI codes", diff --git a/internal/filter/signature_patterns_test.go b/internal/filter/signature_patterns_test.go index 29b3007f8..4fd7b18c4 100644 --- a/internal/filter/signature_patterns_test.go +++ b/internal/filter/signature_patterns_test.go @@ -11,20 +11,20 @@ func TestSignaturePatterns(t *testing.T) { // Test that patterns match expected code tests := []struct { - patternIdx int - input string + patternIdx int + input string shouldMatch bool }{ - {0, "fn main()", true}, // Rust fn (index 0) - {1, "struct Point", true}, // Rust struct (index 1) - {6, "func main()", true}, // Go func (index 6) - {7, "type Point struct", true}, // Go type struct (index 7) - {9, "def hello()", true}, // Python def (index 9) - {10, "async def hello()", true}, // Python async def (index 10) - {11, "class MyClass", true}, // Python class (index 11) - {12, "function hello()", true}, // JS function (index 12) - {16, "interface User", true}, // TS interface (index 16) - {17, "type MyType =", true}, // TS type alias (index 17) + {0, "fn main()", true}, // Rust fn (index 0) + {1, "struct Point", true}, // Rust struct (index 1) + {6, "func main()", true}, // Go func (index 6) + {7, "type Point struct", true}, // Go type struct (index 7) + {9, "def hello()", true}, // Python def (index 9) + {10, "async def hello()", true}, // Python async def (index 10) + {11, "class MyClass", true}, // Python class (index 11) + {12, "function hello()", true}, // JS function (index 12) + {16, "interface User", true}, // TS interface (index 16) + {17, "type MyType =", true}, // TS type alias (index 17) } for _, tt := range tests { @@ -46,7 +46,7 @@ func TestImportPatterns(t *testing.T) { } tests := []struct { - input string + input string shouldMatch bool }{ {"use std::io;", true},