From 97ab601e0e431f9067857d61dcda9fc2f4c05cc9 Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Wed, 24 Jun 2026 10:43:13 -0500 Subject: [PATCH 1/4] fix(features): replace 4 hardcoded volatility stubs with live signal (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VIX level and VIX percentile now come from ^VIX history aligned to the price data date index. implied_volatility, iv_rank, and iv_percentile are derived from rolling HV20 (proxy until a historical options chain source is available — labeled with ponytail: comments). term_structure_slope and put_call_skew remain 0.0: historical options chain data is not available from yfinance and cannot be back-filled. Closes #2. Also resolves #3 (regime classifier blind to vol signal) since the neural net now receives varying volatility features instead of constants. Co-Authored-By: Claude Sonnet 4.6 --- src/features/regime_features.py | 58 +++++++++++++++++++------- tests/features/test_regime_features.py | 52 ++++++++++++++++++++++- 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/src/features/regime_features.py b/src/features/regime_features.py index 7417783..56cfca9 100644 --- a/src/features/regime_features.py +++ b/src/features/regime_features.py @@ -323,31 +323,58 @@ def calculate_atr(self, high: pd.Series, low: pd.Series, close: pd.Series, tr = pd.DataFrame({'tr1': tr1, 'tr2': tr2, 'tr3': tr3}).max(axis=1) return tr.rolling(window=period).mean() + def _rolling_rank(self, series: pd.Series, window: int = 252) -> pd.Series: + """Return each value's rank (0-1) within the preceding window.""" + def _rank(x): + lo, hi = x.min(), x.max() + return (x.iloc[-1] - lo) / (hi - lo) if hi > lo else 0.5 + return series.rolling(window, min_periods=20).apply(_rank, raw=False) + + def _get_vix_aligned(self, dates: pd.DatetimeIndex) -> pd.DataFrame: + """Fetch ^VIX history and align to the given date index.""" + try: + import yfinance as yf + # Strip tz for clean alignment; yfinance dates vary by version + start = dates[0].tz_localize(None) if hasattr(dates[0], 'tz_localize') and dates[0].tzinfo else dates[0] + end = dates[-1].tz_localize(None) if hasattr(dates[-1], 'tz_localize') and dates[-1].tzinfo else dates[-1] + vix_raw = yf.Ticker("^VIX").history(start=start, end=end + pd.Timedelta(days=5)) + vix_close = vix_raw['Close'] + vix_close.index = vix_close.index.tz_localize(None) if vix_close.index.tzinfo else vix_close.index + plain_dates = dates.tz_localize(None) if dates.tzinfo else dates + aligned = vix_close.reindex(plain_dates, method='ffill').bfill().fillna(20.0) + vix_pct = self._rolling_rank(aligned).fillna(0.5) + return pd.DataFrame({'vix': aligned.values, 'vix_pct': vix_pct.values}, index=dates) + except Exception: + n = len(dates) + return pd.DataFrame({'vix': [20.0] * n, 'vix_pct': [0.5] * n}, index=dates) + def calculate(self, symbol: str, data: Optional[pd.DataFrame] = None) -> pd.DataFrame: """Calculate volatility features for a symbol.""" - df = data if data is not None else self.get_data(symbol, days=300) + df = data if data is not None else self.get_data(symbol, days=365) df = self.handle_missing_data(df) features = pd.DataFrame(index=df.index) - # 1. Historical volatility (20-day rolling) + # 1. Historical volatility (20-day rolling, annualized) returns = df['Close'].pct_change() - features['historical_volatility_20'] = returns.rolling(20).std() * np.sqrt(252) # Annualized + hv20 = returns.rolling(20).std() * np.sqrt(252) + features['historical_volatility_20'] = hv20 - # 2-4. Implied volatility features (stubbed - would need options data) - features['implied_volatility'] = 0.2 # Stub with average IV - features['iv_rank'] = 0.5 # Stub with neutral rank - features['iv_percentile'] = 0.5 # Stub with neutral percentile + # 2-4. IV features: use HV20 as proxy; rank/percentile from rolling window + # ponytail: true IV requires historical options chain data not in yfinance + features['implied_volatility'] = hv20 + features['iv_rank'] = self._rolling_rank(hv20) + features['iv_percentile'] = features['iv_rank'] - # 5-6. VIX features (stubbed - would need VIX data) - features['vix_level'] = 20.0 # Stub with average VIX - features['vix_percentile'] = 0.5 # Stub with neutral percentile + # 5-6. VIX level and percentile from live ^VIX history + vix = self._get_vix_aligned(df.index) + features['vix_level'] = vix['vix'].values + features['vix_percentile'] = vix['vix_pct'].values - # 7. Term structure slope (stubbed - would need options data) - features['term_structure_slope'] = 0.0 # Stub with flat term structure - - # 8. Put/call skew (stubbed - would need options data) - features['put_call_skew'] = 0.0 # Stub with neutral skew + # 7-8. Options-derived — require historical chain data yfinance does not provide + # ponytail: wire up when a historical options data source is added + features['term_structure_slope'] = 0.0 + features['put_call_skew'] = 0.0 # 9-10. Bollinger Bands features bb_upper, bb_lower, bb_middle = self.calculate_bollinger_bands(df['Close']) @@ -358,7 +385,6 @@ def calculate(self, symbol: str, data: Optional[pd.DataFrame] = None) -> pd.Data atr = self.calculate_atr(df['High'], df['Low'], df['Close']) features['atr_normalized'] = atr / df['Close'] - # Handle missing data and normalize features = self.handle_missing_data(features) features = features.fillna(0) diff --git a/tests/features/test_regime_features.py b/tests/features/test_regime_features.py index 4cd28ea..a01bbea 100644 --- a/tests/features/test_regime_features.py +++ b/tests/features/test_regime_features.py @@ -17,6 +17,7 @@ PriceStructureFeatures, TrendIndicators, MomentumIndicators, + VolatilityFeatures, RegimeStateVector ) @@ -268,4 +269,53 @@ def test_regime_state_vector_no_nans(self, sample_data): state = rsv.calculate('SPY') - assert not state.isnull().any(), "State vector contains NaN values" \ No newline at end of file + assert not state.isnull().any(), "State vector contains NaN values" + + +class TestVolatilityFeatures: + """Verify live-signal volatility features replaced the hardcoded stubs.""" + + @pytest.fixture + def sample_data(self): + dates = pd.date_range('2022-01-01', periods=300, freq='D') + np.random.seed(7) + close = 100 + np.cumsum(np.random.normal(0, 1, 300)) + high = close * (1 + np.abs(np.random.normal(0, 0.01, 300))) + low = close * (1 - np.abs(np.random.normal(0, 0.01, 300))) + return pd.DataFrame({ + 'Open': close, 'High': high, 'Low': low, + 'Close': close, 'Volume': np.ones(300) * 1e6, + }, index=dates) + + def _fake_vix(self, dates): + """VIX that varies so we can assert it's not the constant 20.0.""" + vix = pd.Series(np.linspace(15, 35, len(dates)), index=dates) + return pd.DataFrame({'vix': vix.values, 'vix_pct': (vix / 35).values}, index=dates) + + def test_vix_level_varies(self, sample_data): + """vix_level must not be a constant — it now comes from live ^VIX data.""" + vf = VolatilityFeatures() + with patch.object(vf, '_get_vix_aligned', side_effect=self._fake_vix): + feats = vf.calculate('SPY', data=sample_data) + assert feats['vix_level'].nunique() > 1, "vix_level is still a constant stub" + + def test_implied_volatility_varies(self, sample_data): + """implied_volatility must not be the hardcoded 0.2 constant.""" + vf = VolatilityFeatures() + with patch.object(vf, '_get_vix_aligned', side_effect=self._fake_vix): + feats = vf.calculate('SPY', data=sample_data) + assert feats['implied_volatility'].nunique() > 1, "implied_volatility is still a constant stub" + + def test_iv_rank_varies(self, sample_data): + """iv_rank must not be the hardcoded 0.5 constant.""" + vf = VolatilityFeatures() + with patch.object(vf, '_get_vix_aligned', side_effect=self._fake_vix): + feats = vf.calculate('SPY', data=sample_data) + assert feats['iv_rank'].nunique() > 1, "iv_rank is still a constant stub" + + def test_no_nans(self, sample_data): + """Feature DataFrame must have no NaN after calculation.""" + vf = VolatilityFeatures() + with patch.object(vf, '_get_vix_aligned', side_effect=self._fake_vix): + feats = vf.calculate('SPY', data=sample_data) + assert not feats.isnull().any().any() \ No newline at end of file From b582b6a8dfdbd33a98b227b3969a3d7188a96b01 Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Wed, 24 Jun 2026 10:51:04 -0500 Subject: [PATCH 2/4] fix(api): wire ML pipeline into live recommendation path (#5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EnhancedStrategyRecommender.get_actionable_recommendations() now routes through RegimeDetector → StrategyFactory for strategy selection instead of a hardcoded list of four strategy types. - _detect_regime(): loads the most recent checkpoint in checkpoints/ and runs RegimeStateVector → RegimeDetector.predict_regime(). Falls back to the existing SMA-crossover heuristic when no checkpoint exists or the pipeline raises — preserving full backward compatibility. - _heuristic_regime(): maps SMA sentiment ('bullish'/'bearish'/'neutral') to RegimeType ints; keeps the fallback logic explicit and testable. - StrategyFactory.get_recommended_strategies(regime) now drives the strategy loop; the builder map dispatches each StrategyType to the appropriate contract-construction method. - _generate_strategy_recommendation() and _find_support_resistance() removed (dead after the refactor). - StrategyFactory imported at module level so it is patchable in tests. Co-Authored-By: Claude Sonnet 4.6 --- src/api/enhanced_strategy_recommender.py | 175 ++++++++++++++--------- tests/test_enhanced_recommender.py | 139 ++++++++++++++++++ 2 files changed, 246 insertions(+), 68 deletions(-) create mode 100644 tests/test_enhanced_recommender.py diff --git a/src/api/enhanced_strategy_recommender.py b/src/api/enhanced_strategy_recommender.py index 118e213..d96c801 100644 --- a/src/api/enhanced_strategy_recommender.py +++ b/src/api/enhanced_strategy_recommender.py @@ -5,13 +5,17 @@ strike prices, expirations, and concrete trading instructions. """ +import glob import logging +import os from datetime import datetime, timedelta from typing import Dict, List, Optional, Any, Union import numpy as np import pandas as pd import yfinance as yf +from ..strategies.factory import StrategyFactory + logger = logging.getLogger(__name__) @@ -84,10 +88,84 @@ def _get_option_price(self, option_row, price_type='mid'): else: return (bid + ask) / 2 + # Map StrategyType enum values to the contract-builder methods we have implemented. + # Types not in this map are skipped — we have no contract-construction logic for them. + _BUILDER_MAP = None # populated lazily after StrategyType is importable + + def _get_builder_map(self) -> Dict: + if self._BUILDER_MAP is None: + from ..strategies.base import StrategyType + EnhancedStrategyRecommender._BUILDER_MAP = { + StrategyType.BULL_CALL_SPREAD: self._create_bull_call_spread, + StrategyType.BEAR_PUT_SPREAD: self._create_bear_put_spread, + StrategyType.IRON_CONDOR: self._create_iron_condor, + # ponytail: covered call ≈ short call against owned shares; no separate StrategyType + StrategyType.SHORT_CALL: self._create_covered_call, + # Butterfly uses the same iron-condor builder as a reasonable approximation + StrategyType.BUTTERFLY: self._create_iron_condor, + } + return self._BUILDER_MAP + + def _find_checkpoint(self) -> Optional[str]: + """Return the most-recently-modified RegimeDetector checkpoint, or None.""" + patterns = ['checkpoints/regime_detector*.pt', 'checkpoints/regime_detector*.pth'] + matches = [f for p in patterns for f in glob.glob(p)] + return max(matches, key=os.path.getmtime) if matches else None + + def _heuristic_regime(self, hist: pd.DataFrame) -> int: + """Map SMA-crossover sentiment to a RegimeType int. Fallback when no ML checkpoint.""" + from ..data.regime_labeler import RegimeType + sentiment = self._analyze_market_sentiment(hist) + return { + 'bullish': int(RegimeType.BULL_TRENDING), + 'bearish': int(RegimeType.BEAR_TRENDING), + }.get(sentiment, int(RegimeType.SIDEWAYS_RANGING)) + + def _detect_regime(self, symbol: str, hist: pd.DataFrame) -> int: + """ + Classify market regime via the ML pipeline. + + Loads the most recent RegimeDetector checkpoint if one exists, runs the + 48-dimensional feature vector through it, and returns the predicted regime + index. Falls back to a simple SMA-crossover heuristic when no checkpoint + is present or the pipeline raises. + + Note: RegimeStateVector.calculate() fetches its own price data internally, + so hist is fetched twice in the ML path. ponytail: consolidate when + RegimeStateVector accepts pre-fetched data. + """ + checkpoint = self._find_checkpoint() + if not checkpoint: + logger.info("No RegimeDetector checkpoint found; using heuristic regime detection") + return self._heuristic_regime(hist) + + try: + import torch + from ..models.regime_detector import RegimeDetector + from ..features.regime_features import RegimeStateVector + + model = RegimeDetector() + model.load_state_dict(torch.load(checkpoint, map_location='cpu', weights_only=True)) + model.eval() + + state_vector = RegimeStateVector().calculate(symbol) + x = torch.tensor(state_vector.values, dtype=torch.float32).unsqueeze(0) + predicted, _, _ = model.predict_regime(x) + regime = int(predicted[0].item()) + logger.info(f"ML regime detection: {regime} for {symbol}") + return regime + except Exception as exc: + logger.warning(f"ML regime detection failed ({exc}); using heuristic") + return self._heuristic_regime(hist) + def get_actionable_recommendations(self, symbol: str, analysis_days: int = 5) -> List[Dict[str, Any]]: """ Get actionable options recommendations with specific contracts and dates. + Uses the ML pipeline (RegimeDetector → StrategyFactory) for regime-aware + strategy selection when a trained checkpoint is available; falls back to + a heuristic SMA-crossover regime when not. + Args: symbol: Stock symbol to analyze analysis_days: Number of days to look ahead for opportunities @@ -96,38 +174,48 @@ def get_actionable_recommendations(self, symbol: str, analysis_days: int = 5) -> List of actionable trading recommendations with specific contracts """ try: - # Get current market data stock = yf.Ticker(symbol) - - # Get current price and historical data hist = stock.history(period="60d", interval="1d") current_price = float(hist['Close'].iloc[-1]) - # Get options chain expirations = stock.options if not expirations: return self._fallback_recommendations(symbol, current_price) - recommendations = [] - - # Analyze market conditions - market_sentiment = self._analyze_market_sentiment(hist) volatility = self._calculate_implied_volatility(hist) - support_resistance = self._find_support_resistance(hist) - - # Generate specific strategy recommendations - for strategy_type in ['bullish_call_spread', 'bearish_put_spread', 'iron_condor', 'covered_call']: - rec = self._generate_strategy_recommendation( - symbol, current_price, expirations, market_sentiment, - volatility, support_resistance, strategy_type, stock - ) + + # Regime detection: try ML pipeline, fall back to heuristic + regime = self._detect_regime(symbol, hist) + + # Strategy selection via StrategyFactory + factory = StrategyFactory() + factory_recs = factory.get_recommended_strategies(regime, max_recommendations=5) + + # Choose expiration (25-50 DTE; fall back to nearest available) + target_expiration = next( + (e for e in expirations + if 25 <= (datetime.strptime(e, '%Y-%m-%d') - datetime.now()).days <= 50), + expirations[0] + ) + + try: + option_chain = stock.option_chain(target_expiration) + except Exception as e: + logger.warning(f"Could not fetch option chain: {e}") + return self._fallback_recommendations(symbol, current_price) + + builder_map = self._get_builder_map() + recommendations = [] + for factory_rec in factory_recs: + builder = builder_map.get(factory_rec.strategy_type) + if builder is None: + continue + rec = builder(symbol, current_price, option_chain, target_expiration, volatility) if rec: recommendations.append(rec) - # Sort by expected profit/risk ratio recommendations.sort(key=lambda x: x['profit_risk_ratio'], reverse=True) - - return recommendations[:3] # Return top 3 recommendations + return recommendations[:3] except Exception as e: logger.error(f"Error generating enhanced recommendations for {symbol}: {e}") @@ -154,55 +242,6 @@ def _calculate_implied_volatility(self, hist: pd.DataFrame) -> float: returns = hist['Close'].pct_change().dropna() return float(returns.std() * np.sqrt(252)) # Annualized volatility - def _find_support_resistance(self, hist: pd.DataFrame) -> Dict[str, float]: - """Identify key support and resistance levels""" - highs = hist['High'].tail(20) - lows = hist['Low'].tail(20) - - resistance = float(highs.quantile(0.9)) - support = float(lows.quantile(0.1)) - - return {'support': support, 'resistance': resistance} - - def _generate_strategy_recommendation(self, symbol: str, current_price: float, - expirations: List[str], market_sentiment: str, - volatility: float, support_resistance: Dict[str, float], - strategy_type: str, stock) -> Optional[Dict[str, Any]]: - """Generate a specific strategy recommendation with contract details""" - - # Choose expiration (30-45 days out is typically optimal) - target_expiration = None - for exp in expirations: - exp_date = datetime.strptime(exp, '%Y-%m-%d') - days_to_exp = (exp_date - datetime.now()).days - if 25 <= days_to_exp <= 50: - target_expiration = exp - break - - if not target_expiration: - target_expiration = expirations[0] if expirations else None - - if not target_expiration: - return None - - try: - option_chain = stock.option_chain(target_expiration) - except Exception as e: - logger.warning(f"Could not fetch option chain for {target_expiration}: {e}") - return None - - # Generate strategy based on market conditions - if strategy_type == 'bullish_call_spread' and market_sentiment in ['bullish', 'neutral']: - return self._create_bull_call_spread(symbol, current_price, option_chain, target_expiration, volatility) - elif strategy_type == 'bearish_put_spread' and market_sentiment in ['bearish', 'neutral']: - return self._create_bear_put_spread(symbol, current_price, option_chain, target_expiration, volatility) - elif strategy_type == 'iron_condor' and market_sentiment == 'neutral': - return self._create_iron_condor(symbol, current_price, option_chain, target_expiration, volatility) - elif strategy_type == 'covered_call' and market_sentiment in ['neutral', 'mildly_bullish']: - return self._create_covered_call(symbol, current_price, option_chain, target_expiration, volatility) - - return None - def _create_bull_call_spread(self, symbol: str, current_price: float, option_chain, expiration: str, volatility: float) -> Dict[str, Any]: """Create a bull call spread recommendation with specific contracts""" diff --git a/tests/test_enhanced_recommender.py b/tests/test_enhanced_recommender.py new file mode 100644 index 0000000..7ffda29 --- /dev/null +++ b/tests/test_enhanced_recommender.py @@ -0,0 +1,139 @@ +""" +Tests for EnhancedStrategyRecommender ML-pipeline wiring (issue #5). + +Verifies that: + - _detect_regime falls back to heuristic when no checkpoint exists + - _heuristic_regime maps SMA sentiment to the correct RegimeType ints + - get_actionable_recommendations drives strategy selection via StrategyFactory + rather than the old hardcoded strategy-type list +""" + +import pandas as pd +import numpy as np +import pytest +from unittest.mock import MagicMock, patch + +from src.api.enhanced_strategy_recommender import EnhancedStrategyRecommender +from src.data.regime_labeler import RegimeType +from src.strategies.base import StrategyType + + +def _make_hist(n=60, trend='flat'): + """Synthetic OHLCV DataFrame.""" + dates = pd.date_range('2024-01-01', periods=n, freq='D') + if trend == 'bullish': + close = np.linspace(100, 120, n) + elif trend == 'bearish': + close = np.linspace(120, 100, n) + else: + close = np.full(n, 110.0) + return pd.DataFrame({ + 'Open': close, 'High': close * 1.01, 'Low': close * 0.99, + 'Close': close, 'Volume': np.ones(n) * 1e6, + }, index=dates) + + +class TestHeuristicRegime: + def test_bullish_sentiment_maps_to_bull_trending(self): + rec = EnhancedStrategyRecommender() + with patch.object(rec, '_analyze_market_sentiment', return_value='bullish'): + regime = rec._heuristic_regime(_make_hist()) + assert regime == int(RegimeType.BULL_TRENDING) + + def test_bearish_sentiment_maps_to_bear_trending(self): + rec = EnhancedStrategyRecommender() + with patch.object(rec, '_analyze_market_sentiment', return_value='bearish'): + regime = rec._heuristic_regime(_make_hist()) + assert regime == int(RegimeType.BEAR_TRENDING) + + def test_neutral_sentiment_maps_to_sideways(self): + rec = EnhancedStrategyRecommender() + with patch.object(rec, '_analyze_market_sentiment', return_value='neutral'): + regime = rec._heuristic_regime(_make_hist()) + assert regime == int(RegimeType.SIDEWAYS_RANGING) + + +class TestDetectRegime: + def test_no_checkpoint_uses_heuristic(self): + rec = EnhancedStrategyRecommender() + hist = _make_hist() + with patch.object(rec, '_find_checkpoint', return_value=None), \ + patch.object(rec, '_heuristic_regime', return_value=int(RegimeType.BULL_TRENDING)) as mock_h: + regime = rec._detect_regime('SPY', hist) + mock_h.assert_called_once_with(hist) + assert regime == int(RegimeType.BULL_TRENDING) + + def test_ml_failure_falls_back_to_heuristic(self): + rec = EnhancedStrategyRecommender() + hist = _make_hist() + with patch.object(rec, '_find_checkpoint', return_value='/fake/checkpoint.pt'), \ + patch('torch.load', side_effect=RuntimeError("corrupt")), \ + patch.object(rec, '_heuristic_regime', return_value=int(RegimeType.BEAR_TRENDING)) as mock_h: + regime = rec._detect_regime('SPY', hist) + mock_h.assert_called_once() + assert regime == int(RegimeType.BEAR_TRENDING) + + +class TestGetActionableRecommendations: + """Verify factory drives strategy selection instead of hardcoded strings.""" + + def _mock_yf(self, hist, expirations, chain): + ticker = MagicMock() + ticker.history.return_value = hist + ticker.options = expirations + ticker.option_chain.return_value = chain + return ticker + + def _fake_chain(self, price=100.0): + strikes = np.arange(price * 0.80, price * 1.20, price * 0.05) + calls = pd.DataFrame({ + 'strike': strikes, 'bid': 2.0, 'ask': 2.5, 'lastPrice': 2.25, + 'impliedVolatility': 0.25, + }) + puts = calls.copy() + return MagicMock(calls=calls, puts=puts) + + def test_factory_is_called_with_detected_regime(self): + rec = EnhancedStrategyRecommender() + hist = _make_hist(60, 'bullish') + chain = self._fake_chain() + mock_ticker = self._mock_yf(hist, ['2026-08-15'], chain) + + with patch('yfinance.Ticker', return_value=mock_ticker), \ + patch.object(rec, '_detect_regime', return_value=int(RegimeType.BULL_TRENDING)) as mock_detect, \ + patch('src.strategies.factory.StrategyFactory') as MockFactory: + + factory_inst = MagicMock() + MockFactory.return_value = factory_inst + fake_rec = MagicMock() + fake_rec.strategy_type = StrategyType.BULL_CALL_SPREAD + factory_inst.get_recommended_strategies.return_value = [fake_rec] + + # Also patch the module-level import that EnhancedStrategyRecommender uses + with patch('src.api.enhanced_strategy_recommender.StrategyFactory', MockFactory): + recs = rec.get_actionable_recommendations('SPY') + + mock_detect.assert_called_once() + factory_inst.get_recommended_strategies.assert_called_once_with( + int(RegimeType.BULL_TRENDING), max_recommendations=5 + ) + + def test_returns_list_on_success(self): + rec = EnhancedStrategyRecommender() + hist = _make_hist(60, 'flat') + chain = self._fake_chain() + mock_ticker = self._mock_yf(hist, ['2026-08-15'], chain) + + with patch('yfinance.Ticker', return_value=mock_ticker), \ + patch.object(rec, '_detect_regime', return_value=int(RegimeType.SIDEWAYS_RANGING)), \ + patch('src.api.enhanced_strategy_recommender.StrategyFactory') as MockFactory: + + factory_inst = MagicMock() + MockFactory.return_value = factory_inst + fake_rec = MagicMock() + fake_rec.strategy_type = StrategyType.IRON_CONDOR + factory_inst.get_recommended_strategies.return_value = [fake_rec] + + recs = rec.get_actionable_recommendations('SPY') + + assert isinstance(recs, list) From 3e1d716f4d9cc0988b4eaa11a2adc81284406fb4 Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Wed, 24 Jun 2026 12:42:12 -0500 Subject: [PATCH 3/4] fix(recommender): repair triple-blocked ML inference path (#43, #44, #45, #46) Three bugs caused _detect_regime() to silently fall back to heuristic every time, even when a trained checkpoint existed: - #43: _find_checkpoint() glob patterns never matched any file the trainer produces; updated to match models/checkpoints/ and training_runs/*/checkpoints/ with correct filenames. - #44: load_state_dict() received the full checkpoint dict instead of the nested state dict; now unpacks model_state_dict (trainer format) or state_dict (inference export format) before loading. - #45: RegimeStateVector().calculate() returned 21 features by default; RegimeDetector expects 48. Fixed by passing include_extended=True. - #46: _BUILDER_MAP class-level cache stored instance-bound methods, causing wrong method bindings on a second instance. Removed class cache; _get_builder_map() now returns a fresh dict per call. Co-Authored-By: Claude Sonnet 4.6 --- src/api/enhanced_strategy_recommender.py | 46 ++++++++++++++---------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/api/enhanced_strategy_recommender.py b/src/api/enhanced_strategy_recommender.py index d96c801..3064686 100644 --- a/src/api/enhanced_strategy_recommender.py +++ b/src/api/enhanced_strategy_recommender.py @@ -88,27 +88,28 @@ def _get_option_price(self, option_row, price_type='mid'): else: return (bid + ask) / 2 - # Map StrategyType enum values to the contract-builder methods we have implemented. - # Types not in this map are skipped — we have no contract-construction logic for them. - _BUILDER_MAP = None # populated lazily after StrategyType is importable - def _get_builder_map(self) -> Dict: - if self._BUILDER_MAP is None: - from ..strategies.base import StrategyType - EnhancedStrategyRecommender._BUILDER_MAP = { - StrategyType.BULL_CALL_SPREAD: self._create_bull_call_spread, - StrategyType.BEAR_PUT_SPREAD: self._create_bear_put_spread, - StrategyType.IRON_CONDOR: self._create_iron_condor, - # ponytail: covered call ≈ short call against owned shares; no separate StrategyType - StrategyType.SHORT_CALL: self._create_covered_call, - # Butterfly uses the same iron-condor builder as a reasonable approximation - StrategyType.BUTTERFLY: self._create_iron_condor, - } - return self._BUILDER_MAP + # ponytail: no class-level cache — bound methods must be per-instance (issue #46) + from ..strategies.base import StrategyType + return { + StrategyType.BULL_CALL_SPREAD: self._create_bull_call_spread, + StrategyType.BEAR_PUT_SPREAD: self._create_bear_put_spread, + StrategyType.IRON_CONDOR: self._create_iron_condor, + StrategyType.SHORT_CALL: self._create_covered_call, + StrategyType.BUTTERFLY: self._create_iron_condor, + } def _find_checkpoint(self) -> Optional[str]: """Return the most-recently-modified RegimeDetector checkpoint, or None.""" - patterns = ['checkpoints/regime_detector*.pt', 'checkpoints/regime_detector*.pth'] + # Matches trainer output (models/checkpoints/) and training script output + # (training_runs//checkpoints/ and .../inference_model.pth) + patterns = [ + 'models/checkpoints/best_model.pth', + 'models/checkpoints/checkpoint_epoch_*.pth', + 'training_runs/*/checkpoints/best_model.pth', + 'training_runs/*/checkpoints/checkpoint_epoch_*.pth', + 'training_runs/*/inference_model.pth', + ] matches = [f for p in patterns for f in glob.glob(p)] return max(matches, key=os.path.getmtime) if matches else None @@ -144,11 +145,18 @@ def _detect_regime(self, symbol: str, hist: pd.DataFrame) -> int: from ..models.regime_detector import RegimeDetector from ..features.regime_features import RegimeStateVector + data = torch.load(checkpoint, map_location='cpu', weights_only=True) + if 'model_state_dict' in data: # trainer checkpoint format + state_dict = data['model_state_dict'] + elif 'state_dict' in data: # inference export format + state_dict = data['state_dict'] + else: + state_dict = data # raw state dict model = RegimeDetector() - model.load_state_dict(torch.load(checkpoint, map_location='cpu', weights_only=True)) + model.load_state_dict(state_dict) model.eval() - state_vector = RegimeStateVector().calculate(symbol) + state_vector = RegimeStateVector(include_extended=True).calculate(symbol) x = torch.tensor(state_vector.values, dtype=torch.float32).unsqueeze(0) predicted, _, _ = model.predict_regime(x) regime = int(predicted[0].item()) From d954d1b70f4e27ad9035f41e0329de5519ca119c Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Wed, 24 Jun 2026 13:00:27 -0500 Subject: [PATCH 4/4] fix: resolve 15 bugs across training, features, data, and recommender layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Training correctness (#47): - Remove F.softmax from RegimeDetector._forward_impl — forward() now returns raw logits so CrossEntropyLoss receives pre-softmax values (was double-softmaxing) - Remove undocumented confidence-logit contamination of regime logits - Fix calculate_uncertainty to apply softmax before entropy computation - Fix compute_loss confidence pseudo-target to use softmax of logits, not raw logits - Update stale tests that assumed softmax output from forward() Regime labeling (#48, #49): - Remove BULL_VOLATILE=2 and SIDEWAYS_LOW_VOL=3 from RegimeType — they were IntEnum aliases for HIGH_VOLATILITY and LOW_VOLATILITY, making list(RegimeType) return 6 members instead of 8 - Fix vol_high_pct threshold from 30 to 0.30 — was comparing decimal volatility against integer 30, making the HIGH_VOLATILITY branch unreachable Training loop (#50): - Initialize val_metrics=None before the epoch loop to prevent UnboundLocalError when checkpoint_frequency < validation_frequency Feature engineering (#52, #53, #54, #12): - EventFeatures: skip minmax standardization for constant columns (was producing all-NaN in 3 of 48 input dimensions) - Stochastic %K: replace inf with NaN then fill 50.0 (flat price guard) - ADX: guard zero denominator in DX calculation with np.where - OBV: fill NaN volume with 0 before accumulation loop Data layer (#55): - batch_get_prices: pass 2-year default start date; was silently returning ~20 rows due to yfinance default 1-month period Recommendation engine (#51): - position_size typed Optional[float] — kelly sizing explicitly returns None when history is absent; callers doing arithmetic were hitting TypeError Enhanced recommender (#57, #56, #58, #60, #61): - _find_checkpoint: anchor glob to Path(__file__).parents[3] so it works from any process cwd, not just repo root - _calculate_probability_above: return 0.5 for ATM at expiry (was returning 0.0) - _days_to_expiration: compare .date() not datetime — eliminates time-of-day off-by-one that misfired at 3pm on expiration day - Iron condor max_loss: use max(call_wing, put_wing) — was ignoring put wing - BUTTERFLY removed from builder map; was silently returning iron condor payload Co-Authored-By: Claude Sonnet 4.6 --- src/api/enhanced_strategy_recommender.py | 24 ++++++++++++------ src/data/providers.py | 7 +++--- src/data/regime_labeler.py | 4 +-- src/features/regime_features.py | 24 ++++++++++++------ src/models/recommendation_engine.py | 2 +- src/models/regime_detector.py | 21 ++++++++-------- src/training/trainer.py | 6 +++-- tests/models/test_regime_detector.py | 31 +++++++++++++++--------- 8 files changed, 71 insertions(+), 48 deletions(-) diff --git a/src/api/enhanced_strategy_recommender.py b/src/api/enhanced_strategy_recommender.py index 3064686..e3abea2 100644 --- a/src/api/enhanced_strategy_recommender.py +++ b/src/api/enhanced_strategy_recommender.py @@ -9,6 +9,7 @@ import logging import os from datetime import datetime, timedelta +from pathlib import Path from typing import Dict, List, Optional, Any, Union import numpy as np import pandas as pd @@ -96,13 +97,14 @@ def _get_builder_map(self) -> Dict: StrategyType.BEAR_PUT_SPREAD: self._create_bear_put_spread, StrategyType.IRON_CONDOR: self._create_iron_condor, StrategyType.SHORT_CALL: self._create_covered_call, - StrategyType.BUTTERFLY: self._create_iron_condor, + # ponytail: BUTTERFLY omitted until _create_butterfly is implemented; + # mapping it to _create_iron_condor returned a factually wrong payload } def _find_checkpoint(self) -> Optional[str]: """Return the most-recently-modified RegimeDetector checkpoint, or None.""" - # Matches trainer output (models/checkpoints/) and training script output - # (training_runs//checkpoints/ and .../inference_model.pth) + # Anchor to repo root so this works regardless of process cwd + repo_root = Path(__file__).parents[3] patterns = [ 'models/checkpoints/best_model.pth', 'models/checkpoints/checkpoint_epoch_*.pth', @@ -110,7 +112,7 @@ def _find_checkpoint(self) -> Optional[str]: 'training_runs/*/checkpoints/checkpoint_epoch_*.pth', 'training_runs/*/inference_model.pth', ] - matches = [f for p in patterns for f in glob.glob(p)] + matches = [str(m) for p in patterns for m in repo_root.glob(p)] return max(matches, key=os.path.getmtime) if matches else None def _heuristic_regime(self, hist: pd.DataFrame) -> int: @@ -429,7 +431,9 @@ def _create_iron_condor(self, symbol: str, current_price: float, net_credit = (short_put_price + short_call_price) - (long_put_price + long_call_price) max_profit = net_credit - max_loss = (short_call_strike - long_call_strike) - net_credit + call_wing = short_call_strike - long_call_strike + put_wing = short_put_strike - long_put_strike + max_loss = max(call_wing, put_wing) - net_credit # Breakevens upper_breakeven = short_call_strike + net_credit @@ -585,14 +589,18 @@ def _format_expiration(self, expiration: str) -> str: def _days_to_expiration(self, expiration: str) -> int: """Calculate days until expiration""" - exp_date = datetime.strptime(expiration, '%Y-%m-%d') - return (exp_date - datetime.now()).days + exp_date = datetime.strptime(expiration, '%Y-%m-%d').date() + return (exp_date - datetime.now().date()).days def _calculate_probability_above(self, current_price: float, target_price: float, volatility: float, days: int) -> float: """Calculate probability stock will be above target using Black-Scholes""" if days <= 0: - return 1.0 if current_price > target_price else 0.0 + if current_price > target_price: + return 1.0 + if current_price == target_price: + return 0.5 # ATM at expiry: equal probability of finishing above or below + return 0.0 t = days / 365.0 d1 = (np.log(current_price / target_price) + (self.risk_free_rate + 0.5 * volatility**2) * t) / (volatility * np.sqrt(t)) diff --git a/src/data/providers.py b/src/data/providers.py index 099cd61..ba32b82 100644 --- a/src/data/providers.py +++ b/src/data/providers.py @@ -9,7 +9,7 @@ import threading import time from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from typing import Dict, List, Optional import yfinance as yf @@ -129,12 +129,13 @@ def get_current_price(self, symbol: str) -> float: raise DataNotAvailable(f"No current price available for {symbol}") return float(history["Close"].iloc[-1]) - def batch_get_prices(self, symbols: List[str]) -> Dict[str, PriceData]: + def batch_get_prices(self, symbols: List[str], days: int = 730) -> Dict[str, PriceData]: """Fetch price history for several symbols, skipping failures.""" + start = datetime.now() - timedelta(days=days) results: Dict[str, PriceData] = {} for symbol in symbols: try: - results[symbol] = self.get_price_history(symbol) + results[symbol] = self.get_price_history(symbol, start=start) except DataProviderError: continue return results diff --git a/src/data/regime_labeler.py b/src/data/regime_labeler.py index 3ff055a..fad51a7 100644 --- a/src/data/regime_labeler.py +++ b/src/data/regime_labeler.py @@ -28,8 +28,6 @@ class RegimeType(IntEnum): RECOVERY = 5 # Post-decline bounce patterns DISTRIBUTION = 6 # Pre-decline weakening CRISIS = 7 # Extreme stress conditions - BULL_VOLATILE = 2 # Alias for high volatility bull environment - SIDEWAYS_LOW_VOL = 3 # Alias for low volatility sideways environment class RegimeMetrics: @@ -100,7 +98,7 @@ def __init__(self, stability_window: int = 252): 'rsi_low': 45, 'vix_high': 25, 'vix_low': 15, - 'vol_high_pct': 30, # 30% annualized volatility + 'vol_high_pct': 0.30, # 30% annualized volatility (decimal, matches volatility units) 'drawdown_crisis': -0.15, # 15% drawdown for crisis 'recovery_bounce_min': 0.05, # 5% bounce minimum for recovery 'distribution_weakness': -0.02, # 2% weakness after strength diff --git a/src/features/regime_features.py b/src/features/regime_features.py index 56cfca9..f5143fc 100644 --- a/src/features/regime_features.py +++ b/src/features/regime_features.py @@ -124,9 +124,10 @@ def calculate_adx(self, high: pd.Series, low: pd.Series, close: pd.Series, di_plus = 100 * (dm_plus_smooth / atr) di_minus = 100 * (dm_minus_smooth / atr) - # ADX calculation - dx = 100 * np.abs(di_plus - di_minus) / (di_plus + di_minus) - adx = dx.ewm(alpha=alpha, adjust=False).mean() + # ADX calculation — guard zero denominator (occurs on first `period` bars) + denom = di_plus + di_minus + dx_values = np.where(denom > 0, 100 * np.abs(di_plus - di_minus) / denom, 0.0) + adx = pd.Series(dx_values, index=di_plus.index).ewm(alpha=alpha, adjust=False).mean() return adx, di_plus, di_minus @@ -237,6 +238,8 @@ def calculate_stochastic(self, high: pd.Series, low: pd.Series, close: pd.Series highest_high = high.rolling(window=k_period).max() k_percent = 100 * (close - lowest_low) / (highest_high - lowest_low) + # flat price produces 0/0 → NaN, or non-zero/0 → inf; replace both with 50 (neutral) + k_percent = k_percent.replace([np.inf, -np.inf], np.nan).fillna(50.0) d_percent = k_percent.rolling(window=d_period).mean() return k_percent, d_percent @@ -400,14 +403,16 @@ class VolumeFeatures(FeatureEngineering): def calculate_obv(self, close: pd.Series, volume: pd.Series) -> pd.Series: """Calculate On Balance Volume.""" + # Fill NaN volume (common for the most recent yfinance bar) with 0 before accumulating + vol = volume.fillna(0) obv = pd.Series(index=close.index, dtype=float) - obv.iloc[0] = volume.iloc[0] + obv.iloc[0] = vol.iloc[0] for i in range(1, len(close)): if close.iloc[i] > close.iloc[i-1]: - obv.iloc[i] = obv.iloc[i-1] + volume.iloc[i] + obv.iloc[i] = obv.iloc[i-1] + vol.iloc[i] elif close.iloc[i] < close.iloc[i-1]: - obv.iloc[i] = obv.iloc[i-1] - volume.iloc[i] + obv.iloc[i] = obv.iloc[i-1] - vol.iloc[i] else: obv.iloc[i] = obv.iloc[i-1] @@ -579,9 +584,12 @@ def calculate(self, symbol: str, data: Optional[pd.DataFrame] = None) -> pd.Data # 3. Days to options expiration (stubbed - would need options calendar) features['days_to_opex'] = 10.0 # Assume average to monthly expiration - # Normalize to [-1, 1] range + # Normalize to [-1, 1] range — skip constant columns (minmax on constant = 0/0) for col in features.columns: - features[col] = self.standardize(features[col], method='minmax') + if features[col].nunique() > 1: + features[col] = self.standardize(features[col], method='minmax') + else: + features[col] = 0.0 # constant stub: map to neutral midpoint self.validate(features) return features diff --git a/src/models/recommendation_engine.py b/src/models/recommendation_engine.py index 61378b4..06caa4e 100644 --- a/src/models/recommendation_engine.py +++ b/src/models/recommendation_engine.py @@ -28,7 +28,7 @@ class StrategyRecommendation: strategy_type: StrategyType score: float # 0-100 normalized score confidence: float # 0-1 confidence level - position_size: float # Recommended position size (fraction of capital) + position_size: Optional[float] # Recommended position size (fraction of capital); None when kelly sizing unavailable expected_return: float # Expected return max_risk: float # Maximum risk (drawdown) regime: RegimeType # Current market regime diff --git a/src/models/regime_detector.py b/src/models/regime_detector.py index c3b8797..3713cdf 100644 --- a/src/models/regime_detector.py +++ b/src/models/regime_detector.py @@ -133,17 +133,13 @@ def _forward_impl(self, x: torch.Tensor) -> torch.Tensor: x = F.relu(x) x = self.dropout(x) - # Output layers + # Output layers — return raw logits so CrossEntropyLoss receives pre-softmax values regime_logits = self.regime_output(x) confidence_logits = self.confidence_output(x) - regime_logits = regime_logits + 0.01 * confidence_logits.expand(-1, self.num_regimes) - - # Apply activations - regime_probs = F.softmax(regime_logits, dim=1) # Regime probabilities sum to 1 - confidence_score = torch.sigmoid(confidence_logits) # Confidence in [0, 1] + confidence_score = torch.sigmoid(confidence_logits) - # Combine outputs - output = torch.cat([regime_probs, confidence_score], dim=1) + # Combine: raw logits + confidence score + output = torch.cat([regime_logits, confidence_score], dim=1) return output @@ -162,9 +158,12 @@ def predict_regime(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, t """ with torch.no_grad(): output = self.forward(x) - regime_probs = output[:, :self.num_regimes] + regime_logits = output[:, :self.num_regimes] confidence = output[:, self.num_regimes:] + # Softmax here only — not in forward() — so CrossEntropyLoss gets raw logits + regime_probs = F.softmax(regime_logits, dim=1) + # Get predicted regime (argmax) predicted_regimes = torch.argmax(regime_probs, dim=1) @@ -184,10 +183,10 @@ def calculate_uncertainty(self, x: torch.Tensor) -> torch.Tensor: """ with torch.no_grad(): output = self.forward(x) - regime_probs = output[:, :self.num_regimes] + # forward() returns raw logits; apply softmax here to get probabilities for entropy + regime_probs = F.softmax(output[:, :self.num_regimes], dim=1) # Calculate entropy: -sum(p * log(p)) - # Add small epsilon to avoid log(0) epsilon = 1e-8 entropy = -torch.sum(regime_probs * torch.log(regime_probs + epsilon), dim=1, keepdim=True) diff --git a/src/training/trainer.py b/src/training/trainer.py index 211ff27..26aae51 100644 --- a/src/training/trainer.py +++ b/src/training/trainer.py @@ -132,8 +132,9 @@ def compute_loss(self, outputs: torch.Tensor, targets: torch.Tensor, # Confidence loss - if no targets provided, use max probability as pseudo-target if confidence_targets is None: - # Use max probability as confidence target (unsupervised confidence learning) - confidence_targets = torch.max(regime_probs, dim=1)[0].unsqueeze(1).detach() + # regime_probs are raw logits; apply softmax to get real probabilities for the target + regime_soft = torch.softmax(regime_probs, dim=1).detach() + confidence_targets = torch.max(regime_soft, dim=1)[0].unsqueeze(1) confidence_loss = self.confidence_loss(predicted_confidence, confidence_targets) @@ -344,6 +345,7 @@ def fit(self, train_loader: DataLoader, val_loader: DataLoader, epochs = epochs or self.config.max_epochs logger.info(f"Starting training for {epochs} epochs") + val_metrics = None # ensure bound even on non-validation checkpoint epochs for epoch in range(epochs): self.current_epoch = epoch start_time = time.time() diff --git a/tests/models/test_regime_detector.py b/tests/models/test_regime_detector.py index b145344..10cb241 100644 --- a/tests/models/test_regime_detector.py +++ b/tests/models/test_regime_detector.py @@ -118,15 +118,16 @@ def test_output_value_ranges(self, model, batch_input): model.eval() output = model(batch_input) - # Split regime probabilities and confidence - regime_probs = output[:, :8] + # Split regime logits and confidence + regime_logits = output[:, :8] confidence = output[:, 8:] - # Test regime probabilities (should be softmax output) - assert torch.all(regime_probs >= 0), "Regime probabilities should be non-negative" - assert torch.all(regime_probs <= 1), "Regime probabilities should be <= 1" + # Regime logits are raw (pre-softmax) — no [0,1] constraint; check finite + assert not torch.any(torch.isnan(regime_logits)), "Regime logits should not be NaN" + assert not torch.any(torch.isinf(regime_logits)), "Regime logits should not be inf" - # Test that probabilities sum to 1 (within tolerance) + # Softmax of logits should sum to 1 + regime_probs = torch.softmax(regime_logits, dim=1) prob_sums = torch.sum(regime_probs, dim=1) torch.testing.assert_close(prob_sums, torch.ones_like(prob_sums), atol=1e-6, rtol=1e-6) @@ -341,8 +342,9 @@ def test_custom_architectures(self, input_dim, hidden_dims, num_regimes): expected_output_dim = num_regimes + 1 # regimes + confidence assert output.shape == (2, expected_output_dim) - # Test regime probabilities sum to 1 - regime_probs = output[:, :num_regimes] + # Regime output is raw logits; softmax of logits should sum to 1 + regime_logits = output[:, :num_regimes] + regime_probs = torch.softmax(regime_logits, dim=1) prob_sums = torch.sum(regime_probs, dim=1) torch.testing.assert_close(prob_sums, torch.ones_like(prob_sums), atol=1e-6, rtol=1e-6) @@ -413,10 +415,15 @@ def test_training_loop_simulation(self): loss.backward() optimizer.step() - # Verify gradients were computed - for param in model.parameters(): - if param.grad is not None: - assert not torch.all(param.grad == 0), "Some gradients should be non-zero" + # Verify gradients were computed for regime-path parameters. + # The confidence_output head gets zero gradients here because only + # the regime loss is used in this standalone test — that's expected. + regime_params_with_grad = [ + p for name, p in model.named_parameters() + if p.grad is not None and 'confidence_output' not in name + ] + assert any(not torch.all(p.grad == 0) for p in regime_params_with_grad), \ + "Regime-path parameters should have non-zero gradients" if __name__ == "__main__": pytest.main([__file__, "-v"]) \ No newline at end of file