⚠️ Sync upstream/integration (4abf058) -> release/5 2026-07-08 (deployment changes present)

.github/workflows/test_optscale_deploy.yaml

@@ -0,0 +1,26 @@
+name: Validate OptScale deployment manifests
+run-name: Validate OptScale deployment manifests - started by ${{ github.actor }}
+permissions: read-all
+on:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+    - 'optscale-deploy/**'
+    - '.github/workflows/test_optscale_deploy.yaml'
+  workflow_dispatch:
+
+
+jobs:
+  validate_deployment_manifests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - name: Install test dependencies
+        run: python -m pip install pytest PyYAML
+      - name: Validate compose and k8s parity
+        run: python -m pytest optscale-deploy/tests

.gitignore

@@ -27,5 +27,8 @@ jira_ui/*/.env
 jira_ui/*/node_modules
 jira_ui/*/build/
 
+# Compose env
+optscale-deploy/compose/.env
+
 # Vagrant
 optscale-deploy/.vagrant/

README.md

@@ -130,6 +130,25 @@ It supports multi-cloud environments and integrates with popular data platforms,
 
 ## Getting started
 
+### Docker Compose quickstart (try mode)
+
+The fastest way to try OptScale locally — no Kubernetes required. Requires Docker Engine 24+ and Docker Compose v2.20+ with at least 8 CPU cores and 16 GB RAM.
+
+```bash
+git clone https://github.com/hystax/optscale.git
+cd optscale/optscale-deploy/compose
+cp .env.example .env
+docker compose up -d
+```
+
+Once the `configurator` container exits successfully (1-2 minutes), open **http://localhost** in your browser.
+
+See [optscale-deploy/compose/README.md](optscale-deploy/compose/README.md) for full details, configuration options, and troubleshooting.
+
+> **Note:** The Docker Compose deployment is for evaluation and development only. For production, use the Kubernetes installation below.
+
+### Production installation (Kubernetes)
+
 The minimum hardware requirements for OptScale cluster: CPU: 8+ cores, RAM: 16Gb, SSD: 150+ Gb.
 
 NVMe SSD is recommended.  

diworker/diworker/importers/aws.py

@@ -11,9 +11,7 @@
 from datetime import datetime, timedelta, timezone
 from functools import cached_property
 
-from diworker.diworker.importers.base import (
-    CSVBaseReportImporter, CSV_REWRITE_DAYS
-)
+from diworker.diworker.importers.base import CSVBaseReportImporter
 import tools.optscale_time as opttime
 import pyarrow.parquet as pq
 
@@ -219,7 +217,7 @@ def min_date_import_threshold(self) -> datetime:
         if self._is_first_import_in_month(last_import_dt):
             # import full previous month on the first import in month
             return last_import_dt.replace(day=1)
-        return last_import_dt - timedelta(days=CSV_REWRITE_DAYS)
+        return last_import_dt - timedelta(days=self.csv_rewrite_days)
 
     def get_raw_upsert_filters(self, expense):
         filters = super().get_raw_upsert_filters(expense)

diworker/diworker/importers/azure.py

@@ -6,9 +6,11 @@
 import tools.optscale_time as opttime
 from diworker.diworker.utils import retry_backoff
 from tools.cloud_adapter.clouds.azure import (
-    AzureConsumptionException, ExpenseImportScheme,
-    AzureErrorResponseException, AzureAuthenticationError,
-    AzureResourceNotFoundError)
+    AzureConsumptionException,
+    ExpenseImportScheme,
+    AzureAuthenticationError,
+    AzureResourceNotFoundError,
+)
 
 from diworker.diworker.importers.base import BaseReportImporter
 
@@ -449,13 +451,10 @@ def load_raw_data(self):
             LOG.info('Downloading PayAsYouGo prices')
             try:
                 prices = self.cloud_adapter.get_public_prices()
-            except AzureErrorResponseException as exc:
-                code = getattr(exc.error, 'additional_properties', {}).get(
-                    'error', {}).get('code')
+            except AzureConsumptionException as exc:
+                code = getattr(exc.error, 'code', None)
                 if code == 'SubscriptionNotFound':
-                    msg = exc.error.additional_properties['error'].get(
-                        'message')
-                    raise AzureResourceNotFoundError(msg)
+                    raise AzureResourceNotFoundError(getattr(exc.error, 'message', str(exc)))
                 else:
                     raise exc
             LOG.info('Fetched %s price entries', len(prices))
@@ -554,14 +553,11 @@ def _load_raw_usage_data(self, prices):
                     if len(chunk) == CHUNK_SIZE:
                         self.update_raw_records(chunk)
                         chunk = []
-            except AzureErrorResponseException as ex:
-                code = getattr(ex.error, 'additional_properties', {}).get(
-                    'error', {}).get('code')
+            except AzureConsumptionException as exc:
+                code = getattr(exc.error, 'code', None)
                 if code == 'SubscriptionNotFound':
-                    msg = ex.error.additional_properties['error'].get(
-                        'message')
-                    raise AzureResourceNotFoundError(msg)
-                error_message = str(ex)
+                    raise AzureResourceNotFoundError(getattr(exc.error, 'message', str(exc)))
+                error_message = str(exc)
                 if 'Unknown error' in error_message:
                     LOG.error('No ready reports yet in cloud for %s. Will '
                               'skip the remaining report import days and try '

diworker/diworker/importers/base.py

@@ -30,7 +30,6 @@
 _THROTTLE_SEMAPHORES: dict[str, threading.Semaphore] = {}
 _THROTTLE_LOCK = threading.Lock()
 
-CSV_REWRITE_DAYS = 5
 GZIP_ENDING = '.gz'
 REPORTS_PATH_PREFIX = 'reports'
 
@@ -47,9 +46,10 @@ class BaseReportImporter:
     def __init__(self, cloud_account_id, rest_cl, config_cl, mongo_raw,
                  mongo_resources, clickhouse_cl, import_file=None,
                  recalculate=False, detect_period_start=True,
-                 max_tenant_concurrent=1):
+                 max_tenant_concurrent=1, csv_rewrite_days=5):
         self.cloud_acc_id = cloud_account_id
         self.max_tenant_concurrent = max_tenant_concurrent
+        self.csv_rewrite_days = csv_rewrite_days
         self.rest_cl = rest_cl
         self.config_cl = config_cl
         self.mongo_raw = mongo_raw
@@ -650,7 +650,7 @@ def min_date_import_threshold(self) -> datetime:
             self.cloud_acc.get('last_import_modified_at', 0), tz=timezone.utc)
         return last_import_dt.replace(
             hour=0, minute=0, second=0, microsecond=0
-        ) - timedelta(days=CSV_REWRITE_DAYS)
+        ) - timedelta(days=self.csv_rewrite_days)
 
     def detect_period_start(self):
         pass

diworker/diworker/main.py

@@ -41,6 +41,8 @@
 HEARTBEAT_INTERVAL = 300
 DEFAULT_MAX_WORKERS = 4
 DEFAULT_MAX_TENANT_WORKERS = 1
+DEFAULT_CSV_REWRITE_DAYS = 10
+MIGRATIONS_READY_FILE = '/tmp/diworker-migrations-ready'
 
 
 def _is_rate_limit_exc(exc):
@@ -178,7 +180,10 @@ def report_import(self, task, config_cl, rest_cl, mongo_cl, clickhouse_cl):
             'import_file': import_dict.get('import_file'),
             'recalculate': is_recalculation,
             'max_tenant_concurrent': int(self.diworker_settings.get(
-                'max_tenant_import_workers', DEFAULT_MAX_TENANT_WORKERS))}
+                'max_tenant_import_workers', DEFAULT_MAX_TENANT_WORKERS)),
+            'csv_rewrite_days': int(self.diworker_settings.get(
+                'csv_rewrite_days') or DEFAULT_CSV_REWRITE_DAYS)
+        }
         importer = None
         ca = None
         previous_attempt_ts = 0
@@ -295,6 +300,8 @@ def _process_task(self, body, message):
     # starting at the same time on cluster
     with EtcdLock(config_cl, 'diworker_migrations'):
         migrator.migrate()
+    with open(MIGRATIONS_READY_FILE, 'w') as ready_file:
+        ready_file.write('ready\n')
     LOG.info("starting worker")
     conn_str = 'amqp://{user}:{pass}@{host}:{port}'.format(
         **config_cl.read_branch('/rabbit'))

diworker/diworker/migrations/202112081330000_move_expenses_clickhouse.py

@@ -51,10 +51,10 @@ def upgrade(self):
             _, resp = rest_client.cloud_account_list(organization_id)
             cloud_account_ids = list(map(
                 lambda x: x['id'], resp['cloud_accounts']))
-            expenses = expenses_collection.find(
-                {'cloud_account_id': {'$in': cloud_account_ids}}
-            )
-            total_expenses = expenses.count_documents()
+            expenses_filter = {'cloud_account_id': {'$in': cloud_account_ids}}
+            total_expenses = expenses_collection.count_documents(
+                expenses_filter)
+            expenses = expenses_collection.find(expenses_filter)
             bulk = []
             total_migrated = 0
             LOG.info('Migrating expenses for org %s (%s/%s)...' % (