From ceb101c9fa3ae87e41b7266854cc0102cd540c6a Mon Sep 17 00:00:00 2001 From: ritik574-coder Date: Fri, 26 Jun 2026 21:50:12 +0530 Subject: [PATCH] feat: implement SCD Type 2 dim_inventory and daily fact tables from bronze intermediate layer --- issues.txt | 286 ------------------ mart_stracture.t | 60 ++++ models/marts/core/dimensions/_dim_models.yml | 16 - .../marts/core/dimensions/dim_customers.sql | 2 +- models/marts/core/dimensions/dim_date.sql | 30 -- .../marts/core/dimensions/dim_employees.sql | 2 +- .../marts/core/dimensions/dim_inventory.sql | 83 +++++ models/marts/core/dimensions/dim_products.sql | 3 +- notes.txt | 266 ---------------- profiles.example.yml | 29 -- .../inventory_snapshot.sql | 0 11 files changed, 146 insertions(+), 631 deletions(-) delete mode 100644 issues.txt create mode 100644 mart_stracture.t delete mode 100644 models/marts/core/dimensions/dim_date.sql delete mode 100644 notes.txt delete mode 100644 profiles.example.yml rename models/intermediate/_int__retail_source.yml => snapshots/inventory_snapshot.sql (100%) diff --git a/issues.txt b/issues.txt deleted file mode 100644 index 0df8968..0000000 --- a/issues.txt +++ /dev/null @@ -1,286 +0,0 @@ -================================================================================ -DBT ANALYTICS ENGINEERING — CI/CD PIPELINE FAILURE ANALYSIS -================================================================================ -Generated: 2026-06-20 -Repository: Ritik574-coder/dbt-analytics-engineering -Branch with failures: dbt_branch (PR to main) - -SUMMARY -------- -All three GitHub Actions workflows are currently failing: - - Workflow Latest failed run Failing job/step - ---------------- ------------------ ------------------------------------------ - dbt CI 27859117241 dbt integration → Load Bronze Data - dbt Docs 27859117248 Generate documentation → Generate dbt docs - dbt CD 27839146670 Deploy (prod) → dbt debug - -The "Lint & parse" CI job passes. Failures are in database connectivity / setup. - - -================================================================================ -ISSUE 1 — DATABASE NAME MISMATCH (CI + DOCS) — PRIMARY BLOCKER -================================================================================ - -SEVERITY: Critical (blocks CI and Docs right now) - -EXACT REASON: - GitHub Actions workflows set: - DBT_SQLSERVER_DATABASE: retail_analytics - - But the database initialization script creates a different database: - scripts/ci-init-db.sql → CREATE DATABASE RetailDB - - When python_load.py and dbt connect, they use the env var - DBT_SQLSERVER_DATABASE=retail_analytics. That database does NOT exist - on the CI SQL Server container. SQL Server returns error 4060: - - Cannot open database "retail_analytics" requested by the login. - The login failed. (4060) - - Evidence from latest CI run (27859117241), step "Load Bronze Data": - - ci-init-db.sql succeeded: "Changed database context to 'RetailDB'." - - python_load.py failed immediately trying to connect to retail_analytics - - Same error in dbt Docs run (27859117248), step "Generate dbt docs": - dbt docs generate cannot connect because retail_analytics does not exist. - -AFFECTED FILES: - - .github/workflows/dbt-ci.yml (remote/pushed version still has retail_analytics) - - .github/workflows/dbt-docs.yml (still has retail_analytics) - - scripts/ci-init-db.sql (creates RetailDB) - - scripts/loading/python_load.py (defaults to RetailDB, but CI env overrides it) - - models/staging/_retail__sources.yml (correctly references database: RetailDB) - - profiles.example.yml / .env.example (default to retail_analytics — confusing) - -NOTE: - You have a LOCAL uncommitted fix in dbt-ci.yml changing retail_analytics → RetailDB, - but it is NOT pushed to GitHub yet, and dbt-docs.yml was NOT updated. - -HOW TO FIX: - 1. Use ONE database name everywhere. Recommended: RetailDB (matches init script, - DDL, sources, and bronze load scripts). - - 2. Update workflow env vars: - .github/workflows/dbt-ci.yml: - DBT_SQLSERVER_DATABASE: RetailDB - - .github/workflows/dbt-docs.yml: - DBT_SQLSERVER_DATABASE: RetailDB - - 3. Commit and push both workflow changes to dbt_branch. - - 4. Optionally align defaults for local dev consistency: - .env.example → DBT_SQLSERVER_DATABASE=RetailDB - profiles.example.yml → default 'RetailDB' instead of 'retail_analytics' - - 5. Re-run CI after push. Bronze load + dbt run should proceed past current failure. - - -================================================================================ -ISSUE 2 — BRONZE TABLES NOT LOADED (CI — secondary failure after Issue 1) -================================================================================ - -SEVERITY: High (appeared in earlier CI runs once DB connection partially worked) - -EXACT REASON: - In CI run 27840802070, dbt run reached execution but all 8 staging models failed: - - Invalid object name 'RetailDB.bronze.customers'. (208) - Invalid object name 'RetailDB.bronze.employees'. (208) - ... (same for all bronze source tables) - - This means dbt connected, but bronze tables were missing or empty. - Root cause chain: - a) python_load.py never loaded data (database name mismatch), OR - b) ddl_bronze.sql was not run before dbt run, OR - c) Load step was missing from an older workflow version - - Current workflow (dbt-ci.yml) does have the correct sequence: - 1. ci-init-db.sql → create RetailDB + schemas - 2. ddl_bronze.sql → create bronze tables - 3. python_load.py → load CSV data into bronze.* - 4. dbt run → build staging/intermediate models - - Fixing Issue 1 should allow steps 1–3 to complete and prevent this error. - -HOW TO FIX: - 1. Fix Issue 1 first (RetailDB name alignment). - 2. Verify CI log shows after python_load.py: - "Loaded N rows into bronze.customers" (for each table) - 3. If tables still missing, confirm CSV files exist in repo: - scripts/raw_dataset/csv_data_file/*.csv (all 8 files present ✓) - 4. Confirm ddl_bronze.sql runs without error before python_load.py. - - -================================================================================ -ISSUE 3 — CD PIPELINE CANNOT REACH PRODUCTION SQL SERVER -================================================================================ - -SEVERITY: Critical (blocks all CD deployments) - -EXACT REASON: - CD workflow (dbt-cd.yml) fails at "dbt debug" with: - - Login timeout expired (HYT00) - Connection test: [ERROR] - - This means the GitHub Actions runner waited ~60+ seconds and could NOT - establish a TCP connection to the SQL Server defined in GitHub Secrets. - - CD does NOT use a local SQL Server container. It connects to a real server - via repository/environment secrets: - DBT_SQLSERVER_HOST - DBT_SQLSERVER_PORT - DBT_SQLSERVER_DATABASE - DBT_SQLSERVER_SCHEMA - DBT_SQLSERVER_USER - DBT_SQLSERVER_PASSWORD - - Login timeout (not "login failed") typically means: - - Host is unreachable (wrong IP/hostname, localhost, private network) - - Firewall blocks GitHub Actions runner IP ranges - - SQL Server not listening on the configured port - - Azure SQL / cloud DB requires firewall rule for GitHub Actions - - Secrets not configured in the "production" GitHub Environment - - All recent CD runs fail the same way (runs: 27839146670, 27750986715, - 27735843782, 27623198237). - -HOW TO FIX: - 1. In GitHub → Settings → Environments → production → verify ALL 6 - DBT_SQLSERVER_* secrets are set (not empty, not localhost). - - 2. DBT_SQLSERVER_HOST must be a publicly reachable hostname/IP - (e.g. your Azure SQL server: myserver.database.windows.net). - localhost / 127.0.0.1 will ALWAYS fail in GitHub Actions. - - 3. Open firewall for GitHub Actions: - - Azure SQL: enable "Allow Azure services" OR add GitHub IP ranges - - On-prem SQL Server: allow inbound TCP 1433 from GitHub Actions IPs - (or use a self-hosted runner inside your network) - - 4. Verify SQL Server accepts SQL authentication (not Windows-only auth). - - 5. Test connectivity manually from outside your network: - sqlcmd -S , -U -P -C -Q "SELECT 1" - - 6. Ensure ODBC Driver 18 settings match your server (trust_cert: true is - already set in profiles.example.yml). - - 7. Re-run CD via Actions → dbt CD → Run workflow. - - -================================================================================ -ISSUE 4 — INCONSISTENT DATABASE NAMING ACROSS PROJECT -================================================================================ - -SEVERITY: Medium (causes confusion and repeat failures) - -EXACT REASON: - The project uses two different database names: - - RetailDB → ci-init-db.sql, ddl_bronze.sql, sources.yml, proc scripts - - retail_analytics → workflows, .env.example, profiles.example.yml - - dbt profile connects to one database (from env var), while source() macros - hardcode database: RetailDB in _retail__sources.yml. This can work in SQL - Server (cross-database queries) ONLY if both databases exist. In CI, only - RetailDB is created — retail_analytics never exists. - -HOW TO FIX: - Standardize on RetailDB everywhere, OR rename ci-init-db.sql to create - retail_analytics instead. Pick one name and update all references. - - -================================================================================ -ISSUE 5 — DOCS WORKFLOW MISSING BRONZE SETUP (minor / future) -================================================================================ - -SEVERITY: Low (docs generate may work without bronze data, but DB must exist) - -EXACT REASON: - dbt-docs.yml only runs ci-init-db.sql, then dbt docs generate. - It does NOT run ddl_bronze.sql or python_load.py. - - dbt docs generate needs a valid DB connection but does not require bronze - data. Fixing the database name (Issue 1) should be enough for Docs to pass. - -HOW TO FIX: - 1. Fix DBT_SQLSERVER_DATABASE in dbt-docs.yml → RetailDB. - 2. No bronze load needed unless you add compile/run steps that query sources. - - -================================================================================ -ISSUE 6 — WORKFLOW HYGIENE (non-blocking but worth fixing) -================================================================================ - -SEVERITY: Low - -EXACT REASON: - a) dbt-ci.yml has two steps both named "Create Bronze Tables" (lines 133–139). - Makes debugging CI logs harder. - - b) SQL Server service health check uses: - /opt/mssql-tools18/bin/sqlcmd - inside the mcr.microsoft.com/mssql/server:2022-latest container. - The container image may ship sqlcmd at a different path. CI currently - works (sqlcmd on the runner host is used for steps), but the service - health check path may be unreliable. - - c) dbt_project.yml has unused config paths (marts, seeds, snapshots) — - warnings only, not causing failures. - -HOW TO FIX: - a) Rename steps: "Initialize CI database" and "Create bronze DDL tables". - b) Change health-cmd to use runner-side check or correct container path: - /opt/mssql-tools18/bin/sqlcmd OR /opt/mssql-tools/bin/sqlcmd - c) Add marts/seeds/snapshots content or remove unused config blocks. - - -================================================================================ -RECOMMENDED FIX ORDER -================================================================================ - - Step Action Expected result - ---- -------------------------------------------------- ------------------------- - 1 Push RetailDB fix to dbt-ci.yml AND dbt-docs.yml CI + Docs pass DB connect - 2 Verify bronze load + dbt run/test in CI logs Full CI green - 3 Fix CD secrets + firewall for production SQL Server CD deploys succeed - 4 Align .env.example / profiles.example.yml Local dev matches CI - - -================================================================================ -WHAT IS ALREADY WORKING -================================================================================ - - ✓ Lint & parse job (YAML validation, dbt parse, sqlfluff) — passes - ✓ Python dependencies install (dbt-core, pandas, sqlalchemy, pyodbc) - ✓ ODBC Driver 18 install on Ubuntu runner - ✓ SQL Server container starts and accepts sqlcmd from runner - ✓ ci-init-db.sql creates RetailDB + bronze + dbt_ci schemas - ✓ ddl_bronze.sql creates bronze tables (when run against RetailDB) - ✓ CSV seed files exist in scripts/raw_dataset/csv_data_file/ - ✓ dbt models compile (35 models found in latest parse) - - -================================================================================ -QUICK REFERENCE — EXACT ERROR MESSAGES FROM GITHUB ACTIONS -================================================================================ - -CI (latest): - sqlalchemy.exc.ProgrammingError: Cannot open database "retail_analytics" - requested by the login. The login failed. (4060) - -Docs (latest): - Database Error: Cannot open database "retail_analytics" requested by the - login. The login failed. (4060) - -CI (earlier, after partial progress): - Invalid object name 'RetailDB.bronze.customers'. (208) - -CD (all recent runs): - Login timeout expired (HYT00) — Connection test: [ERROR] - -================================================================================ -END OF REPORT -================================================================================ diff --git a/mart_stracture.t b/mart_stracture.t new file mode 100644 index 0000000..a7a3e5c --- /dev/null +++ b/mart_stracture.t @@ -0,0 +1,60 @@ +marts/ +├── core/ # Conformed dimensions and core facts +│ ├── dimensions/ # Shared dimensions (used across multiple facts) +│ │ ├── dim_customers.sql +│ │ ├── dim_products.sql +│ │ ├── dim_stores.sql +│ │ ├── dim_date.sql +│ │ ├── dim_employees.sql +│ │ └── _core__dimensions.yml # Documentation for all core dimensions +│ │ +│ ├── facts/ # Core fact tables +│ │ ├── fct_transactions.sql +│ │ ├── fct_returns.sql +│ │ ├── fct_reviews.sql +│ │ ├── fct_inventory.sql +│ │ └── _core__facts.yml +│ │ +│ └── scd/ # Slowly Changing Dimensions +│ ├── scd_dim_customers.sql +│ ├── scd_dim_products.sql +│ └── _core__scd.yml +│ +├── finance/ # Finance-specific marts +│ ├── dimensions/ +│ │ ├── dim_revenue_categories.sql +│ │ └── _finance__dimensions.yml +│ ├── facts/ +│ │ ├── fct_revenue.sql +│ │ ├── fct_costs.sql +│ │ └── _finance__facts.yml +│ └── aggregated/ # Pre-aggregated metrics for dashboards +│ ├── fct_daily_revenue.sql +│ ├── fct_monthly_profitability.sql +│ └── _finance__aggregated.yml +│ +├── marketing/ # Marketing-specific marts +│ ├── dimensions/ +│ │ ├── dim_campaigns.sql +│ │ ├── dim_customer_segments.sql +│ │ └── _marketing__dimensions.yml +│ ├── facts/ +│ │ ├── fct_customer_acquisition.sql +│ │ ├── fct_campaign_performance.sql +│ │ └── _marketing__facts.yml +│ └── aggregated/ +│ ├── fct_customer_lifetime_value.sql +│ └── _marketing__aggregated.yml +│ +├── operations/ # Operations-specific marts +│ ├── dimensions/ +│ │ ├── dim_store_performance.sql +│ │ └── _operations__dimensions.yml +│ ├── facts/ +│ │ ├── fct_inventory_turnover.sql +│ │ └── _operations__facts.yml +│ └── aggregated/ +│ ├── fct_store_performance.sql +│ └── _operations__aggregated.yml +│ +└── _marts__sources.yml # Sources for marts (if needed) \ No newline at end of file diff --git a/models/marts/core/dimensions/_dim_models.yml b/models/marts/core/dimensions/_dim_models.yml index d668e83..7fd2fdd 100644 --- a/models/marts/core/dimensions/_dim_models.yml +++ b/models/marts/core/dimensions/_dim_models.yml @@ -52,19 +52,3 @@ models: - accepted_values: arguments: values: ['True', 'False', 'Unknown'] - - - name: dim_date - description: Calendar dimension built from dates observed across sales, returns, reviews, and inventory snapshots. - columns: - - name: date_day - description: Calendar date. - tests: - - unique - - not_null - - - name: day_type - description: Weekday or weekend indicator. - tests: - - accepted_values: - arguments: - values: ['Weekday', 'Weekend'] diff --git a/models/marts/core/dimensions/dim_customers.sql b/models/marts/core/dimensions/dim_customers.sql index 4eaf3de..ac6667e 100644 --- a/models/marts/core/dimensions/dim_customers.sql +++ b/models/marts/core/dimensions/dim_customers.sql @@ -28,4 +28,4 @@ LEFT JOIN {{ ref('int_customer_location') }} as l ON p.customer_id = l.customer_id LEFT JOIN {{ ref('int_customer_segmentation') }} as s - ON s.customer_id = p.customer_id ; \ No newline at end of file + ON s.customer_id = p.customer_id ; \ No newline at end of file diff --git a/models/marts/core/dimensions/dim_date.sql b/models/marts/core/dimensions/dim_date.sql deleted file mode 100644 index a04a3ae..0000000 --- a/models/marts/core/dimensions/dim_date.sql +++ /dev/null @@ -1,30 +0,0 @@ -WITH dates AS ( - SELECT order_date AS date_day FROM {{ ref('int_transaction_fulfillment') }} - UNION - SELECT ship_date AS date_day FROM {{ ref('int_transaction_fulfillment') }} - UNION - SELECT delivery_date AS date_day FROM {{ ref('int_transaction_fulfillment') }} - UNION - SELECT return_date AS date_day FROM {{ ref('int_return_transaction') }} - UNION - SELECT review_date AS date_day FROM {{ ref('int_review_transaction') }} - UNION - SELECT snapshot_date AS date_day FROM {{ ref('int_inventory_product') }} -) - -SELECT - date_day, - YEAR(date_day) AS year_number, - DATEPART(QUARTER, date_day) AS quarter_number, - MONTH(date_day) AS month_number, - DATENAME(MONTH, date_day) AS month_name, - DAY(date_day) AS day_of_month, - DATEPART(WEEK, date_day) AS week_number, - DATENAME(WEEKDAY, date_day) AS day_name, - DATEPART(WEEKDAY, date_day) AS day_of_week_number, - CASE - WHEN DATEPART(WEEKDAY, date_day) IN (1, 7) THEN 'Weekend' - ELSE 'Weekday' - END AS day_type -FROM dates -WHERE date_day IS NOT NULL; diff --git a/models/marts/core/dimensions/dim_employees.sql b/models/marts/core/dimensions/dim_employees.sql index fd4f407..41d29a6 100644 --- a/models/marts/core/dimensions/dim_employees.sql +++ b/models/marts/core/dimensions/dim_employees.sql @@ -21,4 +21,4 @@ LEFT JOIN {{ ref('int_employee_contact') }} as c ON p.employee_id = c.employee_id LEFT JOIN {{ ref('int_employee_employment') }} as e - ON p.employee_id = e.employee_id ; + ON p.employee_id = e.employee_id ; diff --git a/models/marts/core/dimensions/dim_inventory.sql b/models/marts/core/dimensions/dim_inventory.sql index e69de29..3d6e0fd 100644 --- a/models/marts/core/dimensions/dim_inventory.sql +++ b/models/marts/core/dimensions/dim_inventory.sql @@ -0,0 +1,83 @@ +-- models/marts/core/dimensions/dim_inventory.sql +{{ + config( + materialized='table', + unique_key='product_snapshot_sk', + tags=['marts', 'dimension', 'inventory'] + ) +}} + +/* + Description: Product dimension with SCD Type 2 tracking + Grain: 1 row per product snapshot change + Load: Full refresh daily +*/ + +WITH product_base AS ( + SELECT + p.inventory_snapshot_sk, + p.snapshot_date, + p.product_id, + p.product_name, + p.sku, + p.category, + v.unit_cost, + v.unit_price, + v.warehouse_location, + v.store_id + FROM {{ ref('int_inventory_product') }} p + INNER JOIN {{ ref('int_inventory_valuation') }} v + ON p.inventory_snapshot_sk = v.inventory_snapshot_sk +), + +product_with_lead AS ( + SELECT + *, + LEAD(snapshot_date) OVER ( + PARTITION BY product_id + ORDER BY snapshot_date + ) as next_effective_date + FROM product_base +) + +SELECT + {{ dbt_utils.generate_surrogate_key([ + 'product_id', + 'snapshot_date' + ]) }} as product_snapshot_sk, + + product_id, + inventory_snapshot_sk, + product_name, + sku, + category, + + unit_cost, + unit_price, + unit_price - unit_cost as unit_margin, + CASE + WHEN unit_price = 0 OR unit_price IS NULL THEN 0 + ELSE ROUND(((unit_price - unit_cost) / unit_price) * 100, 2) + END as margin_percentage, + + warehouse_location, + store_id, + + snapshot_date as effective_from_date, + next_effective_date as effective_to_date, + + CASE + WHEN next_effective_date IS NULL THEN 1 + ELSE 0 + END as is_current_record, + + CASE + WHEN unit_cost > unit_price THEN 1 + ELSE 0 + END as is_negative_margin, + + GETDATE() as dbt_created_at, + GETDATE() as dbt_updated_at + +FROM product_with_lead +WHERE snapshot_date IS NOT NULL \ No newline at end of file diff --git a/models/marts/core/dimensions/dim_products.sql b/models/marts/core/dimensions/dim_products.sql index a37da46..4502431 100644 --- a/models/marts/core/dimensions/dim_products.sql +++ b/models/marts/core/dimensions/dim_products.sql @@ -26,5 +26,4 @@ LEFT JOIN {{ ref('int_product_pricing') }} AS p LEFT JOIN {{ ref('int_product_inventory') }} AS i ON a.product_id = i.product_id LEFT JOIN {{ ref('int_product_supplier') }} AS s - ON a.product_id = s.product_id -WHERE a.product_id IS NOT NULL; + ON a.product_id = s.product_id ; diff --git a/notes.txt b/notes.txt deleted file mode 100644 index 339d19a..0000000 --- a/notes.txt +++ /dev/null @@ -1,266 +0,0 @@ - - name: src_inventory - description: > - Cleaned and standardized inventory snapshot data from the bronze layer. - columns: - - name: snapshot_date - description: "ISO-standardized inventory snapshot date (YYYY-MM-DD)" - tests: - - not_null - - name: product_id - description: "Product identifier for inventory tracking" - tests: - - not_null - - name: product_name - description: "Standardized product name" - - name: sku - description: "Stock-keeping unit identifier" - - name: category - description: "Standardized product category" - - name: stock_on_hand - description: "Total physical inventory quantity on hand" - - name: stock_reserved - description: "Quantity of inventory reserved" - - name: stock_available - description: "Available inventory (stock_on_hand - stock_reserved)" - - name: reorder_level - description: "Inventory replenishment threshold" - - name: unit_cost - description: "Standardized unit cost in USD" - - name: unit_price - description: "Standardized unit selling price in USD" - - name: inventory_value - description: "Derived inventory valuation (unit_price * stock_on_hand)" - - name: warehouse_location - description: "Standardized warehouse location code" - - name: store_id - description: "Store-level inventory reference" - - - - name: src_products - description: > - Cleaned and standardized product master data from the bronze layer. - Applies identifier validation, SKU standardization, pricing cleanup, - inventory availability normalization, supplier standardization, and date standardization. - columns: - - name: product_id - description: "Unique product identifier" - tests: - - unique - - not_null - - name: sku - description: "Standardized uppercase SKU (13 characters)" - - name: product_name - description: "Standardized product name (title case)" - - name: brand - description: "Product brand name (title case)" - - name: category - description: "Product category classification" - - name: sub_category - description: "Product sub-category classification" - - name: department - description: "Business department classification" - - name: base_price_usd - description: "Standardized base selling price in USD" - - name: cost_price_usd - description: "Standardized cost price in USD" - - name: gross_margin_pct - description: "Product gross margin percentage" - - name: weight_kg - description: "Product weight in kilograms" - - name: is_available - description: "Inventory availability status" - - name: stock_quantity - description: "Current stock quantity" - - name: reorder_level - description: "Inventory reorder threshold" - - name: supplier_name - description: "Standardized supplier name" - - name: supplier_country - description: "Standardized supplier country" - - name: warranty_years - description: "Product warranty duration in years" - - name: rating_avg - description: "Average customer rating (1-5)" - - name: review_count - description: "Total number of customer reviews" - - name: launched_date - description: "ISO-standardized product launch date (YYYY-MM-DD)" - - name: product_url - description: "Standardized product URL" - - - - name: src_returns - description: > - Cleaned and standardized return transaction data from the bronze layer. - columns: - - name: return_id - description: "Unique return transaction identifier" - tests: - - unique - - not_null - - name: original_txn_id - description: "Original source transaction identifier" - - name: original_order_id - description: "Original order identifier" - - name: customer_id - description: "Customer identifier associated with return" - - name: customer_name - description: "Standardized customer name" - - name: product_id - description: "Returned product identifier" - - name: product_name - description: "Standardized returned product name" - - name: quantity_returned - description: "Quantity of items returned" - - name: return_date - description: "ISO-standardized return date (YYYY-MM-DD)" - - name: return_reason - description: "Standardized return reason" - - name: refund_amount - description: "Standardized refund amount in USD" - - name: refund_method - description: "Refund payment method" - - name: return_channel - description: "Standardized return channel" - tests: - - accepted_values: - values: ["Mobile App", "In Store", "Online", "Phone Call", "Catalog", "Unknown"] - - name: restocked - description: "Restocking indicator (Yes / No / Unknown)" - - name: return_status - description: "Standardized return processing status" - - name: handled_by_emp_id - description: "Employee identifier who handled the return" - - name: notes - description: "Operational return notes" - - - - name: src_reviews - description: > - Cleaned and standardized customer review data from the bronze layer. - columns: - - name: review_id - description: "Unique customer review identifier" - tests: - - unique - - not_null - - name: txn_id - description: "Transaction identifier associated with review" - - name: customer_id - description: "Customer identifier" - - name: customer_name - description: "Standardized customer name" - - name: product_id - description: "Reviewed product identifier" - - name: product_name - description: "Standardized reviewed product name" - - name: rating - description: "Customer rating score (1-5)" - tests: - - not_null - - name: rating_text - description: "Standardized rating sentiment category" - - name: review_date - description: "ISO-standardized review submission date (YYYY-MM-DD)" - - name: verified_purchase - description: "Purchase verification status" - - name: helpful_votes - description: "Count of helpful votes received" - - name: review_channel - description: "Standardized review submission channel" - tests: - - accepted_values: - values: ["Mobile App", "In Store", "Online", "Phone Call", "Catalog", "Unknown"] - - name: review_title - description: "Standardized review title" - - - - name: src_stores - description: > - Cleaned and standardized retail store operational data from the bronze layer. - columns: - - name: store_id - description: "Unique store identifier" - tests: - - unique - - not_null - - name: store_name - description: "Standardized retail store name" - - name: store_type - description: "Standardized store category" - - name: address - description: "Standardized store address" - - name: city - description: "Standardized city name" - - name: state - description: "Standardized two-letter state abbreviation" - - name: state_full - description: "Standardized full state name" - - name: zip_code - description: "Validated ZIP code" - - name: country - description: "Standardized country name" - - name: region - description: "Standardized regional classification" - - name: district - description: "Standardized district classification" - - name: phone - description: "Standardized store phone number: +1 (XXX) XXX-XXXX" - - name: manager_name - description: "Standardized store manager name" - - name: opened_date - description: "ISO-standardized store opening date (YYYY-MM-DD)" - - name: sq_footage - description: "Store area in square feet" - - name: num_employees - description: "Number of employees at store" - - name: annual_rent_usd - description: "Annual rental cost in USD" - - name: is_active - description: "Store operational status (True / False / Unknown)" - - name: has_parking - description: "Parking availability (True / False / Unknown)" - - name: has_cafe - description: "Cafe availability (True / False / Unknown)" - - - - name: src_transactions - description: > - Cleaned and deduplicated sales transaction data from the bronze layer. - columns: - - name: transaction_id - description: "Unique transaction identifier" - tests: - - unique - - not_null - - name: order_id - description: "Order identifier" - - name: order_line_number - description: "Line item number within order" - - name: order_date - description: "ISO-standardized order date (YYYY-MM-DD)" - - name: ship_date - description: "ISO-standardized ship date (YYYY-MM-DD)" - - name: delivery_date - description: "ISO-standardized delivery date (YYYY-MM-DD)" - - name: customer_id - description: "Customer identifier" - - name: product_id - description: "Product identifier" - - name: store_id - description: "Store identifier" - - name: employee_id - description: "Employee identifier" - - name: quantity_ordered - description: "Quantity of items ordered" - - name: unit_selling_price - description: "Standardized unit selling price in USD" - - name: line_total_before_tax - description: "Standardized line total before tax in USD" - - name: line_total_with_tax - description: "Standardized line total with tax in USD" - - name: sales_channel - description: "Standardized sales channel" - - name: order_status - description: "Order processing status" - diff --git a/profiles.example.yml b/profiles.example.yml deleted file mode 100644 index e2750a3..0000000 --- a/profiles.example.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Copy to ~/.dbt/profiles.yml (or set DBT_PROFILES_DIR) and adjust values. -# Never commit real credentials — use environment variables in production/CI. - -dbt_analytics_engineering: - target: dev - outputs: - dev: - type: sqlserver - driver: "ODBC Driver 18 for SQL Server" - server: "{{ env_var('DBT_SQLSERVER_HOST', 'localhost') }}" - port: "{{ env_var('DBT_SQLSERVER_PORT', '1433') | int }}" - database: "{{ env_var('DBT_SQLSERVER_DATABASE', 'retail_analytics') }}" - schema: "{{ env_var('DBT_SQLSERVER_SCHEMA', 'dbt_dev') }}" - user: "{{ env_var('DBT_SQLSERVER_USER', 'sa') }}" - password: "{{ env_var('DBT_SQLSERVER_PASSWORD') }}" - trust_cert: true - encrypt: true - - prod: - type: sqlserver - driver: "ODBC Driver 18 for SQL Server" - server: "{{ env_var('DBT_SQLSERVER_HOST') }}" - port: "{{ env_var('DBT_SQLSERVER_PORT', '1433') | int }}" - database: "{{ env_var('DBT_SQLSERVER_DATABASE') }}" - schema: "{{ env_var('DBT_SQLSERVER_SCHEMA', 'dbt_prod') }}" - user: "{{ env_var('DBT_SQLSERVER_USER') }}" - password: "{{ env_var('DBT_SQLSERVER_PASSWORD') }}" - trust_cert: true - encrypt: true diff --git a/models/intermediate/_int__retail_source.yml b/snapshots/inventory_snapshot.sql similarity index 100% rename from models/intermediate/_int__retail_source.yml rename to snapshots/inventory_snapshot.sql