From 0ed2fbe601ad5bae5fbfd6ab2e70fcb1fcd52243 Mon Sep 17 00:00:00 2001 From: ritik574-coder Date: Thu, 16 Jul 2026 15:50:25 +0530 Subject: [PATCH 1/2] adding mocro in dbt project that enable re-useablity feature and provide me DRY rule --- .gitignore | 6 ++-- macros/standardize_date.sql | 28 +++++++++++++++++++ .../Customers/_int__customers_models.yml | 1 + .../Customers/int_customer_profile.sql | 28 ++++++++++--------- tests/generic/valid_date_format.sql | 9 ++++++ tests/singular/valid_zip_code.sql | 5 ++++ 6 files changed, 62 insertions(+), 15 deletions(-) create mode 100644 macros/standardize_date.sql create mode 100644 tests/generic/valid_date_format.sql create mode 100644 tests/singular/valid_zip_code.sql diff --git a/.gitignore b/.gitignore index 0e8367c..7a73c79 100644 --- a/.gitignore +++ b/.gitignore @@ -50,7 +50,7 @@ python/ .vscode/ .idea/ - +.qodo/ # ========================================== # Operating System Files # ========================================== @@ -108,4 +108,6 @@ docker-data/ # Misc Logs # ========================================== -*.log \ No newline at end of file +*.log +my_intro.md +mart_stracture.t \ No newline at end of file diff --git a/macros/standardize_date.sql b/macros/standardize_date.sql new file mode 100644 index 0000000..0b20e9b --- /dev/null +++ b/macros/standardize_date.sql @@ -0,0 +1,28 @@ +{% macro standardize_date(column_name) %} + +CASE + WHEN TRIM({{ column_name }}) LIKE '[A-Z][a-z][a-z][a-z]% __, ____' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '[A-Z][a-z][a-z] __, ____' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '____-__-__' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '____/__/__' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '__/__/____' + AND TRY_CONVERT(INT, SUBSTRING(TRIM({{ column_name }}), 4, 2)) > 12 + THEN TRY_CONVERT(DATE, TRIM({{ column_name }}), 101) + + WHEN TRIM({{ column_name }}) LIKE '__/__/____' + AND TRY_CONVERT(INT, LEFT(TRIM({{ column_name }}), 2)) > 12 + THEN TRY_CONVERT(DATE, TRIM({{ column_name }}), 103) + + ELSE TRY_CONVERT(DATE, TRIM({{ column_name }}), 101) + +END + +{% endmacro %} \ No newline at end of file diff --git a/models/intermediate/Customers/_int__customers_models.yml b/models/intermediate/Customers/_int__customers_models.yml index 5dca310..ef31c9c 100644 --- a/models/intermediate/Customers/_int__customers_models.yml +++ b/models/intermediate/Customers/_int__customers_models.yml @@ -69,6 +69,7 @@ models: age-based analysis and customer segmentation. tests: - not_null + - valid_date_format #=================================================================================== #============================ Model: int_customers_contact ========================= diff --git a/models/intermediate/Customers/int_customer_profile.sql b/models/intermediate/Customers/int_customer_profile.sql index 6b9472e..04e3ee5 100644 --- a/models/intermediate/Customers/int_customer_profile.sql +++ b/models/intermediate/Customers/int_customer_profile.sql @@ -36,18 +36,20 @@ SELECT ELSE 'Unknown' END as gender, - CASE - WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z][a-z]% __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z] __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - WHEN TRIM(date_of_birth) LIKE '____-__-__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - WHEN TRIM(date_of_birth) LIKE '____/__/__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - - WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 101) - WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 103) - - WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 110) - WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 105) - ELSE TRY_CONVERT(DATE, TRIM(date_of_birth), 101) - END date_of_birth + {{ standardize_date('date_of_birth') }} AS date_of_birth + + --CASE + -- WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z][a-z]% __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + -- WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z] __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + -- WHEN TRIM(date_of_birth) LIKE '____-__-__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + -- WHEN TRIM(date_of_birth) LIKE '____/__/__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + + -- WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 101) + -- WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 103) + + -- WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 110) + -- WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 105) + -- ELSE TRY_CONVERT(DATE, TRIM(date_of_birth), 101) + --END date_of_birth FROM {{ ref('stg_customers') }} ; \ No newline at end of file diff --git a/tests/generic/valid_date_format.sql b/tests/generic/valid_date_format.sql new file mode 100644 index 0000000..feb70d0 --- /dev/null +++ b/tests/generic/valid_date_format.sql @@ -0,0 +1,9 @@ +{% test valid_date_format(model, column_name) %} + +SELECT + * +FROM {{ model }} +WHERE {{ column_name }} NOT LIKE '____-__-__' + OR {{ column_name }} IS NULL + +{% endtest %} \ No newline at end of file diff --git a/tests/singular/valid_zip_code.sql b/tests/singular/valid_zip_code.sql new file mode 100644 index 0000000..cdd38a2 --- /dev/null +++ b/tests/singular/valid_zip_code.sql @@ -0,0 +1,5 @@ +SELECT * +FROM {{ ref('int_customer_location') }} +WHERE zip_code IS NULL + OR LEN(CAST(zip_code AS VARCHAR(10))) <> 5 + OR TRY_CAST(zip_code AS INT) IS NULL \ No newline at end of file From 7007c3dea151ed1b73cee7a5d079af64d55c98a4 Mon Sep 17 00:00:00 2001 From: ritik574-coder Date: Thu, 16 Jul 2026 16:44:18 +0530 Subject: [PATCH 2/2] removing the unwanted test and handling the error --- .../Customers/_int__customers_models.yml | 5 ++--- tests/generic/valid_date_format.sql | 14 ++++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/models/intermediate/Customers/_int__customers_models.yml b/models/intermediate/Customers/_int__customers_models.yml index ef31c9c..e3467db 100644 --- a/models/intermediate/Customers/_int__customers_models.yml +++ b/models/intermediate/Customers/_int__customers_models.yml @@ -67,9 +67,8 @@ models: European, and month-name formats. Invalid or unrecognized values are returned as NULL to preserve data quality while enabling accurate age-based analysis and customer segmentation. - tests: - - not_null - - valid_date_format +# tests: +# - valid_date_format #=================================================================================== #============================ Model: int_customers_contact ========================= diff --git a/tests/generic/valid_date_format.sql b/tests/generic/valid_date_format.sql index feb70d0..76232ef 100644 --- a/tests/generic/valid_date_format.sql +++ b/tests/generic/valid_date_format.sql @@ -1,9 +1,11 @@ -{% test valid_date_format(model, column_name) %} +{% test valid_date_of_birth(model, column_name) %} -SELECT - * +SELECT * FROM {{ model }} -WHERE {{ column_name }} NOT LIKE '____-__-__' - OR {{ column_name }} IS NULL - +WHERE {{ column_name }} IS NOT NULL + AND ( + {{ column_name }} < '1900-01-01' + OR {{ column_name }} > CAST(GETDATE() AS DATE) + ) + {% endtest %} \ No newline at end of file