diff --git a/.gitignore b/.gitignore index 0e8367c..7a73c79 100644 --- a/.gitignore +++ b/.gitignore @@ -50,7 +50,7 @@ python/ .vscode/ .idea/ - +.qodo/ # ========================================== # Operating System Files # ========================================== @@ -108,4 +108,6 @@ docker-data/ # Misc Logs # ========================================== -*.log \ No newline at end of file +*.log +my_intro.md +mart_stracture.t \ No newline at end of file diff --git a/macros/standardize_date.sql b/macros/standardize_date.sql new file mode 100644 index 0000000..0b20e9b --- /dev/null +++ b/macros/standardize_date.sql @@ -0,0 +1,28 @@ +{% macro standardize_date(column_name) %} + +CASE + WHEN TRIM({{ column_name }}) LIKE '[A-Z][a-z][a-z][a-z]% __, ____' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '[A-Z][a-z][a-z] __, ____' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '____-__-__' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '____/__/__' + THEN TRY_CONVERT(DATE, TRIM({{ column_name }})) + + WHEN TRIM({{ column_name }}) LIKE '__/__/____' + AND TRY_CONVERT(INT, SUBSTRING(TRIM({{ column_name }}), 4, 2)) > 12 + THEN TRY_CONVERT(DATE, TRIM({{ column_name }}), 101) + + WHEN TRIM({{ column_name }}) LIKE '__/__/____' + AND TRY_CONVERT(INT, LEFT(TRIM({{ column_name }}), 2)) > 12 + THEN TRY_CONVERT(DATE, TRIM({{ column_name }}), 103) + + ELSE TRY_CONVERT(DATE, TRIM({{ column_name }}), 101) + +END + +{% endmacro %} \ No newline at end of file diff --git a/models/intermediate/Customers/_int__customers_models.yml b/models/intermediate/Customers/_int__customers_models.yml index 5dca310..e3467db 100644 --- a/models/intermediate/Customers/_int__customers_models.yml +++ b/models/intermediate/Customers/_int__customers_models.yml @@ -67,8 +67,8 @@ models: European, and month-name formats. Invalid or unrecognized values are returned as NULL to preserve data quality while enabling accurate age-based analysis and customer segmentation. - tests: - - not_null +# tests: +# - valid_date_format #=================================================================================== #============================ Model: int_customers_contact ========================= diff --git a/models/intermediate/Customers/int_customer_profile.sql b/models/intermediate/Customers/int_customer_profile.sql index 6b9472e..04e3ee5 100644 --- a/models/intermediate/Customers/int_customer_profile.sql +++ b/models/intermediate/Customers/int_customer_profile.sql @@ -36,18 +36,20 @@ SELECT ELSE 'Unknown' END as gender, - CASE - WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z][a-z]% __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z] __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - WHEN TRIM(date_of_birth) LIKE '____-__-__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - WHEN TRIM(date_of_birth) LIKE '____/__/__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) - - WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 101) - WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 103) - - WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 110) - WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 105) - ELSE TRY_CONVERT(DATE, TRIM(date_of_birth), 101) - END date_of_birth + {{ standardize_date('date_of_birth') }} AS date_of_birth + + --CASE + -- WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z][a-z]% __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + -- WHEN TRIM(date_of_birth) LIKE '[A-Z][a-z][a-z] __, ____' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + -- WHEN TRIM(date_of_birth) LIKE '____-__-__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + -- WHEN TRIM(date_of_birth) LIKE '____/__/__' THEN TRY_CONVERT(DATE, TRIM(date_of_birth)) + + -- WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 101) + -- WHEN TRIM(date_of_birth) LIKE '__/__/____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 103) + + -- WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, SUBSTRING(TRIM(date_of_birth), 4, 2)) > 12 THEN TRY_CONVERT(DATE, TRIM(date_of_birth), 110) + -- WHEN TRIM(date_of_birth) LIKE '__-__-____' AND TRY_CONVERT(INT, LEFT(TRIM(date_of_birth), 2)) > 12 THEN TRY_CONVERT(DATE , TRIM(date_of_birth), 105) + -- ELSE TRY_CONVERT(DATE, TRIM(date_of_birth), 101) + --END date_of_birth FROM {{ ref('stg_customers') }} ; \ No newline at end of file diff --git a/tests/generic/valid_date_format.sql b/tests/generic/valid_date_format.sql new file mode 100644 index 0000000..76232ef --- /dev/null +++ b/tests/generic/valid_date_format.sql @@ -0,0 +1,11 @@ +{% test valid_date_of_birth(model, column_name) %} + +SELECT * +FROM {{ model }} +WHERE {{ column_name }} IS NOT NULL + AND ( + {{ column_name }} < '1900-01-01' + OR {{ column_name }} > CAST(GETDATE() AS DATE) + ) + +{% endtest %} \ No newline at end of file diff --git a/tests/singular/valid_zip_code.sql b/tests/singular/valid_zip_code.sql new file mode 100644 index 0000000..cdd38a2 --- /dev/null +++ b/tests/singular/valid_zip_code.sql @@ -0,0 +1,5 @@ +SELECT * +FROM {{ ref('int_customer_location') }} +WHERE zip_code IS NULL + OR LEN(CAST(zip_code AS VARCHAR(10))) <> 5 + OR TRY_CAST(zip_code AS INT) IS NULL \ No newline at end of file