diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index db26a73..d73fe98 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -13,15 +13,6 @@ jobs: fail-fast: false matrix: include: - - PYSPARK_VERSION: "3.1.3" - PYTHON_VERSION: "3.9" - JAVA_VERSION: "11" - - PYSPARK_VERSION: "3.2" - PYTHON_VERSION: "3.9" - JAVA_VERSION: "11" - - PYSPARK_VERSION: "3.3" - PYTHON_VERSION: "3.9" - JAVA_VERSION: "11" - PYSPARK_VERSION: "3.5" PYTHON_VERSION: "3.9" JAVA_VERSION: "17" diff --git a/Dockerfile b/Dockerfile index da30d1f..08db155 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,12 +5,12 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get install -y software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get install -y python3.8 python3-pip -RUN apt-get install -y python3.8-distutils -RUN apt-get install -y openjdk-11-jdk +RUN apt-get install -y python3.9 python3-pip +RUN apt-get install -y python3.9-distutils +RUN apt-get install -y openjdk-17-jdk # Update symlink to point to latest -RUN rm /usr/bin/python3 && ln -s /usr/bin/python3.8 /usr/bin/python3 +RUN rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3 RUN python3 --version RUN pip3 --version RUN java -version @@ -21,8 +21,7 @@ COPY pyproject.toml /python-deequ COPY poetry.lock /python-deequ WORKDIR python-deequ -RUN poetry install -vvv -RUN poetry add pyspark==3.5.0 -vvv +RUN poetry install --extras pyspark -vvv ENV SPARK_VERSION=3.5 COPY . /python-deequ diff --git a/README.md b/README.md index 164b911..c2c1b87 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ You can install [PyDeequ via pip](https://pypi.org/project/pydeequ/). pip install pydeequ ``` +> **Supported Spark version:** PyDeequ 1.7.0+ tracks the Deequ 2.0.21 JVM library, which is published for **Spark 3.5 only**. For Spark 3.1–3.4, use PyDeequ 1.6.0 (the last release supporting them, on Deequ 2.0.8). + ### Set up a PySpark session ```python from pyspark.sql import SparkSession, Row @@ -214,8 +216,8 @@ Install Java Now open favourite terminal and enter the following: List the Apache Spark versions: $ sdk list spark -To install For Spark 3 -$ sdk install spark 3.0.2 +To install For Spark 3.5 +$ sdk install spark 3.5.1 ``` ### Poetry @@ -246,8 +248,8 @@ $ poetry run pytest If you have issues installing the dependencies listed above, another way to run the tests and verify your changes is through Docker. There is a Dockerfile that will install the required dependencies and run the tests in a container. ``` -docker build . -t spark-3.3-docker-test -docker run spark-3.3-docker-test +docker build . -t spark-3.5-docker-test +docker run spark-3.5-docker-test ``` ## Contributing diff --git a/docs/source/conf.py b/docs/source/conf.py index 54a6f9e..7c81117 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,6 +12,7 @@ # import os import sys +from importlib.metadata import PackageNotFoundError, version from recommonmark.parser import CommonMarkParser sys.path.insert(0, os.path.abspath('../..')) @@ -22,8 +23,11 @@ copyright = 'Copyright 2020, Amazon' author = 'Calvin Wang, Chris Ghyzel, Joan Aoanan, Veronika Megler' -# The full version, including alpha/beta/rc tags -release = '0.0.4' +# Track the installed package version so the docs don't drift from releases. +try: + release = version("pydeequ") +except PackageNotFoundError: + release = "0.0.0" # -- General configuration --------------------------------------------------- diff --git a/poetry.lock b/poetry.lock index 62f13bc..a2b8bb9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -815,13 +815,13 @@ virtualenv = ">=20.10.0" [[package]] name = "py4j" -version = "0.10.9.5" +version = "0.10.9.9" description = "Enables Python programs to dynamically access arbitrary Java objects" optional = true python-versions = "*" files = [ - {file = "py4j-0.10.9.5-py2.py3-none-any.whl", hash = "sha256:52d171a6a2b031d8a5d1de6efe451cf4f5baff1a2819aabc3741c8406539ba04"}, - {file = "py4j-0.10.9.5.tar.gz", hash = "sha256:276a4a3c5a2154df1860ef3303a927460e02e97b047dc0a47c1c3fb8cce34db6"}, + {file = "py4j-0.10.9.9-py2.py3-none-any.whl", hash = "sha256:c7c26e4158defb37b0bb124933163641a2ff6e3a3913f7811b0ddbe07ed61533"}, + {file = "py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879"}, ] [[package]] @@ -890,22 +890,23 @@ plugins = ["importlib-metadata"] [[package]] name = "pyspark" -version = "3.3.2" +version = "3.5.8" description = "Apache Spark Python API" optional = true -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pyspark-3.3.2.tar.gz", hash = "sha256:0dfd5db4300c1f6cc9c16d8dbdfb82d881b4b172984da71344ede1a9d4893da8"}, + {file = "pyspark-3.5.8.tar.gz", hash = "sha256:54cca0767b21b40e3953ad1d30f8601c53abf9cbda763653289cdcfcac52313c"}, ] [package.dependencies] -py4j = "0.10.9.5" +py4j = ">=0.10.9.7,<0.10.9.10" [package.extras] -ml = ["numpy (>=1.15)"] -mllib = ["numpy (>=1.15)"] -pandas-on-spark = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] -sql = ["pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] +connect = ["googleapis-common-protos (>=1.56.4)", "grpcio (>=1.56.0)", "grpcio-status (>=1.56.0)", "numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"] +ml = ["numpy (>=1.15,<2)"] +mllib = ["numpy (>=1.15,<2)"] +pandas-on-spark = ["numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"] +sql = ["numpy (>=1.15,<2)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"] [[package]] name = "pytest" @@ -1298,4 +1299,4 @@ pyspark = ["pyspark"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4" -content-hash = "9ca1d776b68eead781bf0161f4f2ee552085c8e201b482b6f78d18c06a2a5d79" +content-hash = "fded78caf1dc1536fcb2034f8c3a21a239d21087f74bca9a02e4d0e32046b273" diff --git a/pydeequ/configs.py b/pydeequ/configs.py index e56c97d..9880d85 100644 --- a/pydeequ/configs.py +++ b/pydeequ/configs.py @@ -4,11 +4,10 @@ import re +# Deequ 2.0.10+ only publishes a spark-3.5 build; support for Spark 3.1/3.2/3.3 +# was dropped upstream, so pydeequ tracks Spark 3.5 only on Deequ 2.0.21. SPARK_TO_DEEQU_COORD_MAPPING = { - "3.5": "com.amazon.deequ:deequ:2.0.8-spark-3.5", - "3.3": "com.amazon.deequ:deequ:2.0.8-spark-3.3", - "3.2": "com.amazon.deequ:deequ:2.0.8-spark-3.2", - "3.1": "com.amazon.deequ:deequ:2.0.8-spark-3.1" + "3.5": "com.amazon.deequ:deequ:2.0.21-spark-3.5", } diff --git a/pyproject.toml b/pyproject.toml index f44b1b6..1543f03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ python = ">=3.9,<4" numpy = ">=1.14.1" pandas = ">=0.23.0" -pyspark = { version = ">=2.4.7,<4.0.0", optional = true } +pyspark = { version = ">=3.5,<3.6", optional = true } [tool.poetry.dev-dependencies] pytest = "^8.0" diff --git a/tests/test_analyzers.py b/tests/test_analyzers.py index 175e8ae..7614cfd 100644 --- a/tests/test_analyzers.py +++ b/tests/test_analyzers.py @@ -411,6 +411,9 @@ def test_fail_Histogram(self): ) def test_Histogram_maxBins(self): + # Deequ 2.0.10+ emits a trailing Histogram.tailCount metric when the + # number of distinct values (3) exceeds maxDetailBins (2): tailCount = + # 3 - 2 = 1, so the final Row(value=1.0) is the rolled-up tail. self.assertEqual( self.Histogram_maxBins("b", maxDetailBins=2), [ @@ -419,6 +422,7 @@ def test_Histogram_maxBins(self): Row(value=0.3333333333333333), Row(value=1.0), Row(value=0.3333333333333333), + Row(value=1.0), ], ) self.assertEqual( @@ -429,6 +433,7 @@ def test_Histogram_maxBins(self): Row(value=0.3333333333333333), Row(value=1.0), Row(value=0.3333333333333333), + Row(value=1.0), ], ) diff --git a/tests/test_anomaly_detection.py b/tests/test_anomaly_detection.py index 98a89c6..cef3deb 100644 --- a/tests/test_anomaly_detection.py +++ b/tests/test_anomaly_detection.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import os import unittest import pytest @@ -282,13 +281,6 @@ def HoltWinters(self, analyzer_func, test, df_prev, df_curr=None): print(df.collect()) return df.select("check_status").collect() - @pytest.mark.xfail( - os.environ.get("SPARK_VERSION", "").startswith(("3.1", "3.2")), - reason=( - "Not supported in Spark < 3.3: breeze.stats.DescriptiveStats " - "is in unnamed module of loader 'app'" - ) - ) def test_BatchNormalStrategy(self): # Interval is inclusive, so meet the requirements upper value is up to 9 @@ -357,13 +349,6 @@ def test_OnlineNormalStrategy(self): [Row(check_status="Success")], ) - @pytest.mark.xfail( - os.environ.get("SPARK_VERSION", "").startswith(("3.1", "3.2")), - reason=( - "Not supported in Spark < 3.3: breeze.stats.DescriptiveStats " - "is in unnamed module of loader 'app'" - ) - ) def test_holtWinters(self): # must have 15 points of data self.assertEqual(self.HoltWinters(Size(), 1, self.df_1), [Row(check_status="Success")]) diff --git a/tests/test_config.py b/tests/test_config.py index c2956b3..03e0a8d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,9 @@ import pytest -from pydeequ.configs import _extract_major_minor_versions +from pydeequ.configs import ( + _extract_major_minor_versions, + _get_deequ_maven_config, + _get_spark_version, +) @pytest.mark.parametrize( @@ -13,3 +17,25 @@ ) def test_extract_major_minor_versions(full_version, major_minor_version): assert _extract_major_minor_versions(full_version) == major_minor_version + + +def test_supported_spark_resolves_to_deequ_coord(monkeypatch): + # _get_spark_version is lru_cached and reads SPARK_VERSION at call time. + monkeypatch.setenv("SPARK_VERSION", "3.5") + _get_spark_version.cache_clear() + try: + assert _get_deequ_maven_config() == "com.amazon.deequ:deequ:2.0.21-spark-3.5" + finally: + _get_spark_version.cache_clear() + + +@pytest.mark.parametrize("unsupported", ["3.3", "3.2", "3.1", "3.4"]) +def test_unsupported_spark_raises(monkeypatch, unsupported): + # Spark versions Deequ no longer publishes a build for must fail loudly. + monkeypatch.setenv("SPARK_VERSION", unsupported) + _get_spark_version.cache_clear() + try: + with pytest.raises(RuntimeError, match="incompatible Spark version"): + _get_deequ_maven_config() + finally: + _get_spark_version.cache_clear()