From 9165c7abe22b25d987784249843557ca2a793c80 Mon Sep 17 00:00:00 2001 From: vc1492a Date: Tue, 16 Jun 2026 09:38:59 -0700 Subject: [PATCH 1/4] feat: adopt scikit-learn API conventions for 1.0.0 Move data parameters (data, distance_matrix, neighbor_matrix, cluster_labels) from __init__() to fit(), aligning with scikit-learn estimator conventions. Configuration parameters (extent, n_neighbors, use_numba, n_jobs, progress_bar) remain in the constructor. Passing data params to __init__() still works with a FutureWarning for backward compatibility. Re-fitting with new data is now supported. Add LoOP class alias and export __version__ from the package. Closes #5, Closes #7 Co-authored-by: Cursor --- PyNomaly/__init__.py | 4 + PyNomaly/loop.py | 148 ++++++++++++++++------- README.md | 52 ++++---- changelog.md | 18 +++ docs/changelog.md | 18 +++ docs/getting-started.md | 16 ++- docs/user-guide.md | 28 ++--- setup.py | 4 +- tests/test_loop.py | 256 +++++++++++++++++++++++++--------------- 9 files changed, 349 insertions(+), 195 deletions(-) diff --git a/PyNomaly/__init__.py b/PyNomaly/__init__.py index 0cd3ebf..9b63740 100644 --- a/PyNomaly/__init__.py +++ b/PyNomaly/__init__.py @@ -3,16 +3,20 @@ from PyNomaly.loop import ( LocalOutlierProbability, + LoOP, PyNomalyError, ValidationError, ClusterSizeError, MissingValuesError, + __version__, ) __all__ = [ "LocalOutlierProbability", + "LoOP", "PyNomalyError", "ValidationError", "ClusterSizeError", "MissingValuesError", + "__version__", ] diff --git a/PyNomaly/loop.py b/PyNomaly/loop.py index 31a32b1..1ea7ab8 100644 --- a/PyNomaly/loop.py +++ b/PyNomaly/loop.py @@ -77,7 +77,7 @@ def _numba_distance_kernel_sequential(clust_points_vector, n_neighbors): pass __author__ = "Valentino Constantinou" -__version__ = "0.4.0" +__version__ = "1.0.0" __license__ = "Apache License, Version 2.0" @@ -131,42 +131,45 @@ def emit_progress_bar(progress: str, index: int, total: int) -> str: class LocalOutlierProbability(object): - """ - :param data: a Pandas DataFrame or Numpy array of float data - :param extent: an integer value [1, 2, 3] that controls the statistical - extent, e.g. lambda times the standard deviation from the mean (optional, + """Local Outlier Probability (LoOP) estimator. + + :param extent: an integer value [1, 2, 3] that controls the statistical + extent, e.g. lambda times the standard deviation from the mean (optional, default 3) - :param n_neighbors: the total number of neighbors to consider w.r.t. each + :param n_neighbors: the total number of neighbors to consider w.r.t. each sample (optional, default 10) - :param cluster_labels: a numpy array of cluster assignments w.r.t. each - sample (optional, default None) + :param use_numba: whether to use Numba JIT acceleration for distance + computation (optional, default False) :param n_jobs: controls Numba thread-level parallelism via prange. Use -1 to use all available CPU cores, or 1 for sequential processing. Only effective when use_numba=True (optional, default 1) - :return: - """ """ + :param progress_bar: whether to display a progress bar during distance + computation (optional, default False) - Based on the work of Kriegel, Kröger, Schubert, and Zimek (2009) in LoOP: + Based on the work of Kriegel, Kröger, Schubert, and Zimek (2009) in LoOP: Local Outlier Probabilities. ---------- References ---------- - .. [1] Breunig M., Kriegel H.-P., Ng R., Sander, J. LOF: Identifying + .. [1] Breunig M., Kriegel H.-P., Ng R., Sander, J. LOF: Identifying Density-based Local Outliers. ACM SIGMOD International Conference on Management of Data (2000). - .. [2] Kriegel H.-P., Kröger P., Schubert E., Zimek A. LoOP: Local Outlier - Probabilities. 18th ACM conference on + .. [2] Kriegel H.-P., Kröger P., Schubert E., Zimek A. LoOP: Local Outlier + Probabilities. 18th ACM conference on Information and knowledge management, CIKM (2009). - .. [3] Goldstein M., Uchida S. A Comparative Evaluation of Unsupervised + .. [3] Goldstein M., Uchida S. A Comparative Evaluation of Unsupervised Anomaly Detection Algorithms for Multivariate Data. PLoS ONE 11(4): e0152173 (2016). - .. [4] Hamlet C., Straub J., Russell M., Kerlin S. An incremental and - approximate local outlier probability algorithm for intrusion - detection and its evaluation. Journal of Cyber Security Technology - (2016). + .. [4] Hamlet C., Straub J., Russell M., Kerlin S. An incremental and + approximate local outlier probability algorithm for intrusion + detection and its evaluation. Journal of Cyber Security Technology + (2016). """ + _DATA_PARAMS = ("data", "distance_matrix", "neighbor_matrix", + "cluster_labels") + """ Validation methods. These methods validate inputs and raise exceptions or warnings as appropriate. @@ -378,17 +381,19 @@ def new_f(*args, **kwds): "Argument %r is not of type %s" % (a, t), UserWarning ) opt_types = { - "distance_matrix": {"type": types[2]}, - "neighbor_matrix": {"type": types[3]}, - "extent": {"type": types[4]}, - "n_neighbors": {"type": types[5]}, - "cluster_labels": {"type": types[6]}, - "use_numba": {"type": types[7]}, - "n_jobs": {"type": types[8]}, - "progress_bar": {"type": types[9]}, + "extent": {"type": (int, np.integer)}, + "n_neighbors": {"type": (int, np.integer)}, + "use_numba": {"type": bool}, + "n_jobs": {"type": (int, np.integer)}, + "progress_bar": {"type": bool}, + "data": {"type": np.ndarray}, + "distance_matrix": {"type": np.ndarray}, + "neighbor_matrix": {"type": np.ndarray}, + "cluster_labels": {"type": list}, } for x in kwds: - opt_types[x]["value"] = kwds[x] + if x in opt_types: + opt_types[x]["value"] = kwds[x] for k in opt_types: try: if ( @@ -411,43 +416,59 @@ def new_f(*args, **kwds): @accepts( object, - np.ndarray, - np.ndarray, - np.ndarray, (int, np.integer), (int, np.integer), - list, bool, (int, np.integer), bool, + np.ndarray, + np.ndarray, + np.ndarray, + list, ) def __init__( self, - data=None, - distance_matrix=None, - neighbor_matrix=None, extent=3, n_neighbors=10, - cluster_labels=None, use_numba=False, n_jobs=1, progress_bar=False, + data=None, + distance_matrix=None, + neighbor_matrix=None, + cluster_labels=None, ) -> None: - self.data = data - self.distance_matrix = distance_matrix - self.neighbor_matrix = neighbor_matrix self.extent = extent self.n_neighbors = n_neighbors - self.cluster_labels = cluster_labels self.use_numba = use_numba self.n_jobs = n_jobs + self.progress_bar = progress_bar + + # Emit deprecation warnings for data params passed to __init__ + _locals = {"data": data, "distance_matrix": distance_matrix, + "neighbor_matrix": neighbor_matrix, + "cluster_labels": cluster_labels} + for param in self._DATA_PARAMS: + if _locals[param] is not None: + warnings.warn( + "Passing '{}' to __init__ is deprecated. " + "Pass it to fit() instead. This will raise an error " + "in a future version.".format(param), + FutureWarning, + stacklevel=2, + ) + + self.data = data + self.distance_matrix = distance_matrix + self.neighbor_matrix = neighbor_matrix + self.cluster_labels = cluster_labels + self.points_vector = None self.prob_distances = None self.prob_distances_ev = None self.norm_prob_local_outlier_factor = None self.local_outlier_probabilities = None self._objects = {} - self.progress_bar = progress_bar self.is_fit = False if self.use_numba is True and "numba" not in sys.modules: @@ -463,7 +484,6 @@ def __init__( ) self.n_jobs = 1 - self._validate_inputs() self._check_extent() """ @@ -954,17 +974,56 @@ def _local_outlier_probabilities(self, data_store: np.ndarray) -> np.ndarray: Public methods """ - def fit(self) -> "LocalOutlierProbability": + def _reset_state(self) -> None: + """Resets computed state to allow re-fitting with new data.""" + self.points_vector = None + self.prob_distances = None + self.prob_distances_ev = None + self.norm_prob_local_outlier_factor = None + self.local_outlier_probabilities = None + self._objects = {} + self.is_fit = False + + def fit( + self, + data=None, + distance_matrix=None, + neighbor_matrix=None, + cluster_labels=None, + ) -> "LocalOutlierProbability": """ Calculates the local outlier probability for each observation in the input data according to the input parameters extent, n_neighbors, and cluster_labels. + :param data: a Pandas DataFrame or Numpy array of float data + (optional, default None) + :param distance_matrix: a precomputed distance matrix of shape + (n_observations, n_neighbors) (optional, default None) + :param neighbor_matrix: a precomputed neighbor index matrix of shape + (n_observations, n_neighbors) (optional, default None) + :param cluster_labels: a numpy array or list of cluster assignments + w.r.t. each sample (optional, default None) :return: self, which contains the local outlier probabilities as self.local_outlier_probabilities. :raises ClusterSizeError: if any cluster is smaller than n_neighbors. :raises MissingValuesError: if data contains missing values. """ + self._reset_state() + + if data is not None: + self.data = data + self.distance_matrix = None + self.neighbor_matrix = None + if distance_matrix is not None: + self.distance_matrix = distance_matrix + if neighbor_matrix is not None: + self.neighbor_matrix = neighbor_matrix + if cluster_labels is not None: + self.cluster_labels = cluster_labels + + if self._validate_inputs() is False: + return self self._check_n_neighbors() self._check_cluster_size() if self.data is not None: @@ -1044,3 +1103,6 @@ def stream(self, x: np.ndarray) -> np.ndarray: self.cluster_labels = orig_cluster_labels return loop + + +LoOP = LocalOutlierProbability diff --git a/README.md b/README.md index 2a1ac49..7a5f7b6 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ scores in the range of [0,1] that are directly interpretable as the probability PyNomaly is a core library of [deepchecks](https://github.com/deepchecks/deepchecks), [OmniDocBench](https://github.com/opendatalab/OmniDocBench) and [pysad](https://github.com/selimfirat/pysad). [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![PyPi](https://img.shields.io/badge/pypi-0.4.0-blue.svg)](https://pypi.python.org/pypi/PyNomaly/0.4.0) +[![PyPi](https://img.shields.io/badge/pypi-1.0.0-blue.svg)](https://pypi.python.org/pypi/PyNomaly/1.0.0) [![Total Downloads](https://static.pepy.tech/badge/pynomaly)](https://pepy.tech/projects/pynomaly) [![Monthly Downloads](https://static.pepy.tech/badge/pynomaly/month)](https://pepy.tech/projects/pynomaly) ![Tests](https://github.com/vc1492a/PyNomaly/actions/workflows/tests.yml/badge.svg) @@ -89,19 +89,19 @@ conda install conda-forge::pynomaly Then you can do something like this: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data).fit() +from PyNomaly import LoOP +m = LoOP(n_neighbors=10).fit(data) scores = m.local_outlier_probabilities print(scores) ``` where *data* is a NxM (N rows, M columns; 2-dimensional) set of data as either a Pandas DataFrame or Numpy array. -LocalOutlierProbability sets the *extent* (in integer in value of 1, 2, or 3) and *n_neighbors* (must be greater than 0) parameters with the default +`LocalOutlierProbability` (also available as `LoOP`) sets the *extent* (an integer value of 1, 2, or 3) and *n_neighbors* (must be greater than 0) parameters with the default values of 3 and 10, respectively. You're free to set these parameters on your own as below: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -111,10 +111,10 @@ of varying density occur within the same set of data. When using *cluster_labels sample is calculated with respect to its cluster assignment. ```python -from PyNomaly import loop +from PyNomaly import LoOP from sklearn.cluster import DBSCAN db = DBSCAN(eps=0.6, min_samples=50).fit(data) -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20, cluster_labels=list(db.labels_)).fit() +m = LoOP(extent=2, n_neighbors=20).fit(data, cluster_labels=list(db.labels_)) scores = m.local_outlier_probabilities print(scores) ``` @@ -130,8 +130,8 @@ For large datasets, Numba's just-in-time (JIT) compilation can significantly speed up distance computation. Enable it with `use_numba=True`: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20, use_numba=True).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20, use_numba=True).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -140,11 +140,8 @@ To go further, set `n_jobs=-1` to enable Numba's thread-level parallelism (`prange`), which distributes work across all available CPU cores: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability( - data, extent=2, n_neighbors=20, - use_numba=True, n_jobs=-1 -).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20, use_numba=True, n_jobs=-1).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -172,8 +169,8 @@ You may choose to print progress bars _with or without_ the use of Numba by passing `progress_bar=True` to `LocalOutlierProbability()`: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, use_numba=True, n_jobs=-1, progress_bar=True).fit() +from PyNomaly import LoOP +m = LoOP(use_numba=True, n_jobs=-1, progress_bar=True).fit(data) ``` Progress bars are supported in both sequential and Numba execution modes. @@ -212,11 +209,10 @@ PyNomaly provides custom exceptions that can be caught and handled in your appli These exceptions are exported from the package and can be imported directly: ```python -from PyNomaly import loop -from PyNomaly.loop import ClusterSizeError, MissingValuesError +from PyNomaly import LoOP, ClusterSizeError, MissingValuesError try: - m = loop.LocalOutlierProbability(data, n_neighbors=50, cluster_labels=labels).fit() + m = LoOP(n_neighbors=50).fit(data, cluster_labels=labels) except ClusterSizeError: print("Reduce n_neighbors or use larger clusters.") except MissingValuesError: @@ -261,9 +257,9 @@ values for both *extent* (0.997) and *n_neighbors* (10). ```python db = DBSCAN(eps=0.9, min_samples=10).fit(iris) -m = loop.LocalOutlierProbability(iris).fit() +m = loop.LocalOutlierProbability().fit(iris) scores_noclust = m.local_outlier_probabilities -m_clust = loop.LocalOutlierProbability(iris, cluster_labels=list(db.labels_)).fit() +m_clust = loop.LocalOutlierProbability().fit(iris, cluster_labels=list(db.labels_)) scores_clust = m_clust.local_outlier_probabilities ``` @@ -348,7 +344,7 @@ data = np.array([ [421.5, 90.3, 50.0] ]) -scores = loop.LocalOutlierProbability(data, n_neighbors=3).fit().local_outlier_probabilities +scores = loop.LocalOutlierProbability(n_neighbors=3).fit(data).local_outlier_probabilities print(scores) ``` @@ -364,7 +360,7 @@ Similar to the above: ```python data = np.random.rand(100, 5) -scores = loop.LocalOutlierProbability(data).fit().local_outlier_probabilities +scores = loop.LocalOutlierProbability().fit(data).local_outlier_probabilities print(scores) ``` @@ -402,7 +398,7 @@ indices = np.delete(indices, 0, 1) distances = np.delete(distances, 0, 1) # Fit and return scores -m = loop.LocalOutlierProbability(distance_matrix=d, neighbor_matrix=idx, n_neighbors=n_neighbors+1).fit() +m = loop.LocalOutlierProbability(n_neighbors=n_neighbors+1).fit(distance_matrix=d, neighbor_matrix=idx) scores = m.local_outlier_probabilities ``` @@ -436,12 +432,12 @@ iris_test = iris.iloc[:, 0:4].tail(30) Fit to each set. ```python -m = loop.LocalOutlierProbability(iris).fit() +m = loop.LocalOutlierProbability().fit(iris) scores_noclust = m.local_outlier_probabilities iris['scores'] = scores_noclust -m_train = loop.LocalOutlierProbability(iris_train, n_neighbors=10) -m_train.fit() +m_train = loop.LocalOutlierProbability(n_neighbors=10) +m_train.fit(iris_train) iris_train_scores = m_train.local_outlier_probabilities ``` diff --git a/changelog.md b/changelog.md index 9b41565..80f809a 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,24 @@ All notable changes to PyNomaly will be documented in this Changelog. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 1.0.0 +### Changed +- **Breaking (with backward compatibility):** Adopted scikit-learn API conventions. + Data parameters (`data`, `distance_matrix`, `neighbor_matrix`, `cluster_labels`) + are now passed to `fit()` instead of `__init__()`. Configuration parameters + (`extent`, `n_neighbors`, `use_numba`, `n_jobs`, `progress_bar`) remain in + the constructor + ([Issue #7](https://github.com/vc1492a/PyNomaly/issues/7)). +- Passing data parameters to `__init__()` still works but emits a + `FutureWarning` and will be removed in a future version. +- Calling `fit()` multiple times with different data is now supported + (re-fitting resets internal state). +### Added +- `LoOP` class alias for `LocalOutlierProbability`, enabling + `from PyNomaly import LoOP` + ([Issue #5](https://github.com/vc1492a/PyNomaly/issues/5)). +- `__version__` is now exported from the package (`from PyNomaly import __version__`). + ## 0.4.0 ### Added - Parallel distance computation via Numba `prange` and the new `n_jobs` diff --git a/docs/changelog.md b/docs/changelog.md index 7214f7b..06b2624 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,24 @@ All notable changes to PyNomaly will be documented in this Changelog. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 1.0.0 +### Changed +- **Breaking (with backward compatibility):** Adopted scikit-learn API conventions. + Data parameters (`data`, `distance_matrix`, `neighbor_matrix`, `cluster_labels`) + are now passed to `fit()` instead of `__init__()`. Configuration parameters + (`extent`, `n_neighbors`, `use_numba`, `n_jobs`, `progress_bar`) remain in + the constructor + ([Issue #7](https://github.com/vc1492a/PyNomaly/issues/7)). +- Passing data parameters to `__init__()` still works but emits a + `FutureWarning` and will be removed in a future version. +- Calling `fit()` multiple times with different data is now supported + (re-fitting resets internal state). +### Added +- `LoOP` class alias for `LocalOutlierProbability`, enabling + `from PyNomaly import LoOP` + ([Issue #5](https://github.com/vc1492a/PyNomaly/issues/5)). +- `__version__` is now exported from the package (`from PyNomaly import __version__`). + ## 0.4.0 ### Added - Parallel distance computation via Numba `prange` and the new `n_jobs` diff --git a/docs/getting-started.md b/docs/getting-started.md index ca72734..e400ec2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -34,19 +34,19 @@ conda install conda-forge::pynomaly ## Quick Start ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data).fit() +from PyNomaly import LoOP +m = LoOP().fit(data) scores = m.local_outlier_probabilities print(scores) ``` where `data` is a NxM (N rows, M columns; 2-dimensional) set of data as either a Pandas DataFrame or Numpy array. -`LocalOutlierProbability` sets the `extent` (an integer value of 1, 2, or 3) and `n_neighbors` (must be greater than 0) parameters with the default values of 3 and 10, respectively: +`LocalOutlierProbability` (also available as `LoOP`) sets the `extent` (an integer value of 1, 2, or 3) and `n_neighbors` (must be greater than 0) parameters with the default values of 3 and 10, respectively: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -56,12 +56,10 @@ print(scores) This implementation of LoOP includes an optional `cluster_labels` parameter. This is useful in cases where regions of varying density occur within the same set of data. When using `cluster_labels`, the Local Outlier Probability of a sample is calculated with respect to its cluster assignment. ```python -from PyNomaly import loop +from PyNomaly import LoOP from sklearn.cluster import DBSCAN db = DBSCAN(eps=0.6, min_samples=50).fit(data) -m = loop.LocalOutlierProbability( - data, extent=2, n_neighbors=20, cluster_labels=list(db.labels_) -).fit() +m = LoOP(extent=2, n_neighbors=20).fit(data, cluster_labels=list(db.labels_)) scores = m.local_outlier_probabilities print(scores) ``` diff --git a/docs/user-guide.md b/docs/user-guide.md index 2933e99..496c1cf 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -29,7 +29,7 @@ For large datasets, Numba's just-in-time (JIT) compilation can significantly spe ```python from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20, use_numba=True).fit() +m = loop.LocalOutlierProbability(extent=2, n_neighbors=20, use_numba=True).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -39,9 +39,8 @@ To go further, set `n_jobs=-1` to enable Numba's thread-level parallelism (`pran ```python from PyNomaly import loop m = loop.LocalOutlierProbability( - data, extent=2, n_neighbors=20, - use_numba=True, n_jobs=-1 -).fit() + extent=2, n_neighbors=20, use_numba=True, n_jobs=-1 +).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -61,7 +60,7 @@ You may choose to print progress bars _with or without_ the use of Numba by pass ```python from PyNomaly import loop -m = loop.LocalOutlierProbability(data, use_numba=True, n_jobs=-1, progress_bar=True).fit() +m = loop.LocalOutlierProbability(use_numba=True, n_jobs=-1, progress_bar=True).fit(data) ``` Progress bars are supported in both sequential and Numba execution modes. @@ -83,7 +82,7 @@ data = np.array([ [421.5, 90.3, 50.0] ]) -scores = loop.LocalOutlierProbability(data, n_neighbors=3).fit().local_outlier_probabilities +scores = loop.LocalOutlierProbability(n_neighbors=3).fit(data).local_outlier_probabilities print(scores) ``` @@ -123,9 +122,9 @@ d, idx = neigh.kneighbors(data, return_distance=True) indices = np.delete(indices, 0, 1) distances = np.delete(distances, 0, 1) -m = loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, n_neighbors=n_neighbors+1 -).fit() +m = loop.LocalOutlierProbability(n_neighbors=n_neighbors+1).fit( + distance_matrix=d, neighbor_matrix=idx +) scores = m.local_outlier_probabilities ``` @@ -150,8 +149,8 @@ iris_test = iris.iloc[:, 0:4].tail(30) Fit the model on training data: ```python -m_train = loop.LocalOutlierProbability(iris_train, n_neighbors=10) -m_train.fit() +m_train = loop.LocalOutlierProbability(n_neighbors=10) +m_train.fit(iris_train) iris_train_scores = m_train.local_outlier_probabilities ``` @@ -189,13 +188,10 @@ PyNomaly provides custom exceptions that can be caught and handled in your appli These exceptions are exported from the package and can be imported directly: ```python -from PyNomaly import loop -from PyNomaly.loop import ClusterSizeError, MissingValuesError +from PyNomaly import LoOP, ClusterSizeError, MissingValuesError try: - m = loop.LocalOutlierProbability( - data, n_neighbors=50, cluster_labels=labels - ).fit() + m = LoOP(n_neighbors=50).fit(data, cluster_labels=labels) except ClusterSizeError: print("Reduce n_neighbors or use larger clusters.") except MissingValuesError: diff --git a/setup.py b/setup.py index a73829b..e701fc2 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='PyNomaly', packages=['PyNomaly'], - version='0.4.0', + version='1.0.0', description='A Python 3 implementation of LoOP: Local Outlier ' 'Probabilities, a local density based outlier detection ' 'method providing an outlier score in the range of [0,1].', @@ -16,7 +16,7 @@ long_description=long_description, long_description_content_type='text/markdown', url='https://github.com/vc1492a/PyNomaly', - download_url='https://github.com/vc1492a/PyNomaly/archive/0.4.0.tar.gz', + download_url='https://github.com/vc1492a/PyNomaly/archive/1.0.0.tar.gz', keywords=['outlier', 'anomaly', 'detection', 'machine', 'learning', 'probability'], classifiers=[], diff --git a/tests/test_loop.py b/tests/test_loop.py index 939a686..138522b 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -6,6 +6,7 @@ import logging from typing import Tuple +import warnings import numpy as np from numpy.testing import assert_array_equal, assert_array_almost_equal import pandas as pd @@ -164,8 +165,8 @@ def test_loop(X_n8) -> None: :return: None """ # Test LocalOutlierProbability: - clf = loop.LocalOutlierProbability(X_n8, n_neighbors=5, use_numba=NUMBA) - score = clf.fit().local_outlier_probabilities + clf = loop.LocalOutlierProbability(n_neighbors=5, use_numba=NUMBA) + score = clf.fit(X_n8).local_outlier_probabilities share_outlier = 2.0 / 8.0 predictions = [-1 if s > share_outlier else 1 for s in score] assert_array_equal(predictions, 6 * [1] + 2 * [-1]) @@ -177,8 +178,8 @@ def test_loop(X_n8) -> None: X_df = pd.DataFrame(X_n8) # Test LocalOutlierProbability: - clf = loop.LocalOutlierProbability(X_df, n_neighbors=5, use_numba=NUMBA) - score = clf.fit().local_outlier_probabilities + clf = loop.LocalOutlierProbability(n_neighbors=5, use_numba=NUMBA) + score = clf.fit(X_df).local_outlier_probabilities share_outlier = 2.0 / 8.0 predictions = [-1 if s > share_outlier else 1 for s in score] assert_array_equal(predictions, 6 * [1] + 2 * [-1]) @@ -194,7 +195,7 @@ def test_regression(X_n20_scores) -> None: the same result given the same dataset """ input_data, expected_scores = X_n20_scores - clf = loop.LocalOutlierProbability(input_data).fit() + clf = loop.LocalOutlierProbability().fit(input_data) scores = clf.local_outlier_probabilities assert_array_almost_equal(scores, expected_scores, 6) @@ -214,15 +215,13 @@ def test_loop_performance(X_n120) -> None: # fit the model clf = loop.LocalOutlierProbability( - X_test, n_neighbors=X_test.shape[0] - 1, - # test the progress bar progress_bar=True, use_numba=NUMBA, ) # predict scores (the lower, the more normal) - score = clf.fit().local_outlier_probabilities + score = clf.fit(X_test).local_outlier_probabilities share_outlier = X_outliers.shape[0] / X_test.shape[0] X_pred = [-1 if s > share_outlier else 1 for s in score] @@ -239,9 +238,10 @@ def test_input_nodata(X_n140_outliers) -> None: """ with pytest.warns(UserWarning) as record: # attempt to fit loop without data or a distance matrix - loop.LocalOutlierProbability( + clf = loop.LocalOutlierProbability( n_neighbors=X_n140_outliers.shape[0] - 1, use_numba=NUMBA ) + clf.fit() # check that only one warning was raised assert len(record) == 1 @@ -259,7 +259,6 @@ def test_input_incorrect_type(X_n140_outliers) -> None: with pytest.warns(UserWarning) as record: # attempt to fit loop with a string input for n_neighbors loop.LocalOutlierProbability( - X_n140_outliers, n_neighbors=str(X_n140_outliers.shape[0] - 1), use_numba=NUMBA, ) @@ -281,11 +280,11 @@ def test_input_neighbor_zero(X_n120) -> None: :param X_n120: A pytest Fixture that generates 120 observations. :return: None """ - clf = loop.LocalOutlierProbability(X_n120, n_neighbors=0, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=0, use_numba=NUMBA) with pytest.warns(UserWarning) as record: # attempt to fit loop with a 0 neighbor count - clf.fit() + clf.fit(X_n120) # check that only one warning was raised assert len(record) == 1 @@ -310,7 +309,8 @@ def test_input_distonly(X_n120) -> None: with pytest.warns(UserWarning) as record: # attempt to fit loop with a distance matrix and no neighbor matrix - loop.LocalOutlierProbability(distance_matrix=d, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + clf.fit(distance_matrix=d) # check that only one warning was raised assert len(record) == 1 @@ -336,7 +336,8 @@ def test_input_neighboronly(X_n120) -> None: with pytest.warns(UserWarning) as record: # attempt to fit loop with a neighbor matrix and no distance matrix - loop.LocalOutlierProbability(neighbor_matrix=idx, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + clf.fit(neighbor_matrix=idx) # check that only one warning was raised assert len(record) == 1 @@ -358,9 +359,8 @@ def test_input_too_many(X_n120) -> None: with pytest.warns(UserWarning) as record: # attempt to fit loop with data and a distance matrix - loop.LocalOutlierProbability( - X_n120, distance_matrix=d, neighbor_matrix=idx, use_numba=NUMBA - ) + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + clf.fit(X_n120, distance_matrix=d, neighbor_matrix=idx) # check that only one warning was raised assert len(record) == 1 @@ -391,9 +391,8 @@ def test_distance_neighbor_shape_mismatch(X_n120) -> None: with pytest.warns(UserWarning) as record: # attempt to fit loop with a mismatch in shapes - loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx_2, n_neighbors=5, use_numba=NUMBA - ) + clf = loop.LocalOutlierProbability(n_neighbors=5, use_numba=NUMBA) + clf.fit(distance_matrix=d, neighbor_matrix=idx_2) # check that only one warning was raised assert len(record) == 1 @@ -418,9 +417,8 @@ def test_input_neighbor_mismatch(X_n120) -> None: with pytest.warns(UserWarning) as record: # attempt to fit loop with a neighbor size mismatch - loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, n_neighbors=10, use_numba=NUMBA - ) + clf = loop.LocalOutlierProbability(n_neighbors=10, use_numba=NUMBA) + clf.fit(distance_matrix=d, neighbor_matrix=idx) # check that only one warning was raised assert len(record) == 1 @@ -446,12 +444,10 @@ def test_loop_dist_matrix(X_n120) -> None: d, idx = neigh.kneighbors(X_n120, n_neighbors=10, return_distance=True) # fit loop using data and distance matrix - clf1 = loop.LocalOutlierProbability(X_n120, use_numba=NUMBA) - clf2 = loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, use_numba=NUMBA - ) - scores1 = clf1.fit().local_outlier_probabilities - scores2 = clf2.fit().local_outlier_probabilities + clf1 = loop.LocalOutlierProbability(use_numba=NUMBA) + clf2 = loop.LocalOutlierProbability(use_numba=NUMBA) + scores1 = clf1.fit(X_n120).local_outlier_probabilities + scores2 = clf2.fit(distance_matrix=d, neighbor_matrix=idx).local_outlier_probabilities # compare the agreement between the results assert np.abs(scores2 - scores1).all() <= 0.1 @@ -466,14 +462,14 @@ def test_lambda_values(X_n140_outliers) -> None: :return: None """ # Fit the model with different extent (lambda) values - clf1 = loop.LocalOutlierProbability(X_n140_outliers, extent=1, use_numba=NUMBA) - clf2 = loop.LocalOutlierProbability(X_n140_outliers, extent=2, use_numba=NUMBA) - clf3 = loop.LocalOutlierProbability(X_n140_outliers, extent=3, use_numba=NUMBA) + clf1 = loop.LocalOutlierProbability(extent=1, use_numba=NUMBA) + clf2 = loop.LocalOutlierProbability(extent=2, use_numba=NUMBA) + clf3 = loop.LocalOutlierProbability(extent=3, use_numba=NUMBA) # predict scores (the lower, the more normal) - score1 = clf1.fit().local_outlier_probabilities - score2 = clf2.fit().local_outlier_probabilities - score3 = clf3.fit().local_outlier_probabilities + score1 = clf1.fit(X_n140_outliers).local_outlier_probabilities + score2 = clf2.fit(X_n140_outliers).local_outlier_probabilities + score3 = clf3.fit(X_n140_outliers).local_outlier_probabilities # Get the mean of all the scores score_mean1 = np.mean(score1) @@ -494,7 +490,7 @@ def test_parameters(X_n120) -> None: :return: None """ # fit the model - clf = loop.LocalOutlierProbability(X_n120, use_numba=NUMBA).fit() + clf = loop.LocalOutlierProbability(use_numba=NUMBA).fit(X_n120) # check that the model has attributes post fit assert hasattr(clf, "n_neighbors") and clf.n_neighbors is not None @@ -520,13 +516,13 @@ def test_n_neighbors() -> None: :return: None """ X = iris.data - clf = loop.LocalOutlierProbability(X, n_neighbors=500, use_numba=NUMBA).fit() + clf = loop.LocalOutlierProbability(n_neighbors=500, use_numba=NUMBA).fit(X) assert clf.n_neighbors == X.shape[0] - 1 - clf = loop.LocalOutlierProbability(X, n_neighbors=500, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=500, use_numba=NUMBA) with pytest.warns(UserWarning) as record: - clf.fit() + clf.fit(X) # check that only one warning was raised assert len(record) == 1 @@ -541,10 +537,10 @@ def test_extent() -> None: :return: None """ X = np.array([[1, 1], [1, 0]]) - clf = loop.LocalOutlierProbability(X, n_neighbors=2, extent=4, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=2, extent=4, use_numba=NUMBA) with pytest.warns(UserWarning) as record: - clf.fit() + clf.fit(X) # check that only one warning was raised assert len(record) == 1 @@ -558,13 +554,15 @@ def test_data_format() -> None: :return: None """ X = [1.3, 1.1, 0.9, 1.4, 1.5, 3.2] - clf = loop.LocalOutlierProbability(X, n_neighbors=3, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=3, use_numba=NUMBA) with pytest.warns(UserWarning) as record: - clf.fit() + clf.fit(X) - # check that only one warning was raised - assert len(record) == 1 + assert any( + "Provided data or distance matrix must be in ndarray" in r.message.args[0] + for r in record + ) def test_missing_values() -> None: @@ -574,10 +572,10 @@ def test_missing_values() -> None: :return: None """ X = np.array([1.3, 1.1, 0.9, 1.4, 1.5, np.nan, 3.2]) - clf = loop.LocalOutlierProbability(X, n_neighbors=3, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=3, use_numba=NUMBA) with pytest.raises(MissingValuesError) as record: - clf.fit() + clf.fit(X) # check that the message matches assert ( @@ -599,11 +597,11 @@ def test_small_cluster_size(X_n140_outliers) -> None: cluster_labels = a + b clf = loop.LocalOutlierProbability( - X_n140_outliers, n_neighbors=50, cluster_labels=cluster_labels, use_numba=NUMBA + n_neighbors=50, use_numba=NUMBA ) with pytest.raises(ClusterSizeError) as record: - clf.fit() + clf.fit(X_n140_outliers, cluster_labels=cluster_labels) # check that the message matches assert ( @@ -625,7 +623,8 @@ def test_stream_fit(X_n140_outliers) -> None: # Fit the model X_train = X_n140_outliers[0:138] X_test = X_n140_outliers[139] - clf = loop.LocalOutlierProbability(X_train, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + clf.data = X_train with pytest.warns(UserWarning) as record: clf.stream(X_test) @@ -655,10 +654,10 @@ def test_stream_distance(X_n140_outliers) -> None: d, idx = neigh.kneighbors(X_train, n_neighbors=10, return_distance=True) # Fit the models in standard and distance matrix form - m = loop.LocalOutlierProbability(X_train, use_numba=NUMBA).fit() + m = loop.LocalOutlierProbability(use_numba=NUMBA).fit(X_train) m_dist = loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, use_numba=NUMBA - ).fit() + use_numba=NUMBA + ).fit(distance_matrix=d, neighbor_matrix=idx) # Collect the scores X_test_scores = [] @@ -694,8 +693,8 @@ def test_stream_cluster(X_n140_outliers) -> None: X_train = X_n140_outliers[0:138] X_test = X_n140_outliers[139] clf = loop.LocalOutlierProbability( - X_train, cluster_labels=cluster_labels, use_numba=NUMBA - ).fit() + use_numba=NUMBA + ).fit(X_train, cluster_labels=cluster_labels) with pytest.warns(UserWarning) as record: clf.stream(X_test) @@ -722,11 +721,11 @@ def test_stream_performance(X_n140_outliers) -> None: X_test = X_n140_outliers[100:140] # Fit the models in standard and stream form - m = loop.LocalOutlierProbability(X_n140_outliers, use_numba=NUMBA).fit() + m = loop.LocalOutlierProbability(use_numba=NUMBA).fit(X_n140_outliers) scores_noclust = m.local_outlier_probabilities - m_train = loop.LocalOutlierProbability(X_train, use_numba=NUMBA) - m_train.fit() + m_train = loop.LocalOutlierProbability(use_numba=NUMBA) + m_train.fit(X_train) X_train_scores = m_train.local_outlier_probabilities X_test_scores = [] @@ -751,7 +750,7 @@ def test_progress_bar(X_n8) -> None: """ # attempt to use the progress bar on a small number of observations - loop.LocalOutlierProbability(X_n8, use_numba=NUMBA, progress_bar=True).fit() + loop.LocalOutlierProbability(use_numba=NUMBA, progress_bar=True).fit(X_n8) def test_data_flipping() -> None: @@ -765,16 +764,14 @@ def test_data_flipping() -> None: np.random.normal(2, 1, [n, 2]), np.random.normal(8, 1, [n, 2]), axis=0 ) clus = np.append(np.ones(n), 2 * np.ones(n)).tolist() - model = loop.LocalOutlierProbability(data, n_neighbors=5, cluster_labels=clus) - fit = model.fit() + model = loop.LocalOutlierProbability(n_neighbors=5) + fit = model.fit(data, cluster_labels=clus) res = fit.local_outlier_probabilities data_flipped = np.flipud(data) clus_flipped = np.flipud(clus).tolist() - model2 = loop.LocalOutlierProbability( - data_flipped, n_neighbors=5, cluster_labels=clus_flipped - ) - fit2 = model2.fit() + model2 = loop.LocalOutlierProbability(n_neighbors=5) + fit2 = model2.fit(data_flipped, cluster_labels=clus_flipped) res2 = np.flipud(fit2.local_outlier_probabilities) assert_array_almost_equal(res, res2, decimal=6) @@ -801,12 +798,12 @@ def test_distance_matrix_consistency(X_n120) -> None: distances = np.delete(distances, 0, 1) # Fit LoOP with and without distance matrix - clf_data = loop.LocalOutlierProbability(X_n120, n_neighbors=10) - clf_dist = loop.LocalOutlierProbability(distance_matrix=distances, neighbor_matrix=indices, n_neighbors=11) + clf_data = loop.LocalOutlierProbability(n_neighbors=10) + clf_dist = loop.LocalOutlierProbability(n_neighbors=10) # Attempt to retrieve scores and check types - scores_data = clf_data.fit().local_outlier_probabilities - scores_dist = clf_dist.fit().local_outlier_probabilities + scores_data = clf_data.fit(X_n120).local_outlier_probabilities + scores_dist = clf_dist.fit(distance_matrix=distances, neighbor_matrix=indices).local_outlier_probabilities # Debugging prints to investigate types and contents print("Type of scores_data:", type(scores_data)) @@ -835,8 +832,8 @@ def test_vectorized_1d_data() -> None: the ndim == 1 reshape branch in _distances_vectorized. """ X = np.array([1.0, 2.0, 3.0, 10.0, 11.0, 12.0, 50.0]) - clf = loop.LocalOutlierProbability(X, n_neighbors=3) - scores = clf.fit().local_outlier_probabilities + clf = loop.LocalOutlierProbability(n_neighbors=3) + scores = clf.fit(X).local_outlier_probabilities assert scores is not None assert len(scores) == len(X) assert scores[-1] > 0 @@ -849,9 +846,9 @@ def test_n_jobs_without_numba_warns(X_n120) -> None: """ with pytest.warns(UserWarning, match="n_jobs > 1 requires use_numba=True"): clf = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, n_jobs=2 + n_neighbors=10, n_jobs=2 ) - scores = clf.fit().local_outlier_probabilities + scores = clf.fit(X_n120).local_outlier_probabilities assert scores is not None assert len(scores) == len(X_n120) @@ -861,10 +858,8 @@ def test_n_jobs_negative_two() -> None: """ Tests that n_jobs=-2 (invalid) produces a warning and defaults to 1. """ - X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) - with pytest.warns(UserWarning) as record: - clf = loop.LocalOutlierProbability(X, n_neighbors=2, n_jobs=-2) + clf = loop.LocalOutlierProbability(n_neighbors=2, n_jobs=-2) messages = [r.message.args[0] for r in record] assert any("n_jobs must be -1 or a positive integer" in m for m in messages) @@ -877,9 +872,9 @@ def test_vectorized_progress_bar_single_cluster(X_n120) -> None: on single-cluster data. """ clf = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, n_jobs=1, progress_bar=True + n_neighbors=10, n_jobs=1, progress_bar=True ) - scores = clf.fit().local_outlier_probabilities + scores = clf.fit(X_n120).local_outlier_probabilities assert scores is not None assert len(scores) == len(X_n120) @@ -888,10 +883,8 @@ def test_n_jobs_invalid() -> None: """ Tests that invalid n_jobs values produce a warning and default to 1. """ - X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) - with pytest.warns(UserWarning) as record: - clf = loop.LocalOutlierProbability(X, n_neighbors=2, n_jobs=0) + clf = loop.LocalOutlierProbability(n_neighbors=2, n_jobs=0) messages = [r.message.args[0] for r in record] assert any("n_jobs must be -1 or a positive integer" in m for m in messages) @@ -915,11 +908,11 @@ def test_numba_sequential_equivalence(X_n8) -> None: Tests that use_numba=True with n_jobs=1 produces equivalent results to the default vectorized path. """ - clf_vec = loop.LocalOutlierProbability(X_n8, n_neighbors=5, use_numba=False) - scores_vec = clf_vec.fit().local_outlier_probabilities + clf_vec = loop.LocalOutlierProbability(n_neighbors=5, use_numba=False) + scores_vec = clf_vec.fit(X_n8).local_outlier_probabilities - clf_numba = loop.LocalOutlierProbability(X_n8, n_neighbors=5, use_numba=True) - scores_numba = clf_numba.fit().local_outlier_probabilities + clf_numba = loop.LocalOutlierProbability(n_neighbors=5, use_numba=True) + scores_numba = clf_numba.fit(X_n8).local_outlier_probabilities assert_array_almost_equal(scores_vec, scores_numba, decimal=6) @@ -935,16 +928,18 @@ def test_numba_parallel_equivalence(X_n140_outliers) -> None: cluster_labels = a + b clf_seq = loop.LocalOutlierProbability( - X_n140_outliers, n_neighbors=10, cluster_labels=cluster_labels, - use_numba=True, n_jobs=1 + n_neighbors=10, use_numba=True, n_jobs=1 ) - scores_seq = clf_seq.fit().local_outlier_probabilities + scores_seq = clf_seq.fit( + X_n140_outliers, cluster_labels=cluster_labels + ).local_outlier_probabilities clf_par = loop.LocalOutlierProbability( - X_n140_outliers, n_neighbors=10, cluster_labels=cluster_labels, - use_numba=True, n_jobs=2 + n_neighbors=10, use_numba=True, n_jobs=2 ) - scores_par = clf_par.fit().local_outlier_probabilities + scores_par = clf_par.fit( + X_n140_outliers, cluster_labels=cluster_labels + ).local_outlier_probabilities assert_array_almost_equal(scores_seq, scores_par, decimal=10) @@ -955,9 +950,9 @@ def test_numba_with_progress_bar(X_n120) -> None: Tests that the Numba path works with the progress bar enabled. """ clf = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, use_numba=True, progress_bar=True + n_neighbors=10, use_numba=True, progress_bar=True ) - scores = clf.fit().local_outlier_probabilities + scores = clf.fit(X_n120).local_outlier_probabilities assert scores is not None assert len(scores) == len(X_n120) @@ -969,13 +964,80 @@ def test_numba_prange_single_cluster(X_n120) -> None: and produces equivalent results to the sequential path. """ clf_seq = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, use_numba=True, n_jobs=1 + n_neighbors=10, use_numba=True, n_jobs=1 ) - scores_seq = clf_seq.fit().local_outlier_probabilities + scores_seq = clf_seq.fit(X_n120).local_outlier_probabilities clf_par = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, use_numba=True, n_jobs=-1 + n_neighbors=10, use_numba=True, n_jobs=-1 ) - scores_par = clf_par.fit().local_outlier_probabilities + scores_par = clf_par.fit(X_n120).local_outlier_probabilities assert_array_almost_equal(scores_seq, scores_par, decimal=6) + + +# --- scikit-learn API convention tests (1.0.0) --- + + +def test_numerical_equivalence_old_style(X_n8) -> None: + """ + Tests that old-style (deprecated) and new-style API produce identical + results. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + clf_old = loop.LocalOutlierProbability(data=X_n8, n_neighbors=5) + scores_old = clf_old.fit().local_outlier_probabilities + + clf_new = loop.LocalOutlierProbability(n_neighbors=5) + scores_new = clf_new.fit(X_n8).local_outlier_probabilities + + assert_array_equal(scores_old, scores_new) + + +def test_old_style_future_warning(X_n8) -> None: + """ + Tests that passing data to __init__ emits a FutureWarning. + """ + with pytest.warns(FutureWarning, match="Passing 'data' to __init__"): + loop.LocalOutlierProbability(data=X_n8, n_neighbors=5) + + +def test_fit_overrides_init(X_n8, X_n120) -> None: + """ + Tests that data passed to fit() takes precedence over data passed + to __init__. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + clf = loop.LocalOutlierProbability(data=X_n8, n_neighbors=5) + + clf.fit(X_n120) + scores = clf.local_outlier_probabilities + assert len(scores) == 120 + + +def test_refit(X_n8, X_n120) -> None: + """ + Tests that calling fit() twice with different data produces correct + results each time. + """ + clf = loop.LocalOutlierProbability(n_neighbors=5) + + clf.fit(X_n8) + scores_first = clf.local_outlier_probabilities.copy() + assert len(scores_first) == 8 + + clf.fit(X_n120) + scores_second = clf.local_outlier_probabilities + assert len(scores_second) == 120 + assert clf.is_fit is True + + +def test_loop_alias_import() -> None: + """ + Tests that LoOP alias can be imported and is the same class. + """ + from PyNomaly import LoOP + from PyNomaly import LocalOutlierProbability + assert LoOP is LocalOutlierProbability From 5eeeff5171aaa129211b4cfdaa1fd3d2bb373755 Mon Sep 17 00:00:00 2001 From: vc1492a Date: Tue, 16 Jun 2026 09:55:38 -0700 Subject: [PATCH 2/4] refactor: split loop.py into focused modules using mixins Extract the monolithic loop.py (~1100 lines) into responsibility-based modules while preserving the identical public API: - exceptions.py: exception hierarchy (PyNomalyError, ValidationError, etc.) - _utils.py: Utils class (progress bar) - _validation.py: ValidationMixin + @accepts decorator - _distance.py: DistanceMixin + Numba JIT kernels - _pipeline.py: PipelineMixin (LoOP scoring math) - loop.py: slim orchestrator composing the class from mixins No API changes. All existing import patterns continue to work. Co-authored-by: Cursor --- PyNomaly/__init__.py | 4 +- PyNomaly/_distance.py | 265 ++++++++++++ PyNomaly/_pipeline.py | 303 ++++++++++++++ PyNomaly/_utils.py | 33 ++ PyNomaly/_validation.py | 248 ++++++++++++ PyNomaly/exceptions.py | 22 + PyNomaly/loop.py | 869 +--------------------------------------- 7 files changed, 888 insertions(+), 856 deletions(-) create mode 100644 PyNomaly/_distance.py create mode 100644 PyNomaly/_pipeline.py create mode 100644 PyNomaly/_utils.py create mode 100644 PyNomaly/_validation.py create mode 100644 PyNomaly/exceptions.py diff --git a/PyNomaly/__init__.py b/PyNomaly/__init__.py index 9b63740..0e94304 100644 --- a/PyNomaly/__init__.py +++ b/PyNomaly/__init__.py @@ -4,11 +4,13 @@ from PyNomaly.loop import ( LocalOutlierProbability, LoOP, + __version__, +) +from PyNomaly.exceptions import ( PyNomalyError, ValidationError, ClusterSizeError, MissingValuesError, - __version__, ) __all__ = [ diff --git a/PyNomaly/_distance.py b/PyNomaly/_distance.py new file mode 100644 index 0000000..cc9ed58 --- /dev/null +++ b/PyNomaly/_distance.py @@ -0,0 +1,265 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +import numpy as np +import os +import sys +from typing import Tuple +import warnings + +from PyNomaly._utils import Utils + +try: + from scipy.spatial.distance import cdist as _scipy_cdist +except ImportError: + _scipy_cdist = None + +try: + import numba + + def _numba_distance_kernel_parallel(clust_points_vector, n_neighbors): + n = clust_points_vector.shape[0] + d_features = clust_points_vector.shape[1] + local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) + local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) + for i in numba.prange(n): + for j in range(n): + if i == j: + continue + d = 0.0 + for f in range(d_features): + diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] + d += diff_val * diff_val + d = d ** 0.5 + idx_max = 0 + for k in range(1, n_neighbors): + if local_distances[i, k] > local_distances[i, idx_max]: + idx_max = k + if d < local_distances[i, idx_max]: + local_distances[i, idx_max] = d + local_indexes[i, idx_max] = j + return local_distances, local_indexes + + _numba_kernel_parallel = numba.jit( + _numba_distance_kernel_parallel, nopython=True, parallel=True, cache=True + ) + + def _numba_distance_kernel_sequential(clust_points_vector, n_neighbors): + n = clust_points_vector.shape[0] + d_features = clust_points_vector.shape[1] + local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) + local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) + for i in range(n): + for j in range(n): + if i == j: + continue + d = 0.0 + for f in range(d_features): + diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] + d += diff_val * diff_val + d = d ** 0.5 + idx_max = 0 + for k in range(1, n_neighbors): + if local_distances[i, k] > local_distances[i, idx_max]: + idx_max = k + if d < local_distances[i, idx_max]: + local_distances[i, idx_max] = d + local_indexes[i, idx_max] = j + return local_distances, local_indexes + + _numba_kernel_sequential = numba.jit( + _numba_distance_kernel_sequential, nopython=True, cache=True + ) + +except ImportError: + pass + + +class DistanceMixin: + """Mixin providing distance computation methods for LocalOutlierProbability.""" + + @staticmethod + def _euclidean(vector1: np.ndarray, vector2: np.ndarray) -> np.ndarray: + """ + Calculates the euclidean distance between two observations in the + input data. + :param vector1: a numpy array corresponding to observation 1. + :param vector2: a numpy array corresponding to observation 2. + :return: the euclidean distance between the two observations. + """ + diff = vector1 - vector2 + return np.dot(diff, diff) ** 0.5 + + def _assign_distances(self, data_store: np.ndarray) -> np.ndarray: + """ + Takes a distance matrix, produced by _distances or provided through + user input, and assigns distances for each observation to the storage + matrix, data_store. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + for vec, cluster_id in zip( + range(self.distance_matrix.shape[0]), self._cluster_labels() + ): + data_store[vec][0] = cluster_id + data_store[vec][1] = self.distance_matrix[vec] + data_store[vec][2] = self.neighbor_matrix[vec] + return data_store + + @staticmethod + def _compute_distance_and_neighbor_matrix( + clust_points_vector: np.ndarray, + indices: np.ndarray, + distances: np.ndarray, + indexes: np.ndarray, + ) -> Tuple[np.ndarray, np.ndarray, int]: + """ + This helper method provides the heavy lifting for the _distances + method and is only intended for use therein. The code has been + written so that it can make full use of Numba's jit capabilities if + desired. + """ + for i in range(clust_points_vector.shape[0]): + for j in range(i + 1, clust_points_vector.shape[0]): + global_i = indices[0][i] + global_j = indices[0][j] + + diff = clust_points_vector[i] - clust_points_vector[j] + d = np.dot(diff, diff) ** 0.5 + + idx_max = distances[global_i].argmax() + if d < distances[global_i][idx_max]: + distances[global_i][idx_max] = d + indexes[global_i][idx_max] = global_j + + idx_max = distances[global_j].argmax() + if d < distances[global_j][idx_max]: + distances[global_j][idx_max] = d + indexes[global_j][idx_max] = global_i + + yield distances, indexes, i + + def _distances_vectorized( + self, clusters, distances, indexes, progress_bar + ) -> None: + """Vectorized kNN distance computation with chunked progress.""" + progress = "=" + total_points = sum(cv.shape[0] for cv, _ in clusters) + completed = 0 + + for clust_points_vector, global_indices in clusters: + n = clust_points_vector.shape[0] + + if clust_points_vector.ndim == 1: + clust_points_vector = clust_points_vector.reshape(-1, 1) + + chunk_size = min(256, n) + for chunk_start in range(0, n, chunk_size): + chunk_end = min(chunk_start + chunk_size, n) + chunk = clust_points_vector[chunk_start:chunk_end] + + if _scipy_cdist is not None: + dist = _scipy_cdist( + chunk, clust_points_vector, metric="euclidean" + ) + else: + diff = ( + chunk[:, np.newaxis, :] + - clust_points_vector[np.newaxis, :, :] + ) + dist = np.sqrt((diff ** 2).sum(axis=2)) + + row_idx = np.arange(chunk_end - chunk_start) + dist[row_idx, row_idx + chunk_start] = np.inf + + knn_idx = np.argpartition(dist, self.n_neighbors, axis=1)[ + :, : self.n_neighbors + ] + knn_dists = np.take_along_axis(dist, knn_idx, axis=1) + + chunk_global = global_indices[chunk_start:chunk_end] + distances[chunk_global] = knn_dists + indexes[chunk_global] = global_indices[knn_idx] + + completed += chunk_end - chunk_start + if progress_bar: + progress = Utils.emit_progress_bar( + progress, completed, total_points + ) + + def _distances_numba( + self, clusters, distances, indexes, progress_bar, parallel=False + ) -> None: + """Numba-accelerated distance computation.""" + progress = "=" + kernel = _numba_kernel_parallel if parallel else _numba_kernel_sequential + + for idx, (clust_points_vector, global_indices) in enumerate(clusters): + if clust_points_vector.ndim == 1: + clust_points_vector = clust_points_vector.reshape(-1, 1) + + local_dists, local_idxs = kernel( + clust_points_vector.astype(np.float64), self.n_neighbors + ) + + distances[global_indices] = local_dists + indexes[global_indices] = global_indices[local_idxs] + + if progress_bar: + progress = Utils.emit_progress_bar( + progress, idx + 1, len(clusters) + ) + + def _distances(self, progress_bar: bool = False) -> None: + """ + Provides the distances between each observation and it's closest + neighbors. When input data is provided, calculates the euclidean + distance between every observation. Otherwise, the user-provided + distance matrix is used. + :return: the updated storage matrix that collects information on + each observation. + """ + distances = np.full( + [self._n_observations(), self.n_neighbors], 9e10, dtype=float + ) + indexes = np.full( + [self._n_observations(), self.n_neighbors], 9e10, dtype=float + ) + self.points_vector = self._convert_to_array(self.data) + + cluster_labels = self._cluster_labels() + cluster_ids = sorted(set(cluster_labels)) + + clusters = [] + for cluster_id in cluster_ids: + indices = np.where(cluster_labels == cluster_id) + clust_points_vector = np.array( + self.points_vector.take(indices, axis=0)[0], dtype=np.float64 + ) + clusters.append((clust_points_vector, indices[0])) + + n_jobs = self.n_jobs + if n_jobs == -1: + n_jobs = os.cpu_count() or 1 + + if self.use_numba: + self._distances_numba( + clusters, distances, indexes, progress_bar, + parallel=(n_jobs > 1) + ) + else: + if n_jobs > 1: + warnings.warn( + "n_jobs > 1 requires use_numba=True for parallel " + "processing. Install Numba and set use_numba=True " + "to enable parallelism. Falling back to sequential.", + UserWarning, + ) + self._distances_vectorized( + clusters, distances, indexes, progress_bar + ) + + self.distance_matrix = distances + self.neighbor_matrix = indexes diff --git a/PyNomaly/_pipeline.py b/PyNomaly/_pipeline.py new file mode 100644 index 0000000..379d8fb --- /dev/null +++ b/PyNomaly/_pipeline.py @@ -0,0 +1,303 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +from math import erf, sqrt +import numpy as np + +try: + from scipy.special import erf as _scipy_erf +except ImportError: + _scipy_erf = None + + +class PipelineMixin: + """Mixin providing the LoOP scoring pipeline for LocalOutlierProbability.""" + + @staticmethod + def _standard_distance(cardinality: float, sum_squared_distance: float) -> float: + """ + Calculates the standard distance of an observation. + :param cardinality: the cardinality of the input observation. + :param sum_squared_distance: the sum squared distance between all + neighbors of the input observation. + :return: the standard distance. + """ + division_result = sum_squared_distance / cardinality + st_dist = sqrt(division_result) + return st_dist + + @staticmethod + def _prob_distance(extent: int, standard_distance: float) -> float: + """ + Calculates the probabilistic distance of an observation. + :param extent: the extent value specified during initialization. + :param standard_distance: the standard distance of the input + observation. + :return: the probabilistic distance. + """ + return extent * standard_distance + + @staticmethod + def _prob_outlier_factor( + probabilistic_distance: np.ndarray, ev_prob_dist: np.ndarray + ) -> np.ndarray: + """ + Calculates the probabilistic outlier factor of an observation. + :param probabilistic_distance: the probabilistic distance of the + input observation. + :param ev_prob_dist: + :return: the probabilistic outlier factor. + """ + if np.all(probabilistic_distance == ev_prob_dist): + return np.zeros(probabilistic_distance.shape) + else: + ev_prob_dist[ev_prob_dist == 0.0] = 1.0e-8 + result = np.divide(probabilistic_distance, ev_prob_dist) - 1.0 + return result + + @staticmethod + def _norm_prob_outlier_factor( + extent: float, ev_probabilistic_outlier_factor: list + ) -> list: + """ + Calculates the normalized probabilistic outlier factor of an + observation. + :param extent: the extent value specified during initialization. + :param ev_probabilistic_outlier_factor: the expected probabilistic + outlier factor of the input observation. + :return: the normalized probabilistic outlier factor. + """ + ev_arr = np.array(ev_probabilistic_outlier_factor, dtype=float) + return (extent * np.sqrt(ev_arr)).tolist() + + @staticmethod + def _local_outlier_probability( + plof_val: np.ndarray, nplof_val: np.ndarray + ) -> np.ndarray: + """ + Calculates the local outlier probability of an observation. + :param plof_val: the probabilistic outlier factor of the input + observation. + :param nplof_val: the normalized probabilistic outlier factor of the + input observation. + :return: the local outlier probability. + """ + if np.all(plof_val == nplof_val): + return np.zeros(plof_val.shape) + plof_f = np.asarray(plof_val, dtype=float) + nplof_f = np.asarray(nplof_val, dtype=float) + if _scipy_erf is not None: + return np.maximum(0, _scipy_erf(plof_f / (nplof_f * np.sqrt(2.0)))) + erf_vec = np.vectorize(erf) + return np.maximum(0, erf_vec(plof_f / (nplof_f * np.sqrt(2.0)))) + + def _n_observations(self) -> int: + """ + Calculates the number of observations in the data. + :return: the number of observations in the input data. + """ + if self.data is not None: + return len(self.data) + return len(self.distance_matrix) + + def _store(self) -> np.ndarray: + """ + Initializes the storage matrix that includes the input value, + cluster labels, local outlier probability, etc. for the input data. + :return: an empty numpy array of shape [n_observations, 3]. + """ + return np.empty([self._n_observations(), 3], dtype=object) + + def _cluster_labels(self) -> np.ndarray: + """ + Returns a numpy array of cluster labels that corresponds to the + input labels or that is an array of all 0 values to indicate all + points belong to the same cluster. + :return: a numpy array of cluster labels. + """ + if self.cluster_labels is None: + if self.data is not None: + return np.array([0] * len(self.data)) + return np.array([0] * len(self.distance_matrix)) + return np.array(self.cluster_labels) + + def _ssd(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the sum squared distance between neighbors for each + observation in the input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + self.cluster_labels_u = np.unique(data_store[:, 0]) + ssd_array = np.empty([self._n_observations(), 1]) + for cluster_id in self.cluster_labels_u: + indices = np.where(data_store[:, 0] == cluster_id) + cluster_distances = np.take(data_store[:, 1], indices).tolist() + ssd = np.power(cluster_distances[0], 2).sum(axis=1) + for i, j in zip(indices[0], ssd): + ssd_array[i] = j + data_store = np.hstack((data_store, ssd_array)) + return data_store + + def _standard_distances(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculated the standard distance for each observation in the input + data. First calculates the cardinality and then calculates the standard + distance with respect to each observation. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + ssd_vals = data_store[:, 3].astype(float) + std_distances = np.sqrt(ssd_vals / self.n_neighbors) + return np.hstack((data_store, std_distances.reshape(-1, 1))) + + def _prob_distances(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the probabilistic distance for each observation in the + input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + prob_distances = self.extent * data_store[:, 4].astype(float) + return np.hstack((data_store, prob_distances.reshape(-1, 1))) + + def _prob_distances_ev(self, data_store) -> np.ndarray: + """ + Calculates the expected value of the probabilistic distance for + each observation in the input data with respect to the cluster the + observation belongs to. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + prob_set_distance_ev = np.empty([self._n_observations(), 1]) + for cluster_id in self.cluster_labels_u: + indices = np.where(data_store[:, 0] == cluster_id)[0] + for index in indices: + nbrhood = data_store[index][2].astype(int) + nbrhood_prob_distances = np.take(data_store[:, 5], nbrhood).astype( + float + ) + nbrhood_prob_distances_nonan = nbrhood_prob_distances[ + np.logical_not(np.isnan(nbrhood_prob_distances)) + ] + prob_set_distance_ev[index] = nbrhood_prob_distances_nonan.mean() + + self.prob_distances_ev = prob_set_distance_ev + return np.hstack((data_store, prob_set_distance_ev)) + + def _prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the probabilistic local outlier factor for each + observation in the input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + return np.hstack( + ( + data_store, + np.array( + [ + np.apply_along_axis( + self._prob_outlier_factor, + 0, + data_store[:, 5], + data_store[:, 6], + ) + ] + ).T, + ) + ) + + def _prob_local_outlier_factors_ev(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the expected value of the probabilistic local outlier factor + for each observation in the input data with respect to the cluster the + observation belongs to. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + prob_local_outlier_factor_ev_dict = {} + for cluster_id in self.cluster_labels_u: + indices = np.where(data_store[:, 0] == cluster_id) + prob_local_outlier_factors = np.take(data_store[:, 7], indices).astype( + float + ) + prob_local_outlier_factors_nonan = prob_local_outlier_factors[ + np.logical_not(np.isnan(prob_local_outlier_factors)) + ] + prob_local_outlier_factor_ev_dict[cluster_id] = np.power( + prob_local_outlier_factors_nonan, 2 + ).sum() / float(prob_local_outlier_factors_nonan.size) + data_store = np.hstack( + ( + data_store, + np.array( + [ + [ + prob_local_outlier_factor_ev_dict[x] + for x in data_store[:, 0].tolist() + ] + ] + ).T, + ) + ) + return data_store + + def _norm_prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the normalized probabilistic local outlier factor for each + observation in the input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + return np.hstack( + ( + data_store, + np.array( + [ + self._norm_prob_outlier_factor( + self.extent, data_store[:, 8].tolist() + ) + ] + ).T, + ) + ) + + def _local_outlier_probabilities(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the local outlier probability for each observation in the + input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + return np.hstack( + ( + data_store, + np.array( + [ + np.apply_along_axis( + self._local_outlier_probability, + 0, + data_store[:, 7], + data_store[:, 9], + ) + ] + ).T, + ) + ) diff --git a/PyNomaly/_utils.py b/PyNomaly/_utils.py new file mode 100644 index 0000000..a8d9a72 --- /dev/null +++ b/PyNomaly/_utils.py @@ -0,0 +1,33 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +import sys +from python_utils.terminal import get_terminal_size + + +class Utils: + @staticmethod + def emit_progress_bar(progress: str, index: int, total: int) -> str: + """ + A progress bar that is continuously updated in Python's standard + out. + :param progress: a string printed to stdout that is updated and later + returned. + :param index: the current index of the iteration within the tracked + process. + :param total: the total length of the tracked process. + :return: progress string. + """ + + w, h = get_terminal_size() + sys.stdout.write("\r") + if total < w: + block_size = int(w / total) + else: + block_size = int(total / w) + if index % block_size == 0: + progress += "=" + percent = index / total + sys.stdout.write("[ %s ] %.2f%%" % (progress, percent * 100)) + sys.stdout.flush() + return progress diff --git a/PyNomaly/_validation.py b/PyNomaly/_validation.py new file mode 100644 index 0000000..75d61a1 --- /dev/null +++ b/PyNomaly/_validation.py @@ -0,0 +1,248 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +import numpy as np +from typing import Union +import warnings + +from PyNomaly.exceptions import ClusterSizeError, MissingValuesError + + +class ValidationMixin: + """Mixin providing input validation methods for LocalOutlierProbability.""" + + @staticmethod + def _convert_to_array(obj: Union["pd.DataFrame", np.ndarray]) -> np.ndarray: + """ + Converts the input data to a numpy array if it is a Pandas DataFrame + or validates it is already a numpy array. + :param obj: user-provided input data. + :return: a vector of values to be used in calculating the local + outlier probability. + """ + if obj.__class__.__name__ == "DataFrame": + points_vector = obj.values + return points_vector + elif obj.__class__.__name__ == "ndarray": + points_vector = obj + return points_vector + else: + warnings.warn( + "Provided data or distance matrix must be in ndarray " + "or DataFrame.", + UserWarning, + ) + if isinstance(obj, list): + points_vector = np.array(obj) + return points_vector + points_vector = np.array([obj]) + return points_vector + + def _validate_inputs(self): + """ + Validates the inputs provided during initialization to ensure + that the needed objects are provided. + :return: a tuple of (data, distance_matrix, neighbor_matrix) or + raises a warning for invalid inputs. + """ + if all(v is None for v in [self.data, self.distance_matrix]): + warnings.warn( + "Data or a distance matrix must be provided.", UserWarning + ) + return False + elif all(v is not None for v in [self.data, self.distance_matrix]): + warnings.warn( + "Only one of the following may be provided: data or a " + "distance matrix (not both).", + UserWarning, + ) + return False + if self.data is not None: + points_vector = self._convert_to_array(self.data) + return points_vector, self.distance_matrix, self.neighbor_matrix + if all( + matrix is not None + for matrix in [self.neighbor_matrix, self.distance_matrix] + ): + dist_vector = self._convert_to_array(self.distance_matrix) + neigh_vector = self._convert_to_array(self.neighbor_matrix) + else: + warnings.warn( + "A neighbor index matrix and distance matrix must both be " + "provided when not using raw input data.", + UserWarning, + ) + return False + if self.distance_matrix.shape != self.neighbor_matrix.shape: + warnings.warn( + "The shape of the distance and neighbor " + "index matrices must match.", + UserWarning, + ) + return False + elif (self.distance_matrix.shape[1] != self.n_neighbors) or ( + self.neighbor_matrix.shape[1] != self.n_neighbors + ): + warnings.warn( + "The shape of the distance or " + "neighbor index matrix does not " + "match the number of neighbors " + "specified.", + UserWarning, + ) + return False + return self.data, dist_vector, neigh_vector + + def _check_cluster_size(self) -> None: + """ + Validates the cluster labels to ensure that the smallest cluster + size (number of observations in the cluster) is larger than the + specified number of neighbors. + :raises ClusterSizeError: if any cluster is too small. + """ + c_labels = self._cluster_labels() + for cluster_id in set(c_labels): + c_size = np.where(c_labels == cluster_id)[0].shape[0] + if c_size <= self.n_neighbors: + raise ClusterSizeError( + "Number of neighbors specified larger than smallest " + "cluster. Specify a number of neighbors smaller than " + "the smallest cluster size (observations in smallest " + "cluster minus one)." + ) + + def _check_n_neighbors(self) -> bool: + """ + Validates the specified number of neighbors to ensure that it is + greater than 0 and that the specified value is less than the total + number of observations. + :return: a boolean indicating whether validation has passed without + adjustment. + """ + if not self.n_neighbors > 0: + self.n_neighbors = 10 + warnings.warn( + "n_neighbors must be greater than 0." + " Fit with " + str(self.n_neighbors) + " instead.", + UserWarning, + ) + return False + elif self.n_neighbors >= self._n_observations(): + self.n_neighbors = self._n_observations() - 1 + warnings.warn( + "n_neighbors must be less than the number of observations." + " Fit with " + str(self.n_neighbors) + " instead.", + UserWarning, + ) + return True + + def _check_extent(self) -> bool: + """ + Validates the specified extent parameter to ensure it is either 1, + 2, or 3. + :return: a boolean indicating whether validation has passed. + """ + if self.extent not in [1, 2, 3]: + warnings.warn( + "extent parameter (lambda) must be 1, 2, or 3.", UserWarning + ) + return False + return True + + def _check_missing_values(self) -> None: + """ + Validates the provided data to ensure that it contains no + missing values. + :raises MissingValuesError: if data contains NaN values. + """ + if np.any(np.isnan(self.data)): + raise MissingValuesError( + "Method does not support missing values in input data." + ) + + def _check_is_fit(self) -> bool: + """ + Checks that the model was fit prior to calling the stream() method. + :return: a boolean indicating whether the model has been fit. + """ + if self.is_fit is False: + warnings.warn( + "Must fit on historical data by calling fit() prior to " + "calling stream(x).", + UserWarning, + ) + return False + return True + + def _check_no_cluster_labels(self) -> bool: + """ + Checks to see if cluster labels are attempting to be used in + stream() and, if so, returns False. As PyNomaly does not accept + clustering algorithms as input, the stream approach does not + support clustering. + :return: a boolean indicating whether single cluster (no labels). + """ + if len(set(self._cluster_labels())) > 1: + warnings.warn( + "Stream approach does not support clustered data. " + "Automatically refit using single cluster of points.", + UserWarning, + ) + return False + return True + + +def accepts(*types): + """ + A decorator that facilitates a form of type checking for the inputs + which can be used in Python 3.4-3.7 in lieu of Python 3.5+'s type + hints. + :param types: the input types of the objects being passed as arguments + in __init__. + :return: a decorator. + """ + + def decorator(f): + assert len(types) == f.__code__.co_argcount + + def new_f(*args, **kwds): + for a, t in zip(args, types): + if type(a).__name__ == "DataFrame": + a = np.array(a) + if isinstance(a, t) is False: + warnings.warn( + "Argument %r is not of type %s" % (a, t), UserWarning + ) + opt_types = { + "extent": {"type": (int, np.integer)}, + "n_neighbors": {"type": (int, np.integer)}, + "use_numba": {"type": bool}, + "n_jobs": {"type": (int, np.integer)}, + "progress_bar": {"type": bool}, + "data": {"type": np.ndarray}, + "distance_matrix": {"type": np.ndarray}, + "neighbor_matrix": {"type": np.ndarray}, + "cluster_labels": {"type": list}, + } + for x in kwds: + if x in opt_types: + opt_types[x]["value"] = kwds[x] + for k in opt_types: + try: + if ( + isinstance(opt_types[k]["value"], opt_types[k]["type"]) + is False + ): + warnings.warn( + "Argument %r is not of type %s." + % (k, opt_types[k]["type"]), + UserWarning, + ) + except KeyError: + pass + return f(*args, **kwds) + + new_f.__name__ = f.__name__ + return new_f + + return decorator diff --git a/PyNomaly/exceptions.py b/PyNomaly/exceptions.py new file mode 100644 index 0000000..7417ddb --- /dev/null +++ b/PyNomaly/exceptions.py @@ -0,0 +1,22 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + + +class PyNomalyError(Exception): + """Base exception for PyNomaly.""" + pass + + +class ValidationError(PyNomalyError): + """Raised when input validation fails.""" + pass + + +class ClusterSizeError(ValidationError): + """Raised when cluster size is smaller than n_neighbors.""" + pass + + +class MissingValuesError(ValidationError): + """Raised when data contains missing values.""" + pass diff --git a/PyNomaly/loop.py b/PyNomaly/loop.py index 1ea7ab8..d5a5074 100644 --- a/PyNomaly/loop.py +++ b/PyNomaly/loop.py @@ -1,136 +1,27 @@ -from math import erf, sqrt +# Authors: Valentino Constantinou +# License: Apache 2.0 + import numpy as np -import os -from python_utils.terminal import get_terminal_size import sys -from typing import Tuple, Union import warnings -try: - from scipy.spatial.distance import cdist as _scipy_cdist -except ImportError: - _scipy_cdist = None - -try: - from scipy.special import erf as _scipy_erf -except ImportError: - _scipy_erf = None - -try: - import numba - - def _numba_distance_kernel_parallel(clust_points_vector, n_neighbors): - n = clust_points_vector.shape[0] - d_features = clust_points_vector.shape[1] - local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) - local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) - for i in numba.prange(n): - for j in range(n): - if i == j: - continue - d = 0.0 - for f in range(d_features): - diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] - d += diff_val * diff_val - d = d ** 0.5 - idx_max = 0 - for k in range(1, n_neighbors): - if local_distances[i, k] > local_distances[i, idx_max]: - idx_max = k - if d < local_distances[i, idx_max]: - local_distances[i, idx_max] = d - local_indexes[i, idx_max] = j - return local_distances, local_indexes - - _numba_kernel_parallel = numba.jit( - _numba_distance_kernel_parallel, nopython=True, parallel=True, cache=True - ) - - def _numba_distance_kernel_sequential(clust_points_vector, n_neighbors): - n = clust_points_vector.shape[0] - d_features = clust_points_vector.shape[1] - local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) - local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) - for i in range(n): - for j in range(n): - if i == j: - continue - d = 0.0 - for f in range(d_features): - diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] - d += diff_val * diff_val - d = d ** 0.5 - idx_max = 0 - for k in range(1, n_neighbors): - if local_distances[i, k] > local_distances[i, idx_max]: - idx_max = k - if d < local_distances[i, idx_max]: - local_distances[i, idx_max] = d - local_indexes[i, idx_max] = j - return local_distances, local_indexes - - _numba_kernel_sequential = numba.jit( - _numba_distance_kernel_sequential, nopython=True, cache=True - ) - -except ImportError: - pass +from PyNomaly.exceptions import ( + PyNomalyError, + ValidationError, + ClusterSizeError, + MissingValuesError, +) +from PyNomaly._utils import Utils +from PyNomaly._validation import ValidationMixin, accepts +from PyNomaly._distance import DistanceMixin +from PyNomaly._pipeline import PipelineMixin __author__ = "Valentino Constantinou" __version__ = "1.0.0" __license__ = "Apache License, Version 2.0" -# Custom Exceptions -class PyNomalyError(Exception): - """Base exception for PyNomaly.""" - pass - - -class ValidationError(PyNomalyError): - """Raised when input validation fails.""" - pass - - -class ClusterSizeError(ValidationError): - """Raised when cluster size is smaller than n_neighbors.""" - pass - - -class MissingValuesError(ValidationError): - """Raised when data contains missing values.""" - pass - - -class Utils: - @staticmethod - def emit_progress_bar(progress: str, index: int, total: int) -> str: - """ - A progress bar that is continuously updated in Python's standard - out. - :param progress: a string printed to stdout that is updated and later - returned. - :param index: the current index of the iteration within the tracked - process. - :param total: the total length of the tracked process. - :return: progress string. - """ - - w, h = get_terminal_size() - sys.stdout.write("\r") - if total < w: - block_size = int(w / total) - else: - block_size = int(total / w) - if index % block_size == 0: - progress += "=" - percent = index / total - sys.stdout.write("[ %s ] %.2f%%" % (progress, percent * 100)) - sys.stdout.flush() - return progress - - -class LocalOutlierProbability(object): +class LocalOutlierProbability(ValidationMixin, DistanceMixin, PipelineMixin): """Local Outlier Probability (LoOP) estimator. :param extent: an integer value [1, 2, 3] that controls the statistical @@ -170,250 +61,6 @@ class LocalOutlierProbability(object): _DATA_PARAMS = ("data", "distance_matrix", "neighbor_matrix", "cluster_labels") - """ - Validation methods. - These methods validate inputs and raise exceptions or warnings as appropriate. - """ - - @staticmethod - def _convert_to_array(obj: Union["pd.DataFrame", np.ndarray]) -> np.ndarray: - """ - Converts the input data to a numpy array if it is a Pandas DataFrame - or validates it is already a numpy array. - :param obj: user-provided input data. - :return: a vector of values to be used in calculating the local - outlier probability. - """ - if obj.__class__.__name__ == "DataFrame": - points_vector = obj.values - return points_vector - elif obj.__class__.__name__ == "ndarray": - points_vector = obj - return points_vector - else: - warnings.warn( - "Provided data or distance matrix must be in ndarray " - "or DataFrame.", - UserWarning, - ) - if isinstance(obj, list): - points_vector = np.array(obj) - return points_vector - points_vector = np.array([obj]) - return points_vector - - def _validate_inputs(self): - """ - Validates the inputs provided during initialization to ensure - that the needed objects are provided. - :return: a tuple of (data, distance_matrix, neighbor_matrix) or - raises a warning for invalid inputs. - """ - if all(v is None for v in [self.data, self.distance_matrix]): - warnings.warn( - "Data or a distance matrix must be provided.", UserWarning - ) - return False - elif all(v is not None for v in [self.data, self.distance_matrix]): - warnings.warn( - "Only one of the following may be provided: data or a " - "distance matrix (not both).", - UserWarning, - ) - return False - if self.data is not None: - points_vector = self._convert_to_array(self.data) - return points_vector, self.distance_matrix, self.neighbor_matrix - if all( - matrix is not None - for matrix in [self.neighbor_matrix, self.distance_matrix] - ): - dist_vector = self._convert_to_array(self.distance_matrix) - neigh_vector = self._convert_to_array(self.neighbor_matrix) - else: - warnings.warn( - "A neighbor index matrix and distance matrix must both be " - "provided when not using raw input data.", - UserWarning, - ) - return False - if self.distance_matrix.shape != self.neighbor_matrix.shape: - warnings.warn( - "The shape of the distance and neighbor " - "index matrices must match.", - UserWarning, - ) - return False - elif (self.distance_matrix.shape[1] != self.n_neighbors) or ( - self.neighbor_matrix.shape[1] != self.n_neighbors - ): - warnings.warn( - "The shape of the distance or " - "neighbor index matrix does not " - "match the number of neighbors " - "specified.", - UserWarning, - ) - return False - return self.data, dist_vector, neigh_vector - - def _check_cluster_size(self) -> None: - """ - Validates the cluster labels to ensure that the smallest cluster - size (number of observations in the cluster) is larger than the - specified number of neighbors. - :raises ClusterSizeError: if any cluster is too small. - """ - c_labels = self._cluster_labels() - for cluster_id in set(c_labels): - c_size = np.where(c_labels == cluster_id)[0].shape[0] - if c_size <= self.n_neighbors: - raise ClusterSizeError( - "Number of neighbors specified larger than smallest " - "cluster. Specify a number of neighbors smaller than " - "the smallest cluster size (observations in smallest " - "cluster minus one)." - ) - - def _check_n_neighbors(self) -> bool: - """ - Validates the specified number of neighbors to ensure that it is - greater than 0 and that the specified value is less than the total - number of observations. - :return: a boolean indicating whether validation has passed without - adjustment. - """ - if not self.n_neighbors > 0: - self.n_neighbors = 10 - warnings.warn( - "n_neighbors must be greater than 0." - " Fit with " + str(self.n_neighbors) + " instead.", - UserWarning, - ) - return False - elif self.n_neighbors >= self._n_observations(): - self.n_neighbors = self._n_observations() - 1 - warnings.warn( - "n_neighbors must be less than the number of observations." - " Fit with " + str(self.n_neighbors) + " instead.", - UserWarning, - ) - return True - - def _check_extent(self) -> bool: - """ - Validates the specified extent parameter to ensure it is either 1, - 2, or 3. - :return: a boolean indicating whether validation has passed. - """ - if self.extent not in [1, 2, 3]: - warnings.warn( - "extent parameter (lambda) must be 1, 2, or 3.", UserWarning - ) - return False - return True - - def _check_missing_values(self) -> None: - """ - Validates the provided data to ensure that it contains no - missing values. - :raises MissingValuesError: if data contains NaN values. - """ - if np.any(np.isnan(self.data)): - raise MissingValuesError( - "Method does not support missing values in input data." - ) - - def _check_is_fit(self) -> bool: - """ - Checks that the model was fit prior to calling the stream() method. - :return: a boolean indicating whether the model has been fit. - """ - if self.is_fit is False: - warnings.warn( - "Must fit on historical data by calling fit() prior to " - "calling stream(x).", - UserWarning, - ) - return False - return True - - def _check_no_cluster_labels(self) -> bool: - """ - Checks to see if cluster labels are attempting to be used in - stream() and, if so, returns False. As PyNomaly does not accept - clustering algorithms as input, the stream approach does not - support clustering. - :return: a boolean indicating whether single cluster (no labels). - """ - if len(set(self._cluster_labels())) > 1: - warnings.warn( - "Stream approach does not support clustered data. " - "Automatically refit using single cluster of points.", - UserWarning, - ) - return False - return True - - """ - Decorators. - """ - - def accepts(*types): - """ - A decorator that facilitates a form of type checking for the inputs - which can be used in Python 3.4-3.7 in lieu of Python 3.5+'s type - hints. - :param types: the input types of the objects being passed as arguments - in __init__. - :return: a decorator. - """ - - def decorator(f): - assert len(types) == f.__code__.co_argcount - - def new_f(*args, **kwds): - for a, t in zip(args, types): - if type(a).__name__ == "DataFrame": - a = np.array(a) - if isinstance(a, t) is False: - warnings.warn( - "Argument %r is not of type %s" % (a, t), UserWarning - ) - opt_types = { - "extent": {"type": (int, np.integer)}, - "n_neighbors": {"type": (int, np.integer)}, - "use_numba": {"type": bool}, - "n_jobs": {"type": (int, np.integer)}, - "progress_bar": {"type": bool}, - "data": {"type": np.ndarray}, - "distance_matrix": {"type": np.ndarray}, - "neighbor_matrix": {"type": np.ndarray}, - "cluster_labels": {"type": list}, - } - for x in kwds: - if x in opt_types: - opt_types[x]["value"] = kwds[x] - for k in opt_types: - try: - if ( - isinstance(opt_types[k]["value"], opt_types[k]["type"]) - is False - ): - warnings.warn( - "Argument %r is not of type %s." - % (k, opt_types[k]["type"]), - UserWarning, - ) - except KeyError: - pass - return f(*args, **kwds) - - new_f.__name__ = f.__name__ - return new_f - - return decorator - @accepts( object, (int, np.integer), @@ -486,494 +133,6 @@ def __init__( self._check_extent() - """ - Private methods. - """ - - @staticmethod - def _standard_distance(cardinality: float, sum_squared_distance: float) -> float: - """ - Calculates the standard distance of an observation. - :param cardinality: the cardinality of the input observation. - :param sum_squared_distance: the sum squared distance between all - neighbors of the input observation. - :return: the standard distance. - #""" - division_result = sum_squared_distance / cardinality - st_dist = sqrt(division_result) - return st_dist - - @staticmethod - def _prob_distance(extent: int, standard_distance: float) -> float: - """ - Calculates the probabilistic distance of an observation. - :param extent: the extent value specified during initialization. - :param standard_distance: the standard distance of the input - observation. - :return: the probabilistic distance. - """ - return extent * standard_distance - - @staticmethod - def _prob_outlier_factor( - probabilistic_distance: np.ndarray, ev_prob_dist: np.ndarray - ) -> np.ndarray: - """ - Calculates the probabilistic outlier factor of an observation. - :param probabilistic_distance: the probabilistic distance of the - input observation. - :param ev_prob_dist: - :return: the probabilistic outlier factor. - """ - if np.all(probabilistic_distance == ev_prob_dist): - return np.zeros(probabilistic_distance.shape) - else: - ev_prob_dist[ev_prob_dist == 0.0] = 1.0e-8 - result = np.divide(probabilistic_distance, ev_prob_dist) - 1.0 - return result - - @staticmethod - def _norm_prob_outlier_factor( - extent: float, ev_probabilistic_outlier_factor: list - ) -> list: - """ - Calculates the normalized probabilistic outlier factor of an - observation. - :param extent: the extent value specified during initialization. - :param ev_probabilistic_outlier_factor: the expected probabilistic - outlier factor of the input observation. - :return: the normalized probabilistic outlier factor. - """ - ev_arr = np.array(ev_probabilistic_outlier_factor, dtype=float) - return (extent * np.sqrt(ev_arr)).tolist() - - @staticmethod - def _local_outlier_probability( - plof_val: np.ndarray, nplof_val: np.ndarray - ) -> np.ndarray: - """ - Calculates the local outlier probability of an observation. - :param plof_val: the probabilistic outlier factor of the input - observation. - :param nplof_val: the normalized probabilistic outlier factor of the - input observation. - :return: the local outlier probability. - """ - if np.all(plof_val == nplof_val): - return np.zeros(plof_val.shape) - plof_f = np.asarray(plof_val, dtype=float) - nplof_f = np.asarray(nplof_val, dtype=float) - if _scipy_erf is not None: - return np.maximum(0, _scipy_erf(plof_f / (nplof_f * np.sqrt(2.0)))) - erf_vec = np.vectorize(erf) - return np.maximum(0, erf_vec(plof_f / (nplof_f * np.sqrt(2.0)))) - - def _n_observations(self) -> int: - """ - Calculates the number of observations in the data. - :return: the number of observations in the input data. - """ - if self.data is not None: - return len(self.data) - return len(self.distance_matrix) - - def _store(self) -> np.ndarray: - """ - Initializes the storage matrix that includes the input value, - cluster labels, local outlier probability, etc. for the input data. - :return: an empty numpy array of shape [n_observations, 3]. - """ - return np.empty([self._n_observations(), 3], dtype=object) - - def _cluster_labels(self) -> np.ndarray: - """ - Returns a numpy array of cluster labels that corresponds to the - input labels or that is an array of all 0 values to indicate all - points belong to the same cluster. - :return: a numpy array of cluster labels. - """ - if self.cluster_labels is None: - if self.data is not None: - return np.array([0] * len(self.data)) - return np.array([0] * len(self.distance_matrix)) - return np.array(self.cluster_labels) - - @staticmethod - def _euclidean(vector1: np.ndarray, vector2: np.ndarray) -> np.ndarray: - """ - Calculates the euclidean distance between two observations in the - input data. - :param vector1: a numpy array corresponding to observation 1. - :param vector2: a numpy array corresponding to observation 2. - :return: the euclidean distance between the two observations. - """ - diff = vector1 - vector2 - return np.dot(diff, diff) ** 0.5 - - def _assign_distances(self, data_store: np.ndarray) -> np.ndarray: - """ - Takes a distance matrix, produced by _distances or provided through - user input, and assigns distances for each observation to the storage - matrix, data_store. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - for vec, cluster_id in zip( - range(self.distance_matrix.shape[0]), self._cluster_labels() - ): - data_store[vec][0] = cluster_id - data_store[vec][1] = self.distance_matrix[vec] - data_store[vec][2] = self.neighbor_matrix[vec] - return data_store - - @staticmethod - def _compute_distance_and_neighbor_matrix( - clust_points_vector: np.ndarray, - indices: np.ndarray, - distances: np.ndarray, - indexes: np.ndarray, - ) -> Tuple[np.ndarray, np.ndarray, int]: - """ - This helper method provides the heavy lifting for the _distances - method and is only intended for use therein. The code has been - written so that it can make full use of Numba's jit capabilities if - desired. - """ - for i in range(clust_points_vector.shape[0]): - for j in range(i + 1, clust_points_vector.shape[0]): - # Global index of the points - global_i = indices[0][i] - global_j = indices[0][j] - - # Compute Euclidean distance - diff = clust_points_vector[i] - clust_points_vector[j] - d = np.dot(diff, diff) ** 0.5 - - # Update distance and neighbor index for global_i - idx_max = distances[global_i].argmax() - if d < distances[global_i][idx_max]: - distances[global_i][idx_max] = d - indexes[global_i][idx_max] = global_j - - # Update distance and neighbor index for global_j - idx_max = distances[global_j].argmax() - if d < distances[global_j][idx_max]: - distances[global_j][idx_max] = d - indexes[global_j][idx_max] = global_i - - yield distances, indexes, i - - def _distances_vectorized( - self, clusters, distances, indexes, progress_bar - ) -> None: - """Vectorized kNN distance computation with chunked progress.""" - progress = "=" - total_points = sum(cv.shape[0] for cv, _ in clusters) - completed = 0 - - for clust_points_vector, global_indices in clusters: - n = clust_points_vector.shape[0] - - if clust_points_vector.ndim == 1: - clust_points_vector = clust_points_vector.reshape(-1, 1) - - chunk_size = min(256, n) - for chunk_start in range(0, n, chunk_size): - chunk_end = min(chunk_start + chunk_size, n) - chunk = clust_points_vector[chunk_start:chunk_end] - - if _scipy_cdist is not None: - dist = _scipy_cdist( - chunk, clust_points_vector, metric="euclidean" - ) - else: - diff = ( - chunk[:, np.newaxis, :] - - clust_points_vector[np.newaxis, :, :] - ) - dist = np.sqrt((diff ** 2).sum(axis=2)) - - row_idx = np.arange(chunk_end - chunk_start) - dist[row_idx, row_idx + chunk_start] = np.inf - - knn_idx = np.argpartition(dist, self.n_neighbors, axis=1)[ - :, : self.n_neighbors - ] - knn_dists = np.take_along_axis(dist, knn_idx, axis=1) - - chunk_global = global_indices[chunk_start:chunk_end] - distances[chunk_global] = knn_dists - indexes[chunk_global] = global_indices[knn_idx] - - completed += chunk_end - chunk_start - if progress_bar: - progress = Utils.emit_progress_bar( - progress, completed, total_points - ) - - def _distances_numba( - self, clusters, distances, indexes, progress_bar, parallel=False - ) -> None: - """Numba-accelerated distance computation.""" - progress = "=" - kernel = _numba_kernel_parallel if parallel else _numba_kernel_sequential - - for idx, (clust_points_vector, global_indices) in enumerate(clusters): - if clust_points_vector.ndim == 1: - clust_points_vector = clust_points_vector.reshape(-1, 1) - - local_dists, local_idxs = kernel( - clust_points_vector.astype(np.float64), self.n_neighbors - ) - - distances[global_indices] = local_dists - indexes[global_indices] = global_indices[local_idxs] - - if progress_bar: - progress = Utils.emit_progress_bar( - progress, idx + 1, len(clusters) - ) - - def _distances(self, progress_bar: bool = False) -> None: - """ - Provides the distances between each observation and it's closest - neighbors. When input data is provided, calculates the euclidean - distance between every observation. Otherwise, the user-provided - distance matrix is used. - :return: the updated storage matrix that collects information on - each observation. - """ - distances = np.full( - [self._n_observations(), self.n_neighbors], 9e10, dtype=float - ) - indexes = np.full( - [self._n_observations(), self.n_neighbors], 9e10, dtype=float - ) - self.points_vector = self._convert_to_array(self.data) - - cluster_labels = self._cluster_labels() - cluster_ids = sorted(set(cluster_labels)) - - clusters = [] - for cluster_id in cluster_ids: - indices = np.where(cluster_labels == cluster_id) - clust_points_vector = np.array( - self.points_vector.take(indices, axis=0)[0], dtype=np.float64 - ) - clusters.append((clust_points_vector, indices[0])) - - n_jobs = self.n_jobs - if n_jobs == -1: - n_jobs = os.cpu_count() or 1 - - if self.use_numba: - self._distances_numba( - clusters, distances, indexes, progress_bar, - parallel=(n_jobs > 1) - ) - else: - if n_jobs > 1: - warnings.warn( - "n_jobs > 1 requires use_numba=True for parallel " - "processing. Install Numba and set use_numba=True " - "to enable parallelism. Falling back to sequential.", - UserWarning, - ) - self._distances_vectorized( - clusters, distances, indexes, progress_bar - ) - - self.distance_matrix = distances - self.neighbor_matrix = indexes - - def _ssd(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the sum squared distance between neighbors for each - observation in the input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - self.cluster_labels_u = np.unique(data_store[:, 0]) - ssd_array = np.empty([self._n_observations(), 1]) - for cluster_id in self.cluster_labels_u: - indices = np.where(data_store[:, 0] == cluster_id) - cluster_distances = np.take(data_store[:, 1], indices).tolist() - ssd = np.power(cluster_distances[0], 2).sum(axis=1) - for i, j in zip(indices[0], ssd): - ssd_array[i] = j - data_store = np.hstack((data_store, ssd_array)) - return data_store - - def _standard_distances(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculated the standard distance for each observation in the input - data. First calculates the cardinality and then calculates the standard - distance with respect to each observation. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - ssd_vals = data_store[:, 3].astype(float) - std_distances = np.sqrt(ssd_vals / self.n_neighbors) - return np.hstack((data_store, std_distances.reshape(-1, 1))) - - def _prob_distances(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the probabilistic distance for each observation in the - input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - prob_distances = self.extent * data_store[:, 4].astype(float) - return np.hstack((data_store, prob_distances.reshape(-1, 1))) - - def _prob_distances_ev(self, data_store) -> np.ndarray: - """ - Calculates the expected value of the probabilistic distance for - each observation in the input data with respect to the cluster the - observation belongs to. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - prob_set_distance_ev = np.empty([self._n_observations(), 1]) - for cluster_id in self.cluster_labels_u: - indices = np.where(data_store[:, 0] == cluster_id)[0] - for index in indices: - # Global neighbor indices for the current point - nbrhood = data_store[index][2].astype(int) # Ensure global indices - nbrhood_prob_distances = np.take(data_store[:, 5], nbrhood).astype( - float - ) - nbrhood_prob_distances_nonan = nbrhood_prob_distances[ - np.logical_not(np.isnan(nbrhood_prob_distances)) - ] - prob_set_distance_ev[index] = nbrhood_prob_distances_nonan.mean() - - self.prob_distances_ev = prob_set_distance_ev - return np.hstack((data_store, prob_set_distance_ev)) - - def _prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the probabilistic local outlier factor for each - observation in the input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - return np.hstack( - ( - data_store, - np.array( - [ - np.apply_along_axis( - self._prob_outlier_factor, - 0, - data_store[:, 5], - data_store[:, 6], - ) - ] - ).T, - ) - ) - - def _prob_local_outlier_factors_ev(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the expected value of the probabilistic local outlier factor - for each observation in the input data with respect to the cluster the - observation belongs to. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - prob_local_outlier_factor_ev_dict = {} - for cluster_id in self.cluster_labels_u: - indices = np.where(data_store[:, 0] == cluster_id) - prob_local_outlier_factors = np.take(data_store[:, 7], indices).astype( - float - ) - prob_local_outlier_factors_nonan = prob_local_outlier_factors[ - np.logical_not(np.isnan(prob_local_outlier_factors)) - ] - prob_local_outlier_factor_ev_dict[cluster_id] = np.power( - prob_local_outlier_factors_nonan, 2 - ).sum() / float(prob_local_outlier_factors_nonan.size) - data_store = np.hstack( - ( - data_store, - np.array( - [ - [ - prob_local_outlier_factor_ev_dict[x] - for x in data_store[:, 0].tolist() - ] - ] - ).T, - ) - ) - return data_store - - def _norm_prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the normalized probabilistic local outlier factor for each - observation in the input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - return np.hstack( - ( - data_store, - np.array( - [ - self._norm_prob_outlier_factor( - self.extent, data_store[:, 8].tolist() - ) - ] - ).T, - ) - ) - - def _local_outlier_probabilities(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the local outlier probability for each observation in the - input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - return np.hstack( - ( - data_store, - np.array( - [ - np.apply_along_axis( - self._local_outlier_probability, - 0, - data_store[:, 7], - data_store[:, 9], - ) - ] - ).T, - ) - ) - - """ - Public methods - """ - def _reset_state(self) -> None: """Resets computed state to allow re-fitting with new data.""" self.points_vector = None From b6d1814cc0245f895f373b983ca04a6ae7787014 Mon Sep 17 00:00:00 2001 From: vc1492a Date: Tue, 16 Jun 2026 10:45:01 -0700 Subject: [PATCH 3/4] fix: clear stale inputs on re-fit and accept DataFrame in type checker - Clear self.data when switching to distance_matrix mode during re-fit, preventing _validate_inputs() from seeing conflicting inputs. - Accept DataFrame and np.ndarray for cluster_labels in the @accepts decorator to avoid spurious type warnings on the deprecated path. Co-authored-by: Cursor --- PyNomaly/_validation.py | 7 +++++-- PyNomaly/loop.py | 14 +++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/PyNomaly/_validation.py b/PyNomaly/_validation.py index 75d61a1..b1f0a76 100644 --- a/PyNomaly/_validation.py +++ b/PyNomaly/_validation.py @@ -222,11 +222,14 @@ def new_f(*args, **kwds): "data": {"type": np.ndarray}, "distance_matrix": {"type": np.ndarray}, "neighbor_matrix": {"type": np.ndarray}, - "cluster_labels": {"type": list}, + "cluster_labels": {"type": (list, np.ndarray)}, } for x in kwds: if x in opt_types: - opt_types[x]["value"] = kwds[x] + v = kwds[x] + if type(v).__name__ == "DataFrame": + v = np.array(v) + opt_types[x]["value"] = v for k in opt_types: try: if ( diff --git a/PyNomaly/loop.py b/PyNomaly/loop.py index d5a5074..a34c518 100644 --- a/PyNomaly/loop.py +++ b/PyNomaly/loop.py @@ -170,13 +170,21 @@ def fit( self._reset_state() - if data is not None: + if data is not None and distance_matrix is not None: + self.data = data + self.distance_matrix = distance_matrix + if neighbor_matrix is not None: + self.neighbor_matrix = neighbor_matrix + elif data is not None: self.data = data self.distance_matrix = None self.neighbor_matrix = None - if distance_matrix is not None: + elif distance_matrix is not None: self.distance_matrix = distance_matrix - if neighbor_matrix is not None: + self.data = None + if neighbor_matrix is not None: + self.neighbor_matrix = neighbor_matrix + elif neighbor_matrix is not None: self.neighbor_matrix = neighbor_matrix if cluster_labels is not None: self.cluster_labels = cluster_labels From ad98c1845bf4f9c24ff7c849d1eded160d22d43f Mon Sep 17 00:00:00 2001 From: SaritNike Date: Fri, 26 Jun 2026 13:25:50 +0200 Subject: [PATCH 4/4] include scikit learn estimator-check in unit tests --- tests/test_loop.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_loop.py b/tests/test_loop.py index 138522b..0ce1be3 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -15,6 +15,8 @@ from sklearn.metrics import roc_auc_score from sklearn.neighbors import NearestNeighbors from sklearn.utils import check_random_state +from sklearn.utils.estimator_checks import check_estimator + import sys logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) @@ -156,6 +158,13 @@ def X_n1000() -> np.ndarray: return X +def test_sklearn_api_compliance(): + """ + Check scikit-learn compatibility as referenced in the developer guide: + https://scikit-learn.org/stable/developers/develop.html#rolling-your-own-estimator + """ + check_estimator(loop.LoOP()) + def test_loop(X_n8) -> None: """ Tests the basic functionality and asserts that the anomalous observations