Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions docs/SDK_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,45 @@ query = (
)
```

### Filtering on Table Calculations

Define a table calculation, add it to the query, then filter on it with the same
operators you use for dimensions. Table-calculation conditions are sent under
`filters.tableCalculations`:

```python
from lightdash import TableCalculation

profit_ratio = TableCalculation(
name="profit_ratio",
sql="${orders.profit} / ${orders.revenue}",
)

query = (
model.query()
.metrics(model.metrics.revenue, model.metrics.profit)
.table_calculations(profit_ratio)
.filter(profit_ratio > 0.2) # only rows where the ratio exceeds 20%
)
```

Dimension and table-calculation filters can be combined freely; each is
serialized under its own key:

```python
query = (
model.query()
.table_calculations(profit_ratio)
.filter((model.dimensions.country == "USA") & (profit_ratio > 0.2))
)
```

> **Note on `type`:** `TableCalculation` defaults `type="number"`. The data
> type must be set for filter operators to compile — an untyped calculation is
> treated as a string by the API and rejects numeric operators like `>` or
> `between`. For a non-numeric calculation, pass `type="string"` (or `date`,
> `timestamp`, `boolean`).

---

## Dimensions and Metrics
Expand Down
5 changes: 4 additions & 1 deletion lightdash/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
)
from lightdash.query import QueryResult
from lightdash.sorting import Sort
from lightdash.filter import DimensionFilter, CompositeFilter
from lightdash.filter import DimensionFilter, TableCalculationFilter, CompositeFilter
from lightdash.table_calculations import TableCalculation
from lightdash.sql_runner import SqlResult
from lightdash.results import ResultSet, BaseResult

Expand All @@ -25,7 +26,9 @@
'QueryResult',
'Sort',
'DimensionFilter',
'TableCalculationFilter',
'CompositeFilter',
'TableCalculation',
'SqlResult',
'ResultSet',
'BaseResult',
Expand Down
98 changes: 79 additions & 19 deletions lightdash/filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

if TYPE_CHECKING:
from lightdash.dimensions import Dimension
from lightdash.table_calculations import TableCalculation

numeric_filters = [
"isNull",
Expand Down Expand Up @@ -57,8 +58,24 @@
)


class _FieldFilterMixin:
"""Shared ``&`` / ``|`` combination behavior for single-field filters."""

def __and__(self, other: Union["FieldFilter", "CompositeFilter"]) -> "CompositeFilter":
"""Combine filters with AND: filter1 & filter2"""
if isinstance(other, CompositeFilter):
return CompositeFilter(filters=[self] + list(other.filters), aggregation="and")
return CompositeFilter(filters=[self, other], aggregation="and")

def __or__(self, other: Union["FieldFilter", "CompositeFilter"]) -> "CompositeFilter":
"""Combine filters with OR: filter1 | filter2"""
if isinstance(other, CompositeFilter):
return CompositeFilter(filters=[self] + list(other.filters), aggregation="or")
return CompositeFilter(filters=[self, other], aggregation="or")


@dataclass
class DimensionFilter:
class DimensionFilter(_FieldFilterMixin):
field: "Dimension"
operator: str
values: Union[str, int, float, List[str], List[int], List[float]]
Expand Down Expand Up @@ -87,27 +104,61 @@ def to_dict(self) -> Dict[str, Union[str, List[str]]]:
"values": self.values,
}

def __and__(self, other: Union["DimensionFilter", "CompositeFilter"]) -> "CompositeFilter":
"""Combine filters with AND: filter1 & filter2"""
if isinstance(other, CompositeFilter):
return CompositeFilter(filters=[self] + list(other.filters), aggregation="and")
return CompositeFilter(filters=[self, other], aggregation="and")

def __or__(self, other: Union["DimensionFilter", "CompositeFilter"]) -> "CompositeFilter":
"""Combine filters with OR: filter1 | filter2"""
if isinstance(other, CompositeFilter):
return CompositeFilter(filters=[self] + list(other.filters), aggregation="or")
return CompositeFilter(filters=[self, other], aggregation="or")
@dataclass
class TableCalculationFilter(_FieldFilterMixin):
"""A filter targeting a table calculation.

Table calculations are referenced by name (no model prefix) and serialize
under ``filters.tableCalculations`` in the query payload.
"""

field: Union[str, "TableCalculation"]
operator: str
values: Union[str, int, float, List[str], List[int], List[float]]

def __post_init__(self):
from lightdash.table_calculations import TableCalculation

if not isinstance(self.values, list):
self.values = [self.values]

if self.operator not in allowed_values:
raise ValueError(
f"Invalid operator '{self.operator}'. "
f"Must be one of: {', '.join(sorted(allowed_values))}"
)

if not isinstance(self.field, (str, TableCalculation)):
raise TypeError(
"field must be a TableCalculation object or table calculation name, "
f"got {type(self.field).__name__}"
)

@property
def field_id(self) -> str:
return self.field if isinstance(self.field, str) else self.field.field_id

def to_dict(self) -> Dict[str, Union[str, List[str]]]:
return {
"target": {"fieldId": self.field_id},
"operator": self.operator,
"values": self.values,
}


FieldFilter = Union[DimensionFilter, TableCalculationFilter]


@dataclass
class CompositeFilter:
"""
Filters are a list of dimension filters that are applied to a query.
Filters are a list of field filters (on dimensions and table calculations)
that are applied to a query.
Later this will also represent complex filters with AND, OR, NOT, etc.
"""

filters: List[DimensionFilter] = field(default_factory=list)
filters: List[FieldFilter] = field(default_factory=list)
aggregation: str = "and"

def __post_init__(self):
Expand All @@ -117,17 +168,26 @@ def __post_init__(self):
)

def to_dict(self):
out = []
dimensions = []
table_calculations = []
for f in self.filters:
# Check that the filter is not a composite filter
if not hasattr(f, "field"):
raise TypeError("Multi-level filter composites not supported yet")
# Multiple filters may target the same field, e.g. a date range
# expressed as (dim >= start) & (dim <= end).
out.append(f.to_dict())
return {"dimensions": {self.aggregation: out}}

def __and__(self, other: Union[DimensionFilter, "CompositeFilter"]) -> "CompositeFilter":
if isinstance(f, TableCalculationFilter):
table_calculations.append(f.to_dict())
else:
dimensions.append(f.to_dict())
out = {"dimensions": {self.aggregation: dimensions}}
# Only include the tableCalculations group when present, so existing
# dimension-only payloads are unchanged.
if table_calculations:
out["tableCalculations"] = {self.aggregation: table_calculations}
return out

def __and__(self, other: Union[FieldFilter, "CompositeFilter"]) -> "CompositeFilter":
"""Combine with another filter using AND: composite & filter"""
if isinstance(other, CompositeFilter):
# Flatten if both are AND composites
Expand All @@ -143,7 +203,7 @@ def __and__(self, other: Union[DimensionFilter, "CompositeFilter"]) -> "Composit
return CompositeFilter(filters=list(self.filters) + [other], aggregation="and")
return CompositeFilter(filters=list(self.filters) + [other], aggregation="and")

def __or__(self, other: Union[DimensionFilter, "CompositeFilter"]) -> "CompositeFilter":
def __or__(self, other: Union[FieldFilter, "CompositeFilter"]) -> "CompositeFilter":
"""Combine with another filter using OR: composite | filter"""
if isinstance(other, CompositeFilter):
# Flatten if both are OR composites
Expand Down
4 changes: 4 additions & 0 deletions lightdash/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ def query(
filters: Optional[Union[DimensionFilter, CompositeFilter]] = None,
sort: Optional[Union[Sort, Sequence[Sort]]] = None,
limit: int = 500,
table_calculations: Optional[Sequence[Any]] = None,
) -> Query:
"""
Create a query against this model.
Expand All @@ -143,6 +144,8 @@ def query(
filters: Optional filters to apply to the query.
sort: Optional Sort object or sequence of Sort objects to order results.
limit: Maximum number of rows to return.
table_calculations: Optional sequence of TableCalculation objects or
raw dicts to include in the query.

Returns:
A Query object that can be used to fetch results or build further.
Expand Down Expand Up @@ -186,6 +189,7 @@ def query(
filters=filters,
sort=sort_seq,
limit=limit,
table_calculations=table_calculations,
)

def list_metrics(self) -> List["Metric"]:
Expand Down
41 changes: 34 additions & 7 deletions lightdash/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from .dimensions import Dimension
from .metrics import Metric
from .filter import DimensionFilter, CompositeFilter
from .filter import DimensionFilter, TableCalculationFilter, CompositeFilter
from .sorting import Sort
from .types import Model
from .exceptions import QueryError, QueryTimeout, QueryCancelled
Expand Down Expand Up @@ -376,10 +376,10 @@ def __init__(
self._dimensions = tuple(dimensions) if dimensions else ()
self._limit = limit

# Handle filters - normalize to CompositeFilter if DimensionFilter is passed
# Handle filters - normalize to CompositeFilter if a single filter is passed
if filters is None:
self._filters = None
elif isinstance(filters, DimensionFilter):
elif isinstance(filters, (DimensionFilter, TableCalculationFilter)):
self._filters = CompositeFilter(filters=[filters])
else:
self._filters = filters
Expand Down Expand Up @@ -448,15 +448,42 @@ def dimensions(self, *dimensions: Union[str, Dimension]) -> "Query":
"""
return self._clone(dimensions=self._dimensions + dimensions)

def filter(self, filter: Union[DimensionFilter, CompositeFilter]) -> "Query":
def table_calculations(self, *table_calculations: Any) -> "Query":
"""
Add table calculations to the query.

Returns a new Query with the specified table calculations added.

Args:
*table_calculations: TableCalculation objects or raw dicts to add

Returns:
A new Query with the table calculations added

Example:
profit_ratio = TableCalculation(
name="profit_ratio",
sql="${orders.profit} / ${orders.revenue}",
)
query = (
model.query()
.metrics(model.metrics.revenue, model.metrics.profit)
.table_calculations(profit_ratio)
.filter(profit_ratio > 0.2)
)
"""
existing = tuple(self._table_calculations) if self._table_calculations else ()
return self._clone(table_calculations=existing + table_calculations)

def filter(self, filter: Union[DimensionFilter, TableCalculationFilter, CompositeFilter]) -> "Query":
"""
Add a filter to the query.

Multiple calls to filter() combine filters with AND logic.
Returns a new Query with the filter added.

Args:
filter: A DimensionFilter or CompositeFilter to apply
filter: A DimensionFilter, TableCalculationFilter or CompositeFilter to apply

Returns:
A new Query with the filter added
Expand All @@ -470,13 +497,13 @@ def filter(self, filter: Union[DimensionFilter, CompositeFilter]) -> "Query":
"""
if self._filters is None:
# First filter
if isinstance(filter, DimensionFilter):
if isinstance(filter, (DimensionFilter, TableCalculationFilter)):
new_filters = CompositeFilter(filters=[filter])
else:
new_filters = filter
else:
# Combine with existing filters using AND
if isinstance(filter, DimensionFilter):
if isinstance(filter, (DimensionFilter, TableCalculationFilter)):
# Add to existing CompositeFilter's list
new_filters = CompositeFilter(
filters=list(self._filters.filters) + [filter],
Expand Down
Loading