diff --git a/docs/tutorials/python/table.md b/docs/tutorials/python/table.md
index dfeebbd84..96f5928af 100644
--- a/docs/tutorials/python/table.md
+++ b/docs/tutorials/python/table.md
@@ -11,6 +11,7 @@ In this tutorial you will:
1. Update your Table
1. Change Table structure
1. Deleting Table rows and Tables
+1. Work with dates and datetimes
## Prerequisites
@@ -24,6 +25,8 @@ In this tutorial you will:
```python
import synapseclient
from synapseclient.models import Column, Project, query, SchemaStorageStrategy, Table
+from datetime import date, datetime, timezone
+import pandas as pd
syn = synapseclient.Synapse()
syn.login()
@@ -216,8 +219,241 @@ Qux2,4,203001,204001,+,False
table.delete()
```
+## 6. Working with dates and datetimes
+
+Synapse `DATE` columns store an exact moment in time, represented as the number
+of milliseconds since `1970-01-01 00:00:00` UTC. Before upload, the client
+converts the datetime values in your data to that representation, and how a
+value is converted depends on whether it carries timezone information. The
+examples below create a small table with a `DATE` column and walk through the
+different ways datetime data can be stored and queried.
+
+```python
+table = Table(
+ name="Sample Collection Dates",
+ parent_id=project.id,
+ columns=[
+ Column(name="sample_id", column_type="STRING", maximum_size=20),
+ Column(name="collected_on", column_type="DATE"),
+ ],
+)
+table = table.store()
+```
+### Storing timezone-aware datetimes (recommended)
+
+A timezone-aware datetime is converted to UTC exactly — the stored value does
+not depend on the timezone settings of the machine performing the upload or on
+the date the upload is run. Localizing with a zone name like
+`"America/Los_Angeles"` interprets each value using the UTC offset in effect on
+that value's own date, so values on either side of a daylight saving switch are
+both stored correctly.
+
+ ```python
+ df = pd.DataFrame(
+ {
+ "sample_id": ["S1", "S2"],
+ "collected_on": [
+ datetime(2017, 2, 14, 11, 23), # winter in Los Angeles (PST, UTC-8)
+ datetime(2018, 10, 1, 9, 30), # summer in Los Angeles (PDT, UTC-7)
+ ],
+ }
+ )
+
+ # Localize the values so each one carries its own timezone information.
+ # tz_localize looks up the UTC offset in effect on each value's own date:
+ # UTC-8 for the February value, UTC-7 for the October value.
+ df["collected_on"] = df["collected_on"].dt.tz_localize("America/Los_Angeles")
+
+ table.store_rows(values=df)
+
+ ```
+
+ The collected_on column will contain timezone-aware datetimes in UTC:
+```
+ sample_id collected_on
+0 S1 2017-02-14 19:23:00+00:00
+1 S2 2018-10-01 16:30:00+00:00
+```
+
+
+
+### Storing naive datetimes
+A naive datetime (one without `tzinfo`) is assumed to be in the local timezone
+of the machine **at the time of upload**, and that single UTC offset is applied
+to every value in the upload.
+
+ ```python
+ naive_df = pd.DataFrame(
+ {
+ "sample_id": ["S3"],
+ "collected_on": [datetime(2017, 2, 14, 11, 23)], # no timezone info; for this example, the local timezone is UTC-7
+
+ }
+ )
+
+ table.store_rows(values=naive_df)
+ ```
+
+
+ The collected_on column will contain timezone-aware datetimes in UTC:
+```
+ sample_id collected_on
+0 S1 2017-02-14 19:23:00+00:00
+1 S2 2018-10-01 16:30:00+00:00
+2 S3 2017-02-14 18:23:00+00:00
+```
+
+
+
+
+This is convenient when your data and your machine share a timezone and daylight
+saving period, but any value whose date falls in a different daylight saving
+period than the upload date is stored shifted by one hour. Here, `2017-02-14
+11:23` is a winter (PST, UTC-8) wall-clock time; uploading it from a machine
+currently on summer time (PDT, UTC-7) stores `18:23` UTC instead of the correct
+`19:23` UTC. Midnight values deserve extra care: the same one-hour shift can
+move a naive midnight to 11 PM of the previous day, changing the calendar date
+the value displays as. To avoid all of this, localize your data first as shown
+in the previous example.
+
+### Storing dates without a time of day
+
+Plain `datetime.date` objects (as opposed to datetimes) are treated as
+`midnight` of that date and converted the same way naive datetimes are: using
+the local timezone of the machine at the time of upload, with the same
+daylight-saving-shift risk described above.
+
+ ```python
+ date_df = pd.DataFrame(
+ {
+ "sample_id": ["S4"],
+ "collected_on": [date(2017, 2, 14)], # a date, not a datetime
+ }
+ )
+
+ table.store_rows(values=date_df)
+
+ query_results = query(
+ f"SELECT * FROM {table.id} ORDER BY collected_on",
+ include_row_id_and_row_version=False,
+ convert_to_datetime=True,
+ )
+ print(query_results)
+ ```
+
+ The collected_on column will contain timezone-aware datetimes in UTC:
+```
+ sample_id collected_on
+0 S1 2017-02-14 19:23:00+00:00
+1 S2 2018-10-01 16:30:00+00:00
+2 S4 2017-02-14 07:00:00+00:00
+3 S3 2017-02-14 18:23:00+00:00
+```
+
+
+
+### Loading date columns from a CSV
+
+When storing rows from a CSV file, use `date_columns` and `date_format` to tell
+the client which columns hold formatted date strings and how to parse them.
+Including the UTC offset in the strings (parsed by `%z`) keeps the values
+timezone-aware, with the same benefits as localizing a DataFrame column.
+
+ ```python
+ csv_content = (
+ "sample_id,collected_on\n"
+ "S5,2017-02-14 11:23 -0800\n"
+ "S6,2018-10-01 09:30 -0700\n"
+ )
+ with open("collection_dates.csv", "w") as f:
+ f.write(csv_content)
+
+ table.store_rows(
+ values="collection_dates.csv",
+ date_columns=["collected_on"],
+ date_format="%Y-%m-%d %H:%M %z",
+ )
+ ```
+
+### Querying datetime data back
+
+Pass `convert_to_datetime=True` to `query` to get `DATE` columns back as
+timezone-aware datetimes in UTC instead of raw epoch-millisecond integers.
+
+ ```python
+ results = query(
+ f"SELECT * FROM {table.id} ORDER BY collected_on",
+ include_row_id_and_row_version=False,
+ convert_to_datetime=True,
+ )
+ print(results)
+ ```
+
+
+ The collected_on column will contain timezone-aware datetimes in UTC:
+```
+ sample_id collected_on
+0 S1 2017-02-14 19:23:00+00:00
+1 S2 2018-10-01 16:30:00+00:00
+2 S3 2017-02-14 18:23:00+00:00
+3 S4 2017-02-14 07:00:00+00:00
+4 S5 2017-02-14 19:23:00+00:00
+5 S6 2018-10-01 16:30:00+00:00
+```
+
+
+~~~python
+# DATE columns are returned as timezone-aware datetimes in UTC. Use tz_convert to view them on another wall clock:
+results["collected_on"] = results["collected_on"].dt.tz_convert("America/Los_Angeles")
+print(results)
+~~~
+
+
+ The collected_on column now displays timezone-aware datetimes converted to America/Los_Angeles time:
+```
+ sample_id collected_on
+0 S1 2017-02-14 19:23:00+00:00
+1 S2 2018-10-01 16:30:00+00:00
+2 S3 2017-02-14 18:23:00+00:00
+3 S4 2017-02-14 07:00:00+00:00
+4 S5 2017-02-14 19:23:00+00:00
+5 S6 2018-10-01 16:30:00+00:00
+```
+
+Note how the timezone-aware rows (`S1`, `S2`, `S5`, `S6`) round-trip back to the
+exact wall-clock times they were created with, while the naive datetime row `S3`
+and date row `S4` were converted using that same upload-time PDT offset. For
+`S4`, that offset is applied to midnight because dates have no time of day.
+
+
+To filter on a `DATE` column in a query, compare it against epoch milliseconds:
+
+```python
+# In a WHERE clause, compare a DATE column against epoch milliseconds:
+# Example: filter for rows where 'collected_on' is on or after January 1, 2018
+cutoff = int(datetime(2018, 1, 1, tzinfo=timezone.utc).timestamp() * 1000)
+recent = query(
+ f"SELECT sample_id FROM {table.id} WHERE collected_on >= {cutoff}",
+ include_row_id_and_row_version=False,
+)
+print(recent)
+```
+
+
+ The printed results will look like:
+```
+ sample_id
+0 S2
+1 S6
+```
+
+
+For the full details of how datetime values are interpreted during upload, see
+the [store_rows][synapseclient.models.Table.store_rows] reference documentation.
+
## References used in this tutorial
- [Project][project-reference-sync]
- [syn.login][synapseclient.Synapse.login]
- [Table][table-reference-sync]
+- [Column][column-reference-sync]
diff --git a/synapseclient/core/utils.py b/synapseclient/core/utils.py
index efe4671d8..2cadfbcb3 100644
--- a/synapseclient/core/utils.py
+++ b/synapseclient/core/utils.py
@@ -686,7 +686,28 @@ def make_bogus_binary_file(
def to_unix_epoch_time(dt: typing.Union[datetime.date, datetime.datetime, str]) -> int:
"""
- Convert either [datetime.date or datetime.datetime objects](http://docs.python.org/2/library/datetime.html) to UNIX time.
+ Convert a datetime, date, or ISO 8601 string to UNIX epoch time in milliseconds.
+
+ Arguments:
+ dt: The value to convert. May be one of:
+
+ - An ISO 8601 formatted string. A trailing `Z` is accepted and treated
+ as UTC (`+00:00`). The parsed value is then handled as a datetime
+ per the rules below.
+ - A `datetime.date` (as opposed to a datetime): interpreted as midnight
+ of that date in the local timezone of the machine at the time of
+ the call.
+ - A naive `datetime.datetime` (no `tzinfo`): assumed to be in the local
+ timezone of the machine at the time of the call. Note that the
+ machine's *current* UTC offset is applied even when the value's own
+ date falls in a different daylight saving period.
+ - A timezone-aware `datetime.datetime`: converted to UTC exactly; the
+ result does not depend on the machine's timezone settings.
+ Subclasses of `datetime.datetime` such as `pandas.Timestamp` are
+ handled this way as well.
+
+ Returns:
+ The number of milliseconds since `1970-01-01 00:00:00` UTC.
"""
if type(dt) == str:
dt = datetime.datetime.fromisoformat(dt.replace("Z", "+00:00"))
diff --git a/synapseclient/models/mixins/table_components.py b/synapseclient/models/mixins/table_components.py
index 44591714d..76b013190 100644
--- a/synapseclient/models/mixins/table_components.py
+++ b/synapseclient/models/mixins/table_components.py
@@ -9,7 +9,7 @@
import uuid
from collections import OrderedDict
from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import date, datetime
from io import BytesIO
from typing import Any, Dict, List, Optional, Protocol, Tuple, Union
@@ -49,6 +49,7 @@
log_dataclass_diff,
merge_dataclass_entities,
test_import_pandas,
+ to_unix_epoch_time,
)
from synapseclient.models import Activity
from synapseclient.models.services.search import get_id
@@ -2277,6 +2278,8 @@ async def _upsert_rows_async(
)
client = Synapse.get_client(synapse_client=synapse_client)
+ # Convert datetime columns to epoch time in milliseconds for Synapse DATE column type
+ values = _convert_df_date_cols_to_epoch_time(df=values)
# Replace pd.NA with None so the columns are converted to object columns instead of 'int64' or 'float64' which are not JSON serializable
values = convert_dtypes_to_json_serializable(values)
@@ -3404,6 +3407,8 @@ async def store_rows_async(
*,
insert_size_bytes: int = 900 * MB,
csv_table_descriptor: Optional[CsvTableDescriptor] = None,
+ date_columns: Optional[List[str]] = None,
+ date_format: Optional[Union[str, Dict[str, str]]] = None,
read_csv_kwargs: Optional[Dict[str, Any]] = None,
to_csv_kwargs: Optional[Dict[str, Any]] = None,
job_timeout: int = 600,
@@ -3434,6 +3439,57 @@ async def store_rows_async(
`INFER_FROM_DATA` the columns will be added at the end of the columns list.
+ **How datetime values are interpreted:**
+
+ Synapse `DATE` columns store an exact moment in time, represented as the
+ number of milliseconds since `1970-01-01 00:00:00` UTC. Before upload,
+ datetime values are converted to that representation as follows:
+
+ - Timezone-aware datetimes (e.g. localized with `zoneinfo.ZoneInfo` or
+ `pandas.Series.dt.tz_localize`) are converted to UTC exactly. This is
+ the recommended way to pass datetime data: the stored value does not
+ depend on the timezone settings of the machine performing the upload
+ or on the date the upload is run.
+ - Naive datetimes (no `tzinfo`) are assumed to be in the local timezone
+ of the machine **at the time of upload**. That single UTC offset is
+ applied to every value, including values whose dates fall in a
+ different daylight saving period, which will be stored shifted by one
+ hour. For example, uploading `2017-02-14 11:23` (a PST date, UTC-8)
+ from a machine currently on PDT (UTC-7) stores `2017-02-14 18:23` UTC
+ instead of the correct `19:23` UTC. To avoid this, localize your data
+ first, e.g. `df["col"] = df["col"].dt.tz_localize("America/Los_Angeles")`.
+ - Plain `datetime.date` objects (as opposed to datetimes) are treated
+ as midnight of that date and converted the same way naive
+ datetimes are: using the local timezone of the machine **at the
+ time of upload**, with the same daylight-saving-shift risk
+ described above.
+ - Midnight values deserve extra care: the one-hour daylight saving
+ shift described above can move a naive midnight datetime — or a
+ `datetime.date`, which is treated as midnight — to 11 PM of the
+ previous day, changing the calendar date the value displays as.
+
+ **Why timezone-aware values are recommended:**
+
+ A naive datetime such as `2017-02-14 11:23` is only "what a clock on the
+ wall said." Before it can be stored as an exact moment in time, something
+ has to answer: *a clock where?* Timezone-aware values answer that
+ question in the data itself; naive values leave the client to guess, and
+ it guesses the uploading machine's current timezone.
+
+ A zone name like `"America/Los_Angeles"` does not mean a fixed offset
+ such as UTC-8. It means "a Los Angeles wall clock" — and those clocks
+ move twice a year: UTC-7 in summer (PDT), UTC-8 in winter (PST). When
+ pandas localizes a column with
+ `df["col"].dt.tz_localize("America/Los_Angeles")`, it looks up what LA
+ clocks were set to on each value's own date:
+
+ - `2017-02-14` → winter → interpreted using UTC-8
+ - `2018-10-01` → summer → interpreted using UTC-7
+
+ When the data is queried back with `query(..., convert_to_datetime=True)`,
+ `DATE` columns are returned as timezone-aware datetimes in UTC. Use
+ `Series.dt.tz_convert` to view them in another timezone.
+
**Limitations:**
- Synapse limits the number of rows that may be stored in a single request to
@@ -3613,10 +3669,38 @@ async def store_rows_async(
[CsvTableDescriptor][synapseclient.models.CsvTableDescriptor]
for more information.
+ date_columns: (CSV file only) The names of columns in your CSV file that
+ contain dates or datetimes stored as formatted strings
+ (e.g. `"2024-01-15"` or `"01/15/2024 13:30"`). The columns are parsed
+ with `pandas.to_datetime` and converted to epoch time in milliseconds
+ before the data is uploaded, which is the format Synapse requires for
+ `DATE` columns. The conversion is done by reading the CSV file into a
+ pandas DataFrame and uploading a temporary copy with the converted
+ values, which requires the CSV file to contain a header row. Because
+ the data passes through pandas type inference, values in the other
+ columns may be normalized (e.g. `"1.10"` becomes `1.1` and literal
+ `"NA"` strings become null values). When `schema_storage_strategy` is
+ set to `INFER_FROM_DATA` the parsed columns will be inferred as `DATE`
+ columns. The parsed values are naive datetimes unless the strings
+ carry a UTC offset — see *How datetime values are interpreted*
+ above for how they are converted to epoch time. When the offsets in
+ a column differ between rows (e.g. a mix of `-0800` and `-0700`
+ values) the values are normalized to UTC, preserving the exact
+ moments in time.
+
+ date_format: (CSV file only) How the strings in `date_columns` are
+ formatted — a
+ [strftime format string](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes)
+ (e.g. `"%m/%d/%Y"`) applied to every column, or a dict mapping column
+ names to their formats. Supply this so that ambiguous dates
+ (e.g. `"01/02/2024"`) are not silently misinterpreted and to optimize the data upload performance. If the values
+ in a column do not match the format a `ValueError` is raised.
+
read_csv_kwargs: Additional arguments to pass to the `pd.read_csv` function
when reading in a CSV file. This is only used when the `values` argument
- is a string holding the path to a CSV file and you have set the
- `schema_storage_strategy` to `INFER_FROM_DATA`. See
+ is a string holding the path to a CSV file and either
+ `schema_storage_strategy` is set to `INFER_FROM_DATA` or `date_columns`
+ is provided. See
for complete list of supported arguments.
@@ -3795,20 +3879,67 @@ async def main():
| B | 2 | 33 |
| C | 3 | 3 |
+ Example: Inserting rows from a CSV file that contains date columns
+
+ This example shows how you may insert rows from a CSV file that contains
+ date columns. The date columns are converted
+ to epoch time in milliseconds before the data is uploaded so that Synapse
+ can store them in `DATE` columns.
+
+ Suppose we have a CSV file with the following data:
+
+ | col1 | date_col1 | date_col2 |
+ |------|------------|------------|
+ | A | 01/15/2024 | 2024-01-20 |
+ | B | 02/20/2024 | 2024-02-25 |
+
+ ```python
+ import asyncio
+ from synapseclient import Synapse
+ from synapseclient.models import Table, SchemaStorageStrategy
+
+ syn = Synapse()
+ syn.login()
+
+ async def main():
+ await Table(id="syn1234").store_rows_async(
+ values="path/to/file.csv",
+ schema_storage_strategy=SchemaStorageStrategy.INFER_FROM_DATA,
+ date_columns=["date_col1", "date_col2"],
+ date_format={"date_col1": "%m/%d/%Y", "date_col2": "%Y-%m-%d"},
+ )
+
+ asyncio.run(main())
+ ```
+
"""
test_import_pandas()
from pandas import DataFrame
to_csv_kwargs = {"escapechar": DEFAULT_ESCAPE_CHAR, **(to_csv_kwargs or {})}
+ read_csv_kwargs = read_csv_kwargs or {}
original_values = values
if isinstance(values, dict):
values = DataFrame(values).convert_dtypes()
- elif (
- isinstance(values, str)
- and schema_storage_strategy == SchemaStorageStrategy.INFER_FROM_DATA
+ elif isinstance(values, str) and (
+ schema_storage_strategy == SchemaStorageStrategy.INFER_FROM_DATA
+ or date_columns
):
- values = csv_to_pandas_df(filepath=values, **(read_csv_kwargs or {}))
+ # ROW_ID/ROW_VERSION must stay as regular columns so they survive the
+ # round-trip to the temporary upload file when `date_columns` is used —
+ # dropping them would turn a row update into an append.
+ values = csv_to_pandas_df(
+ filepath=values,
+ **{"row_id_and_version_in_index": False, **(read_csv_kwargs or {})},
+ )
+ if date_columns:
+ values = _parse_df_date_cols_to_datetime(
+ df=values,
+ date_columns=date_columns,
+ date_format=date_format,
+ synapse_client=synapse_client,
+ )
elif isinstance(values, DataFrame):
values = values.convert_dtypes()
elif isinstance(values, str):
@@ -3862,20 +3993,35 @@ async def main():
raise ValueError(
"The table must have an ID to store rows, or the table could not be found from the given name/parent_id."
)
-
if isinstance(original_values, str):
- with logging_redirect_tqdm(loggers=[client.logger]):
- await self._chunk_and_upload_csv(
- path_to_csv=original_values,
- insert_size_bytes=insert_size_bytes,
+ path_to_csv = original_values
+ temp_csv_with_epoch_dates = None
+ if date_columns:
+ # `values` already has the date columns parsed to datetime by
+ # _parse_df_date_cols_to_datetime
+ temp_csv_with_epoch_dates = _convert_csv_date_cols_to_epoch_time(
+ df=values,
csv_table_descriptor=csv_table_descriptor,
- schema_change_request=schema_change_request,
- client=client,
- additional_changes=additional_changes,
- job_timeout=job_timeout,
)
+ path_to_csv = temp_csv_with_epoch_dates
+ try:
+ with logging_redirect_tqdm(loggers=[client.logger]):
+ await self._chunk_and_upload_csv(
+ path_to_csv=path_to_csv,
+ insert_size_bytes=insert_size_bytes,
+ csv_table_descriptor=csv_table_descriptor,
+ schema_change_request=schema_change_request,
+ client=client,
+ additional_changes=additional_changes,
+ job_timeout=job_timeout,
+ )
+ finally:
+ if temp_csv_with_epoch_dates:
+ os.remove(temp_csv_with_epoch_dates)
elif isinstance(values, DataFrame):
with logging_redirect_tqdm(loggers=[client.logger]):
+ # date columns are converted to epoch time in milliseconds
+ values = _convert_df_date_cols_to_epoch_time(values)
await self._chunk_and_upload_df(
df=values,
insert_size_bytes=insert_size_bytes,
@@ -4658,12 +4804,185 @@ def _convert_df_date_cols_to_datetime(
raise ValueError(
"Cannot convert epoch time to integer. Please make sure that the date columns that you specified contain valid epoch time value"
)
- df[date_columns] = df[date_columns].apply(
- lambda x: to_datetime(x, unit="ms", utc=True)
+ # The trailing astype forces the datetime64 dtype even when df has zero rows —
+ # on an empty DataFrame, apply does not reliably propagate the dtype that
+ # to_datetime returns, leaving the column as float64.
+ df[date_columns] = (
+ df[date_columns]
+ .apply(lambda x: to_datetime(x, unit="ms", utc=True))
+ .astype("datetime64[ns, UTC]")
)
return df
+def _is_date_list_column(series: SERIES_TYPE) -> bool:
+ """
+ Check if a series is a DATE_LIST column.
+
+ Arguments:
+ series: The series to check.
+
+ Returns:
+ True if the series is a DATE_LIST column, False otherwise.
+ """
+ return any(
+ isinstance(cell, (list, tuple))
+ and any(isinstance(item, (date, datetime)) for item in cell if item is not None)
+ for cell in series
+ if cell is not None
+ )
+
+
+def _convert_df_date_cols_to_epoch_time(df: DATA_FRAME_TYPE) -> DATA_FRAME_TYPE:
+ """
+ Convert date columns with datetime values, and DATE_LIST columns holding a
+ Python list of date/datetime values per cell, to epoch time in milliseconds.
+
+ Arguments:
+ df: The pandas dataframe.
+ Returns:
+ A dataframe with datetime columns converted to epoch time in milliseconds
+ """
+ import pandas as pd
+ from pandas.api.types import infer_dtype
+
+ for col in df.columns:
+ dtype = infer_dtype(df[col], skipna=True)
+ if dtype in ("datetime64", "datetime", "date"):
+ df[col] = (
+ df[col]
+ .apply(
+ lambda cell: to_unix_epoch_time(cell) if pd.notna(cell) else None
+ )
+ .astype("Int64")
+ )
+ elif dtype == "mixed" and _is_date_list_column(df[col]):
+ df[col] = df[col].apply(
+ lambda cell: (
+ [
+ to_unix_epoch_time(item) if item is not None else None
+ for item in cell
+ ]
+ if isinstance(cell, (list, tuple))
+ else None
+ )
+ )
+ return df
+
+
+def _parse_df_date_cols_to_datetime(
+ df: DATA_FRAME_TYPE,
+ date_columns: List[str],
+ date_format: Optional[Union[str, Dict[str, str]]] = None,
+ synapse_client: Optional[Synapse] = None,
+) -> DATA_FRAME_TYPE:
+ """
+ Parse date columns holding date strings into datetime values using
+ `pandas.to_datetime`.
+
+ Timezone-naive inputs are converted to timezone-naive; Timezone-aware inputs
+ with constant time offset are converted to timezone-aware. When the offsets in a column
+ differ between rows (e.g. a mix of `-0800` and `-0700` values), the values are normalized to UTC.
+
+ Arguments:
+ df: A pandas dataframe
+ date_columns: The names of the columns holding formatted date strings.
+ date_format: The strftime format of the strings — a single format string
+ applied to every column, or a dict mapping column names to their
+ formats. When `None` the format is inferred by pandas.
+ synapse_client: If not passed in and caching was not disabled by
+ `Synapse.allow_client_caching(False)` this will use the last created
+ instance from the Synapse class constructor.
+ Returns:
+ The dataframe with the date columns parsed to datetime values.
+
+ Raises:
+ ValueError: If a column in `date_columns` is not present in the dataframe,
+ or if the values in a column do not match the supplied format.
+ """
+ test_import_pandas()
+ from pandas import to_datetime
+
+ client = Synapse.get_client(synapse_client=synapse_client)
+
+ missing_cols = list(set(date_columns) - set(df.columns))
+ if missing_cols:
+ raise ValueError(
+ f"The date column(s) {', '.join(missing_cols)} listed in `date_columns` "
+ "are not present in the data. Please ensure that the date columns "
+ "are already in the dataframe."
+ )
+ for col in date_columns:
+ col_format = (
+ date_format.get(col) if isinstance(date_format, dict) else date_format
+ )
+ # TODO: add warning for mixed timezones/offsets if the pandas<3.0 pin is ever lifted.
+ # Mixed offsets will start raising ValueError instead of returning object dtype,
+ # and this spot will need the exception handling back.
+
+ # check if the column holds mixed timezones/offsets by extracting the UTC
+ # offsets; null values have no offset to extract and are dropped so they
+ # don't get counted as a spurious second "offset"
+ offsets = df[col].str.extract(r"([Zz\+\-]\d{2}:?\d{2})$")[0].dropna().unique()
+ if len(offsets) > 1:
+ client.logger.info(
+ f"The date column {col} holds mixed timezones/offsets and will be normalized to UTC."
+ )
+ df[col] = to_datetime(df[col], format=col_format, utc=True)
+ else:
+ df[col] = to_datetime(df[col], format=col_format)
+ return df
+
+
+def _convert_csv_date_cols_to_epoch_time(
+ df: DATA_FRAME_TYPE,
+ csv_table_descriptor: Optional[CsvTableDescriptor] = None,
+) -> str:
+ """
+ Convert the date columns to epoch time in milliseconds and write the result to
+ a temporary CSV file to be uploaded to Synapse. The date columns must already
+ hold datetime values — parse formatted date strings first (e.g. with
+ `_parse_df_date_cols_to_datetime`) before calling this function, as columns
+ of any other type are written out unchanged.
+
+ Arguments:
+ df: The dataframe holding the CSV data to convert.
+ csv_table_descriptor: The descriptor for the CSV file. Used to write the
+ file with the same separator, quote, and escape characters that Synapse
+ will use to parse the uploaded file.
+
+ Returns:
+ The path to a temporary CSV file with the date columns converted to epoch
+ time in milliseconds. The caller is responsible for deleting this file.
+ """
+ test_import_pandas()
+
+ descriptor = csv_table_descriptor or CsvTableDescriptor()
+ if not descriptor.is_first_line_header:
+ raise ValueError(
+ "The CSV file should have a header row to convert date columns to epoch time."
+ )
+
+ df = _convert_df_date_cols_to_epoch_time(df=df)
+
+ fd, temp_path = tempfile.mkstemp(suffix=".csv")
+ os.close(fd)
+ try:
+ df.to_csv(
+ temp_path,
+ index=False,
+ float_format="%.12g",
+ sep=descriptor.separator,
+ quotechar=descriptor.quote_character,
+ escapechar=descriptor.escape_character,
+ lineterminator=descriptor.line_end,
+ )
+ except Exception:
+ os.remove(temp_path)
+ raise
+ return temp_path
+
+
def _row_labels_from_id_and_version(rows: List[Tuple[str, str]]) -> List[str]:
"""
Create a list of row labels from a list of tuples containing row IDs and versions.
diff --git a/synapseclient/models/table.py b/synapseclient/models/table.py
index 4c916849f..9e88a4cba 100644
--- a/synapseclient/models/table.py
+++ b/synapseclient/models/table.py
@@ -412,6 +412,8 @@ def store_rows(
*,
insert_size_bytes: int = 900 * MB,
csv_table_descriptor: Optional[CsvTableDescriptor] = None,
+ date_columns: Optional[List[str]] = None,
+ date_format: Optional[Union[str, Dict[str, str]]] = None,
read_csv_kwargs: Optional[Dict[str, Any]] = None,
to_csv_kwargs: Optional[Dict[str, Any]] = None,
job_timeout: int = 600,
@@ -442,6 +444,66 @@ def store_rows(
`INFER_FROM_DATA` the columns will be added at the end of the columns list.
+ **How datetime values are interpreted:**
+
+ Synapse `DATE` columns store an exact moment in time, represented as the
+ number of milliseconds since `1970-01-01 00:00:00` UTC. Before upload,
+ datetime values are converted to that representation as follows:
+
+ - Timezone-aware datetimes (e.g. localized with `zoneinfo.ZoneInfo` or
+ `pandas.Series.dt.tz_localize`) are converted to UTC exactly. This is
+ the recommended way to pass datetime data: the stored value does not
+ depend on the timezone settings of the machine performing the upload
+ or on the date the upload is run.
+ - Naive datetimes (no `tzinfo`) are assumed to be in the local timezone
+ of the machine **at the time of upload**. That single UTC offset is
+ applied to every value, including values whose dates fall in a
+ different daylight saving period, which will be stored shifted by one
+ hour. For example, uploading `2017-02-14 11:23` (a PST date, UTC-8)
+ from a machine currently on PDT (UTC-7) stores `2017-02-14 18:23` UTC
+ instead of the correct `19:23` UTC. To avoid this, localize your data
+ first, e.g. `df["col"] = df["col"].dt.tz_localize("America/Los_Angeles")`.
+ - Plain `datetime.date` objects (as opposed to datetimes) are treated
+ as midnight of that date and converted the same way naive
+ datetimes are: using the local timezone of the machine **at the
+ time of upload**, with the same daylight-saving-shift risk
+ described above.
+ - Midnight values deserve extra care: the one-hour daylight saving
+ shift described above can move a naive midnight datetime — or a
+ `datetime.date`, which is treated as midnight — to 11 PM of the
+ previous day, changing the calendar date the value displays as.
+
+ **Why timezone-aware values are recommended:**
+
+ A naive datetime such as `2017-02-14 11:23` is only "what a clock on the
+ wall said." Before it can be stored as an exact moment in time, something
+ has to answer: *a clock where?* Timezone-aware values answer that
+ question in the data itself; naive values leave the client to guess, and
+ it guesses the uploading machine's current timezone.
+
+ A zone name like `"America/Los_Angeles"` does not mean a fixed offset
+ such as UTC-8. It means "a Los Angeles wall clock" — and those clocks
+ move twice a year: UTC-7 in summer (PDT), UTC-8 in winter (PST). When
+ pandas localizes a column with
+ `df["col"].dt.tz_localize("America/Los_Angeles")`, it looks up what LA
+ clocks were set to on each value's own date:
+
+ - `2017-02-14` → winter → interpreted using UTC-8
+ - `2018-10-01` → summer → interpreted using UTC-7
+
+ This is like asking "what did a stamp cost when this letter was mailed
+ in 2017?" — you look up the 2017 price, not today's. Naive values skip
+ that lookup: the offset in effect at upload time is applied to every
+ value, like pricing every letter ever mailed at today's stamp rate.
+ Every value whose date falls in the opposite daylight saving period from
+ the upload date is shifted by one hour — a February value uploaded in
+ July displays as `10:23 AM` in the Synapse UI instead of `11:23 AM`, and
+ a midnight value can display as the previous calendar day.
+
+ When the data is queried back with `query(..., convert_to_datetime=True)`,
+ `DATE` columns are returned as timezone-aware datetimes in UTC. Use
+ `Series.dt.tz_convert` to view them in another timezone.
+
**Limitations:**
- Synapse limits the number of rows that may be stored in a single request to
@@ -621,10 +683,38 @@ def store_rows(
[CsvTableDescriptor][synapseclient.models.CsvTableDescriptor]
for more information.
+ date_columns: (CSV file only) The names of columns in your CSV file that
+ contain dates or datetimes stored as formatted strings
+ (e.g. `"2024-01-15"` or `"01/15/2024 13:30"`). The columns are parsed
+ with `pandas.to_datetime` and converted to epoch time in milliseconds
+ before the data is uploaded, which is the format Synapse requires for
+ `DATE` columns. The conversion is done by reading the CSV file into a
+ pandas DataFrame and uploading a temporary copy with the converted
+ values, which requires the CSV file to contain a header row. Because
+ the data passes through pandas type inference, values in the other
+ columns may be normalized (e.g. `"1.10"` becomes `1.1` and literal
+ `"NA"` strings become null values). When `schema_storage_strategy` is
+ set to `INFER_FROM_DATA` the parsed columns will be inferred as `DATE`
+ columns. The parsed values are naive datetimes unless the strings
+ carry a UTC offset — see *How datetime values are interpreted*
+ above for how they are converted to epoch time. When the offsets in
+ a column differ between rows (e.g. a mix of `-0800` and `-0700`
+ values) the values are normalized to UTC, preserving the exact
+ moments in time.
+
+ date_format: (CSV file only) How the strings in `date_columns` are
+ formatted — a
+ [strftime format string](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes)
+ (e.g. `"%m/%d/%Y"`) applied to every column, or a dict mapping column
+ names to their formats. Supply this so that ambiguous dates
+ (e.g. `"01/02/2024"`) are not silently misinterpreted. If the values
+ in a column do not match the format a `ValueError` is raised.
+
read_csv_kwargs: Additional arguments to pass to the `pd.read_csv` function
when reading in a CSV file. This is only used when the `values` argument
- is a string holding the path to a CSV file and you have set the
- `schema_storage_strategy` to `INFER_FROM_DATA`. See
+ is a string holding the path to a CSV file and either
+ `schema_storage_strategy` is set to `INFER_FROM_DATA` or `date_columns`
+ is provided. See
for complete list of supported arguments.
@@ -785,6 +875,35 @@ def store_rows(
| B | 2 | 33 |
| C | 3 | 3 |
+ Example: Inserting rows from a CSV file that contains date columns
+
+ This example shows how you may insert rows from a CSV file that contains
+ date columns. The date columns are converted
+ to epoch time in milliseconds before the data is uploaded so that Synapse
+ can store them in `DATE` columns.
+
+ Suppose we have a CSV file with the following data:
+
+ | col1 | date_col1 | date_col2 |
+ |------|------------|------------|
+ | A | 01/15/2024 | 2024-01-20 |
+ | B | 02/20/2024 | 2024-02-25 |
+
+ ```python
+ from synapseclient import Synapse
+ from synapseclient.models import Table, SchemaStorageStrategy
+
+ syn = Synapse()
+ syn.login()
+
+ Table(id="syn1234").store_rows(
+ values="path/to/file.csv",
+ schema_storage_strategy=SchemaStorageStrategy.INFER_FROM_DATA,
+ date_columns=["date_col1", "date_col2"],
+ date_format={"date_col1": "%m/%d/%Y", "date_col2": "%Y-%m-%d"},
+ )
+ ```
+
"""
return None
diff --git a/tests/integration/synapseclient/models/async/test_table_async.py b/tests/integration/synapseclient/models/async/test_table_async.py
index 5b4fedfc0..58f02a278 100644
--- a/tests/integration/synapseclient/models/async/test_table_async.py
+++ b/tests/integration/synapseclient/models/async/test_table_async.py
@@ -5,8 +5,10 @@
import string
import tempfile
import uuid
+from datetime import date, datetime, timezone
from typing import Callable
from unittest import skip
+from zoneinfo import ZoneInfo
import pandas as pd
import pytest
@@ -917,6 +919,321 @@ async def test_store_rows_as_df_being_split_and_uploaded(
# Note: DataFrames have a minimum of 100 rows per batch
assert spy_send_job.call_count == 3
+ async def test_store_rows_from_df_with_datetime_columns(
+ self, project_model: Project
+ ) -> None:
+ table_name = str(uuid.uuid4())
+ table = Table(
+ name=table_name,
+ parent_id=project_model.id,
+ columns=[
+ Column(name="column_string", column_type=ColumnType.STRING),
+ Column(name="column_date_tz_aware", column_type=ColumnType.DATE),
+ Column(name="column_date_dst", column_type=ColumnType.DATE),
+ Column(name="column_date_naive", column_type=ColumnType.DATE),
+ ],
+ )
+ table = await table.store_async(synapse_client=self.syn)
+ self.schedule_for_cleanup(table.id)
+
+ # AND a DataFrame with tz-aware and naive datetime values, including nulls.
+ # The DST column uses a zone that observes daylight saving time, with one
+ # winter value (PST, UTC-8) and one summer value (PDT, UTC-7)
+ tz_aware_dates = [
+ datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
+ None,
+ datetime(2021, 1, 3, 12, 0, 0, tzinfo=timezone.utc),
+ ]
+ dst_dates = [
+ datetime(2021, 1, 1, 12, 0, 0, tzinfo=ZoneInfo("America/Los_Angeles")),
+ None,
+ datetime(2021, 7, 1, 12, 0, 0, tzinfo=ZoneInfo("America/Los_Angeles")),
+ ]
+ naive_dates = [
+ datetime(2021, 1, 1, 12, 0, 0),
+ None,
+ datetime(2021, 1, 3, 12, 0, 0),
+ ]
+ data_for_table = pd.DataFrame(
+ {
+ "column_string": ["value1", "value2", "value3"],
+ "column_date_tz_aware": tz_aware_dates,
+ "column_date_dst": dst_dates,
+ "column_date_naive": naive_dates,
+ }
+ )
+
+ await table.store_rows_async(
+ values=data_for_table,
+ schema_storage_strategy=None,
+ synapse_client=self.syn,
+ )
+
+ results = await query_async(
+ f"SELECT * FROM {table.id}",
+ synapse_client=self.syn,
+ include_row_id_and_row_version=False,
+ )
+
+ # The DST column expectations are hardcoded to prove each value converts using the
+ # UTC offset in effect on its own date (12:00-08:00 -> 20:00 UTC and
+ # 12:00-07:00 -> 19:00 UTC), independent of the machine running the test
+ expected_results = pd.DataFrame(
+ {
+ "column_string": ["value1", "value2", "value3"],
+ "column_date_tz_aware": [
+ utils.to_unix_epoch_time(date) if date else None
+ for date in tz_aware_dates
+ ],
+ "column_date_dst": [1609531200000, None, 1625166000000],
+ "column_date_naive": [
+ utils.to_unix_epoch_time(date) if date else None
+ for date in naive_dates
+ ],
+ }
+ )
+ pd.testing.assert_frame_equal(results, expected_results, check_dtype=False)
+
+ async def test_store_rows_from_df_with_date_object_column(
+ self, project_model: Project
+ ) -> None:
+ table_name = str(uuid.uuid4())
+ table = Table(
+ name=table_name,
+ parent_id=project_model.id,
+ columns=[
+ Column(name="column_string", column_type=ColumnType.STRING),
+ Column(name="column_date", column_type=ColumnType.DATE),
+ ],
+ )
+ table = await table.store_async(synapse_client=self.syn)
+ self.schedule_for_cleanup(table.id)
+
+ date_values = [date(2021, 1, 1), date(2021, 7, 1)]
+ data_for_table = pd.DataFrame(
+ {
+ "column_string": ["value1", "value2"],
+ "column_date": date_values,
+ }
+ )
+
+ await table.store_rows_async(
+ values=data_for_table,
+ schema_storage_strategy=None,
+ synapse_client=self.syn,
+ )
+
+ results = await query_async(
+ f"SELECT * FROM {table.id}",
+ synapse_client=self.syn,
+ include_row_id_and_row_version=False,
+ )
+
+ expected_results = pd.DataFrame(
+ {
+ "column_string": ["value1", "value2"],
+ "column_date": [
+ utils.to_unix_epoch_time(value)
+ for value in date_values # date columns are converted to epoch ms (midnight local timezone, unit tests run with TZ=UTC)
+ ],
+ }
+ )
+ pd.testing.assert_frame_equal(results, expected_results, check_dtype=False)
+
+ async def test_store_rows_from_csv_with_date_columns(
+ self, project_model: Project
+ ) -> None:
+ table_name = str(uuid.uuid4())
+ table = Table(
+ name=table_name,
+ parent_id=project_model.id,
+ columns=[
+ Column(name="column_string", column_type=ColumnType.STRING),
+ Column(name="column_date", column_type=ColumnType.DATE),
+ ],
+ )
+ table = await table.store_async(synapse_client=self.syn)
+ self.schedule_for_cleanup(table.id)
+
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as csv_file:
+ csv_file.write(
+ "column_string,column_date\n"
+ "value1,01/15/2024\n"
+ "value2,\n"
+ "value3,02/20/2024\n"
+ )
+
+ try:
+ await table.store_rows_async(
+ values=csv_file.name,
+ schema_storage_strategy=None,
+ date_columns=["column_date"],
+ date_format="%m/%d/%Y",
+ synapse_client=self.syn,
+ )
+ finally:
+ os.remove(csv_file.name)
+
+ results = await query_async(
+ f"SELECT * FROM {table.id}",
+ synapse_client=self.syn,
+ include_row_id_and_row_version=False,
+ )
+
+ expected_results = pd.DataFrame(
+ {
+ "column_string": ["value1", "value2", "value3"],
+ "column_date": [
+ utils.to_unix_epoch_time(datetime(2024, 1, 15)),
+ None,
+ utils.to_unix_epoch_time(datetime(2024, 2, 20)),
+ ],
+ }
+ )
+ pd.testing.assert_frame_equal(results, expected_results, check_dtype=False)
+
+ async def test_store_rows_from_csv_with_tz_aware_date_columns(
+ self, project_model: Project
+ ) -> None:
+ table_name = str(uuid.uuid4())
+ table = Table(
+ name=table_name,
+ parent_id=project_model.id,
+ columns=[
+ Column(name="column_string", column_type=ColumnType.STRING),
+ Column(name="column_date", column_type=ColumnType.DATE),
+ ],
+ )
+ table = await table.store_async(synapse_client=self.syn)
+ self.schedule_for_cleanup(table.id)
+
+ # AND a CSV file whose date strings carry an explicit UTC offset. These
+ # parse into timezone-aware values
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as csv_file:
+ csv_file.write(
+ "column_string,column_date\n"
+ "value1,01/15/2024 12:00 -0800\n"
+ "value2,\n"
+ "value3,02/20/2024 12:00 -0800\n"
+ )
+
+ try:
+ await table.store_rows_async(
+ values=csv_file.name,
+ schema_storage_strategy=None,
+ date_columns=["column_date"],
+ date_format="%m/%d/%Y %H:%M %z",
+ synapse_client=self.syn,
+ )
+ finally:
+ os.remove(csv_file.name)
+
+ results = await query_async(
+ f"SELECT * FROM {table.id}",
+ synapse_client=self.syn,
+ include_row_id_and_row_version=False,
+ )
+
+ expected_results = pd.DataFrame(
+ {
+ "column_string": ["value1", "value2", "value3"],
+ "column_date": [
+ 1705348800000,
+ None,
+ 1708459200000,
+ ], # date columns are converted to epoch ms
+ }
+ )
+ pd.testing.assert_frame_equal(results, expected_results, check_dtype=False)
+
+ async def test_store_rows_from_csv_with_row_ids_and_date_columns_updates_rows(
+ self, project_model: Project
+ ) -> None:
+ table_name = str(uuid.uuid4())
+ table = Table(
+ name=table_name,
+ parent_id=project_model.id,
+ columns=[
+ Column(name="column_string", column_type=ColumnType.STRING),
+ Column(name="column_date", column_type=ColumnType.DATE),
+ ],
+ )
+ table = await table.store_async(synapse_client=self.syn)
+ self.schedule_for_cleanup(table.id)
+
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as csv_file:
+ csv_file.write(
+ "column_string,column_date\n"
+ "value1,01/15/2024\n"
+ "value2,02/20/2024\n"
+ )
+ try:
+ await table.store_rows_async(
+ values=csv_file.name,
+ schema_storage_strategy=None,
+ date_columns=["column_date"],
+ date_format="%m/%d/%Y",
+ synapse_client=self.syn,
+ )
+ finally:
+ os.remove(csv_file.name)
+
+ stored_rows = await query_async(
+ f"SELECT * FROM {table.id}",
+ synapse_client=self.syn,
+ )
+
+ updated_values = [("updated1", "03/10/2024"), ("updated2", "04/25/2024")]
+ update_lines = [
+ f"{row.ROW_ID},{row.ROW_VERSION},{new_string},{new_date}"
+ for row, (new_string, new_date) in zip(
+ stored_rows.itertuples(), updated_values
+ )
+ ]
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as update_file:
+ update_file.write(
+ "ROW_ID,ROW_VERSION,column_string,column_date\n"
+ + "\n".join(update_lines)
+ + "\n"
+ )
+
+ try:
+ await table.store_rows_async(
+ values=update_file.name,
+ schema_storage_strategy=None,
+ date_columns=["column_date"],
+ date_format="%m/%d/%Y",
+ synapse_client=self.syn,
+ )
+ finally:
+ os.remove(update_file.name)
+
+ # AND I query the table again
+ results = await query_async(
+ f"SELECT * FROM {table.id}",
+ synapse_client=self.syn,
+ include_row_id_and_row_version=False,
+ )
+
+ expected_results = pd.DataFrame(
+ {
+ "column_string": ["updated1", "updated2"],
+ "column_date": [
+ utils.to_unix_epoch_time(datetime(2024, 3, 10)),
+ utils.to_unix_epoch_time(datetime(2024, 4, 25)),
+ ],
+ }
+ )
+ pd.testing.assert_frame_equal(results, expected_results, check_dtype=False)
+
@skip("Skip in normal testing because the large size makes it slow")
async def test_store_rows_as_large_df_being_split_and_uploaded(
self, project_model: Project, mocker: MockerFixture
@@ -1621,9 +1938,9 @@ async def test_upsert_all_data_types(self, project_model: Project) -> None:
"column_integer": [1, None, 3],
"column_boolean": [True, None, True],
"column_date": [
- utils.to_unix_epoch_time("2021-01-01"),
+ datetime(2021, 1, 1),
None,
- utils.to_unix_epoch_time("2021-01-03"),
+ datetime(2021, 1, 3),
],
# Reference types
"column_filehandleid": [
@@ -1670,13 +1987,13 @@ async def test_upsert_all_data_types(self, project_model: Project) -> None:
],
"column_date_LIST": [
[
- utils.to_unix_epoch_time("2021-01-01"),
- utils.to_unix_epoch_time("2021-01-02"),
+ datetime(2021, 1, 1),
+ datetime(2021, 1, 2),
],
None,
[
- utils.to_unix_epoch_time("2021-01-05"),
- utils.to_unix_epoch_time("2021-01-06"),
+ datetime(2021, 1, 5),
+ datetime(2021, 1, 6),
],
],
"column_entity_id_list": [
@@ -1805,9 +2122,9 @@ async def test_upsert_all_data_types(self, project_model: Project) -> None:
"column_integer": [11, None, 33],
"column_boolean": [False, None, False],
"column_date": [
- utils.to_unix_epoch_time("2022-01-01"),
+ datetime(2022, 1, 1, tzinfo=timezone.utc),
None,
- utils.to_unix_epoch_time("2022-01-03"),
+ datetime(2022, 1, 3, tzinfo=timezone.utc),
],
# Updated references
"column_filehandleid": [
@@ -1854,13 +2171,13 @@ async def test_upsert_all_data_types(self, project_model: Project) -> None:
],
"column_date_LIST": [
[
- utils.to_unix_epoch_time("2022-01-01"),
- utils.to_unix_epoch_time("2022-01-02"),
+ datetime(2022, 1, 1),
+ datetime(2022, 1, 2),
],
None,
[
- utils.to_unix_epoch_time("2022-01-05"),
- utils.to_unix_epoch_time("2022-01-06"),
+ datetime(2022, 1, 5),
+ datetime(2022, 1, 6),
],
],
"column_entity_id_list": [
@@ -1911,9 +2228,9 @@ async def test_upsert_all_data_types(self, project_model: Project) -> None:
"column_integer": [11, None, 33],
"column_boolean": [False, None, False],
"column_date": [
- utils.to_unix_epoch_time("2022-01-01"),
+ utils.to_unix_epoch_time(datetime(2022, 1, 1, tzinfo=timezone.utc)),
None,
- utils.to_unix_epoch_time("2022-01-03"),
+ utils.to_unix_epoch_time(datetime(2022, 1, 3, tzinfo=timezone.utc)),
],
"column_filehandleid": [
file2.file_handle.id,
@@ -1983,9 +2300,9 @@ async def test_upsert_all_data_types(self, project_model: Project) -> None:
"column_integer": [1, 2, 3],
"column_boolean": [True, True, True],
"column_date": [
- utils.to_unix_epoch_time("2021-01-01"),
- utils.to_unix_epoch_time("2021-01-02"),
- utils.to_unix_epoch_time("2021-01-03"),
+ datetime(2021, 1, 1),
+ datetime(2021, 1, 2),
+ datetime(2021, 1, 3),
],
"column_filehandleid": [
file.file_handle.id,
@@ -2029,13 +2346,13 @@ async def test_upsert_all_data_types(self, project_model: Project) -> None:
],
"column_date_LIST": [
[
- utils.to_unix_epoch_time("2023-01-01"),
- utils.to_unix_epoch_time("2023-01-02"),
+ datetime(2023, 1, 1),
+ datetime(2023, 1, 2),
],
None,
[
- utils.to_unix_epoch_time("2023-01-05"),
- utils.to_unix_epoch_time("2023-01-06"),
+ datetime(2023, 1, 5),
+ datetime(2023, 1, 6),
],
],
"column_entity_id_list": [
diff --git a/tests/unit/synapseclient/mixins/unit_test_table_components.py b/tests/unit/synapseclient/mixins/unit_test_table_components.py
index e10270dad..c61b30408 100644
--- a/tests/unit/synapseclient/mixins/unit_test_table_components.py
+++ b/tests/unit/synapseclient/mixins/unit_test_table_components.py
@@ -1,10 +1,13 @@
+import hashlib
import os
import re
+import tempfile
from collections import OrderedDict
from dataclasses import dataclass, field
+from datetime import date, datetime
from io import BytesIO
from typing import Any, Dict, List, Optional
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import numpy as np
import pandas as pd
@@ -29,6 +32,7 @@
SnapshotRequest,
TableDeleteRowMixin,
TableStoreMixin,
+ TableStoreRowMixin,
TableUpdateTransaction,
TableUpsertMixin,
ViewSnapshotMixin,
@@ -39,7 +43,11 @@
_construct_partial_rows_for_upsert,
_construct_select_statement_for_upsert,
_construct_single_key_where_statement,
+ _convert_csv_date_cols_to_epoch_time,
+ _convert_df_date_cols_to_epoch_time,
_format_primary_key_value_for_where,
+ _is_date_list_column,
+ _parse_df_date_cols_to_datetime,
_query_table_csv,
_query_table_next_page,
_query_table_row_set,
@@ -61,6 +69,7 @@
QueryResultOutput,
Row,
RowSet,
+ SchemaStorageStrategy,
SelectColumn,
SumFileSizes,
)
@@ -4503,6 +4512,20 @@ def test_csv_to_pandas_df_with_date_columns(self, csv_with_date_columns):
df["created_date"], pd.Series(expected_dates), check_names=False
)
+ def test_csv_to_pandas_df_with_date_columns_and_no_rows(self):
+ # GIVEN a CSV with a date column but no data rows — e.g. an empty query
+ # result for a table with a DATE column
+ csv_file = BytesIO(b"id,name,created_date\n")
+
+ # WHEN converting the CSV with date_columns specified
+ df = csv_to_pandas_df(filepath=csv_file, date_columns=["created_date"])
+
+ # THEN the date column is still datetime64, not left as the
+ # intermediate float64 used to parse epoch milliseconds — otherwise
+ # a `.dt` accessor on the result would raise an AttributeError
+ assert df.empty
+ assert str(df["created_date"].dtype) == "datetime64[ns, UTC]"
+
def test_csv_to_pandas_df_with_all_list_columns(self, csv_with_list_columns):
"""Test csv_to_pandas_df correctly parses all list column types together."""
# WHEN converting CSV with all list column types
@@ -5028,3 +5051,828 @@ def test_nullable_int64_with_pd_na(self):
).convert_dtypes()
pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
assert is_object_dtype(result.nullable_int_col)
+
+
+@pytest.mark.parametrize(
+ "series,expected",
+ [
+ pytest.param(
+ pd.Series(
+ [[datetime(2021, 1, 1), datetime(2021, 1, 2)], None], dtype=object
+ ),
+ True,
+ id="list_of_datetimes",
+ ),
+ pytest.param(
+ pd.Series([[date(2021, 1, 1), date(2021, 1, 2)], None], dtype=object),
+ True,
+ id="list_of_dates",
+ ),
+ pytest.param(
+ pd.Series([(date(2021, 1, 1), date(2021, 1, 2))], dtype=object),
+ True,
+ id="tuple_of_dates",
+ ),
+ pytest.param(
+ pd.Series([[None, date(2021, 1, 1)]], dtype=object),
+ True,
+ id="list_with_none_items_and_a_date",
+ ),
+ pytest.param(
+ pd.Series([[1, 2], [5, 6], None], dtype=object),
+ False,
+ id="list_of_non_date_values",
+ ),
+ pytest.param(
+ pd.Series([["a", "b"], None], dtype=object),
+ False,
+ id="list_of_strings",
+ ),
+ pytest.param(
+ # a plain (non-list) datetime column, which
+ # _convert_df_date_cols_to_epoch_time handles via a separate branch
+ pd.Series([datetime(2021, 1, 1), None], dtype=object),
+ False,
+ id="scalar_datetime_column",
+ ),
+ pytest.param(
+ pd.Series([None, None], dtype=object),
+ False,
+ id="all_null_column",
+ ),
+ pytest.param(
+ pd.Series([[], []], dtype=object),
+ False,
+ id="empty_lists_only",
+ ),
+ ],
+)
+def test_is_date_list_column(series, expected):
+ assert _is_date_list_column(series) is expected
+
+
+class TestConvertDfDateColsToEpochTime:
+ """Tests for _convert_df_date_cols_to_epoch_time. Unit tests run with
+ TZ=UTC by default, so naive datetimes convert deterministically."""
+
+ def test_dataframe_without_datetime_cols_is_unchanged(self):
+ df = pd.DataFrame({"col1": ["a", "b"], "col2": [1, 2]})
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ pd.testing.assert_frame_equal(result, df)
+
+ def test_empty_datetime_column_is_still_converted(self):
+ # GIVEN a datetime64 column with zero rows — e.g. an upsert/store call
+ # made with an empty dataframe
+ df = pd.DataFrame({"dt": pd.to_datetime([], utc=True)})
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ # THEN the column is still converted to the nullable Int64 dtype used
+ # for epoch milliseconds, not left as datetime64
+ assert result.empty
+ assert is_integer_dtype(result["dt"])
+
+ def test_null_values_keep_integer_dtype(self):
+ df = pd.DataFrame(
+ {
+ "col1": ["a", "b"],
+ "dt": pd.to_datetime([1487071391024, None], unit="ms", utc=True),
+ }
+ )
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ # THEN the values are interpreted as UTC wall-clock times
+ expected_result = pd.DataFrame(
+ {"col1": ["a", "b"], "dt": [1487071391024, None]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+ assert is_integer_dtype(result.dt)
+
+ def test_timezone_aware_datetimes_converted_to_utc_exactly(self):
+ # GIVEN values localized to a zone that observes daylight saving time,
+ # one in winter (PST, UTC-8) and one in summer (PDT, UTC-7)
+ df = pd.DataFrame(
+ {
+ "dt_aware": pd.to_datetime(
+ [
+ datetime(2017, 2, 14, 11, 23, 11, 240000),
+ datetime(2018, 10, 1),
+ ]
+ ).tz_localize("America/Los_Angeles"),
+ }
+ )
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ # THEN each value converts using the UTC offset in effect on its own
+ # date (2017-02-14 19:23:11.240 UTC and 2018-10-01 07:00 UTC), not the
+ # timezone of the machine running the conversion
+ expected_result = pd.DataFrame(
+ {"dt_aware": [1487100191240, 1538377200000]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+
+ def test_naive_datetimes_interpreted_as_local_timezone(self):
+ # GIVEN naive datetimes; unit tests run with TZ=UTC (see conftest.py),
+ # so "the machine's local timezone at upload time" is UTC here
+ df = pd.DataFrame(
+ {
+ "dt_naive": [
+ datetime(2017, 2, 14, 11, 23, 11, 240000),
+ datetime(
+ 2018, 10, 1
+ ), # convert to midnight of the date in the local timezone
+ ]
+ }
+ )
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ # THEN the date column is converted to epoch ms (midnight local timezone,
+ # unit tests run with TZ=UTC)
+ expected_result = pd.DataFrame(
+ {"dt_naive": [1487071391240, 1538352000000]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+
+ def test_date_object_columns_are_converted(self):
+ # GIVEN a column of plain datetime.date objects, conversion should be done to midnight local timezone.
+ df = pd.DataFrame(
+ {"date_col": [date(2017, 2, 14), date(2018, 10, 1)], "other": ["a", "b"]}
+ )
+
+ # WHEN the dataframe is converted
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ # THEN the date column is converted to epoch ms (midnight local timezone,
+ # unit tests run with TZ=UTC)
+ expected_result = pd.DataFrame(
+ {"date_col": [1487030400000, 1538352000000], "other": ["a", "b"]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+ assert is_integer_dtype(result["date_col"])
+
+ def test_date_object_columns_with_nulls_keep_integer_dtype(self):
+ # GIVEN a column of datetime.date objects mixed with a null value
+ df = pd.DataFrame({"date_col": [date(2017, 2, 14), None]})
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ expected_result = pd.DataFrame(
+ {"date_col": [1487030400000, None]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+ assert is_integer_dtype(result["date_col"])
+
+ def test_date_list_columns_are_converted(self):
+ # GIVEN a DATE_LIST column: each cell is a Python list of datetimes,
+ # which pandas can only ever infer as "mixed" dtype, not "datetime"
+ df = pd.DataFrame(
+ {
+ "other": ["a", None],
+ "date_list_col": [
+ [
+ datetime(2017, 2, 14, 11, 23, 11, 240000),
+ datetime(2018, 10, 1),
+ ],
+ None,
+ ],
+ "date_list_col": [[date(2017, 2, 14), date(2018, 10, 1)], None],
+ }
+ )
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ # THEN every element of every list cell is converted to epoch ms
+ # (midnight local timezone for the date-only value; unit tests run
+ # with TZ=UTC)
+ expected_result = pd.DataFrame(
+ {
+ "other": ["a", None],
+ "date_list_col": [
+ [1487071391240, 1538352000000],
+ None,
+ ],
+ "date_list_col": [[1487030400000, 1538352000000], None],
+ }
+ )
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+
+ def test_integer_list_columns_are_unaffected(self):
+ # GIVEN an INTEGER_LIST column, which also infers as "mixed" dtype but
+ # holds no date/datetime values
+ df = pd.DataFrame({"int_list_col": [[1, 2], None, [5, 6]]})
+
+ result = _convert_df_date_cols_to_epoch_time(df=df)
+
+ pd.testing.assert_frame_equal(result, df)
+
+
+class TestParseDfDateColsToDatetime:
+ """Tests for parsing date columns holding formatted date strings via
+ `_parse_df_date_cols_to_datetime`."""
+
+ @pytest.fixture(autouse=True, scope="function")
+ def init_syn(self, syn: Synapse) -> None:
+ self.syn = syn
+
+ def test_date_strings_parsed_with_format(self):
+ csv_buffer = BytesIO(b"col1,date_col\na,01/15/2024\nb,\n")
+ df = csv_to_pandas_df(filepath=csv_buffer, row_id_and_version_in_index=False)
+
+ result = _parse_df_date_cols_to_datetime(
+ df=df,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y",
+ )
+ expected_result = pd.DataFrame(
+ {"col1": ["a", "b"], "date_col": [datetime(2024, 1, 15), None]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+
+ def test_date_format_as_dict_maps_columns_to_formats(self):
+ df = pd.DataFrame(
+ {
+ "date_col1": ["01/15/2024"],
+ "date_col2": ["2024-01-20"],
+ "datetime_col3": ["2024-02-01 14:30:00"],
+ }
+ ).convert_dtypes()
+
+ result = _parse_df_date_cols_to_datetime(
+ df=df,
+ date_columns=["date_col1", "date_col2", "datetime_col3"],
+ date_format={
+ "date_col1": "%m/%d/%Y",
+ "date_col2": "%Y-%m-%d",
+ "datetime_col3": "%Y-%m-%d %H:%M:%S",
+ },
+ )
+ # naive datetime strings are converted to naive datetimes
+ expected_result = pd.DataFrame(
+ {
+ "date_col1": [datetime(2024, 1, 15)],
+ "date_col2": [datetime(2024, 1, 20)],
+ "datetime_col3": [datetime(2024, 2, 1, 14, 30, 0)],
+ }
+ )
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+
+ def test_missing_date_column_raises(self):
+ df = pd.DataFrame({"col1": ["a"]})
+
+ with pytest.raises(
+ ValueError,
+ match=re.escape(
+ "The date column(s) date_col listed in `date_columns` "
+ "are not present in the data. Please ensure that the date columns "
+ "are already in the dataframe."
+ ),
+ ):
+ _parse_df_date_cols_to_datetime(
+ df=df,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y",
+ )
+
+ def test_date_string_not_matching_format_raises(self):
+ # GIVEN date strings that do not match the given format
+ df = pd.DataFrame({"date_col": ["2024-01-15"]})
+
+ with pytest.raises(ValueError):
+ _parse_df_date_cols_to_datetime(
+ df=df,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y",
+ )
+
+ def test_uniform_utc_offsets_parsed_as_tz_aware(self):
+ # GIVEN date strings that all carry the same UTC offset
+ df = pd.DataFrame(
+ {"date_col": ["01/15/2024 12:00 -0800", None, "02/20/2024 12:00 -0800"]}
+ ).convert_dtypes()
+
+ # WHEN the column is parsed
+ with patch.object(self.syn, "logger") as mock_logger:
+ result = _parse_df_date_cols_to_datetime(
+ df=df,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y %H:%M %z",
+ synapse_client=self.syn,
+ )
+
+ # THEN the column holds timezone-aware datetimes carrying that offset
+ assert isinstance(result["date_col"].dtype, pd.DatetimeTZDtype)
+ assert result["date_col"].tolist() == [
+ pd.Timestamp("2024-01-15 12:00:00-0800"),
+ pd.NaT,
+ pd.Timestamp("2024-02-20 12:00:00-0800"),
+ ]
+ # AND no mixed-offset normalization message is logged
+ mock_logger.info.assert_not_called()
+
+ def test_mixed_utc_offsets_normalized_to_utc(self):
+ # GIVEN date strings whose UTC offsets differ between rows, as produced
+ # by exporting a zone that observes daylight saving time with each
+ # value's true offset (winter -0800, summer -0700)
+ df = pd.DataFrame(
+ {"date_col": ["01/15/2024 12:00 -0800", "07/15/2024 12:00 -0700"]}
+ ).convert_dtypes()
+
+ # WHEN the column is parsed
+ with patch.object(self.syn, "logger") as mock_logger:
+ result = _parse_df_date_cols_to_datetime(
+ df=df,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y %H:%M %z",
+ synapse_client=self.syn,
+ )
+
+ # THEN the values are normalized to UTC using each value's own offset,
+ # preserving the exact moments in time
+ assert str(result["date_col"].dtype) == "datetime64[ns, UTC]"
+ assert result["date_col"].tolist() == [
+ pd.Timestamp("2024-01-15 20:00:00", tz="UTC"),
+ pd.Timestamp("2024-07-15 19:00:00", tz="UTC"),
+ ]
+ # AND a message is logged noting the normalization
+ mock_logger.info.assert_called_once_with(
+ "The date column date_col holds mixed timezones/offsets and will be normalized to UTC."
+ )
+
+
+class TestConvertCsvDateColsToEpochTime:
+ """Tests for _convert_csv_date_cols_to_epoch_time. Unit tests run with
+ TZ=UTC, so naive datetimes convert deterministically."""
+
+ def test_date_cols_converted_to_epoch_ms(self):
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as csv_file:
+ csv_file.write("col1,date_col\na,01/15/2024\nb,\nc,02/20/2024\n")
+ result_path = None
+ try:
+ df = csv_to_pandas_df(
+ filepath=csv_file.name,
+ row_id_and_version_in_index=False,
+ )
+ df = _parse_df_date_cols_to_datetime(
+ df=df, date_columns=["date_col"], date_format="%m/%d/%Y"
+ )
+
+ result_path = _convert_csv_date_cols_to_epoch_time(df=df)
+
+ result = pd.read_csv(result_path)
+ expected_result = pd.DataFrame(
+ {
+ "col1": ["a", "b", "c"],
+ "date_col": [
+ 1705276800000,
+ None,
+ 1708387200000,
+ ], # date columns are converted to epoch ms (midnight local timezone, unit tests run with TZ=UTC)
+ }
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+ finally:
+ os.remove(csv_file.name)
+ if result_path:
+ os.remove(result_path)
+
+ def test_temp_file_written_with_csv_table_descriptor_format(self):
+ # GIVEN a DataFrame with a parsed date column and a tab-separated descriptor
+ df = pd.DataFrame(
+ {"col1": ["a", "b"], "date_col": ["01/15/2024", "02/20/2024"]}
+ ).convert_dtypes()
+ df["date_col"] = pd.to_datetime(df["date_col"], format="%m/%d/%Y")
+
+ result_path = _convert_csv_date_cols_to_epoch_time(
+ df=df,
+ csv_table_descriptor=CsvTableDescriptor(separator="\t"),
+ )
+ try:
+ result = pd.read_csv(result_path, sep="\t")
+ expected_result = pd.DataFrame(
+ {
+ "col1": ["a", "b"],
+ "date_col": [1705276800000, 1708387200000],
+ } # date columns are converted to epoch ms (midnight local timezone, unit tests run with TZ=UTC)
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+ finally:
+ os.remove(result_path)
+
+ def test_mixed_utc_offsets_converted_to_exact_epoch_ms(self):
+ # GIVEN date strings whose UTC offsets differ between rows, as produced
+ # by exporting a zone that observes daylight saving time with each
+ # value's true offset (winter -0800, summer -0700). The parse step
+ # normalizes them to UTC
+ csv_buffer = BytesIO(
+ b"date_col\n01/15/2024 12:00 -0800\n07/15/2024 12:00 -0700\n"
+ )
+ df = csv_to_pandas_df(
+ filepath=csv_buffer,
+ row_id_and_version_in_index=False,
+ )
+ df = _parse_df_date_cols_to_datetime(
+ df=df, date_columns=["date_col"], date_format="%m/%d/%Y %H:%M %z"
+ )
+
+ # WHEN the dataframe is written to the upload file
+ result_path = _convert_csv_date_cols_to_epoch_time(df=df)
+
+ try:
+ result = pd.read_csv(result_path)
+ # THEN each value converts using the UTC offset it carried
+ # (12:00-08:00 -> 20:00 UTC, 12:00-07:00 -> 19:00 UTC), preserving
+ # the exact moments in time independent of the machine's timezone
+ expected_result = pd.DataFrame(
+ {"date_col": [1705348800000, 1721070000000]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(result, expected_result, check_dtype=False)
+ finally:
+ os.remove(result_path)
+
+ def test_headerless_descriptor_raises(self):
+ # GIVEN a descriptor stating the CSV file has no header row
+ df = pd.DataFrame({"col1": ["a"], "date_col": ["01/15/2024"]})
+
+ # WHEN the date columns are converted THEN a ValueError is raised
+ with pytest.raises(
+ ValueError,
+ match="The CSV file should have a header row to convert date columns to epoch time.",
+ ):
+ _convert_csv_date_cols_to_epoch_time(
+ df=df,
+ csv_table_descriptor=CsvTableDescriptor(is_first_line_header=False),
+ )
+
+
+class TestTableStoreRowMixin:
+ @pytest.fixture(autouse=True, scope="function")
+ def init_syn(self, syn: Synapse) -> None:
+ self.syn = syn
+
+ @dataclass
+ class ClassForTest(TableStoreRowMixin, TableStoreMixin):
+ id: Optional[str] = None
+ name: Optional[str] = None
+ _last_persistent_instance: Optional[Any] = None
+
+ async def test_store_rows_async_converts_datetime_cols_to_epoch_in_dataframe(self):
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame(
+ {
+ "col1": ["a", "b"],
+ "date_col": pd.to_datetime([1487071391024, None], unit="ms", utc=True),
+ }
+ )
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(values=df, synapse_client=self.syn)
+
+ uploaded_df = mock_upload.call_args.kwargs["df"]
+ expected_df = pd.DataFrame(
+ {"col1": ["a", "b"], "date_col": [1487071391024, None]}
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(uploaded_df, expected_df, check_dtype=False)
+ assert is_integer_dtype(uploaded_df["date_col"])
+
+ async def test_store_rows_async_dict_input_is_converted_to_dataframe(self):
+ # GIVEN a plain dict of columns rather than a DataFrame, including a
+ # datetime column
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ values = {
+ "col1": ["a", "b"],
+ "col2": [1, 2],
+ "date_col": pd.to_datetime([1487071391024, None], unit="ms", utc=True),
+ }
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(values=values, synapse_client=self.syn)
+
+ # THEN it is converted to a DataFrame before reaching the upload step,
+ # and the datetime column is converted to epoch ms just as it would be
+ # for a DataFrame passed in directly
+ uploaded_df = mock_upload.call_args.kwargs["df"]
+ expected_df = pd.DataFrame(
+ {
+ "col1": ["a", "b"],
+ "col2": [1, 2],
+ "date_col": [1487071391024, None],
+ }
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(uploaded_df, expected_df, check_dtype=False)
+ assert is_integer_dtype(uploaded_df["date_col"])
+
+ async def test_store_rows_async_dataframe_defaults_to_csv_kwargs_escapechar(self):
+ # GIVEN a dataframe stored without explicit to_csv_kwargs
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame({"col1": ["a", "b"]})
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(values=df, synapse_client=self.syn)
+
+ # THEN the default escapechar is used
+ assert mock_upload.call_args.kwargs["to_csv_kwargs"] == {"escapechar": "\\"}
+
+ async def test_store_rows_async_dataframe_merges_to_csv_kwargs_with_default(self):
+ # GIVEN to_csv_kwargs that don't override escapechar
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame({"col1": ["a", "b"]})
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(
+ values=df, to_csv_kwargs={"sep": ";"}, synapse_client=self.syn
+ )
+
+ # THEN the caller's kwargs and the default escapechar are both present
+ assert mock_upload.call_args.kwargs["to_csv_kwargs"] == {
+ "escapechar": "\\",
+ "sep": ";",
+ }
+
+ async def test_store_rows_async_dataframe_to_csv_kwargs_can_override_escapechar(
+ self,
+ ):
+ # GIVEN to_csv_kwargs that explicitly override the default escapechar
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame({"col1": ["a", "b"]})
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(
+ values=df, to_csv_kwargs={"escapechar": "|"}, synapse_client=self.syn
+ )
+
+ assert mock_upload.call_args.kwargs["to_csv_kwargs"] == {"escapechar": "|"}
+
+ async def test_store_rows_async_dataframe_forwards_additional_changes(self):
+ # GIVEN additional_changes passed alongside a dataframe
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame({"col1": ["a", "b"]})
+ additional_change = MagicMock(name="additional_change")
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(
+ values=df,
+ additional_changes=[additional_change],
+ synapse_client=self.syn,
+ )
+
+ # THEN they are forwarded to the upload step unchanged
+ assert mock_upload.call_args.kwargs["additional_changes"] == [additional_change]
+
+ async def test_store_rows_async_dataframe_forwards_insert_size_and_timeout(self):
+ # GIVEN a custom insert_size_bytes and job_timeout
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame({"col1": ["a", "b"]})
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(
+ values=df,
+ insert_size_bytes=123,
+ job_timeout=45,
+ synapse_client=self.syn,
+ )
+
+ # THEN both are forwarded to the upload step
+ assert mock_upload.call_args.kwargs["insert_size_bytes"] == 123
+ assert mock_upload.call_args.kwargs["job_timeout"] == 45
+
+ async def test_store_rows_async_dry_run_with_dataframe_does_not_upload(self):
+ # GIVEN dry_run=True with a dataframe
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame({"col1": ["a", "b"]})
+
+ with patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload:
+ await table.store_rows_async(
+ values=df, dry_run=True, synapse_client=self.syn
+ )
+
+ # THEN no data is actually uploaded
+ mock_upload.assert_not_called()
+
+ async def test_store_rows_async_dataframe_raises_without_id(self):
+ # GIVEN a table with no id and no way to resolve one from Synapse
+ table = self.ClassForTest(id=None, name="test_table")
+ df = pd.DataFrame({"col1": ["a", "b"]})
+
+ with (
+ patch(GET_ID_PATCH, return_value=None),
+ patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload,
+ ):
+ with pytest.raises(
+ ValueError,
+ match=(
+ "The table must have an ID to store rows, or the table could "
+ "not be found from the given name/parent_id."
+ ),
+ ):
+ await table.store_rows_async(values=df, synapse_client=self.syn)
+
+ mock_upload.assert_not_called()
+
+ async def test_store_rows_async_dataframe_with_infer_from_data_generates_schema_change(
+ self,
+ ):
+ # GIVEN schema_storage_strategy=INFER_FROM_DATA with a dataframe
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ df = pd.DataFrame({"col1": ["a", "b"]})
+ schema_change_request = MagicMock(name="schema_change_request")
+
+ with (
+ patch.object(table, "_infer_columns_from_data") as mock_infer_columns,
+ patch.object(
+ table,
+ "_generate_schema_change_request",
+ new_callable=AsyncMock,
+ return_value=schema_change_request,
+ ) as mock_generate_schema_change_request,
+ patch.object(
+ table, "_chunk_and_upload_df", new_callable=AsyncMock
+ ) as mock_upload,
+ ):
+ await table.store_rows_async(
+ values=df,
+ schema_storage_strategy=SchemaStorageStrategy.INFER_FROM_DATA,
+ synapse_client=self.syn,
+ )
+
+ # THEN the columns are inferred from the data and the resulting schema
+ # change request is passed through to the upload step
+ mock_infer_columns.assert_called_once()
+ mock_generate_schema_change_request.assert_awaited_once()
+ assert (
+ mock_upload.call_args.kwargs["schema_change_request"]
+ == schema_change_request
+ )
+
+ async def test_store_rows_async_converts_csv_date_cols_to_epoch_in_csv(self):
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as csv_file:
+ csv_file.write("col1,date_col\na,01/15/2024\nb,02/20/2024\n")
+
+ uploaded = {}
+
+ def capture_upload(**kwargs):
+ # The temporary file is deleted after the upload, so read it here
+ uploaded["path"] = kwargs["path_to_csv"]
+ uploaded["df"] = pd.read_csv(kwargs["path_to_csv"])
+
+ try:
+ with patch.object(
+ table, "_chunk_and_upload_csv", new_callable=AsyncMock
+ ) as mock_upload:
+ mock_upload.side_effect = capture_upload
+ await table.store_rows_async(
+ values=csv_file.name,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y",
+ synapse_client=self.syn,
+ )
+
+ assert uploaded["path"] != csv_file.name
+ assert not os.path.exists(uploaded["path"])
+ expected_df = pd.DataFrame(
+ {
+ "col1": ["a", "b"],
+ "date_col": [1705276800000, 1708387200000],
+ } # date columns are converted to epoch ms (midnight local timezone, unit tests run with TZ=UTC)
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(
+ uploaded["df"], expected_df, check_dtype=False
+ )
+ finally:
+ os.remove(csv_file.name)
+
+ async def test_store_rows_async_csv_date_cols_forwards_synapse_client(self):
+ # GIVEN no cached Synapse client is available — matching integration
+ # tests, which run with Synapse.allow_client_caching(False) and never
+ # call Synapse.set_client, so only an explicitly passed client works
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as csv_file:
+ csv_file.write("col1,date_col\na,01/15/2024\n")
+
+ cached_client = Synapse._synapse_client
+ Synapse._synapse_client = None
+ try:
+ with patch.object(table, "_chunk_and_upload_csv", new_callable=AsyncMock):
+ # THEN parsing date_columns must use the explicitly passed
+ # client rather than raising for a missing cached instance
+ await table.store_rows_async(
+ values=csv_file.name,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y",
+ synapse_client=self.syn,
+ )
+ finally:
+ Synapse._synapse_client = cached_client
+ os.remove(csv_file.name)
+
+ async def test_store_rows_async_keeps_row_id_and_version_with_date_cols_in_csv(
+ self,
+ ):
+ table = self.ClassForTest(id="syn123", name="test_table")
+ table._last_persistent_instance = self.ClassForTest(
+ id="syn123", name="test_table"
+ )
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".csv", delete=False
+ ) as csv_file:
+ csv_file.write(
+ "ROW_ID,ROW_VERSION,col1,date_col\n"
+ "1,1,a,01/15/2024\n"
+ "2,1,b,02/20/2024\n"
+ )
+
+ uploaded = {}
+
+ def capture_upload(**kwargs):
+ uploaded["df"] = pd.read_csv(kwargs["path_to_csv"])
+
+ try:
+ with patch.object(
+ table, "_chunk_and_upload_csv", new_callable=AsyncMock
+ ) as mock_upload:
+ mock_upload.side_effect = capture_upload
+ await table.store_rows_async(
+ values=csv_file.name,
+ date_columns=["date_col"],
+ date_format="%m/%d/%Y",
+ synapse_client=self.syn,
+ )
+
+ expected_df = pd.DataFrame(
+ {
+ "ROW_ID": [1, 2],
+ "ROW_VERSION": [1, 1],
+ "col1": ["a", "b"],
+ "date_col": [
+ 1705276800000,
+ 1708387200000,
+ ], # date columns are converted to epoch ms (midnight local timezone, unit tests run with TZ=UTC)
+ }
+ ).convert_dtypes()
+ pd.testing.assert_frame_equal(
+ uploaded["df"], expected_df, check_dtype=False
+ )
+ finally:
+ os.remove(csv_file.name)