From 2ab6a96cde3580141997ff9206b9a37dd11908f0 Mon Sep 17 00:00:00 2001 From: Ben Shaw Date: Fri, 24 Jul 2026 10:02:03 +1200 Subject: [PATCH 1/3] feat: DM-3945 add table reference CRUD client APIs Wraps the new `/api/table-references/` endpoints from datamasque/datamasque!3534, mirroring the existing connection and discovery-config-library client patterns. - New `TableReferenceClient` mixin: list, get, get-by-name, create, update, create-or-update, delete by id/name. - New `TableReference`, `TableReferenceOptions` and `TableReferenceFormat` models. `connection` accepts a `ConnectionId` or a whole `ConnectionConfig`, as run requests do. - The list endpoint is unpaginated and shares the detail serializer, so a by-name lookup needs a single request; the endpoint takes no name filter, so the match is made client-side. - Deletes archive rather than remove, which frees the name for reuse. Co-Authored-By: Claude Opus 4.8 --- HISTORY.rst | 6 + datamasque/client/__init__.py | 10 + datamasque/client/dmclient.py | 2 + datamasque/client/models/table_reference.py | 101 +++++ datamasque/client/table_references.py | 135 ++++++ pyproject.toml | 2 +- setup.cfg | 2 +- tests/test_table_references.py | 472 ++++++++++++++++++++ uv.lock | 2 +- 9 files changed, 729 insertions(+), 3 deletions(-) create mode 100644 datamasque/client/models/table_reference.py create mode 100644 datamasque/client/table_references.py create mode 100644 tests/test_table_references.py diff --git a/HISTORY.rst b/HISTORY.rst index cf9720f..19fd1d7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,12 @@ History ======= +1.1.8 (unreleased) +------------------ + +* Added table-reference management APIs (``list_table_references``, ``create_table_reference``, and friends), + along with the ``TableReference`` and ``TableReferenceOptions`` models. + 1.1.7 (2026-07-14) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index e2275e3..c9aa10b 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -123,6 +123,12 @@ ValidationErrorType, ValidationStatus, ) +from datamasque.client.models.table_reference import ( + TableReference, + TableReferenceFormat, + TableReferenceId, + TableReferenceOptions, +) from datamasque.client.models.user import User, UserId, UserRole __version__ = version("datamasque-python") @@ -230,6 +236,10 @@ "SslZipFile", "SwitchableLicenseMetadata", "TableConstraints", + "TableReference", + "TableReferenceFormat", + "TableReferenceId", + "TableReferenceOptions", "UnfinishedRun", "User", "UserId", diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 4b9ba3d..1b0098f 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -9,6 +9,7 @@ from datamasque.client.rulesets import RulesetClient from datamasque.client.runs import RunClient from datamasque.client.settings import SettingsClient +from datamasque.client.table_references import TableReferenceClient from datamasque.client.users import UserClient __all__ = ["DataMasqueClient", "FileOrContent", "UploadFile"] @@ -24,6 +25,7 @@ class DataMasqueClient( DiscoveryClient, DiscoveryConfigClient, DiscoveryConfigLibraryClient, + TableReferenceClient, UserClient, SettingsClient, ): diff --git a/datamasque/client/models/table_reference.py b/datamasque/client/models/table_reference.py new file mode 100644 index 0000000..fe6f25a --- /dev/null +++ b/datamasque/client/models/table_reference.py @@ -0,0 +1,101 @@ +"""Table reference models for the DataMasque API.""" + +import enum +from datetime import datetime +from typing import Any, NewType, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from datamasque.client.models.connection import ConnectionConfig, ConnectionId, unwrap_connection_id + +TableReferenceId = NewType("TableReferenceId", str) + + +class TableReferenceFormat(enum.Enum): + """ + File format of the data a table reference points at. + + Only meaningful for a table reference on a file connection, + and an explicit choice: the source path's suffix carries no meaning. + """ + + csv = "csv" + parquet = "parquet" + + +class TableReferenceOptions(BaseModel): + """ + Format and CSV parsing options for a table reference. + + These apply to file connections only — + they are ignored for a table reference on a database connection, + and the CSV options are ignored when the `format` is `parquet`. + + There is deliberately no header toggle: + a CSV identity map's columns must be addressable by name, + so the first row is always read as the header. + + Every option has a default, and the server fills in any option omitted from a request, + so the values here are the same defaults the server applies. + """ + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + format: TableReferenceFormat = TableReferenceFormat.csv + delimiter: str = "," + encoding: str = "utf-8" + quotechar: str = '"' + null_string: Optional[str] = None + """The string in the source data to read as a null value, or `None` to read no value as null.""" + + +class TableReference(BaseModel): + """ + Represents a named, persisted reference to a table of identity data held outside DataMasque. + + The data lives in the referenced connection — + a CSV or Parquet file in a file connection, or a table in a database connection — + so a ruleset can reference the identity map once, by name, + instead of re-declaring the source each time. + + A table reference stores no credentials of its own; it borrows the referenced connection's. + + `source` is interpreted according to that connection's own type: + + * file connection — a path to the file within the connection's fileset. + The format comes from `options.format`, never from the path's suffix. + * database connection — a dotted, schema-qualified ``schema.table`` reference. + + Names are unique across live table references (a deleted reference frees its name for reuse), + and the referenced connection must not be archived. + """ + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + name: str + connection: Union[ConnectionId, ConnectionConfig] + """The connection the data lives in. Accepts a `ConnectionId` or a `ConnectionConfig` (its `id` is extracted).""" + source: str + options: Optional[TableReferenceOptions] = None + """Format and CSV parsing options; when `None`, the server applies its defaults on create.""" + + # Server-populated read-only fields, excluded from request bodies. + id: Optional[TableReferenceId] = Field(default=None, exclude=True) + created: Optional[datetime] = Field(default=None, exclude=True) + modified: Optional[datetime] = Field(default=None, exclude=True) + + @field_validator("connection", mode="before") + @classmethod + def _unwrap_connection(cls, value: Any) -> Any: + return unwrap_connection_id(value) + + @property + def connection_id(self) -> ConnectionId: + """ + The ID of the referenced connection. + + `connection` accepts a whole `ConnectionConfig` for convenience but always holds an ID + once validated, so this narrows the declared type back down for callers reading it. + """ + + return ConnectionId(unwrap_connection_id(self.connection)) diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py new file mode 100644 index 0000000..245396d --- /dev/null +++ b/datamasque/client/table_references.py @@ -0,0 +1,135 @@ +import logging +from typing import Optional + +from datamasque.client.base import BaseClient +from datamasque.client.exceptions import DataMasqueException +from datamasque.client.models.table_reference import TableReference, TableReferenceId + +logger = logging.getLogger(__name__) + + +class TableReferenceClient(BaseClient): + """Table reference CRUD API methods. Mixed into `DataMasqueClient`.""" + + def list_table_references(self) -> list[TableReference]: + """ + Lists all table references. + + Unlike most list endpoints, this one is not paginated; + the server returns every table reference in a single response. + Deleted table references are archived rather than removed, and are not listed. + """ + + response = self.make_request("GET", "/api/table-references/") + return [TableReference.model_validate(item) for item in response.json()] + + def get_table_reference(self, table_reference_id: TableReferenceId) -> TableReference: + """Retrieves a single table reference by ID.""" + + response = self.make_request("GET", f"/api/table-references/{table_reference_id}/") + return TableReference.model_validate(response.json()) + + def get_table_reference_by_name(self, name: str) -> Optional[TableReference]: + """ + Looks for a table reference with the given name (case-sensitive, exact match). + + Names are unique across live table references, so at most one can match. + Returns it if found, otherwise `None`. + + The endpoint takes no name filter, so the match is made client-side over the full listing. + """ + + matches = [reference for reference in self.list_table_references() if reference.name == name] + if not matches: + return None + + return matches[0] + + def _get_table_reference_id_by_name(self, name: str) -> Optional[TableReferenceId]: + """Return the ID of the table reference with the given name, or `None` if there is none.""" + + existing = self.get_table_reference_by_name(name) + if existing is None: + return None + + if existing.id is None: + raise DataMasqueException(f'Server returned a table reference named "{name}" without an `id`.') + + return existing.id + + def create_table_reference(self, table_reference: TableReference) -> TableReference: + """ + Creates a new table reference on the server. + + Sets the table reference's server-assigned fields (`id`, `created`, `modified`) + and its `options` as stored by the server (with any omitted option filled in with its default), + then returns the table reference. + """ + + data = table_reference.model_dump(exclude_none=True, by_alias=True, mode="json") + response = self.make_request("POST", "/api/table-references/", data=data) + created = TableReference.model_validate(response.json()) + table_reference.id = created.id + table_reference.options = created.options + table_reference.created = created.created + table_reference.modified = created.modified + logger.info('Creation of table reference "%s" successful', table_reference.name) + return table_reference + + def update_table_reference(self, table_reference: TableReference) -> TableReference: + """ + Performs a full update of the table reference. + + The table reference must have its `id` set + (i.e., it must have been previously created or retrieved from the server). + + `options` are replaced wholesale when set, so an option left at its model default + is sent as that default rather than keeping whatever the server had stored. + Leaving `options` as `None` sends none at all, which leaves the stored options untouched. + """ + + if table_reference.id is None: + raise ValueError("Cannot update a table reference that has not been created yet (id is None)") + + data = table_reference.model_dump(exclude_none=True, by_alias=True, mode="json") + response = self.make_request("PUT", f"/api/table-references/{table_reference.id}/", data=data) + updated = TableReference.model_validate(response.json()) + table_reference.options = updated.options + table_reference.modified = updated.modified + logger.debug('Update of table reference "%s" successful', table_reference.name) + return table_reference + + def create_or_update_table_reference(self, table_reference: TableReference) -> TableReference: + """ + Creates the table reference, or updates the existing one with the same name. + + Sets the table reference's `id` property. + """ + + existing_id = self._get_table_reference_id_by_name(table_reference.name) + if existing_id is not None: + table_reference.id = existing_id + return self.update_table_reference(table_reference) + + return self.create_table_reference(table_reference) + + def delete_table_reference_by_id_if_exists(self, table_reference_id: TableReferenceId) -> None: + """ + Deletes the table reference with the given ID. + + The server archives rather than removes it, which frees its name for reuse. + No-op if the table reference does not exist. + """ + + self._delete_if_exists(f"/api/table-references/{table_reference_id}/") + + def delete_table_reference_by_name_if_exists(self, name: str) -> None: + """ + Deletes the table reference with the given name. + + No-op if no such table reference exists. + """ + + table_reference_id = self._get_table_reference_id_by_name(name) + if table_reference_id is not None: + self.delete_table_reference_by_id_if_exists(table_reference_id) diff --git a/pyproject.toml b/pyproject.toml index 2c40393..dfc3ae7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datamasque-python" -version = "1.1.7" +version = "1.1.8" description = "Official Python client for the DataMasque data-masking API." authors = [ { name = "DataMasque Ltd" }, diff --git a/setup.cfg b/setup.cfg index 0f7c2b8..51bfe02 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.7 +current_version = 1.1.8 commit = True tag = True diff --git a/tests/test_table_references.py b/tests/test_table_references.py new file mode 100644 index 0000000..888c6fd --- /dev/null +++ b/tests/test_table_references.py @@ -0,0 +1,472 @@ +"""Tests for table reference support in the DataMasque client.""" + +from datetime import datetime +from typing import Any + +import pytest +import requests_mock +from pydantic import ValidationError + +from datamasque.client import ( + ConnectionId, + DataMasqueClient, + TableReference, + TableReferenceFormat, + TableReferenceId, + TableReferenceOptions, +) +from datamasque.client.exceptions import DataMasqueException +from tests.helpers import file_connection_config + +TABLE_REFERENCE_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" +TABLE_REFERENCE_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" +CONNECTION_ID_1 = "cccccccc-1111-2222-3333-444444444444" +CONNECTION_ID_2 = "dddddddd-1111-2222-3333-444444444444" + +DEFAULT_OPTIONS_JSON = { + "format": "csv", + "delimiter": ",", + "encoding": "utf-8", + "quotechar": '"', + "null_string": None, +} + + +@pytest.fixture +def sample_table_reference_list_response() -> list[dict[str, Any]]: + """List response: a bare (unpaginated) array of full entries.""" + return [ + { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + "created": "2025-01-01T12:00:00Z", + "modified": "2025-01-02T12:00:00Z", + }, + { + "id": TABLE_REFERENCE_ID_2, + "name": "account_identities", + "connection": CONNECTION_ID_2, + "source": "OPS.ACCOUNTS", + "options": { + "format": "parquet", + "delimiter": ";", + "encoding": "latin-1", + "quotechar": "'", + "null_string": "NULL", + }, + "created": "2025-02-01T12:00:00Z", + "modified": "2025-02-02T12:00:00Z", + }, + ] + + +@pytest.fixture +def table_reference() -> TableReference: + return TableReference( + name="customer_identities", + connection=ConnectionId(CONNECTION_ID_1), + source="identities/customers.csv", + ) + + +def test_name_connection_and_source_are_required_on_the_model() -> None: + with pytest.raises(ValidationError, match="source"): + TableReference(name="customer_identities", connection=ConnectionId(CONNECTION_ID_1)) + + +def test_options_default_to_the_servers_defaults() -> None: + options = TableReferenceOptions() + + assert options.format is TableReferenceFormat.csv + assert options.delimiter == "," + assert options.encoding == "utf-8" + assert options.quotechar == '"' + assert options.null_string is None + + +def test_connection_accepts_a_connection_config() -> None: + connection = file_connection_config() + connection.id = ConnectionId(CONNECTION_ID_1) + + reference = TableReference(name="customer_identities", connection=connection, source="customers.csv") + + assert reference.connection == ConnectionId(CONNECTION_ID_1) + assert reference.connection_id == ConnectionId(CONNECTION_ID_1) + + +def test_connection_config_without_an_id_is_rejected() -> None: + with pytest.raises(ValidationError, match="id is None"): + TableReference(name="customer_identities", connection=file_connection_config(), source="customers.csv") + + +def test_list_table_references( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + references = client.list_table_references() + + assert len(references) == 2 + assert references[0].id == TableReferenceId(TABLE_REFERENCE_ID_1) + assert references[0].name == "customer_identities" + assert references[0].connection == ConnectionId(CONNECTION_ID_1) + assert references[0].source == "identities/customers.csv" + assert references[0].options is not None + assert references[0].options.format is TableReferenceFormat.csv + assert references[0].created == datetime.fromisoformat("2025-01-01T12:00:00+00:00") + assert references[1].id == TableReferenceId(TABLE_REFERENCE_ID_2) + assert references[1].source == "OPS.ACCOUNTS" + assert references[1].options is not None + assert references[1].options.format is TableReferenceFormat.parquet + assert references[1].options.delimiter == ";" + assert references[1].options.encoding == "latin-1" + assert references[1].options.quotechar == "'" + assert references[1].options.null_string == "NULL" + + +def test_list_table_references_empty(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=[], + status_code=200, + ) + references = client.list_table_references() + + assert references == [] + + +def test_get_table_reference( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_1}/", + json=sample_table_reference_list_response[0], + status_code=200, + ) + reference = client.get_table_reference(TableReferenceId(TABLE_REFERENCE_ID_1)) + + assert reference.id == TableReferenceId(TABLE_REFERENCE_ID_1) + assert reference.name == "customer_identities" + assert reference.source == "identities/customers.csv" + + +def test_get_table_reference_by_name_found( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] +) -> None: + """The list response carries every field, so a by-name lookup needs no follow-up detail request.""" + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + reference = client.get_table_reference_by_name("account_identities") + + assert reference is not None + assert reference.id == TableReferenceId(TABLE_REFERENCE_ID_2) + assert m.call_count == 1 + + +def test_get_table_reference_by_name_not_found( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + reference = client.get_table_reference_by_name("nonexistent") + + assert reference is None + + +def test_create_table_reference(client: DataMasqueClient, table_reference: TableReference) -> None: + create_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + "created": "2025-06-01T10:00:00Z", + "modified": "2025-06-01T10:00:00Z", + } + + with requests_mock.Mocker() as m: + m.post( + "http://test-server/api/table-references/", + json=create_response, + status_code=201, + ) + result = client.create_table_reference(table_reference) + + assert result is table_reference + assert result.id == TableReferenceId(TABLE_REFERENCE_ID_1) + assert result.created == datetime.fromisoformat("2025-06-01T10:00:00+00:00") + assert result.modified == datetime.fromisoformat("2025-06-01T10:00:00+00:00") + # The server fills in the options it stored, even though none were sent. + assert result.options is not None + assert result.options.format is TableReferenceFormat.csv + + request_body = m.last_request.json() + assert request_body["name"] == "customer_identities" + assert request_body["connection"] == CONNECTION_ID_1 + assert request_body["source"] == "identities/customers.csv" + assert "options" not in request_body + read_only_fields = {"id", "created", "modified"} + assert not read_only_fields & request_body.keys() + + +def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: + reference = TableReference( + name="customer_identities", + connection=ConnectionId(CONNECTION_ID_1), + source="identities/customers.csv", + options=TableReferenceOptions(format=TableReferenceFormat.parquet, delimiter=";", null_string="NULL"), + ) + create_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": { + "format": "parquet", + "delimiter": ";", + "encoding": "utf-8", + "quotechar": '"', + "null_string": "NULL", + }, + "created": "2025-06-01T10:00:00Z", + "modified": "2025-06-01T10:00:00Z", + } + + with requests_mock.Mocker() as m: + m.post( + "http://test-server/api/table-references/", + json=create_response, + status_code=201, + ) + client.create_table_reference(reference) + + options = m.last_request.json()["options"] + assert options["format"] == "parquet" + assert options["delimiter"] == ";" + assert options["null_string"] == "NULL" + + +def test_update_table_reference(client: DataMasqueClient, table_reference: TableReference) -> None: + table_reference.id = TableReferenceId(TABLE_REFERENCE_ID_1) + table_reference.source = "identities/customers_v2.csv" + + update_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers_v2.csv", + "options": DEFAULT_OPTIONS_JSON, + "created": "2025-06-01T10:00:00Z", + "modified": "2025-06-02T10:00:00Z", + } + + with requests_mock.Mocker() as m: + m.put( + f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_1}/", + json=update_response, + status_code=200, + ) + result = client.update_table_reference(table_reference) + + assert result is table_reference + assert result.modified == datetime.fromisoformat("2025-06-02T10:00:00+00:00") + + request_body = m.last_request.json() + assert request_body["source"] == "identities/customers_v2.csv" + read_only_fields = {"id", "created", "modified"} + assert not read_only_fields & request_body.keys() + + +def test_update_table_reference_without_options_sends_none( + client: DataMasqueClient, table_reference: TableReference +) -> None: + """Options are omitted entirely when unset, so the server keeps the ones it has stored.""" + table_reference.id = TableReferenceId(TABLE_REFERENCE_ID_1) + + update_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": {**DEFAULT_OPTIONS_JSON, "delimiter": ";"}, + "created": "2025-06-01T10:00:00Z", + "modified": "2025-06-02T10:00:00Z", + } + + with requests_mock.Mocker() as m: + m.put( + f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_1}/", + json=update_response, + status_code=200, + ) + result = client.update_table_reference(table_reference) + + assert "options" not in m.last_request.json() + # The server's stored options come back on the updated model. + assert result.options is not None + assert result.options.delimiter == ";" + + +def test_update_table_reference_no_id_raises(client: DataMasqueClient, table_reference: TableReference) -> None: + with pytest.raises(ValueError, match="id is None"): + client.update_table_reference(table_reference) + + +def test_create_or_update_table_reference_create(client: DataMasqueClient, table_reference: TableReference) -> None: + create_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + "created": "2025-06-01T10:00:00Z", + "modified": "2025-06-01T10:00:00Z", + } + + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=[], + status_code=200, + ) + m.post( + "http://test-server/api/table-references/", + json=create_response, + status_code=201, + ) + result = client.create_or_update_table_reference(table_reference) + + assert result.id == TableReferenceId(TABLE_REFERENCE_ID_1) + assert m.request_history[0].method == "GET" + assert m.request_history[1].method == "POST" + + +def test_create_or_update_table_reference_update( + client: DataMasqueClient, + table_reference: TableReference, + sample_table_reference_list_response: list[dict[str, Any]], +) -> None: + update_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + "created": "2025-01-01T12:00:00Z", + "modified": "2025-06-02T10:00:00Z", + } + + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + m.put( + f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_1}/", + json=update_response, + status_code=200, + ) + result = client.create_or_update_table_reference(table_reference) + + assert result.id == TableReferenceId(TABLE_REFERENCE_ID_1) + assert m.call_count == 2 + assert m.request_history[0].method == "GET" + assert m.request_history[1].method == "PUT" + + +def test_create_or_update_table_reference_raises_when_server_omits_id( + client: DataMasqueClient, table_reference: TableReference +) -> None: + list_response_without_id = [ + { + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + "created": "2025-01-01T12:00:00Z", + "modified": "2025-01-02T12:00:00Z", + }, + ] + + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=list_response_without_id, + status_code=200, + ) + with pytest.raises(DataMasqueException, match="without an `id`"): + client.create_or_update_table_reference(table_reference) + + +def test_delete_table_reference_by_id(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.delete( + f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_1}/", + status_code=204, + ) + client.delete_table_reference_by_id_if_exists(TableReferenceId(TABLE_REFERENCE_ID_1)) + + assert m.call_count == 1 + + +def test_delete_table_reference_by_id_not_found(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.delete( + f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_1}/", + status_code=404, + ) + client.delete_table_reference_by_id_if_exists(TableReferenceId(TABLE_REFERENCE_ID_1)) + + +def test_delete_table_reference_by_name( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + m.delete( + f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_2}/", + status_code=204, + ) + client.delete_table_reference_by_name_if_exists("account_identities") + + assert m.call_count == 2 + assert m.request_history[0].method == "GET" + assert m.request_history[1].method == "DELETE" + assert TABLE_REFERENCE_ID_2 in m.request_history[1].url + + +def test_delete_table_reference_by_name_not_found( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + client.delete_table_reference_by_name_if_exists("nonexistent") + + assert m.call_count == 1 + assert m.request_history[0].method == "GET" diff --git a/uv.lock b/uv.lock index 11c8a93..b0f618a 100644 --- a/uv.lock +++ b/uv.lock @@ -428,7 +428,7 @@ toml = [ [[package]] name = "datamasque-python" -version = "1.1.7" +version = "1.1.8" source = { editable = "." } dependencies = [ { name = "pydantic" }, From 70b3544c4e372e7012f3f15a7b1778d74d98728a Mon Sep 17 00:00:00 2001 From: Ben Shaw Date: Fri, 24 Jul 2026 16:38:34 +1200 Subject: [PATCH 2/3] fix: address review comments on table reference API - Use `object` instead of `Any` in the new model validator and test annotations. - Fix docstring formatting for the schema.table reference. - Stop using `exclude_none=True` for request payloads: it silently dropped `options.null_string` (and any other nested `None`) instead of just omitting a wholly-unset `options`. Now only the `options` key itself is dropped when `None`, so explicit `None` values inside it (e.g. `null_string`) are still sent. Co-Authored-By: Claude Sonnet 5 --- datamasque/client/models/table_reference.py | 6 +-- datamasque/client/table_references.py | 8 +++- tests/test_table_references.py | 48 +++++++++++++++++---- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/datamasque/client/models/table_reference.py b/datamasque/client/models/table_reference.py index fe6f25a..ac94fdc 100644 --- a/datamasque/client/models/table_reference.py +++ b/datamasque/client/models/table_reference.py @@ -2,7 +2,7 @@ import enum from datetime import datetime -from typing import Any, NewType, Optional, Union +from typing import NewType, Optional, Union from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -64,7 +64,7 @@ class TableReference(BaseModel): * file connection — a path to the file within the connection's fileset. The format comes from `options.format`, never from the path's suffix. - * database connection — a dotted, schema-qualified ``schema.table`` reference. + * database connection — a dotted, schema-qualified `schema.table` reference. Names are unique across live table references (a deleted reference frees its name for reuse), and the referenced connection must not be archived. @@ -86,7 +86,7 @@ class TableReference(BaseModel): @field_validator("connection", mode="before") @classmethod - def _unwrap_connection(cls, value: Any) -> Any: + def _unwrap_connection(cls, value: object) -> object: return unwrap_connection_id(value) @property diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index 245396d..04d0e29 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -66,7 +66,9 @@ def create_table_reference(self, table_reference: TableReference) -> TableRefere then returns the table reference. """ - data = table_reference.model_dump(exclude_none=True, by_alias=True, mode="json") + data = table_reference.model_dump(by_alias=True, mode="json") + if data.get("options") is None: + data.pop("options", None) response = self.make_request("POST", "/api/table-references/", data=data) created = TableReference.model_validate(response.json()) table_reference.id = created.id @@ -91,7 +93,9 @@ def update_table_reference(self, table_reference: TableReference) -> TableRefere if table_reference.id is None: raise ValueError("Cannot update a table reference that has not been created yet (id is None)") - data = table_reference.model_dump(exclude_none=True, by_alias=True, mode="json") + data = table_reference.model_dump(by_alias=True, mode="json") + if data.get("options") is None: + data.pop("options", None) response = self.make_request("PUT", f"/api/table-references/{table_reference.id}/", data=data) updated = TableReference.model_validate(response.json()) table_reference.options = updated.options diff --git a/tests/test_table_references.py b/tests/test_table_references.py index 888c6fd..1a7968f 100644 --- a/tests/test_table_references.py +++ b/tests/test_table_references.py @@ -1,7 +1,6 @@ """Tests for table reference support in the DataMasque client.""" from datetime import datetime -from typing import Any import pytest import requests_mock @@ -33,7 +32,7 @@ @pytest.fixture -def sample_table_reference_list_response() -> list[dict[str, Any]]: +def sample_table_reference_list_response() -> list[dict[str, object]]: """List response: a bare (unpaginated) array of full entries.""" return [ { @@ -103,7 +102,7 @@ def test_connection_config_without_an_id_is_rejected() -> None: def test_list_table_references( - client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] ) -> None: with requests_mock.Mocker() as m: m.get( @@ -144,7 +143,7 @@ def test_list_table_references_empty(client: DataMasqueClient) -> None: def test_get_table_reference( - client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] ) -> None: with requests_mock.Mocker() as m: m.get( @@ -160,7 +159,7 @@ def test_get_table_reference( def test_get_table_reference_by_name_found( - client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] ) -> None: """The list response carries every field, so a by-name lookup needs no follow-up detail request.""" with requests_mock.Mocker() as m: @@ -177,7 +176,7 @@ def test_get_table_reference_by_name_found( def test_get_table_reference_by_name_not_found( - client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] ) -> None: with requests_mock.Mocker() as m: m.get( @@ -263,6 +262,37 @@ def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: assert options["null_string"] == "NULL" +def test_create_table_reference_sends_null_string_default(client: DataMasqueClient) -> None: + """`null_string`'s default of `None` must be sent explicitly, not dropped from the payload.""" + reference = TableReference( + name="customer_identities", + connection=ConnectionId(CONNECTION_ID_1), + source="identities/customers.csv", + options=TableReferenceOptions(), + ) + create_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + "created": "2025-06-01T10:00:00Z", + "modified": "2025-06-01T10:00:00Z", + } + + with requests_mock.Mocker() as m: + m.post( + "http://test-server/api/table-references/", + json=create_response, + status_code=201, + ) + client.create_table_reference(reference) + + options = m.last_request.json()["options"] + assert "null_string" in options + assert options["null_string"] is None + + def test_update_table_reference(client: DataMasqueClient, table_reference: TableReference) -> None: table_reference.id = TableReferenceId(TABLE_REFERENCE_ID_1) table_reference.source = "identities/customers_v2.csv" @@ -361,7 +391,7 @@ def test_create_or_update_table_reference_create(client: DataMasqueClient, table def test_create_or_update_table_reference_update( client: DataMasqueClient, table_reference: TableReference, - sample_table_reference_list_response: list[dict[str, Any]], + sample_table_reference_list_response: list[dict[str, object]], ) -> None: update_response = { "id": TABLE_REFERENCE_ID_1, @@ -437,7 +467,7 @@ def test_delete_table_reference_by_id_not_found(client: DataMasqueClient) -> Non def test_delete_table_reference_by_name( - client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] ) -> None: with requests_mock.Mocker() as m: m.get( @@ -458,7 +488,7 @@ def test_delete_table_reference_by_name( def test_delete_table_reference_by_name_not_found( - client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, Any]] + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] ) -> None: with requests_mock.Mocker() as m: m.get( From a02300c4332655290d5416b678767f55a0859546 Mon Sep 17 00:00:00 2001 From: Ben Shaw Date: Fri, 24 Jul 2026 16:52:15 +1200 Subject: [PATCH 3/3] fix: raise DataMasqueApiError with response in table reference lookup Match discovery_configs.py's convention: a malformed server response (missing id) raises DataMasqueApiError with the triggering response attached, not the bare DataMasqueException, so callers catching DataMasqueApiError for API-shape problems catch this too. Co-Authored-By: Claude Sonnet 5 --- datamasque/client/table_references.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index 04d0e29..f191abf 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -2,7 +2,7 @@ from typing import Optional from datamasque.client.base import BaseClient -from datamasque.client.exceptions import DataMasqueException +from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.table_reference import TableReference, TableReferenceId logger = logging.getLogger(__name__) @@ -48,14 +48,19 @@ def get_table_reference_by_name(self, name: str) -> Optional[TableReference]: def _get_table_reference_id_by_name(self, name: str) -> Optional[TableReferenceId]: """Return the ID of the table reference with the given name, or `None` if there is none.""" - existing = self.get_table_reference_by_name(name) - if existing is None: + response = self.make_request("GET", "/api/table-references/") + matches = [item for item in response.json() if item.get("name") == name] + if not matches: return None - if existing.id is None: - raise DataMasqueException(f'Server returned a table reference named "{name}" without an `id`.') + existing_id = matches[0].get("id") + if existing_id is None: + raise DataMasqueApiError( + f'Server returned a table reference named "{name}" without an `id`.', + response=response, + ) - return existing.id + return TableReferenceId(existing_id) def create_table_reference(self, table_reference: TableReference) -> TableReference: """