From 45565b534dfe209b6989a7da39769dc1687e1d0c Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Mon, 19 Oct 2026 17:24:40 -0500 Subject: [PATCH 01/14] [Health Deidentification] Remove stale generated files and regenerate Fixes pyright reportArgumentType (issubclass) and pylint C0123 failures in the healthdataaiservices CI pipeline. - Delete stale _serialization.py, _model_base.py, _vendor.py, aio/_vendor.py left over from an older emitter (only referenced each other). - Point tsp-location.yaml at the relocated spec (specification/healthdataaiservices/data-plane/HealthDataAIServices @ c7d9e33) and regenerate with @azure-tools/typespec-python 0.63.10. No API changes. - Emitter moves packaging to pyproject.toml (Python >=3.10, azure-core >=1.37.0) and adds types.py; bump to 1.1.0b2 and note it in CHANGELOG/README. - Tests: exclude Accept header from matching (regenerated delete_job no longer sends it) and disable the background LRO poller in the sync create/cancel/delete test, which otherwise races the test's own GETs in playback. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../CHANGELOG.md | 14 + .../azure-health-deidentification/README.md | 2 +- .../_metadata.json | 5 +- .../apiview-properties.json | 3 +- .../azure/health/deidentification/_client.py | 16 +- .../health/deidentification/_configuration.py | 8 +- .../health/deidentification/_model_base.py | 1232 ---------- .../_operations/_operations.py | 85 +- .../health/deidentification/_serialization.py | 2032 ----------------- .../deidentification/_utils/model_base.py | 799 ++++++- .../deidentification/_utils/serialization.py | 187 +- .../azure/health/deidentification/_vendor.py | 25 - .../azure/health/deidentification/_version.py | 2 +- .../health/deidentification/aio/_client.py | 16 +- .../deidentification/aio/_configuration.py | 8 +- .../aio/_operations/_operations.py | 82 +- .../health/deidentification/aio/_vendor.py | 25 - .../health/deidentification/models/_models.py | 51 +- .../azure/health/deidentification/types.py | 290 +++ .../pyproject.toml | 62 + .../azure-health-deidentification/setup.py | 70 - .../tests/conftest.py | 4 + .../tests/test_create_delete.py | 3 +- .../tsp-location.yaml | 4 +- 24 files changed, 1394 insertions(+), 3631 deletions(-) delete mode 100644 sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_model_base.py delete mode 100644 sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_serialization.py delete mode 100644 sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_vendor.py delete mode 100644 sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_vendor.py create mode 100644 sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py delete mode 100644 sdk/healthdataaiservices/azure-health-deidentification/setup.py diff --git a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md index 0f286e140ea3..7c3939242a74 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md @@ -1,5 +1,19 @@ # Release History +## 1.1.0b2 (Unreleased) + +### Features Added + +### Breaking Changes + +### Bugs Fixed + +### Other Changes + +- Regenerated the client with the latest TypeSpec emitter and removed stale generated files. +- Dropped support for Python 3.9. The minimum supported Python version is now 3.10. +- The minimum required `azure-core` version is now 1.37.0. + ## 1.1.0b1 (2025-08-05) ### Features Added diff --git a/sdk/healthdataaiservices/azure-health-deidentification/README.md b/sdk/healthdataaiservices/azure-health-deidentification/README.md index 87b662118aa9..4beb01f8a12a 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/README.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/README.md @@ -22,7 +22,7 @@ Use the client library for the de-identification service to: ### Prequisites -- Python 3.9 or later is required to use this package. +- Python 3.10 or later is required to use this package. - Install [pip][pip]. - You need an [Azure subscription][azure_sub] to use this package. - [Deploy the de-identification service][deid_quickstart]. diff --git a/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json b/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json index fd321bd8fa9b..7721b6e48d10 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json +++ b/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json @@ -1,3 +1,6 @@ { - "apiVersion": "2025-07-15-preview" + "apiVersion": "2025-07-15-preview", + "apiVersions": { + "HealthDataAIServices.DeidServices": "2025-07-15-preview" + } } \ No newline at end of file diff --git a/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json b/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json index cf39b3fdd7c6..5f7cede3467b 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json +++ b/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json @@ -30,5 +30,6 @@ "azure.health.deidentification.aio.DeidentificationClient.delete_job": "HealthDataAIServices.DeidServices.deleteJob", "azure.health.deidentification.DeidentificationClient.deidentify_text": "HealthDataAIServices.DeidServices.deidentifyText", "azure.health.deidentification.aio.DeidentificationClient.deidentify_text": "HealthDataAIServices.DeidServices.deidentifyText" - } + }, + "CrossLanguageVersion": "8ab7039f42a1" } \ No newline at end of file diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py index 603a33be33d1..c52572072304 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py @@ -7,8 +7,8 @@ # -------------------------------------------------------------------------- from copy import deepcopy +import sys from typing import Any, TYPE_CHECKING -from typing_extensions import Self from azure.core import PipelineClient from azure.core.pipeline import policies @@ -18,19 +18,27 @@ from ._operations import _DeidentificationClientOperationsMixin from ._utils.serialization import Deserializer, Serializer +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self # type: ignore + if TYPE_CHECKING: from azure.core.credentials import TokenCredential -class DeidentificationClient(_DeidentificationClientOperationsMixin): +class DeidentificationClient( + _DeidentificationClientOperationsMixin +): # pylint: disable=docstring-keyword-should-match-keyword-only """DeidentificationClient. :param endpoint: Url of your De-identification Service. Required. :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials.TokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str :keyword int polling_interval: Default waiting time between two polls for LRO operations if no diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py index b2305ff66bc8..04f8590d4a5b 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py @@ -1,3 +1,4 @@ +# pylint: disable=line-too-long,useless-suppression # coding=utf-8 # -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. @@ -16,7 +17,7 @@ from azure.core.credentials import TokenCredential -class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes +class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes,docstring-keyword-should-match-keyword-only """Configuration for DeidentificationClient. Note that all parameters used to create this instance are saved as instance @@ -26,8 +27,9 @@ class DeidentificationClientConfiguration: # pylint: disable=too-many-instance- :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials.TokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str """ diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_model_base.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_model_base.py deleted file mode 100644 index 095e62400e89..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_model_base.py +++ /dev/null @@ -1,1232 +0,0 @@ -# pylint: disable=too-many-lines,line-too-long,useless-suppression -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- -# pylint: disable=protected-access, broad-except - -import copy -import calendar -import decimal -import functools -import sys -import logging -import base64 -import re -import typing -import enum -import email.utils -from datetime import datetime, date, time, timedelta, timezone -from json import JSONEncoder -import xml.etree.ElementTree as ET -from collections.abc import MutableMapping -from typing_extensions import Self -import isodate -from azure.core.exceptions import DeserializationError -from azure.core import CaseInsensitiveEnumMeta -from azure.core.pipeline import PipelineResponse -from azure.core.serialization import _Null - -_LOGGER = logging.getLogger(__name__) - -__all__ = ["SdkJSONEncoder", "Model", "rest_field", "rest_discriminator"] - -TZ_UTC = timezone.utc -_T = typing.TypeVar("_T") - - -def _timedelta_as_isostr(td: timedelta) -> str: - """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S' - - Function adapted from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython - - :param timedelta td: The timedelta to convert - :rtype: str - :return: ISO8601 version of this timedelta - """ - - # Split seconds to larger units - seconds = td.total_seconds() - minutes, seconds = divmod(seconds, 60) - hours, minutes = divmod(minutes, 60) - days, hours = divmod(hours, 24) - - days, hours, minutes = list(map(int, (days, hours, minutes))) - seconds = round(seconds, 6) - - # Build date - date_str = "" - if days: - date_str = "%sD" % days - - if hours or minutes or seconds: - # Build time - time_str = "T" - - # Hours - bigger_exists = date_str or hours - if bigger_exists: - time_str += "{:02}H".format(hours) - - # Minutes - bigger_exists = bigger_exists or minutes - if bigger_exists: - time_str += "{:02}M".format(minutes) - - # Seconds - try: - if seconds.is_integer(): - seconds_string = "{:02}".format(int(seconds)) - else: - # 9 chars long w/ leading 0, 6 digits after decimal - seconds_string = "%09.6f" % seconds - # Remove trailing zeros - seconds_string = seconds_string.rstrip("0") - except AttributeError: # int.is_integer() raises - seconds_string = "{:02}".format(seconds) - - time_str += "{}S".format(seconds_string) - else: - time_str = "" - - return "P" + date_str + time_str - - -def _serialize_bytes(o, format: typing.Optional[str] = None) -> str: - encoded = base64.b64encode(o).decode() - if format == "base64url": - return encoded.strip("=").replace("+", "-").replace("/", "_") - return encoded - - -def _serialize_datetime(o, format: typing.Optional[str] = None): - if hasattr(o, "year") and hasattr(o, "hour"): - if format == "rfc7231": - return email.utils.format_datetime(o, usegmt=True) - if format == "unix-timestamp": - return int(calendar.timegm(o.utctimetuple())) - - # astimezone() fails for naive times in Python 2.7, so make make sure o is aware (tzinfo is set) - if not o.tzinfo: - iso_formatted = o.replace(tzinfo=TZ_UTC).isoformat() - else: - iso_formatted = o.astimezone(TZ_UTC).isoformat() - # Replace the trailing "+00:00" UTC offset with "Z" (RFC 3339: https://www.ietf.org/rfc/rfc3339.txt) - return iso_formatted.replace("+00:00", "Z") - # Next try datetime.date or datetime.time - return o.isoformat() - - -def _is_readonly(p): - try: - return p._visibility == ["read"] - except AttributeError: - return False - - -class SdkJSONEncoder(JSONEncoder): - """A JSON encoder that's capable of serializing datetime objects and bytes.""" - - def __init__(self, *args, exclude_readonly: bool = False, format: typing.Optional[str] = None, **kwargs): - super().__init__(*args, **kwargs) - self.exclude_readonly = exclude_readonly - self.format = format - - def default(self, o): # pylint: disable=too-many-return-statements - if _is_model(o): - if self.exclude_readonly: - readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)] - return {k: v for k, v in o.items() if k not in readonly_props} - return dict(o.items()) - try: - return super(SdkJSONEncoder, self).default(o) - except TypeError: - if isinstance(o, _Null): - return None - if isinstance(o, decimal.Decimal): - return float(o) - if isinstance(o, (bytes, bytearray)): - return _serialize_bytes(o, self.format) - try: - # First try datetime.datetime - return _serialize_datetime(o, self.format) - except AttributeError: - pass - # Last, try datetime.timedelta - try: - return _timedelta_as_isostr(o) - except AttributeError: - # This will be raised when it hits value.total_seconds in the method above - pass - return super(SdkJSONEncoder, self).default(o) - - -_VALID_DATE = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}" + r"\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") -_VALID_RFC7231 = re.compile( - r"(Mon|Tue|Wed|Thu|Fri|Sat|Sun),\s\d{2}\s" - r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT" -) - - -def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: - """Deserialize ISO-8601 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - attr = attr.upper() - match = _VALID_DATE.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - check_decimal = attr.split(".") - if len(check_decimal) > 1: - decimal_str = "" - for digit in check_decimal[1]: - if digit.isdigit(): - decimal_str += digit - else: - break - if len(decimal_str) > 6: - attr = attr.replace(decimal_str, decimal_str[0:6]) - - date_obj = isodate.parse_datetime(attr) - test_utc = date_obj.utctimetuple() - if test_utc.tm_year > 9999 or test_utc.tm_year < 1: - raise OverflowError("Hit max or min date") - return date_obj - - -def _deserialize_datetime_rfc7231(attr: typing.Union[str, datetime]) -> datetime: - """Deserialize RFC7231 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - match = _VALID_RFC7231.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - return email.utils.parsedate_to_datetime(attr) - - -def _deserialize_datetime_unix_timestamp(attr: typing.Union[float, datetime]) -> datetime: - """Deserialize unix timestamp into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - return datetime.fromtimestamp(attr, TZ_UTC) - - -def _deserialize_date(attr: typing.Union[str, date]) -> date: - """Deserialize ISO-8601 formatted string into Date object. - :param str attr: response string to be deserialized. - :rtype: date - :returns: The date object from that input - """ - # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. - if isinstance(attr, date): - return attr - return isodate.parse_date(attr, defaultmonth=None, defaultday=None) # type: ignore - - -def _deserialize_time(attr: typing.Union[str, time]) -> time: - """Deserialize ISO-8601 formatted string into time object. - - :param str attr: response string to be deserialized. - :rtype: datetime.time - :returns: The time object from that input - """ - if isinstance(attr, time): - return attr - return isodate.parse_time(attr) - - -def _deserialize_bytes(attr): - if isinstance(attr, (bytes, bytearray)): - return attr - return bytes(base64.b64decode(attr)) - - -def _deserialize_bytes_base64(attr): - if isinstance(attr, (bytes, bytearray)): - return attr - padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore - attr = attr + padding # type: ignore - encoded = attr.replace("-", "+").replace("_", "/") - return bytes(base64.b64decode(encoded)) - - -def _deserialize_duration(attr): - if isinstance(attr, timedelta): - return attr - return isodate.parse_duration(attr) - - -def _deserialize_decimal(attr): - if isinstance(attr, decimal.Decimal): - return attr - return decimal.Decimal(str(attr)) - - -def _deserialize_int_as_str(attr): - if isinstance(attr, int): - return attr - return int(attr) - - -_DESERIALIZE_MAPPING = { - datetime: _deserialize_datetime, - date: _deserialize_date, - time: _deserialize_time, - bytes: _deserialize_bytes, - bytearray: _deserialize_bytes, - timedelta: _deserialize_duration, - typing.Any: lambda x: x, - decimal.Decimal: _deserialize_decimal, -} - -_DESERIALIZE_MAPPING_WITHFORMAT = { - "rfc3339": _deserialize_datetime, - "rfc7231": _deserialize_datetime_rfc7231, - "unix-timestamp": _deserialize_datetime_unix_timestamp, - "base64": _deserialize_bytes, - "base64url": _deserialize_bytes_base64, -} - - -def get_deserializer(annotation: typing.Any, rf: typing.Optional["_RestField"] = None): - if annotation is int and rf and rf._format == "str": - return _deserialize_int_as_str - if rf and rf._format: - return _DESERIALIZE_MAPPING_WITHFORMAT.get(rf._format) - return _DESERIALIZE_MAPPING.get(annotation) # pyright: ignore - - -def _get_type_alias_type(module_name: str, alias_name: str): - types = { - k: v - for k, v in sys.modules[module_name].__dict__.items() - if isinstance(v, typing._GenericAlias) # type: ignore - } - if alias_name not in types: - return alias_name - return types[alias_name] - - -def _get_model(module_name: str, model_name: str): - models = {k: v for k, v in sys.modules[module_name].__dict__.items() if isinstance(v, type)} - module_end = module_name.rsplit(".", 1)[0] - models.update({k: v for k, v in sys.modules[module_end].__dict__.items() if isinstance(v, type)}) - if isinstance(model_name, str): - model_name = model_name.split(".")[-1] - if model_name not in models: - return model_name - return models[model_name] - - -_UNSET = object() - - -class _MyMutableMapping(MutableMapping[str, typing.Any]): - def __init__(self, data: typing.Dict[str, typing.Any]) -> None: - self._data = data - - def __contains__(self, key: typing.Any) -> bool: - return key in self._data - - def __getitem__(self, key: str) -> typing.Any: - return self._data.__getitem__(key) - - def __setitem__(self, key: str, value: typing.Any) -> None: - self._data.__setitem__(key, value) - - def __delitem__(self, key: str) -> None: - self._data.__delitem__(key) - - def __iter__(self) -> typing.Iterator[typing.Any]: - return self._data.__iter__() - - def __len__(self) -> int: - return self._data.__len__() - - def __ne__(self, other: typing.Any) -> bool: - return not self.__eq__(other) - - def keys(self) -> typing.KeysView[str]: - """ - :returns: a set-like object providing a view on D's keys - :rtype: ~typing.KeysView - """ - return self._data.keys() - - def values(self) -> typing.ValuesView[typing.Any]: - """ - :returns: an object providing a view on D's values - :rtype: ~typing.ValuesView - """ - return self._data.values() - - def items(self) -> typing.ItemsView[str, typing.Any]: - """ - :returns: set-like object providing a view on D's items - :rtype: ~typing.ItemsView - """ - return self._data.items() - - def get(self, key: str, default: typing.Any = None) -> typing.Any: - """ - Get the value for key if key is in the dictionary, else default. - :param str key: The key to look up. - :param any default: The value to return if key is not in the dictionary. Defaults to None - :returns: D[k] if k in D, else d. - :rtype: any - """ - try: - return self[key] - except KeyError: - return default - - @typing.overload - def pop(self, key: str) -> typing.Any: ... # pylint: disable=arguments-differ - - @typing.overload - def pop(self, key: str, default: _T) -> _T: ... # pylint: disable=signature-differs - - @typing.overload - def pop(self, key: str, default: typing.Any) -> typing.Any: ... # pylint: disable=signature-differs - - def pop(self, key: str, default: typing.Any = _UNSET) -> typing.Any: - """ - Removes specified key and return the corresponding value. - :param str key: The key to pop. - :param any default: The value to return if key is not in the dictionary - :returns: The value corresponding to the key. - :rtype: any - :raises KeyError: If key is not found and default is not given. - """ - if default is _UNSET: - return self._data.pop(key) - return self._data.pop(key, default) - - def popitem(self) -> typing.Tuple[str, typing.Any]: - """ - Removes and returns some (key, value) pair - :returns: The (key, value) pair. - :rtype: tuple - :raises KeyError: if D is empty. - """ - return self._data.popitem() - - def clear(self) -> None: - """ - Remove all items from D. - """ - self._data.clear() - - def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: # pylint: disable=arguments-differ - """ - Updates D from mapping/iterable E and F. - :param any args: Either a mapping object or an iterable of key-value pairs. - """ - self._data.update(*args, **kwargs) - - @typing.overload - def setdefault(self, key: str, default: None = None) -> None: ... - - @typing.overload - def setdefault(self, key: str, default: typing.Any) -> typing.Any: ... # pylint: disable=signature-differs - - def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: - """ - Same as calling D.get(k, d), and setting D[k]=d if k not found - :param str key: The key to look up. - :param any default: The value to set if key is not in the dictionary - :returns: D[k] if k in D, else d. - :rtype: any - """ - if default is _UNSET: - return self._data.setdefault(key) - return self._data.setdefault(key, default) - - def __eq__(self, other: typing.Any) -> bool: - try: - other_model = self.__class__(other) - except Exception: - return False - return self._data == other_model._data - - def __repr__(self) -> str: - return str(self._data) - - -def _is_model(obj: typing.Any) -> bool: - return getattr(obj, "_is_model", False) - - -def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-many-return-statements - if isinstance(o, list): - return [_serialize(x, format) for x in o] - if isinstance(o, dict): - return {k: _serialize(v, format) for k, v in o.items()} - if isinstance(o, set): - return {_serialize(x, format) for x in o} - if isinstance(o, tuple): - return tuple(_serialize(x, format) for x in o) - if isinstance(o, (bytes, bytearray)): - return _serialize_bytes(o, format) - if isinstance(o, decimal.Decimal): - return float(o) - if isinstance(o, enum.Enum): - return o.value - if isinstance(o, int): - if format == "str": - return str(o) - return o - try: - # First try datetime.datetime - return _serialize_datetime(o, format) - except AttributeError: - pass - # Last, try datetime.timedelta - try: - return _timedelta_as_isostr(o) - except AttributeError: - # This will be raised when it hits value.total_seconds in the method above - pass - return o - - -def _get_rest_field( - attr_to_rest_field: typing.Dict[str, "_RestField"], rest_name: str -) -> typing.Optional["_RestField"]: - try: - return next(rf for rf in attr_to_rest_field.values() if rf._rest_name == rest_name) - except StopIteration: - return None - - -def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typing.Any: - if not rf: - return _serialize(value, None) - if rf._is_multipart_file_input: - return value - if rf._is_model: - return _deserialize(rf._type, value) - if isinstance(value, ET.Element): - value = _deserialize(rf._type, value) - return _serialize(value, rf._format) - - -class Model(_MyMutableMapping): - _is_model = True - # label whether current class's _attr_to_rest_field has been calculated - # could not see _attr_to_rest_field directly because subclass inherits it from parent class - _calculated: typing.Set[str] = set() - - def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: - class_name = self.__class__.__name__ - if len(args) > 1: - raise TypeError(f"{class_name}.__init__() takes 2 positional arguments but {len(args) + 1} were given") - dict_to_pass = { - rest_field._rest_name: rest_field._default - for rest_field in self._attr_to_rest_field.values() - if rest_field._default is not _UNSET - } - if args: # pylint: disable=too-many-nested-blocks - if isinstance(args[0], ET.Element): - existed_attr_keys = [] - model_meta = getattr(self, "_xml", {}) - - for rf in self._attr_to_rest_field.values(): - prop_meta = getattr(rf, "_xml", {}) - xml_name = prop_meta.get("name", rf._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - - # attribute - if prop_meta.get("attribute", False) and args[0].get(xml_name) is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].get(xml_name)) - continue - - # unwrapped element is array - if prop_meta.get("unwrapped", False): - # unwrapped array could either use prop items meta/prop meta - if prop_meta.get("itemsName"): - xml_name = prop_meta.get("itemsName") - xml_ns = prop_meta.get("itemNs") - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - items = args[0].findall(xml_name) # pyright: ignore - if len(items) > 0: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, items) - continue - - # text element is primitive type - if prop_meta.get("text", False): - if args[0].text is not None: - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].text) - continue - - # wrapped element could be normal property or array, it should only have one element - item = args[0].find(xml_name) - if item is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, item) - - # rest thing is additional properties - for e in args[0]: - if e.tag not in existed_attr_keys: - dict_to_pass[e.tag] = _convert_element(e) - else: - dict_to_pass.update( - {k: _create_value(_get_rest_field(self._attr_to_rest_field, k), v) for k, v in args[0].items()} - ) - else: - non_attr_kwargs = [k for k in kwargs if k not in self._attr_to_rest_field] - if non_attr_kwargs: - # actual type errors only throw the first wrong keyword arg they see, so following that. - raise TypeError(f"{class_name}.__init__() got an unexpected keyword argument '{non_attr_kwargs[0]}'") - dict_to_pass.update( - { - self._attr_to_rest_field[k]._rest_name: _create_value(self._attr_to_rest_field[k], v) - for k, v in kwargs.items() - if v is not None - } - ) - super().__init__(dict_to_pass) - - def copy(self) -> "Model": - return Model(self.__dict__) - - def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: - if f"{cls.__module__}.{cls.__qualname__}" not in cls._calculated: - # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping', - # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object' - mros = cls.__mro__[:-9][::-1] # ignore parents, and reverse the mro order - attr_to_rest_field: typing.Dict[str, _RestField] = { # map attribute name to rest_field property - k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type") - } - annotations = { - k: v - for mro_class in mros - if hasattr(mro_class, "__annotations__") - for k, v in mro_class.__annotations__.items() - } - for attr, rf in attr_to_rest_field.items(): - rf._module = cls.__module__ - if not rf._type: - rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None)) - if not rf._rest_name_input: - rf._rest_name_input = attr - cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items()) - cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}") - - return super().__new__(cls) - - def __init_subclass__(cls, discriminator: typing.Optional[str] = None) -> None: - for base in cls.__bases__: - if hasattr(base, "__mapping__"): - base.__mapping__[discriminator or cls.__name__] = cls # type: ignore - - @classmethod - def _get_discriminator(cls, exist_discriminators) -> typing.Optional["_RestField"]: - for v in cls.__dict__.values(): - if isinstance(v, _RestField) and v._is_discriminator and v._rest_name not in exist_discriminators: - return v - return None - - @classmethod - def _deserialize(cls, data, exist_discriminators): - if not hasattr(cls, "__mapping__"): - return cls(data) - discriminator = cls._get_discriminator(exist_discriminators) - if discriminator is None: - return cls(data) - exist_discriminators.append(discriminator._rest_name) - if isinstance(data, ET.Element): - model_meta = getattr(cls, "_xml", {}) - prop_meta = getattr(discriminator, "_xml", {}) - xml_name = prop_meta.get("name", discriminator._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - - if data.get(xml_name) is not None: - discriminator_value = data.get(xml_name) - else: - discriminator_value = data.find(xml_name).text # pyright: ignore - else: - discriminator_value = data.get(discriminator._rest_name) - mapped_cls = cls.__mapping__.get(discriminator_value, cls) # pyright: ignore # pylint: disable=no-member - return mapped_cls._deserialize(data, exist_discriminators) - - def as_dict(self, *, exclude_readonly: bool = False) -> typing.Dict[str, typing.Any]: - """Return a dict that can be turned into json using json.dump. - - :keyword bool exclude_readonly: Whether to remove the readonly properties. - :returns: A dict JSON compatible object - :rtype: dict - """ - - result = {} - readonly_props = [] - if exclude_readonly: - readonly_props = [p._rest_name for p in self._attr_to_rest_field.values() if _is_readonly(p)] - for k, v in self.items(): - if exclude_readonly and k in readonly_props: # pyright: ignore - continue - is_multipart_file_input = False - try: - is_multipart_file_input = next( - rf for rf in self._attr_to_rest_field.values() if rf._rest_name == k - )._is_multipart_file_input - except StopIteration: - pass - result[k] = v if is_multipart_file_input else Model._as_dict_value(v, exclude_readonly=exclude_readonly) - return result - - @staticmethod - def _as_dict_value(v: typing.Any, exclude_readonly: bool = False) -> typing.Any: - if v is None or isinstance(v, _Null): - return None - if isinstance(v, (list, tuple, set)): - return type(v)(Model._as_dict_value(x, exclude_readonly=exclude_readonly) for x in v) - if isinstance(v, dict): - return {dk: Model._as_dict_value(dv, exclude_readonly=exclude_readonly) for dk, dv in v.items()} - return v.as_dict(exclude_readonly=exclude_readonly) if hasattr(v, "as_dict") else v - - -def _deserialize_model(model_deserializer: typing.Optional[typing.Callable], obj): - if _is_model(obj): - return obj - return _deserialize(model_deserializer, obj) - - -def _deserialize_with_optional(if_obj_deserializer: typing.Optional[typing.Callable], obj): - if obj is None: - return obj - return _deserialize_with_callable(if_obj_deserializer, obj) - - -def _deserialize_with_union(deserializers, obj): - for deserializer in deserializers: - try: - return _deserialize(deserializer, obj) - except DeserializationError: - pass - raise DeserializationError() - - -def _deserialize_dict( - value_deserializer: typing.Optional[typing.Callable], - module: typing.Optional[str], - obj: typing.Dict[typing.Any, typing.Any], -): - if obj is None: - return obj - if isinstance(obj, ET.Element): - obj = {child.tag: child for child in obj} - return {k: _deserialize(value_deserializer, v, module) for k, v in obj.items()} - - -def _deserialize_multiple_sequence( - entry_deserializers: typing.List[typing.Optional[typing.Callable]], - module: typing.Optional[str], - obj, -): - if obj is None: - return obj - return type(obj)(_deserialize(deserializer, entry, module) for entry, deserializer in zip(obj, entry_deserializers)) - - -def _deserialize_sequence( - deserializer: typing.Optional[typing.Callable], - module: typing.Optional[str], - obj, -): - if obj is None: - return obj - if isinstance(obj, ET.Element): - obj = list(obj) - return type(obj)(_deserialize(deserializer, entry, module) for entry in obj) - - -def _sorted_annotations(types: typing.List[typing.Any]) -> typing.List[typing.Any]: - return sorted( - types, - key=lambda x: hasattr(x, "__name__") and x.__name__.lower() in ("str", "float", "int", "bool"), - ) - - -def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-return-statements, too-many-branches - annotation: typing.Any, - module: typing.Optional[str], - rf: typing.Optional["_RestField"] = None, -) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: - if not annotation: - return None - - # is it a type alias? - if isinstance(annotation, str): - if module is not None: - annotation = _get_type_alias_type(module, annotation) - - # is it a forward ref / in quotes? - if isinstance(annotation, (str, typing.ForwardRef)): - try: - model_name = annotation.__forward_arg__ # type: ignore - except AttributeError: - model_name = annotation - if module is not None: - annotation = _get_model(module, model_name) # type: ignore - - try: - if module and _is_model(annotation): - if rf: - rf._is_model = True - - return functools.partial(_deserialize_model, annotation) # pyright: ignore - except Exception: - pass - - # is it a literal? - try: - if annotation.__origin__ is typing.Literal: # pyright: ignore - return None - except AttributeError: - pass - - # is it optional? - try: - if any(a for a in annotation.__args__ if a == type(None)): # pyright: ignore - if len(annotation.__args__) <= 2: # pyright: ignore - if_obj_deserializer = _get_deserialize_callable_from_annotation( - next(a for a in annotation.__args__ if a != type(None)), module, rf # pyright: ignore - ) - - return functools.partial(_deserialize_with_optional, if_obj_deserializer) - # the type is Optional[Union[...]], we need to remove the None type from the Union - annotation_copy = copy.copy(annotation) - annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a != type(None)] # pyright: ignore - return _get_deserialize_callable_from_annotation(annotation_copy, module, rf) - except AttributeError: - pass - - # is it union? - if getattr(annotation, "__origin__", None) is typing.Union: - # initial ordering is we make `string` the last deserialization option, because it is often them most generic - deserializers = [ - _get_deserialize_callable_from_annotation(arg, module, rf) - for arg in _sorted_annotations(annotation.__args__) # pyright: ignore - ] - - return functools.partial(_deserialize_with_union, deserializers) - - try: - if annotation._name == "Dict": # pyright: ignore - value_deserializer = _get_deserialize_callable_from_annotation( - annotation.__args__[1], module, rf # pyright: ignore - ) - - return functools.partial( - _deserialize_dict, - value_deserializer, - module, - ) - except (AttributeError, IndexError): - pass - try: - if annotation._name in ["List", "Set", "Tuple", "Sequence"]: # pyright: ignore - if len(annotation.__args__) > 1: # pyright: ignore - entry_deserializers = [ - _get_deserialize_callable_from_annotation(dt, module, rf) - for dt in annotation.__args__ # pyright: ignore - ] - return functools.partial(_deserialize_multiple_sequence, entry_deserializers, module) - deserializer = _get_deserialize_callable_from_annotation( - annotation.__args__[0], module, rf # pyright: ignore - ) - - return functools.partial(_deserialize_sequence, deserializer, module) - except (TypeError, IndexError, AttributeError, SyntaxError): - pass - - def _deserialize_default( - deserializer, - obj, - ): - if obj is None: - return obj - try: - return _deserialize_with_callable(deserializer, obj) - except Exception: - pass - return obj - - if get_deserializer(annotation, rf): - return functools.partial(_deserialize_default, get_deserializer(annotation, rf)) - - return functools.partial(_deserialize_default, annotation) - - -def _deserialize_with_callable( - deserializer: typing.Optional[typing.Callable[[typing.Any], typing.Any]], - value: typing.Any, -): # pylint: disable=too-many-return-statements - try: - if value is None or isinstance(value, _Null): - return None - if isinstance(value, ET.Element): - if deserializer is str: - return value.text or "" - if deserializer is int: - return int(value.text) if value.text else None - if deserializer is float: - return float(value.text) if value.text else None - if deserializer is bool: - return value.text == "true" if value.text else None - if deserializer is None: - return value - if deserializer in [int, float, bool]: - return deserializer(value) - if isinstance(deserializer, CaseInsensitiveEnumMeta): - try: - return deserializer(value) - except ValueError: - # for unknown value, return raw value - return value - if isinstance(deserializer, type) and issubclass(deserializer, Model): - return deserializer._deserialize(value, []) - return typing.cast(typing.Callable[[typing.Any], typing.Any], deserializer)(value) - except Exception as e: - raise DeserializationError() from e - - -def _deserialize( - deserializer: typing.Any, - value: typing.Any, - module: typing.Optional[str] = None, - rf: typing.Optional["_RestField"] = None, - format: typing.Optional[str] = None, -) -> typing.Any: - if isinstance(value, PipelineResponse): - value = value.http_response.json() - if rf is None and format: - rf = _RestField(format=format) - if not isinstance(deserializer, functools.partial): - deserializer = _get_deserialize_callable_from_annotation(deserializer, module, rf) - return _deserialize_with_callable(deserializer, value) - - -def _failsafe_deserialize( - deserializer: typing.Any, - value: typing.Any, - module: typing.Optional[str] = None, - rf: typing.Optional["_RestField"] = None, - format: typing.Optional[str] = None, -) -> typing.Any: - try: - return _deserialize(deserializer, value, module, rf, format) - except DeserializationError: - _LOGGER.warning( - "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True - ) - return None - - -def _failsafe_deserialize_xml( - deserializer: typing.Any, - value: typing.Any, -) -> typing.Any: - try: - return _deserialize_xml(deserializer, value) - except DeserializationError: - _LOGGER.warning( - "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True - ) - return None - - -class _RestField: - def __init__( - self, - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - is_discriminator: bool = False, - visibility: typing.Optional[typing.List[str]] = None, - default: typing.Any = _UNSET, - format: typing.Optional[str] = None, - is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, - ): - self._type = type - self._rest_name_input = name - self._module: typing.Optional[str] = None - self._is_discriminator = is_discriminator - self._visibility = visibility - self._is_model = False - self._default = default - self._format = format - self._is_multipart_file_input = is_multipart_file_input - self._xml = xml if xml is not None else {} - - @property - def _class_type(self) -> typing.Any: - return getattr(self._type, "args", [None])[0] - - @property - def _rest_name(self) -> str: - if self._rest_name_input is None: - raise ValueError("Rest name was never set") - return self._rest_name_input - - def __get__(self, obj: Model, type=None): # pylint: disable=redefined-builtin - # by this point, type and rest_name will have a value bc we default - # them in __new__ of the Model class - item = obj.get(self._rest_name) - if item is None: - return item - if self._is_model: - return item - return _deserialize(self._type, _serialize(item, self._format), rf=self) - - def __set__(self, obj: Model, value) -> None: - if value is None: - # we want to wipe out entries if users set attr to None - try: - obj.__delitem__(self._rest_name) - except KeyError: - pass - return - if self._is_model: - if not _is_model(value): - value = _deserialize(self._type, value) - obj.__setitem__(self._rest_name, value) - return - obj.__setitem__(self._rest_name, _serialize(value, self._format)) - - def _get_deserialize_callable_from_annotation( - self, annotation: typing.Any - ) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: - return _get_deserialize_callable_from_annotation(annotation, self._module, self) - - -def rest_field( - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, - default: typing.Any = _UNSET, - format: typing.Optional[str] = None, - is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, -) -> typing.Any: - return _RestField( - name=name, - type=type, - visibility=visibility, - default=default, - format=format, - is_multipart_file_input=is_multipart_file_input, - xml=xml, - ) - - -def rest_discriminator( - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, -) -> typing.Any: - return _RestField(name=name, type=type, is_discriminator=True, visibility=visibility, xml=xml) - - -def serialize_xml(model: Model, exclude_readonly: bool = False) -> str: - """Serialize a model to XML. - - :param Model model: The model to serialize. - :param bool exclude_readonly: Whether to exclude readonly properties. - :returns: The XML representation of the model. - :rtype: str - """ - return ET.tostring(_get_element(model, exclude_readonly), encoding="unicode") # type: ignore - - -def _get_element( - o: typing.Any, - exclude_readonly: bool = False, - parent_meta: typing.Optional[typing.Dict[str, typing.Any]] = None, - wrapped_element: typing.Optional[ET.Element] = None, -) -> typing.Union[ET.Element, typing.List[ET.Element]]: - if _is_model(o): - model_meta = getattr(o, "_xml", {}) - - # if prop is a model, then use the prop element directly, else generate a wrapper of model - if wrapped_element is None: - wrapped_element = _create_xml_element( - model_meta.get("name", o.__class__.__name__), - model_meta.get("prefix"), - model_meta.get("ns"), - ) - - readonly_props = [] - if exclude_readonly: - readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)] - - for k, v in o.items(): - # do not serialize readonly properties - if exclude_readonly and k in readonly_props: - continue - - prop_rest_field = _get_rest_field(o._attr_to_rest_field, k) - if prop_rest_field: - prop_meta = getattr(prop_rest_field, "_xml").copy() - # use the wire name as xml name if no specific name is set - if prop_meta.get("name") is None: - prop_meta["name"] = k - else: - # additional properties will not have rest field, use the wire name as xml name - prop_meta = {"name": k} - - # if no ns for prop, use model's - if prop_meta.get("ns") is None and model_meta.get("ns"): - prop_meta["ns"] = model_meta.get("ns") - prop_meta["prefix"] = model_meta.get("prefix") - - if prop_meta.get("unwrapped", False): - # unwrapped could only set on array - wrapped_element.extend(_get_element(v, exclude_readonly, prop_meta)) - elif prop_meta.get("text", False): - # text could only set on primitive type - wrapped_element.text = _get_primitive_type_value(v) - elif prop_meta.get("attribute", False): - xml_name = prop_meta.get("name", k) - if prop_meta.get("ns"): - ET.register_namespace(prop_meta.get("prefix"), prop_meta.get("ns")) # pyright: ignore - xml_name = "{" + prop_meta.get("ns") + "}" + xml_name # pyright: ignore - # attribute should be primitive type - wrapped_element.set(xml_name, _get_primitive_type_value(v)) - else: - # other wrapped prop element - wrapped_element.append(_get_wrapped_element(v, exclude_readonly, prop_meta)) - return wrapped_element - if isinstance(o, list): - return [_get_element(x, exclude_readonly, parent_meta) for x in o] # type: ignore - if isinstance(o, dict): - result = [] - for k, v in o.items(): - result.append( - _get_wrapped_element( - v, - exclude_readonly, - { - "name": k, - "ns": parent_meta.get("ns") if parent_meta else None, - "prefix": parent_meta.get("prefix") if parent_meta else None, - }, - ) - ) - return result - - # primitive case need to create element based on parent_meta - if parent_meta: - return _get_wrapped_element( - o, - exclude_readonly, - { - "name": parent_meta.get("itemsName", parent_meta.get("name")), - "prefix": parent_meta.get("itemsPrefix", parent_meta.get("prefix")), - "ns": parent_meta.get("itemsNs", parent_meta.get("ns")), - }, - ) - - raise ValueError("Could not serialize value into xml: " + o) - - -def _get_wrapped_element( - v: typing.Any, - exclude_readonly: bool, - meta: typing.Optional[typing.Dict[str, typing.Any]], -) -> ET.Element: - wrapped_element = _create_xml_element( - meta.get("name") if meta else None, meta.get("prefix") if meta else None, meta.get("ns") if meta else None - ) - if isinstance(v, (dict, list)): - wrapped_element.extend(_get_element(v, exclude_readonly, meta)) - elif _is_model(v): - _get_element(v, exclude_readonly, meta, wrapped_element) - else: - wrapped_element.text = _get_primitive_type_value(v) - return wrapped_element - - -def _get_primitive_type_value(v) -> str: - if v is True: - return "true" - if v is False: - return "false" - if isinstance(v, _Null): - return "" - return str(v) - - -def _create_xml_element(tag, prefix=None, ns=None): - if prefix and ns: - ET.register_namespace(prefix, ns) - if ns: - return ET.Element("{" + ns + "}" + tag) - return ET.Element(tag) - - -def _deserialize_xml( - deserializer: typing.Any, - value: str, -) -> typing.Any: - element = ET.fromstring(value) # nosec - return _deserialize(deserializer, element) - - -def _convert_element(e: ET.Element): - # dict case - if len(e.attrib) > 0 or len({child.tag for child in e}) > 1: - dict_result: typing.Dict[str, typing.Any] = {} - for child in e: - if dict_result.get(child.tag) is not None: - if isinstance(dict_result[child.tag], list): - dict_result[child.tag].append(_convert_element(child)) - else: - dict_result[child.tag] = [dict_result[child.tag], _convert_element(child)] - else: - dict_result[child.tag] = _convert_element(child) - dict_result.update(e.attrib) - return dict_result - # array case - if len(e) > 0: - array_result: typing.List[typing.Any] = [] - for child in e: - array_result.append(_convert_element(child)) - return array_result - # primitive case - return e.text diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py index 88d6f825118d..7d9a97cc1d51 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py @@ -8,7 +8,7 @@ from collections.abc import MutableMapping from io import IOBase import json -from typing import Any, Callable, Dict, IO, Iterator, List, Optional, TypeVar, Union, cast, overload +from typing import Any, Callable, IO, Iterator, Optional, TypeVar, Union, cast, overload import urllib.parse from azure.core import PipelineClient @@ -30,15 +30,14 @@ from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict -from .. import models as _models +from .. import models as _models, types as _types from .._configuration import DeidentificationClientConfiguration from .._utils.model_base import SdkJSONEncoder, _deserialize from .._utils.serialization import Serializer from .._utils.utils import ClientMixinABC -JSON = MutableMapping[str, Any] T = TypeVar("T") -ClsType = Optional[Callable[[PipelineResponse[HttpRequest, HttpResponse], T, Dict[str, Any]], Any]] +ClsType = Optional[Callable[[PipelineResponse[HttpRequest, HttpResponse], T, dict[str, Any]], Any]] _SERIALIZER = Serializer() _SERIALIZER.client_side_validation = False @@ -193,8 +192,6 @@ def build_deidentification_delete_job_request( # pylint: disable=name-too-long _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2025-07-15-preview")) - accept = _headers.pop("Accept", "application/json") - # Construct URL _url = "/jobs/{name}" path_format_arguments = { @@ -207,7 +204,6 @@ def build_deidentification_delete_job_request( # pylint: disable=name-too-long _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str") # Construct headers - _headers["Accept"] = _SERIALIZER.header("accept", accept, "str") return HttpRequest(method="DELETE", url=_url, params=_params, headers=_headers, **kwargs) @@ -274,6 +270,7 @@ def get_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJob: } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -296,7 +293,7 @@ def get_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJob: ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -306,7 +303,10 @@ def get_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJob: return deserialized # type: ignore def _deidentify_documents_initial( - self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any + self, + job_name: str, + resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], + **kwargs: Any ) -> Iterator[bytes]: error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -342,6 +342,7 @@ def _deidentify_documents_initial( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -363,7 +364,7 @@ def _deidentify_documents_initial( ) response_headers["Operation-Location"] = self._deserialize("str", response.headers.get("Operation-Location")) - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -399,7 +400,12 @@ def begin_deidentify_documents( @overload def begin_deidentify_documents( - self, job_name: str, resource: JSON, *, content_type: str = "application/json", **kwargs: Any + self, + job_name: str, + resource: _types.DeidentificationJob, + *, + content_type: str = "application/json", + **kwargs: Any ) -> LROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -408,7 +414,7 @@ def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str :param resource: The resource instance. Required. - :type resource: JSON + :type resource: ~azure.health.deidentification.types.DeidentificationJob :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -443,7 +449,10 @@ def begin_deidentify_documents( @distributed_trace def begin_deidentify_documents( - self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any + self, + job_name: str, + resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], + **kwargs: Any ) -> LROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -451,9 +460,10 @@ def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str - :param resource: The resource instance. Is one of the following types: DeidentificationJob, - JSON, IO[bytes] Required. - :type resource: ~azure.health.deidentification.models.DeidentificationJob or JSON or IO[bytes] + :param resource: The resource instance. Is either a DeidentificationJob type or a IO[bytes] + type. Required. + :type resource: ~azure.health.deidentification.models.DeidentificationJob or + ~azure.health.deidentification.types.DeidentificationJob or IO[bytes] :return: An instance of LROPoller that returns DeidentificationJob. The DeidentificationJob is compatible with MutableMapping :rtype: @@ -538,7 +548,7 @@ def _list_jobs_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationJob]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationJob]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -576,7 +586,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -589,7 +602,10 @@ def prepare_request(next_link=None): def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationJob], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationJob], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, iter(list_of_elem) @@ -633,7 +649,7 @@ def _list_job_documents_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -672,7 +688,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -685,7 +704,10 @@ def prepare_request(next_link=None): def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationDocumentDetails], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationDocumentDetails], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, iter(list_of_elem) @@ -748,6 +770,7 @@ def cancel_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJo } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -770,7 +793,7 @@ def cancel_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJo ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -854,14 +877,14 @@ def deidentify_text( @overload def deidentify_text( - self, body: JSON, *, content_type: str = "application/json", **kwargs: Any + self, body: _types.DeidentificationContent, *, content_type: str = "application/json", **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. :param body: Request body for de-identification operation. Required. - :type body: JSON + :type body: ~azure.health.deidentification.types.DeidentificationContent :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -890,15 +913,16 @@ def deidentify_text( @distributed_trace def deidentify_text( - self, body: Union[_models.DeidentificationContent, JSON, IO[bytes]], **kwargs: Any + self, body: Union[_models.DeidentificationContent, _types.DeidentificationContent, IO[bytes]], **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. - :param body: Request body for de-identification operation. Is one of the following types: - DeidentificationContent, JSON, IO[bytes] Required. - :type body: ~azure.health.deidentification.models.DeidentificationContent or JSON or IO[bytes] + :param body: Request body for de-identification operation. Is either a DeidentificationContent + type or a IO[bytes] type. Required. + :type body: ~azure.health.deidentification.models.DeidentificationContent or + ~azure.health.deidentification.types.DeidentificationContent or IO[bytes] :return: DeidentificationResult. The DeidentificationResult is compatible with MutableMapping :rtype: ~azure.health.deidentification.models.DeidentificationResult :raises ~azure.core.exceptions.HttpResponseError: @@ -936,6 +960,7 @@ def deidentify_text( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -958,7 +983,7 @@ def deidentify_text( ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationResult, response.json()) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_serialization.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_serialization.py deleted file mode 100644 index eb86ea23c965..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_serialization.py +++ /dev/null @@ -1,2032 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression,too-many-lines -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -# pyright: reportUnnecessaryTypeIgnoreComment=false - -from base64 import b64decode, b64encode -import calendar -import datetime -import decimal -import email -from enum import Enum -import json -import logging -import re -import sys -import codecs -from typing import ( - Dict, - Any, - cast, - Optional, - Union, - AnyStr, - IO, - Mapping, - Callable, - MutableMapping, - List, -) - -try: - from urllib import quote # type: ignore -except ImportError: - from urllib.parse import quote -import xml.etree.ElementTree as ET - -import isodate # type: ignore -from typing_extensions import Self - -from azure.core.exceptions import DeserializationError, SerializationError -from azure.core.serialization import NULL as CoreNull - -_BOM = codecs.BOM_UTF8.decode(encoding="utf-8") - -JSON = MutableMapping[str, Any] - - -class RawDeserializer: - - # Accept "text" because we're open minded people... - JSON_REGEXP = re.compile(r"^(application|text)/([a-z+.]+\+)?json$") - - # Name used in context - CONTEXT_NAME = "deserialized_data" - - @classmethod - def deserialize_from_text(cls, data: Optional[Union[AnyStr, IO]], content_type: Optional[str] = None) -> Any: - """Decode data according to content-type. - - Accept a stream of data as well, but will be load at once in memory for now. - - If no content-type, will return the string version (not bytes, not stream) - - :param data: Input, could be bytes or stream (will be decoded with UTF8) or text - :type data: str or bytes or IO - :param str content_type: The content type. - :return: The deserialized data. - :rtype: object - """ - if hasattr(data, "read"): - # Assume a stream - data = cast(IO, data).read() - - if isinstance(data, bytes): - data_as_str = data.decode(encoding="utf-8-sig") - else: - # Explain to mypy the correct type. - data_as_str = cast(str, data) - - # Remove Byte Order Mark if present in string - data_as_str = data_as_str.lstrip(_BOM) - - if content_type is None: - return data - - if cls.JSON_REGEXP.match(content_type): - try: - return json.loads(data_as_str) - except ValueError as err: - raise DeserializationError("JSON is invalid: {}".format(err), err) from err - elif "xml" in (content_type or []): - try: - - try: - if isinstance(data, unicode): # type: ignore - # If I'm Python 2.7 and unicode XML will scream if I try a "fromstring" on unicode string - data_as_str = data_as_str.encode(encoding="utf-8") # type: ignore - except NameError: - pass - - return ET.fromstring(data_as_str) # nosec - except ET.ParseError as err: - # It might be because the server has an issue, and returned JSON with - # content-type XML.... - # So let's try a JSON load, and if it's still broken - # let's flow the initial exception - def _json_attemp(data): - try: - return True, json.loads(data) - except ValueError: - return False, None # Don't care about this one - - success, json_result = _json_attemp(data) - if success: - return json_result - # If i'm here, it's not JSON, it's not XML, let's scream - # and raise the last context in this block (the XML exception) - # The function hack is because Py2.7 messes up with exception - # context otherwise. - _LOGGER.critical("Wasn't XML not JSON, failing") - raise DeserializationError("XML is invalid") from err - elif content_type.startswith("text/"): - return data_as_str - raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) - - @classmethod - def deserialize_from_http_generics(cls, body_bytes: Optional[Union[AnyStr, IO]], headers: Mapping) -> Any: - """Deserialize from HTTP response. - - Use bytes and headers to NOT use any requests/aiohttp or whatever - specific implementation. - Headers will tested for "content-type" - - :param bytes body_bytes: The body of the response. - :param dict headers: The headers of the response. - :returns: The deserialized data. - :rtype: object - """ - # Try to use content-type from headers if available - content_type = None - if "content-type" in headers: - content_type = headers["content-type"].split(";")[0].strip().lower() - # Ouch, this server did not declare what it sent... - # Let's guess it's JSON... - # Also, since Autorest was considering that an empty body was a valid JSON, - # need that test as well.... - else: - content_type = "application/json" - - if body_bytes: - return cls.deserialize_from_text(body_bytes, content_type) - return None - - -_LOGGER = logging.getLogger(__name__) - -try: - _long_type = long # type: ignore -except NameError: - _long_type = int - -TZ_UTC = datetime.timezone.utc - -_FLATTEN = re.compile(r"(? None: - self.additional_properties: Optional[Dict[str, Any]] = {} - for k in kwargs: # pylint: disable=consider-using-dict-items - if k not in self._attribute_map: - _LOGGER.warning("%s is not a known attribute of class %s and will be ignored", k, self.__class__) - elif k in self._validation and self._validation[k].get("readonly", False): - _LOGGER.warning("Readonly attribute %s will be ignored in class %s", k, self.__class__) - else: - setattr(self, k, kwargs[k]) - - def __eq__(self, other: Any) -> bool: - """Compare objects by comparing all attributes. - - :param object other: The object to compare - :returns: True if objects are equal - :rtype: bool - """ - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return False - - def __ne__(self, other: Any) -> bool: - """Compare objects by comparing all attributes. - - :param object other: The object to compare - :returns: True if objects are not equal - :rtype: bool - """ - return not self.__eq__(other) - - def __str__(self) -> str: - return str(self.__dict__) - - @classmethod - def enable_additional_properties_sending(cls) -> None: - cls._attribute_map["additional_properties"] = {"key": "", "type": "{object}"} - - @classmethod - def is_xml_model(cls) -> bool: - try: - cls._xml_map # type: ignore - except AttributeError: - return False - return True - - @classmethod - def _create_xml_node(cls): - """Create XML node. - - :returns: The XML node - :rtype: xml.etree.ElementTree.Element - """ - try: - xml_map = cls._xml_map # type: ignore - except AttributeError: - xml_map = {} - - return _create_xml_node(xml_map.get("name", cls.__name__), xml_map.get("prefix", None), xml_map.get("ns", None)) - - def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> JSON: - """Return the JSON that would be sent to server from this model. - - This is an alias to `as_dict(full_restapi_key_transformer, keep_readonly=False)`. - - If you want XML serialization, you can pass the kwargs is_xml=True. - - :param bool keep_readonly: If you want to serialize the readonly attributes - :returns: A dict JSON compatible object - :rtype: dict - """ - serializer = Serializer(self._infer_class_models()) - return serializer._serialize( # type: ignore # pylint: disable=protected-access - self, keep_readonly=keep_readonly, **kwargs - ) - - def as_dict( - self, - keep_readonly: bool = True, - key_transformer: Callable[[str, Dict[str, Any], Any], Any] = attribute_transformer, - **kwargs: Any - ) -> JSON: - """Return a dict that can be serialized using json.dump. - - Advanced usage might optionally use a callback as parameter: - - .. code::python - - def my_key_transformer(key, attr_desc, value): - return key - - Key is the attribute name used in Python. Attr_desc - is a dict of metadata. Currently contains 'type' with the - msrest type and 'key' with the RestAPI encoded key. - Value is the current value in this object. - - The string returned will be used to serialize the key. - If the return type is a list, this is considered hierarchical - result dict. - - See the three examples in this file: - - - attribute_transformer - - full_restapi_key_transformer - - last_restapi_key_transformer - - If you want XML serialization, you can pass the kwargs is_xml=True. - - :param bool keep_readonly: If you want to serialize the readonly attributes - :param function key_transformer: A key transformer function. - :returns: A dict JSON compatible object - :rtype: dict - """ - serializer = Serializer(self._infer_class_models()) - return serializer._serialize( # type: ignore # pylint: disable=protected-access - self, key_transformer=key_transformer, keep_readonly=keep_readonly, **kwargs - ) - - @classmethod - def _infer_class_models(cls): - try: - str_models = cls.__module__.rsplit(".", 1)[0] - models = sys.modules[str_models] - client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)} - if cls.__name__ not in client_models: - raise ValueError("Not Autorest generated code") - except Exception: # pylint: disable=broad-exception-caught - # Assume it's not Autorest generated (tests?). Add ourselves as dependencies. - client_models = {cls.__name__: cls} - return client_models - - @classmethod - def deserialize(cls, data: Any, content_type: Optional[str] = None) -> Self: - """Parse a str using the RestAPI syntax and return a model. - - :param str data: A str using RestAPI structure. JSON by default. - :param str content_type: JSON by default, set application/xml if XML. - :returns: An instance of this model - :raises DeserializationError: if something went wrong - :rtype: Self - """ - deserializer = Deserializer(cls._infer_class_models()) - return deserializer(cls.__name__, data, content_type=content_type) # type: ignore - - @classmethod - def from_dict( - cls, - data: Any, - key_extractors: Optional[Callable[[str, Dict[str, Any], Any], Any]] = None, - content_type: Optional[str] = None, - ) -> Self: - """Parse a dict using given key extractor return a model. - - By default consider key - extractors (rest_key_case_insensitive_extractor, attribute_key_case_insensitive_extractor - and last_rest_key_case_insensitive_extractor) - - :param dict data: A dict using RestAPI structure - :param function key_extractors: A key extractor function. - :param str content_type: JSON by default, set application/xml if XML. - :returns: An instance of this model - :raises DeserializationError: if something went wrong - :rtype: Self - """ - deserializer = Deserializer(cls._infer_class_models()) - deserializer.key_extractors = ( # type: ignore - [ # type: ignore - attribute_key_case_insensitive_extractor, - rest_key_case_insensitive_extractor, - last_rest_key_case_insensitive_extractor, - ] - if key_extractors is None - else key_extractors - ) - return deserializer(cls.__name__, data, content_type=content_type) # type: ignore - - @classmethod - def _flatten_subtype(cls, key, objects): - if "_subtype_map" not in cls.__dict__: - return {} - result = dict(cls._subtype_map[key]) - for valuetype in cls._subtype_map[key].values(): - result.update(objects[valuetype]._flatten_subtype(key, objects)) # pylint: disable=protected-access - return result - - @classmethod - def _classify(cls, response, objects): - """Check the class _subtype_map for any child classes. - We want to ignore any inherited _subtype_maps. - - :param dict response: The initial data - :param dict objects: The class objects - :returns: The class to be used - :rtype: class - """ - for subtype_key in cls.__dict__.get("_subtype_map", {}).keys(): - subtype_value = None - - if not isinstance(response, ET.Element): - rest_api_response_key = cls._get_rest_key_parts(subtype_key)[-1] - subtype_value = response.get(rest_api_response_key, None) or response.get(subtype_key, None) - else: - subtype_value = xml_key_extractor(subtype_key, cls._attribute_map[subtype_key], response) - if subtype_value: - # Try to match base class. Can be class name only - # (bug to fix in Autorest to support x-ms-discriminator-name) - if cls.__name__ == subtype_value: - return cls - flatten_mapping_type = cls._flatten_subtype(subtype_key, objects) - try: - return objects[flatten_mapping_type[subtype_value]] # type: ignore - except KeyError: - _LOGGER.warning( - "Subtype value %s has no mapping, use base class %s.", - subtype_value, - cls.__name__, - ) - break - else: - _LOGGER.warning("Discriminator %s is absent or null, use base class %s.", subtype_key, cls.__name__) - break - return cls - - @classmethod - def _get_rest_key_parts(cls, attr_key): - """Get the RestAPI key of this attr, split it and decode part - :param str attr_key: Attribute key must be in attribute_map. - :returns: A list of RestAPI part - :rtype: list - """ - rest_split_key = _FLATTEN.split(cls._attribute_map[attr_key]["key"]) - return [_decode_attribute_map_key(key_part) for key_part in rest_split_key] - - -def _decode_attribute_map_key(key): - """This decode a key in an _attribute_map to the actual key we want to look at - inside the received data. - - :param str key: A key string from the generated code - :returns: The decoded key - :rtype: str - """ - return key.replace("\\.", ".") - - -class Serializer: # pylint: disable=too-many-public-methods - """Request object model serializer.""" - - basic_types = {str: "str", int: "int", bool: "bool", float: "float"} - - _xml_basic_types_serializers = {"bool": lambda x: str(x).lower()} - days = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 4: "Fri", 5: "Sat", 6: "Sun"} - months = { - 1: "Jan", - 2: "Feb", - 3: "Mar", - 4: "Apr", - 5: "May", - 6: "Jun", - 7: "Jul", - 8: "Aug", - 9: "Sep", - 10: "Oct", - 11: "Nov", - 12: "Dec", - } - validation = { - "min_length": lambda x, y: len(x) < y, - "max_length": lambda x, y: len(x) > y, - "minimum": lambda x, y: x < y, - "maximum": lambda x, y: x > y, - "minimum_ex": lambda x, y: x <= y, - "maximum_ex": lambda x, y: x >= y, - "min_items": lambda x, y: len(x) < y, - "max_items": lambda x, y: len(x) > y, - "pattern": lambda x, y: not re.match(y, x, re.UNICODE), - "unique": lambda x, y: len(x) != len(set(x)), - "multiple": lambda x, y: x % y != 0, - } - - def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: - self.serialize_type = { - "iso-8601": Serializer.serialize_iso, - "rfc-1123": Serializer.serialize_rfc, - "unix-time": Serializer.serialize_unix, - "duration": Serializer.serialize_duration, - "date": Serializer.serialize_date, - "time": Serializer.serialize_time, - "decimal": Serializer.serialize_decimal, - "long": Serializer.serialize_long, - "bytearray": Serializer.serialize_bytearray, - "base64": Serializer.serialize_base64, - "object": self.serialize_object, - "[]": self.serialize_iter, - "{}": self.serialize_dict, - } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} - self.key_transformer = full_restapi_key_transformer - self.client_side_validation = True - - def _serialize( # pylint: disable=too-many-nested-blocks, too-many-branches, too-many-statements, too-many-locals - self, target_obj, data_type=None, **kwargs - ): - """Serialize data into a string according to type. - - :param object target_obj: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str, dict - :raises SerializationError: if serialization fails. - :returns: The serialized data. - """ - key_transformer = kwargs.get("key_transformer", self.key_transformer) - keep_readonly = kwargs.get("keep_readonly", False) - if target_obj is None: - return None - - attr_name = None - class_name = target_obj.__class__.__name__ - - if data_type: - return self.serialize_data(target_obj, data_type, **kwargs) - - if not hasattr(target_obj, "_attribute_map"): - data_type = type(target_obj).__name__ - if data_type in self.basic_types.values(): - return self.serialize_data(target_obj, data_type, **kwargs) - - # Force "is_xml" kwargs if we detect a XML model - try: - is_xml_model_serialization = kwargs["is_xml"] - except KeyError: - is_xml_model_serialization = kwargs.setdefault("is_xml", target_obj.is_xml_model()) - - serialized = {} - if is_xml_model_serialization: - serialized = target_obj._create_xml_node() # pylint: disable=protected-access - try: - attributes = target_obj._attribute_map # pylint: disable=protected-access - for attr, attr_desc in attributes.items(): - attr_name = attr - if not keep_readonly and target_obj._validation.get( # pylint: disable=protected-access - attr_name, {} - ).get("readonly", False): - continue - - if attr_name == "additional_properties" and attr_desc["key"] == "": - if target_obj.additional_properties is not None: - serialized.update(target_obj.additional_properties) - continue - try: - - orig_attr = getattr(target_obj, attr) - if is_xml_model_serialization: - pass # Don't provide "transformer" for XML for now. Keep "orig_attr" - else: # JSON - keys, orig_attr = key_transformer(attr, attr_desc.copy(), orig_attr) - keys = keys if isinstance(keys, list) else [keys] - - kwargs["serialization_ctxt"] = attr_desc - new_attr = self.serialize_data(orig_attr, attr_desc["type"], **kwargs) - - if is_xml_model_serialization: - xml_desc = attr_desc.get("xml", {}) - xml_name = xml_desc.get("name", attr_desc["key"]) - xml_prefix = xml_desc.get("prefix", None) - xml_ns = xml_desc.get("ns", None) - if xml_desc.get("attr", False): - if xml_ns: - ET.register_namespace(xml_prefix, xml_ns) - xml_name = "{{{}}}{}".format(xml_ns, xml_name) - serialized.set(xml_name, new_attr) # type: ignore - continue - if xml_desc.get("text", False): - serialized.text = new_attr # type: ignore - continue - if isinstance(new_attr, list): - serialized.extend(new_attr) # type: ignore - elif isinstance(new_attr, ET.Element): - # If the down XML has no XML/Name, - # we MUST replace the tag with the local tag. But keeping the namespaces. - if "name" not in getattr(orig_attr, "_xml_map", {}): - splitted_tag = new_attr.tag.split("}") - if len(splitted_tag) == 2: # Namespace - new_attr.tag = "}".join([splitted_tag[0], xml_name]) - else: - new_attr.tag = xml_name - serialized.append(new_attr) # type: ignore - else: # That's a basic type - # Integrate namespace if necessary - local_node = _create_xml_node(xml_name, xml_prefix, xml_ns) - local_node.text = str(new_attr) - serialized.append(local_node) # type: ignore - else: # JSON - for k in reversed(keys): # type: ignore - new_attr = {k: new_attr} - - _new_attr = new_attr - _serialized = serialized - for k in keys: # type: ignore - if k not in _serialized: - _serialized.update(_new_attr) # type: ignore - _new_attr = _new_attr[k] # type: ignore - _serialized = _serialized[k] - except ValueError as err: - if isinstance(err, SerializationError): - raise - - except (AttributeError, KeyError, TypeError) as err: - msg = "Attribute {} in object {} cannot be serialized.\n{}".format(attr_name, class_name, str(target_obj)) - raise SerializationError(msg) from err - return serialized - - def body(self, data, data_type, **kwargs): - """Serialize data intended for a request body. - - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: dict - :raises SerializationError: if serialization fails. - :raises ValueError: if data is None - :returns: The serialized request body - """ - - # Just in case this is a dict - internal_data_type_str = data_type.strip("[]{}") - internal_data_type = self.dependencies.get(internal_data_type_str, None) - try: - is_xml_model_serialization = kwargs["is_xml"] - except KeyError: - if internal_data_type and issubclass(internal_data_type, Model): - is_xml_model_serialization = kwargs.setdefault("is_xml", internal_data_type.is_xml_model()) - else: - is_xml_model_serialization = False - if internal_data_type and not isinstance(internal_data_type, Enum): - try: - deserializer = Deserializer(self.dependencies) - # Since it's on serialization, it's almost sure that format is not JSON REST - # We're not able to deal with additional properties for now. - deserializer.additional_properties_detection = False - if is_xml_model_serialization: - deserializer.key_extractors = [ # type: ignore - attribute_key_case_insensitive_extractor, - ] - else: - deserializer.key_extractors = [ - rest_key_case_insensitive_extractor, - attribute_key_case_insensitive_extractor, - last_rest_key_case_insensitive_extractor, - ] - data = deserializer._deserialize(data_type, data) # pylint: disable=protected-access - except DeserializationError as err: - raise SerializationError("Unable to build a model: " + str(err)) from err - - return self._serialize(data, data_type, **kwargs) - - def url(self, name, data, data_type, **kwargs): - """Serialize data intended for a URL path. - - :param str name: The name of the URL path parameter. - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str - :returns: The serialized URL path - :raises TypeError: if serialization fails. - :raises ValueError: if data is None - """ - try: - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - - if kwargs.get("skip_quote") is True: - output = str(output) - output = output.replace("{", quote("{")).replace("}", quote("}")) - else: - output = quote(str(output), safe="") - except SerializationError as exc: - raise TypeError("{} must be type {}.".format(name, data_type)) from exc - return output - - def query(self, name, data, data_type, **kwargs): - """Serialize data intended for a URL query. - - :param str name: The name of the query parameter. - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str, list - :raises TypeError: if serialization fails. - :raises ValueError: if data is None - :returns: The serialized query parameter - """ - try: - # Treat the list aside, since we don't want to encode the div separator - if data_type.startswith("["): - internal_data_type = data_type[1:-1] - do_quote = not kwargs.get("skip_quote", False) - return self.serialize_iter(data, internal_data_type, do_quote=do_quote, **kwargs) - - # Not a list, regular serialization - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - if kwargs.get("skip_quote") is True: - output = str(output) - else: - output = quote(str(output), safe="") - except SerializationError as exc: - raise TypeError("{} must be type {}.".format(name, data_type)) from exc - return str(output) - - def header(self, name, data, data_type, **kwargs): - """Serialize data intended for a request header. - - :param str name: The name of the header. - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str - :raises TypeError: if serialization fails. - :raises ValueError: if data is None - :returns: The serialized header - """ - try: - if data_type in ["[str]"]: - data = ["" if d is None else d for d in data] - - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - except SerializationError as exc: - raise TypeError("{} must be type {}.".format(name, data_type)) from exc - return str(output) - - def serialize_data(self, data, data_type, **kwargs): - """Serialize generic data according to supplied data type. - - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :raises AttributeError: if required data is None. - :raises ValueError: if data is None - :raises SerializationError: if serialization fails. - :returns: The serialized data. - :rtype: str, int, float, bool, dict, list - """ - if data is None: - raise ValueError("No value for given attribute") - - try: - if data is CoreNull: - return None - if data_type in self.basic_types.values(): - return self.serialize_basic(data, data_type, **kwargs) - - if data_type in self.serialize_type: - return self.serialize_type[data_type](data, **kwargs) - - # If dependencies is empty, try with current data class - # It has to be a subclass of Enum anyway - enum_type = self.dependencies.get(data_type, data.__class__) - if issubclass(enum_type, Enum): - return Serializer.serialize_enum(data, enum_obj=enum_type) - - iter_type = data_type[0] + data_type[-1] - if iter_type in self.serialize_type: - return self.serialize_type[iter_type](data, data_type[1:-1], **kwargs) - - except (ValueError, TypeError) as err: - msg = "Unable to serialize value: {!r} as type: {!r}." - raise SerializationError(msg.format(data, data_type)) from err - return self._serialize(data, **kwargs) - - @classmethod - def _get_custom_serializers(cls, data_type, **kwargs): # pylint: disable=inconsistent-return-statements - custom_serializer = kwargs.get("basic_types_serializers", {}).get(data_type) - if custom_serializer: - return custom_serializer - if kwargs.get("is_xml", False): - return cls._xml_basic_types_serializers.get(data_type) - - @classmethod - def serialize_basic(cls, data, data_type, **kwargs): - """Serialize basic builting data type. - Serializes objects to str, int, float or bool. - - Possible kwargs: - - basic_types_serializers dict[str, callable] : If set, use the callable as serializer - - is_xml bool : If set, use xml_basic_types_serializers - - :param obj data: Object to be serialized. - :param str data_type: Type of object in the iterable. - :rtype: str, int, float, bool - :return: serialized object - """ - custom_serializer = cls._get_custom_serializers(data_type, **kwargs) - if custom_serializer: - return custom_serializer(data) - if data_type == "str": - return cls.serialize_unicode(data) - return eval(data_type)(data) # nosec # pylint: disable=eval-used - - @classmethod - def serialize_unicode(cls, data): - """Special handling for serializing unicode strings in Py2. - Encode to UTF-8 if unicode, otherwise handle as a str. - - :param str data: Object to be serialized. - :rtype: str - :return: serialized object - """ - try: # If I received an enum, return its value - return data.value - except AttributeError: - pass - - try: - if isinstance(data, unicode): # type: ignore - # Don't change it, JSON and XML ElementTree are totally able - # to serialize correctly u'' strings - return data - except NameError: - return str(data) - return str(data) - - def serialize_iter(self, data, iter_type, div=None, **kwargs): - """Serialize iterable. - - Supported kwargs: - - serialization_ctxt dict : The current entry of _attribute_map, or same format. - serialization_ctxt['type'] should be same as data_type. - - is_xml bool : If set, serialize as XML - - :param list data: Object to be serialized. - :param str iter_type: Type of object in the iterable. - :param str div: If set, this str will be used to combine the elements - in the iterable into a combined string. Default is 'None'. - Defaults to False. - :rtype: list, str - :return: serialized iterable - """ - if isinstance(data, str): - raise SerializationError("Refuse str type as a valid iter type.") - - serialization_ctxt = kwargs.get("serialization_ctxt", {}) - is_xml = kwargs.get("is_xml", False) - - serialized = [] - for d in data: - try: - serialized.append(self.serialize_data(d, iter_type, **kwargs)) - except ValueError as err: - if isinstance(err, SerializationError): - raise - serialized.append(None) - - if kwargs.get("do_quote", False): - serialized = ["" if s is None else quote(str(s), safe="") for s in serialized] - - if div: - serialized = ["" if s is None else str(s) for s in serialized] - serialized = div.join(serialized) - - if "xml" in serialization_ctxt or is_xml: - # XML serialization is more complicated - xml_desc = serialization_ctxt.get("xml", {}) - xml_name = xml_desc.get("name") - if not xml_name: - xml_name = serialization_ctxt["key"] - - # Create a wrap node if necessary (use the fact that Element and list have "append") - is_wrapped = xml_desc.get("wrapped", False) - node_name = xml_desc.get("itemsName", xml_name) - if is_wrapped: - final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - else: - final_result = [] - # All list elements to "local_node" - for el in serialized: - if isinstance(el, ET.Element): - el_node = el - else: - el_node = _create_xml_node(node_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - if el is not None: # Otherwise it writes "None" :-p - el_node.text = str(el) - final_result.append(el_node) - return final_result - return serialized - - def serialize_dict(self, attr, dict_type, **kwargs): - """Serialize a dictionary of objects. - - :param dict attr: Object to be serialized. - :param str dict_type: Type of object in the dictionary. - :rtype: dict - :return: serialized dictionary - """ - serialization_ctxt = kwargs.get("serialization_ctxt", {}) - serialized = {} - for key, value in attr.items(): - try: - serialized[self.serialize_unicode(key)] = self.serialize_data(value, dict_type, **kwargs) - except ValueError as err: - if isinstance(err, SerializationError): - raise - serialized[self.serialize_unicode(key)] = None - - if "xml" in serialization_ctxt: - # XML serialization is more complicated - xml_desc = serialization_ctxt["xml"] - xml_name = xml_desc["name"] - - final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - for key, value in serialized.items(): - ET.SubElement(final_result, key).text = value - return final_result - - return serialized - - def serialize_object(self, attr, **kwargs): # pylint: disable=too-many-return-statements - """Serialize a generic object. - This will be handled as a dictionary. If object passed in is not - a basic type (str, int, float, dict, list) it will simply be - cast to str. - - :param dict attr: Object to be serialized. - :rtype: dict or str - :return: serialized object - """ - if attr is None: - return None - if isinstance(attr, ET.Element): - return attr - obj_type = type(attr) - if obj_type in self.basic_types: - return self.serialize_basic(attr, self.basic_types[obj_type], **kwargs) - if obj_type is _long_type: - return self.serialize_long(attr) - if obj_type is str: - return self.serialize_unicode(attr) - if obj_type is datetime.datetime: - return self.serialize_iso(attr) - if obj_type is datetime.date: - return self.serialize_date(attr) - if obj_type is datetime.time: - return self.serialize_time(attr) - if obj_type is datetime.timedelta: - return self.serialize_duration(attr) - if obj_type is decimal.Decimal: - return self.serialize_decimal(attr) - - # If it's a model or I know this dependency, serialize as a Model - if obj_type in self.dependencies.values() or isinstance(attr, Model): - return self._serialize(attr) - - if obj_type == dict: - serialized = {} - for key, value in attr.items(): - try: - serialized[self.serialize_unicode(key)] = self.serialize_object(value, **kwargs) - except ValueError: - serialized[self.serialize_unicode(key)] = None - return serialized - - if obj_type == list: - serialized = [] - for obj in attr: - try: - serialized.append(self.serialize_object(obj, **kwargs)) - except ValueError: - pass - return serialized - return str(attr) - - @staticmethod - def serialize_enum(attr, enum_obj=None): - try: - result = attr.value - except AttributeError: - result = attr - try: - enum_obj(result) # type: ignore - return result - except ValueError as exc: - for enum_value in enum_obj: # type: ignore - if enum_value.value.lower() == str(attr).lower(): - return enum_value.value - error = "{!r} is not valid value for enum {!r}" - raise SerializationError(error.format(attr, enum_obj)) from exc - - @staticmethod - def serialize_bytearray(attr, **kwargs): # pylint: disable=unused-argument - """Serialize bytearray into base-64 string. - - :param str attr: Object to be serialized. - :rtype: str - :return: serialized base64 - """ - return b64encode(attr).decode() - - @staticmethod - def serialize_base64(attr, **kwargs): # pylint: disable=unused-argument - """Serialize str into base-64 string. - - :param str attr: Object to be serialized. - :rtype: str - :return: serialized base64 - """ - encoded = b64encode(attr).decode("ascii") - return encoded.strip("=").replace("+", "-").replace("/", "_") - - @staticmethod - def serialize_decimal(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Decimal object to float. - - :param decimal attr: Object to be serialized. - :rtype: float - :return: serialized decimal - """ - return float(attr) - - @staticmethod - def serialize_long(attr, **kwargs): # pylint: disable=unused-argument - """Serialize long (Py2) or int (Py3). - - :param int attr: Object to be serialized. - :rtype: int/long - :return: serialized long - """ - return _long_type(attr) - - @staticmethod - def serialize_date(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Date object into ISO-8601 formatted string. - - :param Date attr: Object to be serialized. - :rtype: str - :return: serialized date - """ - if isinstance(attr, str): - attr = isodate.parse_date(attr) - t = "{:04}-{:02}-{:02}".format(attr.year, attr.month, attr.day) - return t - - @staticmethod - def serialize_time(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Time object into ISO-8601 formatted string. - - :param datetime.time attr: Object to be serialized. - :rtype: str - :return: serialized time - """ - if isinstance(attr, str): - attr = isodate.parse_time(attr) - t = "{:02}:{:02}:{:02}".format(attr.hour, attr.minute, attr.second) - if attr.microsecond: - t += ".{:02}".format(attr.microsecond) - return t - - @staticmethod - def serialize_duration(attr, **kwargs): # pylint: disable=unused-argument - """Serialize TimeDelta object into ISO-8601 formatted string. - - :param TimeDelta attr: Object to be serialized. - :rtype: str - :return: serialized duration - """ - if isinstance(attr, str): - attr = isodate.parse_duration(attr) - return isodate.duration_isoformat(attr) - - @staticmethod - def serialize_rfc(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Datetime object into RFC-1123 formatted string. - - :param Datetime attr: Object to be serialized. - :rtype: str - :raises TypeError: if format invalid. - :return: serialized rfc - """ - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - utc = attr.utctimetuple() - except AttributeError as exc: - raise TypeError("RFC1123 object must be valid Datetime object.") from exc - - return "{}, {:02} {} {:04} {:02}:{:02}:{:02} GMT".format( - Serializer.days[utc.tm_wday], - utc.tm_mday, - Serializer.months[utc.tm_mon], - utc.tm_year, - utc.tm_hour, - utc.tm_min, - utc.tm_sec, - ) - - @staticmethod - def serialize_iso(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Datetime object into ISO-8601 formatted string. - - :param Datetime attr: Object to be serialized. - :rtype: str - :raises SerializationError: if format invalid. - :return: serialized iso - """ - if isinstance(attr, str): - attr = isodate.parse_datetime(attr) - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - utc = attr.utctimetuple() - if utc.tm_year > 9999 or utc.tm_year < 1: - raise OverflowError("Hit max or min date") - - microseconds = str(attr.microsecond).rjust(6, "0").rstrip("0").ljust(3, "0") - if microseconds: - microseconds = "." + microseconds - date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format( - utc.tm_year, utc.tm_mon, utc.tm_mday, utc.tm_hour, utc.tm_min, utc.tm_sec - ) - return date + microseconds + "Z" - except (ValueError, OverflowError) as err: - msg = "Unable to serialize datetime object." - raise SerializationError(msg) from err - except AttributeError as err: - msg = "ISO-8601 object must be valid Datetime object." - raise TypeError(msg) from err - - @staticmethod - def serialize_unix(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Datetime object into IntTime format. - This is represented as seconds. - - :param Datetime attr: Object to be serialized. - :rtype: int - :raises SerializationError: if format invalid - :return: serialied unix - """ - if isinstance(attr, int): - return attr - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - return int(calendar.timegm(attr.utctimetuple())) - except AttributeError as exc: - raise TypeError("Unix time object must be valid Datetime object.") from exc - - -def rest_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argument - key = attr_desc["key"] - working_data = data - - while "." in key: - # Need the cast, as for some reasons "split" is typed as list[str | Any] - dict_keys = cast(List[str], _FLATTEN.split(key)) - if len(dict_keys) == 1: - key = _decode_attribute_map_key(dict_keys[0]) - break - working_key = _decode_attribute_map_key(dict_keys[0]) - working_data = working_data.get(working_key, data) - if working_data is None: - # If at any point while following flatten JSON path see None, it means - # that all properties under are None as well - return None - key = ".".join(dict_keys[1:]) - - return working_data.get(key) - - -def rest_key_case_insensitive_extractor( # pylint: disable=unused-argument, inconsistent-return-statements - attr, attr_desc, data -): - key = attr_desc["key"] - working_data = data - - while "." in key: - dict_keys = _FLATTEN.split(key) - if len(dict_keys) == 1: - key = _decode_attribute_map_key(dict_keys[0]) - break - working_key = _decode_attribute_map_key(dict_keys[0]) - working_data = attribute_key_case_insensitive_extractor(working_key, None, working_data) - if working_data is None: - # If at any point while following flatten JSON path see None, it means - # that all properties under are None as well - return None - key = ".".join(dict_keys[1:]) - - if working_data: - return attribute_key_case_insensitive_extractor(key, None, working_data) - - -def last_rest_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argument - """Extract the attribute in "data" based on the last part of the JSON path key. - - :param str attr: The attribute to extract - :param dict attr_desc: The attribute description - :param dict data: The data to extract from - :rtype: object - :returns: The extracted attribute - """ - key = attr_desc["key"] - dict_keys = _FLATTEN.split(key) - return attribute_key_extractor(dict_keys[-1], None, data) - - -def last_rest_key_case_insensitive_extractor(attr, attr_desc, data): # pylint: disable=unused-argument - """Extract the attribute in "data" based on the last part of the JSON path key. - - This is the case insensitive version of "last_rest_key_extractor" - :param str attr: The attribute to extract - :param dict attr_desc: The attribute description - :param dict data: The data to extract from - :rtype: object - :returns: The extracted attribute - """ - key = attr_desc["key"] - dict_keys = _FLATTEN.split(key) - return attribute_key_case_insensitive_extractor(dict_keys[-1], None, data) - - -def attribute_key_extractor(attr, _, data): - return data.get(attr) - - -def attribute_key_case_insensitive_extractor(attr, _, data): - found_key = None - lower_attr = attr.lower() - for key in data: - if lower_attr == key.lower(): - found_key = key - break - - return data.get(found_key) - - -def _extract_name_from_internal_type(internal_type): - """Given an internal type XML description, extract correct XML name with namespace. - - :param dict internal_type: An model type - :rtype: tuple - :returns: A tuple XML name + namespace dict - """ - internal_type_xml_map = getattr(internal_type, "_xml_map", {}) - xml_name = internal_type_xml_map.get("name", internal_type.__name__) - xml_ns = internal_type_xml_map.get("ns", None) - if xml_ns: - xml_name = "{{{}}}{}".format(xml_ns, xml_name) - return xml_name - - -def xml_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argument,too-many-return-statements - if isinstance(data, dict): - return None - - # Test if this model is XML ready first - if not isinstance(data, ET.Element): - return None - - xml_desc = attr_desc.get("xml", {}) - xml_name = xml_desc.get("name", attr_desc["key"]) - - # Look for a children - is_iter_type = attr_desc["type"].startswith("[") - is_wrapped = xml_desc.get("wrapped", False) - internal_type = attr_desc.get("internalType", None) - internal_type_xml_map = getattr(internal_type, "_xml_map", {}) - - # Integrate namespace if necessary - xml_ns = xml_desc.get("ns", internal_type_xml_map.get("ns", None)) - if xml_ns: - xml_name = "{{{}}}{}".format(xml_ns, xml_name) - - # If it's an attribute, that's simple - if xml_desc.get("attr", False): - return data.get(xml_name) - - # If it's x-ms-text, that's simple too - if xml_desc.get("text", False): - return data.text - - # Scenario where I take the local name: - # - Wrapped node - # - Internal type is an enum (considered basic types) - # - Internal type has no XML/Name node - if is_wrapped or (internal_type and (issubclass(internal_type, Enum) or "name" not in internal_type_xml_map)): - children = data.findall(xml_name) - # If internal type has a local name and it's not a list, I use that name - elif not is_iter_type and internal_type and "name" in internal_type_xml_map: - xml_name = _extract_name_from_internal_type(internal_type) - children = data.findall(xml_name) - # That's an array - else: - if internal_type: # Complex type, ignore itemsName and use the complex type name - items_name = _extract_name_from_internal_type(internal_type) - else: - items_name = xml_desc.get("itemsName", xml_name) - children = data.findall(items_name) - - if len(children) == 0: - if is_iter_type: - if is_wrapped: - return None # is_wrapped no node, we want None - return [] # not wrapped, assume empty list - return None # Assume it's not there, maybe an optional node. - - # If is_iter_type and not wrapped, return all found children - if is_iter_type: - if not is_wrapped: - return children - # Iter and wrapped, should have found one node only (the wrap one) - if len(children) != 1: - raise DeserializationError( - "Tried to deserialize an array not wrapped, and found several nodes '{}'. Maybe you should declare this array as wrapped?".format( - xml_name - ) - ) - return list(children[0]) # Might be empty list and that's ok. - - # Here it's not a itertype, we should have found one element only or empty - if len(children) > 1: - raise DeserializationError("Find several XML '{}' where it was not expected".format(xml_name)) - return children[0] - - -class Deserializer: - """Response object model deserializer. - - :param dict classes: Class type dictionary for deserializing complex types. - :ivar list key_extractors: Ordered list of extractors to be used by this deserializer. - """ - - basic_types = {str: "str", int: "int", bool: "bool", float: "float"} - - valid_date = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") - - def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: - self.deserialize_type = { - "iso-8601": Deserializer.deserialize_iso, - "rfc-1123": Deserializer.deserialize_rfc, - "unix-time": Deserializer.deserialize_unix, - "duration": Deserializer.deserialize_duration, - "date": Deserializer.deserialize_date, - "time": Deserializer.deserialize_time, - "decimal": Deserializer.deserialize_decimal, - "long": Deserializer.deserialize_long, - "bytearray": Deserializer.deserialize_bytearray, - "base64": Deserializer.deserialize_base64, - "object": self.deserialize_object, - "[]": self.deserialize_iter, - "{}": self.deserialize_dict, - } - self.deserialize_expected_types = { - "duration": (isodate.Duration, datetime.timedelta), - "iso-8601": (datetime.datetime), - } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} - self.key_extractors = [rest_key_extractor, xml_key_extractor] - # Additional properties only works if the "rest_key_extractor" is used to - # extract the keys. Making it to work whatever the key extractor is too much - # complicated, with no real scenario for now. - # So adding a flag to disable additional properties detection. This flag should be - # used if your expect the deserialization to NOT come from a JSON REST syntax. - # Otherwise, result are unexpected - self.additional_properties_detection = True - - def __call__(self, target_obj, response_data, content_type=None): - """Call the deserializer to process a REST response. - - :param str target_obj: Target data type to deserialize to. - :param requests.Response response_data: REST response object. - :param str content_type: Swagger "produces" if available. - :raises DeserializationError: if deserialization fails. - :return: Deserialized object. - :rtype: object - """ - data = self._unpack_content(response_data, content_type) - return self._deserialize(target_obj, data) - - def _deserialize(self, target_obj, data): # pylint: disable=inconsistent-return-statements - """Call the deserializer on a model. - - Data needs to be already deserialized as JSON or XML ElementTree - - :param str target_obj: Target data type to deserialize to. - :param object data: Object to deserialize. - :raises DeserializationError: if deserialization fails. - :return: Deserialized object. - :rtype: object - """ - # This is already a model, go recursive just in case - if hasattr(data, "_attribute_map"): - constants = [name for name, config in getattr(data, "_validation", {}).items() if config.get("constant")] - try: - for attr, mapconfig in data._attribute_map.items(): # pylint: disable=protected-access - if attr in constants: - continue - value = getattr(data, attr) - if value is None: - continue - local_type = mapconfig["type"] - internal_data_type = local_type.strip("[]{}") - if internal_data_type not in self.dependencies or isinstance(internal_data_type, Enum): - continue - setattr(data, attr, self._deserialize(local_type, value)) - return data - except AttributeError: - return - - response, class_name = self._classify_target(target_obj, data) - - if isinstance(response, str): - return self.deserialize_data(data, response) - if isinstance(response, type) and issubclass(response, Enum): - return self.deserialize_enum(data, response) - - if data is None or data is CoreNull: - return data - try: - attributes = response._attribute_map # type: ignore # pylint: disable=protected-access - d_attrs = {} - for attr, attr_desc in attributes.items(): - # Check empty string. If it's not empty, someone has a real "additionalProperties"... - if attr == "additional_properties" and attr_desc["key"] == "": - continue - raw_value = None - # Enhance attr_desc with some dynamic data - attr_desc = attr_desc.copy() # Do a copy, do not change the real one - internal_data_type = attr_desc["type"].strip("[]{}") - if internal_data_type in self.dependencies: - attr_desc["internalType"] = self.dependencies[internal_data_type] - - for key_extractor in self.key_extractors: - found_value = key_extractor(attr, attr_desc, data) - if found_value is not None: - if raw_value is not None and raw_value != found_value: - msg = ( - "Ignoring extracted value '%s' from %s for key '%s'" - " (duplicate extraction, follow extractors order)" - ) - _LOGGER.warning(msg, found_value, key_extractor, attr) - continue - raw_value = found_value - - value = self.deserialize_data(raw_value, attr_desc["type"]) - d_attrs[attr] = value - except (AttributeError, TypeError, KeyError) as err: - msg = "Unable to deserialize to object: " + class_name # type: ignore - raise DeserializationError(msg) from err - additional_properties = self._build_additional_properties(attributes, data) - return self._instantiate_model(response, d_attrs, additional_properties) - - def _build_additional_properties(self, attribute_map, data): - if not self.additional_properties_detection: - return None - if "additional_properties" in attribute_map and attribute_map.get("additional_properties", {}).get("key") != "": - # Check empty string. If it's not empty, someone has a real "additionalProperties" - return None - if isinstance(data, ET.Element): - data = {el.tag: el.text for el in data} - - known_keys = { - _decode_attribute_map_key(_FLATTEN.split(desc["key"])[0]) - for desc in attribute_map.values() - if desc["key"] != "" - } - present_keys = set(data.keys()) - missing_keys = present_keys - known_keys - return {key: data[key] for key in missing_keys} - - def _classify_target(self, target, data): - """Check to see whether the deserialization target object can - be classified into a subclass. - Once classification has been determined, initialize object. - - :param str target: The target object type to deserialize to. - :param str/dict data: The response data to deserialize. - :return: The classified target object and its class name. - :rtype: tuple - """ - if target is None: - return None, None - - if isinstance(target, str): - try: - target = self.dependencies[target] - except KeyError: - return target, target - - try: - target = target._classify(data, self.dependencies) # type: ignore # pylint: disable=protected-access - except AttributeError: - pass # Target is not a Model, no classify - return target, target.__class__.__name__ # type: ignore - - def failsafe_deserialize(self, target_obj, data, content_type=None): - """Ignores any errors encountered in deserialization, - and falls back to not deserializing the object. Recommended - for use in error deserialization, as we want to return the - HttpResponseError to users, and not have them deal with - a deserialization error. - - :param str target_obj: The target object type to deserialize to. - :param str/dict data: The response data to deserialize. - :param str content_type: Swagger "produces" if available. - :return: Deserialized object. - :rtype: object - """ - try: - return self(target_obj, data, content_type=content_type) - except: # pylint: disable=bare-except - _LOGGER.debug( - "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True - ) - return None - - @staticmethod - def _unpack_content(raw_data, content_type=None): - """Extract the correct structure for deserialization. - - If raw_data is a PipelineResponse, try to extract the result of RawDeserializer. - if we can't, raise. Your Pipeline should have a RawDeserializer. - - If not a pipeline response and raw_data is bytes or string, use content-type - to decode it. If no content-type, try JSON. - - If raw_data is something else, bypass all logic and return it directly. - - :param obj raw_data: Data to be processed. - :param str content_type: How to parse if raw_data is a string/bytes. - :raises JSONDecodeError: If JSON is requested and parsing is impossible. - :raises UnicodeDecodeError: If bytes is not UTF8 - :rtype: object - :return: Unpacked content. - """ - # Assume this is enough to detect a Pipeline Response without importing it - context = getattr(raw_data, "context", {}) - if context: - if RawDeserializer.CONTEXT_NAME in context: - return context[RawDeserializer.CONTEXT_NAME] - raise ValueError("This pipeline didn't have the RawDeserializer policy; can't deserialize") - - # Assume this is enough to recognize universal_http.ClientResponse without importing it - if hasattr(raw_data, "body"): - return RawDeserializer.deserialize_from_http_generics(raw_data.text(), raw_data.headers) - - # Assume this enough to recognize requests.Response without importing it. - if hasattr(raw_data, "_content_consumed"): - return RawDeserializer.deserialize_from_http_generics(raw_data.text, raw_data.headers) - - if isinstance(raw_data, (str, bytes)) or hasattr(raw_data, "read"): - return RawDeserializer.deserialize_from_text(raw_data, content_type) # type: ignore - return raw_data - - def _instantiate_model(self, response, attrs, additional_properties=None): - """Instantiate a response model passing in deserialized args. - - :param Response response: The response model class. - :param dict attrs: The deserialized response attributes. - :param dict additional_properties: Additional properties to be set. - :rtype: Response - :return: The instantiated response model. - """ - if callable(response): - subtype = getattr(response, "_subtype_map", {}) - try: - readonly = [ - k - for k, v in response._validation.items() # pylint: disable=protected-access # type: ignore - if v.get("readonly") - ] - const = [ - k - for k, v in response._validation.items() # pylint: disable=protected-access # type: ignore - if v.get("constant") - ] - kwargs = {k: v for k, v in attrs.items() if k not in subtype and k not in readonly + const} - response_obj = response(**kwargs) - for attr in readonly: - setattr(response_obj, attr, attrs.get(attr)) - if additional_properties: - response_obj.additional_properties = additional_properties # type: ignore - return response_obj - except TypeError as err: - msg = "Unable to deserialize {} into model {}. ".format(kwargs, response) # type: ignore - raise DeserializationError(msg + str(err)) from err - else: - try: - for attr, value in attrs.items(): - setattr(response, attr, value) - return response - except Exception as exp: - msg = "Unable to populate response model. " - msg += "Type: {}, Error: {}".format(type(response), exp) - raise DeserializationError(msg) from exp - - def deserialize_data(self, data, data_type): # pylint: disable=too-many-return-statements - """Process data for deserialization according to data type. - - :param str data: The response string to be deserialized. - :param str data_type: The type to deserialize to. - :raises DeserializationError: if deserialization fails. - :return: Deserialized object. - :rtype: object - """ - if data is None: - return data - - try: - if not data_type: - return data - if data_type in self.basic_types.values(): - return self.deserialize_basic(data, data_type) - if data_type in self.deserialize_type: - if isinstance(data, self.deserialize_expected_types.get(data_type, tuple())): - return data - - is_a_text_parsing_type = lambda x: x not in [ # pylint: disable=unnecessary-lambda-assignment - "object", - "[]", - r"{}", - ] - if isinstance(data, ET.Element) and is_a_text_parsing_type(data_type) and not data.text: - return None - data_val = self.deserialize_type[data_type](data) - return data_val - - iter_type = data_type[0] + data_type[-1] - if iter_type in self.deserialize_type: - return self.deserialize_type[iter_type](data, data_type[1:-1]) - - obj_type = self.dependencies[data_type] - if issubclass(obj_type, Enum): - if isinstance(data, ET.Element): - data = data.text - return self.deserialize_enum(data, obj_type) - - except (ValueError, TypeError, AttributeError) as err: - msg = "Unable to deserialize response data." - msg += " Data: {}, {}".format(data, data_type) - raise DeserializationError(msg) from err - return self._deserialize(obj_type, data) - - def deserialize_iter(self, attr, iter_type): - """Deserialize an iterable. - - :param list attr: Iterable to be deserialized. - :param str iter_type: The type of object in the iterable. - :return: Deserialized iterable. - :rtype: list - """ - if attr is None: - return None - if isinstance(attr, ET.Element): # If I receive an element here, get the children - attr = list(attr) - if not isinstance(attr, (list, set)): - raise DeserializationError("Cannot deserialize as [{}] an object of type {}".format(iter_type, type(attr))) - return [self.deserialize_data(a, iter_type) for a in attr] - - def deserialize_dict(self, attr, dict_type): - """Deserialize a dictionary. - - :param dict/list attr: Dictionary to be deserialized. Also accepts - a list of key, value pairs. - :param str dict_type: The object type of the items in the dictionary. - :return: Deserialized dictionary. - :rtype: dict - """ - if isinstance(attr, list): - return {x["key"]: self.deserialize_data(x["value"], dict_type) for x in attr} - - if isinstance(attr, ET.Element): - # Transform value into {"Key": "value"} - attr = {el.tag: el.text for el in attr} - return {k: self.deserialize_data(v, dict_type) for k, v in attr.items()} - - def deserialize_object(self, attr, **kwargs): # pylint: disable=too-many-return-statements - """Deserialize a generic object. - This will be handled as a dictionary. - - :param dict attr: Dictionary to be deserialized. - :return: Deserialized object. - :rtype: dict - :raises TypeError: if non-builtin datatype encountered. - """ - if attr is None: - return None - if isinstance(attr, ET.Element): - # Do no recurse on XML, just return the tree as-is - return attr - if isinstance(attr, str): - return self.deserialize_basic(attr, "str") - obj_type = type(attr) - if obj_type in self.basic_types: - return self.deserialize_basic(attr, self.basic_types[obj_type]) - if obj_type is _long_type: - return self.deserialize_long(attr) - - if obj_type == dict: - deserialized = {} - for key, value in attr.items(): - try: - deserialized[key] = self.deserialize_object(value, **kwargs) - except ValueError: - deserialized[key] = None - return deserialized - - if obj_type == list: - deserialized = [] - for obj in attr: - try: - deserialized.append(self.deserialize_object(obj, **kwargs)) - except ValueError: - pass - return deserialized - - error = "Cannot deserialize generic object with type: " - raise TypeError(error + str(obj_type)) - - def deserialize_basic(self, attr, data_type): # pylint: disable=too-many-return-statements - """Deserialize basic builtin data type from string. - Will attempt to convert to str, int, float and bool. - This function will also accept '1', '0', 'true' and 'false' as - valid bool values. - - :param str attr: response string to be deserialized. - :param str data_type: deserialization data type. - :return: Deserialized basic type. - :rtype: str, int, float or bool - :raises TypeError: if string format is not valid. - """ - # If we're here, data is supposed to be a basic type. - # If it's still an XML node, take the text - if isinstance(attr, ET.Element): - attr = attr.text - if not attr: - if data_type == "str": - # None or '', node is empty string. - return "" - # None or '', node with a strong type is None. - # Don't try to model "empty bool" or "empty int" - return None - - if data_type == "bool": - if attr in [True, False, 1, 0]: - return bool(attr) - if isinstance(attr, str): - if attr.lower() in ["true", "1"]: - return True - if attr.lower() in ["false", "0"]: - return False - raise TypeError("Invalid boolean value: {}".format(attr)) - - if data_type == "str": - return self.deserialize_unicode(attr) - return eval(data_type)(attr) # nosec # pylint: disable=eval-used - - @staticmethod - def deserialize_unicode(data): - """Preserve unicode objects in Python 2, otherwise return data - as a string. - - :param str data: response string to be deserialized. - :return: Deserialized string. - :rtype: str or unicode - """ - # We might be here because we have an enum modeled as string, - # and we try to deserialize a partial dict with enum inside - if isinstance(data, Enum): - return data - - # Consider this is real string - try: - if isinstance(data, unicode): # type: ignore - return data - except NameError: - return str(data) - return str(data) - - @staticmethod - def deserialize_enum(data, enum_obj): - """Deserialize string into enum object. - - If the string is not a valid enum value it will be returned as-is - and a warning will be logged. - - :param str data: Response string to be deserialized. If this value is - None or invalid it will be returned as-is. - :param Enum enum_obj: Enum object to deserialize to. - :return: Deserialized enum object. - :rtype: Enum - """ - if isinstance(data, enum_obj) or data is None: - return data - if isinstance(data, Enum): - data = data.value - if isinstance(data, int): - # Workaround. We might consider remove it in the future. - try: - return list(enum_obj.__members__.values())[data] - except IndexError as exc: - error = "{!r} is not a valid index for enum {!r}" - raise DeserializationError(error.format(data, enum_obj)) from exc - try: - return enum_obj(str(data)) - except ValueError: - for enum_value in enum_obj: - if enum_value.value.lower() == str(data).lower(): - return enum_value - # We don't fail anymore for unknown value, we deserialize as a string - _LOGGER.warning("Deserializer is not able to find %s as valid enum in %s", data, enum_obj) - return Deserializer.deserialize_unicode(data) - - @staticmethod - def deserialize_bytearray(attr): - """Deserialize string into bytearray. - - :param str attr: response string to be deserialized. - :return: Deserialized bytearray - :rtype: bytearray - :raises TypeError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - return bytearray(b64decode(attr)) # type: ignore - - @staticmethod - def deserialize_base64(attr): - """Deserialize base64 encoded string into string. - - :param str attr: response string to be deserialized. - :return: Deserialized base64 string - :rtype: bytearray - :raises TypeError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore - attr = attr + padding # type: ignore - encoded = attr.replace("-", "+").replace("_", "/") - return b64decode(encoded) - - @staticmethod - def deserialize_decimal(attr): - """Deserialize string into Decimal object. - - :param str attr: response string to be deserialized. - :return: Deserialized decimal - :raises DeserializationError: if string format invalid. - :rtype: decimal - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - return decimal.Decimal(str(attr)) # type: ignore - except decimal.DecimalException as err: - msg = "Invalid decimal {}".format(attr) - raise DeserializationError(msg) from err - - @staticmethod - def deserialize_long(attr): - """Deserialize string into long (Py2) or int (Py3). - - :param str attr: response string to be deserialized. - :return: Deserialized int - :rtype: long or int - :raises ValueError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - return _long_type(attr) # type: ignore - - @staticmethod - def deserialize_duration(attr): - """Deserialize ISO-8601 formatted string into TimeDelta object. - - :param str attr: response string to be deserialized. - :return: Deserialized duration - :rtype: TimeDelta - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - duration = isodate.parse_duration(attr) - except (ValueError, OverflowError, AttributeError) as err: - msg = "Cannot deserialize duration object." - raise DeserializationError(msg) from err - return duration - - @staticmethod - def deserialize_date(attr): - """Deserialize ISO-8601 formatted string into Date object. - - :param str attr: response string to be deserialized. - :return: Deserialized date - :rtype: Date - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore - raise DeserializationError("Date must have only digits and -. Received: %s" % attr) - # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. - return isodate.parse_date(attr, defaultmonth=0, defaultday=0) - - @staticmethod - def deserialize_time(attr): - """Deserialize ISO-8601 formatted string into time object. - - :param str attr: response string to be deserialized. - :return: Deserialized time - :rtype: datetime.time - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore - raise DeserializationError("Date must have only digits and -. Received: %s" % attr) - return isodate.parse_time(attr) - - @staticmethod - def deserialize_rfc(attr): - """Deserialize RFC-1123 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :return: Deserialized RFC datetime - :rtype: Datetime - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - parsed_date = email.utils.parsedate_tz(attr) # type: ignore - date_obj = datetime.datetime( - *parsed_date[:6], tzinfo=datetime.timezone(datetime.timedelta(minutes=(parsed_date[9] or 0) / 60)) - ) - if not date_obj.tzinfo: - date_obj = date_obj.astimezone(tz=TZ_UTC) - except ValueError as err: - msg = "Cannot deserialize to rfc datetime object." - raise DeserializationError(msg) from err - return date_obj - - @staticmethod - def deserialize_iso(attr): - """Deserialize ISO-8601 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :return: Deserialized ISO datetime - :rtype: Datetime - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - attr = attr.upper() # type: ignore - match = Deserializer.valid_date.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - check_decimal = attr.split(".") - if len(check_decimal) > 1: - decimal_str = "" - for digit in check_decimal[1]: - if digit.isdigit(): - decimal_str += digit - else: - break - if len(decimal_str) > 6: - attr = attr.replace(decimal_str, decimal_str[0:6]) - - date_obj = isodate.parse_datetime(attr) - test_utc = date_obj.utctimetuple() - if test_utc.tm_year > 9999 or test_utc.tm_year < 1: - raise OverflowError("Hit max or min date") - except (ValueError, OverflowError, AttributeError) as err: - msg = "Cannot deserialize datetime object." - raise DeserializationError(msg) from err - return date_obj - - @staticmethod - def deserialize_unix(attr): - """Serialize Datetime object into IntTime format. - This is represented as seconds. - - :param int attr: Object to be serialized. - :return: Deserialized datetime - :rtype: Datetime - :raises DeserializationError: if format invalid - """ - if isinstance(attr, ET.Element): - attr = int(attr.text) # type: ignore - try: - attr = int(attr) - date_obj = datetime.datetime.fromtimestamp(attr, TZ_UTC) - except ValueError as err: - msg = "Cannot deserialize to unix datetime object." - raise DeserializationError(msg) from err - return date_obj diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py index aaa6692b2346..35d5fc024978 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py @@ -23,19 +23,26 @@ from json import JSONEncoder import xml.etree.ElementTree as ET from collections.abc import MutableMapping -from typing_extensions import Self import isodate from azure.core.exceptions import DeserializationError from azure.core import CaseInsensitiveEnumMeta from azure.core.pipeline import PipelineResponse from azure.core.serialization import _Null +from azure.core.rest import HttpResponse + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + _LOGGER = logging.getLogger(__name__) __all__ = ["SdkJSONEncoder", "Model", "rest_field", "rest_discriminator"] TZ_UTC = timezone.utc _T = typing.TypeVar("_T") +_NONE_TYPE = type(None) def _timedelta_as_isostr(td: timedelta) -> str: @@ -102,6 +109,29 @@ def _serialize_bytes(o, format: typing.Optional[str] = None) -> str: return encoded +def _serialize_duration(td: timedelta, format: typing.Optional[str] = None): + """Serialize a timedelta to its wire representation. + + For the ``seconds``/``milliseconds`` encodings the value is converted to a + numeric value, otherwise it falls back to an ISO 8601 duration string. + + :param timedelta td: The timedelta to serialize. + :param str format: The duration encoding format. + :rtype: int or float or str + :return: serialized duration + """ + seconds = td.total_seconds() + if format == "duration-seconds-int": + return int(seconds) + if format == "duration-seconds-float": + return seconds + if format == "duration-milliseconds-int": + return int(seconds * 1000) + if format == "duration-milliseconds-float": + return seconds * 1000 + return _timedelta_as_isostr(td) + + def _serialize_datetime(o, format: typing.Optional[str] = None): if hasattr(o, "year") and hasattr(o, "hour"): if format == "rfc7231": @@ -128,7 +158,15 @@ def _is_readonly(p): class SdkJSONEncoder(JSONEncoder): - """A JSON encoder that's capable of serializing datetime objects and bytes.""" + """A JSON encoder that's capable of serializing datetime objects and bytes. + + :param args: Additional positional arguments passed to the base ``JSONEncoder``. + :type args: typing.Any + :keyword exclude_readonly: Whether to exclude readonly properties. Defaults to False. + :paramtype exclude_readonly: bool + :keyword format: The format to use for serialization. Defaults to None. + :paramtype format: typing.Optional[str] + """ def __init__(self, *args, exclude_readonly: bool = False, format: typing.Optional[str] = None, **kwargs): super().__init__(*args, **kwargs) @@ -170,6 +208,21 @@ def default(self, o): # pylint: disable=too-many-return-statements r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT" ) +_ARRAY_ENCODE_MAPPING = { + "pipeDelimited": "|", + "spaceDelimited": " ", + "commaDelimited": ",", + "newlineDelimited": "\n", +} + + +def _deserialize_array_encoded(delimit: str, attr): + if isinstance(attr, str): + if attr == "": + return [] + return attr.split(delimit) + return attr + def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: """Deserialize ISO-8601 formatted string into Datetime object. @@ -201,7 +254,7 @@ def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: test_utc = date_obj.utctimetuple() if test_utc.tm_year > 9999 or test_utc.tm_year < 1: raise OverflowError("Hit max or min date") - return date_obj + return date_obj # type: ignore[no-any-return] def _deserialize_datetime_rfc7231(attr: typing.Union[str, datetime]) -> datetime: @@ -255,7 +308,7 @@ def _deserialize_time(attr: typing.Union[str, time]) -> time: """ if isinstance(attr, time): return attr - return isodate.parse_time(attr) + return isodate.parse_time(attr) # type: ignore[no-any-return] def _deserialize_bytes(attr): @@ -279,6 +332,12 @@ def _deserialize_duration(attr): return isodate.parse_duration(attr) +def _deserialize_duration_numeric(attr, unit): + if isinstance(attr, timedelta): + return attr + return timedelta(**{unit: float(attr)}) + + def _deserialize_decimal(attr): if isinstance(attr, decimal.Decimal): return attr @@ -291,6 +350,12 @@ def _deserialize_int_as_str(attr): return int(attr) +def _deserialize_bool_as_str(attr): + if isinstance(attr, bool): + return attr + return attr.lower() == "true" + + _DESERIALIZE_MAPPING = { datetime: _deserialize_datetime, date: _deserialize_date, @@ -308,12 +373,20 @@ def _deserialize_int_as_str(attr): "unix-timestamp": _deserialize_datetime_unix_timestamp, "base64": _deserialize_bytes, "base64url": _deserialize_bytes_base64, + "duration-seconds-int": functools.partial(_deserialize_duration_numeric, unit="seconds"), + "duration-seconds-float": functools.partial(_deserialize_duration_numeric, unit="seconds"), + "duration-milliseconds-int": functools.partial(_deserialize_duration_numeric, unit="milliseconds"), + "duration-milliseconds-float": functools.partial(_deserialize_duration_numeric, unit="milliseconds"), } def get_deserializer(annotation: typing.Any, rf: typing.Optional["_RestField"] = None): if annotation is int and rf and rf._format == "str": return _deserialize_int_as_str + if annotation is bool and rf and rf._format == "str": + return _deserialize_bool_as_str + if annotation is str and rf and rf._format in _ARRAY_ENCODE_MAPPING: + return functools.partial(_deserialize_array_encoded, _ARRAY_ENCODE_MAPPING[rf._format]) if rf and rf._format: return _DESERIALIZE_MAPPING_WITHFORMAT.get(rf._format) return _DESERIALIZE_MAPPING.get(annotation) # pyright: ignore @@ -345,16 +418,46 @@ def _get_model(module_name: str, model_name: str): class _MyMutableMapping(MutableMapping[str, typing.Any]): - def __init__(self, data: typing.Dict[str, typing.Any]) -> None: + def __init__(self, data: dict[str, typing.Any]) -> None: self._data = data def __contains__(self, key: typing.Any) -> bool: return key in self._data def __getitem__(self, key: str) -> typing.Any: + # If this key has been deserialized (for mutable types), we need to handle serialization + if hasattr(self, "_attr_to_rest_field"): + cache_attr = f"_deserialized_{key}" + if hasattr(self, cache_attr): + rf = _get_rest_field(getattr(self, "_attr_to_rest_field"), key) + if rf: + value = self._data.get(key) + if isinstance(value, (dict, list, set)): + # For mutable types, serialize and return + # But also update _data with serialized form and clear flag + # so mutations via this returned value affect _data + serialized = _serialize(value, rf._format) + # If serialized form is same type (no transformation needed), + # return _data directly so mutations work + if isinstance(serialized, type(value)) and serialized == value: + return self._data.get(key) + # Otherwise return serialized copy and clear flag + try: + object.__delattr__(self, cache_attr) + except AttributeError: + pass + # Store serialized form back + self._data[key] = serialized + return serialized return self._data.__getitem__(key) def __setitem__(self, key: str, value: typing.Any) -> None: + # Clear any cached deserialized value when setting through dictionary access + cache_attr = f"_deserialized_{key}" + try: + object.__delattr__(self, cache_attr) + except AttributeError: + pass self._data.__setitem__(key, value) def __delitem__(self, key: str) -> None: @@ -371,21 +474,21 @@ def __ne__(self, other: typing.Any) -> bool: def keys(self) -> typing.KeysView[str]: """ - :returns: a set-like object providing a view on D's keys + :returns: a set-like object providing a view on the mapping's keys :rtype: ~typing.KeysView """ return self._data.keys() def values(self) -> typing.ValuesView[typing.Any]: """ - :returns: an object providing a view on D's values + :returns: an object providing a view on the mapping's values :rtype: ~typing.ValuesView """ return self._data.values() def items(self) -> typing.ItemsView[str, typing.Any]: """ - :returns: set-like object providing a view on D's items + :returns: a set-like object providing a view on the mapping's items :rtype: ~typing.ItemsView """ return self._data.items() @@ -395,7 +498,7 @@ def get(self, key: str, default: typing.Any = None) -> typing.Any: Get the value for key if key is in the dictionary, else default. :param str key: The key to look up. :param any default: The value to return if key is not in the dictionary. Defaults to None - :returns: D[k] if k in D, else d. + :returns: The value for key if key is in the dictionary, else default. :rtype: any """ try: @@ -425,24 +528,24 @@ def pop(self, key: str, default: typing.Any = _UNSET) -> typing.Any: return self._data.pop(key) return self._data.pop(key, default) - def popitem(self) -> typing.Tuple[str, typing.Any]: + def popitem(self) -> tuple[str, typing.Any]: """ Removes and returns some (key, value) pair :returns: The (key, value) pair. :rtype: tuple - :raises KeyError: if D is empty. + :raises KeyError: if the dictionary is empty. """ return self._data.popitem() def clear(self) -> None: """ - Remove all items from D. + Remove all items from the dictionary. """ self._data.clear() def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: # pylint: disable=arguments-differ """ - Updates D from mapping/iterable E and F. + Update the dictionary from a mapping or an iterable of key-value pairs. :param any args: Either a mapping object or an iterable of key-value pairs. """ self._data.update(*args, **kwargs) @@ -455,10 +558,11 @@ def setdefault(self, key: str, default: typing.Any) -> typing.Any: ... # pylint def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: """ - Same as calling D.get(k, d), and setting D[k]=d if k not found + Return the value for key if key is in the dictionary; otherwise set the key to + default and return default. :param str key: The key to look up. :param any default: The value to set if key is not in the dictionary - :returns: D[k] if k in D, else d. + :returns: The value for key if key is in the dictionary, else default. :rtype: any """ if default is _UNSET: @@ -466,6 +570,8 @@ def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: return self._data.setdefault(key, default) def __eq__(self, other: typing.Any) -> bool: + if isinstance(other, _MyMutableMapping): + return self._data == other._data try: other_model = self.__class__(other) except Exception: @@ -482,6 +588,8 @@ def _is_model(obj: typing.Any) -> bool: def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-many-return-statements if isinstance(o, list): + if format in _ARRAY_ENCODE_MAPPING and all(isinstance(x, str) for x in o): + return _ARRAY_ENCODE_MAPPING[format].join(o) return [_serialize(x, format) for x in o] if isinstance(o, dict): return {k: _serialize(v, format) for k, v in o.items()} @@ -506,16 +614,14 @@ def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-m pass # Last, try datetime.timedelta try: - return _timedelta_as_isostr(o) + return _serialize_duration(o, format) except AttributeError: # This will be raised when it hits value.total_seconds in the method above pass return o -def _get_rest_field( - attr_to_rest_field: typing.Dict[str, "_RestField"], rest_name: str -) -> typing.Optional["_RestField"]: +def _get_rest_field(attr_to_rest_field: dict[str, "_RestField"], rest_name: str) -> typing.Optional["_RestField"]: try: return next(rf for rf in attr_to_rest_field.values() if rf._rest_name == rest_name) except StopIteration: @@ -534,69 +640,253 @@ def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typin return _serialize(value, rf._format) +# ============================================================================ +# Fast-path scalar deserializer functions for rest_field(deserializer=...) +# These are referenced from rest_field declarations to bypass the generic +# _deserialize -> _deserialize_with_callable chain. +# Only simple/primitive types — no models or container types. +# ============================================================================ + + +def _xml_deser_str(value): + if isinstance(value, ET.Element): + return value.text or "" + return str(value) if value is not None else None + + +def _xml_deser_int(value): + if isinstance(value, ET.Element): + return int(value.text) if value.text else None + return int(value) if value is not None else None + + +def _xml_deser_float(value): + if isinstance(value, ET.Element): + return float(value.text) if value.text else None + return float(value) if value is not None else None + + +def _xml_deser_bool(value): + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + if text in (True, False): + return text + return text.lower() == "true" + + +# pylint: disable=docstring-missing-param +def _xml_deser_bytes(value): + """Deserialize bytes from XML (base64).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_bytes(text) + + +def _xml_deser_bytes_base64url(value): + """Deserialize bytes from XML (base64url).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_bytes_base64(text) + + +def _xml_deser_datetime(value): + """Deserialize a datetime from XML (ISO 8601 / rfc3339).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_datetime(text) + + +def _xml_deser_datetime_rfc7231(value): + """Deserialize a datetime from XML (RFC7231 format).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_datetime_rfc7231(text) + + +def _xml_deser_datetime_unix_timestamp(value): + """Deserialize a datetime from XML (Unix timestamp).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_datetime_unix_timestamp(float(text)) + + +def _xml_deser_date(value): + """Deserialize a date from XML (ISO 8601).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_date(text) + + +def _xml_deser_time(value): + """Deserialize a time from XML (ISO 8601).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_time(text) + + +def _xml_deser_duration(value): + """Deserialize a timedelta from XML (ISO 8601 duration).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_duration(text) + + +def _xml_deser_decimal(value): + """Deserialize a Decimal from XML.""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_decimal(text) + + +def _xml_deser_enum_or_str(enum_cls, value): + """Deserialize a Union[EnumType, str] from XML.""" + text = value.text if isinstance(value, ET.Element) else value + if text is None: + return None + try: + return enum_cls(text) + except ValueError: + return text + + +def _extract_xml_model_type(rf_type): + """Extract the concrete Model class from a resolved rf._type partial chain. + + Unwraps ``Optional[Model]`` and ``_deserialize_model(Model, ...)`` + wrappers. Only handles Model and Optional[Model] — other composite + types (List, Dict, Union, etc.) return None and fall through to the + generic ``_deserialize`` path at runtime. + """ + if rf_type is None: + return None + if isinstance(rf_type, type) and _is_model(rf_type): + return rf_type + if not isinstance(rf_type, functools.partial): + return None + func = rf_type.func + args = rf_type.args + if func is _deserialize_with_optional and args: + return _extract_xml_model_type(args[0]) + if func is _deserialize_model and args: + cls = args[0] + return cls if isinstance(cls, type) and _is_model(cls) else None + return None + + +def _build_xml_field_plan( # pylint: disable=docstring-missing-return, docstring-missing-rtype, unused-variable + cls, attr_to_rest_field: dict +) -> list: + """Build a precomputed XML field plan for fast _init_from_xml iteration. + + Called once per model class in __new__. Returns a list of tuples: + (rest_name, xml_name, kind, deser, rf_type, is_optional, items_name) + + kind: 0=wrapped, 1=attribute, 2=unwrapped, 3=text + + For Model and Optional[Model] fields that lack a scalar + ``_deserializer``, this function precomputes the Model class as the + deserializer so ``_init_from_xml`` can call ``ModelClass(element)`` + directly instead of going through the expensive + ``_get_deserialize_callable_from_annotation`` chain at runtime. + """ + model_meta = getattr(cls, "_xml", {}) + model_ns = model_meta.get("ns") or model_meta.get("namespace") + plan = [] + + for rf in attr_to_rest_field.values(): + prop_meta = getattr(rf, "_xml", {}) + deser = rf._deserializer + + xml_name = prop_meta.get("name", rf._rest_name) + xml_ns = _resolve_xml_ns(prop_meta, model_meta) + if xml_ns: + xml_name = "{" + xml_ns + "}" + xml_name + + is_optional = rf._is_optional + + # For Model / Optional[Model] fields without a scalar deserializer, + # precompute the Model class as the deserializer. + if deser is None and rf._type is not None: + model_cls = _extract_xml_model_type(rf._type) + if model_cls is not None: + deser = model_cls + + if prop_meta.get("attribute", False): + plan.append((rf._rest_name, xml_name, 1, deser, rf._type, is_optional, None)) + elif prop_meta.get("unwrapped", False): + items_name = prop_meta.get("itemsName") + if items_name: + items_ns = prop_meta.get("itemsNs") + if items_ns is not None: + xml_ns = items_ns + if xml_ns: + items_name = "{" + xml_ns + "}" + items_name + else: + items_name = xml_name + plan.append((rf._rest_name, xml_name, 2, deser, rf._type, is_optional, items_name)) + elif prop_meta.get("text", False): + plan.append((rf._rest_name, xml_name, 3, deser, rf._type, is_optional, None)) + else: + plan.append((rf._rest_name, xml_name, 0, deser, rf._type, is_optional, None)) + + return plan + + +# pylint: enable=docstring-missing-param class Model(_MyMutableMapping): _is_model = True # label whether current class's _attr_to_rest_field has been calculated # could not see _attr_to_rest_field directly because subclass inherits it from parent class - _calculated: typing.Set[str] = set() + _calculated: set[str] = set() def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: class_name = self.__class__.__name__ if len(args) > 1: raise TypeError(f"{class_name}.__init__() takes 2 positional arguments but {len(args) + 1} were given") - dict_to_pass = { - rest_field._rest_name: rest_field._default - for rest_field in self._attr_to_rest_field.values() - if rest_field._default is not _UNSET - } - if args: # pylint: disable=too-many-nested-blocks + dict_to_pass: dict[str, typing.Any] = {} + if args: if isinstance(args[0], ET.Element): - existed_attr_keys = [] - model_meta = getattr(self, "_xml", {}) - - for rf in self._attr_to_rest_field.values(): - prop_meta = getattr(rf, "_xml", {}) - xml_name = prop_meta.get("name", rf._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - - # attribute - if prop_meta.get("attribute", False) and args[0].get(xml_name) is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].get(xml_name)) - continue - - # unwrapped element is array - if prop_meta.get("unwrapped", False): - # unwrapped array could either use prop items meta/prop meta - if prop_meta.get("itemsName"): - xml_name = prop_meta.get("itemsName") - xml_ns = prop_meta.get("itemNs") - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - items = args[0].findall(xml_name) # pyright: ignore - if len(items) > 0: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, items) - continue - - # text element is primitive type - if prop_meta.get("text", False): - if args[0].text is not None: - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].text) - continue - - # wrapped element could be normal property or array, it should only have one element - item = args[0].find(xml_name) - if item is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, item) - - # rest thing is additional properties - for e in args[0]: - if e.tag not in existed_attr_keys: - dict_to_pass[e.tag] = _convert_element(e) + dict_to_pass.update(self._init_from_xml(args[0])) else: dict_to_pass.update( {k: _create_value(_get_rest_field(self._attr_to_rest_field, k), v) for k, v in args[0].items()} @@ -613,8 +903,117 @@ def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: if v is not None } ) + # Apply client default values for fields the caller didn't set so that + # defaults are part of `_data` and therefore included during serialization. + for rf in self._attr_to_rest_field.values(): + if rf._default is _UNSET: + continue + if rf._rest_name in dict_to_pass: + continue + dict_to_pass[rf._rest_name] = _create_value(rf, rf._default) super().__init__(dict_to_pass) + def _init_from_xml( # pylint: disable=too-many-branches, too-many-statements + self, element: ET.Element + ) -> dict[str, typing.Any]: + """Deserialize an XML element into a dict mapping rest field names to values. + + :param ET.Element element: The XML element to deserialize from. + :returns: A dictionary of rest_name to deserialized value pairs. + :rtype: dict + """ + result: dict[str, typing.Any] = {} + existed_attr_keys: list[str] = [] + + field_plan = getattr(self, "_xml_field_plan", None) + if field_plan: + for rest_name, xml_name, kind, deser, rf_type, is_optional, items_name in field_plan: + if kind == 0: # wrapped element (most common) + item = element.find(xml_name) + if item is not None: + existed_attr_keys.append(xml_name) + if deser: + result[rest_name] = deser(item) + else: + result[rest_name] = _deserialize(rf_type, item) + elif kind == 1: # attribute + attr_val = element.get(xml_name) + if attr_val is not None: + existed_attr_keys.append(xml_name) + if deser: + result[rest_name] = deser(attr_val) + else: + result[rest_name] = attr_val + elif kind == 2: # unwrapped array + items = element.findall(items_name) # pyright: ignore + if len(items) > 0: + existed_attr_keys.append(items_name) + if deser: + result[rest_name] = deser(items) + else: + result[rest_name] = _deserialize(rf_type, items) + elif not is_optional: + existed_attr_keys.append(items_name) + result[rest_name] = [] + elif kind == 3: # text + if element.text is not None: + if deser: + result[rest_name] = deser(element.text) + else: + result[rest_name] = element.text + else: + model_meta = getattr(self, "_xml", {}) + for rf in self._attr_to_rest_field.values(): + prop_meta = getattr(rf, "_xml", {}) + xml_name = prop_meta.get("name", rf._rest_name) + xml_ns = _resolve_xml_ns(prop_meta, model_meta) + if xml_ns: + xml_name = "{" + xml_ns + "}" + xml_name + + # attribute + if prop_meta.get("attribute", False) and element.get(xml_name) is not None: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = _deserialize(rf._type, element.get(xml_name)) + continue + + # unwrapped element is array + if prop_meta.get("unwrapped", False): + _items_name = prop_meta.get("itemsName") + if _items_name: + xml_name = _items_name + _items_ns = prop_meta.get("itemsNs") + if _items_ns is not None: + xml_ns = _items_ns + if xml_ns: + xml_name = "{" + xml_ns + "}" + xml_name + items = element.findall(xml_name) # pyright: ignore + if len(items) > 0: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = _deserialize(rf._type, items) + elif not rf._is_optional: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = [] + continue + + # text element is primitive type + if prop_meta.get("text", False): + if element.text is not None: + result[rf._rest_name] = _deserialize(rf._type, element.text) + continue + + # wrapped element could be normal property or array + item = element.find(xml_name) + if item is not None: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = _deserialize(rf._type, item) + + # rest thing is additional properties + for e in element: + if e.tag not in existed_attr_keys: + result[e.tag] = _convert_element(e) + + return result + def copy(self) -> "Model": return Model(self.__dict__) @@ -623,7 +1022,7 @@ def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping', # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object' mros = cls.__mro__[:-9][::-1] # ignore parents, and reverse the mro order - attr_to_rest_field: typing.Dict[str, _RestField] = { # map attribute name to rest_field property + attr_to_rest_field: dict[str, _RestField] = { # map attribute name to rest_field property k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type") } annotations = { @@ -638,7 +1037,10 @@ def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None)) if not rf._rest_name_input: rf._rest_name_input = attr - cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items()) + cls._attr_to_rest_field: dict[str, _RestField] = dict(attr_to_rest_field.items()) + # Build XML field plan for fast _init_from_xml (only for XML models) + if getattr(cls, "_xml", None): + cls._xml_field_plan = _build_xml_field_plan(cls, attr_to_rest_field) cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}") return super().__new__(cls) @@ -667,7 +1069,7 @@ def _deserialize(cls, data, exist_discriminators): model_meta = getattr(cls, "_xml", {}) prop_meta = getattr(discriminator, "_xml", {}) xml_name = prop_meta.get("name", discriminator._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) + xml_ns = _resolve_xml_ns(prop_meta, model_meta) if xml_ns: xml_name = "{" + xml_ns + "}" + xml_name @@ -680,7 +1082,7 @@ def _deserialize(cls, data, exist_discriminators): mapped_cls = cls.__mapping__.get(discriminator_value, cls) # pyright: ignore # pylint: disable=no-member return mapped_cls._deserialize(data, exist_discriminators) - def as_dict(self, *, exclude_readonly: bool = False) -> typing.Dict[str, typing.Any]: + def as_dict(self, *, exclude_readonly: bool = False) -> dict[str, typing.Any]: """Return a dict that can be turned into json using json.dump. :keyword bool exclude_readonly: Whether to remove the readonly properties. @@ -740,7 +1142,7 @@ def _deserialize_with_union(deserializers, obj): def _deserialize_dict( value_deserializer: typing.Optional[typing.Callable], module: typing.Optional[str], - obj: typing.Dict[typing.Any, typing.Any], + obj: dict[typing.Any, typing.Any], ): if obj is None: return obj @@ -750,7 +1152,7 @@ def _deserialize_dict( def _deserialize_multiple_sequence( - entry_deserializers: typing.List[typing.Optional[typing.Callable]], + entry_deserializers: list[typing.Optional[typing.Callable]], module: typing.Optional[str], obj, ): @@ -759,6 +1161,14 @@ def _deserialize_multiple_sequence( return type(obj)(_deserialize(deserializer, entry, module) for entry, deserializer in zip(obj, entry_deserializers)) +def _is_array_encoded_deserializer(deserializer: functools.partial) -> bool: + return ( + isinstance(deserializer, functools.partial) + and isinstance(deserializer.args[0], functools.partial) + and deserializer.args[0].func == _deserialize_array_encoded # pylint: disable=comparison-with-callable + ) + + def _deserialize_sequence( deserializer: typing.Optional[typing.Callable], module: typing.Optional[str], @@ -768,17 +1178,30 @@ def _deserialize_sequence( return obj if isinstance(obj, ET.Element): obj = list(obj) + + # encoded string may be deserialized to sequence + if isinstance(obj, str) and isinstance(deserializer, functools.partial): + # for list[str] + if _is_array_encoded_deserializer(deserializer): + return deserializer(obj) + + # for list[Union[...]] + if isinstance(deserializer.args[0], list): + for sub_deserializer in deserializer.args[0]: + if _is_array_encoded_deserializer(sub_deserializer): + return sub_deserializer(obj) + return type(obj)(_deserialize(deserializer, entry, module) for entry in obj) -def _sorted_annotations(types: typing.List[typing.Any]) -> typing.List[typing.Any]: +def _sorted_annotations(types: list[typing.Any]) -> list[typing.Any]: return sorted( types, key=lambda x: hasattr(x, "__name__") and x.__name__.lower() in ("str", "float", "int", "bool"), ) -def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-return-statements, too-many-branches +def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-return-statements, too-many-statements, too-many-branches annotation: typing.Any, module: typing.Optional[str], rf: typing.Optional["_RestField"] = None, @@ -818,16 +1241,18 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur # is it optional? try: - if any(a for a in annotation.__args__ if a == type(None)): # pyright: ignore + if any(a is _NONE_TYPE for a in annotation.__args__): # pyright: ignore + if rf: + rf._is_optional = True if len(annotation.__args__) <= 2: # pyright: ignore if_obj_deserializer = _get_deserialize_callable_from_annotation( - next(a for a in annotation.__args__ if a != type(None)), module, rf # pyright: ignore + next(a for a in annotation.__args__ if a is not _NONE_TYPE), module, rf # pyright: ignore ) return functools.partial(_deserialize_with_optional, if_obj_deserializer) # the type is Optional[Union[...]], we need to remove the None type from the Union annotation_copy = copy.copy(annotation) - annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a != type(None)] # pyright: ignore + annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a is not _NONE_TYPE] # pyright: ignore return _get_deserialize_callable_from_annotation(annotation_copy, module, rf) except AttributeError: pass @@ -843,7 +1268,10 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur return functools.partial(_deserialize_with_union, deserializers) try: - if annotation._name == "Dict": # pyright: ignore + annotation_name = ( + annotation.__name__ if hasattr(annotation, "__name__") else annotation._name # pyright: ignore + ) + if annotation_name.lower() == "dict": value_deserializer = _get_deserialize_callable_from_annotation( annotation.__args__[1], module, rf # pyright: ignore ) @@ -856,7 +1284,10 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur except (AttributeError, IndexError): pass try: - if annotation._name in ["List", "Set", "Tuple", "Sequence"]: # pyright: ignore + annotation_name = ( + annotation.__name__ if hasattr(annotation, "__name__") else annotation._name # pyright: ignore + ) + if annotation_name.lower() in ["list", "set", "tuple", "sequence"]: if len(annotation.__args__) > 1: # pyright: ignore entry_deserializers = [ _get_deserialize_callable_from_annotation(dt, module, rf) @@ -905,16 +1336,20 @@ def _deserialize_with_callable( return float(value.text) if value.text else None if deserializer is bool: return value.text == "true" if value.text else None + if deserializer and deserializer in _DESERIALIZE_MAPPING.values(): + return deserializer(value.text) if value.text else None + if deserializer and deserializer in _DESERIALIZE_MAPPING_WITHFORMAT.values(): + return deserializer(value.text) if value.text else None if deserializer is None: return value if deserializer in [int, float, bool]: return deserializer(value) if isinstance(deserializer, CaseInsensitiveEnumMeta): try: - return deserializer(value) + return deserializer(value.text if isinstance(value, ET.Element) else value) except ValueError: # for unknown value, return raw value - return value + return value.text if isinstance(value, ET.Element) else value if isinstance(deserializer, type) and issubclass(deserializer, Model): return deserializer._deserialize(value, []) return typing.cast(typing.Callable[[typing.Any], typing.Any], deserializer)(value) @@ -940,14 +1375,14 @@ def _deserialize( def _failsafe_deserialize( deserializer: typing.Any, - value: typing.Any, + response: HttpResponse, module: typing.Optional[str] = None, rf: typing.Optional["_RestField"] = None, format: typing.Optional[str] = None, ) -> typing.Any: try: - return _deserialize(deserializer, value, module, rf, format) - except DeserializationError: + return _deserialize(deserializer, response.json(), module, rf, format) + except Exception: # pylint: disable=broad-except _LOGGER.warning( "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True ) @@ -956,17 +1391,18 @@ def _failsafe_deserialize( def _failsafe_deserialize_xml( deserializer: typing.Any, - value: typing.Any, + response: HttpResponse, ) -> typing.Any: try: - return _deserialize_xml(deserializer, value) - except DeserializationError: + return _deserialize_xml(deserializer, response.text()) + except Exception: # pylint: disable=broad-except _LOGGER.warning( "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True ) return None +# pylint: disable=too-many-instance-attributes class _RestField: def __init__( self, @@ -974,11 +1410,12 @@ def __init__( name: typing.Optional[str] = None, type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin is_discriminator: bool = False, - visibility: typing.Optional[typing.List[str]] = None, + visibility: typing.Optional[list[str]] = None, default: typing.Any = _UNSET, format: typing.Optional[str] = None, is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, + xml: typing.Optional[dict[str, typing.Any]] = None, + deserializer: typing.Optional[typing.Callable] = None, ): self._type = type self._rest_name_input = name @@ -986,14 +1423,20 @@ def __init__( self._is_discriminator = is_discriminator self._visibility = visibility self._is_model = False + self._is_optional = False self._default = default self._format = format self._is_multipart_file_input = is_multipart_file_input self._xml = xml if xml is not None else {} + self._deserializer = deserializer @property def _class_type(self) -> typing.Any: - return getattr(self._type, "args", [None])[0] + result = getattr(self._type, "args", [None])[0] + # type may be wrapped by nested functools.partial so we need to check for that + if isinstance(result, functools.partial): + return getattr(result, "args", [None])[0] + return result @property def _rest_name(self) -> str: @@ -1004,14 +1447,44 @@ def _rest_name(self) -> str: def __get__(self, obj: Model, type=None): # pylint: disable=redefined-builtin # by this point, type and rest_name will have a value bc we default # them in __new__ of the Model class - item = obj.get(self._rest_name) + # Use _data.get() directly to avoid triggering __getitem__ which clears the cache + item = obj._data.get(self._rest_name, _UNSET) + if item is _UNSET: + # Field not set by user; return the client default if one exists, otherwise None + return self._default if self._default is not _UNSET else None if item is None: return item if self._is_model: return item - return _deserialize(self._type, _serialize(item, self._format), rf=self) + + # For mutable types, we want mutations to directly affect _data + # Check if we've already deserialized this value + cache_attr = f"_deserialized_{self._rest_name}" + if hasattr(obj, cache_attr): + # Return the value from _data directly (it's been deserialized in place) + return obj._data.get(self._rest_name) + + # Fast path: use _deserializer directly (avoids _serialize/_deserialize chain) + if self._deserializer: + deserialized = self._deserializer(item) + else: + deserialized = _deserialize(self._type, _serialize(item, self._format), rf=self) + + # For mutable types, store the deserialized value back in _data + # so mutations directly affect _data + if isinstance(deserialized, (dict, list, set)): + obj._data[self._rest_name] = deserialized + object.__setattr__(obj, cache_attr, True) # Mark as deserialized + return deserialized + + return deserialized def __set__(self, obj: Model, value) -> None: + # Clear the cached deserialized object when setting a new value + cache_attr = f"_deserialized_{self._rest_name}" + if hasattr(obj, cache_attr): + object.__delattr__(obj, cache_attr) + if value is None: # we want to wipe out entries if users set attr to None try: @@ -1036,11 +1509,12 @@ def rest_field( *, name: typing.Optional[str] = None, type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, + visibility: typing.Optional[list[str]] = None, default: typing.Any = _UNSET, format: typing.Optional[str] = None, is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, + xml: typing.Optional[dict[str, typing.Any]] = None, + deserializer: typing.Optional[typing.Callable] = None, ) -> typing.Any: return _RestField( name=name, @@ -1050,6 +1524,7 @@ def rest_field( format=format, is_multipart_file_input=is_multipart_file_input, xml=xml, + deserializer=deserializer, ) @@ -1057,8 +1532,8 @@ def rest_discriminator( *, name: typing.Optional[str] = None, type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, + visibility: typing.Optional[list[str]] = None, + xml: typing.Optional[dict[str, typing.Any]] = None, ) -> typing.Any: return _RestField(name=name, type=type, is_discriminator=True, visibility=visibility, xml=xml) @@ -1074,21 +1549,77 @@ def serialize_xml(model: Model, exclude_readonly: bool = False) -> str: return ET.tostring(_get_element(model, exclude_readonly), encoding="unicode") # type: ignore +def _get_xml_ns(meta: dict[str, typing.Any]) -> typing.Optional[str]: + """Return the XML namespace from a metadata dict, checking both 'ns' (old-style) and 'namespace' (DPG) keys. + + :param dict meta: The metadata dictionary to extract namespace from. + :returns: The namespace string if 'ns' or 'namespace' key is present, None otherwise. + :rtype: str or None + """ + ns = meta.get("ns") + if ns is None: + ns = meta.get("namespace") + return ns + + +def _resolve_xml_ns( + prop_meta: dict[str, typing.Any], model_meta: typing.Optional[dict[str, typing.Any]] = None +) -> typing.Optional[str]: + """Resolve XML namespace for a property, falling back to model namespace when appropriate. + + Checks the property metadata first; if no namespace is found and the model does not declare + an explicit prefix, falls back to the model-level namespace. + + :param dict prop_meta: The property metadata dictionary. + :param dict model_meta: The model metadata dictionary, used as fallback. + :returns: The resolved namespace string, or None. + :rtype: str or None + """ + ns = _get_xml_ns(prop_meta) + if ns is None and model_meta is not None and not model_meta.get("prefix"): + ns = _get_xml_ns(model_meta) + return ns + + +def _set_xml_attribute(element: ET.Element, name: str, value: typing.Any, prop_meta: dict[str, typing.Any]) -> None: + """Set an XML attribute on an element, handling namespace prefix registration. + + :param ET.Element element: The element to set the attribute on. + :param str name: The default attribute name (wire name). + :param any value: The attribute value. + :param dict prop_meta: The property metadata dictionary. + """ + xml_name = prop_meta.get("name", name) + _attr_ns = _get_xml_ns(prop_meta) + if _attr_ns: + _attr_prefix = prop_meta.get("prefix") + if _attr_prefix: + _safe_register_namespace(_attr_prefix, _attr_ns) + xml_name = "{" + _attr_ns + "}" + xml_name + element.set(xml_name, _get_primitive_type_value(value)) + + def _get_element( o: typing.Any, exclude_readonly: bool = False, - parent_meta: typing.Optional[typing.Dict[str, typing.Any]] = None, + parent_meta: typing.Optional[dict[str, typing.Any]] = None, wrapped_element: typing.Optional[ET.Element] = None, -) -> typing.Union[ET.Element, typing.List[ET.Element]]: +) -> typing.Union[ET.Element, list[ET.Element]]: if _is_model(o): model_meta = getattr(o, "_xml", {}) # if prop is a model, then use the prop element directly, else generate a wrapper of model if wrapped_element is None: + # When serializing as an array item (parent_meta is set), check if the parent has an + # explicit itemsName. This ensures correct element names for unwrapped arrays (where + # the element tag is the property/items name, not the model type name). + _items_name = parent_meta.get("itemsName") if parent_meta is not None else None + element_name = _items_name if _items_name else (model_meta.get("name") or o.__class__.__name__) + _model_ns = _get_xml_ns(model_meta) wrapped_element = _create_xml_element( - model_meta.get("name", o.__class__.__name__), + element_name, model_meta.get("prefix"), - model_meta.get("ns"), + _model_ns, ) readonly_props = [] @@ -1110,7 +1641,9 @@ def _get_element( # additional properties will not have rest field, use the wire name as xml name prop_meta = {"name": k} - # if no ns for prop, use model's + # Propagate model namespace to properties only for old-style "ns"-keyed models. + # DPG-generated models use the "namespace" key and explicitly declare namespace on + # each property that needs it, so propagation is intentionally skipped for them. if prop_meta.get("ns") is None and model_meta.get("ns"): prop_meta["ns"] = model_meta.get("ns") prop_meta["prefix"] = model_meta.get("prefix") @@ -1122,12 +1655,7 @@ def _get_element( # text could only set on primitive type wrapped_element.text = _get_primitive_type_value(v) elif prop_meta.get("attribute", False): - xml_name = prop_meta.get("name", k) - if prop_meta.get("ns"): - ET.register_namespace(prop_meta.get("prefix"), prop_meta.get("ns")) # pyright: ignore - xml_name = "{" + prop_meta.get("ns") + "}" + xml_name # pyright: ignore - # attribute should be primitive type - wrapped_element.set(xml_name, _get_primitive_type_value(v)) + _set_xml_attribute(wrapped_element, k, v, prop_meta) else: # other wrapped prop element wrapped_element.append(_get_wrapped_element(v, exclude_readonly, prop_meta)) @@ -1136,6 +1664,7 @@ def _get_element( return [_get_element(x, exclude_readonly, parent_meta) for x in o] # type: ignore if isinstance(o, dict): result = [] + _dict_ns = _get_xml_ns(parent_meta) if parent_meta else None for k, v in o.items(): result.append( _get_wrapped_element( @@ -1143,7 +1672,7 @@ def _get_element( exclude_readonly, { "name": k, - "ns": parent_meta.get("ns") if parent_meta else None, + "ns": _dict_ns, "prefix": parent_meta.get("prefix") if parent_meta else None, }, ) @@ -1152,13 +1681,16 @@ def _get_element( # primitive case need to create element based on parent_meta if parent_meta: + _items_ns = parent_meta.get("itemsNs") + if _items_ns is None: + _items_ns = _get_xml_ns(parent_meta) return _get_wrapped_element( o, exclude_readonly, { "name": parent_meta.get("itemsName", parent_meta.get("name")), "prefix": parent_meta.get("itemsPrefix", parent_meta.get("prefix")), - "ns": parent_meta.get("itemsNs", parent_meta.get("ns")), + "ns": _items_ns, }, ) @@ -1168,10 +1700,11 @@ def _get_element( def _get_wrapped_element( v: typing.Any, exclude_readonly: bool, - meta: typing.Optional[typing.Dict[str, typing.Any]], + meta: typing.Optional[dict[str, typing.Any]], ) -> ET.Element: + _meta_ns = _get_xml_ns(meta) if meta else None wrapped_element = _create_xml_element( - meta.get("name") if meta else None, meta.get("prefix") if meta else None, meta.get("ns") if meta else None + meta.get("name") if meta else None, meta.get("prefix") if meta else None, _meta_ns ) if isinstance(v, (dict, list)): wrapped_element.extend(_get_element(v, exclude_readonly, meta)) @@ -1179,7 +1712,7 @@ def _get_wrapped_element( _get_element(v, exclude_readonly, meta, wrapped_element) else: wrapped_element.text = _get_primitive_type_value(v) - return wrapped_element + return wrapped_element # type: ignore[no-any-return] def _get_primitive_type_value(v) -> str: @@ -1192,9 +1725,29 @@ def _get_primitive_type_value(v) -> str: return str(v) -def _create_xml_element(tag, prefix=None, ns=None): - if prefix and ns: +def _safe_register_namespace(prefix: str, ns: str) -> None: + """Register an XML namespace prefix, handling reserved prefix patterns. + + Some prefixes (e.g. 'ns2') match Python's reserved 'ns\\d+' pattern used for + auto-generated prefixes, causing register_namespace to raise ValueError. + Falls back to directly registering in the internal namespace map. + + :param str prefix: The namespace prefix to register. + :param str ns: The namespace URI. + """ + try: ET.register_namespace(prefix, ns) + except ValueError: + _ns_map = getattr(ET, "_namespace_map", None) + if _ns_map is not None: + _ns_map[ns] = prefix + + +def _create_xml_element( + tag: typing.Any, prefix: typing.Optional[str] = None, ns: typing.Optional[str] = None +) -> ET.Element: + if prefix and ns: + _safe_register_namespace(prefix, ns) if ns: return ET.Element("{" + ns + "}" + tag) return ET.Element(tag) @@ -1205,13 +1758,15 @@ def _deserialize_xml( value: str, ) -> typing.Any: element = ET.fromstring(value) # nosec + if _is_model(deserializer): + return deserializer._deserialize(element, []) return _deserialize(deserializer, element) def _convert_element(e: ET.Element): # dict case if len(e.attrib) > 0 or len({child.tag for child in e}) > 1: - dict_result: typing.Dict[str, typing.Any] = {} + dict_result: dict[str, typing.Any] = {} for child in e: if dict_result.get(child.tag) is not None: if isinstance(dict_result[child.tag], list): @@ -1224,7 +1779,7 @@ def _convert_element(e: ET.Element): return dict_result # array case if len(e) > 0: - array_result: typing.List[typing.Any] = [] + array_result: list[typing.Any] = [] for child in e: array_result.append(_convert_element(child)) return array_result diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py index eb86ea23c965..ae08f9d89f74 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py @@ -21,7 +21,6 @@ import sys import codecs from typing import ( - Dict, Any, cast, Optional, @@ -31,7 +30,6 @@ Mapping, Callable, MutableMapping, - List, ) try: @@ -41,11 +39,15 @@ import xml.etree.ElementTree as ET import isodate # type: ignore -from typing_extensions import Self from azure.core.exceptions import DeserializationError, SerializationError from azure.core.serialization import NULL as CoreNull +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + _BOM = codecs.BOM_UTF8.decode(encoding="utf-8") JSON = MutableMapping[str, Any] @@ -229,12 +231,12 @@ class Model: serialization and deserialization. """ - _subtype_map: Dict[str, Dict[str, Any]] = {} - _attribute_map: Dict[str, Dict[str, Any]] = {} - _validation: Dict[str, Dict[str, Any]] = {} + _subtype_map: dict[str, dict[str, Any]] = {} + _attribute_map: dict[str, dict[str, Any]] = {} + _validation: dict[str, dict[str, Any]] = {} def __init__(self, **kwargs: Any) -> None: - self.additional_properties: Optional[Dict[str, Any]] = {} + self.additional_properties: Optional[dict[str, Any]] = {} for k in kwargs: # pylint: disable=consider-using-dict-items if k not in self._attribute_map: _LOGGER.warning("%s is not a known attribute of class %s and will be ignored", k, self.__class__) @@ -311,7 +313,7 @@ def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> JSON: def as_dict( self, keep_readonly: bool = True, - key_transformer: Callable[[str, Dict[str, Any], Any], Any] = attribute_transformer, + key_transformer: Callable[[str, dict[str, Any], Any], Any] = attribute_transformer, **kwargs: Any ) -> JSON: """Return a dict that can be serialized using json.dump. @@ -380,7 +382,7 @@ def deserialize(cls, data: Any, content_type: Optional[str] = None) -> Self: def from_dict( cls, data: Any, - key_extractors: Optional[Callable[[str, Dict[str, Any], Any], Any]] = None, + key_extractors: Optional[Callable[[str, dict[str, Any], Any], Any]] = None, content_type: Optional[str] = None, ) -> Self: """Parse a dict using given key extractor return a model. @@ -414,7 +416,7 @@ def _flatten_subtype(cls, key, objects): return {} result = dict(cls._subtype_map[key]) for valuetype in cls._subtype_map[key].values(): - result.update(objects[valuetype]._flatten_subtype(key, objects)) # pylint: disable=protected-access + result |= objects[valuetype]._flatten_subtype(key, objects) # pylint: disable=protected-access return result @classmethod @@ -478,7 +480,11 @@ def _decode_attribute_map_key(key): class Serializer: # pylint: disable=too-many-public-methods - """Request object model serializer.""" + """Request object model serializer. + + :param classes: Mapping of model names to model types, used to resolve models during serialization. + :type classes: typing.Optional[typing.Mapping[str, type]] + """ basic_types = {str: "str", int: "int", bool: "bool", float: "float"} @@ -518,6 +524,10 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: "rfc-1123": Serializer.serialize_rfc, "unix-time": Serializer.serialize_unix, "duration": Serializer.serialize_duration, + "duration-seconds-int": Serializer.serialize_duration_seconds_int, + "duration-seconds-float": Serializer.serialize_duration_seconds_float, + "duration-milliseconds-int": Serializer.serialize_duration_milliseconds_int, + "duration-milliseconds-float": Serializer.serialize_duration_milliseconds_float, "date": Serializer.serialize_date, "time": Serializer.serialize_time, "decimal": Serializer.serialize_decimal, @@ -528,7 +538,7 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: "[]": self.serialize_iter, "{}": self.serialize_dict, } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} + self.dependencies: dict[str, type] = dict(classes) if classes else {} self.key_transformer = full_restapi_key_transformer self.client_side_validation = True @@ -579,7 +589,7 @@ def _serialize( # pylint: disable=too-many-nested-blocks, too-many-branches, to if attr_name == "additional_properties" and attr_desc["key"] == "": if target_obj.additional_properties is not None: - serialized.update(target_obj.additional_properties) + serialized |= target_obj.additional_properties continue try: @@ -789,7 +799,7 @@ def serialize_data(self, data, data_type, **kwargs): # If dependencies is empty, try with current data class # It has to be a subclass of Enum anyway - enum_type = self.dependencies.get(data_type, data.__class__) + enum_type = self.dependencies.get(data_type, cast(type, data.__class__)) if issubclass(enum_type, Enum): return Serializer.serialize_enum(data, enum_obj=enum_type) @@ -823,13 +833,20 @@ def serialize_basic(cls, data, data_type, **kwargs): :param str data_type: Type of object in the iterable. :rtype: str, int, float, bool :return: serialized object + :raises TypeError: raise if data_type is not one of str, int, float, bool. """ custom_serializer = cls._get_custom_serializers(data_type, **kwargs) if custom_serializer: return custom_serializer(data) if data_type == "str": return cls.serialize_unicode(data) - return eval(data_type)(data) # nosec # pylint: disable=eval-used + if data_type == "int": + return int(data) + if data_type == "float": + return float(data) + if data_type == "bool": + return bool(data) + raise TypeError("Unknown basic data type: {}".format(data_type)) @classmethod def serialize_unicode(cls, data): @@ -1100,6 +1117,61 @@ def serialize_duration(attr, **kwargs): # pylint: disable=unused-argument attr = isodate.parse_duration(attr) return isodate.duration_isoformat(attr) + @staticmethod + def _serialize_duration_numeric(attr, scale, as_int): + """Serialize a TimeDelta into a numeric value scaled to the wire unit. + + :param TimeDelta attr: Object to be serialized. + :param int scale: Multiplier applied to total seconds (1 for seconds, 1000 for milliseconds). + :param bool as_int: Whether to truncate the result to an int. + :rtype: int or float + :return: serialized duration + """ + if isinstance(attr, str): + attr = isodate.parse_duration(attr) + value = attr.total_seconds() * scale if isinstance(attr, datetime.timedelta) else attr + return int(value) if as_int else float(value) + + @staticmethod + def serialize_duration_seconds_int(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into an integer number of seconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: int + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1, True) + + @staticmethod + def serialize_duration_seconds_float(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into a floating point number of seconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: float + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1, False) + + @staticmethod + def serialize_duration_milliseconds_int(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into an integer number of milliseconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: int + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1000, True) + + @staticmethod + def serialize_duration_milliseconds_float(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into a floating point number of milliseconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: float + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1000, False) + @staticmethod def serialize_rfc(attr, **kwargs): # pylint: disable=unused-argument """Serialize Datetime object into RFC-1123 formatted string. @@ -1184,7 +1256,7 @@ def rest_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argumen while "." in key: # Need the cast, as for some reasons "split" is typed as list[str | Any] - dict_keys = cast(List[str], _FLATTEN.split(key)) + dict_keys = cast(list[str], _FLATTEN.split(key)) if len(dict_keys) == 1: key = _decode_attribute_map_key(dict_keys[0]) break @@ -1372,6 +1444,10 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: "rfc-1123": Deserializer.deserialize_rfc, "unix-time": Deserializer.deserialize_unix, "duration": Deserializer.deserialize_duration, + "duration-seconds-int": Deserializer.deserialize_duration_seconds, + "duration-seconds-float": Deserializer.deserialize_duration_seconds, + "duration-milliseconds-int": Deserializer.deserialize_duration_milliseconds, + "duration-milliseconds-float": Deserializer.deserialize_duration_milliseconds, "date": Deserializer.deserialize_date, "time": Deserializer.deserialize_time, "decimal": Deserializer.deserialize_decimal, @@ -1384,9 +1460,13 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: } self.deserialize_expected_types = { "duration": (isodate.Duration, datetime.timedelta), + "duration-seconds-int": (isodate.Duration, datetime.timedelta), + "duration-seconds-float": (isodate.Duration, datetime.timedelta), + "duration-milliseconds-int": (isodate.Duration, datetime.timedelta), + "duration-milliseconds-float": (isodate.Duration, datetime.timedelta), "iso-8601": (datetime.datetime), } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} + self.dependencies: dict[str, type] = dict(classes) if classes else {} self.key_extractors = [rest_key_extractor, xml_key_extractor] # Additional properties only works if the "rest_key_extractor" is used to # extract the keys. Making it to work whatever the key extractor is too much @@ -1396,7 +1476,7 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: # Otherwise, result are unexpected self.additional_properties_detection = True - def __call__(self, target_obj, response_data, content_type=None): + def __call__(self, target_obj, response_data, content_type=None): # pylint: disable=too-many-return-statements """Call the deserializer to process a REST response. :param str target_obj: Target data type to deserialize to. @@ -1406,6 +1486,27 @@ def __call__(self, target_obj, response_data, content_type=None): :return: Deserialized object. :rtype: object """ + # Fast path for header deserialization: response_data is a plain str or None + # and target_obj is a simple scalar type. This avoids the expensive + # _unpack_content → _deserialize → _classify_target → deserialize_data chain. + if response_data is None: + return None + if target_obj == "str" and isinstance(response_data, str): + return response_data + if isinstance(response_data, str): + if target_obj == "int": + return int(response_data) + if target_obj == "bool": + if response_data in ("true", "1", "True"): + return True + if response_data in ("false", "0", "False"): + return False + return bool(response_data) + if target_obj == "rfc-1123": + return Deserializer.deserialize_rfc(response_data) + if target_obj == "bytearray": + return Deserializer.deserialize_bytearray(response_data) + data = self._unpack_content(response_data, content_type) return self._deserialize(target_obj, data) @@ -1759,7 +1860,7 @@ def deserialize_basic(self, attr, data_type): # pylint: disable=too-many-return :param str data_type: deserialization data type. :return: Deserialized basic type. :rtype: str, int, float or bool - :raises TypeError: if string format is not valid. + :raises TypeError: if string format is not valid or data_type is not one of str, int, float, bool. """ # If we're here, data is supposed to be a basic type. # If it's still an XML node, take the text @@ -1785,7 +1886,11 @@ def deserialize_basic(self, attr, data_type): # pylint: disable=too-many-return if data_type == "str": return self.deserialize_unicode(attr) - return eval(data_type)(attr) # nosec # pylint: disable=eval-used + if data_type == "int": + return int(attr) + if data_type == "float": + return float(attr) + raise TypeError("Unknown basic data type: {}".format(data_type)) @staticmethod def deserialize_unicode(data): @@ -1920,6 +2025,48 @@ def deserialize_duration(attr): raise DeserializationError(msg) from err return duration + @staticmethod + def _deserialize_duration_numeric(attr, unit): + """Deserialize a numeric duration value into a TimeDelta object. + + :param float attr: response value to be deserialized. + :param str unit: The wire unit, used as the ``timedelta`` keyword + (``"seconds"`` or ``"milliseconds"``). + :return: Deserialized duration + :rtype: TimeDelta + :raises DeserializationError: if value is invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + try: + duration = datetime.timedelta(**{unit: float(attr)}) # type: ignore + except (ValueError, OverflowError, TypeError) as err: + msg = "Cannot deserialize duration object." + raise DeserializationError(msg) from err + return duration + + @staticmethod + def deserialize_duration_seconds(attr): + """Deserialize a numeric number of seconds into a TimeDelta object. + + :param float attr: response value to be deserialized. + :return: Deserialized duration + :rtype: TimeDelta + :raises DeserializationError: if value is invalid. + """ + return Deserializer._deserialize_duration_numeric(attr, "seconds") + + @staticmethod + def deserialize_duration_milliseconds(attr): + """Deserialize a numeric number of milliseconds into a TimeDelta object. + + :param float attr: response value to be deserialized. + :return: Deserialized duration + :rtype: TimeDelta + :raises DeserializationError: if value is invalid. + """ + return Deserializer._deserialize_duration_numeric(attr, "milliseconds") + @staticmethod def deserialize_date(attr): """Deserialize ISO-8601 formatted string into Date object. diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_vendor.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_vendor.py deleted file mode 100644 index 5af45cbb9df5..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_vendor.py +++ /dev/null @@ -1,25 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from abc import ABC -from typing import TYPE_CHECKING - -from ._configuration import DeidentificationClientConfiguration - -if TYPE_CHECKING: - from azure.core import PipelineClient - - from ._serialization import Deserializer, Serializer - - -class DeidentificationClientMixinABC(ABC): - """DO NOT use this class. It is for internal typing use only.""" - - _client: "PipelineClient" - _config: DeidentificationClientConfiguration - _serialize: "Serializer" - _deserialize: "Deserializer" diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py index a1f432eddc4e..f6cd39818b76 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py @@ -6,4 +6,4 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -VERSION = "1.1.0b1" +VERSION = "1.1.0b2" diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py index 7f413f77a0fb..085d9fa3c313 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py @@ -7,8 +7,8 @@ # -------------------------------------------------------------------------- from copy import deepcopy +import sys from typing import Any, Awaitable, TYPE_CHECKING -from typing_extensions import Self from azure.core import AsyncPipelineClient from azure.core.pipeline import policies @@ -18,19 +18,27 @@ from ._configuration import DeidentificationClientConfiguration from ._operations import _DeidentificationClientOperationsMixin +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self # type: ignore + if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential -class DeidentificationClient(_DeidentificationClientOperationsMixin): +class DeidentificationClient( + _DeidentificationClientOperationsMixin +): # pylint: disable=docstring-keyword-should-match-keyword-only """DeidentificationClient. :param endpoint: Url of your De-identification Service. Required. :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials_async.AsyncTokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str :keyword int polling_interval: Default waiting time between two polls for LRO operations if no diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py index f389c8624735..087ecd2812ca 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py @@ -1,3 +1,4 @@ +# pylint: disable=line-too-long,useless-suppression # coding=utf-8 # -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. @@ -16,7 +17,7 @@ from azure.core.credentials_async import AsyncTokenCredential -class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes +class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes,docstring-keyword-should-match-keyword-only """Configuration for DeidentificationClient. Note that all parameters used to create this instance are saved as instance @@ -26,8 +27,9 @@ class DeidentificationClientConfiguration: # pylint: disable=too-many-instance- :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials_async.AsyncTokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str """ diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py index fe6452408aba..d218f58752bf 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py @@ -9,7 +9,7 @@ from collections.abc import MutableMapping from io import IOBase import json -from typing import Any, AsyncIterator, Callable, Dict, IO, List, Optional, TypeVar, Union, cast, overload +from typing import Any, AsyncIterator, Callable, IO, Optional, TypeVar, Union, cast, overload import urllib.parse from azure.core import AsyncPipelineClient @@ -32,7 +32,7 @@ from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict -from ... import models as _models +from ... import models as _models, types as _types from ..._operations._operations import ( build_deidentification_cancel_job_request, build_deidentification_deidentify_documents_request, @@ -46,9 +46,8 @@ from ..._utils.utils import ClientMixinABC from .._configuration import DeidentificationClientConfiguration -JSON = MutableMapping[str, Any] T = TypeVar("T") -ClsType = Optional[Callable[[PipelineResponse[HttpRequest, AsyncHttpResponse], T, Dict[str, Any]], Any]] +ClsType = Optional[Callable[[PipelineResponse[HttpRequest, AsyncHttpResponse], T, dict[str, Any]], Any]] class _DeidentificationClientOperationsMixin( @@ -91,6 +90,7 @@ async def get_job(self, job_name: str, **kwargs: Any) -> _models.Deidentificatio } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -113,7 +113,7 @@ async def get_job(self, job_name: str, **kwargs: Any) -> _models.Deidentificatio ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -123,7 +123,10 @@ async def get_job(self, job_name: str, **kwargs: Any) -> _models.Deidentificatio return deserialized # type: ignore async def _deidentify_documents_initial( - self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any + self, + job_name: str, + resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], + **kwargs: Any ) -> AsyncIterator[bytes]: error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -159,6 +162,7 @@ async def _deidentify_documents_initial( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -180,7 +184,7 @@ async def _deidentify_documents_initial( ) response_headers["Operation-Location"] = self._deserialize("str", response.headers.get("Operation-Location")) - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -216,7 +220,12 @@ async def begin_deidentify_documents( @overload async def begin_deidentify_documents( - self, job_name: str, resource: JSON, *, content_type: str = "application/json", **kwargs: Any + self, + job_name: str, + resource: _types.DeidentificationJob, + *, + content_type: str = "application/json", + **kwargs: Any ) -> AsyncLROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -225,7 +234,7 @@ async def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str :param resource: The resource instance. Required. - :type resource: JSON + :type resource: ~azure.health.deidentification.types.DeidentificationJob :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -260,7 +269,10 @@ async def begin_deidentify_documents( @distributed_trace_async async def begin_deidentify_documents( - self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any + self, + job_name: str, + resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], + **kwargs: Any ) -> AsyncLROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -268,9 +280,10 @@ async def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str - :param resource: The resource instance. Is one of the following types: DeidentificationJob, - JSON, IO[bytes] Required. - :type resource: ~azure.health.deidentification.models.DeidentificationJob or JSON or IO[bytes] + :param resource: The resource instance. Is either a DeidentificationJob type or a IO[bytes] + type. Required. + :type resource: ~azure.health.deidentification.models.DeidentificationJob or + ~azure.health.deidentification.types.DeidentificationJob or IO[bytes] :return: An instance of AsyncLROPoller that returns DeidentificationJob. The DeidentificationJob is compatible with MutableMapping :rtype: @@ -357,7 +370,7 @@ def _list_jobs_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationJob]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationJob]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -395,7 +408,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -408,7 +424,10 @@ def prepare_request(next_link=None): async def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationJob], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationJob], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, AsyncList(list_of_elem) @@ -452,7 +471,7 @@ def _list_job_documents_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -491,7 +510,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -504,7 +526,10 @@ def prepare_request(next_link=None): async def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationDocumentDetails], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationDocumentDetails], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, AsyncList(list_of_elem) @@ -567,6 +592,7 @@ async def cancel_job(self, job_name: str, **kwargs: Any) -> _models.Deidentifica } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -589,7 +615,7 @@ async def cancel_job(self, job_name: str, **kwargs: Any) -> _models.Deidentifica ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -673,14 +699,14 @@ async def deidentify_text( @overload async def deidentify_text( - self, body: JSON, *, content_type: str = "application/json", **kwargs: Any + self, body: _types.DeidentificationContent, *, content_type: str = "application/json", **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. :param body: Request body for de-identification operation. Required. - :type body: JSON + :type body: ~azure.health.deidentification.types.DeidentificationContent :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -709,15 +735,16 @@ async def deidentify_text( @distributed_trace_async async def deidentify_text( - self, body: Union[_models.DeidentificationContent, JSON, IO[bytes]], **kwargs: Any + self, body: Union[_models.DeidentificationContent, _types.DeidentificationContent, IO[bytes]], **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. - :param body: Request body for de-identification operation. Is one of the following types: - DeidentificationContent, JSON, IO[bytes] Required. - :type body: ~azure.health.deidentification.models.DeidentificationContent or JSON or IO[bytes] + :param body: Request body for de-identification operation. Is either a DeidentificationContent + type or a IO[bytes] type. Required. + :type body: ~azure.health.deidentification.models.DeidentificationContent or + ~azure.health.deidentification.types.DeidentificationContent or IO[bytes] :return: DeidentificationResult. The DeidentificationResult is compatible with MutableMapping :rtype: ~azure.health.deidentification.models.DeidentificationResult :raises ~azure.core.exceptions.HttpResponseError: @@ -755,6 +782,7 @@ async def deidentify_text( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -777,7 +805,7 @@ async def deidentify_text( ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationResult, response.json()) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_vendor.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_vendor.py deleted file mode 100644 index 0afc83d417cc..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_vendor.py +++ /dev/null @@ -1,25 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from abc import ABC -from typing import TYPE_CHECKING - -from ._configuration import DeidentificationClientConfiguration - -if TYPE_CHECKING: - from azure.core import AsyncPipelineClient - - from .._serialization import Deserializer, Serializer - - -class DeidentificationClientMixinABC(ABC): - """DO NOT use this class. It is for internal typing use only.""" - - _client: "AsyncPipelineClient" - _config: DeidentificationClientConfiguration - _serialize: "Serializer" - _deserialize: "Deserializer" diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py index 4cbd2dc93f44..633ff8e199e5 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py @@ -1,4 +1,3 @@ -# pylint: disable=line-too-long,useless-suppression # coding=utf-8 # -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. @@ -9,7 +8,7 @@ # pylint: disable=useless-super-delegation import datetime -from typing import Any, List, Mapping, Optional, TYPE_CHECKING, Union, overload +from typing import Any, Mapping, Optional, TYPE_CHECKING, Union, overload from azure.core.exceptions import ODataV4Format @@ -19,7 +18,7 @@ from .. import models as _models -class DeidentificationContent(_Model): +class DeidentificationContent(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Request body for de-identification operation. :ivar input_text: Input text to de-identify. Required. @@ -73,7 +72,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationCustomizationOptions(_Model): +class DeidentificationCustomizationOptions(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Customizations options to override default service behaviors for synchronous usage. :ivar redaction_format: Format of the redacted output. Only valid when Operation is Redact. @@ -92,8 +91,7 @@ class DeidentificationCustomizationOptions(_Model): redaction_format: Optional[str] = rest_field( name="redactionFormat", visibility=["read", "create", "update", "delete", "query"] ) - """Format of the redacted output. Only valid when Operation is Redact. - Please refer to + """Format of the redacted output. Only valid when Operation is Redact. Please refer to `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format `_ for more details.""" @@ -126,7 +124,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationDocumentDetails(_Model): +class DeidentificationDocumentDetails(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Details of a single document in a job. :ivar id: Id of the document details. Required. @@ -182,7 +180,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationDocumentLocation(_Model): +class DeidentificationDocumentLocation(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Location of a document. :ivar location: Location of document in storage. Required. @@ -214,7 +212,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationJob(_Model): +class DeidentificationJob(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """A job containing a batch of documents to de-identify. :ivar job_name: The name of a job. Required. @@ -307,7 +305,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationJobCustomizationOptions(_Model): +class DeidentificationJobCustomizationOptions(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Customizations options to override default service behaviors for job usage. :ivar redaction_format: Format of the redacted output. Only valid when Operation is Redact. @@ -326,8 +324,7 @@ class DeidentificationJobCustomizationOptions(_Model): redaction_format: Optional[str] = rest_field( name="redactionFormat", visibility=["read", "create", "update", "delete", "query"] ) - """Format of the redacted output. Only valid when Operation is Redact. - Please refer to + """Format of the redacted output. Only valid when Operation is Redact. Please refer to `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format `_ for more details.""" @@ -360,7 +357,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationJobSummary(_Model): +class DeidentificationJobSummary(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Summary metrics of a job. :ivar successful_count: Number of documents that have completed. Required. @@ -408,7 +405,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationResult(_Model): +class DeidentificationResult(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Response body for de-identification operation. :ivar output_text: Output text after de-identification. Not available for "Tag" operation. @@ -445,7 +442,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class PhiEntity(_Model): +class PhiEntity(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """PHI Entity tag in the input. :ivar category: PHI Category of the entity. Required. Known values are: "Unknown", "Account", @@ -503,21 +500,21 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class PhiTaggerResult(_Model): +class PhiTaggerResult(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Result of the "Tag" operation. :ivar entities: List of entities detected in the input. Required. :vartype entities: list[~azure.health.deidentification.models.PhiEntity] """ - entities: List["_models.PhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + entities: list["_models.PhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) """List of entities detected in the input. Required.""" @overload def __init__( self, *, - entities: List["_models.PhiEntity"], + entities: list["_models.PhiEntity"], ) -> None: ... @overload @@ -531,7 +528,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class SimplePhiEntity(_Model): +class SimplePhiEntity(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Simple PHI entity with encoding-specific offset and length values. :ivar category: PHI Category of the entity. Required. Known values are: "Unknown", "Account", @@ -583,7 +580,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class SourceStorageLocation(_Model): +class SourceStorageLocation(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Storage location. :ivar location: URL to storage location. Required. @@ -598,7 +595,7 @@ class SourceStorageLocation(_Model): """URL to storage location. Required.""" prefix: str = rest_field(visibility=["read", "create", "update", "delete", "query"]) """Prefix to filter path by. Required.""" - extensions: Optional[List[str]] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + extensions: Optional[list[str]] = rest_field(visibility=["read", "create", "update", "delete", "query"]) """List of extensions to filter path by.""" @overload @@ -607,7 +604,7 @@ def __init__( *, location: str, prefix: str, - extensions: Optional[List[str]] = None, + extensions: Optional[list[str]] = None, ) -> None: ... @overload @@ -621,7 +618,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class StringIndex(_Model): +class StringIndex(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """String index encoding model. :ivar utf8: The offset or length of the substring in UTF-8 encoding. Required. @@ -667,7 +664,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class TaggedPhiEntities(_Model): +class TaggedPhiEntities(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Grouped PHI entities with shared encoding specification. :ivar encoding: The encoding type used for all entities in this group. Required. Known values @@ -682,7 +679,7 @@ class TaggedPhiEntities(_Model): ) """The encoding type used for all entities in this group. Required. Known values are: \"Utf8\", \"Utf16\", and \"CodePoint\".""" - entities: List["_models.SimplePhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + entities: list["_models.SimplePhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) """List of PHI entities using the specified encoding. Required.""" @overload @@ -690,7 +687,7 @@ def __init__( self, *, encoding: Union[str, "_models.TextEncodingType"], - entities: List["_models.SimplePhiEntity"], + entities: list["_models.SimplePhiEntity"], ) -> None: ... @overload @@ -704,7 +701,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class TargetStorageLocation(_Model): +class TargetStorageLocation(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Storage location. :ivar location: URL to storage location. Required. diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py new file mode 100644 index 000000000000..6e1feec90b13 --- /dev/null +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py @@ -0,0 +1,290 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# Code generated by Microsoft (R) Python Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is regenerated. +# -------------------------------------------------------------------------- + +from typing import TYPE_CHECKING, Union +from typing_extensions import Required, TypedDict + +from azure.core.exceptions import ODataV4Format + +if TYPE_CHECKING: + from .models import DeidentificationOperationType, OperationStatus, PhiCategory, TextEncodingType + + +class DeidentificationContent(TypedDict, total=False): + """Request body for de-identification operation. + + :ivar inputText: Input text to de-identify. Required. + :vartype inputText: str + :ivar operation: Operation to perform on the input documents. Known values are: "Redact", + "Surrogate", "Tag", and "SurrogateOnly". + :vartype operation: Union[str, "DeidentificationOperationType"] + :ivar taggedEntities: Grouped PHI entities with single encoding specification for SurrogateOnly + operation. + :vartype taggedEntities: "TaggedPhiEntities" + :ivar customizations: Customization parameters to override default service behaviors. + :vartype customizations: "DeidentificationCustomizationOptions" + """ + + inputText: Required[str] + """Input text to de-identify. Required.""" + operation: Union[str, "DeidentificationOperationType"] + """Operation to perform on the input documents. Known values are: \"Redact\", \"Surrogate\", + \"Tag\", and \"SurrogateOnly\".""" + taggedEntities: "TaggedPhiEntities" + """Grouped PHI entities with single encoding specification for SurrogateOnly operation.""" + customizations: "DeidentificationCustomizationOptions" + """Customization parameters to override default service behaviors.""" + + +class DeidentificationCustomizationOptions(TypedDict, total=False): + """Customizations options to override default service behaviors for synchronous usage. + + :ivar redactionFormat: Format of the redacted output. Only valid when Operation is Redact. + Please refer to + `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format + `_ for + more details. + :vartype redactionFormat: str + :ivar surrogateLocale: Locale in which the output surrogates are written. + :vartype surrogateLocale: str + :ivar inputLocale: Locale of the input text. Used for better PHI detection. Defaults to + 'en-US'. + :vartype inputLocale: str + """ + + redactionFormat: str + """Format of the redacted output. Only valid when Operation is Redact. Please refer to + `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format + `_ for + more details.""" + surrogateLocale: str + """Locale in which the output surrogates are written.""" + inputLocale: str + """Locale of the input text. Used for better PHI detection. Defaults to 'en-US'.""" + + +class DeidentificationJob(TypedDict, total=False): + """A job containing a batch of documents to de-identify. + + :ivar name: The name of a job. Required. + :vartype name: str + :ivar operation: Operation to perform on the input documents. Known values are: "Redact", + "Surrogate", "Tag", and "SurrogateOnly". + :vartype operation: Union[str, "DeidentificationOperationType"] + :ivar sourceLocation: Storage location to perform the operation on. Required. + :vartype sourceLocation: "SourceStorageLocation" + :ivar targetLocation: Target location to store output of operation. Required. + :vartype targetLocation: "TargetStorageLocation" + :ivar customizations: Customization parameters to override default service behaviors. + :vartype customizations: "DeidentificationJobCustomizationOptions" + :ivar status: Current status of a job. Required. Known values are: "NotStarted", "Running", + "Succeeded", "Failed", and "Canceled". + :vartype status: Union[str, "OperationStatus"] + :ivar error: Error when job fails in it's entirety. + :vartype error: ODataV4Format + :ivar lastUpdatedAt: Date and time when the job was completed. + + If the job is canceled, this is the time when the job was canceled. + + If the job failed, this is the time when the job failed. Required. + :vartype lastUpdatedAt: str + :ivar createdAt: Date and time when the job was created. Required. + :vartype createdAt: str + :ivar startedAt: Date and time when the job was started. + :vartype startedAt: str + :ivar summary: Summary of a job. Exists only when the job is completed. + :vartype summary: "DeidentificationJobSummary" + """ + + name: Required[str] + """The name of a job. Required.""" + operation: Union[str, "DeidentificationOperationType"] + """Operation to perform on the input documents. Known values are: \"Redact\", \"Surrogate\", + \"Tag\", and \"SurrogateOnly\".""" + sourceLocation: Required["SourceStorageLocation"] + """Storage location to perform the operation on. Required.""" + targetLocation: Required["TargetStorageLocation"] + """Target location to store output of operation. Required.""" + customizations: "DeidentificationJobCustomizationOptions" + """Customization parameters to override default service behaviors.""" + status: Required[Union[str, "OperationStatus"]] + """Current status of a job. Required. Known values are: \"NotStarted\", \"Running\", + \"Succeeded\", \"Failed\", and \"Canceled\".""" + error: ODataV4Format + """Error when job fails in it's entirety.""" + lastUpdatedAt: Required[str] + """Date and time when the job was completed. + + If the job is canceled, this is the time when the job was canceled. + + If the job failed, this is the time when the job failed. Required.""" + createdAt: Required[str] + """Date and time when the job was created. Required.""" + startedAt: str + """Date and time when the job was started.""" + summary: "DeidentificationJobSummary" + """Summary of a job. Exists only when the job is completed.""" + + +class DeidentificationJobCustomizationOptions(TypedDict, total=False): + """Customizations options to override default service behaviors for job usage. + + :ivar redactionFormat: Format of the redacted output. Only valid when Operation is Redact. + Please refer to + `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format + `_ for + more details. + :vartype redactionFormat: str + :ivar surrogateLocale: Locale in which the output surrogates are written. + :vartype surrogateLocale: str + :ivar inputLocale: Locale of the input text. Used for better PHI detection. Defaults to + 'en-US'. + :vartype inputLocale: str + """ + + redactionFormat: str + """Format of the redacted output. Only valid when Operation is Redact. Please refer to + `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format + `_ for + more details.""" + surrogateLocale: str + """Locale in which the output surrogates are written.""" + inputLocale: str + """Locale of the input text. Used for better PHI detection. Defaults to 'en-US'.""" + + +class DeidentificationJobSummary(TypedDict, total=False): + """Summary metrics of a job. + + :ivar successful: Number of documents that have completed. Required. + :vartype successful: int + :ivar failed: Number of documents that have failed. Required. + :vartype failed: int + :ivar canceled: Number of documents that have been canceled. Required. + :vartype canceled: int + :ivar total: Number of documents total. Required. + :vartype total: int + :ivar bytesProcessed: Number of bytes processed. Required. + :vartype bytesProcessed: int + """ + + successful: Required[int] + """Number of documents that have completed. Required.""" + failed: Required[int] + """Number of documents that have failed. Required.""" + canceled: Required[int] + """Number of documents that have been canceled. Required.""" + total: Required[int] + """Number of documents total. Required.""" + bytesProcessed: Required[int] + """Number of bytes processed. Required.""" + + +class SimplePhiEntity(TypedDict, total=False): + """Simple PHI entity with encoding-specific offset and length values. + + :ivar category: PHI Category of the entity. Required. Known values are: "Unknown", "Account", + "Age", "BioID", "City", "CountryOrRegion", "Date", "Device", "Doctor", "Email", "Fax", + "HealthPlan", "Hospital", "IDNum", "IPAddress", "License", "LocationOther", "MedicalRecord", + "Organization", "Patient", "Phone", "Profession", "SocialSecurity", "State", "Street", "Url", + "Username", "Vehicle", and "Zip". + :vartype category: Union[str, "PhiCategory"] + :ivar offset: Starting index of the location from within the input text using the group's + encoding. Required. + :vartype offset: int + :ivar length: Length of the input text using the group's encoding. Required. + :vartype length: int + :ivar text: Text of the entity (optional). + :vartype text: str + """ + + category: Required[Union[str, "PhiCategory"]] + """PHI Category of the entity. Required. Known values are: \"Unknown\", \"Account\", \"Age\", + \"BioID\", \"City\", \"CountryOrRegion\", \"Date\", \"Device\", \"Doctor\", \"Email\", \"Fax\", + \"HealthPlan\", \"Hospital\", \"IDNum\", \"IPAddress\", \"License\", \"LocationOther\", + \"MedicalRecord\", \"Organization\", \"Patient\", \"Phone\", \"Profession\", + \"SocialSecurity\", \"State\", \"Street\", \"Url\", \"Username\", \"Vehicle\", and \"Zip\".""" + offset: Required[int] + """Starting index of the location from within the input text using the group's encoding. Required.""" + length: Required[int] + """Length of the input text using the group's encoding. Required.""" + text: str + """Text of the entity (optional).""" + + +class SourceStorageLocation(TypedDict, total=False): + """Storage location. + + :ivar location: URL to storage location. Required. + :vartype location: str + :ivar prefix: Prefix to filter path by. Required. + :vartype prefix: str + :ivar extensions: List of extensions to filter path by. + :vartype extensions: list[str] + """ + + location: Required[str] + """URL to storage location. Required.""" + prefix: Required[str] + """Prefix to filter path by. Required.""" + extensions: list[str] + """List of extensions to filter path by.""" + + +class TaggedPhiEntities(TypedDict, total=False): + """Grouped PHI entities with shared encoding specification. + + :ivar encoding: The encoding type used for all entities in this group. Required. Known values + are: "Utf8", "Utf16", and "CodePoint". + :vartype encoding: Union[str, "TextEncodingType"] + :ivar entities: List of PHI entities using the specified encoding. Required. + :vartype entities: list["SimplePhiEntity"] + """ + + encoding: Required[Union[str, "TextEncodingType"]] + """The encoding type used for all entities in this group. Required. Known values are: \"Utf8\", + \"Utf16\", and \"CodePoint\".""" + entities: Required[list["SimplePhiEntity"]] + """List of PHI entities using the specified encoding. Required.""" + + +class TargetStorageLocation(TypedDict, total=False): + """Storage location. + + :ivar location: URL to storage location. Required. + :vartype location: str + :ivar prefix: Replaces the input prefix of a file path with the output prefix, preserving the + rest of the path structure. + + Example: + File full path: documents/user/note.txt + Input Prefix: "documents/user/" + Output Prefix: "output_docs/" + + Output file: "output_docs/note.txt". Required. + :vartype prefix: str + :ivar overwrite: When set to true during a job, the service will overwrite the output location + if it already exists. + :vartype overwrite: bool + """ + + location: Required[str] + """URL to storage location. Required.""" + prefix: Required[str] + """Replaces the input prefix of a file path with the output prefix, preserving the rest of the + path structure. + + Example: + File full path: documents/user/note.txt + Input Prefix: \"documents/user/\" + Output Prefix: \"output_docs/\" + + Output file: \"output_docs/note.txt\". Required.""" + overwrite: bool + """When set to true during a job, the service will overwrite the output location if it already + exists.""" diff --git a/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml b/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml index f1c500aa2f77..171eea727a09 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml +++ b/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml @@ -1,3 +1,65 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# Code generated by Microsoft (R) Python Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is regenerated. +# -------------------------------------------------------------------------- + +[build-system] +requires = ["setuptools>=77.0.3", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-health-deidentification" +authors = [ + { name = "Microsoft Corporation", email = "azpysdkhelp@microsoft.com" }, +] +description = "Microsoft Corporation Azure Health Deidentification Client Library for Python" +license = "MIT" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +requires-python = ">=3.10" +keywords = ["azure", "azure sdk"] + +dependencies = [ + "isodate>=0.6.1", + "azure-core>=1.37.0", + "typing-extensions>=4.6.0", +] +dynamic = [ +"version", "readme" +] + +[project.urls] +repository = "https://github.com/Azure/azure-sdk-for-python" + +[tool.setuptools.dynamic] +version = {attr = "azure.health.deidentification._version.VERSION"} +readme = {file = ["README.md", "CHANGELOG.md"], content-type = "text/markdown"} + +[tool.setuptools.packages.find] +exclude = [ + "tests*", + "generated_tests*", + "samples*", + "generated_samples*", + "doc*", + "azure", + "azure.health", +] + +[tool.setuptools.package-data] +pytyped = ["py.typed"] + [tool.azure-sdk-build] black = true diff --git a/sdk/healthdataaiservices/azure-health-deidentification/setup.py b/sdk/healthdataaiservices/azure-health-deidentification/setup.py deleted file mode 100644 index 1fd15892a972..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - - -import os -import re -from setuptools import setup, find_packages - - -PACKAGE_NAME = "azure-health-deidentification" -PACKAGE_PPRINT_NAME = "Azure Health Deidentification" - -# a-b-c => a/b/c -package_folder_path = PACKAGE_NAME.replace("-", "/") - -# Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, "_version.py"), "r") as fd: - version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) - -if not version: - raise RuntimeError("Cannot find version information") - - -setup( - name=PACKAGE_NAME, - version=version, - description="Microsoft Corporation {} Client Library for Python".format(PACKAGE_PPRINT_NAME), - long_description=open("README.md", "r").read(), - long_description_content_type="text/markdown", - license="MIT License", - author="Microsoft Corporation", - author_email="azpysdkhelp@microsoft.com", - url="https://github.com/Azure/azure-sdk-for-python/tree/main/sdk", - keywords="azure, azure sdk", - classifiers=[ - "Development Status :: 4 - Beta", - "Programming Language :: Python", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "License :: OSI Approved :: MIT License", - ], - zip_safe=False, - packages=find_packages( - exclude=[ - "tests", - # Exclude packages that will be covered by PEP420 or nspkg - "azure", - "azure.health", - ] - ), - include_package_data=True, - package_data={ - "azure.health.deidentification": ["py.typed"], - }, - install_requires=[ - "isodate>=0.6.1", - "azure-core>=1.30.0", - "typing-extensions>=4.6.0", - ], - python_requires=">=3.9", -) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py index 625400893a1f..0f66d3d44ae4 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py @@ -4,6 +4,7 @@ from devtools_testutils import ( add_uri_regex_sanitizer, remove_batch_sanitizers, + set_custom_default_matcher, test_proxy, ) @@ -44,3 +45,6 @@ def add_sanitizers(test_proxy): # uri sanitization in favor of substitution remove_batch_sanitizers(["AZSDK3493", "AZSDK3430", "AZSDK4001"]) add_uri_regex_sanitizer(regex='continuationToken=[^&"}]*', value="continuationToken=Sanitized") + # Regenerated delete_job no longer sends "Accept: application/json" for its 204 response; existing + # recordings still contain it. + set_custom_default_matcher(excluded_headers="Accept") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py index 9274dab6f955..e6be763939aa 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py @@ -30,7 +30,8 @@ def test_create_cancel_delete(self, **kwargs): operation_type=DeidentificationOperationType.SURROGATE, ) - client.begin_deidentify_documents(jobname, job) + # The test polls the job itself; a background LRO poller would race it for recorded GET responses. + client.begin_deidentify_documents(jobname, job, polling=False) job = client.get_job(jobname) while job.status == OperationStatus.NOT_STARTED: diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml index 2a223df43022..bd8d338fb2c0 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml +++ b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml @@ -1,4 +1,4 @@ -directory: specification/healthdataaiservices/HealthDataAIServices.DeidServices -commit: a624d7fb58b59abadcd29c84268561c887bfee23 +directory: specification/healthdataaiservices/data-plane/HealthDataAIServices +commit: c7d9e3379dd76fe55ce476f7aa312a4fb938615c repo: Azure/azure-rest-api-specs additionalDirectories: From e3186e1dc5d355c7aa0f88abe74db13c1445cc59 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 10:08:23 -0500 Subject: [PATCH 02/14] [Health Deidentification] Note new types module in CHANGELOG Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../azure-health-deidentification/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md index 7c3939242a74..7f22afb9a42b 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added the `azure.health.deidentification.types` module with `TypedDict` definitions for request bodies. `begin_deidentify_documents` and `deidentify_text` accept these as typed alternatives to the models. + ### Breaking Changes ### Bugs Fixed @@ -13,6 +15,7 @@ - Regenerated the client with the latest TypeSpec emitter and removed stale generated files. - Dropped support for Python 3.9. The minimum supported Python version is now 3.10. - The minimum required `azure-core` version is now 1.37.0. +- The `JSON` overloads of `begin_deidentify_documents` and `deidentify_text` are now typed with the `TypedDict`s in `azure.health.deidentification.types`. Runtime behavior is unchanged, but static type checkers may flag arguments typed as `dict[str, Any]`. ## 1.1.0b1 (2025-08-05) From dc8531877be62a3d750e8ecacfe3fa402e0c85e2 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 10:48:06 -0500 Subject: [PATCH 03/14] Opt out of TypedDict generation for azure-health-deidentification Regenerated with generate-typeddict: false so the JSON overloads match 1.1.0b1 and the inaccurate request-body TypedDicts (types.py) are removed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../CHANGELOG.md | 3 - .../_operations/_operations.py | 42 +-- .../aio/_operations/_operations.py | 42 +-- .../azure/health/deidentification/types.py | 290 ------------------ 4 files changed, 30 insertions(+), 347 deletions(-) delete mode 100644 sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py diff --git a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md index 7f22afb9a42b..7c3939242a74 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md @@ -4,8 +4,6 @@ ### Features Added -- Added the `azure.health.deidentification.types` module with `TypedDict` definitions for request bodies. `begin_deidentify_documents` and `deidentify_text` accept these as typed alternatives to the models. - ### Breaking Changes ### Bugs Fixed @@ -15,7 +13,6 @@ - Regenerated the client with the latest TypeSpec emitter and removed stale generated files. - Dropped support for Python 3.9. The minimum supported Python version is now 3.10. - The minimum required `azure-core` version is now 1.37.0. -- The `JSON` overloads of `begin_deidentify_documents` and `deidentify_text` are now typed with the `TypedDict`s in `azure.health.deidentification.types`. Runtime behavior is unchanged, but static type checkers may flag arguments typed as `dict[str, Any]`. ## 1.1.0b1 (2025-08-05) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py index 7d9a97cc1d51..61815308ceca 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py @@ -30,12 +30,13 @@ from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict -from .. import models as _models, types as _types +from .. import models as _models from .._configuration import DeidentificationClientConfiguration from .._utils.model_base import SdkJSONEncoder, _deserialize from .._utils.serialization import Serializer from .._utils.utils import ClientMixinABC +JSON = MutableMapping[str, Any] T = TypeVar("T") ClsType = Optional[Callable[[PipelineResponse[HttpRequest, HttpResponse], T, dict[str, Any]], Any]] @@ -303,10 +304,7 @@ def get_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJob: return deserialized # type: ignore def _deidentify_documents_initial( - self, - job_name: str, - resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], - **kwargs: Any + self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any ) -> Iterator[bytes]: error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -400,12 +398,7 @@ def begin_deidentify_documents( @overload def begin_deidentify_documents( - self, - job_name: str, - resource: _types.DeidentificationJob, - *, - content_type: str = "application/json", - **kwargs: Any + self, job_name: str, resource: JSON, *, content_type: str = "application/json", **kwargs: Any ) -> LROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -414,7 +407,7 @@ def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str :param resource: The resource instance. Required. - :type resource: ~azure.health.deidentification.types.DeidentificationJob + :type resource: JSON :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -449,10 +442,7 @@ def begin_deidentify_documents( @distributed_trace def begin_deidentify_documents( - self, - job_name: str, - resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], - **kwargs: Any + self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any ) -> LROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -460,10 +450,9 @@ def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str - :param resource: The resource instance. Is either a DeidentificationJob type or a IO[bytes] - type. Required. - :type resource: ~azure.health.deidentification.models.DeidentificationJob or - ~azure.health.deidentification.types.DeidentificationJob or IO[bytes] + :param resource: The resource instance. Is one of the following types: DeidentificationJob, + JSON, IO[bytes] Required. + :type resource: ~azure.health.deidentification.models.DeidentificationJob or JSON or IO[bytes] :return: An instance of LROPoller that returns DeidentificationJob. The DeidentificationJob is compatible with MutableMapping :rtype: @@ -877,14 +866,14 @@ def deidentify_text( @overload def deidentify_text( - self, body: _types.DeidentificationContent, *, content_type: str = "application/json", **kwargs: Any + self, body: JSON, *, content_type: str = "application/json", **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. :param body: Request body for de-identification operation. Required. - :type body: ~azure.health.deidentification.types.DeidentificationContent + :type body: JSON :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -913,16 +902,15 @@ def deidentify_text( @distributed_trace def deidentify_text( - self, body: Union[_models.DeidentificationContent, _types.DeidentificationContent, IO[bytes]], **kwargs: Any + self, body: Union[_models.DeidentificationContent, JSON, IO[bytes]], **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. - :param body: Request body for de-identification operation. Is either a DeidentificationContent - type or a IO[bytes] type. Required. - :type body: ~azure.health.deidentification.models.DeidentificationContent or - ~azure.health.deidentification.types.DeidentificationContent or IO[bytes] + :param body: Request body for de-identification operation. Is one of the following types: + DeidentificationContent, JSON, IO[bytes] Required. + :type body: ~azure.health.deidentification.models.DeidentificationContent or JSON or IO[bytes] :return: DeidentificationResult. The DeidentificationResult is compatible with MutableMapping :rtype: ~azure.health.deidentification.models.DeidentificationResult :raises ~azure.core.exceptions.HttpResponseError: diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py index d218f58752bf..39d077dd229f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py @@ -32,7 +32,7 @@ from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict -from ... import models as _models, types as _types +from ... import models as _models from ..._operations._operations import ( build_deidentification_cancel_job_request, build_deidentification_deidentify_documents_request, @@ -46,6 +46,7 @@ from ..._utils.utils import ClientMixinABC from .._configuration import DeidentificationClientConfiguration +JSON = MutableMapping[str, Any] T = TypeVar("T") ClsType = Optional[Callable[[PipelineResponse[HttpRequest, AsyncHttpResponse], T, dict[str, Any]], Any]] @@ -123,10 +124,7 @@ async def get_job(self, job_name: str, **kwargs: Any) -> _models.Deidentificatio return deserialized # type: ignore async def _deidentify_documents_initial( - self, - job_name: str, - resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], - **kwargs: Any + self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any ) -> AsyncIterator[bytes]: error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -220,12 +218,7 @@ async def begin_deidentify_documents( @overload async def begin_deidentify_documents( - self, - job_name: str, - resource: _types.DeidentificationJob, - *, - content_type: str = "application/json", - **kwargs: Any + self, job_name: str, resource: JSON, *, content_type: str = "application/json", **kwargs: Any ) -> AsyncLROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -234,7 +227,7 @@ async def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str :param resource: The resource instance. Required. - :type resource: ~azure.health.deidentification.types.DeidentificationJob + :type resource: JSON :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -269,10 +262,7 @@ async def begin_deidentify_documents( @distributed_trace_async async def begin_deidentify_documents( - self, - job_name: str, - resource: Union[_models.DeidentificationJob, _types.DeidentificationJob, IO[bytes]], - **kwargs: Any + self, job_name: str, resource: Union[_models.DeidentificationJob, JSON, IO[bytes]], **kwargs: Any ) -> AsyncLROPoller[_models.DeidentificationJob]: """Create a de-identification job. @@ -280,10 +270,9 @@ async def begin_deidentify_documents( :param job_name: The name of a job. Required. :type job_name: str - :param resource: The resource instance. Is either a DeidentificationJob type or a IO[bytes] - type. Required. - :type resource: ~azure.health.deidentification.models.DeidentificationJob or - ~azure.health.deidentification.types.DeidentificationJob or IO[bytes] + :param resource: The resource instance. Is one of the following types: DeidentificationJob, + JSON, IO[bytes] Required. + :type resource: ~azure.health.deidentification.models.DeidentificationJob or JSON or IO[bytes] :return: An instance of AsyncLROPoller that returns DeidentificationJob. The DeidentificationJob is compatible with MutableMapping :rtype: @@ -699,14 +688,14 @@ async def deidentify_text( @overload async def deidentify_text( - self, body: _types.DeidentificationContent, *, content_type: str = "application/json", **kwargs: Any + self, body: JSON, *, content_type: str = "application/json", **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. :param body: Request body for de-identification operation. Required. - :type body: ~azure.health.deidentification.types.DeidentificationContent + :type body: JSON :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. Default value is "application/json". :paramtype content_type: str @@ -735,16 +724,15 @@ async def deidentify_text( @distributed_trace_async async def deidentify_text( - self, body: Union[_models.DeidentificationContent, _types.DeidentificationContent, IO[bytes]], **kwargs: Any + self, body: Union[_models.DeidentificationContent, JSON, IO[bytes]], **kwargs: Any ) -> _models.DeidentificationResult: """De-identify text. A remote procedure call (RPC) operation. - :param body: Request body for de-identification operation. Is either a DeidentificationContent - type or a IO[bytes] type. Required. - :type body: ~azure.health.deidentification.models.DeidentificationContent or - ~azure.health.deidentification.types.DeidentificationContent or IO[bytes] + :param body: Request body for de-identification operation. Is one of the following types: + DeidentificationContent, JSON, IO[bytes] Required. + :type body: ~azure.health.deidentification.models.DeidentificationContent or JSON or IO[bytes] :return: DeidentificationResult. The DeidentificationResult is compatible with MutableMapping :rtype: ~azure.health.deidentification.models.DeidentificationResult :raises ~azure.core.exceptions.HttpResponseError: diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py deleted file mode 100644 index 6e1feec90b13..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/types.py +++ /dev/null @@ -1,290 +0,0 @@ -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from typing import TYPE_CHECKING, Union -from typing_extensions import Required, TypedDict - -from azure.core.exceptions import ODataV4Format - -if TYPE_CHECKING: - from .models import DeidentificationOperationType, OperationStatus, PhiCategory, TextEncodingType - - -class DeidentificationContent(TypedDict, total=False): - """Request body for de-identification operation. - - :ivar inputText: Input text to de-identify. Required. - :vartype inputText: str - :ivar operation: Operation to perform on the input documents. Known values are: "Redact", - "Surrogate", "Tag", and "SurrogateOnly". - :vartype operation: Union[str, "DeidentificationOperationType"] - :ivar taggedEntities: Grouped PHI entities with single encoding specification for SurrogateOnly - operation. - :vartype taggedEntities: "TaggedPhiEntities" - :ivar customizations: Customization parameters to override default service behaviors. - :vartype customizations: "DeidentificationCustomizationOptions" - """ - - inputText: Required[str] - """Input text to de-identify. Required.""" - operation: Union[str, "DeidentificationOperationType"] - """Operation to perform on the input documents. Known values are: \"Redact\", \"Surrogate\", - \"Tag\", and \"SurrogateOnly\".""" - taggedEntities: "TaggedPhiEntities" - """Grouped PHI entities with single encoding specification for SurrogateOnly operation.""" - customizations: "DeidentificationCustomizationOptions" - """Customization parameters to override default service behaviors.""" - - -class DeidentificationCustomizationOptions(TypedDict, total=False): - """Customizations options to override default service behaviors for synchronous usage. - - :ivar redactionFormat: Format of the redacted output. Only valid when Operation is Redact. - Please refer to - `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format - `_ for - more details. - :vartype redactionFormat: str - :ivar surrogateLocale: Locale in which the output surrogates are written. - :vartype surrogateLocale: str - :ivar inputLocale: Locale of the input text. Used for better PHI detection. Defaults to - 'en-US'. - :vartype inputLocale: str - """ - - redactionFormat: str - """Format of the redacted output. Only valid when Operation is Redact. Please refer to - `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format - `_ for - more details.""" - surrogateLocale: str - """Locale in which the output surrogates are written.""" - inputLocale: str - """Locale of the input text. Used for better PHI detection. Defaults to 'en-US'.""" - - -class DeidentificationJob(TypedDict, total=False): - """A job containing a batch of documents to de-identify. - - :ivar name: The name of a job. Required. - :vartype name: str - :ivar operation: Operation to perform on the input documents. Known values are: "Redact", - "Surrogate", "Tag", and "SurrogateOnly". - :vartype operation: Union[str, "DeidentificationOperationType"] - :ivar sourceLocation: Storage location to perform the operation on. Required. - :vartype sourceLocation: "SourceStorageLocation" - :ivar targetLocation: Target location to store output of operation. Required. - :vartype targetLocation: "TargetStorageLocation" - :ivar customizations: Customization parameters to override default service behaviors. - :vartype customizations: "DeidentificationJobCustomizationOptions" - :ivar status: Current status of a job. Required. Known values are: "NotStarted", "Running", - "Succeeded", "Failed", and "Canceled". - :vartype status: Union[str, "OperationStatus"] - :ivar error: Error when job fails in it's entirety. - :vartype error: ODataV4Format - :ivar lastUpdatedAt: Date and time when the job was completed. - - If the job is canceled, this is the time when the job was canceled. - - If the job failed, this is the time when the job failed. Required. - :vartype lastUpdatedAt: str - :ivar createdAt: Date and time when the job was created. Required. - :vartype createdAt: str - :ivar startedAt: Date and time when the job was started. - :vartype startedAt: str - :ivar summary: Summary of a job. Exists only when the job is completed. - :vartype summary: "DeidentificationJobSummary" - """ - - name: Required[str] - """The name of a job. Required.""" - operation: Union[str, "DeidentificationOperationType"] - """Operation to perform on the input documents. Known values are: \"Redact\", \"Surrogate\", - \"Tag\", and \"SurrogateOnly\".""" - sourceLocation: Required["SourceStorageLocation"] - """Storage location to perform the operation on. Required.""" - targetLocation: Required["TargetStorageLocation"] - """Target location to store output of operation. Required.""" - customizations: "DeidentificationJobCustomizationOptions" - """Customization parameters to override default service behaviors.""" - status: Required[Union[str, "OperationStatus"]] - """Current status of a job. Required. Known values are: \"NotStarted\", \"Running\", - \"Succeeded\", \"Failed\", and \"Canceled\".""" - error: ODataV4Format - """Error when job fails in it's entirety.""" - lastUpdatedAt: Required[str] - """Date and time when the job was completed. - - If the job is canceled, this is the time when the job was canceled. - - If the job failed, this is the time when the job failed. Required.""" - createdAt: Required[str] - """Date and time when the job was created. Required.""" - startedAt: str - """Date and time when the job was started.""" - summary: "DeidentificationJobSummary" - """Summary of a job. Exists only when the job is completed.""" - - -class DeidentificationJobCustomizationOptions(TypedDict, total=False): - """Customizations options to override default service behaviors for job usage. - - :ivar redactionFormat: Format of the redacted output. Only valid when Operation is Redact. - Please refer to - `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format - `_ for - more details. - :vartype redactionFormat: str - :ivar surrogateLocale: Locale in which the output surrogates are written. - :vartype surrogateLocale: str - :ivar inputLocale: Locale of the input text. Used for better PHI detection. Defaults to - 'en-US'. - :vartype inputLocale: str - """ - - redactionFormat: str - """Format of the redacted output. Only valid when Operation is Redact. Please refer to - `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format - `_ for - more details.""" - surrogateLocale: str - """Locale in which the output surrogates are written.""" - inputLocale: str - """Locale of the input text. Used for better PHI detection. Defaults to 'en-US'.""" - - -class DeidentificationJobSummary(TypedDict, total=False): - """Summary metrics of a job. - - :ivar successful: Number of documents that have completed. Required. - :vartype successful: int - :ivar failed: Number of documents that have failed. Required. - :vartype failed: int - :ivar canceled: Number of documents that have been canceled. Required. - :vartype canceled: int - :ivar total: Number of documents total. Required. - :vartype total: int - :ivar bytesProcessed: Number of bytes processed. Required. - :vartype bytesProcessed: int - """ - - successful: Required[int] - """Number of documents that have completed. Required.""" - failed: Required[int] - """Number of documents that have failed. Required.""" - canceled: Required[int] - """Number of documents that have been canceled. Required.""" - total: Required[int] - """Number of documents total. Required.""" - bytesProcessed: Required[int] - """Number of bytes processed. Required.""" - - -class SimplePhiEntity(TypedDict, total=False): - """Simple PHI entity with encoding-specific offset and length values. - - :ivar category: PHI Category of the entity. Required. Known values are: "Unknown", "Account", - "Age", "BioID", "City", "CountryOrRegion", "Date", "Device", "Doctor", "Email", "Fax", - "HealthPlan", "Hospital", "IDNum", "IPAddress", "License", "LocationOther", "MedicalRecord", - "Organization", "Patient", "Phone", "Profession", "SocialSecurity", "State", "Street", "Url", - "Username", "Vehicle", and "Zip". - :vartype category: Union[str, "PhiCategory"] - :ivar offset: Starting index of the location from within the input text using the group's - encoding. Required. - :vartype offset: int - :ivar length: Length of the input text using the group's encoding. Required. - :vartype length: int - :ivar text: Text of the entity (optional). - :vartype text: str - """ - - category: Required[Union[str, "PhiCategory"]] - """PHI Category of the entity. Required. Known values are: \"Unknown\", \"Account\", \"Age\", - \"BioID\", \"City\", \"CountryOrRegion\", \"Date\", \"Device\", \"Doctor\", \"Email\", \"Fax\", - \"HealthPlan\", \"Hospital\", \"IDNum\", \"IPAddress\", \"License\", \"LocationOther\", - \"MedicalRecord\", \"Organization\", \"Patient\", \"Phone\", \"Profession\", - \"SocialSecurity\", \"State\", \"Street\", \"Url\", \"Username\", \"Vehicle\", and \"Zip\".""" - offset: Required[int] - """Starting index of the location from within the input text using the group's encoding. Required.""" - length: Required[int] - """Length of the input text using the group's encoding. Required.""" - text: str - """Text of the entity (optional).""" - - -class SourceStorageLocation(TypedDict, total=False): - """Storage location. - - :ivar location: URL to storage location. Required. - :vartype location: str - :ivar prefix: Prefix to filter path by. Required. - :vartype prefix: str - :ivar extensions: List of extensions to filter path by. - :vartype extensions: list[str] - """ - - location: Required[str] - """URL to storage location. Required.""" - prefix: Required[str] - """Prefix to filter path by. Required.""" - extensions: list[str] - """List of extensions to filter path by.""" - - -class TaggedPhiEntities(TypedDict, total=False): - """Grouped PHI entities with shared encoding specification. - - :ivar encoding: The encoding type used for all entities in this group. Required. Known values - are: "Utf8", "Utf16", and "CodePoint". - :vartype encoding: Union[str, "TextEncodingType"] - :ivar entities: List of PHI entities using the specified encoding. Required. - :vartype entities: list["SimplePhiEntity"] - """ - - encoding: Required[Union[str, "TextEncodingType"]] - """The encoding type used for all entities in this group. Required. Known values are: \"Utf8\", - \"Utf16\", and \"CodePoint\".""" - entities: Required[list["SimplePhiEntity"]] - """List of PHI entities using the specified encoding. Required.""" - - -class TargetStorageLocation(TypedDict, total=False): - """Storage location. - - :ivar location: URL to storage location. Required. - :vartype location: str - :ivar prefix: Replaces the input prefix of a file path with the output prefix, preserving the - rest of the path structure. - - Example: - File full path: documents/user/note.txt - Input Prefix: "documents/user/" - Output Prefix: "output_docs/" - - Output file: "output_docs/note.txt". Required. - :vartype prefix: str - :ivar overwrite: When set to true during a job, the service will overwrite the output location - if it already exists. - :vartype overwrite: bool - """ - - location: Required[str] - """URL to storage location. Required.""" - prefix: Required[str] - """Replaces the input prefix of a file path with the output prefix, preserving the rest of the - path structure. - - Example: - File full path: documents/user/note.txt - Input Prefix: \"documents/user/\" - Output Prefix: \"output_docs/\" - - Output file: \"output_docs/note.txt\". Required.""" - overwrite: bool - """When set to true during a job, the service will overwrite the output location if it already - exists.""" From 54c06ccc8488033f1d2eac646751180020812bbc Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Mon, 19 Oct 2026 16:45:49 -0500 Subject: [PATCH 04/14] [Health Deid] Replace fixed sleep with retry/backoff in test-resources-post.ps1 Role assignment and network ACL propagation can take minutes after deployment, causing intermittent 403 AuthorizationFailure on upload. Retry container access and each blob upload with exponential backoff (5s initial, 60s cap, 5 min budget) and fail with a clear error. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d9b0092-996f-4265-b2da-1effbf3945c6 --- .../test-resources-post.ps1 | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 index c162558889fd..2fe997c58ed7 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 +++ b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 @@ -27,6 +27,44 @@ if ([string]::IsNullOrWhiteSpace($storageAccountName)) { exit 1 } +# Retries a script block with exponential backoff until it succeeds or the time budget is exhausted. +# Role assignments and network ACL changes can take minutes to propagate after deployment. +function Invoke-WithRetry { + param ( + [Parameter(Mandatory = $true)] + [scriptblock] $ScriptBlock, + [Parameter(Mandatory = $true)] + [string] $Operation, + [int] $MaxTotalSeconds = 300, + [int] $InitialDelaySeconds = 5, + [int] $MaxDelaySeconds = 60 + ) + + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + $delay = $InitialDelaySeconds + $attempt = 0 + while ($true) { + $attempt++ + try { + $result = & $ScriptBlock + if ($attempt -gt 1) { + Write-Host "$Operation succeeded on attempt $attempt after $([int]$stopwatch.Elapsed.TotalSeconds)s." + } + return $result + } + catch { + $lastError = $_ + $elapsed = $stopwatch.Elapsed.TotalSeconds + if ($elapsed + $delay -gt $MaxTotalSeconds) { + throw "$Operation failed after $attempt attempt(s) over $([int]$elapsed)s. This usually means the storage role assignment or network ACL change has not propagated. Last error: $($lastError.Exception.Message)" + } + Write-Host "$Operation failed (attempt $attempt, $([int]$elapsed)s elapsed): $($lastError.Exception.Message). Retrying in ${delay}s." + Start-Sleep -Seconds $delay + $delay = [Math]::Min($delay * 2, $MaxDelaySeconds) + } + } +} + # Load the Azure Storage module Import-Module Az.Storage @@ -36,25 +74,26 @@ $storageContext = New-AzStorageContext -StorageAccountName $storageAccountName - # FIXME Remove once vpn team fixes the network acl issue $networkRuleSet = New-Object -TypeName Microsoft.Azure.Commands.Management.Storage.Models.PSNetworkRuleSet $networkRuleSet.DefaultAction = "Allow" -Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet +Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet -ErrorAction Stop -# Sleep for 30 seconds to allow the network rule to take effect -Write-Host "[Fix] Temporary sleep to allow network rule to take effect." -Start-Sleep -Seconds 30 - -Get-AzStorageContainer -Name $containerName -Context $storageContext +# Wait until the container is reachable through the data plane (RBAC + network ACL propagated) +Invoke-WithRetry -Operation "Accessing container '$containerName'" -ScriptBlock { + Get-AzStorageContainer -Name $containerName -Context $storageContext -ErrorAction Stop +} # Upload the folder and its contents to the container # Gets last folder name + filename. example_patient_1/doctor_dictation.txt -Get-ChildItem -Path $localFolderPath -Recurse | ForEach-Object { - $relativePath = $_.FullName - $folderName = Split-Path -Path (Split-Path -Path $relativePath -Parent) -Leaf - $blobName = Split-Path -Path $relativePath -Leaf +Get-ChildItem -Path $localFolderPath -Recurse -File | ForEach-Object { + $filePath = $_.FullName + $folderName = Split-Path -Path (Split-Path -Path $filePath -Parent) -Leaf + $blobName = Split-Path -Path $filePath -Leaf $destinationBlob = $blobName -replace ":", "" $destinationBlob = "$folderName/$destinationBlob" Write-Host "Uploading file '$destinationBlob'" - Set-AzStorageBlobContent -File $_.FullName -Container $containerName -Blob $destinationBlob -Context $storageContext -Force + Invoke-WithRetry -Operation "Uploading blob '$destinationBlob'" -ScriptBlock { + Set-AzStorageBlobContent -File $filePath -Container $containerName -Blob $destinationBlob -Context $storageContext -Force -ErrorAction Stop + } } Write-Host "Folder '$localFolderPath' uploaded to container '$containerName' successfully." From af712e4ff5197a01e50055118d46151c24c2b862 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 10:44:01 -0500 Subject: [PATCH 05/14] [Health Deid] Keep a 30s minimum wait after the network rule update The retry loop proves the test principal's RBAC and the network rule, but not the DeID managed identity's role assignment. Restore the previous 30s wait as a minimum measured from the network rule update; time spent on retries and uploads counts toward it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d9b0092-996f-4265-b2da-1effbf3945c6 --- .../test-resources-post.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 index 2fe997c58ed7..084db7465cfa 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 +++ b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 @@ -75,6 +75,7 @@ $storageContext = New-AzStorageContext -StorageAccountName $storageAccountName - $networkRuleSet = New-Object -TypeName Microsoft.Azure.Commands.Management.Storage.Models.PSNetworkRuleSet $networkRuleSet.DefaultAction = "Allow" Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet -ErrorAction Stop +$sinceNetworkRuleUpdate = [System.Diagnostics.Stopwatch]::StartNew() # Wait until the container is reachable through the data plane (RBAC + network ACL propagated) Invoke-WithRetry -Operation "Accessing container '$containerName'" -ScriptBlock { @@ -97,3 +98,12 @@ Get-ChildItem -Path $localFolderPath -Recurse -File | ForEach-Object { } Write-Host "Folder '$localFolderPath' uploaded to container '$containerName' successfully." + +# Our own access proves the test principal's RBAC and the network rule, but not the DeID service's managed +# identity role assignment. Keep a minimum wait after the network rule update for it to propagate. +$minimumWaitSeconds = 30 +$remainingSeconds = [int][Math]::Ceiling($minimumWaitSeconds - $sinceNetworkRuleUpdate.Elapsed.TotalSeconds) +if ($remainingSeconds -gt 0) { + Write-Host "Waiting ${remainingSeconds}s more so at least ${minimumWaitSeconds}s pass after the network rule update for role assignments to propagate." + Start-Sleep -Seconds $remainingSeconds +} From 6b57ee151e98df70a4988ae09393383090076ed9 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Mon, 19 Oct 2026 16:57:26 -0500 Subject: [PATCH 06/14] [DeID] Surface job errors in live test failures; random live job names - Add wait_for_job/wait_for_job_async test helpers that report job status, error, summary, and unsuccessful documents instead of azure-core's generic 'Operation returned an invalid status OK', and fail explicitly when the job is not terminal after the timeout (now 600s). - Generate a fresh uniquifier for every live run (including AZURE_SKIP_LIVE_RECORDING); only persist it when recording. - Samples print job.error before re-raising on job failure. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 40cdcc81-accc-4b2e-af24-fd79c6480740 --- .../deidentify_documents_async.py | 13 +- .../async_samples/list_job_documents_async.py | 10 +- .../samples/deidentify_documents.py | 9 +- .../samples/list_job_documents.py | 9 +- .../tests/conftest.py | 18 +-- .../tests/deid_base_test_case.py | 115 ++++++++++++++++++ .../tests/test_create_wait_finish.py | 6 +- .../tests/test_create_wait_finish_async.py | 6 +- .../tests/test_list_job_docs_pagination.py | 2 +- 9 files changed, 169 insertions(+), 19 deletions(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py index fe065600659e..d6d2d44e915f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py @@ -23,6 +23,7 @@ import asyncio +from azure.core.exceptions import HttpResponseError from azure.core.polling import AsyncLROPoller from azure.health.deidentification.aio import DeidentificationClient from azure.health.deidentification.models import ( @@ -56,9 +57,15 @@ async def deidentify_documents_async(): async with client: lro: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = await lro.result() - - await credential.close() + try: + finished_job: DeidentificationJob = await lro.result() + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise + finally: + await credential.close() print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") # Succeeded diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py index 54ea3ec0464c..b17baa6ab8c0 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py @@ -22,6 +22,7 @@ import asyncio +from azure.core.exceptions import HttpResponseError from azure.core.polling import AsyncLROPoller from azure.health.deidentification.aio import DeidentificationClient from azure.health.deidentification.models import ( @@ -56,7 +57,14 @@ async def list_job_documents_async(): print(f"Creating job with name: {jobname}") async with client: poller: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - job = await poller.result() + try: + job = await poller.result() + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + await credential.close() + raise print(f"Job Status: {job.status}") files = client.list_job_documents(jobname) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py index ac6e2c05026d..4657bf075ef1 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py @@ -26,6 +26,7 @@ def deidentify_documents(): # [START sample] + from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -57,7 +58,13 @@ def deidentify_documents(): target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) - finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) + try: + finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py index 1322e0a75285..edaaa20d7b38 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py @@ -23,6 +23,7 @@ """ +from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -56,7 +57,13 @@ def list_job_documents(): print(f"Creating job with name: {jobname}") - job = client.begin_deidentify_documents(jobname, job).result(timeout=60) + try: + job = client.begin_deidentify_documents(jobname, job).result(timeout=60) + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise print(f"Job Status: {job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py index 0f66d3d44ae4..285b4a1f016a 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py @@ -22,16 +22,18 @@ def start_proxy(test_proxy, patch_sleep, patch_async_sleep): @pytest.fixture(scope="session", autouse=True) def create_session_uniquifier(): - if ( - os.environ.get("AZURE_TEST_RUN_LIVE", "false").lower() == "true" # Don't override uniquifier by default - and os.environ.get("AZURE_SKIP_LIVE_RECORDING", "false").lower() != "true" - and os.environ.get("AZURE_TEST_KEEP_UNIQUIFIER", "false").lower() != "true" - ): - print("Creating new uniquifier for live test run.") + is_live = os.environ.get("AZURE_TEST_RUN_LIVE", "false").lower() == "true" + skip_recording = os.environ.get("AZURE_SKIP_LIVE_RECORDING", "false").lower() == "true" + keep_uniquifier = os.environ.get("AZURE_TEST_KEEP_UNIQUIFIER", "false").lower() == "true" + if is_live and not keep_uniquifier: + # Every live run gets fresh job names so reused resources or parallel matrix legs can't collide. uniquifier = uuid.uuid4().hex[:6] + print(f"Creating new uniquifier for live test run: {uniquifier}") os.environ["HEALTHDATAAISERVICES_UNIQUIFIER"] = uniquifier - with open(uniquifier_file, "w") as file: - file.write(uniquifier) + if not skip_recording: + # Persist only when recording so playback replays the job names in the new recordings. + with open(uniquifier_file, "w") as file: + file.write(uniquifier) else: with open(uniquifier_file, "r") as file: uniquifier = file.read() diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py index cb9bb597f468..a8e0143cb6a2 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py @@ -1,13 +1,24 @@ +import asyncio import functools import inspect import os import random import datetime import uuid +from typing import List, Optional + +from azure.core.exceptions import HttpResponseError +from azure.core.polling import AsyncLROPoller, LROPoller +from azure.core.polling.base_polling import OperationFailed from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.aio import ( DeidentificationClient as DeidentificationClientAsync, ) +from azure.health.deidentification.models import ( + DeidentificationDocumentDetails, + DeidentificationJob, + OperationStatus, +) from devtools_testutils import ( @@ -30,9 +41,113 @@ ) +JOB_TIMEOUT_SECONDS = 600 +MAX_DIAGNOSTIC_DOCUMENTS = 20 + + +def format_job_diagnostics(job: DeidentificationJob, documents: List[DeidentificationDocumentDetails]) -> str: + lines = [ + f"Job '{job.job_name}' status: {job.status}", + f" created_at={job.created_at} started_at={job.started_at} last_updated_at={job.last_updated_at}", + ] + if job.summary is not None: + s = job.summary + lines.append( + f" summary: total={s.total_count} successful={s.successful_count} " + f"failed={s.failed_count} canceled={s.canceled_count} bytes_processed={s.bytes_processed}" + ) + else: + lines.append(" summary: None") + lines.append(f" error: {job.error if job.error is not None else 'None'}") + if documents: + lines.append(f" unsuccessful documents (up to {MAX_DIAGNOSTIC_DOCUMENTS}):") + for doc in documents: + location = doc.input_location.location if doc.input_location is not None else doc.id + lines.append(f" - {location} status={doc.status} error={doc.error}") + return "\n".join(lines) + + +def _is_unsuccessful(doc: DeidentificationDocumentDetails) -> bool: + return doc.status != OperationStatus.SUCCEEDED or doc.error is not None + + class DeidBaseTestCase(AzureRecordedTestCase): OUTPUT_PATH = "_output" + def describe_job(self, client: DeidentificationClient, job_name: str) -> str: + """Fetch the job and its unsuccessful documents and format them for an assertion message.""" + try: + job = client.get_job(job_name) + documents: List[DeidentificationDocumentDetails] = [] + for doc in client.list_job_documents(job_name): + if _is_unsuccessful(doc): + documents.append(doc) + if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: + break + return format_job_diagnostics(job, documents) + except Exception as e: # pylint: disable=broad-except + # Diagnostics must never mask the original failure. + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + + async def describe_job_async(self, client: DeidentificationClientAsync, job_name: str) -> str: + """Async version of describe_job.""" + try: + job = await client.get_job(job_name) + documents: List[DeidentificationDocumentDetails] = [] + async for doc in client.list_job_documents(job_name): + if _is_unsuccessful(doc): + documents.append(doc) + if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: + break + return format_job_diagnostics(job, documents) + except Exception as e: # pylint: disable=broad-except + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + + def wait_for_job( + self, + client: DeidentificationClient, + poller: LROPoller[DeidentificationJob], + job_name: str, + timeout: Optional[float] = JOB_TIMEOUT_SECONDS, + ) -> DeidentificationJob: + """Wait for a job LRO and fail with the job's real error instead of azure-core's generic polling error. + + azure-core surfaces a Failed/Canceled job as HttpResponseError("Operation returned an invalid status 'OK'"), + and LROPoller.result(timeout) silently returns the in-progress job when the timeout expires. + """ + try: + job = poller.result(timeout=timeout) + except (HttpResponseError, OperationFailed) as e: + raise AssertionError( + f"Job '{job_name}' did not succeed ({e}).\n{self.describe_job(client, job_name)}" + ) from e + if not poller.done(): + raise AssertionError( + f"Job '{job_name}' did not reach a terminal state within {timeout}s.\n" + f"{self.describe_job(client, job_name)}" + ) + return job + + async def wait_for_job_async( + self, + client: DeidentificationClientAsync, + poller: AsyncLROPoller[DeidentificationJob], + job_name: str, + timeout: Optional[float] = JOB_TIMEOUT_SECONDS, + ) -> DeidentificationJob: + """Async version of wait_for_job.""" + try: + return await asyncio.wait_for(poller.result(), timeout=timeout) + except asyncio.TimeoutError as e: + raise AssertionError( + f"Job '{job_name}' did not reach a terminal state within {timeout}s.\n" + f"{await self.describe_job_async(client, job_name)}" + ) from e + except (HttpResponseError, OperationFailed) as e: + raise AssertionError( + f"Job '{job_name}' did not succeed ({e}).\n{await self.describe_job_async(client, job_name)}" + ) from e + def make_client(self, endpoint) -> DeidentificationClient: credential = self.get_credential(DeidentificationClient) connection_verify = not "localhost" in endpoint diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py index aaebe91349a6..1da0e802c658 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py @@ -29,7 +29,7 @@ def test_create_wait_finish(self, **kwargs): ) lro: LROPoller = client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = lro.result(timeout=120) + finished_job: DeidentificationJob = self.wait_for_job(client, lro, jobname) assert finished_job.status == OperationStatus.SUCCEEDED assert finished_job.job_name == jobname @@ -50,7 +50,9 @@ def test_create_wait_finish(self, **kwargs): for my_file in files: assert len(my_file.id) == 36 # GUID assert input_prefix in my_file.input_location.location - assert my_file.status == OperationStatus.SUCCEEDED + assert ( + my_file.status == OperationStatus.SUCCEEDED + ), f"{my_file.input_location.location}: status={my_file.status} error={my_file.error}" assert my_file.output_location is not None assert self.OUTPUT_PATH in my_file.output_location.location count += 1 diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py index be9bae5499fc..7545a9f3b0cd 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py @@ -32,7 +32,7 @@ async def test_create_wait_finish_async(self, **kwargs): lro: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = await lro.result() + finished_job: DeidentificationJob = await self.wait_for_job_async(client, lro, jobname) assert finished_job.status == OperationStatus.SUCCEEDED assert finished_job.job_name == jobname @@ -54,7 +54,9 @@ async def test_create_wait_finish_async(self, **kwargs): async for my_file in files: assert len(my_file.id) == 36 # GUID assert input_prefix in my_file.input_location.location - assert my_file.status == OperationStatus.SUCCEEDED + assert ( + my_file.status == OperationStatus.SUCCEEDED + ), f"{my_file.input_location.location}: status={my_file.status} error={my_file.error}" assert my_file.output_location is not None assert self.OUTPUT_PATH in my_file.output_location.location count += 1 diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py index ddb7f06ca83f..bffb32dbcdd4 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py @@ -33,7 +33,7 @@ def test_list_job_docs_pagination(self, **kwargs): customizations=DeidentificationJobCustomizationOptions(redaction_format="[{type}]"), ) - client.begin_deidentify_documents(jobname, job).result(180) + self.wait_for_job(client, client.begin_deidentify_documents(jobname, job), jobname) job_documents = client.list_job_documents(job_name=jobname, maxpagesize=2) _get_next = job_documents._args[0] # type: ignore From 185512553e6816e519095129a33bc201f7c59bfa Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Tue, 20 Oct 2026 16:40:10 -0500 Subject: [PATCH 07/14] [DeID] Sync README snippet with deidentify_documents sample Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 40cdcc81-accc-4b2e-af24-fd79c6480740 --- .../azure-health-deidentification/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/README.md b/sdk/healthdataaiservices/azure-health-deidentification/README.md index 4beb01f8a12a..4737c1382f84 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/README.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/README.md @@ -152,6 +152,7 @@ The client exposes a `begin_deidentify_documents` method that returns a [LROPoll ```python +from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -183,7 +184,13 @@ job = DeidentificationJob( target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) -finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) +try: + finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) +except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") From d742b3c10df52554d5d03616baeca80d3d9ca28e Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 14:05:12 -0500 Subject: [PATCH 08/14] [DeID] Make sample job diagnostics best-effort; always close async credentials The diagnostic get_job lookup in the samples' error handling could raise (for example 404 when creation was rejected) and replace the original error. It is now wrapped so the original exception is always re-raised. The async samples close the credential via async with. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../azure-health-deidentification/README.md | 8 ++++++-- .../async_samples/deidentify_documents_async.py | 12 +++++++----- .../async_samples/list_job_documents_async.py | 13 +++++++------ .../samples/deidentify_documents.py | 8 ++++++-- .../samples/list_job_documents.py | 8 ++++++-- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/README.md b/sdk/healthdataaiservices/azure-health-deidentification/README.md index 4737c1382f84..7411b6fff35f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/README.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/README.md @@ -188,8 +188,12 @@ try: finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) except HttpResponseError: # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. - failed_job = client.get_job(jobname) - print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") raise print(f"Job Name: {finished_job.job_name}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py index d6d2d44e915f..9a7bc853a453 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py @@ -55,17 +55,19 @@ async def deidentify_documents_async(): target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) - async with client: + async with credential, client: lro: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) try: finished_job: DeidentificationJob = await lro.result() except HttpResponseError: # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. - failed_job = await client.get_job(jobname) - print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") raise - finally: - await credential.close() print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") # Succeeded diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py index b17baa6ab8c0..99fc5fb234eb 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py @@ -55,15 +55,18 @@ async def list_job_documents_async(): ) print(f"Creating job with name: {jobname}") - async with client: + async with credential, client: poller: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) try: job = await poller.result() except HttpResponseError: # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. - failed_job = await client.get_job(jobname) - print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") - await credential.close() + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") raise print(f"Job Status: {job.status}") @@ -78,8 +81,6 @@ async def list_job_documents_async(): if filesToLookThrough <= 0: break - await credential.close() - async def main(): await list_job_documents_async() diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py index 4657bf075ef1..e85e61245f9e 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py @@ -62,8 +62,12 @@ def deidentify_documents(): finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) except HttpResponseError: # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. - failed_job = client.get_job(jobname) - print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") raise print(f"Job Name: {finished_job.job_name}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py index edaaa20d7b38..c79acadd732f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py @@ -61,8 +61,12 @@ def list_job_documents(): job = client.begin_deidentify_documents(jobname, job).result(timeout=60) except HttpResponseError: # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. - failed_job = client.get_job(jobname) - print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") raise print(f"Job Status: {job.status}") From 2dee9159d469e6b6e9f9118d540cb69df5b1730a Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 14:49:59 -0500 Subject: [PATCH 09/14] [DeID] Point tsp-location at spec commit with generate-typeddict: false Spec PR Azure/azure-rest-api-specs#47496 adds generate-typeddict: false to the Python emitter options. Regenerating from that commit produces no changes to the generated code. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../azure-health-deidentification/tsp-location.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml index bd8d338fb2c0..c9cbfa0aa555 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml +++ b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml @@ -1,4 +1,4 @@ directory: specification/healthdataaiservices/data-plane/HealthDataAIServices -commit: c7d9e3379dd76fe55ce476f7aa312a4fb938615c +commit: 876d6affeb52e0a66d9bc660e18b1332c0240f33 repo: Azure/azure-rest-api-specs additionalDirectories: From 29fe2f4da0fbb9fe52b2d2359cc90bc4a5f41f22 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 17:11:35 -0500 Subject: [PATCH 10/14] [DeID] Point tsp-location at merged spec commit Azure/azure-rest-api-specs#47496 merged as 6de3840. Regenerating from it produces no changes to the generated code. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../azure-health-deidentification/tsp-location.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml index c9cbfa0aa555..7c077c9b9c46 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml +++ b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml @@ -1,4 +1,4 @@ directory: specification/healthdataaiservices/data-plane/HealthDataAIServices -commit: 876d6affeb52e0a66d9bc660e18b1332c0240f33 +commit: 6de384008f55f42ea97ee0654f237c74fb05ee89 repo: Azure/azure-rest-api-specs additionalDirectories: From b3bd9a64544a1cf1229fbfc31d1c2a3afeae93b8 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 17:11:35 -0500 Subject: [PATCH 11/14] [DeID] Keep job diagnostics when document listing fails describe_job(_async) now fetches the job and lists documents in separate try blocks, so a listing failure (even on a later page) no longer discards the job status/error or documents already collected; the listing error is appended instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../tests/deid_base_test_case.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py index a8e0143cb6a2..1f79b796cf24 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py @@ -45,7 +45,11 @@ MAX_DIAGNOSTIC_DOCUMENTS = 20 -def format_job_diagnostics(job: DeidentificationJob, documents: List[DeidentificationDocumentDetails]) -> str: +def format_job_diagnostics( + job: DeidentificationJob, + documents: List[DeidentificationDocumentDetails], + listing_error: Optional[Exception] = None, +) -> str: lines = [ f"Job '{job.job_name}' status: {job.status}", f" created_at={job.created_at} started_at={job.started_at} last_updated_at={job.last_updated_at}", @@ -64,6 +68,8 @@ def format_job_diagnostics(job: DeidentificationJob, documents: List[Deidentific for doc in documents: location = doc.input_location.location if doc.input_location is not None else doc.id lines.append(f" - {location} status={doc.status} error={doc.error}") + if listing_error is not None: + lines.append(f" unable to list all job documents: {listing_error!r}") return "\n".join(lines) @@ -76,32 +82,41 @@ class DeidBaseTestCase(AzureRecordedTestCase): def describe_job(self, client: DeidentificationClient, job_name: str) -> str: """Fetch the job and its unsuccessful documents and format them for an assertion message.""" + # Diagnostics must never mask the original failure, and a document-listing failure (even on a later page) + # must not discard the job details or documents already collected. try: job = client.get_job(job_name) - documents: List[DeidentificationDocumentDetails] = [] + except Exception as e: # pylint: disable=broad-except + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + documents: List[DeidentificationDocumentDetails] = [] + listing_error: Optional[Exception] = None + try: for doc in client.list_job_documents(job_name): if _is_unsuccessful(doc): documents.append(doc) if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: break - return format_job_diagnostics(job, documents) except Exception as e: # pylint: disable=broad-except - # Diagnostics must never mask the original failure. - return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + listing_error = e + return format_job_diagnostics(job, documents, listing_error) async def describe_job_async(self, client: DeidentificationClientAsync, job_name: str) -> str: """Async version of describe_job.""" try: job = await client.get_job(job_name) - documents: List[DeidentificationDocumentDetails] = [] + except Exception as e: # pylint: disable=broad-except + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + documents: List[DeidentificationDocumentDetails] = [] + listing_error: Optional[Exception] = None + try: async for doc in client.list_job_documents(job_name): if _is_unsuccessful(doc): documents.append(doc) if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: break - return format_job_diagnostics(job, documents) except Exception as e: # pylint: disable=broad-except - return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + listing_error = e + return format_job_diagnostics(job, documents, listing_error) def wait_for_job( self, From 34a97bc588ed8534d3965c40342318338593a252 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Thu, 22 Oct 2026 10:52:45 -0500 Subject: [PATCH 12/14] [DeID] Preserve model container identity across dict and property access The generated _RestField.__get__ replaces the backing list/dict/set in _data when a mutable field is first read as a property, detaching any reference previously returned by model["field"]. Wrap __get__ in patch_sdk() to copy the deserialized contents into the original container instead, until the emitter template is fixed. Adds unit tests for all dict/property access patterns plus a guard that fails once the generated code no longer needs the shim. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../CHANGELOG.md | 2 + .../azure/health/deidentification/_patch.py | 51 ++++++++ .../tests/test_model_container_identity.py | 115 ++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 sdk/healthdataaiservices/azure-health-deidentification/tests/test_model_container_identity.py diff --git a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md index 7c3939242a74..bbc2c5ed4a46 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md @@ -8,6 +8,8 @@ ### Bugs Fixed +- List, dict, and set fields on models now consistently reflect in-place mutations, whether the value was obtained through dictionary access (`model["field"]`) or property access (`model.field`). + ### Other Changes - Regenerated the client with the latest TypeSpec emitter and removed stale generated files. diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py index bb09ee2cbf77..9b37d22c7fed 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py @@ -6,10 +6,12 @@ Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize """ +import functools from typing import Any, Iterable, List from azure.core.tracing.decorator import distributed_trace from azure.health.deidentification.models import DeidentificationDocumentDetails, DeidentificationJob from ._client import DeidentificationClient as DeidentificationClientGenerated +from ._utils import model_base as _model_base __all__: List[str] = [ "DeidentificationClient", @@ -43,6 +45,54 @@ def list_job_documents(self, job_name: str, **kwargs: Any) -> Iterable[Deidentif return super()._list_job_documents_internal(job_name=job_name, continuation_token_parameter=None, **kwargs) +_MISSING = object() +_IDENTITY_SHIM_MARKER = "_deid_preserves_container_identity" + + +def _preserve_container_identity() -> None: + """Keep list/dict/set fields bound to one container across dict and property access. + + The generated ``_RestField.__get__`` stores a newly deserialized container back into + ``_data``, detaching any container previously returned by ``model["field"]``, so later + mutations through that reference are dropped on serialization. This wrapper copies the + new contents into the original container and keeps it in ``_data``. + + TODO: remove once the emitter template is fixed (microsoft/typespec, + packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2, + ``_RestField.__get__``). See https://github.com/Azure/azure-sdk-for-python/pull/49627#discussion_r4342560283 + """ + rest_field_cls = _model_base._RestField # pylint: disable=protected-access + original_get: Any = rest_field_cls.__get__ + if getattr(original_get, _IDENTITY_SHIM_MARKER, False): + return + + @functools.wraps(original_get) + def __get__(self: Any, obj: Any, owner: Any = None) -> Any: + if obj is None: + return original_get(self, obj, owner) + data = obj._data # pylint: disable=protected-access + name = self._rest_name # pylint: disable=protected-access + before = data.get(name, _MISSING) + result = original_get(self, obj, owner) + after = data.get(name, _MISSING) + if before is after or result is not after: + return result + if isinstance(before, list) and isinstance(after, list): + before[:] = after + elif (isinstance(before, dict) and isinstance(after, dict)) or ( + isinstance(before, set) and isinstance(after, set) + ): + before.clear() + before.update(after) + else: + return result + data[name] = before + return before + + setattr(__get__, _IDENTITY_SHIM_MARKER, True) + setattr(rest_field_cls, "__get__", __get__) + + def patch_sdk(): """Do not remove from this file. @@ -50,3 +100,4 @@ def patch_sdk(): you can't accomplish using the techniques described in https://aka.ms/azsdk/python/dpcodegen/python/customize """ + _preserve_container_identity() diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_model_container_identity.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_model_container_identity.py new file mode 100644 index 000000000000..c43404c84530 --- /dev/null +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_model_container_identity.py @@ -0,0 +1,115 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Unit tests for the _patch.py shim that keeps model container fields bound to one object. + +No service calls or recordings are involved. +""" +import pytest + +import azure.health.deidentification.aio # pylint: disable=unused-import +from azure.health.deidentification import _patch +from azure.health.deidentification._utils import model_base +from azure.health.deidentification.models import ( + SimplePhiEntity, + SourceStorageLocation, + TaggedPhiEntities, +) + +# Each pattern lists how references are obtained, in order; the first reference is then mutated. +PATTERNS = { + "A_dict_property_mutate_dict_ref": ["dict", "property"], + "B_property_dict_mutate_property_ref": ["property", "dict"], + "C_property_mutate": ["property"], + "D_dict_mutate": ["dict"], + "E_property_property_mutate_first_ref": ["property", "property"], +} + + +def _access(model, attr, key, how): + return model[key] if how == "dict" else getattr(model, attr) + + +def _make_source(): + return SourceStorageLocation(location="https://example.blob.core.windows.net/c", prefix="p", extensions=[".txt"]) + + +def _make_tagged_from_kwargs(): + return TaggedPhiEntities( + encoding="CodePoint", + entities=[SimplePhiEntity(category="Patient", offset=0, length=4, text="John")], + ) + + +def _make_tagged_from_mapping(): + return TaggedPhiEntities( + {"encoding": "CodePoint", "entities": [{"category": "Patient", "offset": 0, "length": 4, "text": "John"}]} + ) + + +def _mutate_extensions(source, accesses): + refs = [_access(source, "extensions", "extensions", how) for how in accesses] + refs[0].append(".csv") + return source.as_dict()["extensions"] + + +@pytest.mark.parametrize("accesses", PATTERNS.values(), ids=PATTERNS.keys()) +def test_list_of_str_mutation_is_serialized(accesses): + assert _mutate_extensions(_make_source(), accesses) == [".txt", ".csv"] + + +@pytest.mark.parametrize( + "make_tagged", [_make_tagged_from_kwargs, _make_tagged_from_mapping], ids=["kwargs", "mapping"] +) +@pytest.mark.parametrize("accesses", PATTERNS.values(), ids=PATTERNS.keys()) +def test_list_of_models_mutation_is_serialized(accesses, make_tagged): + tagged = make_tagged() + refs = [_access(tagged, "entities", "entities", how) for how in accesses] + refs[0].append(SimplePhiEntity(category="Date", offset=9, length=10)) + refs[0][0]["text"] = "Jane" + + assert tagged.as_dict() == { + "encoding": "CodePoint", + "entities": [ + {"category": "Patient", "offset": 0, "length": 4, "text": "Jane"}, + {"category": "Date", "offset": 9, "length": 10}, + ], + } + assert [type(e) for e in tagged.entities] == [SimplePhiEntity, SimplePhiEntity] + assert tagged.entities[0].text == "Jane" + + +def test_property_and_dict_access_share_one_list(): + source = _make_source() + via_dict = source["extensions"] + via_property = source.extensions + assert via_property is via_dict + assert source["extensions"] is via_dict + + +def test_setting_field_replaces_container(): + source = _make_source() + old = source.extensions + source.extensions = [".json"] + assert source.extensions == [".json"] + assert old == [".txt"] + assert source.as_dict()["extensions"] == [".json"] + + +def test_shim_is_applied_once_for_sync_and_aio(): + _patch.patch_sdk() + _patch.patch_sdk() + get = model_base._RestField.__get__ # pylint: disable=protected-access + assert getattr(get, _patch._IDENTITY_SHIM_MARKER, False) # pylint: disable=protected-access + assert not getattr(get.__wrapped__, _patch._IDENTITY_SHIM_MARKER, False) # pylint: disable=protected-access + + +def test_generated_get_still_needs_shim(monkeypatch): + """Fails once the emitter preserves container identity itself; then remove the _patch.py shim.""" + rest_field = model_base._RestField # pylint: disable=protected-access + monkeypatch.setattr(rest_field, "__get__", rest_field.__get__.__wrapped__) + assert _mutate_extensions(_make_source(), PATTERNS["A_dict_property_mutate_dict_ref"]) == [".txt"], ( + "Generated _RestField.__get__ now preserves container identity; " + "remove _preserve_container_identity from azure/health/deidentification/_patch.py." + ) From f4ed24e3be0c4fc6c0903ef514c4038b87b17050 Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Thu, 22 Oct 2026 11:00:16 -0500 Subject: [PATCH 13/14] [DeID] Link microsoft/typespec#12359 as the shim removal tracker Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../azure/health/deidentification/_patch.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py index 9b37d22c7fed..853a762370ac 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py @@ -57,9 +57,8 @@ def _preserve_container_identity() -> None: mutations through that reference are dropped on serialization. This wrapper copies the new contents into the original container and keeps it in ``_data``. - TODO: remove once the emitter template is fixed (microsoft/typespec, - packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2, - ``_RestField.__get__``). See https://github.com/Azure/azure-sdk-for-python/pull/49627#discussion_r4342560283 + TODO: remove once https://github.com/microsoft/typespec/pull/12359 fixes the emitter template + (packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2, ``_RestField.__get__``). """ rest_field_cls = _model_base._RestField # pylint: disable=protected-access original_get: Any = rest_field_cls.__get__ From 0504930167e2ec4e41152a7af6984f7c13e563bf Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Fri, 23 Oct 2026 15:12:55 -0500 Subject: [PATCH 14/14] [DeID] Fix cspell failure in container-identity shim comment The shim's TODO quoted the emitter template's repository path, which contains "pygen" and fails the cspell check. Describe the template by name instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ef2497f1-4ec3-4283-9448-f9679c8b7849 --- .../azure/health/deidentification/_patch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py index 853a762370ac..c65684174efc 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py @@ -57,8 +57,8 @@ def _preserve_container_identity() -> None: mutations through that reference are dropped on serialization. This wrapper copies the new contents into the original container and keeps it in ``_data``. - TODO: remove once https://github.com/microsoft/typespec/pull/12359 fixes the emitter template - (packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2, ``_RestField.__get__``). + TODO: remove once https://github.com/microsoft/typespec/pull/12359 fixes ``_RestField.__get__`` in the + http-client-python emitter's ``model_base.py.jinja2`` template. """ rest_field_cls = _model_base._RestField # pylint: disable=protected-access original_get: Any = rest_field_cls.__get__