diff --git a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md index 0f286e140ea3..bbc2c5ed4a46 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/CHANGELOG.md @@ -1,5 +1,21 @@ # Release History +## 1.1.0b2 (Unreleased) + +### Features Added + +### Breaking Changes + +### Bugs Fixed + +- List, dict, and set fields on models now consistently reflect in-place mutations, whether the value was obtained through dictionary access (`model["field"]`) or property access (`model.field`). + +### Other Changes + +- Regenerated the client with the latest TypeSpec emitter and removed stale generated files. +- Dropped support for Python 3.9. The minimum supported Python version is now 3.10. +- The minimum required `azure-core` version is now 1.37.0. + ## 1.1.0b1 (2025-08-05) ### Features Added diff --git a/sdk/healthdataaiservices/azure-health-deidentification/README.md b/sdk/healthdataaiservices/azure-health-deidentification/README.md index 87b662118aa9..7411b6fff35f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/README.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/README.md @@ -22,7 +22,7 @@ Use the client library for the de-identification service to: ### Prequisites -- Python 3.9 or later is required to use this package. +- Python 3.10 or later is required to use this package. - Install [pip][pip]. - You need an [Azure subscription][azure_sub] to use this package. - [Deploy the de-identification service][deid_quickstart]. @@ -152,6 +152,7 @@ The client exposes a `begin_deidentify_documents` method that returns a [LROPoll ```python +from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -183,7 +184,17 @@ job = DeidentificationJob( target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) -finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) +try: + finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) +except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") + raise print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json b/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json index fd321bd8fa9b..7721b6e48d10 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json +++ b/sdk/healthdataaiservices/azure-health-deidentification/_metadata.json @@ -1,3 +1,6 @@ { - "apiVersion": "2025-07-15-preview" + "apiVersion": "2025-07-15-preview", + "apiVersions": { + "HealthDataAIServices.DeidServices": "2025-07-15-preview" + } } \ No newline at end of file diff --git a/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json b/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json index cf39b3fdd7c6..5f7cede3467b 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json +++ b/sdk/healthdataaiservices/azure-health-deidentification/apiview-properties.json @@ -30,5 +30,6 @@ "azure.health.deidentification.aio.DeidentificationClient.delete_job": "HealthDataAIServices.DeidServices.deleteJob", "azure.health.deidentification.DeidentificationClient.deidentify_text": "HealthDataAIServices.DeidServices.deidentifyText", "azure.health.deidentification.aio.DeidentificationClient.deidentify_text": "HealthDataAIServices.DeidServices.deidentifyText" - } + }, + "CrossLanguageVersion": "8ab7039f42a1" } \ No newline at end of file diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py index 603a33be33d1..c52572072304 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_client.py @@ -7,8 +7,8 @@ # -------------------------------------------------------------------------- from copy import deepcopy +import sys from typing import Any, TYPE_CHECKING -from typing_extensions import Self from azure.core import PipelineClient from azure.core.pipeline import policies @@ -18,19 +18,27 @@ from ._operations import _DeidentificationClientOperationsMixin from ._utils.serialization import Deserializer, Serializer +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self # type: ignore + if TYPE_CHECKING: from azure.core.credentials import TokenCredential -class DeidentificationClient(_DeidentificationClientOperationsMixin): +class DeidentificationClient( + _DeidentificationClientOperationsMixin +): # pylint: disable=docstring-keyword-should-match-keyword-only """DeidentificationClient. :param endpoint: Url of your De-identification Service. Required. :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials.TokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str :keyword int polling_interval: Default waiting time between two polls for LRO operations if no diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py index b2305ff66bc8..04f8590d4a5b 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_configuration.py @@ -1,3 +1,4 @@ +# pylint: disable=line-too-long,useless-suppression # coding=utf-8 # -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. @@ -16,7 +17,7 @@ from azure.core.credentials import TokenCredential -class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes +class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes,docstring-keyword-should-match-keyword-only """Configuration for DeidentificationClient. Note that all parameters used to create this instance are saved as instance @@ -26,8 +27,9 @@ class DeidentificationClientConfiguration: # pylint: disable=too-many-instance- :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials.TokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str """ diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_model_base.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_model_base.py deleted file mode 100644 index 095e62400e89..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_model_base.py +++ /dev/null @@ -1,1232 +0,0 @@ -# pylint: disable=too-many-lines,line-too-long,useless-suppression -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- -# pylint: disable=protected-access, broad-except - -import copy -import calendar -import decimal -import functools -import sys -import logging -import base64 -import re -import typing -import enum -import email.utils -from datetime import datetime, date, time, timedelta, timezone -from json import JSONEncoder -import xml.etree.ElementTree as ET -from collections.abc import MutableMapping -from typing_extensions import Self -import isodate -from azure.core.exceptions import DeserializationError -from azure.core import CaseInsensitiveEnumMeta -from azure.core.pipeline import PipelineResponse -from azure.core.serialization import _Null - -_LOGGER = logging.getLogger(__name__) - -__all__ = ["SdkJSONEncoder", "Model", "rest_field", "rest_discriminator"] - -TZ_UTC = timezone.utc -_T = typing.TypeVar("_T") - - -def _timedelta_as_isostr(td: timedelta) -> str: - """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S' - - Function adapted from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython - - :param timedelta td: The timedelta to convert - :rtype: str - :return: ISO8601 version of this timedelta - """ - - # Split seconds to larger units - seconds = td.total_seconds() - minutes, seconds = divmod(seconds, 60) - hours, minutes = divmod(minutes, 60) - days, hours = divmod(hours, 24) - - days, hours, minutes = list(map(int, (days, hours, minutes))) - seconds = round(seconds, 6) - - # Build date - date_str = "" - if days: - date_str = "%sD" % days - - if hours or minutes or seconds: - # Build time - time_str = "T" - - # Hours - bigger_exists = date_str or hours - if bigger_exists: - time_str += "{:02}H".format(hours) - - # Minutes - bigger_exists = bigger_exists or minutes - if bigger_exists: - time_str += "{:02}M".format(minutes) - - # Seconds - try: - if seconds.is_integer(): - seconds_string = "{:02}".format(int(seconds)) - else: - # 9 chars long w/ leading 0, 6 digits after decimal - seconds_string = "%09.6f" % seconds - # Remove trailing zeros - seconds_string = seconds_string.rstrip("0") - except AttributeError: # int.is_integer() raises - seconds_string = "{:02}".format(seconds) - - time_str += "{}S".format(seconds_string) - else: - time_str = "" - - return "P" + date_str + time_str - - -def _serialize_bytes(o, format: typing.Optional[str] = None) -> str: - encoded = base64.b64encode(o).decode() - if format == "base64url": - return encoded.strip("=").replace("+", "-").replace("/", "_") - return encoded - - -def _serialize_datetime(o, format: typing.Optional[str] = None): - if hasattr(o, "year") and hasattr(o, "hour"): - if format == "rfc7231": - return email.utils.format_datetime(o, usegmt=True) - if format == "unix-timestamp": - return int(calendar.timegm(o.utctimetuple())) - - # astimezone() fails for naive times in Python 2.7, so make make sure o is aware (tzinfo is set) - if not o.tzinfo: - iso_formatted = o.replace(tzinfo=TZ_UTC).isoformat() - else: - iso_formatted = o.astimezone(TZ_UTC).isoformat() - # Replace the trailing "+00:00" UTC offset with "Z" (RFC 3339: https://www.ietf.org/rfc/rfc3339.txt) - return iso_formatted.replace("+00:00", "Z") - # Next try datetime.date or datetime.time - return o.isoformat() - - -def _is_readonly(p): - try: - return p._visibility == ["read"] - except AttributeError: - return False - - -class SdkJSONEncoder(JSONEncoder): - """A JSON encoder that's capable of serializing datetime objects and bytes.""" - - def __init__(self, *args, exclude_readonly: bool = False, format: typing.Optional[str] = None, **kwargs): - super().__init__(*args, **kwargs) - self.exclude_readonly = exclude_readonly - self.format = format - - def default(self, o): # pylint: disable=too-many-return-statements - if _is_model(o): - if self.exclude_readonly: - readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)] - return {k: v for k, v in o.items() if k not in readonly_props} - return dict(o.items()) - try: - return super(SdkJSONEncoder, self).default(o) - except TypeError: - if isinstance(o, _Null): - return None - if isinstance(o, decimal.Decimal): - return float(o) - if isinstance(o, (bytes, bytearray)): - return _serialize_bytes(o, self.format) - try: - # First try datetime.datetime - return _serialize_datetime(o, self.format) - except AttributeError: - pass - # Last, try datetime.timedelta - try: - return _timedelta_as_isostr(o) - except AttributeError: - # This will be raised when it hits value.total_seconds in the method above - pass - return super(SdkJSONEncoder, self).default(o) - - -_VALID_DATE = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}" + r"\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") -_VALID_RFC7231 = re.compile( - r"(Mon|Tue|Wed|Thu|Fri|Sat|Sun),\s\d{2}\s" - r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT" -) - - -def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: - """Deserialize ISO-8601 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - attr = attr.upper() - match = _VALID_DATE.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - check_decimal = attr.split(".") - if len(check_decimal) > 1: - decimal_str = "" - for digit in check_decimal[1]: - if digit.isdigit(): - decimal_str += digit - else: - break - if len(decimal_str) > 6: - attr = attr.replace(decimal_str, decimal_str[0:6]) - - date_obj = isodate.parse_datetime(attr) - test_utc = date_obj.utctimetuple() - if test_utc.tm_year > 9999 or test_utc.tm_year < 1: - raise OverflowError("Hit max or min date") - return date_obj - - -def _deserialize_datetime_rfc7231(attr: typing.Union[str, datetime]) -> datetime: - """Deserialize RFC7231 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - match = _VALID_RFC7231.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - return email.utils.parsedate_to_datetime(attr) - - -def _deserialize_datetime_unix_timestamp(attr: typing.Union[float, datetime]) -> datetime: - """Deserialize unix timestamp into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - return datetime.fromtimestamp(attr, TZ_UTC) - - -def _deserialize_date(attr: typing.Union[str, date]) -> date: - """Deserialize ISO-8601 formatted string into Date object. - :param str attr: response string to be deserialized. - :rtype: date - :returns: The date object from that input - """ - # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. - if isinstance(attr, date): - return attr - return isodate.parse_date(attr, defaultmonth=None, defaultday=None) # type: ignore - - -def _deserialize_time(attr: typing.Union[str, time]) -> time: - """Deserialize ISO-8601 formatted string into time object. - - :param str attr: response string to be deserialized. - :rtype: datetime.time - :returns: The time object from that input - """ - if isinstance(attr, time): - return attr - return isodate.parse_time(attr) - - -def _deserialize_bytes(attr): - if isinstance(attr, (bytes, bytearray)): - return attr - return bytes(base64.b64decode(attr)) - - -def _deserialize_bytes_base64(attr): - if isinstance(attr, (bytes, bytearray)): - return attr - padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore - attr = attr + padding # type: ignore - encoded = attr.replace("-", "+").replace("_", "/") - return bytes(base64.b64decode(encoded)) - - -def _deserialize_duration(attr): - if isinstance(attr, timedelta): - return attr - return isodate.parse_duration(attr) - - -def _deserialize_decimal(attr): - if isinstance(attr, decimal.Decimal): - return attr - return decimal.Decimal(str(attr)) - - -def _deserialize_int_as_str(attr): - if isinstance(attr, int): - return attr - return int(attr) - - -_DESERIALIZE_MAPPING = { - datetime: _deserialize_datetime, - date: _deserialize_date, - time: _deserialize_time, - bytes: _deserialize_bytes, - bytearray: _deserialize_bytes, - timedelta: _deserialize_duration, - typing.Any: lambda x: x, - decimal.Decimal: _deserialize_decimal, -} - -_DESERIALIZE_MAPPING_WITHFORMAT = { - "rfc3339": _deserialize_datetime, - "rfc7231": _deserialize_datetime_rfc7231, - "unix-timestamp": _deserialize_datetime_unix_timestamp, - "base64": _deserialize_bytes, - "base64url": _deserialize_bytes_base64, -} - - -def get_deserializer(annotation: typing.Any, rf: typing.Optional["_RestField"] = None): - if annotation is int and rf and rf._format == "str": - return _deserialize_int_as_str - if rf and rf._format: - return _DESERIALIZE_MAPPING_WITHFORMAT.get(rf._format) - return _DESERIALIZE_MAPPING.get(annotation) # pyright: ignore - - -def _get_type_alias_type(module_name: str, alias_name: str): - types = { - k: v - for k, v in sys.modules[module_name].__dict__.items() - if isinstance(v, typing._GenericAlias) # type: ignore - } - if alias_name not in types: - return alias_name - return types[alias_name] - - -def _get_model(module_name: str, model_name: str): - models = {k: v for k, v in sys.modules[module_name].__dict__.items() if isinstance(v, type)} - module_end = module_name.rsplit(".", 1)[0] - models.update({k: v for k, v in sys.modules[module_end].__dict__.items() if isinstance(v, type)}) - if isinstance(model_name, str): - model_name = model_name.split(".")[-1] - if model_name not in models: - return model_name - return models[model_name] - - -_UNSET = object() - - -class _MyMutableMapping(MutableMapping[str, typing.Any]): - def __init__(self, data: typing.Dict[str, typing.Any]) -> None: - self._data = data - - def __contains__(self, key: typing.Any) -> bool: - return key in self._data - - def __getitem__(self, key: str) -> typing.Any: - return self._data.__getitem__(key) - - def __setitem__(self, key: str, value: typing.Any) -> None: - self._data.__setitem__(key, value) - - def __delitem__(self, key: str) -> None: - self._data.__delitem__(key) - - def __iter__(self) -> typing.Iterator[typing.Any]: - return self._data.__iter__() - - def __len__(self) -> int: - return self._data.__len__() - - def __ne__(self, other: typing.Any) -> bool: - return not self.__eq__(other) - - def keys(self) -> typing.KeysView[str]: - """ - :returns: a set-like object providing a view on D's keys - :rtype: ~typing.KeysView - """ - return self._data.keys() - - def values(self) -> typing.ValuesView[typing.Any]: - """ - :returns: an object providing a view on D's values - :rtype: ~typing.ValuesView - """ - return self._data.values() - - def items(self) -> typing.ItemsView[str, typing.Any]: - """ - :returns: set-like object providing a view on D's items - :rtype: ~typing.ItemsView - """ - return self._data.items() - - def get(self, key: str, default: typing.Any = None) -> typing.Any: - """ - Get the value for key if key is in the dictionary, else default. - :param str key: The key to look up. - :param any default: The value to return if key is not in the dictionary. Defaults to None - :returns: D[k] if k in D, else d. - :rtype: any - """ - try: - return self[key] - except KeyError: - return default - - @typing.overload - def pop(self, key: str) -> typing.Any: ... # pylint: disable=arguments-differ - - @typing.overload - def pop(self, key: str, default: _T) -> _T: ... # pylint: disable=signature-differs - - @typing.overload - def pop(self, key: str, default: typing.Any) -> typing.Any: ... # pylint: disable=signature-differs - - def pop(self, key: str, default: typing.Any = _UNSET) -> typing.Any: - """ - Removes specified key and return the corresponding value. - :param str key: The key to pop. - :param any default: The value to return if key is not in the dictionary - :returns: The value corresponding to the key. - :rtype: any - :raises KeyError: If key is not found and default is not given. - """ - if default is _UNSET: - return self._data.pop(key) - return self._data.pop(key, default) - - def popitem(self) -> typing.Tuple[str, typing.Any]: - """ - Removes and returns some (key, value) pair - :returns: The (key, value) pair. - :rtype: tuple - :raises KeyError: if D is empty. - """ - return self._data.popitem() - - def clear(self) -> None: - """ - Remove all items from D. - """ - self._data.clear() - - def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: # pylint: disable=arguments-differ - """ - Updates D from mapping/iterable E and F. - :param any args: Either a mapping object or an iterable of key-value pairs. - """ - self._data.update(*args, **kwargs) - - @typing.overload - def setdefault(self, key: str, default: None = None) -> None: ... - - @typing.overload - def setdefault(self, key: str, default: typing.Any) -> typing.Any: ... # pylint: disable=signature-differs - - def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: - """ - Same as calling D.get(k, d), and setting D[k]=d if k not found - :param str key: The key to look up. - :param any default: The value to set if key is not in the dictionary - :returns: D[k] if k in D, else d. - :rtype: any - """ - if default is _UNSET: - return self._data.setdefault(key) - return self._data.setdefault(key, default) - - def __eq__(self, other: typing.Any) -> bool: - try: - other_model = self.__class__(other) - except Exception: - return False - return self._data == other_model._data - - def __repr__(self) -> str: - return str(self._data) - - -def _is_model(obj: typing.Any) -> bool: - return getattr(obj, "_is_model", False) - - -def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-many-return-statements - if isinstance(o, list): - return [_serialize(x, format) for x in o] - if isinstance(o, dict): - return {k: _serialize(v, format) for k, v in o.items()} - if isinstance(o, set): - return {_serialize(x, format) for x in o} - if isinstance(o, tuple): - return tuple(_serialize(x, format) for x in o) - if isinstance(o, (bytes, bytearray)): - return _serialize_bytes(o, format) - if isinstance(o, decimal.Decimal): - return float(o) - if isinstance(o, enum.Enum): - return o.value - if isinstance(o, int): - if format == "str": - return str(o) - return o - try: - # First try datetime.datetime - return _serialize_datetime(o, format) - except AttributeError: - pass - # Last, try datetime.timedelta - try: - return _timedelta_as_isostr(o) - except AttributeError: - # This will be raised when it hits value.total_seconds in the method above - pass - return o - - -def _get_rest_field( - attr_to_rest_field: typing.Dict[str, "_RestField"], rest_name: str -) -> typing.Optional["_RestField"]: - try: - return next(rf for rf in attr_to_rest_field.values() if rf._rest_name == rest_name) - except StopIteration: - return None - - -def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typing.Any: - if not rf: - return _serialize(value, None) - if rf._is_multipart_file_input: - return value - if rf._is_model: - return _deserialize(rf._type, value) - if isinstance(value, ET.Element): - value = _deserialize(rf._type, value) - return _serialize(value, rf._format) - - -class Model(_MyMutableMapping): - _is_model = True - # label whether current class's _attr_to_rest_field has been calculated - # could not see _attr_to_rest_field directly because subclass inherits it from parent class - _calculated: typing.Set[str] = set() - - def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: - class_name = self.__class__.__name__ - if len(args) > 1: - raise TypeError(f"{class_name}.__init__() takes 2 positional arguments but {len(args) + 1} were given") - dict_to_pass = { - rest_field._rest_name: rest_field._default - for rest_field in self._attr_to_rest_field.values() - if rest_field._default is not _UNSET - } - if args: # pylint: disable=too-many-nested-blocks - if isinstance(args[0], ET.Element): - existed_attr_keys = [] - model_meta = getattr(self, "_xml", {}) - - for rf in self._attr_to_rest_field.values(): - prop_meta = getattr(rf, "_xml", {}) - xml_name = prop_meta.get("name", rf._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - - # attribute - if prop_meta.get("attribute", False) and args[0].get(xml_name) is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].get(xml_name)) - continue - - # unwrapped element is array - if prop_meta.get("unwrapped", False): - # unwrapped array could either use prop items meta/prop meta - if prop_meta.get("itemsName"): - xml_name = prop_meta.get("itemsName") - xml_ns = prop_meta.get("itemNs") - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - items = args[0].findall(xml_name) # pyright: ignore - if len(items) > 0: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, items) - continue - - # text element is primitive type - if prop_meta.get("text", False): - if args[0].text is not None: - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].text) - continue - - # wrapped element could be normal property or array, it should only have one element - item = args[0].find(xml_name) - if item is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, item) - - # rest thing is additional properties - for e in args[0]: - if e.tag not in existed_attr_keys: - dict_to_pass[e.tag] = _convert_element(e) - else: - dict_to_pass.update( - {k: _create_value(_get_rest_field(self._attr_to_rest_field, k), v) for k, v in args[0].items()} - ) - else: - non_attr_kwargs = [k for k in kwargs if k not in self._attr_to_rest_field] - if non_attr_kwargs: - # actual type errors only throw the first wrong keyword arg they see, so following that. - raise TypeError(f"{class_name}.__init__() got an unexpected keyword argument '{non_attr_kwargs[0]}'") - dict_to_pass.update( - { - self._attr_to_rest_field[k]._rest_name: _create_value(self._attr_to_rest_field[k], v) - for k, v in kwargs.items() - if v is not None - } - ) - super().__init__(dict_to_pass) - - def copy(self) -> "Model": - return Model(self.__dict__) - - def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: - if f"{cls.__module__}.{cls.__qualname__}" not in cls._calculated: - # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping', - # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object' - mros = cls.__mro__[:-9][::-1] # ignore parents, and reverse the mro order - attr_to_rest_field: typing.Dict[str, _RestField] = { # map attribute name to rest_field property - k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type") - } - annotations = { - k: v - for mro_class in mros - if hasattr(mro_class, "__annotations__") - for k, v in mro_class.__annotations__.items() - } - for attr, rf in attr_to_rest_field.items(): - rf._module = cls.__module__ - if not rf._type: - rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None)) - if not rf._rest_name_input: - rf._rest_name_input = attr - cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items()) - cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}") - - return super().__new__(cls) - - def __init_subclass__(cls, discriminator: typing.Optional[str] = None) -> None: - for base in cls.__bases__: - if hasattr(base, "__mapping__"): - base.__mapping__[discriminator or cls.__name__] = cls # type: ignore - - @classmethod - def _get_discriminator(cls, exist_discriminators) -> typing.Optional["_RestField"]: - for v in cls.__dict__.values(): - if isinstance(v, _RestField) and v._is_discriminator and v._rest_name not in exist_discriminators: - return v - return None - - @classmethod - def _deserialize(cls, data, exist_discriminators): - if not hasattr(cls, "__mapping__"): - return cls(data) - discriminator = cls._get_discriminator(exist_discriminators) - if discriminator is None: - return cls(data) - exist_discriminators.append(discriminator._rest_name) - if isinstance(data, ET.Element): - model_meta = getattr(cls, "_xml", {}) - prop_meta = getattr(discriminator, "_xml", {}) - xml_name = prop_meta.get("name", discriminator._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - - if data.get(xml_name) is not None: - discriminator_value = data.get(xml_name) - else: - discriminator_value = data.find(xml_name).text # pyright: ignore - else: - discriminator_value = data.get(discriminator._rest_name) - mapped_cls = cls.__mapping__.get(discriminator_value, cls) # pyright: ignore # pylint: disable=no-member - return mapped_cls._deserialize(data, exist_discriminators) - - def as_dict(self, *, exclude_readonly: bool = False) -> typing.Dict[str, typing.Any]: - """Return a dict that can be turned into json using json.dump. - - :keyword bool exclude_readonly: Whether to remove the readonly properties. - :returns: A dict JSON compatible object - :rtype: dict - """ - - result = {} - readonly_props = [] - if exclude_readonly: - readonly_props = [p._rest_name for p in self._attr_to_rest_field.values() if _is_readonly(p)] - for k, v in self.items(): - if exclude_readonly and k in readonly_props: # pyright: ignore - continue - is_multipart_file_input = False - try: - is_multipart_file_input = next( - rf for rf in self._attr_to_rest_field.values() if rf._rest_name == k - )._is_multipart_file_input - except StopIteration: - pass - result[k] = v if is_multipart_file_input else Model._as_dict_value(v, exclude_readonly=exclude_readonly) - return result - - @staticmethod - def _as_dict_value(v: typing.Any, exclude_readonly: bool = False) -> typing.Any: - if v is None or isinstance(v, _Null): - return None - if isinstance(v, (list, tuple, set)): - return type(v)(Model._as_dict_value(x, exclude_readonly=exclude_readonly) for x in v) - if isinstance(v, dict): - return {dk: Model._as_dict_value(dv, exclude_readonly=exclude_readonly) for dk, dv in v.items()} - return v.as_dict(exclude_readonly=exclude_readonly) if hasattr(v, "as_dict") else v - - -def _deserialize_model(model_deserializer: typing.Optional[typing.Callable], obj): - if _is_model(obj): - return obj - return _deserialize(model_deserializer, obj) - - -def _deserialize_with_optional(if_obj_deserializer: typing.Optional[typing.Callable], obj): - if obj is None: - return obj - return _deserialize_with_callable(if_obj_deserializer, obj) - - -def _deserialize_with_union(deserializers, obj): - for deserializer in deserializers: - try: - return _deserialize(deserializer, obj) - except DeserializationError: - pass - raise DeserializationError() - - -def _deserialize_dict( - value_deserializer: typing.Optional[typing.Callable], - module: typing.Optional[str], - obj: typing.Dict[typing.Any, typing.Any], -): - if obj is None: - return obj - if isinstance(obj, ET.Element): - obj = {child.tag: child for child in obj} - return {k: _deserialize(value_deserializer, v, module) for k, v in obj.items()} - - -def _deserialize_multiple_sequence( - entry_deserializers: typing.List[typing.Optional[typing.Callable]], - module: typing.Optional[str], - obj, -): - if obj is None: - return obj - return type(obj)(_deserialize(deserializer, entry, module) for entry, deserializer in zip(obj, entry_deserializers)) - - -def _deserialize_sequence( - deserializer: typing.Optional[typing.Callable], - module: typing.Optional[str], - obj, -): - if obj is None: - return obj - if isinstance(obj, ET.Element): - obj = list(obj) - return type(obj)(_deserialize(deserializer, entry, module) for entry in obj) - - -def _sorted_annotations(types: typing.List[typing.Any]) -> typing.List[typing.Any]: - return sorted( - types, - key=lambda x: hasattr(x, "__name__") and x.__name__.lower() in ("str", "float", "int", "bool"), - ) - - -def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-return-statements, too-many-branches - annotation: typing.Any, - module: typing.Optional[str], - rf: typing.Optional["_RestField"] = None, -) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: - if not annotation: - return None - - # is it a type alias? - if isinstance(annotation, str): - if module is not None: - annotation = _get_type_alias_type(module, annotation) - - # is it a forward ref / in quotes? - if isinstance(annotation, (str, typing.ForwardRef)): - try: - model_name = annotation.__forward_arg__ # type: ignore - except AttributeError: - model_name = annotation - if module is not None: - annotation = _get_model(module, model_name) # type: ignore - - try: - if module and _is_model(annotation): - if rf: - rf._is_model = True - - return functools.partial(_deserialize_model, annotation) # pyright: ignore - except Exception: - pass - - # is it a literal? - try: - if annotation.__origin__ is typing.Literal: # pyright: ignore - return None - except AttributeError: - pass - - # is it optional? - try: - if any(a for a in annotation.__args__ if a == type(None)): # pyright: ignore - if len(annotation.__args__) <= 2: # pyright: ignore - if_obj_deserializer = _get_deserialize_callable_from_annotation( - next(a for a in annotation.__args__ if a != type(None)), module, rf # pyright: ignore - ) - - return functools.partial(_deserialize_with_optional, if_obj_deserializer) - # the type is Optional[Union[...]], we need to remove the None type from the Union - annotation_copy = copy.copy(annotation) - annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a != type(None)] # pyright: ignore - return _get_deserialize_callable_from_annotation(annotation_copy, module, rf) - except AttributeError: - pass - - # is it union? - if getattr(annotation, "__origin__", None) is typing.Union: - # initial ordering is we make `string` the last deserialization option, because it is often them most generic - deserializers = [ - _get_deserialize_callable_from_annotation(arg, module, rf) - for arg in _sorted_annotations(annotation.__args__) # pyright: ignore - ] - - return functools.partial(_deserialize_with_union, deserializers) - - try: - if annotation._name == "Dict": # pyright: ignore - value_deserializer = _get_deserialize_callable_from_annotation( - annotation.__args__[1], module, rf # pyright: ignore - ) - - return functools.partial( - _deserialize_dict, - value_deserializer, - module, - ) - except (AttributeError, IndexError): - pass - try: - if annotation._name in ["List", "Set", "Tuple", "Sequence"]: # pyright: ignore - if len(annotation.__args__) > 1: # pyright: ignore - entry_deserializers = [ - _get_deserialize_callable_from_annotation(dt, module, rf) - for dt in annotation.__args__ # pyright: ignore - ] - return functools.partial(_deserialize_multiple_sequence, entry_deserializers, module) - deserializer = _get_deserialize_callable_from_annotation( - annotation.__args__[0], module, rf # pyright: ignore - ) - - return functools.partial(_deserialize_sequence, deserializer, module) - except (TypeError, IndexError, AttributeError, SyntaxError): - pass - - def _deserialize_default( - deserializer, - obj, - ): - if obj is None: - return obj - try: - return _deserialize_with_callable(deserializer, obj) - except Exception: - pass - return obj - - if get_deserializer(annotation, rf): - return functools.partial(_deserialize_default, get_deserializer(annotation, rf)) - - return functools.partial(_deserialize_default, annotation) - - -def _deserialize_with_callable( - deserializer: typing.Optional[typing.Callable[[typing.Any], typing.Any]], - value: typing.Any, -): # pylint: disable=too-many-return-statements - try: - if value is None or isinstance(value, _Null): - return None - if isinstance(value, ET.Element): - if deserializer is str: - return value.text or "" - if deserializer is int: - return int(value.text) if value.text else None - if deserializer is float: - return float(value.text) if value.text else None - if deserializer is bool: - return value.text == "true" if value.text else None - if deserializer is None: - return value - if deserializer in [int, float, bool]: - return deserializer(value) - if isinstance(deserializer, CaseInsensitiveEnumMeta): - try: - return deserializer(value) - except ValueError: - # for unknown value, return raw value - return value - if isinstance(deserializer, type) and issubclass(deserializer, Model): - return deserializer._deserialize(value, []) - return typing.cast(typing.Callable[[typing.Any], typing.Any], deserializer)(value) - except Exception as e: - raise DeserializationError() from e - - -def _deserialize( - deserializer: typing.Any, - value: typing.Any, - module: typing.Optional[str] = None, - rf: typing.Optional["_RestField"] = None, - format: typing.Optional[str] = None, -) -> typing.Any: - if isinstance(value, PipelineResponse): - value = value.http_response.json() - if rf is None and format: - rf = _RestField(format=format) - if not isinstance(deserializer, functools.partial): - deserializer = _get_deserialize_callable_from_annotation(deserializer, module, rf) - return _deserialize_with_callable(deserializer, value) - - -def _failsafe_deserialize( - deserializer: typing.Any, - value: typing.Any, - module: typing.Optional[str] = None, - rf: typing.Optional["_RestField"] = None, - format: typing.Optional[str] = None, -) -> typing.Any: - try: - return _deserialize(deserializer, value, module, rf, format) - except DeserializationError: - _LOGGER.warning( - "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True - ) - return None - - -def _failsafe_deserialize_xml( - deserializer: typing.Any, - value: typing.Any, -) -> typing.Any: - try: - return _deserialize_xml(deserializer, value) - except DeserializationError: - _LOGGER.warning( - "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True - ) - return None - - -class _RestField: - def __init__( - self, - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - is_discriminator: bool = False, - visibility: typing.Optional[typing.List[str]] = None, - default: typing.Any = _UNSET, - format: typing.Optional[str] = None, - is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, - ): - self._type = type - self._rest_name_input = name - self._module: typing.Optional[str] = None - self._is_discriminator = is_discriminator - self._visibility = visibility - self._is_model = False - self._default = default - self._format = format - self._is_multipart_file_input = is_multipart_file_input - self._xml = xml if xml is not None else {} - - @property - def _class_type(self) -> typing.Any: - return getattr(self._type, "args", [None])[0] - - @property - def _rest_name(self) -> str: - if self._rest_name_input is None: - raise ValueError("Rest name was never set") - return self._rest_name_input - - def __get__(self, obj: Model, type=None): # pylint: disable=redefined-builtin - # by this point, type and rest_name will have a value bc we default - # them in __new__ of the Model class - item = obj.get(self._rest_name) - if item is None: - return item - if self._is_model: - return item - return _deserialize(self._type, _serialize(item, self._format), rf=self) - - def __set__(self, obj: Model, value) -> None: - if value is None: - # we want to wipe out entries if users set attr to None - try: - obj.__delitem__(self._rest_name) - except KeyError: - pass - return - if self._is_model: - if not _is_model(value): - value = _deserialize(self._type, value) - obj.__setitem__(self._rest_name, value) - return - obj.__setitem__(self._rest_name, _serialize(value, self._format)) - - def _get_deserialize_callable_from_annotation( - self, annotation: typing.Any - ) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: - return _get_deserialize_callable_from_annotation(annotation, self._module, self) - - -def rest_field( - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, - default: typing.Any = _UNSET, - format: typing.Optional[str] = None, - is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, -) -> typing.Any: - return _RestField( - name=name, - type=type, - visibility=visibility, - default=default, - format=format, - is_multipart_file_input=is_multipart_file_input, - xml=xml, - ) - - -def rest_discriminator( - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, -) -> typing.Any: - return _RestField(name=name, type=type, is_discriminator=True, visibility=visibility, xml=xml) - - -def serialize_xml(model: Model, exclude_readonly: bool = False) -> str: - """Serialize a model to XML. - - :param Model model: The model to serialize. - :param bool exclude_readonly: Whether to exclude readonly properties. - :returns: The XML representation of the model. - :rtype: str - """ - return ET.tostring(_get_element(model, exclude_readonly), encoding="unicode") # type: ignore - - -def _get_element( - o: typing.Any, - exclude_readonly: bool = False, - parent_meta: typing.Optional[typing.Dict[str, typing.Any]] = None, - wrapped_element: typing.Optional[ET.Element] = None, -) -> typing.Union[ET.Element, typing.List[ET.Element]]: - if _is_model(o): - model_meta = getattr(o, "_xml", {}) - - # if prop is a model, then use the prop element directly, else generate a wrapper of model - if wrapped_element is None: - wrapped_element = _create_xml_element( - model_meta.get("name", o.__class__.__name__), - model_meta.get("prefix"), - model_meta.get("ns"), - ) - - readonly_props = [] - if exclude_readonly: - readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)] - - for k, v in o.items(): - # do not serialize readonly properties - if exclude_readonly and k in readonly_props: - continue - - prop_rest_field = _get_rest_field(o._attr_to_rest_field, k) - if prop_rest_field: - prop_meta = getattr(prop_rest_field, "_xml").copy() - # use the wire name as xml name if no specific name is set - if prop_meta.get("name") is None: - prop_meta["name"] = k - else: - # additional properties will not have rest field, use the wire name as xml name - prop_meta = {"name": k} - - # if no ns for prop, use model's - if prop_meta.get("ns") is None and model_meta.get("ns"): - prop_meta["ns"] = model_meta.get("ns") - prop_meta["prefix"] = model_meta.get("prefix") - - if prop_meta.get("unwrapped", False): - # unwrapped could only set on array - wrapped_element.extend(_get_element(v, exclude_readonly, prop_meta)) - elif prop_meta.get("text", False): - # text could only set on primitive type - wrapped_element.text = _get_primitive_type_value(v) - elif prop_meta.get("attribute", False): - xml_name = prop_meta.get("name", k) - if prop_meta.get("ns"): - ET.register_namespace(prop_meta.get("prefix"), prop_meta.get("ns")) # pyright: ignore - xml_name = "{" + prop_meta.get("ns") + "}" + xml_name # pyright: ignore - # attribute should be primitive type - wrapped_element.set(xml_name, _get_primitive_type_value(v)) - else: - # other wrapped prop element - wrapped_element.append(_get_wrapped_element(v, exclude_readonly, prop_meta)) - return wrapped_element - if isinstance(o, list): - return [_get_element(x, exclude_readonly, parent_meta) for x in o] # type: ignore - if isinstance(o, dict): - result = [] - for k, v in o.items(): - result.append( - _get_wrapped_element( - v, - exclude_readonly, - { - "name": k, - "ns": parent_meta.get("ns") if parent_meta else None, - "prefix": parent_meta.get("prefix") if parent_meta else None, - }, - ) - ) - return result - - # primitive case need to create element based on parent_meta - if parent_meta: - return _get_wrapped_element( - o, - exclude_readonly, - { - "name": parent_meta.get("itemsName", parent_meta.get("name")), - "prefix": parent_meta.get("itemsPrefix", parent_meta.get("prefix")), - "ns": parent_meta.get("itemsNs", parent_meta.get("ns")), - }, - ) - - raise ValueError("Could not serialize value into xml: " + o) - - -def _get_wrapped_element( - v: typing.Any, - exclude_readonly: bool, - meta: typing.Optional[typing.Dict[str, typing.Any]], -) -> ET.Element: - wrapped_element = _create_xml_element( - meta.get("name") if meta else None, meta.get("prefix") if meta else None, meta.get("ns") if meta else None - ) - if isinstance(v, (dict, list)): - wrapped_element.extend(_get_element(v, exclude_readonly, meta)) - elif _is_model(v): - _get_element(v, exclude_readonly, meta, wrapped_element) - else: - wrapped_element.text = _get_primitive_type_value(v) - return wrapped_element - - -def _get_primitive_type_value(v) -> str: - if v is True: - return "true" - if v is False: - return "false" - if isinstance(v, _Null): - return "" - return str(v) - - -def _create_xml_element(tag, prefix=None, ns=None): - if prefix and ns: - ET.register_namespace(prefix, ns) - if ns: - return ET.Element("{" + ns + "}" + tag) - return ET.Element(tag) - - -def _deserialize_xml( - deserializer: typing.Any, - value: str, -) -> typing.Any: - element = ET.fromstring(value) # nosec - return _deserialize(deserializer, element) - - -def _convert_element(e: ET.Element): - # dict case - if len(e.attrib) > 0 or len({child.tag for child in e}) > 1: - dict_result: typing.Dict[str, typing.Any] = {} - for child in e: - if dict_result.get(child.tag) is not None: - if isinstance(dict_result[child.tag], list): - dict_result[child.tag].append(_convert_element(child)) - else: - dict_result[child.tag] = [dict_result[child.tag], _convert_element(child)] - else: - dict_result[child.tag] = _convert_element(child) - dict_result.update(e.attrib) - return dict_result - # array case - if len(e) > 0: - array_result: typing.List[typing.Any] = [] - for child in e: - array_result.append(_convert_element(child)) - return array_result - # primitive case - return e.text diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py index 88d6f825118d..61815308ceca 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_operations/_operations.py @@ -8,7 +8,7 @@ from collections.abc import MutableMapping from io import IOBase import json -from typing import Any, Callable, Dict, IO, Iterator, List, Optional, TypeVar, Union, cast, overload +from typing import Any, Callable, IO, Iterator, Optional, TypeVar, Union, cast, overload import urllib.parse from azure.core import PipelineClient @@ -38,7 +38,7 @@ JSON = MutableMapping[str, Any] T = TypeVar("T") -ClsType = Optional[Callable[[PipelineResponse[HttpRequest, HttpResponse], T, Dict[str, Any]], Any]] +ClsType = Optional[Callable[[PipelineResponse[HttpRequest, HttpResponse], T, dict[str, Any]], Any]] _SERIALIZER = Serializer() _SERIALIZER.client_side_validation = False @@ -193,8 +193,6 @@ def build_deidentification_delete_job_request( # pylint: disable=name-too-long _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2025-07-15-preview")) - accept = _headers.pop("Accept", "application/json") - # Construct URL _url = "/jobs/{name}" path_format_arguments = { @@ -207,7 +205,6 @@ def build_deidentification_delete_job_request( # pylint: disable=name-too-long _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str") # Construct headers - _headers["Accept"] = _SERIALIZER.header("accept", accept, "str") return HttpRequest(method="DELETE", url=_url, params=_params, headers=_headers, **kwargs) @@ -274,6 +271,7 @@ def get_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJob: } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -296,7 +294,7 @@ def get_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJob: ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -342,6 +340,7 @@ def _deidentify_documents_initial( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -363,7 +362,7 @@ def _deidentify_documents_initial( ) response_headers["Operation-Location"] = self._deserialize("str", response.headers.get("Operation-Location")) - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -538,7 +537,7 @@ def _list_jobs_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationJob]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationJob]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -576,7 +575,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -589,7 +591,10 @@ def prepare_request(next_link=None): def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationJob], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationJob], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, iter(list_of_elem) @@ -633,7 +638,7 @@ def _list_job_documents_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -672,7 +677,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -685,7 +693,10 @@ def prepare_request(next_link=None): def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationDocumentDetails], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationDocumentDetails], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, iter(list_of_elem) @@ -748,6 +759,7 @@ def cancel_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJo } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -770,7 +782,7 @@ def cancel_job(self, job_name: str, **kwargs: Any) -> _models.DeidentificationJo ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -936,6 +948,7 @@ def deidentify_text( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -958,7 +971,7 @@ def deidentify_text( ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationResult, response.json()) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py index bb09ee2cbf77..c65684174efc 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_patch.py @@ -6,10 +6,12 @@ Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize """ +import functools from typing import Any, Iterable, List from azure.core.tracing.decorator import distributed_trace from azure.health.deidentification.models import DeidentificationDocumentDetails, DeidentificationJob from ._client import DeidentificationClient as DeidentificationClientGenerated +from ._utils import model_base as _model_base __all__: List[str] = [ "DeidentificationClient", @@ -43,6 +45,53 @@ def list_job_documents(self, job_name: str, **kwargs: Any) -> Iterable[Deidentif return super()._list_job_documents_internal(job_name=job_name, continuation_token_parameter=None, **kwargs) +_MISSING = object() +_IDENTITY_SHIM_MARKER = "_deid_preserves_container_identity" + + +def _preserve_container_identity() -> None: + """Keep list/dict/set fields bound to one container across dict and property access. + + The generated ``_RestField.__get__`` stores a newly deserialized container back into + ``_data``, detaching any container previously returned by ``model["field"]``, so later + mutations through that reference are dropped on serialization. This wrapper copies the + new contents into the original container and keeps it in ``_data``. + + TODO: remove once https://github.com/microsoft/typespec/pull/12359 fixes ``_RestField.__get__`` in the + http-client-python emitter's ``model_base.py.jinja2`` template. + """ + rest_field_cls = _model_base._RestField # pylint: disable=protected-access + original_get: Any = rest_field_cls.__get__ + if getattr(original_get, _IDENTITY_SHIM_MARKER, False): + return + + @functools.wraps(original_get) + def __get__(self: Any, obj: Any, owner: Any = None) -> Any: + if obj is None: + return original_get(self, obj, owner) + data = obj._data # pylint: disable=protected-access + name = self._rest_name # pylint: disable=protected-access + before = data.get(name, _MISSING) + result = original_get(self, obj, owner) + after = data.get(name, _MISSING) + if before is after or result is not after: + return result + if isinstance(before, list) and isinstance(after, list): + before[:] = after + elif (isinstance(before, dict) and isinstance(after, dict)) or ( + isinstance(before, set) and isinstance(after, set) + ): + before.clear() + before.update(after) + else: + return result + data[name] = before + return before + + setattr(__get__, _IDENTITY_SHIM_MARKER, True) + setattr(rest_field_cls, "__get__", __get__) + + def patch_sdk(): """Do not remove from this file. @@ -50,3 +99,4 @@ def patch_sdk(): you can't accomplish using the techniques described in https://aka.ms/azsdk/python/dpcodegen/python/customize """ + _preserve_container_identity() diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_serialization.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_serialization.py deleted file mode 100644 index eb86ea23c965..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_serialization.py +++ /dev/null @@ -1,2032 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression,too-many-lines -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -# pyright: reportUnnecessaryTypeIgnoreComment=false - -from base64 import b64decode, b64encode -import calendar -import datetime -import decimal -import email -from enum import Enum -import json -import logging -import re -import sys -import codecs -from typing import ( - Dict, - Any, - cast, - Optional, - Union, - AnyStr, - IO, - Mapping, - Callable, - MutableMapping, - List, -) - -try: - from urllib import quote # type: ignore -except ImportError: - from urllib.parse import quote -import xml.etree.ElementTree as ET - -import isodate # type: ignore -from typing_extensions import Self - -from azure.core.exceptions import DeserializationError, SerializationError -from azure.core.serialization import NULL as CoreNull - -_BOM = codecs.BOM_UTF8.decode(encoding="utf-8") - -JSON = MutableMapping[str, Any] - - -class RawDeserializer: - - # Accept "text" because we're open minded people... - JSON_REGEXP = re.compile(r"^(application|text)/([a-z+.]+\+)?json$") - - # Name used in context - CONTEXT_NAME = "deserialized_data" - - @classmethod - def deserialize_from_text(cls, data: Optional[Union[AnyStr, IO]], content_type: Optional[str] = None) -> Any: - """Decode data according to content-type. - - Accept a stream of data as well, but will be load at once in memory for now. - - If no content-type, will return the string version (not bytes, not stream) - - :param data: Input, could be bytes or stream (will be decoded with UTF8) or text - :type data: str or bytes or IO - :param str content_type: The content type. - :return: The deserialized data. - :rtype: object - """ - if hasattr(data, "read"): - # Assume a stream - data = cast(IO, data).read() - - if isinstance(data, bytes): - data_as_str = data.decode(encoding="utf-8-sig") - else: - # Explain to mypy the correct type. - data_as_str = cast(str, data) - - # Remove Byte Order Mark if present in string - data_as_str = data_as_str.lstrip(_BOM) - - if content_type is None: - return data - - if cls.JSON_REGEXP.match(content_type): - try: - return json.loads(data_as_str) - except ValueError as err: - raise DeserializationError("JSON is invalid: {}".format(err), err) from err - elif "xml" in (content_type or []): - try: - - try: - if isinstance(data, unicode): # type: ignore - # If I'm Python 2.7 and unicode XML will scream if I try a "fromstring" on unicode string - data_as_str = data_as_str.encode(encoding="utf-8") # type: ignore - except NameError: - pass - - return ET.fromstring(data_as_str) # nosec - except ET.ParseError as err: - # It might be because the server has an issue, and returned JSON with - # content-type XML.... - # So let's try a JSON load, and if it's still broken - # let's flow the initial exception - def _json_attemp(data): - try: - return True, json.loads(data) - except ValueError: - return False, None # Don't care about this one - - success, json_result = _json_attemp(data) - if success: - return json_result - # If i'm here, it's not JSON, it's not XML, let's scream - # and raise the last context in this block (the XML exception) - # The function hack is because Py2.7 messes up with exception - # context otherwise. - _LOGGER.critical("Wasn't XML not JSON, failing") - raise DeserializationError("XML is invalid") from err - elif content_type.startswith("text/"): - return data_as_str - raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) - - @classmethod - def deserialize_from_http_generics(cls, body_bytes: Optional[Union[AnyStr, IO]], headers: Mapping) -> Any: - """Deserialize from HTTP response. - - Use bytes and headers to NOT use any requests/aiohttp or whatever - specific implementation. - Headers will tested for "content-type" - - :param bytes body_bytes: The body of the response. - :param dict headers: The headers of the response. - :returns: The deserialized data. - :rtype: object - """ - # Try to use content-type from headers if available - content_type = None - if "content-type" in headers: - content_type = headers["content-type"].split(";")[0].strip().lower() - # Ouch, this server did not declare what it sent... - # Let's guess it's JSON... - # Also, since Autorest was considering that an empty body was a valid JSON, - # need that test as well.... - else: - content_type = "application/json" - - if body_bytes: - return cls.deserialize_from_text(body_bytes, content_type) - return None - - -_LOGGER = logging.getLogger(__name__) - -try: - _long_type = long # type: ignore -except NameError: - _long_type = int - -TZ_UTC = datetime.timezone.utc - -_FLATTEN = re.compile(r"(? None: - self.additional_properties: Optional[Dict[str, Any]] = {} - for k in kwargs: # pylint: disable=consider-using-dict-items - if k not in self._attribute_map: - _LOGGER.warning("%s is not a known attribute of class %s and will be ignored", k, self.__class__) - elif k in self._validation and self._validation[k].get("readonly", False): - _LOGGER.warning("Readonly attribute %s will be ignored in class %s", k, self.__class__) - else: - setattr(self, k, kwargs[k]) - - def __eq__(self, other: Any) -> bool: - """Compare objects by comparing all attributes. - - :param object other: The object to compare - :returns: True if objects are equal - :rtype: bool - """ - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return False - - def __ne__(self, other: Any) -> bool: - """Compare objects by comparing all attributes. - - :param object other: The object to compare - :returns: True if objects are not equal - :rtype: bool - """ - return not self.__eq__(other) - - def __str__(self) -> str: - return str(self.__dict__) - - @classmethod - def enable_additional_properties_sending(cls) -> None: - cls._attribute_map["additional_properties"] = {"key": "", "type": "{object}"} - - @classmethod - def is_xml_model(cls) -> bool: - try: - cls._xml_map # type: ignore - except AttributeError: - return False - return True - - @classmethod - def _create_xml_node(cls): - """Create XML node. - - :returns: The XML node - :rtype: xml.etree.ElementTree.Element - """ - try: - xml_map = cls._xml_map # type: ignore - except AttributeError: - xml_map = {} - - return _create_xml_node(xml_map.get("name", cls.__name__), xml_map.get("prefix", None), xml_map.get("ns", None)) - - def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> JSON: - """Return the JSON that would be sent to server from this model. - - This is an alias to `as_dict(full_restapi_key_transformer, keep_readonly=False)`. - - If you want XML serialization, you can pass the kwargs is_xml=True. - - :param bool keep_readonly: If you want to serialize the readonly attributes - :returns: A dict JSON compatible object - :rtype: dict - """ - serializer = Serializer(self._infer_class_models()) - return serializer._serialize( # type: ignore # pylint: disable=protected-access - self, keep_readonly=keep_readonly, **kwargs - ) - - def as_dict( - self, - keep_readonly: bool = True, - key_transformer: Callable[[str, Dict[str, Any], Any], Any] = attribute_transformer, - **kwargs: Any - ) -> JSON: - """Return a dict that can be serialized using json.dump. - - Advanced usage might optionally use a callback as parameter: - - .. code::python - - def my_key_transformer(key, attr_desc, value): - return key - - Key is the attribute name used in Python. Attr_desc - is a dict of metadata. Currently contains 'type' with the - msrest type and 'key' with the RestAPI encoded key. - Value is the current value in this object. - - The string returned will be used to serialize the key. - If the return type is a list, this is considered hierarchical - result dict. - - See the three examples in this file: - - - attribute_transformer - - full_restapi_key_transformer - - last_restapi_key_transformer - - If you want XML serialization, you can pass the kwargs is_xml=True. - - :param bool keep_readonly: If you want to serialize the readonly attributes - :param function key_transformer: A key transformer function. - :returns: A dict JSON compatible object - :rtype: dict - """ - serializer = Serializer(self._infer_class_models()) - return serializer._serialize( # type: ignore # pylint: disable=protected-access - self, key_transformer=key_transformer, keep_readonly=keep_readonly, **kwargs - ) - - @classmethod - def _infer_class_models(cls): - try: - str_models = cls.__module__.rsplit(".", 1)[0] - models = sys.modules[str_models] - client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)} - if cls.__name__ not in client_models: - raise ValueError("Not Autorest generated code") - except Exception: # pylint: disable=broad-exception-caught - # Assume it's not Autorest generated (tests?). Add ourselves as dependencies. - client_models = {cls.__name__: cls} - return client_models - - @classmethod - def deserialize(cls, data: Any, content_type: Optional[str] = None) -> Self: - """Parse a str using the RestAPI syntax and return a model. - - :param str data: A str using RestAPI structure. JSON by default. - :param str content_type: JSON by default, set application/xml if XML. - :returns: An instance of this model - :raises DeserializationError: if something went wrong - :rtype: Self - """ - deserializer = Deserializer(cls._infer_class_models()) - return deserializer(cls.__name__, data, content_type=content_type) # type: ignore - - @classmethod - def from_dict( - cls, - data: Any, - key_extractors: Optional[Callable[[str, Dict[str, Any], Any], Any]] = None, - content_type: Optional[str] = None, - ) -> Self: - """Parse a dict using given key extractor return a model. - - By default consider key - extractors (rest_key_case_insensitive_extractor, attribute_key_case_insensitive_extractor - and last_rest_key_case_insensitive_extractor) - - :param dict data: A dict using RestAPI structure - :param function key_extractors: A key extractor function. - :param str content_type: JSON by default, set application/xml if XML. - :returns: An instance of this model - :raises DeserializationError: if something went wrong - :rtype: Self - """ - deserializer = Deserializer(cls._infer_class_models()) - deserializer.key_extractors = ( # type: ignore - [ # type: ignore - attribute_key_case_insensitive_extractor, - rest_key_case_insensitive_extractor, - last_rest_key_case_insensitive_extractor, - ] - if key_extractors is None - else key_extractors - ) - return deserializer(cls.__name__, data, content_type=content_type) # type: ignore - - @classmethod - def _flatten_subtype(cls, key, objects): - if "_subtype_map" not in cls.__dict__: - return {} - result = dict(cls._subtype_map[key]) - for valuetype in cls._subtype_map[key].values(): - result.update(objects[valuetype]._flatten_subtype(key, objects)) # pylint: disable=protected-access - return result - - @classmethod - def _classify(cls, response, objects): - """Check the class _subtype_map for any child classes. - We want to ignore any inherited _subtype_maps. - - :param dict response: The initial data - :param dict objects: The class objects - :returns: The class to be used - :rtype: class - """ - for subtype_key in cls.__dict__.get("_subtype_map", {}).keys(): - subtype_value = None - - if not isinstance(response, ET.Element): - rest_api_response_key = cls._get_rest_key_parts(subtype_key)[-1] - subtype_value = response.get(rest_api_response_key, None) or response.get(subtype_key, None) - else: - subtype_value = xml_key_extractor(subtype_key, cls._attribute_map[subtype_key], response) - if subtype_value: - # Try to match base class. Can be class name only - # (bug to fix in Autorest to support x-ms-discriminator-name) - if cls.__name__ == subtype_value: - return cls - flatten_mapping_type = cls._flatten_subtype(subtype_key, objects) - try: - return objects[flatten_mapping_type[subtype_value]] # type: ignore - except KeyError: - _LOGGER.warning( - "Subtype value %s has no mapping, use base class %s.", - subtype_value, - cls.__name__, - ) - break - else: - _LOGGER.warning("Discriminator %s is absent or null, use base class %s.", subtype_key, cls.__name__) - break - return cls - - @classmethod - def _get_rest_key_parts(cls, attr_key): - """Get the RestAPI key of this attr, split it and decode part - :param str attr_key: Attribute key must be in attribute_map. - :returns: A list of RestAPI part - :rtype: list - """ - rest_split_key = _FLATTEN.split(cls._attribute_map[attr_key]["key"]) - return [_decode_attribute_map_key(key_part) for key_part in rest_split_key] - - -def _decode_attribute_map_key(key): - """This decode a key in an _attribute_map to the actual key we want to look at - inside the received data. - - :param str key: A key string from the generated code - :returns: The decoded key - :rtype: str - """ - return key.replace("\\.", ".") - - -class Serializer: # pylint: disable=too-many-public-methods - """Request object model serializer.""" - - basic_types = {str: "str", int: "int", bool: "bool", float: "float"} - - _xml_basic_types_serializers = {"bool": lambda x: str(x).lower()} - days = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 4: "Fri", 5: "Sat", 6: "Sun"} - months = { - 1: "Jan", - 2: "Feb", - 3: "Mar", - 4: "Apr", - 5: "May", - 6: "Jun", - 7: "Jul", - 8: "Aug", - 9: "Sep", - 10: "Oct", - 11: "Nov", - 12: "Dec", - } - validation = { - "min_length": lambda x, y: len(x) < y, - "max_length": lambda x, y: len(x) > y, - "minimum": lambda x, y: x < y, - "maximum": lambda x, y: x > y, - "minimum_ex": lambda x, y: x <= y, - "maximum_ex": lambda x, y: x >= y, - "min_items": lambda x, y: len(x) < y, - "max_items": lambda x, y: len(x) > y, - "pattern": lambda x, y: not re.match(y, x, re.UNICODE), - "unique": lambda x, y: len(x) != len(set(x)), - "multiple": lambda x, y: x % y != 0, - } - - def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: - self.serialize_type = { - "iso-8601": Serializer.serialize_iso, - "rfc-1123": Serializer.serialize_rfc, - "unix-time": Serializer.serialize_unix, - "duration": Serializer.serialize_duration, - "date": Serializer.serialize_date, - "time": Serializer.serialize_time, - "decimal": Serializer.serialize_decimal, - "long": Serializer.serialize_long, - "bytearray": Serializer.serialize_bytearray, - "base64": Serializer.serialize_base64, - "object": self.serialize_object, - "[]": self.serialize_iter, - "{}": self.serialize_dict, - } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} - self.key_transformer = full_restapi_key_transformer - self.client_side_validation = True - - def _serialize( # pylint: disable=too-many-nested-blocks, too-many-branches, too-many-statements, too-many-locals - self, target_obj, data_type=None, **kwargs - ): - """Serialize data into a string according to type. - - :param object target_obj: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str, dict - :raises SerializationError: if serialization fails. - :returns: The serialized data. - """ - key_transformer = kwargs.get("key_transformer", self.key_transformer) - keep_readonly = kwargs.get("keep_readonly", False) - if target_obj is None: - return None - - attr_name = None - class_name = target_obj.__class__.__name__ - - if data_type: - return self.serialize_data(target_obj, data_type, **kwargs) - - if not hasattr(target_obj, "_attribute_map"): - data_type = type(target_obj).__name__ - if data_type in self.basic_types.values(): - return self.serialize_data(target_obj, data_type, **kwargs) - - # Force "is_xml" kwargs if we detect a XML model - try: - is_xml_model_serialization = kwargs["is_xml"] - except KeyError: - is_xml_model_serialization = kwargs.setdefault("is_xml", target_obj.is_xml_model()) - - serialized = {} - if is_xml_model_serialization: - serialized = target_obj._create_xml_node() # pylint: disable=protected-access - try: - attributes = target_obj._attribute_map # pylint: disable=protected-access - for attr, attr_desc in attributes.items(): - attr_name = attr - if not keep_readonly and target_obj._validation.get( # pylint: disable=protected-access - attr_name, {} - ).get("readonly", False): - continue - - if attr_name == "additional_properties" and attr_desc["key"] == "": - if target_obj.additional_properties is not None: - serialized.update(target_obj.additional_properties) - continue - try: - - orig_attr = getattr(target_obj, attr) - if is_xml_model_serialization: - pass # Don't provide "transformer" for XML for now. Keep "orig_attr" - else: # JSON - keys, orig_attr = key_transformer(attr, attr_desc.copy(), orig_attr) - keys = keys if isinstance(keys, list) else [keys] - - kwargs["serialization_ctxt"] = attr_desc - new_attr = self.serialize_data(orig_attr, attr_desc["type"], **kwargs) - - if is_xml_model_serialization: - xml_desc = attr_desc.get("xml", {}) - xml_name = xml_desc.get("name", attr_desc["key"]) - xml_prefix = xml_desc.get("prefix", None) - xml_ns = xml_desc.get("ns", None) - if xml_desc.get("attr", False): - if xml_ns: - ET.register_namespace(xml_prefix, xml_ns) - xml_name = "{{{}}}{}".format(xml_ns, xml_name) - serialized.set(xml_name, new_attr) # type: ignore - continue - if xml_desc.get("text", False): - serialized.text = new_attr # type: ignore - continue - if isinstance(new_attr, list): - serialized.extend(new_attr) # type: ignore - elif isinstance(new_attr, ET.Element): - # If the down XML has no XML/Name, - # we MUST replace the tag with the local tag. But keeping the namespaces. - if "name" not in getattr(orig_attr, "_xml_map", {}): - splitted_tag = new_attr.tag.split("}") - if len(splitted_tag) == 2: # Namespace - new_attr.tag = "}".join([splitted_tag[0], xml_name]) - else: - new_attr.tag = xml_name - serialized.append(new_attr) # type: ignore - else: # That's a basic type - # Integrate namespace if necessary - local_node = _create_xml_node(xml_name, xml_prefix, xml_ns) - local_node.text = str(new_attr) - serialized.append(local_node) # type: ignore - else: # JSON - for k in reversed(keys): # type: ignore - new_attr = {k: new_attr} - - _new_attr = new_attr - _serialized = serialized - for k in keys: # type: ignore - if k not in _serialized: - _serialized.update(_new_attr) # type: ignore - _new_attr = _new_attr[k] # type: ignore - _serialized = _serialized[k] - except ValueError as err: - if isinstance(err, SerializationError): - raise - - except (AttributeError, KeyError, TypeError) as err: - msg = "Attribute {} in object {} cannot be serialized.\n{}".format(attr_name, class_name, str(target_obj)) - raise SerializationError(msg) from err - return serialized - - def body(self, data, data_type, **kwargs): - """Serialize data intended for a request body. - - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: dict - :raises SerializationError: if serialization fails. - :raises ValueError: if data is None - :returns: The serialized request body - """ - - # Just in case this is a dict - internal_data_type_str = data_type.strip("[]{}") - internal_data_type = self.dependencies.get(internal_data_type_str, None) - try: - is_xml_model_serialization = kwargs["is_xml"] - except KeyError: - if internal_data_type and issubclass(internal_data_type, Model): - is_xml_model_serialization = kwargs.setdefault("is_xml", internal_data_type.is_xml_model()) - else: - is_xml_model_serialization = False - if internal_data_type and not isinstance(internal_data_type, Enum): - try: - deserializer = Deserializer(self.dependencies) - # Since it's on serialization, it's almost sure that format is not JSON REST - # We're not able to deal with additional properties for now. - deserializer.additional_properties_detection = False - if is_xml_model_serialization: - deserializer.key_extractors = [ # type: ignore - attribute_key_case_insensitive_extractor, - ] - else: - deserializer.key_extractors = [ - rest_key_case_insensitive_extractor, - attribute_key_case_insensitive_extractor, - last_rest_key_case_insensitive_extractor, - ] - data = deserializer._deserialize(data_type, data) # pylint: disable=protected-access - except DeserializationError as err: - raise SerializationError("Unable to build a model: " + str(err)) from err - - return self._serialize(data, data_type, **kwargs) - - def url(self, name, data, data_type, **kwargs): - """Serialize data intended for a URL path. - - :param str name: The name of the URL path parameter. - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str - :returns: The serialized URL path - :raises TypeError: if serialization fails. - :raises ValueError: if data is None - """ - try: - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - - if kwargs.get("skip_quote") is True: - output = str(output) - output = output.replace("{", quote("{")).replace("}", quote("}")) - else: - output = quote(str(output), safe="") - except SerializationError as exc: - raise TypeError("{} must be type {}.".format(name, data_type)) from exc - return output - - def query(self, name, data, data_type, **kwargs): - """Serialize data intended for a URL query. - - :param str name: The name of the query parameter. - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str, list - :raises TypeError: if serialization fails. - :raises ValueError: if data is None - :returns: The serialized query parameter - """ - try: - # Treat the list aside, since we don't want to encode the div separator - if data_type.startswith("["): - internal_data_type = data_type[1:-1] - do_quote = not kwargs.get("skip_quote", False) - return self.serialize_iter(data, internal_data_type, do_quote=do_quote, **kwargs) - - # Not a list, regular serialization - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - if kwargs.get("skip_quote") is True: - output = str(output) - else: - output = quote(str(output), safe="") - except SerializationError as exc: - raise TypeError("{} must be type {}.".format(name, data_type)) from exc - return str(output) - - def header(self, name, data, data_type, **kwargs): - """Serialize data intended for a request header. - - :param str name: The name of the header. - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str - :raises TypeError: if serialization fails. - :raises ValueError: if data is None - :returns: The serialized header - """ - try: - if data_type in ["[str]"]: - data = ["" if d is None else d for d in data] - - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - except SerializationError as exc: - raise TypeError("{} must be type {}.".format(name, data_type)) from exc - return str(output) - - def serialize_data(self, data, data_type, **kwargs): - """Serialize generic data according to supplied data type. - - :param object data: The data to be serialized. - :param str data_type: The type to be serialized from. - :raises AttributeError: if required data is None. - :raises ValueError: if data is None - :raises SerializationError: if serialization fails. - :returns: The serialized data. - :rtype: str, int, float, bool, dict, list - """ - if data is None: - raise ValueError("No value for given attribute") - - try: - if data is CoreNull: - return None - if data_type in self.basic_types.values(): - return self.serialize_basic(data, data_type, **kwargs) - - if data_type in self.serialize_type: - return self.serialize_type[data_type](data, **kwargs) - - # If dependencies is empty, try with current data class - # It has to be a subclass of Enum anyway - enum_type = self.dependencies.get(data_type, data.__class__) - if issubclass(enum_type, Enum): - return Serializer.serialize_enum(data, enum_obj=enum_type) - - iter_type = data_type[0] + data_type[-1] - if iter_type in self.serialize_type: - return self.serialize_type[iter_type](data, data_type[1:-1], **kwargs) - - except (ValueError, TypeError) as err: - msg = "Unable to serialize value: {!r} as type: {!r}." - raise SerializationError(msg.format(data, data_type)) from err - return self._serialize(data, **kwargs) - - @classmethod - def _get_custom_serializers(cls, data_type, **kwargs): # pylint: disable=inconsistent-return-statements - custom_serializer = kwargs.get("basic_types_serializers", {}).get(data_type) - if custom_serializer: - return custom_serializer - if kwargs.get("is_xml", False): - return cls._xml_basic_types_serializers.get(data_type) - - @classmethod - def serialize_basic(cls, data, data_type, **kwargs): - """Serialize basic builting data type. - Serializes objects to str, int, float or bool. - - Possible kwargs: - - basic_types_serializers dict[str, callable] : If set, use the callable as serializer - - is_xml bool : If set, use xml_basic_types_serializers - - :param obj data: Object to be serialized. - :param str data_type: Type of object in the iterable. - :rtype: str, int, float, bool - :return: serialized object - """ - custom_serializer = cls._get_custom_serializers(data_type, **kwargs) - if custom_serializer: - return custom_serializer(data) - if data_type == "str": - return cls.serialize_unicode(data) - return eval(data_type)(data) # nosec # pylint: disable=eval-used - - @classmethod - def serialize_unicode(cls, data): - """Special handling for serializing unicode strings in Py2. - Encode to UTF-8 if unicode, otherwise handle as a str. - - :param str data: Object to be serialized. - :rtype: str - :return: serialized object - """ - try: # If I received an enum, return its value - return data.value - except AttributeError: - pass - - try: - if isinstance(data, unicode): # type: ignore - # Don't change it, JSON and XML ElementTree are totally able - # to serialize correctly u'' strings - return data - except NameError: - return str(data) - return str(data) - - def serialize_iter(self, data, iter_type, div=None, **kwargs): - """Serialize iterable. - - Supported kwargs: - - serialization_ctxt dict : The current entry of _attribute_map, or same format. - serialization_ctxt['type'] should be same as data_type. - - is_xml bool : If set, serialize as XML - - :param list data: Object to be serialized. - :param str iter_type: Type of object in the iterable. - :param str div: If set, this str will be used to combine the elements - in the iterable into a combined string. Default is 'None'. - Defaults to False. - :rtype: list, str - :return: serialized iterable - """ - if isinstance(data, str): - raise SerializationError("Refuse str type as a valid iter type.") - - serialization_ctxt = kwargs.get("serialization_ctxt", {}) - is_xml = kwargs.get("is_xml", False) - - serialized = [] - for d in data: - try: - serialized.append(self.serialize_data(d, iter_type, **kwargs)) - except ValueError as err: - if isinstance(err, SerializationError): - raise - serialized.append(None) - - if kwargs.get("do_quote", False): - serialized = ["" if s is None else quote(str(s), safe="") for s in serialized] - - if div: - serialized = ["" if s is None else str(s) for s in serialized] - serialized = div.join(serialized) - - if "xml" in serialization_ctxt or is_xml: - # XML serialization is more complicated - xml_desc = serialization_ctxt.get("xml", {}) - xml_name = xml_desc.get("name") - if not xml_name: - xml_name = serialization_ctxt["key"] - - # Create a wrap node if necessary (use the fact that Element and list have "append") - is_wrapped = xml_desc.get("wrapped", False) - node_name = xml_desc.get("itemsName", xml_name) - if is_wrapped: - final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - else: - final_result = [] - # All list elements to "local_node" - for el in serialized: - if isinstance(el, ET.Element): - el_node = el - else: - el_node = _create_xml_node(node_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - if el is not None: # Otherwise it writes "None" :-p - el_node.text = str(el) - final_result.append(el_node) - return final_result - return serialized - - def serialize_dict(self, attr, dict_type, **kwargs): - """Serialize a dictionary of objects. - - :param dict attr: Object to be serialized. - :param str dict_type: Type of object in the dictionary. - :rtype: dict - :return: serialized dictionary - """ - serialization_ctxt = kwargs.get("serialization_ctxt", {}) - serialized = {} - for key, value in attr.items(): - try: - serialized[self.serialize_unicode(key)] = self.serialize_data(value, dict_type, **kwargs) - except ValueError as err: - if isinstance(err, SerializationError): - raise - serialized[self.serialize_unicode(key)] = None - - if "xml" in serialization_ctxt: - # XML serialization is more complicated - xml_desc = serialization_ctxt["xml"] - xml_name = xml_desc["name"] - - final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - for key, value in serialized.items(): - ET.SubElement(final_result, key).text = value - return final_result - - return serialized - - def serialize_object(self, attr, **kwargs): # pylint: disable=too-many-return-statements - """Serialize a generic object. - This will be handled as a dictionary. If object passed in is not - a basic type (str, int, float, dict, list) it will simply be - cast to str. - - :param dict attr: Object to be serialized. - :rtype: dict or str - :return: serialized object - """ - if attr is None: - return None - if isinstance(attr, ET.Element): - return attr - obj_type = type(attr) - if obj_type in self.basic_types: - return self.serialize_basic(attr, self.basic_types[obj_type], **kwargs) - if obj_type is _long_type: - return self.serialize_long(attr) - if obj_type is str: - return self.serialize_unicode(attr) - if obj_type is datetime.datetime: - return self.serialize_iso(attr) - if obj_type is datetime.date: - return self.serialize_date(attr) - if obj_type is datetime.time: - return self.serialize_time(attr) - if obj_type is datetime.timedelta: - return self.serialize_duration(attr) - if obj_type is decimal.Decimal: - return self.serialize_decimal(attr) - - # If it's a model or I know this dependency, serialize as a Model - if obj_type in self.dependencies.values() or isinstance(attr, Model): - return self._serialize(attr) - - if obj_type == dict: - serialized = {} - for key, value in attr.items(): - try: - serialized[self.serialize_unicode(key)] = self.serialize_object(value, **kwargs) - except ValueError: - serialized[self.serialize_unicode(key)] = None - return serialized - - if obj_type == list: - serialized = [] - for obj in attr: - try: - serialized.append(self.serialize_object(obj, **kwargs)) - except ValueError: - pass - return serialized - return str(attr) - - @staticmethod - def serialize_enum(attr, enum_obj=None): - try: - result = attr.value - except AttributeError: - result = attr - try: - enum_obj(result) # type: ignore - return result - except ValueError as exc: - for enum_value in enum_obj: # type: ignore - if enum_value.value.lower() == str(attr).lower(): - return enum_value.value - error = "{!r} is not valid value for enum {!r}" - raise SerializationError(error.format(attr, enum_obj)) from exc - - @staticmethod - def serialize_bytearray(attr, **kwargs): # pylint: disable=unused-argument - """Serialize bytearray into base-64 string. - - :param str attr: Object to be serialized. - :rtype: str - :return: serialized base64 - """ - return b64encode(attr).decode() - - @staticmethod - def serialize_base64(attr, **kwargs): # pylint: disable=unused-argument - """Serialize str into base-64 string. - - :param str attr: Object to be serialized. - :rtype: str - :return: serialized base64 - """ - encoded = b64encode(attr).decode("ascii") - return encoded.strip("=").replace("+", "-").replace("/", "_") - - @staticmethod - def serialize_decimal(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Decimal object to float. - - :param decimal attr: Object to be serialized. - :rtype: float - :return: serialized decimal - """ - return float(attr) - - @staticmethod - def serialize_long(attr, **kwargs): # pylint: disable=unused-argument - """Serialize long (Py2) or int (Py3). - - :param int attr: Object to be serialized. - :rtype: int/long - :return: serialized long - """ - return _long_type(attr) - - @staticmethod - def serialize_date(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Date object into ISO-8601 formatted string. - - :param Date attr: Object to be serialized. - :rtype: str - :return: serialized date - """ - if isinstance(attr, str): - attr = isodate.parse_date(attr) - t = "{:04}-{:02}-{:02}".format(attr.year, attr.month, attr.day) - return t - - @staticmethod - def serialize_time(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Time object into ISO-8601 formatted string. - - :param datetime.time attr: Object to be serialized. - :rtype: str - :return: serialized time - """ - if isinstance(attr, str): - attr = isodate.parse_time(attr) - t = "{:02}:{:02}:{:02}".format(attr.hour, attr.minute, attr.second) - if attr.microsecond: - t += ".{:02}".format(attr.microsecond) - return t - - @staticmethod - def serialize_duration(attr, **kwargs): # pylint: disable=unused-argument - """Serialize TimeDelta object into ISO-8601 formatted string. - - :param TimeDelta attr: Object to be serialized. - :rtype: str - :return: serialized duration - """ - if isinstance(attr, str): - attr = isodate.parse_duration(attr) - return isodate.duration_isoformat(attr) - - @staticmethod - def serialize_rfc(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Datetime object into RFC-1123 formatted string. - - :param Datetime attr: Object to be serialized. - :rtype: str - :raises TypeError: if format invalid. - :return: serialized rfc - """ - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - utc = attr.utctimetuple() - except AttributeError as exc: - raise TypeError("RFC1123 object must be valid Datetime object.") from exc - - return "{}, {:02} {} {:04} {:02}:{:02}:{:02} GMT".format( - Serializer.days[utc.tm_wday], - utc.tm_mday, - Serializer.months[utc.tm_mon], - utc.tm_year, - utc.tm_hour, - utc.tm_min, - utc.tm_sec, - ) - - @staticmethod - def serialize_iso(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Datetime object into ISO-8601 formatted string. - - :param Datetime attr: Object to be serialized. - :rtype: str - :raises SerializationError: if format invalid. - :return: serialized iso - """ - if isinstance(attr, str): - attr = isodate.parse_datetime(attr) - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - utc = attr.utctimetuple() - if utc.tm_year > 9999 or utc.tm_year < 1: - raise OverflowError("Hit max or min date") - - microseconds = str(attr.microsecond).rjust(6, "0").rstrip("0").ljust(3, "0") - if microseconds: - microseconds = "." + microseconds - date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format( - utc.tm_year, utc.tm_mon, utc.tm_mday, utc.tm_hour, utc.tm_min, utc.tm_sec - ) - return date + microseconds + "Z" - except (ValueError, OverflowError) as err: - msg = "Unable to serialize datetime object." - raise SerializationError(msg) from err - except AttributeError as err: - msg = "ISO-8601 object must be valid Datetime object." - raise TypeError(msg) from err - - @staticmethod - def serialize_unix(attr, **kwargs): # pylint: disable=unused-argument - """Serialize Datetime object into IntTime format. - This is represented as seconds. - - :param Datetime attr: Object to be serialized. - :rtype: int - :raises SerializationError: if format invalid - :return: serialied unix - """ - if isinstance(attr, int): - return attr - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - return int(calendar.timegm(attr.utctimetuple())) - except AttributeError as exc: - raise TypeError("Unix time object must be valid Datetime object.") from exc - - -def rest_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argument - key = attr_desc["key"] - working_data = data - - while "." in key: - # Need the cast, as for some reasons "split" is typed as list[str | Any] - dict_keys = cast(List[str], _FLATTEN.split(key)) - if len(dict_keys) == 1: - key = _decode_attribute_map_key(dict_keys[0]) - break - working_key = _decode_attribute_map_key(dict_keys[0]) - working_data = working_data.get(working_key, data) - if working_data is None: - # If at any point while following flatten JSON path see None, it means - # that all properties under are None as well - return None - key = ".".join(dict_keys[1:]) - - return working_data.get(key) - - -def rest_key_case_insensitive_extractor( # pylint: disable=unused-argument, inconsistent-return-statements - attr, attr_desc, data -): - key = attr_desc["key"] - working_data = data - - while "." in key: - dict_keys = _FLATTEN.split(key) - if len(dict_keys) == 1: - key = _decode_attribute_map_key(dict_keys[0]) - break - working_key = _decode_attribute_map_key(dict_keys[0]) - working_data = attribute_key_case_insensitive_extractor(working_key, None, working_data) - if working_data is None: - # If at any point while following flatten JSON path see None, it means - # that all properties under are None as well - return None - key = ".".join(dict_keys[1:]) - - if working_data: - return attribute_key_case_insensitive_extractor(key, None, working_data) - - -def last_rest_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argument - """Extract the attribute in "data" based on the last part of the JSON path key. - - :param str attr: The attribute to extract - :param dict attr_desc: The attribute description - :param dict data: The data to extract from - :rtype: object - :returns: The extracted attribute - """ - key = attr_desc["key"] - dict_keys = _FLATTEN.split(key) - return attribute_key_extractor(dict_keys[-1], None, data) - - -def last_rest_key_case_insensitive_extractor(attr, attr_desc, data): # pylint: disable=unused-argument - """Extract the attribute in "data" based on the last part of the JSON path key. - - This is the case insensitive version of "last_rest_key_extractor" - :param str attr: The attribute to extract - :param dict attr_desc: The attribute description - :param dict data: The data to extract from - :rtype: object - :returns: The extracted attribute - """ - key = attr_desc["key"] - dict_keys = _FLATTEN.split(key) - return attribute_key_case_insensitive_extractor(dict_keys[-1], None, data) - - -def attribute_key_extractor(attr, _, data): - return data.get(attr) - - -def attribute_key_case_insensitive_extractor(attr, _, data): - found_key = None - lower_attr = attr.lower() - for key in data: - if lower_attr == key.lower(): - found_key = key - break - - return data.get(found_key) - - -def _extract_name_from_internal_type(internal_type): - """Given an internal type XML description, extract correct XML name with namespace. - - :param dict internal_type: An model type - :rtype: tuple - :returns: A tuple XML name + namespace dict - """ - internal_type_xml_map = getattr(internal_type, "_xml_map", {}) - xml_name = internal_type_xml_map.get("name", internal_type.__name__) - xml_ns = internal_type_xml_map.get("ns", None) - if xml_ns: - xml_name = "{{{}}}{}".format(xml_ns, xml_name) - return xml_name - - -def xml_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argument,too-many-return-statements - if isinstance(data, dict): - return None - - # Test if this model is XML ready first - if not isinstance(data, ET.Element): - return None - - xml_desc = attr_desc.get("xml", {}) - xml_name = xml_desc.get("name", attr_desc["key"]) - - # Look for a children - is_iter_type = attr_desc["type"].startswith("[") - is_wrapped = xml_desc.get("wrapped", False) - internal_type = attr_desc.get("internalType", None) - internal_type_xml_map = getattr(internal_type, "_xml_map", {}) - - # Integrate namespace if necessary - xml_ns = xml_desc.get("ns", internal_type_xml_map.get("ns", None)) - if xml_ns: - xml_name = "{{{}}}{}".format(xml_ns, xml_name) - - # If it's an attribute, that's simple - if xml_desc.get("attr", False): - return data.get(xml_name) - - # If it's x-ms-text, that's simple too - if xml_desc.get("text", False): - return data.text - - # Scenario where I take the local name: - # - Wrapped node - # - Internal type is an enum (considered basic types) - # - Internal type has no XML/Name node - if is_wrapped or (internal_type and (issubclass(internal_type, Enum) or "name" not in internal_type_xml_map)): - children = data.findall(xml_name) - # If internal type has a local name and it's not a list, I use that name - elif not is_iter_type and internal_type and "name" in internal_type_xml_map: - xml_name = _extract_name_from_internal_type(internal_type) - children = data.findall(xml_name) - # That's an array - else: - if internal_type: # Complex type, ignore itemsName and use the complex type name - items_name = _extract_name_from_internal_type(internal_type) - else: - items_name = xml_desc.get("itemsName", xml_name) - children = data.findall(items_name) - - if len(children) == 0: - if is_iter_type: - if is_wrapped: - return None # is_wrapped no node, we want None - return [] # not wrapped, assume empty list - return None # Assume it's not there, maybe an optional node. - - # If is_iter_type and not wrapped, return all found children - if is_iter_type: - if not is_wrapped: - return children - # Iter and wrapped, should have found one node only (the wrap one) - if len(children) != 1: - raise DeserializationError( - "Tried to deserialize an array not wrapped, and found several nodes '{}'. Maybe you should declare this array as wrapped?".format( - xml_name - ) - ) - return list(children[0]) # Might be empty list and that's ok. - - # Here it's not a itertype, we should have found one element only or empty - if len(children) > 1: - raise DeserializationError("Find several XML '{}' where it was not expected".format(xml_name)) - return children[0] - - -class Deserializer: - """Response object model deserializer. - - :param dict classes: Class type dictionary for deserializing complex types. - :ivar list key_extractors: Ordered list of extractors to be used by this deserializer. - """ - - basic_types = {str: "str", int: "int", bool: "bool", float: "float"} - - valid_date = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") - - def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: - self.deserialize_type = { - "iso-8601": Deserializer.deserialize_iso, - "rfc-1123": Deserializer.deserialize_rfc, - "unix-time": Deserializer.deserialize_unix, - "duration": Deserializer.deserialize_duration, - "date": Deserializer.deserialize_date, - "time": Deserializer.deserialize_time, - "decimal": Deserializer.deserialize_decimal, - "long": Deserializer.deserialize_long, - "bytearray": Deserializer.deserialize_bytearray, - "base64": Deserializer.deserialize_base64, - "object": self.deserialize_object, - "[]": self.deserialize_iter, - "{}": self.deserialize_dict, - } - self.deserialize_expected_types = { - "duration": (isodate.Duration, datetime.timedelta), - "iso-8601": (datetime.datetime), - } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} - self.key_extractors = [rest_key_extractor, xml_key_extractor] - # Additional properties only works if the "rest_key_extractor" is used to - # extract the keys. Making it to work whatever the key extractor is too much - # complicated, with no real scenario for now. - # So adding a flag to disable additional properties detection. This flag should be - # used if your expect the deserialization to NOT come from a JSON REST syntax. - # Otherwise, result are unexpected - self.additional_properties_detection = True - - def __call__(self, target_obj, response_data, content_type=None): - """Call the deserializer to process a REST response. - - :param str target_obj: Target data type to deserialize to. - :param requests.Response response_data: REST response object. - :param str content_type: Swagger "produces" if available. - :raises DeserializationError: if deserialization fails. - :return: Deserialized object. - :rtype: object - """ - data = self._unpack_content(response_data, content_type) - return self._deserialize(target_obj, data) - - def _deserialize(self, target_obj, data): # pylint: disable=inconsistent-return-statements - """Call the deserializer on a model. - - Data needs to be already deserialized as JSON or XML ElementTree - - :param str target_obj: Target data type to deserialize to. - :param object data: Object to deserialize. - :raises DeserializationError: if deserialization fails. - :return: Deserialized object. - :rtype: object - """ - # This is already a model, go recursive just in case - if hasattr(data, "_attribute_map"): - constants = [name for name, config in getattr(data, "_validation", {}).items() if config.get("constant")] - try: - for attr, mapconfig in data._attribute_map.items(): # pylint: disable=protected-access - if attr in constants: - continue - value = getattr(data, attr) - if value is None: - continue - local_type = mapconfig["type"] - internal_data_type = local_type.strip("[]{}") - if internal_data_type not in self.dependencies or isinstance(internal_data_type, Enum): - continue - setattr(data, attr, self._deserialize(local_type, value)) - return data - except AttributeError: - return - - response, class_name = self._classify_target(target_obj, data) - - if isinstance(response, str): - return self.deserialize_data(data, response) - if isinstance(response, type) and issubclass(response, Enum): - return self.deserialize_enum(data, response) - - if data is None or data is CoreNull: - return data - try: - attributes = response._attribute_map # type: ignore # pylint: disable=protected-access - d_attrs = {} - for attr, attr_desc in attributes.items(): - # Check empty string. If it's not empty, someone has a real "additionalProperties"... - if attr == "additional_properties" and attr_desc["key"] == "": - continue - raw_value = None - # Enhance attr_desc with some dynamic data - attr_desc = attr_desc.copy() # Do a copy, do not change the real one - internal_data_type = attr_desc["type"].strip("[]{}") - if internal_data_type in self.dependencies: - attr_desc["internalType"] = self.dependencies[internal_data_type] - - for key_extractor in self.key_extractors: - found_value = key_extractor(attr, attr_desc, data) - if found_value is not None: - if raw_value is not None and raw_value != found_value: - msg = ( - "Ignoring extracted value '%s' from %s for key '%s'" - " (duplicate extraction, follow extractors order)" - ) - _LOGGER.warning(msg, found_value, key_extractor, attr) - continue - raw_value = found_value - - value = self.deserialize_data(raw_value, attr_desc["type"]) - d_attrs[attr] = value - except (AttributeError, TypeError, KeyError) as err: - msg = "Unable to deserialize to object: " + class_name # type: ignore - raise DeserializationError(msg) from err - additional_properties = self._build_additional_properties(attributes, data) - return self._instantiate_model(response, d_attrs, additional_properties) - - def _build_additional_properties(self, attribute_map, data): - if not self.additional_properties_detection: - return None - if "additional_properties" in attribute_map and attribute_map.get("additional_properties", {}).get("key") != "": - # Check empty string. If it's not empty, someone has a real "additionalProperties" - return None - if isinstance(data, ET.Element): - data = {el.tag: el.text for el in data} - - known_keys = { - _decode_attribute_map_key(_FLATTEN.split(desc["key"])[0]) - for desc in attribute_map.values() - if desc["key"] != "" - } - present_keys = set(data.keys()) - missing_keys = present_keys - known_keys - return {key: data[key] for key in missing_keys} - - def _classify_target(self, target, data): - """Check to see whether the deserialization target object can - be classified into a subclass. - Once classification has been determined, initialize object. - - :param str target: The target object type to deserialize to. - :param str/dict data: The response data to deserialize. - :return: The classified target object and its class name. - :rtype: tuple - """ - if target is None: - return None, None - - if isinstance(target, str): - try: - target = self.dependencies[target] - except KeyError: - return target, target - - try: - target = target._classify(data, self.dependencies) # type: ignore # pylint: disable=protected-access - except AttributeError: - pass # Target is not a Model, no classify - return target, target.__class__.__name__ # type: ignore - - def failsafe_deserialize(self, target_obj, data, content_type=None): - """Ignores any errors encountered in deserialization, - and falls back to not deserializing the object. Recommended - for use in error deserialization, as we want to return the - HttpResponseError to users, and not have them deal with - a deserialization error. - - :param str target_obj: The target object type to deserialize to. - :param str/dict data: The response data to deserialize. - :param str content_type: Swagger "produces" if available. - :return: Deserialized object. - :rtype: object - """ - try: - return self(target_obj, data, content_type=content_type) - except: # pylint: disable=bare-except - _LOGGER.debug( - "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True - ) - return None - - @staticmethod - def _unpack_content(raw_data, content_type=None): - """Extract the correct structure for deserialization. - - If raw_data is a PipelineResponse, try to extract the result of RawDeserializer. - if we can't, raise. Your Pipeline should have a RawDeserializer. - - If not a pipeline response and raw_data is bytes or string, use content-type - to decode it. If no content-type, try JSON. - - If raw_data is something else, bypass all logic and return it directly. - - :param obj raw_data: Data to be processed. - :param str content_type: How to parse if raw_data is a string/bytes. - :raises JSONDecodeError: If JSON is requested and parsing is impossible. - :raises UnicodeDecodeError: If bytes is not UTF8 - :rtype: object - :return: Unpacked content. - """ - # Assume this is enough to detect a Pipeline Response without importing it - context = getattr(raw_data, "context", {}) - if context: - if RawDeserializer.CONTEXT_NAME in context: - return context[RawDeserializer.CONTEXT_NAME] - raise ValueError("This pipeline didn't have the RawDeserializer policy; can't deserialize") - - # Assume this is enough to recognize universal_http.ClientResponse without importing it - if hasattr(raw_data, "body"): - return RawDeserializer.deserialize_from_http_generics(raw_data.text(), raw_data.headers) - - # Assume this enough to recognize requests.Response without importing it. - if hasattr(raw_data, "_content_consumed"): - return RawDeserializer.deserialize_from_http_generics(raw_data.text, raw_data.headers) - - if isinstance(raw_data, (str, bytes)) or hasattr(raw_data, "read"): - return RawDeserializer.deserialize_from_text(raw_data, content_type) # type: ignore - return raw_data - - def _instantiate_model(self, response, attrs, additional_properties=None): - """Instantiate a response model passing in deserialized args. - - :param Response response: The response model class. - :param dict attrs: The deserialized response attributes. - :param dict additional_properties: Additional properties to be set. - :rtype: Response - :return: The instantiated response model. - """ - if callable(response): - subtype = getattr(response, "_subtype_map", {}) - try: - readonly = [ - k - for k, v in response._validation.items() # pylint: disable=protected-access # type: ignore - if v.get("readonly") - ] - const = [ - k - for k, v in response._validation.items() # pylint: disable=protected-access # type: ignore - if v.get("constant") - ] - kwargs = {k: v for k, v in attrs.items() if k not in subtype and k not in readonly + const} - response_obj = response(**kwargs) - for attr in readonly: - setattr(response_obj, attr, attrs.get(attr)) - if additional_properties: - response_obj.additional_properties = additional_properties # type: ignore - return response_obj - except TypeError as err: - msg = "Unable to deserialize {} into model {}. ".format(kwargs, response) # type: ignore - raise DeserializationError(msg + str(err)) from err - else: - try: - for attr, value in attrs.items(): - setattr(response, attr, value) - return response - except Exception as exp: - msg = "Unable to populate response model. " - msg += "Type: {}, Error: {}".format(type(response), exp) - raise DeserializationError(msg) from exp - - def deserialize_data(self, data, data_type): # pylint: disable=too-many-return-statements - """Process data for deserialization according to data type. - - :param str data: The response string to be deserialized. - :param str data_type: The type to deserialize to. - :raises DeserializationError: if deserialization fails. - :return: Deserialized object. - :rtype: object - """ - if data is None: - return data - - try: - if not data_type: - return data - if data_type in self.basic_types.values(): - return self.deserialize_basic(data, data_type) - if data_type in self.deserialize_type: - if isinstance(data, self.deserialize_expected_types.get(data_type, tuple())): - return data - - is_a_text_parsing_type = lambda x: x not in [ # pylint: disable=unnecessary-lambda-assignment - "object", - "[]", - r"{}", - ] - if isinstance(data, ET.Element) and is_a_text_parsing_type(data_type) and not data.text: - return None - data_val = self.deserialize_type[data_type](data) - return data_val - - iter_type = data_type[0] + data_type[-1] - if iter_type in self.deserialize_type: - return self.deserialize_type[iter_type](data, data_type[1:-1]) - - obj_type = self.dependencies[data_type] - if issubclass(obj_type, Enum): - if isinstance(data, ET.Element): - data = data.text - return self.deserialize_enum(data, obj_type) - - except (ValueError, TypeError, AttributeError) as err: - msg = "Unable to deserialize response data." - msg += " Data: {}, {}".format(data, data_type) - raise DeserializationError(msg) from err - return self._deserialize(obj_type, data) - - def deserialize_iter(self, attr, iter_type): - """Deserialize an iterable. - - :param list attr: Iterable to be deserialized. - :param str iter_type: The type of object in the iterable. - :return: Deserialized iterable. - :rtype: list - """ - if attr is None: - return None - if isinstance(attr, ET.Element): # If I receive an element here, get the children - attr = list(attr) - if not isinstance(attr, (list, set)): - raise DeserializationError("Cannot deserialize as [{}] an object of type {}".format(iter_type, type(attr))) - return [self.deserialize_data(a, iter_type) for a in attr] - - def deserialize_dict(self, attr, dict_type): - """Deserialize a dictionary. - - :param dict/list attr: Dictionary to be deserialized. Also accepts - a list of key, value pairs. - :param str dict_type: The object type of the items in the dictionary. - :return: Deserialized dictionary. - :rtype: dict - """ - if isinstance(attr, list): - return {x["key"]: self.deserialize_data(x["value"], dict_type) for x in attr} - - if isinstance(attr, ET.Element): - # Transform value into {"Key": "value"} - attr = {el.tag: el.text for el in attr} - return {k: self.deserialize_data(v, dict_type) for k, v in attr.items()} - - def deserialize_object(self, attr, **kwargs): # pylint: disable=too-many-return-statements - """Deserialize a generic object. - This will be handled as a dictionary. - - :param dict attr: Dictionary to be deserialized. - :return: Deserialized object. - :rtype: dict - :raises TypeError: if non-builtin datatype encountered. - """ - if attr is None: - return None - if isinstance(attr, ET.Element): - # Do no recurse on XML, just return the tree as-is - return attr - if isinstance(attr, str): - return self.deserialize_basic(attr, "str") - obj_type = type(attr) - if obj_type in self.basic_types: - return self.deserialize_basic(attr, self.basic_types[obj_type]) - if obj_type is _long_type: - return self.deserialize_long(attr) - - if obj_type == dict: - deserialized = {} - for key, value in attr.items(): - try: - deserialized[key] = self.deserialize_object(value, **kwargs) - except ValueError: - deserialized[key] = None - return deserialized - - if obj_type == list: - deserialized = [] - for obj in attr: - try: - deserialized.append(self.deserialize_object(obj, **kwargs)) - except ValueError: - pass - return deserialized - - error = "Cannot deserialize generic object with type: " - raise TypeError(error + str(obj_type)) - - def deserialize_basic(self, attr, data_type): # pylint: disable=too-many-return-statements - """Deserialize basic builtin data type from string. - Will attempt to convert to str, int, float and bool. - This function will also accept '1', '0', 'true' and 'false' as - valid bool values. - - :param str attr: response string to be deserialized. - :param str data_type: deserialization data type. - :return: Deserialized basic type. - :rtype: str, int, float or bool - :raises TypeError: if string format is not valid. - """ - # If we're here, data is supposed to be a basic type. - # If it's still an XML node, take the text - if isinstance(attr, ET.Element): - attr = attr.text - if not attr: - if data_type == "str": - # None or '', node is empty string. - return "" - # None or '', node with a strong type is None. - # Don't try to model "empty bool" or "empty int" - return None - - if data_type == "bool": - if attr in [True, False, 1, 0]: - return bool(attr) - if isinstance(attr, str): - if attr.lower() in ["true", "1"]: - return True - if attr.lower() in ["false", "0"]: - return False - raise TypeError("Invalid boolean value: {}".format(attr)) - - if data_type == "str": - return self.deserialize_unicode(attr) - return eval(data_type)(attr) # nosec # pylint: disable=eval-used - - @staticmethod - def deserialize_unicode(data): - """Preserve unicode objects in Python 2, otherwise return data - as a string. - - :param str data: response string to be deserialized. - :return: Deserialized string. - :rtype: str or unicode - """ - # We might be here because we have an enum modeled as string, - # and we try to deserialize a partial dict with enum inside - if isinstance(data, Enum): - return data - - # Consider this is real string - try: - if isinstance(data, unicode): # type: ignore - return data - except NameError: - return str(data) - return str(data) - - @staticmethod - def deserialize_enum(data, enum_obj): - """Deserialize string into enum object. - - If the string is not a valid enum value it will be returned as-is - and a warning will be logged. - - :param str data: Response string to be deserialized. If this value is - None or invalid it will be returned as-is. - :param Enum enum_obj: Enum object to deserialize to. - :return: Deserialized enum object. - :rtype: Enum - """ - if isinstance(data, enum_obj) or data is None: - return data - if isinstance(data, Enum): - data = data.value - if isinstance(data, int): - # Workaround. We might consider remove it in the future. - try: - return list(enum_obj.__members__.values())[data] - except IndexError as exc: - error = "{!r} is not a valid index for enum {!r}" - raise DeserializationError(error.format(data, enum_obj)) from exc - try: - return enum_obj(str(data)) - except ValueError: - for enum_value in enum_obj: - if enum_value.value.lower() == str(data).lower(): - return enum_value - # We don't fail anymore for unknown value, we deserialize as a string - _LOGGER.warning("Deserializer is not able to find %s as valid enum in %s", data, enum_obj) - return Deserializer.deserialize_unicode(data) - - @staticmethod - def deserialize_bytearray(attr): - """Deserialize string into bytearray. - - :param str attr: response string to be deserialized. - :return: Deserialized bytearray - :rtype: bytearray - :raises TypeError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - return bytearray(b64decode(attr)) # type: ignore - - @staticmethod - def deserialize_base64(attr): - """Deserialize base64 encoded string into string. - - :param str attr: response string to be deserialized. - :return: Deserialized base64 string - :rtype: bytearray - :raises TypeError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore - attr = attr + padding # type: ignore - encoded = attr.replace("-", "+").replace("_", "/") - return b64decode(encoded) - - @staticmethod - def deserialize_decimal(attr): - """Deserialize string into Decimal object. - - :param str attr: response string to be deserialized. - :return: Deserialized decimal - :raises DeserializationError: if string format invalid. - :rtype: decimal - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - return decimal.Decimal(str(attr)) # type: ignore - except decimal.DecimalException as err: - msg = "Invalid decimal {}".format(attr) - raise DeserializationError(msg) from err - - @staticmethod - def deserialize_long(attr): - """Deserialize string into long (Py2) or int (Py3). - - :param str attr: response string to be deserialized. - :return: Deserialized int - :rtype: long or int - :raises ValueError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - return _long_type(attr) # type: ignore - - @staticmethod - def deserialize_duration(attr): - """Deserialize ISO-8601 formatted string into TimeDelta object. - - :param str attr: response string to be deserialized. - :return: Deserialized duration - :rtype: TimeDelta - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - duration = isodate.parse_duration(attr) - except (ValueError, OverflowError, AttributeError) as err: - msg = "Cannot deserialize duration object." - raise DeserializationError(msg) from err - return duration - - @staticmethod - def deserialize_date(attr): - """Deserialize ISO-8601 formatted string into Date object. - - :param str attr: response string to be deserialized. - :return: Deserialized date - :rtype: Date - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore - raise DeserializationError("Date must have only digits and -. Received: %s" % attr) - # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. - return isodate.parse_date(attr, defaultmonth=0, defaultday=0) - - @staticmethod - def deserialize_time(attr): - """Deserialize ISO-8601 formatted string into time object. - - :param str attr: response string to be deserialized. - :return: Deserialized time - :rtype: datetime.time - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore - raise DeserializationError("Date must have only digits and -. Received: %s" % attr) - return isodate.parse_time(attr) - - @staticmethod - def deserialize_rfc(attr): - """Deserialize RFC-1123 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :return: Deserialized RFC datetime - :rtype: Datetime - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - parsed_date = email.utils.parsedate_tz(attr) # type: ignore - date_obj = datetime.datetime( - *parsed_date[:6], tzinfo=datetime.timezone(datetime.timedelta(minutes=(parsed_date[9] or 0) / 60)) - ) - if not date_obj.tzinfo: - date_obj = date_obj.astimezone(tz=TZ_UTC) - except ValueError as err: - msg = "Cannot deserialize to rfc datetime object." - raise DeserializationError(msg) from err - return date_obj - - @staticmethod - def deserialize_iso(attr): - """Deserialize ISO-8601 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :return: Deserialized ISO datetime - :rtype: Datetime - :raises DeserializationError: if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - attr = attr.upper() # type: ignore - match = Deserializer.valid_date.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - check_decimal = attr.split(".") - if len(check_decimal) > 1: - decimal_str = "" - for digit in check_decimal[1]: - if digit.isdigit(): - decimal_str += digit - else: - break - if len(decimal_str) > 6: - attr = attr.replace(decimal_str, decimal_str[0:6]) - - date_obj = isodate.parse_datetime(attr) - test_utc = date_obj.utctimetuple() - if test_utc.tm_year > 9999 or test_utc.tm_year < 1: - raise OverflowError("Hit max or min date") - except (ValueError, OverflowError, AttributeError) as err: - msg = "Cannot deserialize datetime object." - raise DeserializationError(msg) from err - return date_obj - - @staticmethod - def deserialize_unix(attr): - """Serialize Datetime object into IntTime format. - This is represented as seconds. - - :param int attr: Object to be serialized. - :return: Deserialized datetime - :rtype: Datetime - :raises DeserializationError: if format invalid - """ - if isinstance(attr, ET.Element): - attr = int(attr.text) # type: ignore - try: - attr = int(attr) - date_obj = datetime.datetime.fromtimestamp(attr, TZ_UTC) - except ValueError as err: - msg = "Cannot deserialize to unix datetime object." - raise DeserializationError(msg) from err - return date_obj diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py index aaa6692b2346..35d5fc024978 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/model_base.py @@ -23,19 +23,26 @@ from json import JSONEncoder import xml.etree.ElementTree as ET from collections.abc import MutableMapping -from typing_extensions import Self import isodate from azure.core.exceptions import DeserializationError from azure.core import CaseInsensitiveEnumMeta from azure.core.pipeline import PipelineResponse from azure.core.serialization import _Null +from azure.core.rest import HttpResponse + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + _LOGGER = logging.getLogger(__name__) __all__ = ["SdkJSONEncoder", "Model", "rest_field", "rest_discriminator"] TZ_UTC = timezone.utc _T = typing.TypeVar("_T") +_NONE_TYPE = type(None) def _timedelta_as_isostr(td: timedelta) -> str: @@ -102,6 +109,29 @@ def _serialize_bytes(o, format: typing.Optional[str] = None) -> str: return encoded +def _serialize_duration(td: timedelta, format: typing.Optional[str] = None): + """Serialize a timedelta to its wire representation. + + For the ``seconds``/``milliseconds`` encodings the value is converted to a + numeric value, otherwise it falls back to an ISO 8601 duration string. + + :param timedelta td: The timedelta to serialize. + :param str format: The duration encoding format. + :rtype: int or float or str + :return: serialized duration + """ + seconds = td.total_seconds() + if format == "duration-seconds-int": + return int(seconds) + if format == "duration-seconds-float": + return seconds + if format == "duration-milliseconds-int": + return int(seconds * 1000) + if format == "duration-milliseconds-float": + return seconds * 1000 + return _timedelta_as_isostr(td) + + def _serialize_datetime(o, format: typing.Optional[str] = None): if hasattr(o, "year") and hasattr(o, "hour"): if format == "rfc7231": @@ -128,7 +158,15 @@ def _is_readonly(p): class SdkJSONEncoder(JSONEncoder): - """A JSON encoder that's capable of serializing datetime objects and bytes.""" + """A JSON encoder that's capable of serializing datetime objects and bytes. + + :param args: Additional positional arguments passed to the base ``JSONEncoder``. + :type args: typing.Any + :keyword exclude_readonly: Whether to exclude readonly properties. Defaults to False. + :paramtype exclude_readonly: bool + :keyword format: The format to use for serialization. Defaults to None. + :paramtype format: typing.Optional[str] + """ def __init__(self, *args, exclude_readonly: bool = False, format: typing.Optional[str] = None, **kwargs): super().__init__(*args, **kwargs) @@ -170,6 +208,21 @@ def default(self, o): # pylint: disable=too-many-return-statements r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT" ) +_ARRAY_ENCODE_MAPPING = { + "pipeDelimited": "|", + "spaceDelimited": " ", + "commaDelimited": ",", + "newlineDelimited": "\n", +} + + +def _deserialize_array_encoded(delimit: str, attr): + if isinstance(attr, str): + if attr == "": + return [] + return attr.split(delimit) + return attr + def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: """Deserialize ISO-8601 formatted string into Datetime object. @@ -201,7 +254,7 @@ def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: test_utc = date_obj.utctimetuple() if test_utc.tm_year > 9999 or test_utc.tm_year < 1: raise OverflowError("Hit max or min date") - return date_obj + return date_obj # type: ignore[no-any-return] def _deserialize_datetime_rfc7231(attr: typing.Union[str, datetime]) -> datetime: @@ -255,7 +308,7 @@ def _deserialize_time(attr: typing.Union[str, time]) -> time: """ if isinstance(attr, time): return attr - return isodate.parse_time(attr) + return isodate.parse_time(attr) # type: ignore[no-any-return] def _deserialize_bytes(attr): @@ -279,6 +332,12 @@ def _deserialize_duration(attr): return isodate.parse_duration(attr) +def _deserialize_duration_numeric(attr, unit): + if isinstance(attr, timedelta): + return attr + return timedelta(**{unit: float(attr)}) + + def _deserialize_decimal(attr): if isinstance(attr, decimal.Decimal): return attr @@ -291,6 +350,12 @@ def _deserialize_int_as_str(attr): return int(attr) +def _deserialize_bool_as_str(attr): + if isinstance(attr, bool): + return attr + return attr.lower() == "true" + + _DESERIALIZE_MAPPING = { datetime: _deserialize_datetime, date: _deserialize_date, @@ -308,12 +373,20 @@ def _deserialize_int_as_str(attr): "unix-timestamp": _deserialize_datetime_unix_timestamp, "base64": _deserialize_bytes, "base64url": _deserialize_bytes_base64, + "duration-seconds-int": functools.partial(_deserialize_duration_numeric, unit="seconds"), + "duration-seconds-float": functools.partial(_deserialize_duration_numeric, unit="seconds"), + "duration-milliseconds-int": functools.partial(_deserialize_duration_numeric, unit="milliseconds"), + "duration-milliseconds-float": functools.partial(_deserialize_duration_numeric, unit="milliseconds"), } def get_deserializer(annotation: typing.Any, rf: typing.Optional["_RestField"] = None): if annotation is int and rf and rf._format == "str": return _deserialize_int_as_str + if annotation is bool and rf and rf._format == "str": + return _deserialize_bool_as_str + if annotation is str and rf and rf._format in _ARRAY_ENCODE_MAPPING: + return functools.partial(_deserialize_array_encoded, _ARRAY_ENCODE_MAPPING[rf._format]) if rf and rf._format: return _DESERIALIZE_MAPPING_WITHFORMAT.get(rf._format) return _DESERIALIZE_MAPPING.get(annotation) # pyright: ignore @@ -345,16 +418,46 @@ def _get_model(module_name: str, model_name: str): class _MyMutableMapping(MutableMapping[str, typing.Any]): - def __init__(self, data: typing.Dict[str, typing.Any]) -> None: + def __init__(self, data: dict[str, typing.Any]) -> None: self._data = data def __contains__(self, key: typing.Any) -> bool: return key in self._data def __getitem__(self, key: str) -> typing.Any: + # If this key has been deserialized (for mutable types), we need to handle serialization + if hasattr(self, "_attr_to_rest_field"): + cache_attr = f"_deserialized_{key}" + if hasattr(self, cache_attr): + rf = _get_rest_field(getattr(self, "_attr_to_rest_field"), key) + if rf: + value = self._data.get(key) + if isinstance(value, (dict, list, set)): + # For mutable types, serialize and return + # But also update _data with serialized form and clear flag + # so mutations via this returned value affect _data + serialized = _serialize(value, rf._format) + # If serialized form is same type (no transformation needed), + # return _data directly so mutations work + if isinstance(serialized, type(value)) and serialized == value: + return self._data.get(key) + # Otherwise return serialized copy and clear flag + try: + object.__delattr__(self, cache_attr) + except AttributeError: + pass + # Store serialized form back + self._data[key] = serialized + return serialized return self._data.__getitem__(key) def __setitem__(self, key: str, value: typing.Any) -> None: + # Clear any cached deserialized value when setting through dictionary access + cache_attr = f"_deserialized_{key}" + try: + object.__delattr__(self, cache_attr) + except AttributeError: + pass self._data.__setitem__(key, value) def __delitem__(self, key: str) -> None: @@ -371,21 +474,21 @@ def __ne__(self, other: typing.Any) -> bool: def keys(self) -> typing.KeysView[str]: """ - :returns: a set-like object providing a view on D's keys + :returns: a set-like object providing a view on the mapping's keys :rtype: ~typing.KeysView """ return self._data.keys() def values(self) -> typing.ValuesView[typing.Any]: """ - :returns: an object providing a view on D's values + :returns: an object providing a view on the mapping's values :rtype: ~typing.ValuesView """ return self._data.values() def items(self) -> typing.ItemsView[str, typing.Any]: """ - :returns: set-like object providing a view on D's items + :returns: a set-like object providing a view on the mapping's items :rtype: ~typing.ItemsView """ return self._data.items() @@ -395,7 +498,7 @@ def get(self, key: str, default: typing.Any = None) -> typing.Any: Get the value for key if key is in the dictionary, else default. :param str key: The key to look up. :param any default: The value to return if key is not in the dictionary. Defaults to None - :returns: D[k] if k in D, else d. + :returns: The value for key if key is in the dictionary, else default. :rtype: any """ try: @@ -425,24 +528,24 @@ def pop(self, key: str, default: typing.Any = _UNSET) -> typing.Any: return self._data.pop(key) return self._data.pop(key, default) - def popitem(self) -> typing.Tuple[str, typing.Any]: + def popitem(self) -> tuple[str, typing.Any]: """ Removes and returns some (key, value) pair :returns: The (key, value) pair. :rtype: tuple - :raises KeyError: if D is empty. + :raises KeyError: if the dictionary is empty. """ return self._data.popitem() def clear(self) -> None: """ - Remove all items from D. + Remove all items from the dictionary. """ self._data.clear() def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: # pylint: disable=arguments-differ """ - Updates D from mapping/iterable E and F. + Update the dictionary from a mapping or an iterable of key-value pairs. :param any args: Either a mapping object or an iterable of key-value pairs. """ self._data.update(*args, **kwargs) @@ -455,10 +558,11 @@ def setdefault(self, key: str, default: typing.Any) -> typing.Any: ... # pylint def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: """ - Same as calling D.get(k, d), and setting D[k]=d if k not found + Return the value for key if key is in the dictionary; otherwise set the key to + default and return default. :param str key: The key to look up. :param any default: The value to set if key is not in the dictionary - :returns: D[k] if k in D, else d. + :returns: The value for key if key is in the dictionary, else default. :rtype: any """ if default is _UNSET: @@ -466,6 +570,8 @@ def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: return self._data.setdefault(key, default) def __eq__(self, other: typing.Any) -> bool: + if isinstance(other, _MyMutableMapping): + return self._data == other._data try: other_model = self.__class__(other) except Exception: @@ -482,6 +588,8 @@ def _is_model(obj: typing.Any) -> bool: def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-many-return-statements if isinstance(o, list): + if format in _ARRAY_ENCODE_MAPPING and all(isinstance(x, str) for x in o): + return _ARRAY_ENCODE_MAPPING[format].join(o) return [_serialize(x, format) for x in o] if isinstance(o, dict): return {k: _serialize(v, format) for k, v in o.items()} @@ -506,16 +614,14 @@ def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-m pass # Last, try datetime.timedelta try: - return _timedelta_as_isostr(o) + return _serialize_duration(o, format) except AttributeError: # This will be raised when it hits value.total_seconds in the method above pass return o -def _get_rest_field( - attr_to_rest_field: typing.Dict[str, "_RestField"], rest_name: str -) -> typing.Optional["_RestField"]: +def _get_rest_field(attr_to_rest_field: dict[str, "_RestField"], rest_name: str) -> typing.Optional["_RestField"]: try: return next(rf for rf in attr_to_rest_field.values() if rf._rest_name == rest_name) except StopIteration: @@ -534,69 +640,253 @@ def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typin return _serialize(value, rf._format) +# ============================================================================ +# Fast-path scalar deserializer functions for rest_field(deserializer=...) +# These are referenced from rest_field declarations to bypass the generic +# _deserialize -> _deserialize_with_callable chain. +# Only simple/primitive types — no models or container types. +# ============================================================================ + + +def _xml_deser_str(value): + if isinstance(value, ET.Element): + return value.text or "" + return str(value) if value is not None else None + + +def _xml_deser_int(value): + if isinstance(value, ET.Element): + return int(value.text) if value.text else None + return int(value) if value is not None else None + + +def _xml_deser_float(value): + if isinstance(value, ET.Element): + return float(value.text) if value.text else None + return float(value) if value is not None else None + + +def _xml_deser_bool(value): + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + if text in (True, False): + return text + return text.lower() == "true" + + +# pylint: disable=docstring-missing-param +def _xml_deser_bytes(value): + """Deserialize bytes from XML (base64).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_bytes(text) + + +def _xml_deser_bytes_base64url(value): + """Deserialize bytes from XML (base64url).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_bytes_base64(text) + + +def _xml_deser_datetime(value): + """Deserialize a datetime from XML (ISO 8601 / rfc3339).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_datetime(text) + + +def _xml_deser_datetime_rfc7231(value): + """Deserialize a datetime from XML (RFC7231 format).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_datetime_rfc7231(text) + + +def _xml_deser_datetime_unix_timestamp(value): + """Deserialize a datetime from XML (Unix timestamp).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_datetime_unix_timestamp(float(text)) + + +def _xml_deser_date(value): + """Deserialize a date from XML (ISO 8601).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_date(text) + + +def _xml_deser_time(value): + """Deserialize a time from XML (ISO 8601).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_time(text) + + +def _xml_deser_duration(value): + """Deserialize a timedelta from XML (ISO 8601 duration).""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_duration(text) + + +def _xml_deser_decimal(value): + """Deserialize a Decimal from XML.""" + if isinstance(value, ET.Element): + text = value.text + else: + text = value + if text is None: + return None + return _deserialize_decimal(text) + + +def _xml_deser_enum_or_str(enum_cls, value): + """Deserialize a Union[EnumType, str] from XML.""" + text = value.text if isinstance(value, ET.Element) else value + if text is None: + return None + try: + return enum_cls(text) + except ValueError: + return text + + +def _extract_xml_model_type(rf_type): + """Extract the concrete Model class from a resolved rf._type partial chain. + + Unwraps ``Optional[Model]`` and ``_deserialize_model(Model, ...)`` + wrappers. Only handles Model and Optional[Model] — other composite + types (List, Dict, Union, etc.) return None and fall through to the + generic ``_deserialize`` path at runtime. + """ + if rf_type is None: + return None + if isinstance(rf_type, type) and _is_model(rf_type): + return rf_type + if not isinstance(rf_type, functools.partial): + return None + func = rf_type.func + args = rf_type.args + if func is _deserialize_with_optional and args: + return _extract_xml_model_type(args[0]) + if func is _deserialize_model and args: + cls = args[0] + return cls if isinstance(cls, type) and _is_model(cls) else None + return None + + +def _build_xml_field_plan( # pylint: disable=docstring-missing-return, docstring-missing-rtype, unused-variable + cls, attr_to_rest_field: dict +) -> list: + """Build a precomputed XML field plan for fast _init_from_xml iteration. + + Called once per model class in __new__. Returns a list of tuples: + (rest_name, xml_name, kind, deser, rf_type, is_optional, items_name) + + kind: 0=wrapped, 1=attribute, 2=unwrapped, 3=text + + For Model and Optional[Model] fields that lack a scalar + ``_deserializer``, this function precomputes the Model class as the + deserializer so ``_init_from_xml`` can call ``ModelClass(element)`` + directly instead of going through the expensive + ``_get_deserialize_callable_from_annotation`` chain at runtime. + """ + model_meta = getattr(cls, "_xml", {}) + model_ns = model_meta.get("ns") or model_meta.get("namespace") + plan = [] + + for rf in attr_to_rest_field.values(): + prop_meta = getattr(rf, "_xml", {}) + deser = rf._deserializer + + xml_name = prop_meta.get("name", rf._rest_name) + xml_ns = _resolve_xml_ns(prop_meta, model_meta) + if xml_ns: + xml_name = "{" + xml_ns + "}" + xml_name + + is_optional = rf._is_optional + + # For Model / Optional[Model] fields without a scalar deserializer, + # precompute the Model class as the deserializer. + if deser is None and rf._type is not None: + model_cls = _extract_xml_model_type(rf._type) + if model_cls is not None: + deser = model_cls + + if prop_meta.get("attribute", False): + plan.append((rf._rest_name, xml_name, 1, deser, rf._type, is_optional, None)) + elif prop_meta.get("unwrapped", False): + items_name = prop_meta.get("itemsName") + if items_name: + items_ns = prop_meta.get("itemsNs") + if items_ns is not None: + xml_ns = items_ns + if xml_ns: + items_name = "{" + xml_ns + "}" + items_name + else: + items_name = xml_name + plan.append((rf._rest_name, xml_name, 2, deser, rf._type, is_optional, items_name)) + elif prop_meta.get("text", False): + plan.append((rf._rest_name, xml_name, 3, deser, rf._type, is_optional, None)) + else: + plan.append((rf._rest_name, xml_name, 0, deser, rf._type, is_optional, None)) + + return plan + + +# pylint: enable=docstring-missing-param class Model(_MyMutableMapping): _is_model = True # label whether current class's _attr_to_rest_field has been calculated # could not see _attr_to_rest_field directly because subclass inherits it from parent class - _calculated: typing.Set[str] = set() + _calculated: set[str] = set() def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: class_name = self.__class__.__name__ if len(args) > 1: raise TypeError(f"{class_name}.__init__() takes 2 positional arguments but {len(args) + 1} were given") - dict_to_pass = { - rest_field._rest_name: rest_field._default - for rest_field in self._attr_to_rest_field.values() - if rest_field._default is not _UNSET - } - if args: # pylint: disable=too-many-nested-blocks + dict_to_pass: dict[str, typing.Any] = {} + if args: if isinstance(args[0], ET.Element): - existed_attr_keys = [] - model_meta = getattr(self, "_xml", {}) - - for rf in self._attr_to_rest_field.values(): - prop_meta = getattr(rf, "_xml", {}) - xml_name = prop_meta.get("name", rf._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - - # attribute - if prop_meta.get("attribute", False) and args[0].get(xml_name) is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].get(xml_name)) - continue - - # unwrapped element is array - if prop_meta.get("unwrapped", False): - # unwrapped array could either use prop items meta/prop meta - if prop_meta.get("itemsName"): - xml_name = prop_meta.get("itemsName") - xml_ns = prop_meta.get("itemNs") - if xml_ns: - xml_name = "{" + xml_ns + "}" + xml_name - items = args[0].findall(xml_name) # pyright: ignore - if len(items) > 0: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, items) - continue - - # text element is primitive type - if prop_meta.get("text", False): - if args[0].text is not None: - dict_to_pass[rf._rest_name] = _deserialize(rf._type, args[0].text) - continue - - # wrapped element could be normal property or array, it should only have one element - item = args[0].find(xml_name) - if item is not None: - existed_attr_keys.append(xml_name) - dict_to_pass[rf._rest_name] = _deserialize(rf._type, item) - - # rest thing is additional properties - for e in args[0]: - if e.tag not in existed_attr_keys: - dict_to_pass[e.tag] = _convert_element(e) + dict_to_pass.update(self._init_from_xml(args[0])) else: dict_to_pass.update( {k: _create_value(_get_rest_field(self._attr_to_rest_field, k), v) for k, v in args[0].items()} @@ -613,8 +903,117 @@ def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: if v is not None } ) + # Apply client default values for fields the caller didn't set so that + # defaults are part of `_data` and therefore included during serialization. + for rf in self._attr_to_rest_field.values(): + if rf._default is _UNSET: + continue + if rf._rest_name in dict_to_pass: + continue + dict_to_pass[rf._rest_name] = _create_value(rf, rf._default) super().__init__(dict_to_pass) + def _init_from_xml( # pylint: disable=too-many-branches, too-many-statements + self, element: ET.Element + ) -> dict[str, typing.Any]: + """Deserialize an XML element into a dict mapping rest field names to values. + + :param ET.Element element: The XML element to deserialize from. + :returns: A dictionary of rest_name to deserialized value pairs. + :rtype: dict + """ + result: dict[str, typing.Any] = {} + existed_attr_keys: list[str] = [] + + field_plan = getattr(self, "_xml_field_plan", None) + if field_plan: + for rest_name, xml_name, kind, deser, rf_type, is_optional, items_name in field_plan: + if kind == 0: # wrapped element (most common) + item = element.find(xml_name) + if item is not None: + existed_attr_keys.append(xml_name) + if deser: + result[rest_name] = deser(item) + else: + result[rest_name] = _deserialize(rf_type, item) + elif kind == 1: # attribute + attr_val = element.get(xml_name) + if attr_val is not None: + existed_attr_keys.append(xml_name) + if deser: + result[rest_name] = deser(attr_val) + else: + result[rest_name] = attr_val + elif kind == 2: # unwrapped array + items = element.findall(items_name) # pyright: ignore + if len(items) > 0: + existed_attr_keys.append(items_name) + if deser: + result[rest_name] = deser(items) + else: + result[rest_name] = _deserialize(rf_type, items) + elif not is_optional: + existed_attr_keys.append(items_name) + result[rest_name] = [] + elif kind == 3: # text + if element.text is not None: + if deser: + result[rest_name] = deser(element.text) + else: + result[rest_name] = element.text + else: + model_meta = getattr(self, "_xml", {}) + for rf in self._attr_to_rest_field.values(): + prop_meta = getattr(rf, "_xml", {}) + xml_name = prop_meta.get("name", rf._rest_name) + xml_ns = _resolve_xml_ns(prop_meta, model_meta) + if xml_ns: + xml_name = "{" + xml_ns + "}" + xml_name + + # attribute + if prop_meta.get("attribute", False) and element.get(xml_name) is not None: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = _deserialize(rf._type, element.get(xml_name)) + continue + + # unwrapped element is array + if prop_meta.get("unwrapped", False): + _items_name = prop_meta.get("itemsName") + if _items_name: + xml_name = _items_name + _items_ns = prop_meta.get("itemsNs") + if _items_ns is not None: + xml_ns = _items_ns + if xml_ns: + xml_name = "{" + xml_ns + "}" + xml_name + items = element.findall(xml_name) # pyright: ignore + if len(items) > 0: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = _deserialize(rf._type, items) + elif not rf._is_optional: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = [] + continue + + # text element is primitive type + if prop_meta.get("text", False): + if element.text is not None: + result[rf._rest_name] = _deserialize(rf._type, element.text) + continue + + # wrapped element could be normal property or array + item = element.find(xml_name) + if item is not None: + existed_attr_keys.append(xml_name) + result[rf._rest_name] = _deserialize(rf._type, item) + + # rest thing is additional properties + for e in element: + if e.tag not in existed_attr_keys: + result[e.tag] = _convert_element(e) + + return result + def copy(self) -> "Model": return Model(self.__dict__) @@ -623,7 +1022,7 @@ def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping', # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object' mros = cls.__mro__[:-9][::-1] # ignore parents, and reverse the mro order - attr_to_rest_field: typing.Dict[str, _RestField] = { # map attribute name to rest_field property + attr_to_rest_field: dict[str, _RestField] = { # map attribute name to rest_field property k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type") } annotations = { @@ -638,7 +1037,10 @@ def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None)) if not rf._rest_name_input: rf._rest_name_input = attr - cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items()) + cls._attr_to_rest_field: dict[str, _RestField] = dict(attr_to_rest_field.items()) + # Build XML field plan for fast _init_from_xml (only for XML models) + if getattr(cls, "_xml", None): + cls._xml_field_plan = _build_xml_field_plan(cls, attr_to_rest_field) cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}") return super().__new__(cls) @@ -667,7 +1069,7 @@ def _deserialize(cls, data, exist_discriminators): model_meta = getattr(cls, "_xml", {}) prop_meta = getattr(discriminator, "_xml", {}) xml_name = prop_meta.get("name", discriminator._rest_name) - xml_ns = prop_meta.get("ns", model_meta.get("ns", None)) + xml_ns = _resolve_xml_ns(prop_meta, model_meta) if xml_ns: xml_name = "{" + xml_ns + "}" + xml_name @@ -680,7 +1082,7 @@ def _deserialize(cls, data, exist_discriminators): mapped_cls = cls.__mapping__.get(discriminator_value, cls) # pyright: ignore # pylint: disable=no-member return mapped_cls._deserialize(data, exist_discriminators) - def as_dict(self, *, exclude_readonly: bool = False) -> typing.Dict[str, typing.Any]: + def as_dict(self, *, exclude_readonly: bool = False) -> dict[str, typing.Any]: """Return a dict that can be turned into json using json.dump. :keyword bool exclude_readonly: Whether to remove the readonly properties. @@ -740,7 +1142,7 @@ def _deserialize_with_union(deserializers, obj): def _deserialize_dict( value_deserializer: typing.Optional[typing.Callable], module: typing.Optional[str], - obj: typing.Dict[typing.Any, typing.Any], + obj: dict[typing.Any, typing.Any], ): if obj is None: return obj @@ -750,7 +1152,7 @@ def _deserialize_dict( def _deserialize_multiple_sequence( - entry_deserializers: typing.List[typing.Optional[typing.Callable]], + entry_deserializers: list[typing.Optional[typing.Callable]], module: typing.Optional[str], obj, ): @@ -759,6 +1161,14 @@ def _deserialize_multiple_sequence( return type(obj)(_deserialize(deserializer, entry, module) for entry, deserializer in zip(obj, entry_deserializers)) +def _is_array_encoded_deserializer(deserializer: functools.partial) -> bool: + return ( + isinstance(deserializer, functools.partial) + and isinstance(deserializer.args[0], functools.partial) + and deserializer.args[0].func == _deserialize_array_encoded # pylint: disable=comparison-with-callable + ) + + def _deserialize_sequence( deserializer: typing.Optional[typing.Callable], module: typing.Optional[str], @@ -768,17 +1178,30 @@ def _deserialize_sequence( return obj if isinstance(obj, ET.Element): obj = list(obj) + + # encoded string may be deserialized to sequence + if isinstance(obj, str) and isinstance(deserializer, functools.partial): + # for list[str] + if _is_array_encoded_deserializer(deserializer): + return deserializer(obj) + + # for list[Union[...]] + if isinstance(deserializer.args[0], list): + for sub_deserializer in deserializer.args[0]: + if _is_array_encoded_deserializer(sub_deserializer): + return sub_deserializer(obj) + return type(obj)(_deserialize(deserializer, entry, module) for entry in obj) -def _sorted_annotations(types: typing.List[typing.Any]) -> typing.List[typing.Any]: +def _sorted_annotations(types: list[typing.Any]) -> list[typing.Any]: return sorted( types, key=lambda x: hasattr(x, "__name__") and x.__name__.lower() in ("str", "float", "int", "bool"), ) -def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-return-statements, too-many-branches +def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-return-statements, too-many-statements, too-many-branches annotation: typing.Any, module: typing.Optional[str], rf: typing.Optional["_RestField"] = None, @@ -818,16 +1241,18 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur # is it optional? try: - if any(a for a in annotation.__args__ if a == type(None)): # pyright: ignore + if any(a is _NONE_TYPE for a in annotation.__args__): # pyright: ignore + if rf: + rf._is_optional = True if len(annotation.__args__) <= 2: # pyright: ignore if_obj_deserializer = _get_deserialize_callable_from_annotation( - next(a for a in annotation.__args__ if a != type(None)), module, rf # pyright: ignore + next(a for a in annotation.__args__ if a is not _NONE_TYPE), module, rf # pyright: ignore ) return functools.partial(_deserialize_with_optional, if_obj_deserializer) # the type is Optional[Union[...]], we need to remove the None type from the Union annotation_copy = copy.copy(annotation) - annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a != type(None)] # pyright: ignore + annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a is not _NONE_TYPE] # pyright: ignore return _get_deserialize_callable_from_annotation(annotation_copy, module, rf) except AttributeError: pass @@ -843,7 +1268,10 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur return functools.partial(_deserialize_with_union, deserializers) try: - if annotation._name == "Dict": # pyright: ignore + annotation_name = ( + annotation.__name__ if hasattr(annotation, "__name__") else annotation._name # pyright: ignore + ) + if annotation_name.lower() == "dict": value_deserializer = _get_deserialize_callable_from_annotation( annotation.__args__[1], module, rf # pyright: ignore ) @@ -856,7 +1284,10 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur except (AttributeError, IndexError): pass try: - if annotation._name in ["List", "Set", "Tuple", "Sequence"]: # pyright: ignore + annotation_name = ( + annotation.__name__ if hasattr(annotation, "__name__") else annotation._name # pyright: ignore + ) + if annotation_name.lower() in ["list", "set", "tuple", "sequence"]: if len(annotation.__args__) > 1: # pyright: ignore entry_deserializers = [ _get_deserialize_callable_from_annotation(dt, module, rf) @@ -905,16 +1336,20 @@ def _deserialize_with_callable( return float(value.text) if value.text else None if deserializer is bool: return value.text == "true" if value.text else None + if deserializer and deserializer in _DESERIALIZE_MAPPING.values(): + return deserializer(value.text) if value.text else None + if deserializer and deserializer in _DESERIALIZE_MAPPING_WITHFORMAT.values(): + return deserializer(value.text) if value.text else None if deserializer is None: return value if deserializer in [int, float, bool]: return deserializer(value) if isinstance(deserializer, CaseInsensitiveEnumMeta): try: - return deserializer(value) + return deserializer(value.text if isinstance(value, ET.Element) else value) except ValueError: # for unknown value, return raw value - return value + return value.text if isinstance(value, ET.Element) else value if isinstance(deserializer, type) and issubclass(deserializer, Model): return deserializer._deserialize(value, []) return typing.cast(typing.Callable[[typing.Any], typing.Any], deserializer)(value) @@ -940,14 +1375,14 @@ def _deserialize( def _failsafe_deserialize( deserializer: typing.Any, - value: typing.Any, + response: HttpResponse, module: typing.Optional[str] = None, rf: typing.Optional["_RestField"] = None, format: typing.Optional[str] = None, ) -> typing.Any: try: - return _deserialize(deserializer, value, module, rf, format) - except DeserializationError: + return _deserialize(deserializer, response.json(), module, rf, format) + except Exception: # pylint: disable=broad-except _LOGGER.warning( "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True ) @@ -956,17 +1391,18 @@ def _failsafe_deserialize( def _failsafe_deserialize_xml( deserializer: typing.Any, - value: typing.Any, + response: HttpResponse, ) -> typing.Any: try: - return _deserialize_xml(deserializer, value) - except DeserializationError: + return _deserialize_xml(deserializer, response.text()) + except Exception: # pylint: disable=broad-except _LOGGER.warning( "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True ) return None +# pylint: disable=too-many-instance-attributes class _RestField: def __init__( self, @@ -974,11 +1410,12 @@ def __init__( name: typing.Optional[str] = None, type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin is_discriminator: bool = False, - visibility: typing.Optional[typing.List[str]] = None, + visibility: typing.Optional[list[str]] = None, default: typing.Any = _UNSET, format: typing.Optional[str] = None, is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, + xml: typing.Optional[dict[str, typing.Any]] = None, + deserializer: typing.Optional[typing.Callable] = None, ): self._type = type self._rest_name_input = name @@ -986,14 +1423,20 @@ def __init__( self._is_discriminator = is_discriminator self._visibility = visibility self._is_model = False + self._is_optional = False self._default = default self._format = format self._is_multipart_file_input = is_multipart_file_input self._xml = xml if xml is not None else {} + self._deserializer = deserializer @property def _class_type(self) -> typing.Any: - return getattr(self._type, "args", [None])[0] + result = getattr(self._type, "args", [None])[0] + # type may be wrapped by nested functools.partial so we need to check for that + if isinstance(result, functools.partial): + return getattr(result, "args", [None])[0] + return result @property def _rest_name(self) -> str: @@ -1004,14 +1447,44 @@ def _rest_name(self) -> str: def __get__(self, obj: Model, type=None): # pylint: disable=redefined-builtin # by this point, type and rest_name will have a value bc we default # them in __new__ of the Model class - item = obj.get(self._rest_name) + # Use _data.get() directly to avoid triggering __getitem__ which clears the cache + item = obj._data.get(self._rest_name, _UNSET) + if item is _UNSET: + # Field not set by user; return the client default if one exists, otherwise None + return self._default if self._default is not _UNSET else None if item is None: return item if self._is_model: return item - return _deserialize(self._type, _serialize(item, self._format), rf=self) + + # For mutable types, we want mutations to directly affect _data + # Check if we've already deserialized this value + cache_attr = f"_deserialized_{self._rest_name}" + if hasattr(obj, cache_attr): + # Return the value from _data directly (it's been deserialized in place) + return obj._data.get(self._rest_name) + + # Fast path: use _deserializer directly (avoids _serialize/_deserialize chain) + if self._deserializer: + deserialized = self._deserializer(item) + else: + deserialized = _deserialize(self._type, _serialize(item, self._format), rf=self) + + # For mutable types, store the deserialized value back in _data + # so mutations directly affect _data + if isinstance(deserialized, (dict, list, set)): + obj._data[self._rest_name] = deserialized + object.__setattr__(obj, cache_attr, True) # Mark as deserialized + return deserialized + + return deserialized def __set__(self, obj: Model, value) -> None: + # Clear the cached deserialized object when setting a new value + cache_attr = f"_deserialized_{self._rest_name}" + if hasattr(obj, cache_attr): + object.__delattr__(obj, cache_attr) + if value is None: # we want to wipe out entries if users set attr to None try: @@ -1036,11 +1509,12 @@ def rest_field( *, name: typing.Optional[str] = None, type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, + visibility: typing.Optional[list[str]] = None, default: typing.Any = _UNSET, format: typing.Optional[str] = None, is_multipart_file_input: bool = False, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, + xml: typing.Optional[dict[str, typing.Any]] = None, + deserializer: typing.Optional[typing.Callable] = None, ) -> typing.Any: return _RestField( name=name, @@ -1050,6 +1524,7 @@ def rest_field( format=format, is_multipart_file_input=is_multipart_file_input, xml=xml, + deserializer=deserializer, ) @@ -1057,8 +1532,8 @@ def rest_discriminator( *, name: typing.Optional[str] = None, type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, - xml: typing.Optional[typing.Dict[str, typing.Any]] = None, + visibility: typing.Optional[list[str]] = None, + xml: typing.Optional[dict[str, typing.Any]] = None, ) -> typing.Any: return _RestField(name=name, type=type, is_discriminator=True, visibility=visibility, xml=xml) @@ -1074,21 +1549,77 @@ def serialize_xml(model: Model, exclude_readonly: bool = False) -> str: return ET.tostring(_get_element(model, exclude_readonly), encoding="unicode") # type: ignore +def _get_xml_ns(meta: dict[str, typing.Any]) -> typing.Optional[str]: + """Return the XML namespace from a metadata dict, checking both 'ns' (old-style) and 'namespace' (DPG) keys. + + :param dict meta: The metadata dictionary to extract namespace from. + :returns: The namespace string if 'ns' or 'namespace' key is present, None otherwise. + :rtype: str or None + """ + ns = meta.get("ns") + if ns is None: + ns = meta.get("namespace") + return ns + + +def _resolve_xml_ns( + prop_meta: dict[str, typing.Any], model_meta: typing.Optional[dict[str, typing.Any]] = None +) -> typing.Optional[str]: + """Resolve XML namespace for a property, falling back to model namespace when appropriate. + + Checks the property metadata first; if no namespace is found and the model does not declare + an explicit prefix, falls back to the model-level namespace. + + :param dict prop_meta: The property metadata dictionary. + :param dict model_meta: The model metadata dictionary, used as fallback. + :returns: The resolved namespace string, or None. + :rtype: str or None + """ + ns = _get_xml_ns(prop_meta) + if ns is None and model_meta is not None and not model_meta.get("prefix"): + ns = _get_xml_ns(model_meta) + return ns + + +def _set_xml_attribute(element: ET.Element, name: str, value: typing.Any, prop_meta: dict[str, typing.Any]) -> None: + """Set an XML attribute on an element, handling namespace prefix registration. + + :param ET.Element element: The element to set the attribute on. + :param str name: The default attribute name (wire name). + :param any value: The attribute value. + :param dict prop_meta: The property metadata dictionary. + """ + xml_name = prop_meta.get("name", name) + _attr_ns = _get_xml_ns(prop_meta) + if _attr_ns: + _attr_prefix = prop_meta.get("prefix") + if _attr_prefix: + _safe_register_namespace(_attr_prefix, _attr_ns) + xml_name = "{" + _attr_ns + "}" + xml_name + element.set(xml_name, _get_primitive_type_value(value)) + + def _get_element( o: typing.Any, exclude_readonly: bool = False, - parent_meta: typing.Optional[typing.Dict[str, typing.Any]] = None, + parent_meta: typing.Optional[dict[str, typing.Any]] = None, wrapped_element: typing.Optional[ET.Element] = None, -) -> typing.Union[ET.Element, typing.List[ET.Element]]: +) -> typing.Union[ET.Element, list[ET.Element]]: if _is_model(o): model_meta = getattr(o, "_xml", {}) # if prop is a model, then use the prop element directly, else generate a wrapper of model if wrapped_element is None: + # When serializing as an array item (parent_meta is set), check if the parent has an + # explicit itemsName. This ensures correct element names for unwrapped arrays (where + # the element tag is the property/items name, not the model type name). + _items_name = parent_meta.get("itemsName") if parent_meta is not None else None + element_name = _items_name if _items_name else (model_meta.get("name") or o.__class__.__name__) + _model_ns = _get_xml_ns(model_meta) wrapped_element = _create_xml_element( - model_meta.get("name", o.__class__.__name__), + element_name, model_meta.get("prefix"), - model_meta.get("ns"), + _model_ns, ) readonly_props = [] @@ -1110,7 +1641,9 @@ def _get_element( # additional properties will not have rest field, use the wire name as xml name prop_meta = {"name": k} - # if no ns for prop, use model's + # Propagate model namespace to properties only for old-style "ns"-keyed models. + # DPG-generated models use the "namespace" key and explicitly declare namespace on + # each property that needs it, so propagation is intentionally skipped for them. if prop_meta.get("ns") is None and model_meta.get("ns"): prop_meta["ns"] = model_meta.get("ns") prop_meta["prefix"] = model_meta.get("prefix") @@ -1122,12 +1655,7 @@ def _get_element( # text could only set on primitive type wrapped_element.text = _get_primitive_type_value(v) elif prop_meta.get("attribute", False): - xml_name = prop_meta.get("name", k) - if prop_meta.get("ns"): - ET.register_namespace(prop_meta.get("prefix"), prop_meta.get("ns")) # pyright: ignore - xml_name = "{" + prop_meta.get("ns") + "}" + xml_name # pyright: ignore - # attribute should be primitive type - wrapped_element.set(xml_name, _get_primitive_type_value(v)) + _set_xml_attribute(wrapped_element, k, v, prop_meta) else: # other wrapped prop element wrapped_element.append(_get_wrapped_element(v, exclude_readonly, prop_meta)) @@ -1136,6 +1664,7 @@ def _get_element( return [_get_element(x, exclude_readonly, parent_meta) for x in o] # type: ignore if isinstance(o, dict): result = [] + _dict_ns = _get_xml_ns(parent_meta) if parent_meta else None for k, v in o.items(): result.append( _get_wrapped_element( @@ -1143,7 +1672,7 @@ def _get_element( exclude_readonly, { "name": k, - "ns": parent_meta.get("ns") if parent_meta else None, + "ns": _dict_ns, "prefix": parent_meta.get("prefix") if parent_meta else None, }, ) @@ -1152,13 +1681,16 @@ def _get_element( # primitive case need to create element based on parent_meta if parent_meta: + _items_ns = parent_meta.get("itemsNs") + if _items_ns is None: + _items_ns = _get_xml_ns(parent_meta) return _get_wrapped_element( o, exclude_readonly, { "name": parent_meta.get("itemsName", parent_meta.get("name")), "prefix": parent_meta.get("itemsPrefix", parent_meta.get("prefix")), - "ns": parent_meta.get("itemsNs", parent_meta.get("ns")), + "ns": _items_ns, }, ) @@ -1168,10 +1700,11 @@ def _get_element( def _get_wrapped_element( v: typing.Any, exclude_readonly: bool, - meta: typing.Optional[typing.Dict[str, typing.Any]], + meta: typing.Optional[dict[str, typing.Any]], ) -> ET.Element: + _meta_ns = _get_xml_ns(meta) if meta else None wrapped_element = _create_xml_element( - meta.get("name") if meta else None, meta.get("prefix") if meta else None, meta.get("ns") if meta else None + meta.get("name") if meta else None, meta.get("prefix") if meta else None, _meta_ns ) if isinstance(v, (dict, list)): wrapped_element.extend(_get_element(v, exclude_readonly, meta)) @@ -1179,7 +1712,7 @@ def _get_wrapped_element( _get_element(v, exclude_readonly, meta, wrapped_element) else: wrapped_element.text = _get_primitive_type_value(v) - return wrapped_element + return wrapped_element # type: ignore[no-any-return] def _get_primitive_type_value(v) -> str: @@ -1192,9 +1725,29 @@ def _get_primitive_type_value(v) -> str: return str(v) -def _create_xml_element(tag, prefix=None, ns=None): - if prefix and ns: +def _safe_register_namespace(prefix: str, ns: str) -> None: + """Register an XML namespace prefix, handling reserved prefix patterns. + + Some prefixes (e.g. 'ns2') match Python's reserved 'ns\\d+' pattern used for + auto-generated prefixes, causing register_namespace to raise ValueError. + Falls back to directly registering in the internal namespace map. + + :param str prefix: The namespace prefix to register. + :param str ns: The namespace URI. + """ + try: ET.register_namespace(prefix, ns) + except ValueError: + _ns_map = getattr(ET, "_namespace_map", None) + if _ns_map is not None: + _ns_map[ns] = prefix + + +def _create_xml_element( + tag: typing.Any, prefix: typing.Optional[str] = None, ns: typing.Optional[str] = None +) -> ET.Element: + if prefix and ns: + _safe_register_namespace(prefix, ns) if ns: return ET.Element("{" + ns + "}" + tag) return ET.Element(tag) @@ -1205,13 +1758,15 @@ def _deserialize_xml( value: str, ) -> typing.Any: element = ET.fromstring(value) # nosec + if _is_model(deserializer): + return deserializer._deserialize(element, []) return _deserialize(deserializer, element) def _convert_element(e: ET.Element): # dict case if len(e.attrib) > 0 or len({child.tag for child in e}) > 1: - dict_result: typing.Dict[str, typing.Any] = {} + dict_result: dict[str, typing.Any] = {} for child in e: if dict_result.get(child.tag) is not None: if isinstance(dict_result[child.tag], list): @@ -1224,7 +1779,7 @@ def _convert_element(e: ET.Element): return dict_result # array case if len(e) > 0: - array_result: typing.List[typing.Any] = [] + array_result: list[typing.Any] = [] for child in e: array_result.append(_convert_element(child)) return array_result diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py index eb86ea23c965..ae08f9d89f74 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_utils/serialization.py @@ -21,7 +21,6 @@ import sys import codecs from typing import ( - Dict, Any, cast, Optional, @@ -31,7 +30,6 @@ Mapping, Callable, MutableMapping, - List, ) try: @@ -41,11 +39,15 @@ import xml.etree.ElementTree as ET import isodate # type: ignore -from typing_extensions import Self from azure.core.exceptions import DeserializationError, SerializationError from azure.core.serialization import NULL as CoreNull +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + _BOM = codecs.BOM_UTF8.decode(encoding="utf-8") JSON = MutableMapping[str, Any] @@ -229,12 +231,12 @@ class Model: serialization and deserialization. """ - _subtype_map: Dict[str, Dict[str, Any]] = {} - _attribute_map: Dict[str, Dict[str, Any]] = {} - _validation: Dict[str, Dict[str, Any]] = {} + _subtype_map: dict[str, dict[str, Any]] = {} + _attribute_map: dict[str, dict[str, Any]] = {} + _validation: dict[str, dict[str, Any]] = {} def __init__(self, **kwargs: Any) -> None: - self.additional_properties: Optional[Dict[str, Any]] = {} + self.additional_properties: Optional[dict[str, Any]] = {} for k in kwargs: # pylint: disable=consider-using-dict-items if k not in self._attribute_map: _LOGGER.warning("%s is not a known attribute of class %s and will be ignored", k, self.__class__) @@ -311,7 +313,7 @@ def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> JSON: def as_dict( self, keep_readonly: bool = True, - key_transformer: Callable[[str, Dict[str, Any], Any], Any] = attribute_transformer, + key_transformer: Callable[[str, dict[str, Any], Any], Any] = attribute_transformer, **kwargs: Any ) -> JSON: """Return a dict that can be serialized using json.dump. @@ -380,7 +382,7 @@ def deserialize(cls, data: Any, content_type: Optional[str] = None) -> Self: def from_dict( cls, data: Any, - key_extractors: Optional[Callable[[str, Dict[str, Any], Any], Any]] = None, + key_extractors: Optional[Callable[[str, dict[str, Any], Any], Any]] = None, content_type: Optional[str] = None, ) -> Self: """Parse a dict using given key extractor return a model. @@ -414,7 +416,7 @@ def _flatten_subtype(cls, key, objects): return {} result = dict(cls._subtype_map[key]) for valuetype in cls._subtype_map[key].values(): - result.update(objects[valuetype]._flatten_subtype(key, objects)) # pylint: disable=protected-access + result |= objects[valuetype]._flatten_subtype(key, objects) # pylint: disable=protected-access return result @classmethod @@ -478,7 +480,11 @@ def _decode_attribute_map_key(key): class Serializer: # pylint: disable=too-many-public-methods - """Request object model serializer.""" + """Request object model serializer. + + :param classes: Mapping of model names to model types, used to resolve models during serialization. + :type classes: typing.Optional[typing.Mapping[str, type]] + """ basic_types = {str: "str", int: "int", bool: "bool", float: "float"} @@ -518,6 +524,10 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: "rfc-1123": Serializer.serialize_rfc, "unix-time": Serializer.serialize_unix, "duration": Serializer.serialize_duration, + "duration-seconds-int": Serializer.serialize_duration_seconds_int, + "duration-seconds-float": Serializer.serialize_duration_seconds_float, + "duration-milliseconds-int": Serializer.serialize_duration_milliseconds_int, + "duration-milliseconds-float": Serializer.serialize_duration_milliseconds_float, "date": Serializer.serialize_date, "time": Serializer.serialize_time, "decimal": Serializer.serialize_decimal, @@ -528,7 +538,7 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: "[]": self.serialize_iter, "{}": self.serialize_dict, } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} + self.dependencies: dict[str, type] = dict(classes) if classes else {} self.key_transformer = full_restapi_key_transformer self.client_side_validation = True @@ -579,7 +589,7 @@ def _serialize( # pylint: disable=too-many-nested-blocks, too-many-branches, to if attr_name == "additional_properties" and attr_desc["key"] == "": if target_obj.additional_properties is not None: - serialized.update(target_obj.additional_properties) + serialized |= target_obj.additional_properties continue try: @@ -789,7 +799,7 @@ def serialize_data(self, data, data_type, **kwargs): # If dependencies is empty, try with current data class # It has to be a subclass of Enum anyway - enum_type = self.dependencies.get(data_type, data.__class__) + enum_type = self.dependencies.get(data_type, cast(type, data.__class__)) if issubclass(enum_type, Enum): return Serializer.serialize_enum(data, enum_obj=enum_type) @@ -823,13 +833,20 @@ def serialize_basic(cls, data, data_type, **kwargs): :param str data_type: Type of object in the iterable. :rtype: str, int, float, bool :return: serialized object + :raises TypeError: raise if data_type is not one of str, int, float, bool. """ custom_serializer = cls._get_custom_serializers(data_type, **kwargs) if custom_serializer: return custom_serializer(data) if data_type == "str": return cls.serialize_unicode(data) - return eval(data_type)(data) # nosec # pylint: disable=eval-used + if data_type == "int": + return int(data) + if data_type == "float": + return float(data) + if data_type == "bool": + return bool(data) + raise TypeError("Unknown basic data type: {}".format(data_type)) @classmethod def serialize_unicode(cls, data): @@ -1100,6 +1117,61 @@ def serialize_duration(attr, **kwargs): # pylint: disable=unused-argument attr = isodate.parse_duration(attr) return isodate.duration_isoformat(attr) + @staticmethod + def _serialize_duration_numeric(attr, scale, as_int): + """Serialize a TimeDelta into a numeric value scaled to the wire unit. + + :param TimeDelta attr: Object to be serialized. + :param int scale: Multiplier applied to total seconds (1 for seconds, 1000 for milliseconds). + :param bool as_int: Whether to truncate the result to an int. + :rtype: int or float + :return: serialized duration + """ + if isinstance(attr, str): + attr = isodate.parse_duration(attr) + value = attr.total_seconds() * scale if isinstance(attr, datetime.timedelta) else attr + return int(value) if as_int else float(value) + + @staticmethod + def serialize_duration_seconds_int(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into an integer number of seconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: int + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1, True) + + @staticmethod + def serialize_duration_seconds_float(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into a floating point number of seconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: float + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1, False) + + @staticmethod + def serialize_duration_milliseconds_int(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into an integer number of milliseconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: int + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1000, True) + + @staticmethod + def serialize_duration_milliseconds_float(attr, **kwargs): # pylint: disable=unused-argument + """Serialize TimeDelta object into a floating point number of milliseconds. + + :param TimeDelta attr: Object to be serialized. + :rtype: float + :return: serialized duration + """ + return Serializer._serialize_duration_numeric(attr, 1000, False) + @staticmethod def serialize_rfc(attr, **kwargs): # pylint: disable=unused-argument """Serialize Datetime object into RFC-1123 formatted string. @@ -1184,7 +1256,7 @@ def rest_key_extractor(attr, attr_desc, data): # pylint: disable=unused-argumen while "." in key: # Need the cast, as for some reasons "split" is typed as list[str | Any] - dict_keys = cast(List[str], _FLATTEN.split(key)) + dict_keys = cast(list[str], _FLATTEN.split(key)) if len(dict_keys) == 1: key = _decode_attribute_map_key(dict_keys[0]) break @@ -1372,6 +1444,10 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: "rfc-1123": Deserializer.deserialize_rfc, "unix-time": Deserializer.deserialize_unix, "duration": Deserializer.deserialize_duration, + "duration-seconds-int": Deserializer.deserialize_duration_seconds, + "duration-seconds-float": Deserializer.deserialize_duration_seconds, + "duration-milliseconds-int": Deserializer.deserialize_duration_milliseconds, + "duration-milliseconds-float": Deserializer.deserialize_duration_milliseconds, "date": Deserializer.deserialize_date, "time": Deserializer.deserialize_time, "decimal": Deserializer.deserialize_decimal, @@ -1384,9 +1460,13 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: } self.deserialize_expected_types = { "duration": (isodate.Duration, datetime.timedelta), + "duration-seconds-int": (isodate.Duration, datetime.timedelta), + "duration-seconds-float": (isodate.Duration, datetime.timedelta), + "duration-milliseconds-int": (isodate.Duration, datetime.timedelta), + "duration-milliseconds-float": (isodate.Duration, datetime.timedelta), "iso-8601": (datetime.datetime), } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} + self.dependencies: dict[str, type] = dict(classes) if classes else {} self.key_extractors = [rest_key_extractor, xml_key_extractor] # Additional properties only works if the "rest_key_extractor" is used to # extract the keys. Making it to work whatever the key extractor is too much @@ -1396,7 +1476,7 @@ def __init__(self, classes: Optional[Mapping[str, type]] = None) -> None: # Otherwise, result are unexpected self.additional_properties_detection = True - def __call__(self, target_obj, response_data, content_type=None): + def __call__(self, target_obj, response_data, content_type=None): # pylint: disable=too-many-return-statements """Call the deserializer to process a REST response. :param str target_obj: Target data type to deserialize to. @@ -1406,6 +1486,27 @@ def __call__(self, target_obj, response_data, content_type=None): :return: Deserialized object. :rtype: object """ + # Fast path for header deserialization: response_data is a plain str or None + # and target_obj is a simple scalar type. This avoids the expensive + # _unpack_content → _deserialize → _classify_target → deserialize_data chain. + if response_data is None: + return None + if target_obj == "str" and isinstance(response_data, str): + return response_data + if isinstance(response_data, str): + if target_obj == "int": + return int(response_data) + if target_obj == "bool": + if response_data in ("true", "1", "True"): + return True + if response_data in ("false", "0", "False"): + return False + return bool(response_data) + if target_obj == "rfc-1123": + return Deserializer.deserialize_rfc(response_data) + if target_obj == "bytearray": + return Deserializer.deserialize_bytearray(response_data) + data = self._unpack_content(response_data, content_type) return self._deserialize(target_obj, data) @@ -1759,7 +1860,7 @@ def deserialize_basic(self, attr, data_type): # pylint: disable=too-many-return :param str data_type: deserialization data type. :return: Deserialized basic type. :rtype: str, int, float or bool - :raises TypeError: if string format is not valid. + :raises TypeError: if string format is not valid or data_type is not one of str, int, float, bool. """ # If we're here, data is supposed to be a basic type. # If it's still an XML node, take the text @@ -1785,7 +1886,11 @@ def deserialize_basic(self, attr, data_type): # pylint: disable=too-many-return if data_type == "str": return self.deserialize_unicode(attr) - return eval(data_type)(attr) # nosec # pylint: disable=eval-used + if data_type == "int": + return int(attr) + if data_type == "float": + return float(attr) + raise TypeError("Unknown basic data type: {}".format(data_type)) @staticmethod def deserialize_unicode(data): @@ -1920,6 +2025,48 @@ def deserialize_duration(attr): raise DeserializationError(msg) from err return duration + @staticmethod + def _deserialize_duration_numeric(attr, unit): + """Deserialize a numeric duration value into a TimeDelta object. + + :param float attr: response value to be deserialized. + :param str unit: The wire unit, used as the ``timedelta`` keyword + (``"seconds"`` or ``"milliseconds"``). + :return: Deserialized duration + :rtype: TimeDelta + :raises DeserializationError: if value is invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + try: + duration = datetime.timedelta(**{unit: float(attr)}) # type: ignore + except (ValueError, OverflowError, TypeError) as err: + msg = "Cannot deserialize duration object." + raise DeserializationError(msg) from err + return duration + + @staticmethod + def deserialize_duration_seconds(attr): + """Deserialize a numeric number of seconds into a TimeDelta object. + + :param float attr: response value to be deserialized. + :return: Deserialized duration + :rtype: TimeDelta + :raises DeserializationError: if value is invalid. + """ + return Deserializer._deserialize_duration_numeric(attr, "seconds") + + @staticmethod + def deserialize_duration_milliseconds(attr): + """Deserialize a numeric number of milliseconds into a TimeDelta object. + + :param float attr: response value to be deserialized. + :return: Deserialized duration + :rtype: TimeDelta + :raises DeserializationError: if value is invalid. + """ + return Deserializer._deserialize_duration_numeric(attr, "milliseconds") + @staticmethod def deserialize_date(attr): """Deserialize ISO-8601 formatted string into Date object. diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_vendor.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_vendor.py deleted file mode 100644 index 5af45cbb9df5..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_vendor.py +++ /dev/null @@ -1,25 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from abc import ABC -from typing import TYPE_CHECKING - -from ._configuration import DeidentificationClientConfiguration - -if TYPE_CHECKING: - from azure.core import PipelineClient - - from ._serialization import Deserializer, Serializer - - -class DeidentificationClientMixinABC(ABC): - """DO NOT use this class. It is for internal typing use only.""" - - _client: "PipelineClient" - _config: DeidentificationClientConfiguration - _serialize: "Serializer" - _deserialize: "Deserializer" diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py index a1f432eddc4e..f6cd39818b76 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/_version.py @@ -6,4 +6,4 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -VERSION = "1.1.0b1" +VERSION = "1.1.0b2" diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py index 7f413f77a0fb..085d9fa3c313 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_client.py @@ -7,8 +7,8 @@ # -------------------------------------------------------------------------- from copy import deepcopy +import sys from typing import Any, Awaitable, TYPE_CHECKING -from typing_extensions import Self from azure.core import AsyncPipelineClient from azure.core.pipeline import policies @@ -18,19 +18,27 @@ from ._configuration import DeidentificationClientConfiguration from ._operations import _DeidentificationClientOperationsMixin +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self # type: ignore + if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential -class DeidentificationClient(_DeidentificationClientOperationsMixin): +class DeidentificationClient( + _DeidentificationClientOperationsMixin +): # pylint: disable=docstring-keyword-should-match-keyword-only """DeidentificationClient. :param endpoint: Url of your De-identification Service. Required. :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials_async.AsyncTokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str :keyword int polling_interval: Default waiting time between two polls for LRO operations if no diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py index f389c8624735..087ecd2812ca 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_configuration.py @@ -1,3 +1,4 @@ +# pylint: disable=line-too-long,useless-suppression # coding=utf-8 # -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. @@ -16,7 +17,7 @@ from azure.core.credentials_async import AsyncTokenCredential -class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes +class DeidentificationClientConfiguration: # pylint: disable=too-many-instance-attributes,docstring-keyword-should-match-keyword-only """Configuration for DeidentificationClient. Note that all parameters used to create this instance are saved as instance @@ -26,8 +27,9 @@ class DeidentificationClientConfiguration: # pylint: disable=too-many-instance- :type endpoint: str :param credential: Credential used to authenticate requests to the service. Required. :type credential: ~azure.core.credentials_async.AsyncTokenCredential - :keyword api_version: The API version to use for this operation. Default value is - "2025-07-15-preview". Note that overriding this default value may result in unsupported + :keyword api_version: The API version to use for this operation. Known values are + "2025-07-15-preview" and None. Default value is None. If not set, the operation's default API + version will be used. Note that overriding this default value may result in unsupported behavior. :paramtype api_version: str """ diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py index fe6452408aba..39d077dd229f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_operations/_operations.py @@ -9,7 +9,7 @@ from collections.abc import MutableMapping from io import IOBase import json -from typing import Any, AsyncIterator, Callable, Dict, IO, List, Optional, TypeVar, Union, cast, overload +from typing import Any, AsyncIterator, Callable, IO, Optional, TypeVar, Union, cast, overload import urllib.parse from azure.core import AsyncPipelineClient @@ -48,7 +48,7 @@ JSON = MutableMapping[str, Any] T = TypeVar("T") -ClsType = Optional[Callable[[PipelineResponse[HttpRequest, AsyncHttpResponse], T, Dict[str, Any]], Any]] +ClsType = Optional[Callable[[PipelineResponse[HttpRequest, AsyncHttpResponse], T, dict[str, Any]], Any]] class _DeidentificationClientOperationsMixin( @@ -91,6 +91,7 @@ async def get_job(self, job_name: str, **kwargs: Any) -> _models.Deidentificatio } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -113,7 +114,7 @@ async def get_job(self, job_name: str, **kwargs: Any) -> _models.Deidentificatio ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -159,6 +160,7 @@ async def _deidentify_documents_initial( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -180,7 +182,7 @@ async def _deidentify_documents_initial( ) response_headers["Operation-Location"] = self._deserialize("str", response.headers.get("Operation-Location")) - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -357,7 +359,7 @@ def _list_jobs_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationJob]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationJob]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -395,7 +397,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -408,7 +413,10 @@ def prepare_request(next_link=None): async def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationJob], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationJob], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, AsyncList(list_of_elem) @@ -452,7 +460,7 @@ def _list_job_documents_internal( _params = kwargs.pop("params", {}) or {} maxpagesize = kwargs.pop("maxpagesize", None) - cls: ClsType[List[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) + cls: ClsType[list[_models.DeidentificationDocumentDetails]] = kwargs.pop("cls", None) error_map: MutableMapping = { 401: ClientAuthenticationError, @@ -491,7 +499,10 @@ def prepare_request(next_link=None): ) _next_request_params["api-version"] = self._config.api_version _request = HttpRequest( - "GET", urllib.parse.urljoin(next_link, _parsed_next_link.path), params=_next_request_params + "GET", + urllib.parse.urljoin(next_link, _parsed_next_link.path), + headers=_headers, + params=_next_request_params, ) path_format_arguments = { "endpoint": self._serialize.url( @@ -504,7 +515,10 @@ def prepare_request(next_link=None): async def extract_data(pipeline_response): deserialized = pipeline_response.http_response.json() - list_of_elem = _deserialize(List[_models.DeidentificationDocumentDetails], deserialized.get("value", [])) + list_of_elem = _deserialize( + list[_models.DeidentificationDocumentDetails], + deserialized.get("value", []), + ) if cls: list_of_elem = cls(list_of_elem) # type: ignore return deserialized.get("nextLink") or None, AsyncList(list_of_elem) @@ -567,6 +581,7 @@ async def cancel_job(self, job_name: str, **kwargs: Any) -> _models.Deidentifica } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -589,7 +604,7 @@ async def cancel_job(self, job_name: str, **kwargs: Any) -> _models.Deidentifica ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationJob, response.json()) @@ -755,6 +770,7 @@ async def deidentify_text( } _request.url = self._client.format_url(_request.url, **path_format_arguments) + _decompress = kwargs.pop("decompress", True) _stream = kwargs.pop("stream", False) pipeline_response: PipelineResponse = await self._client._pipeline.run( # type: ignore # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -777,7 +793,7 @@ async def deidentify_text( ) if _stream: - deserialized = response.iter_bytes() + deserialized = response.iter_bytes() if _decompress else response.iter_raw() else: deserialized = _deserialize(_models.DeidentificationResult, response.json()) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_vendor.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_vendor.py deleted file mode 100644 index 0afc83d417cc..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/aio/_vendor.py +++ /dev/null @@ -1,25 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from abc import ABC -from typing import TYPE_CHECKING - -from ._configuration import DeidentificationClientConfiguration - -if TYPE_CHECKING: - from azure.core import AsyncPipelineClient - - from .._serialization import Deserializer, Serializer - - -class DeidentificationClientMixinABC(ABC): - """DO NOT use this class. It is for internal typing use only.""" - - _client: "AsyncPipelineClient" - _config: DeidentificationClientConfiguration - _serialize: "Serializer" - _deserialize: "Deserializer" diff --git a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py index 4cbd2dc93f44..633ff8e199e5 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py @@ -1,4 +1,3 @@ -# pylint: disable=line-too-long,useless-suppression # coding=utf-8 # -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. @@ -9,7 +8,7 @@ # pylint: disable=useless-super-delegation import datetime -from typing import Any, List, Mapping, Optional, TYPE_CHECKING, Union, overload +from typing import Any, Mapping, Optional, TYPE_CHECKING, Union, overload from azure.core.exceptions import ODataV4Format @@ -19,7 +18,7 @@ from .. import models as _models -class DeidentificationContent(_Model): +class DeidentificationContent(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Request body for de-identification operation. :ivar input_text: Input text to de-identify. Required. @@ -73,7 +72,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationCustomizationOptions(_Model): +class DeidentificationCustomizationOptions(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Customizations options to override default service behaviors for synchronous usage. :ivar redaction_format: Format of the redacted output. Only valid when Operation is Redact. @@ -92,8 +91,7 @@ class DeidentificationCustomizationOptions(_Model): redaction_format: Optional[str] = rest_field( name="redactionFormat", visibility=["read", "create", "update", "delete", "query"] ) - """Format of the redacted output. Only valid when Operation is Redact. - Please refer to + """Format of the redacted output. Only valid when Operation is Redact. Please refer to `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format `_ for more details.""" @@ -126,7 +124,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationDocumentDetails(_Model): +class DeidentificationDocumentDetails(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Details of a single document in a job. :ivar id: Id of the document details. Required. @@ -182,7 +180,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationDocumentLocation(_Model): +class DeidentificationDocumentLocation(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Location of a document. :ivar location: Location of document in storage. Required. @@ -214,7 +212,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationJob(_Model): +class DeidentificationJob(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """A job containing a batch of documents to de-identify. :ivar job_name: The name of a job. Required. @@ -307,7 +305,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationJobCustomizationOptions(_Model): +class DeidentificationJobCustomizationOptions(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Customizations options to override default service behaviors for job usage. :ivar redaction_format: Format of the redacted output. Only valid when Operation is Redact. @@ -326,8 +324,7 @@ class DeidentificationJobCustomizationOptions(_Model): redaction_format: Optional[str] = rest_field( name="redactionFormat", visibility=["read", "create", "update", "delete", "query"] ) - """Format of the redacted output. Only valid when Operation is Redact. - Please refer to + """Format of the redacted output. Only valid when Operation is Redact. Please refer to `https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format `_ for more details.""" @@ -360,7 +357,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationJobSummary(_Model): +class DeidentificationJobSummary(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Summary metrics of a job. :ivar successful_count: Number of documents that have completed. Required. @@ -408,7 +405,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class DeidentificationResult(_Model): +class DeidentificationResult(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Response body for de-identification operation. :ivar output_text: Output text after de-identification. Not available for "Tag" operation. @@ -445,7 +442,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class PhiEntity(_Model): +class PhiEntity(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """PHI Entity tag in the input. :ivar category: PHI Category of the entity. Required. Known values are: "Unknown", "Account", @@ -503,21 +500,21 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class PhiTaggerResult(_Model): +class PhiTaggerResult(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Result of the "Tag" operation. :ivar entities: List of entities detected in the input. Required. :vartype entities: list[~azure.health.deidentification.models.PhiEntity] """ - entities: List["_models.PhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + entities: list["_models.PhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) """List of entities detected in the input. Required.""" @overload def __init__( self, *, - entities: List["_models.PhiEntity"], + entities: list["_models.PhiEntity"], ) -> None: ... @overload @@ -531,7 +528,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class SimplePhiEntity(_Model): +class SimplePhiEntity(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Simple PHI entity with encoding-specific offset and length values. :ivar category: PHI Category of the entity. Required. Known values are: "Unknown", "Account", @@ -583,7 +580,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class SourceStorageLocation(_Model): +class SourceStorageLocation(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Storage location. :ivar location: URL to storage location. Required. @@ -598,7 +595,7 @@ class SourceStorageLocation(_Model): """URL to storage location. Required.""" prefix: str = rest_field(visibility=["read", "create", "update", "delete", "query"]) """Prefix to filter path by. Required.""" - extensions: Optional[List[str]] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + extensions: Optional[list[str]] = rest_field(visibility=["read", "create", "update", "delete", "query"]) """List of extensions to filter path by.""" @overload @@ -607,7 +604,7 @@ def __init__( *, location: str, prefix: str, - extensions: Optional[List[str]] = None, + extensions: Optional[list[str]] = None, ) -> None: ... @overload @@ -621,7 +618,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class StringIndex(_Model): +class StringIndex(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """String index encoding model. :ivar utf8: The offset or length of the substring in UTF-8 encoding. Required. @@ -667,7 +664,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class TaggedPhiEntities(_Model): +class TaggedPhiEntities(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Grouped PHI entities with shared encoding specification. :ivar encoding: The encoding type used for all entities in this group. Required. Known values @@ -682,7 +679,7 @@ class TaggedPhiEntities(_Model): ) """The encoding type used for all entities in this group. Required. Known values are: \"Utf8\", \"Utf16\", and \"CodePoint\".""" - entities: List["_models.SimplePhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + entities: list["_models.SimplePhiEntity"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) """List of PHI entities using the specified encoding. Required.""" @overload @@ -690,7 +687,7 @@ def __init__( self, *, encoding: Union[str, "_models.TextEncodingType"], - entities: List["_models.SimplePhiEntity"], + entities: list["_models.SimplePhiEntity"], ) -> None: ... @overload @@ -704,7 +701,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class TargetStorageLocation(_Model): +class TargetStorageLocation(_Model): # pylint: disable=docstring-keyword-should-match-keyword-only """Storage location. :ivar location: URL to storage location. Required. diff --git a/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml b/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml index f1c500aa2f77..171eea727a09 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml +++ b/sdk/healthdataaiservices/azure-health-deidentification/pyproject.toml @@ -1,3 +1,65 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# Code generated by Microsoft (R) Python Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is regenerated. +# -------------------------------------------------------------------------- + +[build-system] +requires = ["setuptools>=77.0.3", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-health-deidentification" +authors = [ + { name = "Microsoft Corporation", email = "azpysdkhelp@microsoft.com" }, +] +description = "Microsoft Corporation Azure Health Deidentification Client Library for Python" +license = "MIT" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +requires-python = ">=3.10" +keywords = ["azure", "azure sdk"] + +dependencies = [ + "isodate>=0.6.1", + "azure-core>=1.37.0", + "typing-extensions>=4.6.0", +] +dynamic = [ +"version", "readme" +] + +[project.urls] +repository = "https://github.com/Azure/azure-sdk-for-python" + +[tool.setuptools.dynamic] +version = {attr = "azure.health.deidentification._version.VERSION"} +readme = {file = ["README.md", "CHANGELOG.md"], content-type = "text/markdown"} + +[tool.setuptools.packages.find] +exclude = [ + "tests*", + "generated_tests*", + "samples*", + "generated_samples*", + "doc*", + "azure", + "azure.health", +] + +[tool.setuptools.package-data] +pytyped = ["py.typed"] + [tool.azure-sdk-build] black = true diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py index fe065600659e..9a7bc853a453 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py @@ -23,6 +23,7 @@ import asyncio +from azure.core.exceptions import HttpResponseError from azure.core.polling import AsyncLROPoller from azure.health.deidentification.aio import DeidentificationClient from azure.health.deidentification.models import ( @@ -54,11 +55,19 @@ async def deidentify_documents_async(): target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) - async with client: + async with credential, client: lro: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = await lro.result() - - await credential.close() + try: + finished_job: DeidentificationJob = await lro.result() + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") + raise print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") # Succeeded diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py index 54ea3ec0464c..99fc5fb234eb 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py @@ -22,6 +22,7 @@ import asyncio +from azure.core.exceptions import HttpResponseError from azure.core.polling import AsyncLROPoller from azure.health.deidentification.aio import DeidentificationClient from azure.health.deidentification.models import ( @@ -54,9 +55,19 @@ async def list_job_documents_async(): ) print(f"Creating job with name: {jobname}") - async with client: + async with credential, client: poller: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - job = await poller.result() + try: + job = await poller.result() + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") + raise print(f"Job Status: {job.status}") files = client.list_job_documents(jobname) @@ -70,8 +81,6 @@ async def list_job_documents_async(): if filesToLookThrough <= 0: break - await credential.close() - async def main(): await list_job_documents_async() diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py index ac6e2c05026d..e85e61245f9e 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py @@ -26,6 +26,7 @@ def deidentify_documents(): # [START sample] + from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -57,7 +58,17 @@ def deidentify_documents(): target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) - finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) + try: + finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") + raise print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py index 1322e0a75285..c79acadd732f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py @@ -23,6 +23,7 @@ """ +from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -56,7 +57,17 @@ def list_job_documents(): print(f"Creating job with name: {jobname}") - job = client.begin_deidentify_documents(jobname, job).result(timeout=60) + try: + job = client.begin_deidentify_documents(jobname, job).result(timeout=60) + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + # The lookup is best-effort so the original error is always re-raised. + try: + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + except Exception as lookup_error: + print(f"Could not retrieve job {jobname} for diagnostics: {lookup_error}") + raise print(f"Job Status: {job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/setup.py b/sdk/healthdataaiservices/azure-health-deidentification/setup.py deleted file mode 100644 index 1fd15892a972..000000000000 --- a/sdk/healthdataaiservices/azure-health-deidentification/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) Python Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - - -import os -import re -from setuptools import setup, find_packages - - -PACKAGE_NAME = "azure-health-deidentification" -PACKAGE_PPRINT_NAME = "Azure Health Deidentification" - -# a-b-c => a/b/c -package_folder_path = PACKAGE_NAME.replace("-", "/") - -# Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, "_version.py"), "r") as fd: - version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) - -if not version: - raise RuntimeError("Cannot find version information") - - -setup( - name=PACKAGE_NAME, - version=version, - description="Microsoft Corporation {} Client Library for Python".format(PACKAGE_PPRINT_NAME), - long_description=open("README.md", "r").read(), - long_description_content_type="text/markdown", - license="MIT License", - author="Microsoft Corporation", - author_email="azpysdkhelp@microsoft.com", - url="https://github.com/Azure/azure-sdk-for-python/tree/main/sdk", - keywords="azure, azure sdk", - classifiers=[ - "Development Status :: 4 - Beta", - "Programming Language :: Python", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "License :: OSI Approved :: MIT License", - ], - zip_safe=False, - packages=find_packages( - exclude=[ - "tests", - # Exclude packages that will be covered by PEP420 or nspkg - "azure", - "azure.health", - ] - ), - include_package_data=True, - package_data={ - "azure.health.deidentification": ["py.typed"], - }, - install_requires=[ - "isodate>=0.6.1", - "azure-core>=1.30.0", - "typing-extensions>=4.6.0", - ], - python_requires=">=3.9", -) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 index c162558889fd..084db7465cfa 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 +++ b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 @@ -27,6 +27,44 @@ if ([string]::IsNullOrWhiteSpace($storageAccountName)) { exit 1 } +# Retries a script block with exponential backoff until it succeeds or the time budget is exhausted. +# Role assignments and network ACL changes can take minutes to propagate after deployment. +function Invoke-WithRetry { + param ( + [Parameter(Mandatory = $true)] + [scriptblock] $ScriptBlock, + [Parameter(Mandatory = $true)] + [string] $Operation, + [int] $MaxTotalSeconds = 300, + [int] $InitialDelaySeconds = 5, + [int] $MaxDelaySeconds = 60 + ) + + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + $delay = $InitialDelaySeconds + $attempt = 0 + while ($true) { + $attempt++ + try { + $result = & $ScriptBlock + if ($attempt -gt 1) { + Write-Host "$Operation succeeded on attempt $attempt after $([int]$stopwatch.Elapsed.TotalSeconds)s." + } + return $result + } + catch { + $lastError = $_ + $elapsed = $stopwatch.Elapsed.TotalSeconds + if ($elapsed + $delay -gt $MaxTotalSeconds) { + throw "$Operation failed after $attempt attempt(s) over $([int]$elapsed)s. This usually means the storage role assignment or network ACL change has not propagated. Last error: $($lastError.Exception.Message)" + } + Write-Host "$Operation failed (attempt $attempt, $([int]$elapsed)s elapsed): $($lastError.Exception.Message). Retrying in ${delay}s." + Start-Sleep -Seconds $delay + $delay = [Math]::Min($delay * 2, $MaxDelaySeconds) + } + } +} + # Load the Azure Storage module Import-Module Az.Storage @@ -36,25 +74,36 @@ $storageContext = New-AzStorageContext -StorageAccountName $storageAccountName - # FIXME Remove once vpn team fixes the network acl issue $networkRuleSet = New-Object -TypeName Microsoft.Azure.Commands.Management.Storage.Models.PSNetworkRuleSet $networkRuleSet.DefaultAction = "Allow" -Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet - -# Sleep for 30 seconds to allow the network rule to take effect -Write-Host "[Fix] Temporary sleep to allow network rule to take effect." -Start-Sleep -Seconds 30 +Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet -ErrorAction Stop +$sinceNetworkRuleUpdate = [System.Diagnostics.Stopwatch]::StartNew() -Get-AzStorageContainer -Name $containerName -Context $storageContext +# Wait until the container is reachable through the data plane (RBAC + network ACL propagated) +Invoke-WithRetry -Operation "Accessing container '$containerName'" -ScriptBlock { + Get-AzStorageContainer -Name $containerName -Context $storageContext -ErrorAction Stop +} # Upload the folder and its contents to the container # Gets last folder name + filename. example_patient_1/doctor_dictation.txt -Get-ChildItem -Path $localFolderPath -Recurse | ForEach-Object { - $relativePath = $_.FullName - $folderName = Split-Path -Path (Split-Path -Path $relativePath -Parent) -Leaf - $blobName = Split-Path -Path $relativePath -Leaf +Get-ChildItem -Path $localFolderPath -Recurse -File | ForEach-Object { + $filePath = $_.FullName + $folderName = Split-Path -Path (Split-Path -Path $filePath -Parent) -Leaf + $blobName = Split-Path -Path $filePath -Leaf $destinationBlob = $blobName -replace ":", "" $destinationBlob = "$folderName/$destinationBlob" Write-Host "Uploading file '$destinationBlob'" - Set-AzStorageBlobContent -File $_.FullName -Container $containerName -Blob $destinationBlob -Context $storageContext -Force + Invoke-WithRetry -Operation "Uploading blob '$destinationBlob'" -ScriptBlock { + Set-AzStorageBlobContent -File $filePath -Container $containerName -Blob $destinationBlob -Context $storageContext -Force -ErrorAction Stop + } } Write-Host "Folder '$localFolderPath' uploaded to container '$containerName' successfully." + +# Our own access proves the test principal's RBAC and the network rule, but not the DeID service's managed +# identity role assignment. Keep a minimum wait after the network rule update for it to propagate. +$minimumWaitSeconds = 30 +$remainingSeconds = [int][Math]::Ceiling($minimumWaitSeconds - $sinceNetworkRuleUpdate.Elapsed.TotalSeconds) +if ($remainingSeconds -gt 0) { + Write-Host "Waiting ${remainingSeconds}s more so at least ${minimumWaitSeconds}s pass after the network rule update for role assignments to propagate." + Start-Sleep -Seconds $remainingSeconds +} diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py index 625400893a1f..285b4a1f016a 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py @@ -4,6 +4,7 @@ from devtools_testutils import ( add_uri_regex_sanitizer, remove_batch_sanitizers, + set_custom_default_matcher, test_proxy, ) @@ -21,16 +22,18 @@ def start_proxy(test_proxy, patch_sleep, patch_async_sleep): @pytest.fixture(scope="session", autouse=True) def create_session_uniquifier(): - if ( - os.environ.get("AZURE_TEST_RUN_LIVE", "false").lower() == "true" # Don't override uniquifier by default - and os.environ.get("AZURE_SKIP_LIVE_RECORDING", "false").lower() != "true" - and os.environ.get("AZURE_TEST_KEEP_UNIQUIFIER", "false").lower() != "true" - ): - print("Creating new uniquifier for live test run.") + is_live = os.environ.get("AZURE_TEST_RUN_LIVE", "false").lower() == "true" + skip_recording = os.environ.get("AZURE_SKIP_LIVE_RECORDING", "false").lower() == "true" + keep_uniquifier = os.environ.get("AZURE_TEST_KEEP_UNIQUIFIER", "false").lower() == "true" + if is_live and not keep_uniquifier: + # Every live run gets fresh job names so reused resources or parallel matrix legs can't collide. uniquifier = uuid.uuid4().hex[:6] + print(f"Creating new uniquifier for live test run: {uniquifier}") os.environ["HEALTHDATAAISERVICES_UNIQUIFIER"] = uniquifier - with open(uniquifier_file, "w") as file: - file.write(uniquifier) + if not skip_recording: + # Persist only when recording so playback replays the job names in the new recordings. + with open(uniquifier_file, "w") as file: + file.write(uniquifier) else: with open(uniquifier_file, "r") as file: uniquifier = file.read() @@ -44,3 +47,6 @@ def add_sanitizers(test_proxy): # uri sanitization in favor of substitution remove_batch_sanitizers(["AZSDK3493", "AZSDK3430", "AZSDK4001"]) add_uri_regex_sanitizer(regex='continuationToken=[^&"}]*', value="continuationToken=Sanitized") + # Regenerated delete_job no longer sends "Accept: application/json" for its 204 response; existing + # recordings still contain it. + set_custom_default_matcher(excluded_headers="Accept") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py index cb9bb597f468..1f79b796cf24 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py @@ -1,13 +1,24 @@ +import asyncio import functools import inspect import os import random import datetime import uuid +from typing import List, Optional + +from azure.core.exceptions import HttpResponseError +from azure.core.polling import AsyncLROPoller, LROPoller +from azure.core.polling.base_polling import OperationFailed from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.aio import ( DeidentificationClient as DeidentificationClientAsync, ) +from azure.health.deidentification.models import ( + DeidentificationDocumentDetails, + DeidentificationJob, + OperationStatus, +) from devtools_testutils import ( @@ -30,9 +41,128 @@ ) +JOB_TIMEOUT_SECONDS = 600 +MAX_DIAGNOSTIC_DOCUMENTS = 20 + + +def format_job_diagnostics( + job: DeidentificationJob, + documents: List[DeidentificationDocumentDetails], + listing_error: Optional[Exception] = None, +) -> str: + lines = [ + f"Job '{job.job_name}' status: {job.status}", + f" created_at={job.created_at} started_at={job.started_at} last_updated_at={job.last_updated_at}", + ] + if job.summary is not None: + s = job.summary + lines.append( + f" summary: total={s.total_count} successful={s.successful_count} " + f"failed={s.failed_count} canceled={s.canceled_count} bytes_processed={s.bytes_processed}" + ) + else: + lines.append(" summary: None") + lines.append(f" error: {job.error if job.error is not None else 'None'}") + if documents: + lines.append(f" unsuccessful documents (up to {MAX_DIAGNOSTIC_DOCUMENTS}):") + for doc in documents: + location = doc.input_location.location if doc.input_location is not None else doc.id + lines.append(f" - {location} status={doc.status} error={doc.error}") + if listing_error is not None: + lines.append(f" unable to list all job documents: {listing_error!r}") + return "\n".join(lines) + + +def _is_unsuccessful(doc: DeidentificationDocumentDetails) -> bool: + return doc.status != OperationStatus.SUCCEEDED or doc.error is not None + + class DeidBaseTestCase(AzureRecordedTestCase): OUTPUT_PATH = "_output" + def describe_job(self, client: DeidentificationClient, job_name: str) -> str: + """Fetch the job and its unsuccessful documents and format them for an assertion message.""" + # Diagnostics must never mask the original failure, and a document-listing failure (even on a later page) + # must not discard the job details or documents already collected. + try: + job = client.get_job(job_name) + except Exception as e: # pylint: disable=broad-except + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + documents: List[DeidentificationDocumentDetails] = [] + listing_error: Optional[Exception] = None + try: + for doc in client.list_job_documents(job_name): + if _is_unsuccessful(doc): + documents.append(doc) + if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: + break + except Exception as e: # pylint: disable=broad-except + listing_error = e + return format_job_diagnostics(job, documents, listing_error) + + async def describe_job_async(self, client: DeidentificationClientAsync, job_name: str) -> str: + """Async version of describe_job.""" + try: + job = await client.get_job(job_name) + except Exception as e: # pylint: disable=broad-except + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + documents: List[DeidentificationDocumentDetails] = [] + listing_error: Optional[Exception] = None + try: + async for doc in client.list_job_documents(job_name): + if _is_unsuccessful(doc): + documents.append(doc) + if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: + break + except Exception as e: # pylint: disable=broad-except + listing_error = e + return format_job_diagnostics(job, documents, listing_error) + + def wait_for_job( + self, + client: DeidentificationClient, + poller: LROPoller[DeidentificationJob], + job_name: str, + timeout: Optional[float] = JOB_TIMEOUT_SECONDS, + ) -> DeidentificationJob: + """Wait for a job LRO and fail with the job's real error instead of azure-core's generic polling error. + + azure-core surfaces a Failed/Canceled job as HttpResponseError("Operation returned an invalid status 'OK'"), + and LROPoller.result(timeout) silently returns the in-progress job when the timeout expires. + """ + try: + job = poller.result(timeout=timeout) + except (HttpResponseError, OperationFailed) as e: + raise AssertionError( + f"Job '{job_name}' did not succeed ({e}).\n{self.describe_job(client, job_name)}" + ) from e + if not poller.done(): + raise AssertionError( + f"Job '{job_name}' did not reach a terminal state within {timeout}s.\n" + f"{self.describe_job(client, job_name)}" + ) + return job + + async def wait_for_job_async( + self, + client: DeidentificationClientAsync, + poller: AsyncLROPoller[DeidentificationJob], + job_name: str, + timeout: Optional[float] = JOB_TIMEOUT_SECONDS, + ) -> DeidentificationJob: + """Async version of wait_for_job.""" + try: + return await asyncio.wait_for(poller.result(), timeout=timeout) + except asyncio.TimeoutError as e: + raise AssertionError( + f"Job '{job_name}' did not reach a terminal state within {timeout}s.\n" + f"{await self.describe_job_async(client, job_name)}" + ) from e + except (HttpResponseError, OperationFailed) as e: + raise AssertionError( + f"Job '{job_name}' did not succeed ({e}).\n{await self.describe_job_async(client, job_name)}" + ) from e + def make_client(self, endpoint) -> DeidentificationClient: credential = self.get_credential(DeidentificationClient) connection_verify = not "localhost" in endpoint diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py index 9274dab6f955..e6be763939aa 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_delete.py @@ -30,7 +30,8 @@ def test_create_cancel_delete(self, **kwargs): operation_type=DeidentificationOperationType.SURROGATE, ) - client.begin_deidentify_documents(jobname, job) + # The test polls the job itself; a background LRO poller would race it for recorded GET responses. + client.begin_deidentify_documents(jobname, job, polling=False) job = client.get_job(jobname) while job.status == OperationStatus.NOT_STARTED: diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py index aaebe91349a6..1da0e802c658 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py @@ -29,7 +29,7 @@ def test_create_wait_finish(self, **kwargs): ) lro: LROPoller = client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = lro.result(timeout=120) + finished_job: DeidentificationJob = self.wait_for_job(client, lro, jobname) assert finished_job.status == OperationStatus.SUCCEEDED assert finished_job.job_name == jobname @@ -50,7 +50,9 @@ def test_create_wait_finish(self, **kwargs): for my_file in files: assert len(my_file.id) == 36 # GUID assert input_prefix in my_file.input_location.location - assert my_file.status == OperationStatus.SUCCEEDED + assert ( + my_file.status == OperationStatus.SUCCEEDED + ), f"{my_file.input_location.location}: status={my_file.status} error={my_file.error}" assert my_file.output_location is not None assert self.OUTPUT_PATH in my_file.output_location.location count += 1 diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py index be9bae5499fc..7545a9f3b0cd 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py @@ -32,7 +32,7 @@ async def test_create_wait_finish_async(self, **kwargs): lro: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = await lro.result() + finished_job: DeidentificationJob = await self.wait_for_job_async(client, lro, jobname) assert finished_job.status == OperationStatus.SUCCEEDED assert finished_job.job_name == jobname @@ -54,7 +54,9 @@ async def test_create_wait_finish_async(self, **kwargs): async for my_file in files: assert len(my_file.id) == 36 # GUID assert input_prefix in my_file.input_location.location - assert my_file.status == OperationStatus.SUCCEEDED + assert ( + my_file.status == OperationStatus.SUCCEEDED + ), f"{my_file.input_location.location}: status={my_file.status} error={my_file.error}" assert my_file.output_location is not None assert self.OUTPUT_PATH in my_file.output_location.location count += 1 diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py index ddb7f06ca83f..bffb32dbcdd4 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py @@ -33,7 +33,7 @@ def test_list_job_docs_pagination(self, **kwargs): customizations=DeidentificationJobCustomizationOptions(redaction_format="[{type}]"), ) - client.begin_deidentify_documents(jobname, job).result(180) + self.wait_for_job(client, client.begin_deidentify_documents(jobname, job), jobname) job_documents = client.list_job_documents(job_name=jobname, maxpagesize=2) _get_next = job_documents._args[0] # type: ignore diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_model_container_identity.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_model_container_identity.py new file mode 100644 index 000000000000..c43404c84530 --- /dev/null +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_model_container_identity.py @@ -0,0 +1,115 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Unit tests for the _patch.py shim that keeps model container fields bound to one object. + +No service calls or recordings are involved. +""" +import pytest + +import azure.health.deidentification.aio # pylint: disable=unused-import +from azure.health.deidentification import _patch +from azure.health.deidentification._utils import model_base +from azure.health.deidentification.models import ( + SimplePhiEntity, + SourceStorageLocation, + TaggedPhiEntities, +) + +# Each pattern lists how references are obtained, in order; the first reference is then mutated. +PATTERNS = { + "A_dict_property_mutate_dict_ref": ["dict", "property"], + "B_property_dict_mutate_property_ref": ["property", "dict"], + "C_property_mutate": ["property"], + "D_dict_mutate": ["dict"], + "E_property_property_mutate_first_ref": ["property", "property"], +} + + +def _access(model, attr, key, how): + return model[key] if how == "dict" else getattr(model, attr) + + +def _make_source(): + return SourceStorageLocation(location="https://example.blob.core.windows.net/c", prefix="p", extensions=[".txt"]) + + +def _make_tagged_from_kwargs(): + return TaggedPhiEntities( + encoding="CodePoint", + entities=[SimplePhiEntity(category="Patient", offset=0, length=4, text="John")], + ) + + +def _make_tagged_from_mapping(): + return TaggedPhiEntities( + {"encoding": "CodePoint", "entities": [{"category": "Patient", "offset": 0, "length": 4, "text": "John"}]} + ) + + +def _mutate_extensions(source, accesses): + refs = [_access(source, "extensions", "extensions", how) for how in accesses] + refs[0].append(".csv") + return source.as_dict()["extensions"] + + +@pytest.mark.parametrize("accesses", PATTERNS.values(), ids=PATTERNS.keys()) +def test_list_of_str_mutation_is_serialized(accesses): + assert _mutate_extensions(_make_source(), accesses) == [".txt", ".csv"] + + +@pytest.mark.parametrize( + "make_tagged", [_make_tagged_from_kwargs, _make_tagged_from_mapping], ids=["kwargs", "mapping"] +) +@pytest.mark.parametrize("accesses", PATTERNS.values(), ids=PATTERNS.keys()) +def test_list_of_models_mutation_is_serialized(accesses, make_tagged): + tagged = make_tagged() + refs = [_access(tagged, "entities", "entities", how) for how in accesses] + refs[0].append(SimplePhiEntity(category="Date", offset=9, length=10)) + refs[0][0]["text"] = "Jane" + + assert tagged.as_dict() == { + "encoding": "CodePoint", + "entities": [ + {"category": "Patient", "offset": 0, "length": 4, "text": "Jane"}, + {"category": "Date", "offset": 9, "length": 10}, + ], + } + assert [type(e) for e in tagged.entities] == [SimplePhiEntity, SimplePhiEntity] + assert tagged.entities[0].text == "Jane" + + +def test_property_and_dict_access_share_one_list(): + source = _make_source() + via_dict = source["extensions"] + via_property = source.extensions + assert via_property is via_dict + assert source["extensions"] is via_dict + + +def test_setting_field_replaces_container(): + source = _make_source() + old = source.extensions + source.extensions = [".json"] + assert source.extensions == [".json"] + assert old == [".txt"] + assert source.as_dict()["extensions"] == [".json"] + + +def test_shim_is_applied_once_for_sync_and_aio(): + _patch.patch_sdk() + _patch.patch_sdk() + get = model_base._RestField.__get__ # pylint: disable=protected-access + assert getattr(get, _patch._IDENTITY_SHIM_MARKER, False) # pylint: disable=protected-access + assert not getattr(get.__wrapped__, _patch._IDENTITY_SHIM_MARKER, False) # pylint: disable=protected-access + + +def test_generated_get_still_needs_shim(monkeypatch): + """Fails once the emitter preserves container identity itself; then remove the _patch.py shim.""" + rest_field = model_base._RestField # pylint: disable=protected-access + monkeypatch.setattr(rest_field, "__get__", rest_field.__get__.__wrapped__) + assert _mutate_extensions(_make_source(), PATTERNS["A_dict_property_mutate_dict_ref"]) == [".txt"], ( + "Generated _RestField.__get__ now preserves container identity; " + "remove _preserve_container_identity from azure/health/deidentification/_patch.py." + ) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml index 2a223df43022..7c077c9b9c46 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml +++ b/sdk/healthdataaiservices/azure-health-deidentification/tsp-location.yaml @@ -1,4 +1,4 @@ -directory: specification/healthdataaiservices/HealthDataAIServices.DeidServices -commit: a624d7fb58b59abadcd29c84268561c887bfee23 +directory: specification/healthdataaiservices/data-plane/HealthDataAIServices +commit: 6de384008f55f42ea97ee0654f237c74fb05ee89 repo: Azure/azure-rest-api-specs additionalDirectories: