diff --git a/.chronus/changes/python-lro-status-metadata-operation-id.md b/.chronus/changes/python-lro-status-metadata-operation-id.md new file mode 100644 index 00000000000..67b398332b9 --- /dev/null +++ b/.chronus/changes/python-lro-status-metadata-operation-id.md @@ -0,0 +1,14 @@ +--- +changeKind: feature +packages: + - "@typespec/http-client-python" +--- + +Generate `MetadataLROPoller` and `AsyncMetadataLROPoller` for Azure operations using `@lroStatusMetadata` or `@lroOperationId`. Their `details` and `operation_id` properties expose selected monitor metadata and server-issued identifiers, refresh on polling, and repopulate after resuming from a continuation token. Operations without these annotations retain their existing pollers. + +```python +poller = client.begin_analyze(document) +result = poller.result() +print(poller.operation_id) +print(poller.details) +``` diff --git a/cspell.yaml b/cspell.yaml index 806c508b70d..a3a3d03ac86 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -214,6 +214,7 @@ words: - onwarn - openapi - opfields + - operationid - oxlint - openapiv - Packument diff --git a/packages/http-client-python/ARCHITECTURE.md b/packages/http-client-python/ARCHITECTURE.md index 69b4122745a..32404b48bec 100644 --- a/packages/http-client-python/ARCHITECTURE.md +++ b/packages/http-client-python/ARCHITECTURE.md @@ -308,6 +308,21 @@ The emitter supports different operation patterns: - **Paging**: Operations that return paginated results - **LRO + Paging**: Combined pattern +### Selected LRO Metadata + +For Azure LROs annotated with `@lroStatusMetadata` or `@lroOperationId`, the emitter preserves the selected properties and status-monitor model in the intermediate code model. The generator exports `MetadataLROPoller` from the SDK package and `AsyncMetadataLROPoller` from its `aio` package. Unannotated operations retain their existing poller types. + +The poller's `details` mapping exposes selected values by their Python client names, using the generated status-monitor deserializer. It reflects the latest successfully decoded monitor response: changed values replace earlier values, absent selected properties are `None`, and failed or canceled responses remain inspectable. Non-success HTTP responses, decoding failures, and a subsequent final-resource request do not replace the monitor snapshot. The `operation_id` property exposes a selected monitor-body ID, falling back to the selected parameter of the initial polling URI. When no selected property is named `operation_id`, `details["operation_id"]` also provides the identifier when available. Polling still uses the complete URL, including its query parameters. + +Continuation tokens retain Azure Core's existing format. After resuming through the generated begin method, the URI identifier is immediately available, but selected monitor metadata is repopulated only after another monitor response. The default generated polling strategy refreshes metadata; an explicitly supplied custom polling strategy remains responsible for its own behavior. + +```python +poller = client.begin_analyze(document, polling_interval=1) +result = poller.result() +print(poller.operation_id) +print(poller.details.get("percent_complete")) +``` + ### Model Modes Two model generation modes: diff --git a/packages/http-client-python/emitter/src/http.ts b/packages/http-client-python/emitter/src/http.ts index e1fe7efb351..d3d1adfc806 100644 --- a/packages/http-client-python/emitter/src/http.ts +++ b/packages/http-client-python/emitter/src/http.ts @@ -23,6 +23,7 @@ import type { } from "@azure-tools/typespec-client-generator-core"; import { getHttpOperationParameter, UsageFlags } from "@azure-tools/typespec-client-generator-core"; import type { HttpStatusCodeRange } from "@typespec/http"; +import { getHeaderFieldName } from "@typespec/http"; import type { PythonSdkContext } from "./lib.js"; import { reportDiagnostic } from "./lib.js"; import { getType, KnownTypes } from "./types.js"; @@ -285,6 +286,70 @@ function emitInitialLroHttpMethod( }; } +function emitLroMetadata( + context: PythonSdkContext, + method: SdkLroServiceMethod | SdkLroPagingServiceMethod, +) { + const metadata = method.lroMetadata; + if (!metadata?.statusMetadata?.length && !metadata?.operationId?.length) return undefined; + const wirePath = (segments: SdkModelPropertyType[]) => + segments.map((property) => property.serializationOptions.json?.name ?? property.name); + const pathId = metadata.operationId?.find((id) => id.kind === "path"); + const bodyId = metadata.operationId?.find((id) => id.kind === "property"); + const step = metadata.statusMonitorStep; + const pollingOperation = + step?.kind === "nextOperationLink" ? step.operation : step?.target.operation; + let operationIdUri: { location: string; property: string; pattern: string } | undefined; + if (pathId) { + const template = pollingOperation?.uriTemplate; + const path = template?.split(/\{[?&]/, 1)[0].split("?", 1)[0]; + const segments = path?.split(/(\{[^{}]*\})/); + const parameter = `{${pathId.pathParameter.serializedName}}`; + const valid = + segments && + segments.filter((segment) => segment === parameter).length === 1 && + segments.every((segment) => /^\{[a-zA-Z0-9._%~-]+\}$/.test(segment) || !/[{}]/.test(segment)); + if (valid) { + const pattern = + segments + .map((segment) => + segment === parameter + ? "(?P[^/]+)" + : segment.startsWith("{") + ? "[^/]+" + : segment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), + ) + .join("") + "/?$"; + operationIdUri = { + location: pathId.source.location, + property: + pathId.source.location === "ResponseHeader" && pathId.source.property.__raw + ? getHeaderFieldName(context.program, pathId.source.property.__raw) + : (pathId.source.property.serializationOptions.json?.name ?? + pathId.source.property.name), + pattern, + }; + } else { + reportDiagnostic(context.program, { + code: "unsupported-lro-operation-id-template", + target: NoTarget, + format: { operationId: method.name, template: template ?? "(missing)" }, + }); + } + } + return { + responseModel: getType(context, metadata.pollingInfo.responseModel), + statusMetadata: metadata.statusMetadata?.map((selection) => ({ + propertySegments: wirePath(selection.propertySegments), + })), + operationIdPath: bodyId ? wirePath(bodyId.propertySegments) : undefined, + operationIdUri, + initialResponseIsMonitor: method.operation.responses.some( + (response) => response.type === metadata.pollingInfo.responseModel, + ), + }; +} + function addLroInformation( context: PythonSdkContext, rootClient: SdkClientType, @@ -313,6 +378,7 @@ function addLroInformation( exposeStreamKeyword: false, description: method.doc ?? "", summary: method.summary, + lroMetadata: emitLroMetadata(context, method), }; } diff --git a/packages/http-client-python/emitter/src/lib.ts b/packages/http-client-python/emitter/src/lib.ts index c7974a621ad..15147a92bcd 100644 --- a/packages/http-client-python/emitter/src/lib.ts +++ b/packages/http-client-python/emitter/src/lib.ts @@ -218,6 +218,12 @@ const libDef = { default: paramMessage`No valid LRO result for operation '${"operationId"}'.`, }, }, + "unsupported-lro-operation-id-template": { + severity: "warning", + messages: { + default: paramMessage`Cannot extract the LRO operation ID for operation '${"operationId"}' from URI template '${"template"}'.`, + }, + }, "invalid-continuation-token": { severity: "warning", messages: { diff --git a/packages/http-client-python/emitter/test/lro-metadata.test.ts b/packages/http-client-python/emitter/test/lro-metadata.test.ts new file mode 100644 index 00000000000..30680dd0a30 --- /dev/null +++ b/packages/http-client-python/emitter/test/lro-metadata.test.ts @@ -0,0 +1,126 @@ +import { expectDiagnostics } from "@typespec/compiler/testing"; +import { expect, it } from "vitest"; +import { emitCodeModel } from "./test-host.js"; + +it.each([true, false])( + "emits selected LRO paths without changing unannotated operations (%s)", + async (annotated) => { + const { codeModel, diagnostics } = await emitCodeModel(` + import "@azure-tools/typespec-azure-core"; + import "@azure-tools/typespec-client-generator-core"; + using TypeSpec.Http; + using Azure.Core; + using Azure.ClientGenerator.Core; + + @service namespace Test; + model Usage { tokens: int32; } + model BaseStatus { + ${annotated ? "@lroStatusMetadata" : ""} + @clientName("progress") + @encodedName("application/json", "wire.progress") + usage?: Usage; + } + model Monitor extends BaseStatus { + ${annotated ? "@lroOperationId" : ""} + @encodedName("application/json", "server-id") + id: string; + status: Foundations.OperationState; + result?: string; + } + @get @route("/operations/{id}/status") + op poll(${annotated ? "@lroOperationId" : ""} @path id: string): Monitor; + @post @route("/jobs") + @pollingOperation(poll) + op start(): Foundations.AcceptedResponse & + Foundations.LongRunningStatusLocation; + `); + expectDiagnostics(diagnostics, []); + const client = codeModel.clients[0] as { + operationGroups: { operations: { name: string; lroMetadata?: unknown }[] }[]; + }; + const start = client.operationGroups + .flatMap((group) => group.operations) + .find((op) => op.name === "start"); + expect(start).toBeDefined(); + if (!annotated) { + expect(start).not.toHaveProperty("lroMetadata"); + return; + } + expect(start).toEqual( + expect.objectContaining({ + name: "start", + discriminator: "lro", + lroMetadata: expect.objectContaining({ + responseModel: expect.objectContaining({ name: "Monitor" }), + statusMetadata: [{ propertySegments: ["wire.progress"] }], + operationIdPath: ["server-id"], + operationIdUri: { + location: "ResponseHeader", + property: "Operation-Location", + pattern: "/operations/(?P[^/]+)/status/?$", + }, + initialResponseIsMonitor: true, + }), + }), + ); + }, +); + +it.each(["{+id}", "{id*}"])( + "diagnoses unsupported operation-ID URI templates (%s)", + async (segment) => { + const { diagnostics } = await emitCodeModel(` + import "@azure-tools/typespec-azure-core"; + import "@azure-tools/typespec-client-generator-core"; + using TypeSpec.Http; + using Azure.Core; + using Azure.ClientGenerator.Core; + + @service namespace Test; + model Monitor { status: Foundations.OperationState; result?: string; } + @get @route("/operations/${segment}/status") + op poll(@lroOperationId @path id: string): Monitor; + @post @route("/jobs") + @pollingOperation(poll) + op start(): Foundations.AcceptedResponse & + Foundations.LongRunningStatusLocation; + `); + expectDiagnostics(diagnostics, { + code: "@typespec/http-client-python/unsupported-lro-operation-id-template", + severity: "warning", + }); + }, +); + +it("emits a path pattern for URI templates with other parameters and query expansions", async () => { + const { codeModel, diagnostics } = await emitCodeModel(` + import "@azure-tools/typespec-azure-core"; + import "@azure-tools/typespec-client-generator-core"; + using TypeSpec.Http; + using Azure.Core; + using Azure.ClientGenerator.Core; + + @service namespace Test; + model Monitor { status: Foundations.OperationState; result?: string; } + @get @route("/v1.0/{account}/operations/{id}/status") + op poll(@path account: string, @lroOperationId @path id: string, @query apiVersion?: string): Monitor; + @post @route("/v1.0/{account}/jobs") + @pollingOperation(poll) + op start(@path account: string): Foundations.AcceptedResponse & + Foundations.LongRunningStatusLocation; + `); + expectDiagnostics(diagnostics, []); + const client = codeModel.clients[0] as { + operationGroups: { + operations: { name: string; lroMetadata?: { operationIdUri?: unknown } }[]; + }[]; + }; + const start = client.operationGroups + .flatMap((group) => group.operations) + .find((op) => op.name === "start"); + expect(start?.lroMetadata?.operationIdUri).toEqual({ + location: "ResponseHeader", + property: "Operation-Location", + pattern: "/v1\\.0/[^/]+/operations/(?P[^/]+)/status/?$", + }); +}); diff --git a/packages/http-client-python/emitter/test/test-host.ts b/packages/http-client-python/emitter/test/test-host.ts index 2b5a087881c..def6abc30a7 100644 --- a/packages/http-client-python/emitter/test/test-host.ts +++ b/packages/http-client-python/emitter/test/test-host.ts @@ -9,7 +9,11 @@ import { $onEmit } from "../src/emitter.js"; import { $lib, type PythonEmitterOptions } from "../src/lib.js"; const PythonTester = createTester(resolvePath(import.meta.dirname, "../.."), { - libraries: ["@azure-tools/typespec-client-generator-core", "@typespec/http"], + libraries: [ + "@azure-tools/typespec-client-generator-core", + "@azure-tools/typespec-azure-core", + "@typespec/http", + ], }); export const Tester = PythonTester; diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts index be0ab9c8c23..4bc9ea9a47a 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -129,6 +129,12 @@ export const AZURE_EMITTER_OPTIONS: Record BaseType: not code_model.options["models-mode"] and not code_model.generate_typeddict_only ): model_type = JSONModelType - elif yaml_data["base"] == "dpg": + elif yaml_data["base"] == "dpg" and code_model.options["models-mode"] != "msrest": model_type = DPGModelType # type: ignore else: model_type = MsrestModelType # type: ignore diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 5618f156055..ce80c33998f 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -14,6 +14,7 @@ from .client import Client from .request_builder import RequestBuilder, OverloadedRequestBuilder from .operation_group import OperationGroup +from .lro_operation import LROOperationBase from .utils import NamespaceType from .._utils import DEFAULT_HEADER_TEXT, DEFAULT_LICENSE_DESCRIPTION from ... import OptionsDict @@ -295,6 +296,14 @@ def has_structured_stream(self) -> bool: for op in og.operations ) + @property + def has_lro_metadata(self) -> bool: + return any( + isinstance(operation, LROOperationBase) and operation.lro_metadata + for client in self.clients + for operation in client.lro_operations + ) + def need_utils_utils(self, async_mode: bool, client_namespace: str) -> bool: return ( self.need_utils_form_data(async_mode, client_namespace) diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index b7673d4d865..f55b332f04e 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -10,6 +10,8 @@ from .imports import ImportType, TypingSection from .request_builder import RequestBuilder from .parameter_list import ParameterList +from .model_type import ModelType +from .property import Property if TYPE_CHECKING: from .code_model import CodeModel @@ -47,8 +49,43 @@ def __init__( if not self.name.lstrip("_").startswith("begin"): self.name = ("_begin" if self.internal else "begin_") + self.name self.lro_options: dict[str, Any] = self.yaml_data.get("lroOptions", {}) + self.lro_metadata: dict[str, Any] = self.yaml_data.get("lroMetadata") or {} + self.status_monitor_type = ( + code_model.lookup_type(id(self.lro_metadata["responseModel"])) if self.lro_metadata else None + ) + self.status_metadata: list[tuple[str, list[str]]] = [] + self.operation_id_path: Optional[list[str]] = None + if self.lro_metadata: + for selection in self.lro_metadata.get("statusMetadata") or []: + properties = self._metadata_properties(selection["propertySegments"]) + self.status_metadata.append((properties[-1].client_name, self._metadata_path(properties))) + if self.lro_metadata.get("operationIdPath"): + self.operation_id_path = self._metadata_path( + self._metadata_properties(self.lro_metadata["operationIdPath"]) + ) + for response in self.responses: + response.has_lro_metadata = True self._initial_operation: Optional["OperationType"] = None + def _metadata_properties(self, wire_path: list[str]) -> list[Property]: + model = self.status_monitor_type + properties = [] + for wire_name in wire_path: + if not isinstance(model, ModelType): + raise ValueError(f"Status-monitor metadata path {wire_path!r} does not refer to a model property") + prop = next((p for p in model.properties if p.wire_name == wire_name), None) + if prop is None: + raise ValueError(f"Status-monitor metadata property {wire_name!r} was not found on {model.name}") + properties.append(prop) + model = prop.type + return properties + + def _metadata_path(self, properties: list[Property]) -> list[str]: + use_models = self.code_model.options["models-mode"] in ("dpg", "msrest") and not ( + self.code_model.generate_typeddict_only + ) + return [prop.client_name if use_models else prop.wire_name for prop in properties] + @property def initial_operation(self) -> "OperationType": if not self._initial_operation: @@ -140,6 +177,25 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ) file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.REGULAR) file_import.add_submodule_import("typing", "cast", ImportType.STDLIB) + if self.lro_metadata: + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace, module_name="_polling"), + "_LroMetadata", + ImportType.LOCAL, + ) + if ( + (self.status_metadata or self.operation_id_path) + and self.code_model.options["models-mode"] == "dpg" + and not self.code_model.generate_typeddict_only + ): + if self.status_monitor_type: + file_import.merge(self.status_monitor_type.imports(**kwargs)) + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace, module_name="_utils.model_base"), + "_deserialize", + ImportType.LOCAL, + ) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/response.py b/packages/http-client-python/generator/pygen/codegen/models/response.py index ba6014eb69d..60751a19c89 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/response.py +++ b/packages/http-client-python/generator/pygen/codegen/models/response.py @@ -388,7 +388,19 @@ def imports(self, **kwargs: Any) -> FileImport: class LROResponse(Response): + has_lro_metadata: bool = False + def get_poller_path(self, async_mode: bool) -> str: + if self.has_lro_metadata: + subdir = ( + f".{self.code_model.options['generation-subdir']}" + if self.code_model.options.get("generation-subdir") + else "" + ) + return ( + f"{self.code_model.namespace}{subdir}" + f"{'.aio' if async_mode else ''}.{'Async' if async_mode else ''}MetadataLROPoller" + ) return self.yaml_data["pollerAsync"] if async_mode else self.yaml_data["pollerSync"] def get_poller(self, async_mode: bool) -> str: @@ -397,6 +409,13 @@ def get_poller(self, async_mode: bool) -> str: def get_polling_method_path(self, async_mode: bool) -> str: """Get the full name of the poller path. Default are the azure core pollers""" + if self.has_lro_metadata: + subdir = ( + f".{self.code_model.options['generation-subdir']}" + if self.code_model.options.get("generation-subdir") + else "" + ) + return f"{self.code_model.namespace}{subdir}._polling._{'Async' if async_mode else ''}MetadataPolling" return self.yaml_data["pollingMethodAsync"] if async_mode else self.yaml_data["pollingMethodSync"] def get_polling_method(self, async_mode: bool) -> str: @@ -439,15 +458,25 @@ def imports(self, **kwargs: Any) -> FileImport: async_mode = kwargs["async_mode"] poller_import_path = ".".join(self.get_poller_path(async_mode).split(".")[:-1]) poller = self.get_poller(async_mode) - file_import.add_submodule_import(poller_import_path, poller, self._get_import_type(poller_import_path)) + if self.has_lro_metadata: + poller_import_path = self.code_model.get_relative_import_path( + kwargs.get("serialize_namespace", self.code_model.namespace), module_name="_polling" + ) + file_import.add_submodule_import( + poller_import_path, + poller, + ImportType.LOCAL if self.has_lro_metadata else self._get_import_type(poller_import_path), + ) async_mode = kwargs["async_mode"] default_polling_method_import_path = ".".join(self.get_polling_method_path(async_mode).split(".")[:-1]) default_polling_method = self.get_polling_method(async_mode) + if self.has_lro_metadata: + default_polling_method_import_path = poller_import_path file_import.add_submodule_import( default_polling_method_import_path, default_polling_method, - self._get_import_type(default_polling_method_import_path), + (ImportType.LOCAL if self.has_lro_metadata else self._get_import_type(default_polling_method_import_path)), ) default_no_polling_method_import_path = ".".join(self.get_no_polling_method_path(async_mode).split(".")[:-1]) default_no_polling_method = self.get_no_polling_method(async_mode) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index e73e3866970..8c0c7519f36 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -196,10 +196,18 @@ def serialize(self) -> None: ) else: # add pkgutil init file if no clients in this namespace + export_lro_metadata = ( + self.code_model.is_top_namespace(client_namespace) and self.code_model.has_lro_metadata + ) self.write_file( generation_path / Path("__init__.py"), - general_serializer.serialize_pkgutil_init_file(), + general_serializer.serialize_pkgutil_init_file(export_lro_metadata=export_lro_metadata), ) + if export_lro_metadata and self.has_aio_folder: + self.write_file( + generation_path / Path("aio/__init__.py"), + general_serializer.serialize_pkgutil_init_file(export_lro_metadata=True, async_mode=True), + ) # _utils/py.typed/_unions.py/types.py/_validation.py # is always put in top level namespace @@ -255,7 +263,11 @@ def serialize(self) -> None: if not client_namespace_type.clients and client_namespace_type.operation_groups and self.has_aio_folder: self.write_file( generation_path / Path("aio/__init__.py"), - general_serializer.serialize_pkgutil_init_file(), + general_serializer.serialize_pkgutil_init_file( + export_lro_metadata=self.code_model.is_top_namespace(client_namespace) + and self.code_model.has_lro_metadata, + async_mode=True, + ), ) # path where README.md is @@ -501,6 +513,11 @@ def _serialize_and_write_utils_folder(self, env: Environment, namespace: str): generation_dir = self.code_model.get_generation_dir(namespace) general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) utils_folder_path = generation_dir / Path("_utils") + if namespace == self.code_model.namespace and self.code_model.has_lro_metadata: + self.write_file( + generation_dir / "_polling.py", + env.get_template("polling.py.jinja2").render(code_model=self.code_model), + ) if self.code_model.need_utils_folder(async_mode=False, client_namespace=self.code_model.namespace): self.write_file( utils_folder_path / Path("__init__.py"), diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 94e866079e5..3878cdc6821 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -1106,7 +1106,12 @@ def response_deserialization( # pylint: disable=too-many-statements deserialize_code.append("deserialized = self._deserialize(") serialization_type = response.serialization_type(serialize_namespace=self.serialize_namespace) deserialize_code.append(f" '{serialization_type}',{pylint_disable}") - deserialize_code.append(" pipeline_response.http_response") + deserialize_input = ( + f"response.json(){response.result_property}" + if response.result_property + else "pipeline_response.http_response" + ) + deserialize_code.append(f" {deserialize_input}") deserialize_code.append(")") elif self.code_model.options["models-mode"] == "dpg": if builder.has_stream_response: @@ -1850,6 +1855,33 @@ def initial_call(self, builder: LROOperationType) -> list[str]: def return_lro_poller(self, builder: LROOperationType) -> list[str]: retval = [] + metadata_argument = "lro_metadata=lro_metadata," if builder.lro_metadata else "" + if builder.lro_metadata: + retval.extend(["lro_metadata = _LroMetadata(", " status_metadata={"]) + for name, path in builder.status_metadata: + retval.append(f" {name!r}: {path!r},") + deserialize = "value" + if (builder.status_metadata or builder.operation_id_path) and builder.status_monitor_type: + if self.code_model.options["models-mode"] == "msrest": + serialization_type = builder.status_monitor_type.serialization_type( + serialize_namespace=self.serialize_namespace + ) + deserialize = f"self._deserialize({serialization_type!r}, value)" + elif self.code_model.options["models-mode"] == "dpg" and not self.code_model.generate_typeddict_only: + annotation = builder.status_monitor_type.type_annotation( + serialize_namespace=self.serialize_namespace, is_operation_file=True + ) + deserialize = f"_deserialize({annotation}, value)" + retval.extend( + [ + " },", + f" deserialize_monitor=lambda value: {deserialize},", + f" operation_id_path={builder.operation_id_path!r},", + f" operation_id_uri={builder.lro_metadata.get('operationIdUri')!r},", + f" initial_response_is_monitor={builder.lro_metadata['initialResponseIsMonitor']!r},", + ")", + ] + ) lro_options_str = ( "lro_options={'final-state-via': '" + builder.lro_options["final-state-via"] + "'}," if builder.lro_options @@ -1867,6 +1899,7 @@ def return_lro_poller(self, builder: LROOperationType) -> list[str]: + f"= cast({builder.get_base_polling_method(self.async_mode)}, " f"{builder.get_polling_method(self.async_mode)}(", " lro_delay,", + f" {metadata_argument}", f" {lro_options_str}", f" {path_format_arguments_str}", " **kwargs", @@ -1883,10 +1916,23 @@ def return_lro_poller(self, builder: LROOperationType) -> list[str]: retval.append(" polling_method=polling_method,") retval.append(" continuation_token=cont_token,") retval.append(" client=self._client,") - retval.append(" deserialization_callback=get_long_running_output") + retval.append(" deserialization_callback=get_long_running_output" + ("," if metadata_argument else "")) + if metadata_argument: + retval.append(f" {metadata_argument}") retval.append(" )") retval.append(f"return {builder.get_poller_with_response_type(self.async_mode)}(") - retval.append(" self._client, raw_result, get_long_running_output, polling_method # type: ignore") + if metadata_argument: + retval.extend( + [ + " self._client,", + " raw_result, # type: ignore", + " get_long_running_output,", + " polling_method,", + f" {metadata_argument}", + ] + ) + else: + retval.append(" self._client, raw_result, get_long_running_output, polling_method # type: ignore") retval.append(" )") return retval diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index b35cffc746d..a01f29114cb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -194,9 +194,9 @@ def serialize_package_file(self, template_name: str, file_content: str, **kwargs params |= kwargs return template.render(file_import=FileImport(self.code_model), **params) - def serialize_pkgutil_init_file(self) -> str: + def serialize_pkgutil_init_file(self, *, export_lro_metadata: bool = False, async_mode: bool = False) -> str: template = self.env.get_template("pkgutil_init.py.jinja2") - return template.render() + return template.render(export_lro_metadata=export_lro_metadata, async_mode=async_mode) def serialize_init_file(self, clients: list[Client]) -> str: template = self.env.get_template("init.py.jinja2") diff --git a/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 index 007009e4bc9..634f7aeafeb 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/init.py.jinja2 @@ -14,12 +14,18 @@ from {{ code_model.get_relative_import_path(serialize_namespace, module_name="_v __version__ = VERSION {% endif %} +{% if code_model.has_lro_metadata and serialize_namespace == code_model.namespace + ('.aio' if async_mode else '') %} +from {{ '..' if async_mode else '.' }}_polling import {{ 'Async' if async_mode else '' }}MetadataLROPoller +{% endif %} {{ keywords.patch_imports(try_except=True) }} __all__ = [ {% for client in clients %} {{ keywords.escape_str(client.name) }}, {% endfor %} + {% if code_model.has_lro_metadata and serialize_namespace == code_model.namespace + ('.aio' if async_mode else '') %} + "{{ 'Async' if async_mode else '' }}MetadataLROPoller", + {% endif %} ] {{ keywords.extend_all }} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 index 5960c353a89..0852fb59b48 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 @@ -1 +1,7 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore \ No newline at end of file +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +{% if export_lro_metadata %} + +from {{ '..' if async_mode else '.' }}_polling import {{ 'Async' if async_mode else '' }}MetadataLROPoller + +__all__ = ["{{ 'Async' if async_mode else '' }}MetadataLROPoller"] +{% endif %} \ No newline at end of file diff --git a/packages/http-client-python/generator/pygen/codegen/templates/polling.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/polling.py.jinja2 new file mode 100644 index 00000000000..4db2015e24b --- /dev/null +++ b/packages/http-client-python/generator/pygen/codegen/templates/polling.py.jinja2 @@ -0,0 +1,194 @@ +{% if code_model.license_header %} +{{ code_model.license_header }} +{% endif %} +"""Pollers exposing selected status-monitor metadata and operation identity.""" + +import logging +import re +from typing import Any, Callable, Mapping, Optional, TypeVar +from urllib.parse import unquote, urlsplit + +from azure.core.polling import LROPoller +{% if not code_model.options.get("no-async") %} +from azure.core.polling import AsyncLROPoller +{% endif %} +{% if code_model.options["azure-arm"] %} +from azure.mgmt.core.polling.arm_polling import ARMPolling +{% if not code_model.options.get("no-async") %} +from azure.mgmt.core.polling.async_arm_polling import AsyncARMPolling +{% endif %} +{% else %} +from azure.core.polling.base_polling import LROBasePolling +{% if not code_model.options.get("no-async") %} +from azure.core.polling.async_base_polling import AsyncLROBasePolling +{% endif %} +{% endif %} + +__all__ = ["MetadataLROPoller"{% if not code_model.options.get("no-async") %}, "AsyncMetadataLROPoller"{% endif %}] + +T = TypeVar("T") +_MISSING = object() +_LOGGER = logging.getLogger(__name__) + + +def _read_property(body: Any, path: list[str]) -> Any: + for segment in path: + if isinstance(body, dict): + body = body.get(segment, _MISSING) + else: + body = getattr(body, segment, _MISSING) + if body is _MISSING or body is None: + return _MISSING + return body + + +class _LroMetadata: + def __init__( + self, + status_metadata: Mapping[str, list[str]], + *, + deserialize_monitor: Callable[[Any], Any], + operation_id_path: Optional[list[str]] = None, + operation_id_uri: Optional[dict[str, str]] = None, + initial_response_is_monitor: bool = False, + ) -> None: + self._status_metadata = status_metadata + self._deserialize_monitor = deserialize_monitor + self._operation_id_path = operation_id_path + self._operation_id_uri = operation_id_uri + self._initial_response_is_monitor = initial_response_is_monitor + self._snapshot: tuple[dict[str, Any], Optional[str]] = ({name: None for name in status_metadata}, None) + self._uri_id: Optional[str] = None + self.rehydrated = False + + def initialize(self, response: Any) -> None: + if self._operation_id_uri: + source = self._operation_id_uri + try: + if source["location"] == "ResponseHeader": + url = response.headers.get(source["property"]) + elif source["location"] == "Self": + url = response.json() if response.content else None + else: + url = _read_property(response.json(), [source["property"]]) if response.content else None + if isinstance(url, str): + match = re.search(source["pattern"], urlsplit(url).path) + self._uri_id = unquote(match.group("operation_id")) if match else None + except Exception: # pylint: disable=broad-exception-caught + _LOGGER.warning("Unable to extract the LRO operation ID from the polling URI.", exc_info=True) + if self._initial_response_is_monitor and not self.rehydrated: + self.update(response) + + def update(self, response: Any) -> None: + if not self._status_metadata and not self._operation_id_path: + return + try: + body = response.json() if response.content else {} + monitor = self._deserialize_monitor(body) + details = {} + for name, path in self._status_metadata.items(): + value = _read_property(monitor, path) + details[name] = None if value is _MISSING else value + body_id = _read_property(monitor, self._operation_id_path) if self._operation_id_path else None + except Exception: # pylint: disable=broad-exception-caught + _LOGGER.warning("Unable to deserialize LRO status-monitor metadata.", exc_info=True) + return + self._snapshot = (details, body_id if isinstance(body_id, str) else None) + + @property + def operation_id(self) -> Optional[str]: + body_id = self._snapshot[1] + return body_id if body_id is not None else self._uri_id + + @property + def details(self) -> Mapping[str, Any]: + values, body_id = self._snapshot + details = values.copy() + operation_id = body_id if body_id is not None else self._uri_id + if operation_id is not None and "operation_id" not in self._status_metadata: + details["operation_id"] = operation_id + return details + + +{% for async_mode in ([false] if code_model.options.get("no-async") else [false, true]) %} +{% set prefix = "Async" if async_mode else "" %} +class _{{ prefix }}MetadataPolling({{ prefix }}{{ "ARMPolling" if code_model.options["azure-arm"] else "LROBasePolling[T, Any, Any]" }}): + def __init__(self, *args: Any, lro_metadata: _LroMetadata, **kwargs: Any) -> None: + self._lro_metadata = lro_metadata + super().__init__(*args, **kwargs) + + {{ "async " if async_mode else "" }}def update_status(self) -> None: + {{ "await " if async_mode else "" }}super().update_status() + self._lro_metadata.update(self._pipeline_response.http_response) + + +class {{ prefix }}MetadataLROPoller({{ prefix }}LROPoller[T]): + """A poller with metadata from the latest received status-monitor response. + + :param client: The pipeline client used for polling. + :type client: any + :param initial_response: The initial pipeline response. + :type initial_response: any + :param deserialization_callback: Callback for the final result. + :type deserialization_callback: callable + :param polling_method: The polling strategy. + :type polling_method: ~azure.core.polling.{{ prefix }}PollingMethod + :keyword lro_metadata: Generated metadata selection and deserializers. + :paramtype lro_metadata: any + """ + + def __init__( + self, client: Any, initial_response: Any, deserialization_callback: Any, + polling_method: Any, *, lro_metadata: _LroMetadata, + ) -> None: + self._lro_metadata = lro_metadata + lro_metadata.initialize(initial_response.http_response) + super().__init__(client, initial_response, deserialization_callback, polling_method) + + @property + def details(self) -> Mapping[str, Any]: + """Selected metadata, decoded using the generated model types. + + Absent selected values are None. Values are refreshed after each monitor response, + including failed or canceled responses. After resuming, the next poll + repopulates metadata; it is not stored separately in continuation tokens. + + :rtype: Mapping[str, Any] + :return: The latest received selected metadata and operation ID, when available. + """ + return self._lro_metadata.details + + @property + def operation_id(self) -> Optional[str]: + """The selected server-issued ID, or None until it is available. + + A received body ID takes precedence over the polling URI ID. + This value is never used as the polling URL. + + :rtype: str or None + :return: The selected server-issued operation identifier. + """ + return self._lro_metadata.operation_id + + @classmethod + def from_continuation_token( + cls, polling_method: Any, continuation_token: str, **kwargs: Any, + ) -> "{{ prefix }}MetadataLROPoller[T]": + """Resume polling, repopulating metadata on the next monitor response. + + :param polling_method: The polling strategy. + :type polling_method: ~azure.core.polling.{{ prefix }}PollingMethod + :param str continuation_token: The opaque continuation token. + :keyword dict[str, Any] ``**kwargs``: The ``client`` pipeline client, + ``deserialization_callback`` for the final result, and generated + ``lro_metadata`` selection and deserializers. + :rtype: ~{{ prefix }}MetadataLROPoller + :return: The resumed poller. + """ + metadata = kwargs.pop("lro_metadata") + metadata.rehydrated = True + client, initial_response, callback = polling_method.from_continuation_token(continuation_token, **kwargs) + return cls(client, initial_response, callback, polling_method, lro_metadata=metadata) + + +{% endfor %} diff --git a/packages/http-client-python/tests/mock_api/azure/asynctests/test_client_generator_core_lro_metadata_async.py b/packages/http-client-python/tests/mock_api/azure/asynctests/test_client_generator_core_lro_metadata_async.py new file mode 100644 index 00000000000..aceb2476ef0 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/azure/asynctests/test_client_generator_core_lro_metadata_async.py @@ -0,0 +1,64 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from datetime import datetime, timezone + +import pytest +import pytest_asyncio +from azure.core.exceptions import HttpResponseError +from specs.azure.clientgenerator.core.lro.metadata import models +from specs.azure.clientgenerator.core.lro.metadata.aio import AsyncMetadataLROPoller, MetadataClient + +pytestmark = pytest.mark.xdist_group(name="lro_metadata") + + +@pytest_asyncio.fixture +async def client(): + async with MetadataClient() as client: + yield client + + +@pytest.mark.asyncio +async def test_usage(client): + poller = await client.begin_analyze_with_usage("analyzer-1", polling_interval=0) + assert isinstance(poller, AsyncMetadataLROPoller) + assert poller.details == {"usage": None} + await poller.polling_method().update_status() + assert poller.details["usage"] == models.UsageDetails(document_pages_basic=1, tokens={"input": 10}) + assert await poller.result() == models.ContentAnalysisResult(contents="The document contains three sections.") + assert poller.details["usage"].tokens == {"input": 10, "output": 4} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fails", [False, True]) +async def test_latest_job_metadata_including_failure(client, fails): + begin = client.begin_analyze_job_with_failure if fails else client.begin_analyze_job + poller = await begin(polling_interval=0) + assert poller.details["job_id"] == ("job-failure" if fails else "job-success") + assert poller.details["statistics"] is None + assert poller.details["errors"] is None + await poller.polling_method().update_status() + assert poller.details["statistics"] == models.JobStatistics(total=3, succeeded=1, failed=0) + assert poller.details["last_updated_date_time"] == datetime(2025, 1, 1, 10, 1, tzinfo=timezone.utc) + if fails: + with pytest.raises(HttpResponseError): + await poller.result() + else: + assert await poller.result() == models.LanguageJobResult(documents=["document-1", "document-2", "document-3"]) + assert poller.details["statistics"].failed == (2 if fails else 0) + assert poller.details["last_updated_date_time"] == datetime(2025, 1, 1, 10, 2, tzinfo=timezone.utc) + assert len(poller.details["errors"]) == (2 if fails else 0) + assert "error" not in poller.details + + +@pytest.mark.asyncio +async def test_metadata_repopulates_after_resume(client): + poller = await client.begin_analyze_job(polling_interval=0) + await poller.polling_method().update_status() + token = poller.continuation_token() + resumed = await client.begin_analyze_job(continuation_token=token, polling_interval=0) + assert all(value is None for value in resumed.details.values()) + await resumed.result() + assert resumed.details["statistics"] == models.JobStatistics(total=3, succeeded=3, failed=0) diff --git a/packages/http-client-python/tests/mock_api/azure/asynctests/test_client_generator_core_lro_operation_id_async.py b/packages/http-client-python/tests/mock_api/azure/asynctests/test_client_generator_core_lro_operation_id_async.py new file mode 100644 index 00000000000..d5e3dec0ef2 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/azure/asynctests/test_client_generator_core_lro_operation_id_async.py @@ -0,0 +1,40 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +import pytest_asyncio +from specs.azure.clientgenerator.core.lro.operationid import models +from specs.azure.clientgenerator.core.lro.operationid.aio import OperationIdClient + +pytestmark = pytest.mark.xdist_group(name="lro_metadata") + + +@pytest_asyncio.fixture +async def client(): + async with OperationIdClient() as client: + yield client + + +@pytest.mark.asyncio +async def test_polling_uri_id_and_resume(client): + poller = await client.begin_analyze("document-1", polling_interval=0) + assert poller.operation_id == "analysis-123" + assert poller.details == {"operation_id": "analysis-123"} + resumed = await client.begin_analyze( + "document-1", continuation_token=poller.continuation_token(), polling_interval=0 + ) + assert resumed.operation_id == "analysis-123" + assert await resumed.result() == models.DocumentAnalysisResult(summary="The document contains three sections.") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_path", [False, True]) +async def test_monitor_body_id(client, with_path): + begin = client.begin_start_translation_with_path if with_path else client.begin_start_translation + poller = await begin(polling_interval=0) + assert poller.operation_id == ("translation-456" if with_path else None) + await poller.polling_method().update_status() + assert poller.operation_id == ("translation-456" if with_path else "translation-123") + assert await poller.result() == models.TranslationResult(translated_characters=84 if with_path else 42) diff --git a/packages/http-client-python/tests/mock_api/azure/test_client_generator_core_lro_metadata.py b/packages/http-client-python/tests/mock_api/azure/test_client_generator_core_lro_metadata.py new file mode 100644 index 00000000000..3da06062187 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/azure/test_client_generator_core_lro_metadata.py @@ -0,0 +1,58 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from datetime import datetime, timezone + +import pytest +from azure.core.exceptions import HttpResponseError +from specs.azure.clientgenerator.core.lro.metadata import MetadataClient, MetadataLROPoller, models + +pytestmark = pytest.mark.xdist_group(name="lro_metadata") + + +@pytest.fixture +def client(): + with MetadataClient() as client: + yield client + + +def test_usage(client): + poller = client.begin_analyze_with_usage("analyzer-1", polling_interval=0) + assert isinstance(poller, MetadataLROPoller) + assert poller.result() == models.ContentAnalysisResult(contents="The document contains three sections.") + assert poller.details["usage"] == models.UsageDetails(document_pages_basic=1, tokens={"input": 10, "output": 4}) + + +def test_job_initial_metadata(client): + poller = client.begin_analyze_job(polling=False) + assert poller.details["job_id"] == "job-success" + assert poller.details["created_date_time"] == datetime(2025, 1, 1, 10, tzinfo=timezone.utc) + assert poller.details["statistics"] is None + assert poller.details["errors"] is None + + +@pytest.mark.parametrize("fails", [False, True]) +def test_latest_job_metadata_including_failure(client, fails): + begin = client.begin_analyze_job_with_failure if fails else client.begin_analyze_job + poller = begin(polling_interval=0) + if fails: + with pytest.raises(HttpResponseError): + poller.result() + else: + assert poller.result() == models.LanguageJobResult(documents=["document-1", "document-2", "document-3"]) + assert poller.details["statistics"] == models.JobStatistics( + total=3, succeeded=1 if fails else 3, failed=2 if fails else 0 + ) + assert poller.details["last_updated_date_time"] == datetime(2025, 1, 1, 10, 2, tzinfo=timezone.utc) + assert len(poller.details["errors"]) == (2 if fails else 0) + assert "error" not in poller.details + + +def test_metadata_repopulates_after_resume(client): + poller = client.begin_analyze_job(polling_interval=0) + poller.result() + resumed = client.begin_analyze_job(continuation_token=poller.continuation_token(), polling_interval=0) + resumed.result() + assert resumed.details["statistics"] == models.JobStatistics(total=3, succeeded=3, failed=0) diff --git a/packages/http-client-python/tests/mock_api/azure/test_client_generator_core_lro_operation_id.py b/packages/http-client-python/tests/mock_api/azure/test_client_generator_core_lro_operation_id.py new file mode 100644 index 00000000000..704f2da5698 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/azure/test_client_generator_core_lro_operation_id.py @@ -0,0 +1,36 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from specs.azure.clientgenerator.core.lro.operationid import OperationIdClient, models + +pytestmark = pytest.mark.xdist_group(name="lro_metadata") + + +@pytest.fixture +def client(): + with OperationIdClient() as client: + yield client + + +def test_polling_uri_id_and_resume(client): + initial = client.begin_analyze("document-1", polling=False) + assert initial.operation_id == "analysis-123" + assert initial.details == {"operation_id": "analysis-123"} + poller = client.begin_analyze("document-1", polling_interval=0) + assert poller.result() == models.DocumentAnalysisResult(summary="The document contains three sections.") + resumed = client.begin_analyze("document-1", continuation_token=poller.continuation_token(), polling_interval=0) + assert resumed.operation_id == "analysis-123" + resumed.result() + + +@pytest.mark.parametrize("with_path", [False, True]) +def test_monitor_body_id(client, with_path): + begin = client.begin_start_translation_with_path if with_path else client.begin_start_translation + initial = begin(polling=False) + assert initial.operation_id == ("translation-456" if with_path else None) + poller = begin(polling_interval=0) + assert poller.result() == models.TranslationResult(translated_characters=84 if with_path else 42) + assert poller.operation_id == ("translation-456" if with_path else "translation-123") diff --git a/packages/http-client-python/tests/mock_api/azure/test_polling_template.py b/packages/http-client-python/tests/mock_api/azure/test_polling_template.py new file mode 100644 index 00000000000..dedd677f0ce --- /dev/null +++ b/packages/http-client-python/tests/mock_api/azure/test_polling_template.py @@ -0,0 +1,400 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock +import json + +import pytest +from azure.core.pipeline import PipelineContext, PipelineResponse +from azure.core.pipeline.transport import HttpRequest +from azure.core.polling.base_polling import BadStatus +from azure.core.utils import case_insensitive_dict +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +from pygen import OptionsDict +from pygen.codegen.models import CodeModel, build_type +from pygen.codegen.models.lro_operation import LROOperationBase +from pygen.codegen.models.response import LROResponse +from pygen.codegen.serializers.builder_serializer import LROOperationSerializer + + +@pytest.fixture(scope="session", autouse=True) +def testserver(): + yield + + +@pytest.fixture(params=[False, True], ids=["data-plane", "arm"]) +def polling(request): + env = Environment( + loader=FileSystemLoader(Path(__file__).parents[3] / "generator/pygen/codegen/templates"), + undefined=StrictUndefined, + trim_blocks=True, + lstrip_blocks=True, + ) + namespace = {} + exec( + env.get_template("polling.py.jinja2").render( + code_model=SimpleNamespace(license_header="", options={"azure-arm": request.param}) + ), + namespace, + ) + return SimpleNamespace(**namespace) + + +def response(body, headers=None, status_code=200): + return SimpleNamespace( + json=lambda: body, + content=b"json" if body is not None else b"", + headers=headers or {}, + status_code=status_code, + ) + + +def test_latest_metadata_replaces_missing_and_changed_values(polling): + state = polling._LroMetadata({k: [k] for k in ("progress", "errors")}, deserialize_monitor=lambda body: body) + assert state.details == {"progress": None, "errors": None} + state.update(response({"progress": 10})) + assert state.details == {"progress": 10, "errors": None} + for terminal_state in ("Failed", "Canceled"): + state.update(response({"status": terminal_state, "errors": ["item error"]})) + assert state.details == {"progress": None, "errors": ["item error"]} + state.update(response(None)) + assert state.details == {"progress": None, "errors": None} + + +def test_monitor_is_deserialized_once_and_selected_client_properties_are_preserved(polling): + calls = [] + + def deserialize(body): + calls.append(body) + return SimpleNamespace( + operation_id=body["metadataId"], + progress=SimpleNamespace(completed=body["progress"]["done"]), + server_id=body["serverId"], + ) + + state = polling._LroMetadata( + {"operation_id": ["operation_id"], "completed": ["progress", "completed"]}, + deserialize_monitor=deserialize, + operation_id_path=["server_id"], + ) + state.update(response({"metadataId": "metadata", "progress": {"done": 2}, "serverId": "server"})) + assert len(calls) == 1 + assert state.details == {"operation_id": "metadata", "completed": 2} + assert state.operation_id == "server" + + +def test_metadata_decode_errors_preserve_last_snapshot(polling, caplog): + def deserialize(body): + if body.get("bad"): + raise ValueError("invalid monitor value") + return body + + state = polling._LroMetadata({"progress": ["progress"]}, deserialize_monitor=deserialize) + state.update(response({"progress": 1})) + invalid_json = response({}) + invalid_json.json = lambda: json.loads("not json") + state.update(invalid_json) + state.update(response({"bad": True})) + assert state.details == {"progress": 1} + assert "metadata" in caplog.text.lower() + state.update(response({"progress": 2})) + assert state.details == {"progress": 2} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) +async def test_non_success_poll_preserves_last_metadata(polling, monkeypatch, async_mode): + state = polling._LroMetadata({"progress": ["progress"]}, deserialize_monitor=lambda body: body) + state.update(response({"progress": 1})) + cls = polling._AsyncMetadataPolling if async_mode else polling._MetadataPolling + method = cls(0, lro_metadata=state) + method._operation = SimpleNamespace(get_polling_url=lambda: POLL_URL) + failed_response = pipeline_response({"error": "failed"}) + failed_response.http_response.status_code = 400 + if async_mode: + + async def request_status(*args, **kwargs): + return failed_response + + else: + + def request_status(*args, **kwargs): + return failed_response + + monkeypatch.setattr(cls.__mro__[1], "request_status", request_status) + with pytest.raises(BadStatus): + result = method.update_status() + if async_mode: + await result + assert state.details == {"progress": 1} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) +async def test_metadata_decode_error_does_not_fail_polling(polling, monkeypatch, caplog, async_mode): + def deserialize(body): + if body.get("bad"): + raise ValueError("invalid monitor metadata") + return body + + state = polling._LroMetadata({"progress": ["progress"]}, deserialize_monitor=deserialize) + state.update(response({"progress": 1})) + cls = polling._AsyncMetadataPolling if async_mode else polling._MetadataPolling + method = cls(0, lro_metadata=state) + method._operation = SimpleNamespace(get_polling_url=lambda: POLL_URL, get_status=lambda _: "Succeeded") + response_mock = AsyncMock if async_mode else Mock + monkeypatch.setattr( + cls.__mro__[1], + "request_status", + response_mock(return_value=pipeline_response({"status": "Succeeded", "bad": True})), + ) + + result = method.update_status() + if async_mode: + await result + assert method.status() == "Succeeded" + assert state.details == {"progress": 1} + assert "metadata" in caplog.text.lower() + + +@pytest.mark.parametrize("location", ["ResponseHeader", "ResponseBody", "Self"]) +def test_selected_uri_segment_and_body_precedence(polling, location): + state = polling._LroMetadata( + {}, + deserialize_monitor=lambda body: body, + operation_id_path=["wire-id"], + operation_id_uri={ + "location": location, + "property": "monitor", + "pattern": r"/operations/(?P[^/]+)/status/?$", + }, + ) + url = "https://example.test/prefix/operations/uri%20id/status?api-version=1#fragment" + initial = response(url if location == "Self" else {"monitor": url}, {"monitor": url}) + state.initialize(initial) + assert state.operation_id == "uri id" + state.update(response({"wire-id": "body-id"})) + assert state.operation_id == "body-id" + state.update(response({"wire-id": None})) + assert state.operation_id == "uri id" + assert state.details["operation_id"] == "uri id" + + +def test_unmatched_uri_does_not_invent_an_operation_id(polling): + state = polling._LroMetadata( + {}, + deserialize_monitor=lambda body: body, + operation_id_uri={ + "location": "ResponseHeader", + "property": "Operation-Location", + "pattern": r"/operations/(?P[^/]+)/status/?$", + }, + ) + state.initialize(response(None, {"Operation-Location": "https://example.test/unrelated"})) + assert state.operation_id is None + + +def test_initial_monitor_and_rehydration(polling): + state = polling._LroMetadata( + {"progress": ["progress"]}, deserialize_monitor=lambda body: body, initial_response_is_monitor=True + ) + state.initialize(response({"progress": 1})) + assert state.details == {"progress": 1} + resumed = polling._LroMetadata( + {"progress": ["progress"]}, deserialize_monitor=lambda body: body, initial_response_is_monitor=True + ) + resumed.rehydrated = True + resumed.initialize(response({"progress": 1})) + assert resumed.details == {"progress": None} + resumed.update(response({"progress": 2})) + assert resumed.details == {"progress": 2} + + +POLL_URL = "https://example.test/operations/uri%20id/status?api-version=1&token=opaque" +FINAL_URL = "https://example.test/result" + + +class _HttpResponse: + def __init__(self, request, body, initial): + self.content = json.dumps(body).encode() + self.request = request + self.status_code = 202 if initial else 200 + self.reason = "Accepted" if initial else "OK" + self.headers = case_insensitive_dict( + {"Operation-Location": POLL_URL, "Azure-AsyncOperation": POLL_URL, "Location": FINAL_URL} if initial else {} + ) + + def json(self): + return json.loads(self.content) + + def body(self): + return self.content + + def text(self): + return self.content.decode() + + +def pipeline_response(body, *, initial=False): + request = HttpRequest( + "POST" if initial else "GET", + "https://example.test/start" if initial else POLL_URL, + headers={"x-ms-client-request-id": "request-id"}, + ) + http_response = _HttpResponse(request, body, initial) + return PipelineResponse(request, http_response, PipelineContext(None)) + + +def metadata_state(polling): + return polling._LroMetadata( + {"progress": ["progress"]}, + deserialize_monitor=lambda body: body, + operation_id_path=["id"], + operation_id_uri={ + "location": "ResponseHeader", + "property": "Operation-Location", + "pattern": r"/operations/(?P[^/]+)/status/?$", + }, + initial_response_is_monitor=True, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) +async def test_final_get_at_polling_url_preserves_monitor_metadata(polling, async_mode): + urls = [] + bodies = iter([{"status": "Succeeded", "progress": 2, "id": "body-id"}, {"final": True}]) + + def run_sync(request, **kwargs): + urls.append(request.url) + return pipeline_response(next(bodies)) + + async def run_async(request, **kwargs): + return run_sync(request, **kwargs) + + run = run_async if async_mode else run_sync + client = SimpleNamespace( + send_request=run, + get=lambda url: HttpRequest("GET", url), + _pipeline=SimpleNamespace(run=run, _transport=SimpleNamespace(sleep=lambda _: None)), + ) + initial = pipeline_response({"status": "Running", "progress": 0}, initial=True) + initial.http_response.headers["Location"] = POLL_URL + state = metadata_state(polling) + method_class = polling._AsyncMetadataPolling if async_mode else polling._MetadataPolling + poller_class = polling.AsyncMetadataLROPoller if async_mode else polling.MetadataLROPoller + poller = poller_class( + client, + initial, + lambda value: value.http_response.json(), + method_class(0, lro_metadata=state), + lro_metadata=state, + ) + + result = await poller.result() if async_mode else poller.result() + assert result == {"final": True} + assert urls == [POLL_URL, POLL_URL] + assert poller.details == {"progress": 2, "operation_id": "body-id"} + + +class _TestLROOperationSerializer(LROOperationSerializer): + def response_headers_and_deserialization(self, builder, response): + return ["deserialized = response.json()"] + + +@pytest.mark.parametrize("models_mode", [None, "dpg"]) +@pytest.mark.parametrize("generate_typeddict_only", [False, True]) +def test_lro_response_is_available_for_deserialization(models_mode, generate_typeddict_only): + code_model = SimpleNamespace( + options={"models-mode": models_mode}, + generate_typeddict_only=generate_typeddict_only, + get_serialize_namespace=lambda *args, **kwargs: "test", + ) + serializer = _TestLROOperationSerializer(code_model, async_mode=False, client_namespace="test") + operation = SimpleNamespace( + lro_response=SimpleNamespace(headers=[], type=object()), + ) + + generated = serializer.get_long_running_output(operation) + + assert " response = pipeline_response.http_response" in generated + assert " deserialized = response.json()" in generated + + +@pytest.mark.parametrize( + "models_mode, expected", + [("dpg", ["parent_", "class_"]), ("msrest", ["parent_", "class_"]), ("none", ["wire.parent", "wire.class"])], +) +def test_monitor_metadata_paths_use_model_names_or_wire_names(models_mode, expected): + code_model = CodeModel( + { + "namespace": "sample", + "clients": [ + { + "name": "client", + "namespace": "sample", + "moduleName": "sample", + "parameters": [], + "url": "", + "operationGroups": [], + } + ], + }, + OptionsDict({"models-mode": models_mode, "flavor": "azure", "tsp_file": True}), + ) + monitor = build_type( + { + "type": "model", + "base": "dpg", + "name": "Monitor", + "snakeCaseName": "monitor", + "properties": [ + { + "wireName": "wire.parent", + "clientName": "parent_", + "optional": True, + "type": { + "type": "model", + "base": "dpg", + "name": "Details", + "snakeCaseName": "details", + "properties": [ + { + "wireName": "wire.class", + "clientName": "class_", + "optional": True, + "type": {"type": "string"}, + } + ], + }, + } + ], + }, + code_model, + ) + operation = LROOperationBase.__new__(LROOperationBase) + operation.code_model = code_model + operation.status_monitor_type = monitor + properties = operation._metadata_properties(["wire.parent", "wire.class"]) + assert properties[-1].client_name == "class_" + assert operation._metadata_path(properties) == expected + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("subdir", [None, "_generated"]) +def test_metadata_poller_paths_use_the_public_package(async_mode, subdir): + response = LROResponse.__new__(LROResponse) + response.code_model = SimpleNamespace(namespace="sample", options={"generation-subdir": subdir}) + response.has_lro_metadata = True + package = f"sample.{subdir}" if subdir else "sample" + + assert response.get_poller_path(async_mode) == ( + f"{package}.aio.AsyncMetadataLROPoller" if async_mode else f"{package}.MetadataLROPoller" + ) + assert response.get_polling_method_path(async_mode) == ( + f"{package}._polling._AsyncMetadataPolling" if async_mode else f"{package}._polling._MetadataPolling" + ) diff --git a/packages/http-client-python/tests/tox.ini b/packages/http-client-python/tests/tox.ini index 29dde03686a..daaf804fcd2 100644 --- a/packages/http-client-python/tests/tox.ini +++ b/packages/http-client-python/tests/tox.ini @@ -34,7 +34,7 @@ deps = -e {tox_root}/../generator commands = python {tox_root}/install_packages.py azure {tox_root} - pytest mock_api/azure mock_api/shared -v -n auto {posargs} + pytest mock_api/azure mock_api/shared -v -n auto --dist=loadgroup {posargs} [testenv:test-unbranded] description = Run tests for unbranded flavor diff --git a/packages/http-client-python/tests/unit/test_lro_response.py b/packages/http-client-python/tests/unit/test_lro_response.py deleted file mode 100644 index f7754cbe72c..00000000000 --- a/packages/http-client-python/tests/unit/test_lro_response.py +++ /dev/null @@ -1,40 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from types import SimpleNamespace - -import pytest - -from pygen.codegen.serializers.builder_serializer import LROOperationSerializer - - -@pytest.fixture(scope="session", autouse=True) -def testserver(): - yield - - -class _TestLROOperationSerializer(LROOperationSerializer): - def response_headers_and_deserialization(self, builder, response): - return ["deserialized = response.json()"] - - -@pytest.mark.parametrize("models_mode", [None, "dpg"]) -@pytest.mark.parametrize("generate_typeddict_only", [False, True]) -def test_lro_response_is_available_for_deserialization(models_mode, generate_typeddict_only): - code_model = SimpleNamespace( - options={"models-mode": models_mode}, - generate_typeddict_only=generate_typeddict_only, - get_serialize_namespace=lambda *args, **kwargs: "test", - ) - serializer = _TestLROOperationSerializer(code_model, async_mode=False, client_namespace="test") - operation = SimpleNamespace( - lro_response=SimpleNamespace(headers=[], type=object()), - ) - - generated = serializer.get_long_running_output(operation) - - assert " response = pipeline_response.http_response" in generated - assert " deserialized = response.json()" in generated