diff --git a/sdk/healthdataaiservices/azure-health-deidentification/README.md b/sdk/healthdataaiservices/azure-health-deidentification/README.md index 87b662118aa9..808d56f9ac41 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/README.md +++ b/sdk/healthdataaiservices/azure-health-deidentification/README.md @@ -152,6 +152,7 @@ The client exposes a `begin_deidentify_documents` method that returns a [LROPoll ```python +from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -183,7 +184,13 @@ job = DeidentificationJob( target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) -finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) +try: + finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) +except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py index fe065600659e..d6d2d44e915f 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py @@ -23,6 +23,7 @@ import asyncio +from azure.core.exceptions import HttpResponseError from azure.core.polling import AsyncLROPoller from azure.health.deidentification.aio import DeidentificationClient from azure.health.deidentification.models import ( @@ -56,9 +57,15 @@ async def deidentify_documents_async(): async with client: lro: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = await lro.result() - - await credential.close() + try: + finished_job: DeidentificationJob = await lro.result() + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise + finally: + await credential.close() print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") # Succeeded diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py index 54ea3ec0464c..b17baa6ab8c0 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py @@ -22,6 +22,7 @@ import asyncio +from azure.core.exceptions import HttpResponseError from azure.core.polling import AsyncLROPoller from azure.health.deidentification.aio import DeidentificationClient from azure.health.deidentification.models import ( @@ -56,7 +57,14 @@ async def list_job_documents_async(): print(f"Creating job with name: {jobname}") async with client: poller: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - job = await poller.result() + try: + job = await poller.result() + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = await client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + await credential.close() + raise print(f"Job Status: {job.status}") files = client.list_job_documents(jobname) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py index ac6e2c05026d..4657bf075ef1 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py @@ -26,6 +26,7 @@ def deidentify_documents(): # [START sample] + from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -57,7 +58,13 @@ def deidentify_documents(): target_location=TargetStorageLocation(location=storage_location, prefix=outputPrefix, overwrite=True), ) - finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) + try: + finished_job: DeidentificationJob = client.begin_deidentify_documents(jobname, job).result(timeout=120) + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise print(f"Job Name: {finished_job.job_name}") print(f"Job Status: {finished_job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py index 1322e0a75285..edaaa20d7b38 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/samples/list_job_documents.py @@ -23,6 +23,7 @@ """ +from azure.core.exceptions import HttpResponseError from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.models import ( DeidentificationJob, @@ -56,7 +57,13 @@ def list_job_documents(): print(f"Creating job with name: {jobname}") - job = client.begin_deidentify_documents(jobname, job).result(timeout=60) + try: + job = client.begin_deidentify_documents(jobname, job).result(timeout=60) + except HttpResponseError: + # A failed or canceled job surfaces as a generic polling error; the job's own error explains why. + failed_job = client.get_job(jobname) + print(f"Job {jobname} ended with status {failed_job.status}: {failed_job.error}") + raise print(f"Job Status: {job.status}") diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py index 625400893a1f..9ff83df79fa4 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/conftest.py @@ -21,16 +21,18 @@ def start_proxy(test_proxy, patch_sleep, patch_async_sleep): @pytest.fixture(scope="session", autouse=True) def create_session_uniquifier(): - if ( - os.environ.get("AZURE_TEST_RUN_LIVE", "false").lower() == "true" # Don't override uniquifier by default - and os.environ.get("AZURE_SKIP_LIVE_RECORDING", "false").lower() != "true" - and os.environ.get("AZURE_TEST_KEEP_UNIQUIFIER", "false").lower() != "true" - ): - print("Creating new uniquifier for live test run.") + is_live = os.environ.get("AZURE_TEST_RUN_LIVE", "false").lower() == "true" + skip_recording = os.environ.get("AZURE_SKIP_LIVE_RECORDING", "false").lower() == "true" + keep_uniquifier = os.environ.get("AZURE_TEST_KEEP_UNIQUIFIER", "false").lower() == "true" + if is_live and not keep_uniquifier: + # Every live run gets fresh job names so reused resources or parallel matrix legs can't collide. uniquifier = uuid.uuid4().hex[:6] + print(f"Creating new uniquifier for live test run: {uniquifier}") os.environ["HEALTHDATAAISERVICES_UNIQUIFIER"] = uniquifier - with open(uniquifier_file, "w") as file: - file.write(uniquifier) + if not skip_recording: + # Persist only when recording so playback replays the job names in the new recordings. + with open(uniquifier_file, "w") as file: + file.write(uniquifier) else: with open(uniquifier_file, "r") as file: uniquifier = file.read() diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py index cb9bb597f468..a8e0143cb6a2 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/deid_base_test_case.py @@ -1,13 +1,24 @@ +import asyncio import functools import inspect import os import random import datetime import uuid +from typing import List, Optional + +from azure.core.exceptions import HttpResponseError +from azure.core.polling import AsyncLROPoller, LROPoller +from azure.core.polling.base_polling import OperationFailed from azure.health.deidentification import DeidentificationClient from azure.health.deidentification.aio import ( DeidentificationClient as DeidentificationClientAsync, ) +from azure.health.deidentification.models import ( + DeidentificationDocumentDetails, + DeidentificationJob, + OperationStatus, +) from devtools_testutils import ( @@ -30,9 +41,113 @@ ) +JOB_TIMEOUT_SECONDS = 600 +MAX_DIAGNOSTIC_DOCUMENTS = 20 + + +def format_job_diagnostics(job: DeidentificationJob, documents: List[DeidentificationDocumentDetails]) -> str: + lines = [ + f"Job '{job.job_name}' status: {job.status}", + f" created_at={job.created_at} started_at={job.started_at} last_updated_at={job.last_updated_at}", + ] + if job.summary is not None: + s = job.summary + lines.append( + f" summary: total={s.total_count} successful={s.successful_count} " + f"failed={s.failed_count} canceled={s.canceled_count} bytes_processed={s.bytes_processed}" + ) + else: + lines.append(" summary: None") + lines.append(f" error: {job.error if job.error is not None else 'None'}") + if documents: + lines.append(f" unsuccessful documents (up to {MAX_DIAGNOSTIC_DOCUMENTS}):") + for doc in documents: + location = doc.input_location.location if doc.input_location is not None else doc.id + lines.append(f" - {location} status={doc.status} error={doc.error}") + return "\n".join(lines) + + +def _is_unsuccessful(doc: DeidentificationDocumentDetails) -> bool: + return doc.status != OperationStatus.SUCCEEDED or doc.error is not None + + class DeidBaseTestCase(AzureRecordedTestCase): OUTPUT_PATH = "_output" + def describe_job(self, client: DeidentificationClient, job_name: str) -> str: + """Fetch the job and its unsuccessful documents and format them for an assertion message.""" + try: + job = client.get_job(job_name) + documents: List[DeidentificationDocumentDetails] = [] + for doc in client.list_job_documents(job_name): + if _is_unsuccessful(doc): + documents.append(doc) + if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: + break + return format_job_diagnostics(job, documents) + except Exception as e: # pylint: disable=broad-except + # Diagnostics must never mask the original failure. + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + + async def describe_job_async(self, client: DeidentificationClientAsync, job_name: str) -> str: + """Async version of describe_job.""" + try: + job = await client.get_job(job_name) + documents: List[DeidentificationDocumentDetails] = [] + async for doc in client.list_job_documents(job_name): + if _is_unsuccessful(doc): + documents.append(doc) + if len(documents) >= MAX_DIAGNOSTIC_DOCUMENTS: + break + return format_job_diagnostics(job, documents) + except Exception as e: # pylint: disable=broad-except + return f"Unable to fetch diagnostics for job '{job_name}': {e!r}" + + def wait_for_job( + self, + client: DeidentificationClient, + poller: LROPoller[DeidentificationJob], + job_name: str, + timeout: Optional[float] = JOB_TIMEOUT_SECONDS, + ) -> DeidentificationJob: + """Wait for a job LRO and fail with the job's real error instead of azure-core's generic polling error. + + azure-core surfaces a Failed/Canceled job as HttpResponseError("Operation returned an invalid status 'OK'"), + and LROPoller.result(timeout) silently returns the in-progress job when the timeout expires. + """ + try: + job = poller.result(timeout=timeout) + except (HttpResponseError, OperationFailed) as e: + raise AssertionError( + f"Job '{job_name}' did not succeed ({e}).\n{self.describe_job(client, job_name)}" + ) from e + if not poller.done(): + raise AssertionError( + f"Job '{job_name}' did not reach a terminal state within {timeout}s.\n" + f"{self.describe_job(client, job_name)}" + ) + return job + + async def wait_for_job_async( + self, + client: DeidentificationClientAsync, + poller: AsyncLROPoller[DeidentificationJob], + job_name: str, + timeout: Optional[float] = JOB_TIMEOUT_SECONDS, + ) -> DeidentificationJob: + """Async version of wait_for_job.""" + try: + return await asyncio.wait_for(poller.result(), timeout=timeout) + except asyncio.TimeoutError as e: + raise AssertionError( + f"Job '{job_name}' did not reach a terminal state within {timeout}s.\n" + f"{await self.describe_job_async(client, job_name)}" + ) from e + except (HttpResponseError, OperationFailed) as e: + raise AssertionError( + f"Job '{job_name}' did not succeed ({e}).\n{await self.describe_job_async(client, job_name)}" + ) from e + def make_client(self, endpoint) -> DeidentificationClient: credential = self.get_credential(DeidentificationClient) connection_verify = not "localhost" in endpoint diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py index aaebe91349a6..1da0e802c658 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish.py @@ -29,7 +29,7 @@ def test_create_wait_finish(self, **kwargs): ) lro: LROPoller = client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = lro.result(timeout=120) + finished_job: DeidentificationJob = self.wait_for_job(client, lro, jobname) assert finished_job.status == OperationStatus.SUCCEEDED assert finished_job.job_name == jobname @@ -50,7 +50,9 @@ def test_create_wait_finish(self, **kwargs): for my_file in files: assert len(my_file.id) == 36 # GUID assert input_prefix in my_file.input_location.location - assert my_file.status == OperationStatus.SUCCEEDED + assert ( + my_file.status == OperationStatus.SUCCEEDED + ), f"{my_file.input_location.location}: status={my_file.status} error={my_file.error}" assert my_file.output_location is not None assert self.OUTPUT_PATH in my_file.output_location.location count += 1 diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py index be9bae5499fc..7545a9f3b0cd 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_create_wait_finish_async.py @@ -32,7 +32,7 @@ async def test_create_wait_finish_async(self, **kwargs): lro: AsyncLROPoller = await client.begin_deidentify_documents(jobname, job) - finished_job: DeidentificationJob = await lro.result() + finished_job: DeidentificationJob = await self.wait_for_job_async(client, lro, jobname) assert finished_job.status == OperationStatus.SUCCEEDED assert finished_job.job_name == jobname @@ -54,7 +54,9 @@ async def test_create_wait_finish_async(self, **kwargs): async for my_file in files: assert len(my_file.id) == 36 # GUID assert input_prefix in my_file.input_location.location - assert my_file.status == OperationStatus.SUCCEEDED + assert ( + my_file.status == OperationStatus.SUCCEEDED + ), f"{my_file.input_location.location}: status={my_file.status} error={my_file.error}" assert my_file.output_location is not None assert self.OUTPUT_PATH in my_file.output_location.location count += 1 diff --git a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py index ddb7f06ca83f..bffb32dbcdd4 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py +++ b/sdk/healthdataaiservices/azure-health-deidentification/tests/test_list_job_docs_pagination.py @@ -33,7 +33,7 @@ def test_list_job_docs_pagination(self, **kwargs): customizations=DeidentificationJobCustomizationOptions(redaction_format="[{type}]"), ) - client.begin_deidentify_documents(jobname, job).result(180) + self.wait_for_job(client, client.begin_deidentify_documents(jobname, job), jobname) job_documents = client.list_job_documents(job_name=jobname, maxpagesize=2) _get_next = job_documents._args[0] # type: ignore