From ca0b95219b9ecedaf6da7bfdae948c5a9ca7e02c Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Mon, 19 Oct 2026 16:45:49 -0500 Subject: [PATCH 1/2] [Health Deid] Replace fixed sleep with retry/backoff in test-resources-post.ps1 Role assignment and network ACL propagation can take minutes after deployment, causing intermittent 403 AuthorizationFailure on upload. Retry container access and each blob upload with exponential backoff (5s initial, 60s cap, 5 min budget) and fail with a clear error. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d9b0092-996f-4265-b2da-1effbf3945c6 --- .../test-resources-post.ps1 | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 index c162558889fd..2fe997c58ed7 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 +++ b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 @@ -27,6 +27,44 @@ if ([string]::IsNullOrWhiteSpace($storageAccountName)) { exit 1 } +# Retries a script block with exponential backoff until it succeeds or the time budget is exhausted. +# Role assignments and network ACL changes can take minutes to propagate after deployment. +function Invoke-WithRetry { + param ( + [Parameter(Mandatory = $true)] + [scriptblock] $ScriptBlock, + [Parameter(Mandatory = $true)] + [string] $Operation, + [int] $MaxTotalSeconds = 300, + [int] $InitialDelaySeconds = 5, + [int] $MaxDelaySeconds = 60 + ) + + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + $delay = $InitialDelaySeconds + $attempt = 0 + while ($true) { + $attempt++ + try { + $result = & $ScriptBlock + if ($attempt -gt 1) { + Write-Host "$Operation succeeded on attempt $attempt after $([int]$stopwatch.Elapsed.TotalSeconds)s." + } + return $result + } + catch { + $lastError = $_ + $elapsed = $stopwatch.Elapsed.TotalSeconds + if ($elapsed + $delay -gt $MaxTotalSeconds) { + throw "$Operation failed after $attempt attempt(s) over $([int]$elapsed)s. This usually means the storage role assignment or network ACL change has not propagated. Last error: $($lastError.Exception.Message)" + } + Write-Host "$Operation failed (attempt $attempt, $([int]$elapsed)s elapsed): $($lastError.Exception.Message). Retrying in ${delay}s." + Start-Sleep -Seconds $delay + $delay = [Math]::Min($delay * 2, $MaxDelaySeconds) + } + } +} + # Load the Azure Storage module Import-Module Az.Storage @@ -36,25 +74,26 @@ $storageContext = New-AzStorageContext -StorageAccountName $storageAccountName - # FIXME Remove once vpn team fixes the network acl issue $networkRuleSet = New-Object -TypeName Microsoft.Azure.Commands.Management.Storage.Models.PSNetworkRuleSet $networkRuleSet.DefaultAction = "Allow" -Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet +Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet -ErrorAction Stop -# Sleep for 30 seconds to allow the network rule to take effect -Write-Host "[Fix] Temporary sleep to allow network rule to take effect." -Start-Sleep -Seconds 30 - -Get-AzStorageContainer -Name $containerName -Context $storageContext +# Wait until the container is reachable through the data plane (RBAC + network ACL propagated) +Invoke-WithRetry -Operation "Accessing container '$containerName'" -ScriptBlock { + Get-AzStorageContainer -Name $containerName -Context $storageContext -ErrorAction Stop +} # Upload the folder and its contents to the container # Gets last folder name + filename. example_patient_1/doctor_dictation.txt -Get-ChildItem -Path $localFolderPath -Recurse | ForEach-Object { - $relativePath = $_.FullName - $folderName = Split-Path -Path (Split-Path -Path $relativePath -Parent) -Leaf - $blobName = Split-Path -Path $relativePath -Leaf +Get-ChildItem -Path $localFolderPath -Recurse -File | ForEach-Object { + $filePath = $_.FullName + $folderName = Split-Path -Path (Split-Path -Path $filePath -Parent) -Leaf + $blobName = Split-Path -Path $filePath -Leaf $destinationBlob = $blobName -replace ":", "" $destinationBlob = "$folderName/$destinationBlob" Write-Host "Uploading file '$destinationBlob'" - Set-AzStorageBlobContent -File $_.FullName -Container $containerName -Blob $destinationBlob -Context $storageContext -Force + Invoke-WithRetry -Operation "Uploading blob '$destinationBlob'" -ScriptBlock { + Set-AzStorageBlobContent -File $filePath -Container $containerName -Blob $destinationBlob -Context $storageContext -Force -ErrorAction Stop + } } Write-Host "Folder '$localFolderPath' uploaded to container '$containerName' successfully." From 2e9ce5522e403801d930db7081567fe38a4933ad Mon Sep 17 00:00:00 2001 From: Josiah Vinson Date: Wed, 21 Oct 2026 10:44:01 -0500 Subject: [PATCH 2/2] [Health Deid] Keep a 30s minimum wait after the network rule update The retry loop proves the test principal's RBAC and the network rule, but not the DeID managed identity's role assignment. Restore the previous 30s wait as a minimum measured from the network rule update; time spent on retries and uploads counts toward it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d9b0092-996f-4265-b2da-1effbf3945c6 --- .../test-resources-post.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 index 2fe997c58ed7..084db7465cfa 100644 --- a/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 +++ b/sdk/healthdataaiservices/azure-health-deidentification/test-resources-post.ps1 @@ -75,6 +75,7 @@ $storageContext = New-AzStorageContext -StorageAccountName $storageAccountName - $networkRuleSet = New-Object -TypeName Microsoft.Azure.Commands.Management.Storage.Models.PSNetworkRuleSet $networkRuleSet.DefaultAction = "Allow" Set-AzStorageAccount -ResourceGroupName $resourceGroup -Name $storageAccountName -NetworkRuleSet $networkRuleSet -ErrorAction Stop +$sinceNetworkRuleUpdate = [System.Diagnostics.Stopwatch]::StartNew() # Wait until the container is reachable through the data plane (RBAC + network ACL propagated) Invoke-WithRetry -Operation "Accessing container '$containerName'" -ScriptBlock { @@ -97,3 +98,12 @@ Get-ChildItem -Path $localFolderPath -Recurse -File | ForEach-Object { } Write-Host "Folder '$localFolderPath' uploaded to container '$containerName' successfully." + +# Our own access proves the test principal's RBAC and the network rule, but not the DeID service's managed +# identity role assignment. Keep a minimum wait after the network rule update for it to propagate. +$minimumWaitSeconds = 30 +$remainingSeconds = [int][Math]::Ceiling($minimumWaitSeconds - $sinceNetworkRuleUpdate.Elapsed.TotalSeconds) +if ($remainingSeconds -gt 0) { + Write-Host "Waiting ${remainingSeconds}s more so at least ${minimumWaitSeconds}s pass after the network rule update for role assignments to propagate." + Start-Sleep -Seconds $remainingSeconds +}