From 3fd5c48ac001bbf6c1c4277934f098dca4fe5442 Mon Sep 17 00:00:00 2001 From: Florian BRACQ Date: Thu, 20 Aug 2026 12:18:47 +0000 Subject: [PATCH 1/2] Replace datetime.utcnow() with datetime.now(tz=dt.timezone.utc) --- .../azure/kusto/ingest/_resource_manager.py | 10 ++++++---- .../azure/kusto/ingest/ingestion_blob_info.py | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py b/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py index 50c769dde..66e529bad 100644 --- a/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py +++ b/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import List, Dict from urllib.parse import urlparse @@ -85,7 +85,8 @@ def __set_throttling_settings(self, num_of_attempts: int = 4, max_seconds_per_re def _refresh_ingest_client_resources(self): if ( not self._ingest_client_resources - or (self._ingest_client_resources_last_update + self._refresh_period) <= datetime.utcnow() + or (self._ingest_client_resources_last_update + self._refresh_period) + <= datetime.now(tz=timezone.utc) or not self._ingest_client_resources.is_applicable() ): self._ingest_client_resources = self._get_ingest_client_resources_from_service() @@ -119,10 +120,11 @@ def _refresh_authorization_context(self): if ( not self._authorization_context or self._authorization_context.isspace() - or (self._authorization_context_last_update + self._refresh_period) <= datetime.utcnow() + or (self._authorization_context_last_update + self._refresh_period) + <= datetime.now(tz=timezone.utc) ): self._authorization_context = self._get_authorization_context_from_service() - self._authorization_context_last_update = datetime.utcnow() + self._authorization_context_last_update = datetime.now(tz=timezone.utc) def _get_authorization_context_from_service(self): # trace all calls to get identity token diff --git a/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py b/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py index 29ce9a63b..9496267c3 100644 --- a/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py +++ b/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License import json -from datetime import datetime +from datetime import datetime, timezone from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -28,7 +28,9 @@ def __init__( self.properties["IgnoreSizeLimit"] = False self.properties["ReportLevel"] = ingestion_properties.report_level.value self.properties["ReportMethod"] = ingestion_properties.report_method.value - self.properties["SourceMessageCreationTime"] = datetime.utcnow().isoformat() + self.properties["SourceMessageCreationTime"] = datetime.now( + tz=timezone.utc + ).isoformat() self.properties["Id"] = str(blob_descriptor.source_id) self.properties["ApplicationForTracing"] = application_for_tracing self.properties["ClientVersionForTracing"] = client_version_for_tracing From 907f63a54dc73bd0704879f096805abf7b597d6a Mon Sep 17 00:00:00 2001 From: Aasav Badera Date: Fri, 20 Nov 2026 14:06:38 +0530 Subject: [PATCH 2/2] Fix UTC timestamps in ingestion resource cache Keep resource cache writes and comparisons in aware UTC. Cover successive resource lookups and cache-expiry boundaries, verify UTC ingestion message serialization, and format the PR changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f4a536a-7f7a-4d39-b60e-96db0cb49c1c --- .../azure/kusto/ingest/_resource_manager.py | 8 ++--- .../azure/kusto/ingest/ingestion_blob_info.py | 4 +-- .../tests/test_ingestion_blob_info.py | 2 ++ .../tests/test_kusto_ingest_client.py | 34 +++++++++++++++++++ 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py b/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py index 66e529bad..94ab33be4 100644 --- a/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py +++ b/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py @@ -85,12 +85,11 @@ def __set_throttling_settings(self, num_of_attempts: int = 4, max_seconds_per_re def _refresh_ingest_client_resources(self): if ( not self._ingest_client_resources - or (self._ingest_client_resources_last_update + self._refresh_period) - <= datetime.now(tz=timezone.utc) + or (self._ingest_client_resources_last_update + self._refresh_period) <= datetime.now(tz=timezone.utc) or not self._ingest_client_resources.is_applicable() ): self._ingest_client_resources = self._get_ingest_client_resources_from_service() - self._ingest_client_resources_last_update = datetime.utcnow() + self._ingest_client_resources_last_update = datetime.now(tz=timezone.utc) self._populate_ranked_storage_account_set() def _get_resource_by_name(self, table: KustoResultTable, resource_name: str): @@ -120,8 +119,7 @@ def _refresh_authorization_context(self): if ( not self._authorization_context or self._authorization_context.isspace() - or (self._authorization_context_last_update + self._refresh_period) - <= datetime.now(tz=timezone.utc) + or (self._authorization_context_last_update + self._refresh_period) <= datetime.now(tz=timezone.utc) ): self._authorization_context = self._get_authorization_context_from_service() self._authorization_context_last_update = datetime.now(tz=timezone.utc) diff --git a/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py b/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py index 9496267c3..efbeaa8d4 100644 --- a/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py +++ b/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py @@ -28,9 +28,7 @@ def __init__( self.properties["IgnoreSizeLimit"] = False self.properties["ReportLevel"] = ingestion_properties.report_level.value self.properties["ReportMethod"] = ingestion_properties.report_method.value - self.properties["SourceMessageCreationTime"] = datetime.now( - tz=timezone.utc - ).isoformat() + self.properties["SourceMessageCreationTime"] = datetime.now(tz=timezone.utc).isoformat() self.properties["Id"] = str(blob_descriptor.source_id) self.properties["ApplicationForTracing"] = application_for_tracing self.properties["ClientVersionForTracing"] = client_version_for_tracing diff --git a/azure-kusto-ingest/tests/test_ingestion_blob_info.py b/azure-kusto-ingest/tests/test_ingestion_blob_info.py index 73ed8eb66..bd9cc05a8 100644 --- a/azure-kusto-ingest/tests/test_ingestion_blob_info.py +++ b/azure-kusto-ingest/tests/test_ingestion_blob_info.py @@ -3,6 +3,7 @@ import json import re import unittest +from datetime import datetime, timedelta from uuid import UUID from azure.kusto.data.data_format import DataFormat @@ -125,6 +126,7 @@ def _verify_ingestion_blob_info_result(self, ingestion_blob_info): assert isinstance(result["ReportLevel"], int) assert isinstance(UUID(result["Id"]), UUID) assert re.match(TIMESTAMP_REGEX, result["SourceMessageCreationTime"]) + assert datetime.fromisoformat(result["SourceMessageCreationTime"]).utcoffset() == timedelta(0) assert result["AdditionalProperties"]["authorizationContext"] == "authorizationContextText" assert result["AdditionalProperties"]["ingestIfNotExists"] == '["ingestIfNotExistTags"]' assert result["AdditionalProperties"]["ValidationPolicy"] in ( diff --git a/azure-kusto-ingest/tests/test_kusto_ingest_client.py b/azure-kusto-ingest/tests/test_kusto_ingest_client.py index 4726fd8d3..3f9d91b50 100644 --- a/azure-kusto-ingest/tests/test_kusto_ingest_client.py +++ b/azure-kusto-ingest/tests/test_kusto_ingest_client.py @@ -4,6 +4,7 @@ import json import os import uuid +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import patch @@ -568,6 +569,39 @@ def test_containers(self): assert containers_selected_with_round_robin[1].storage_account_name == containers_selected_with_round_robin[3].storage_account_name assert containers_selected_with_round_robin[1].storage_account_name != containers_selected_with_round_robin[2].storage_account_name + @pytest.mark.parametrize( + ("elapsed", "expected_fetches"), + [ + (timedelta(minutes=59, seconds=59), 1), + (timedelta(hours=1), 2), + (timedelta(hours=1, seconds=1), 2), + ], + ) + def test_ingestion_resources_cache_refresh(self, elapsed: timedelta, expected_fetches: int) -> None: + container = _resource_manager._ResourceUri(TEMP_STORAGE_URL) + queue = _resource_manager._ResourceUri(STORAGE_QUEUE_URL) + resources = _resource_manager._IngestClientResources( + secured_ready_for_aggregation_queues=[queue], + failed_ingestions_queues=[queue], + successful_ingestions_queues=[queue], + containers=[container], + status_tables=[container], + ) + start = datetime(2025, 1, 1, tzinfo=timezone.utc) + + with _resource_manager.KustoClient("https://ingest-somecluster.kusto.windows.net") as client: + resource_manager = _resource_manager._ResourceManager(client) + with patch.object(resource_manager, "_get_ingest_client_resources_from_service", return_value=resources) as get_resources: + with patch("azure.kusto.ingest._resource_manager.datetime", wraps=datetime) as mock_datetime: + mock_datetime.now.return_value = start + assert resource_manager.get_containers() == [container] + + mock_datetime.now.return_value = start + elapsed + assert resource_manager.get_ingestion_queues() == [queue] + assert resource_manager.get_containers() == [container] + + assert get_resources.call_count == expected_fetches + @responses.activate @patch( "azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0