diff --git a/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py b/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py index 50c769dd..94ab33be 100644 --- a/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py +++ b/azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import List, Dict from urllib.parse import urlparse @@ -85,11 +85,11 @@ def __set_throttling_settings(self, num_of_attempts: int = 4, max_seconds_per_re def _refresh_ingest_client_resources(self): if ( not self._ingest_client_resources - or (self._ingest_client_resources_last_update + self._refresh_period) <= datetime.utcnow() + or (self._ingest_client_resources_last_update + self._refresh_period) <= datetime.now(tz=timezone.utc) or not self._ingest_client_resources.is_applicable() ): self._ingest_client_resources = self._get_ingest_client_resources_from_service() - self._ingest_client_resources_last_update = datetime.utcnow() + self._ingest_client_resources_last_update = datetime.now(tz=timezone.utc) self._populate_ranked_storage_account_set() def _get_resource_by_name(self, table: KustoResultTable, resource_name: str): @@ -119,10 +119,10 @@ def _refresh_authorization_context(self): if ( not self._authorization_context or self._authorization_context.isspace() - or (self._authorization_context_last_update + self._refresh_period) <= datetime.utcnow() + or (self._authorization_context_last_update + self._refresh_period) <= datetime.now(tz=timezone.utc) ): self._authorization_context = self._get_authorization_context_from_service() - self._authorization_context_last_update = datetime.utcnow() + self._authorization_context_last_update = datetime.now(tz=timezone.utc) def _get_authorization_context_from_service(self): # trace all calls to get identity token diff --git a/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py b/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py index 29ce9a63..efbeaa8d 100644 --- a/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py +++ b/azure-kusto-ingest/azure/kusto/ingest/ingestion_blob_info.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License import json -from datetime import datetime +from datetime import datetime, timezone from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -28,7 +28,7 @@ def __init__( self.properties["IgnoreSizeLimit"] = False self.properties["ReportLevel"] = ingestion_properties.report_level.value self.properties["ReportMethod"] = ingestion_properties.report_method.value - self.properties["SourceMessageCreationTime"] = datetime.utcnow().isoformat() + self.properties["SourceMessageCreationTime"] = datetime.now(tz=timezone.utc).isoformat() self.properties["Id"] = str(blob_descriptor.source_id) self.properties["ApplicationForTracing"] = application_for_tracing self.properties["ClientVersionForTracing"] = client_version_for_tracing diff --git a/azure-kusto-ingest/tests/test_ingestion_blob_info.py b/azure-kusto-ingest/tests/test_ingestion_blob_info.py index 73ed8eb6..bd9cc05a 100644 --- a/azure-kusto-ingest/tests/test_ingestion_blob_info.py +++ b/azure-kusto-ingest/tests/test_ingestion_blob_info.py @@ -3,6 +3,7 @@ import json import re import unittest +from datetime import datetime, timedelta from uuid import UUID from azure.kusto.data.data_format import DataFormat @@ -125,6 +126,7 @@ def _verify_ingestion_blob_info_result(self, ingestion_blob_info): assert isinstance(result["ReportLevel"], int) assert isinstance(UUID(result["Id"]), UUID) assert re.match(TIMESTAMP_REGEX, result["SourceMessageCreationTime"]) + assert datetime.fromisoformat(result["SourceMessageCreationTime"]).utcoffset() == timedelta(0) assert result["AdditionalProperties"]["authorizationContext"] == "authorizationContextText" assert result["AdditionalProperties"]["ingestIfNotExists"] == '["ingestIfNotExistTags"]' assert result["AdditionalProperties"]["ValidationPolicy"] in ( diff --git a/azure-kusto-ingest/tests/test_kusto_ingest_client.py b/azure-kusto-ingest/tests/test_kusto_ingest_client.py index 4726fd8d..3f9d91b5 100644 --- a/azure-kusto-ingest/tests/test_kusto_ingest_client.py +++ b/azure-kusto-ingest/tests/test_kusto_ingest_client.py @@ -4,6 +4,7 @@ import json import os import uuid +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import patch @@ -568,6 +569,39 @@ def test_containers(self): assert containers_selected_with_round_robin[1].storage_account_name == containers_selected_with_round_robin[3].storage_account_name assert containers_selected_with_round_robin[1].storage_account_name != containers_selected_with_round_robin[2].storage_account_name + @pytest.mark.parametrize( + ("elapsed", "expected_fetches"), + [ + (timedelta(minutes=59, seconds=59), 1), + (timedelta(hours=1), 2), + (timedelta(hours=1, seconds=1), 2), + ], + ) + def test_ingestion_resources_cache_refresh(self, elapsed: timedelta, expected_fetches: int) -> None: + container = _resource_manager._ResourceUri(TEMP_STORAGE_URL) + queue = _resource_manager._ResourceUri(STORAGE_QUEUE_URL) + resources = _resource_manager._IngestClientResources( + secured_ready_for_aggregation_queues=[queue], + failed_ingestions_queues=[queue], + successful_ingestions_queues=[queue], + containers=[container], + status_tables=[container], + ) + start = datetime(2025, 1, 1, tzinfo=timezone.utc) + + with _resource_manager.KustoClient("https://ingest-somecluster.kusto.windows.net") as client: + resource_manager = _resource_manager._ResourceManager(client) + with patch.object(resource_manager, "_get_ingest_client_resources_from_service", return_value=resources) as get_resources: + with patch("azure.kusto.ingest._resource_manager.datetime", wraps=datetime) as mock_datetime: + mock_datetime.now.return_value = start + assert resource_manager.get_containers() == [container] + + mock_datetime.now.return_value = start + elapsed + assert resource_manager.get_ingestion_queues() == [queue] + assert resource_manager.get_containers() == [container] + + assert get_resources.call_count == expected_fetches + @responses.activate @patch( "azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0