From ab9b9d3f84d431ee7158631e1e2045928118e324 Mon Sep 17 00:00:00 2001 From: Michael Richey Date: Tue, 25 Aug 2026 11:21:03 -0400 Subject: [PATCH] Skip read-only integration pipelines instead of failing Integration pipelines (is_read_only=true) are auto-managed by Datadog and cannot be created or modified via the public API. The create_resource method already attempts to trigger creation via the logs intake API, but when that fails it raises a bare Exception, causing 66 persistent failures on every sync run for orgs like Allstate that have many integration pipelines not present at the destination. Replace both Exception raises in create_resource with SkipResource: 1. When source extraction from the filter query fails 2. When the logs-intake trigger doesn't create the pipeline within the polling window This matches the existing behavior in update_resource, which already raises SkipResource for read-only pipelines. --- datadog_sync/model/logs_pipelines.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/datadog_sync/model/logs_pipelines.py b/datadog_sync/model/logs_pipelines.py index 7c306425..0959f1f7 100644 --- a/datadog_sync/model/logs_pipelines.py +++ b/datadog_sync/model/logs_pipelines.py @@ -104,7 +104,12 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: # Extract the source from the query source = self.extract_source_from_query(resource.get("filter", {}).get("query")) if not source: - raise Exception(f"Source not found in the query for integration pipeline '{resource['name']}'") + raise SkipResource( + _id, + self.resource_type, + f"Source not found in the query for integration pipeline '{resource['name']}'. " + "Integration pipelines are auto-managed by Datadog and cannot be created via the public API.", + ) payload = { "ddsource": source, "ddtags": ",".join(DEFAULT_TAGS), @@ -134,9 +139,20 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: await sleep(5) if not created: - raise Exception( - f"Integration pipeline '{resource['name']}' is not created after x seconds. " - "It will be rechecked in the next sync." + # Integration pipelines are auto-managed by Datadog and cannot be + # created via the public API. The logs-intake trigger above is a + # best-effort attempt to prompt the destination org's integration + # system to provision the pipeline; when it does not appear within + # the polling window the pipeline is almost certainly one that the + # destination org will never have (e.g. an integration that is not + # enabled there). Skip instead of failing so the run does not + # report 66 persistent failures every cycle. + raise SkipResource( + _id, + self.resource_type, + f"Integration pipeline '{resource['name']}' could not be triggered at the destination " + "after polling timeout. Integration pipelines are auto-managed by Datadog and may " + "not be available at the destination org.", ) self.config.state.destination[self.resource_type][_id] = self.destination_integration_pipelines[