diff --git a/docs/guides/trace-and-monitor-crawlers.mdx b/docs/guides/trace-and-monitor-crawlers.mdx new file mode 100644 index 000000000000..23ae6db87f08 --- /dev/null +++ b/docs/guides/trace-and-monitor-crawlers.mdx @@ -0,0 +1,279 @@ +--- +id: trace-and-monitor-crawlers +title: Trace and monitor crawlers +description: How to use OpenTelemetry to trace and monitor your crawlers +--- + +import CodeBlock from '@theme/CodeBlock'; + +import RegisterHookSource from '!!raw-loader!./trace_and_monitor_register_hook.ts'; +import SetupSource from '!!raw-loader!./trace_and_monitor_setup.ts'; +import BasicExampleSource from '!!raw-loader!./trace_and_monitor_basic.ts'; +import WrapWithSpanSource from '!!raw-loader!./trace_and_monitor_wrap_with_span.ts'; +import CustomInstrumentationSource from '!!raw-loader!./trace_and_monitor_custom.ts'; + +[OpenTelemetry](https://opentelemetry.io/) is a collection of APIs, SDKs, and tools to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) to help you analyze your software's performance and behavior. You can learn more about its basic concepts in the [OpenTelemetry documentation](https://opentelemetry.io/docs/concepts/). + +In this guide, we'll show you how to set up OpenTelemetry and instrument your Crawlee crawlers to see traces of individual requests as they are processed. OpenTelemetry on its own does not provide visualization tools, so we'll use [Jaeger](https://www.jaegertracing.io/) as our tracing backend. Feel free to use any other OpenTelemetry-compatible backend. Check the [OpenTelemetry vendors list](https://opentelemetry.io/ecosystem/vendors/) for more options. + +## Set up Jaeger + +This guide will show you how to set up the environment locally to run the example code and visualize the telemetry data in Jaeger running in a [Docker](https://docs.docker.com/engine/install/) container. +To start the preconfigured Docker container, create a `docker-compose.yml` file: + +```yaml +services: + jaeger: + image: jaegertracing/all-in-one:1.53 + container_name: jaeger + ports: + # Jaeger UI + - "16686:16686" + # OTLP gRPC + - "4317:4317" + # OTLP HTTP + - "4318:4318" + environment: + - COLLECTOR_OTLP_ENABLED=true + restart: unless-stopped +``` + +Then start it with: + +```bash +docker compose up -d +``` + +For more details about the Jaeger setup, see the [getting started section](https://www.jaegertracing.io/docs/latest/getting-started/) in their documentation. You can see the Jaeger UI in your browser by navigating to [http://localhost:16686](http://localhost:16686). + +## Install dependencies + +To instrument your Crawlee crawler, you need to install the `@crawlee/otel` package along with the OpenTelemetry SDK packages: + +```bash npm2yarn +npm install @crawlee/otel @opentelemetry/api @opentelemetry/api-logs @opentelemetry/sdk-node @opentelemetry/sdk-trace-base @opentelemetry/resources @opentelemetry/semantic-conventions @opentelemetry/exporter-trace-otlp-grpc +``` + +## Instrument the crawler + +OpenTelemetry instrumentation must be set up **before** importing Crawlee or any other instrumented modules. The easiest way to do this is to create a separate setup file and import it first using Node.js's `--import` flag. + +### Module hook + +Crawlee is published as ECMAScript modules, so the automatic instrumentation can only patch the crawler classes through +Node's module hook. Register it in its own file, which is preloaded ahead of everything else: + + + {RegisterHookSource} + + +### Setup file + +Create a setup file that initializes OpenTelemetry with the Crawlee instrumentation. Because the exporters buffer +data, the setup file is also where the SDK is shut down, so that everything is flushed before the process exits: + + + {SetupSource} + + +### Main crawler file + +Now create your crawler. The `CrawleeInstrumentation` will automatically instrument the core crawler methods: + + + {BasicExampleSource} + + +### Run the crawler + +Run your crawler with the setup file imported first: + +```bash +npx tsx --import ./src/register-hook.ts --import ./src/setup.ts ./src/main.ts +``` + +The `--import` flags run in order, before any of your own code: the hook is installed first, then the OpenTelemetry SDK +starts, and only then is the crawler loaded and patched. + +The examples on this page live in the Crawlee repository, so to run this one from a checkout, from the repository root: + +```bash +pnpm exec tsx --import ./docs/guides/trace_and_monitor_register_hook.ts \ + --import ./docs/guides/trace_and_monitor_setup.ts \ + ./docs/guides/trace_and_monitor_basic.ts +``` + +## Troubleshoot instrumentation setup + +Proper setup of instrumentation depends on your specific environment setup. Please refer to the [OTEL documentation](https://github.com/open-telemetry/opentelemetry-js/blob/main/doc/esm-support.md). + + +## Analyze the results + +In the Jaeger UI, you can search for different traces, apply filtering, compare traces, view their detailed attributes, view timing details, and more. For a detailed description of the tool's capabilities, please refer to the [Jaeger documentation](https://www.jaegertracing.io/docs/latest/). + +![Jaeger search view](/img/jaeger-search.png) + +## Customize the instrumentation + +The `CrawleeInstrumentation` class provides several configuration options to customize what gets instrumented: + +| Option | Default | Description | +|--------|---------|-------------| +| `enabled` | `true` | Enable or disable the instrumentation entirely | +| `requestHandlingInstrumentation` | `true` | Instrument the core request handling methods of the crawlers | +| `logInstrumentation` | `true` | Forward Crawlee logs to OpenTelemetry logs | +| `customInstrumentation` | `[]` | Array of custom class methods to instrument | + +### Configuration example + +```ts +import { CrawleeInstrumentation } from '@crawlee/otel'; + +const crawleeInstrumentation = new CrawleeInstrumentation({ + // Disable automatic request handling instrumentation + requestHandlingInstrumentation: false, + // Disable log forwarding + logInstrumentation: false, + // Add custom instrumentation + customInstrumentation: [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'my-custom-span-name', + }, + ], +}); +``` + +## Manual span instrumentation with wrapWithSpan + +For more fine-grained control, you can use the `wrapWithSpan` utility to wrap specific functions with OpenTelemetry spans. This is particularly useful for instrumenting request handlers, hooks, and error handlers. + + + {WrapWithSpanSource} + + +### wrapWithSpan options + +The `wrapWithSpan` function accepts these options: + +| Option | Type | Description | +|--------|------|-------------| +| `spanName` | `string \| ((...args) => string)` | Static name or function that receives the handler arguments and returns a span name | +| `spanOptions` | `SpanOptions \| ((...args) => SpanOptions)` | Static options or function that returns OpenTelemetry SpanOptions including attributes | +| `tracer` | `Tracer` | Custom tracer instance. Defaults to the tracer of the registered `CrawleeInstrumentation`, or to a tracer from the global provider when no instrumentation is registered. | + +### Accessing the current span + +Inside a wrapped function, you can access the current span to add additional attributes or events: + +```ts +import { context, trace } from '@opentelemetry/api'; + +requestHandler: wrapWithSpan( + async ({ request, $ }) => { + const span = trace.getSpan(context.active()); + + const title = $('title').text(); + + if (span) { + span.setAttribute('page.title', title); + span.addEvent('page_scraped', { url: request.url }); + } + + // ... rest of your handler + }, + { spanName: 'request-handler' } +), +``` + +## Custom class instrumentation + +You can also create your instrumentation by selecting only the methods you want to instrument. Here's an example of adding custom instrumentation for specific crawler methods: + + + {CustomInstrumentationSource} + + +## What gets instrumented automatically + +When `requestHandlingInstrumentation` is enabled (the default), the following methods are automatically instrumented: + +| Crawler | Method | Span Name | +|---------|--------|-----------| +| `BasicCrawler` | `run` | `crawlee.crawler.run` | +| `BasicCrawler` | `handleRequest` | `crawlee.crawler.handleRequest` | +| `BasicCrawler` | `runRequestHandler` | `crawlee.crawler.runRequestHandler` | +| `BasicCrawler` | `requestFunctionErrorHandler` | `crawlee.crawler.requestFunctionErrorHandler` | +| `BasicCrawler` | `handleFailedRequestHandler` | `crawlee.crawler.handleFailedRequestHandler` | +| `HttpCrawler` | `makeHttpRequest` | `crawlee.http.makeHttpRequest` | +| `BrowserCrawler` | `navigate` | `crawlee.browser.navigate` | +| `AdaptivePlaywrightCrawler` | `runRequestHandler` | `crawlee.crawler.runRequestHandler` | + +`crawlee.http.makeHttpRequest` and `crawlee.browser.navigate` are recorded as client spans, since they are the calls +that leave your process. The rest are internal spans. + +Every crawler inherits the `BasicCrawler` methods, so the table covers all of them. `AdaptivePlaywrightCrawler` is +listed separately only because it replaces `runRequestHandler` with its own implementation, which is instrumented in +its place - a run of it still produces one `crawlee.crawler.runRequestHandler` span per request. + +Every automatically instrumented span carries the `code.function.name` attribute - spans you create yourself with +`wrapWithSpan` carry only the attributes you give them. The `crawlee.crawler.run` span additionally carries +`crawlee.crawler.type` with the class name of the crawler that is running, and spans of the methods that receive a +crawling context (the request handlers, navigation handlers and error handlers) also include: + +- `url.full` - the request URL +- `http.request.method` - the request method +- `crawlee.request.id` - the Crawlee request ID +- `crawlee.request.retry_count` - how many times the request has been retried + +`url.full` and `http.request.method` are the stable [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/), +so traces stay comparable with the rest of your instrumented stack. Crawlee-specific data that has no semantic +convention keeps the `crawlee.` prefix. + +## Forwarded logs + +With `logInstrumentation` enabled (the default), Crawlee logs are emitted as OpenTelemetry log records. The Crawlee log +level is mapped onto the OpenTelemetry severity (`SOFT_FAIL` and `WARNING` both become `WARN`, `PERF` becomes `DEBUG`), +the structured `data` of the log call becomes the log record attributes, and any `Error` in it is recorded as the +`exception.type`, `exception.message` and `exception.stacktrace` attributes. + +This works for whichever logger you configure. The instrumentation patches the logging methods Crawlee derives in +`BaseCrawleeLogger`, so a Winston, Pino or hand-written adapter is forwarded just like the default one. + +Crawlee leaves level filtering to the underlying logging library, so every message is forwarded regardless of the +configured log level - filter them in your OpenTelemetry pipeline instead. + +The record body is the message as it was passed to the log call, not the line the logger prints: a `perf` record has no +`[PERF]` prefix, and an `exception` record carries the message on its own with the error in the `exception.*` +attributes instead of appended to it. Match the two by attribute rather than by grepping for the printed line. + +:::caution Jaeger does not accept OpenTelemetry logs + +The records only go somewhere once you add a log record processor to the SDK, and the backend has to implement the +OTLP logs service. Jaeger is a tracing backend and does not - pointing a log exporter at the `jaegertracing/all-in-one` +container above makes every log batch fail with +`UNIMPLEMENTED: unknown service opentelemetry.proto.collector.logs.v1.LogsService`. Send the logs to an +[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or a backend that ingests them instead. + +::: + +To export the logs, install `@opentelemetry/sdk-logs` along with a log exporter and add a log record processor to the +setup file: + +```ts title="src/setup.ts" +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc'; +import { BatchLogRecordProcessor } from '@opentelemetry/sdk-logs'; + +export const sdk = new NodeSDK({ + // ... the trace configuration from above + logRecordProcessors: [ + new BatchLogRecordProcessor(new OTLPLogExporter({ url: 'http://localhost:4317' })), + ], +}); +``` + +Set `logInstrumentation: false` if you would rather keep the Crawlee logs out of OpenTelemetry entirely. + diff --git a/docs/guides/trace_and_monitor_basic.ts b/docs/guides/trace_and_monitor_basic.ts new file mode 100644 index 000000000000..eba24f7b6ea8 --- /dev/null +++ b/docs/guides/trace_and_monitor_basic.ts @@ -0,0 +1,19 @@ +import { CheerioCrawler } from 'crawlee'; + +const crawler = new CheerioCrawler({ + maxRequestsPerCrawl: 10, + + async requestHandler({ request, $, enqueueLinks, log }) { + const title = $('title').text(); + log.info(`Crawled ${request.url}`, { title }); + + await enqueueLinks({ + include: ['https://crawlee.dev/**'], + }); + }, +}); + +await crawler.run(['https://crawlee.dev']); + +// The setup file flushes the telemetry on exit. +console.log('Crawl complete. View traces at http://localhost:16686'); diff --git a/docs/guides/trace_and_monitor_custom.ts b/docs/guides/trace_and_monitor_custom.ts new file mode 100644 index 000000000000..4b51f2245f40 --- /dev/null +++ b/docs/guides/trace_and_monitor_custom.ts @@ -0,0 +1,78 @@ +import { CrawleeInstrumentation } from '@crawlee/otel'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { ATTR_HTTP_REQUEST_METHOD, ATTR_SERVICE_NAME, ATTR_URL_FULL } from '@opentelemetry/semantic-conventions'; + +const crawleeInstrumentation = new CrawleeInstrumentation({ + // Disable default request handling instrumentation + requestHandlingInstrumentation: false, + // Disable log forwarding to OpenTelemetry + logInstrumentation: false, + // Define custom methods to instrument + customInstrumentation: [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'crawler.run', + spanOptions() { + return { + attributes: { + 'crawler.type': this.constructor.name, + }, + }; + }, + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'runRequestHandler', + // Dynamic span name using the context argument + spanName(context: any) { + return `request ${context.request.url}`; + }, + spanOptions(context: any) { + return { + attributes: { + [ATTR_URL_FULL]: context.request.url, + [ATTR_HTTP_REQUEST_METHOD]: context.request.method, + }, + }; + }, + }, + ], +}); + +const resource = resourceFromAttributes({ + [ATTR_SERVICE_NAME]: 'custom-instrumented-crawler', +}); + +const traceExporter = new OTLPTraceExporter({ + url: 'http://localhost:4317', +}); + +export const sdk = new NodeSDK({ + resource, + spanProcessors: [new BatchSpanProcessor(traceExporter)], + instrumentations: [crawleeInstrumentation], +}); + +sdk.start(); + +// Like the setup file above, this one is preloaded before the crawler, so it also owns flushing the buffered +// telemetry on the way out - without this the batched spans are dropped when the process exits. +const shutdown = async () => { + await sdk.shutdown(); +}; + +// `beforeExit` covers a script that simply runs to completion... +process.once('beforeExit', () => { + void shutdown(); +}); + +// ...while signals have to be handled separately, as they do not emit `beforeExit`. +process.once('SIGTERM', () => { + void shutdown().then(() => process.exit(0)); +}); diff --git a/docs/guides/trace_and_monitor_register_hook.ts b/docs/guides/trace_and_monitor_register_hook.ts new file mode 100644 index 000000000000..57cdbd493a44 --- /dev/null +++ b/docs/guides/trace_and_monitor_register_hook.ts @@ -0,0 +1,6 @@ +import { register } from 'node:module'; +import { pathToFileURL } from 'node:url'; + +// Installs the OpenTelemetry module hook, which the automatic instrumentation needs in order to patch the Crawlee +// classes as they are imported. This file must be preloaded before the OpenTelemetry setup and before the crawler. +register('@opentelemetry/instrumentation/hook.mjs', pathToFileURL('./')); diff --git a/docs/guides/trace_and_monitor_setup.ts b/docs/guides/trace_and_monitor_setup.ts new file mode 100644 index 000000000000..09159363b8cc --- /dev/null +++ b/docs/guides/trace_and_monitor_setup.ts @@ -0,0 +1,59 @@ +import { CrawleeInstrumentation } from '@crawlee/otel'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; + +// Create a resource that identifies your service +const resource = resourceFromAttributes({ + [ATTR_SERVICE_NAME]: 'my-crawler', + [ATTR_SERVICE_VERSION]: '1.0.0', + 'deployment.environment': 'development', +}); + +// Configure exporters to send data to Jaeger via OTLP +// The gRPC exporter takes the collector endpoint without a signal path - unlike the HTTP one, +// which would use `http://localhost:4318/v1/traces`. +const traceExporter = new OTLPTraceExporter({ + url: 'http://localhost:4317', +}); + +// Create the Crawlee instrumentation +const crawleeInstrumentation = new CrawleeInstrumentation(); + +// Initialize the OpenTelemetry SDK +export const sdk = new NodeSDK({ + resource, + spanProcessors: [new BatchSpanProcessor(traceExporter)], + instrumentations: [crawleeInstrumentation], +}); + +// Start the SDK +sdk.start(); + +console.log('OpenTelemetry initialized'); + +// This file is preloaded before the crawler, so it also owns flushing the buffered telemetry on the way out. +let shuttingDown: Promise | undefined; + +const shutdown = () => { + // Every handler below can fire, and the SDK must only be shut down once. + shuttingDown ??= sdk.shutdown(); + return shuttingDown; +}; + +// `beforeExit` covers a script that simply runs to completion. The flush it starts is async work, so Node keeps the +// process alive for it and then fires `beforeExit` once more - hence `on` rather than `once`, and hence `shutdown` +// having to be idempotent. +process.on('beforeExit', () => { + void shutdown(); +}); + +// Signals have to be handled separately, as they do not emit `beforeExit`. `SIGINT` is the one you send by pressing +// Ctrl-C, so without it a local run loses whatever the exporter had not sent yet. +for (const signal of ['SIGINT', 'SIGTERM'] as const) { + process.once(signal, () => { + void shutdown().then(() => process.exit(0)); + }); +} diff --git a/docs/guides/trace_and_monitor_wrap_with_span.ts b/docs/guides/trace_and_monitor_wrap_with_span.ts new file mode 100644 index 000000000000..ffe342a5f845 --- /dev/null +++ b/docs/guides/trace_and_monitor_wrap_with_span.ts @@ -0,0 +1,87 @@ +import { wrapWithSpan } from '@crawlee/otel'; +import { context, trace } from '@opentelemetry/api'; +import { ATTR_EXCEPTION_MESSAGE, ATTR_HTTP_REQUEST_METHOD, ATTR_URL_FULL } from '@opentelemetry/semantic-conventions'; +import type { CheerioCrawlingContext, CrawlingContext } from 'crawlee'; +import { CheerioCrawler } from 'crawlee'; + +const crawler = new CheerioCrawler({ + maxRequestsPerCrawl: 10, + + // Wrap the request handler with a custom span + requestHandler: wrapWithSpan( + async ({ request, $, enqueueLinks, log }: CheerioCrawlingContext) => { + // Access the current span to add custom attributes + const span = trace.getSpan(context.active()); + + const title = $('title').text(); + const headings = $('h1, h2').length; + const links = $('a').length; + + if (span) { + span.setAttribute('page.title', title); + span.setAttribute('page.headings_count', headings); + span.setAttribute('page.links_count', links); + } + + log.info(`Scraped page`, { url: request.url, title }); + + await enqueueLinks({ + include: ['https://crawlee.dev/**'], + }); + }, + { + // Dynamic span name based on the request + spanName: ({ request }: CheerioCrawlingContext) => `scrape ${request.url}`, + // Add attributes to the span + spanOptions: ({ request }: CheerioCrawlingContext) => ({ + attributes: { + [ATTR_URL_FULL]: request.url, + [ATTR_HTTP_REQUEST_METHOD]: request.method, + }, + }), + }, + ), + + // Wrap hooks with spans + preNavigationHooks: [ + wrapWithSpan( + ({ log }: CheerioCrawlingContext) => { + log.debug('Pre-navigation hook executed'); + }, + { + spanName: 'pre-navigation-hook', + }, + ), + ], + + // Wrap error handlers + errorHandler: wrapWithSpan( + ({ request, log }: CrawlingContext, error: Error) => { + log.error(`Request failed: ${request.url}`, { + error: error.message, + }); + }, + { + spanName: ({ request }: CrawlingContext) => `error-handler ${request.url}`, + spanOptions: ({ request }: CrawlingContext, error: Error) => ({ + attributes: { + [ATTR_URL_FULL]: request.url, + [ATTR_EXCEPTION_MESSAGE]: error.message, + }, + }), + }, + ), + + failedRequestHandler: wrapWithSpan( + ({ request, log }: CrawlingContext, error: Error) => { + log.error(`Request permanently failed: ${request.url}`, { + error: error.message, + }); + }, + { + spanName: 'failed-request-handler', + }, + ), +}); + +await crawler.run(['https://crawlee.dev']); diff --git a/docs/package.json b/docs/package.json index 5e9c57042805..f028fef954e5 100644 --- a/docs/package.json +++ b/docs/package.json @@ -14,7 +14,14 @@ "@crawlee/got-scraping-client": "workspace:*", "@crawlee/http-client": "workspace:*", "@crawlee/impit-client": "workspace:*", + "@crawlee/otel": "workspace:*", "@crawlee/stagehand": "workspace:*", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.210.0", + "@opentelemetry/resources": "^2.0.0", + "@opentelemetry/sdk-node": "^0.210.0", + "@opentelemetry/sdk-trace-base": "^2.0.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "apify": "*", "crawlee": "workspace:*", "impit": "^0.14.2", diff --git a/docs/public-api/crawlee-otel.api.md b/docs/public-api/crawlee-otel.api.md new file mode 100644 index 000000000000..08f1e1296b77 --- /dev/null +++ b/docs/public-api/crawlee-otel.api.md @@ -0,0 +1,50 @@ +## Public API Report File for "@crawlee/otel" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { InstrumentationBase } from '@opentelemetry/instrumentation'; +import type { InstrumentationConfig } from '@opentelemetry/instrumentation'; +import type { InstrumentationModuleDefinition } from '@opentelemetry/instrumentation'; +import type { SpanOptions } from '@opentelemetry/api'; +import type { Tracer } from '@opentelemetry/api'; +import type { TracerProvider } from '@opentelemetry/api'; + +// @public +export interface ClassMethodToInstrument { + className: string; + methodName: string; + moduleName: string; + spanName?: string | ((this: any, ...args: any[]) => string); + spanOptions?: SpanOptions | ((this: any, ...args: any[]) => SpanOptions); +} + +// @public (undocumented) +export class CrawleeInstrumentation extends InstrumentationBase { + constructor(config?: CrawleeInstrumentationConfig); + // (undocumented) + protected init(): InstrumentationModuleDefinition[]; + setTracerProvider(tracerProvider: TracerProvider): void; +} + +// @public (undocumented) +export interface CrawleeInstrumentationConfig extends InstrumentationConfig { + // (undocumented) + customInstrumentation?: ClassMethodToInstrument[]; + // (undocumented) + logInstrumentation?: boolean; + // (undocumented) + requestHandlingInstrumentation?: boolean; +} + +// @public +export function wrapWithSpan(fn: (...args: Args) => Return, options?: { + spanName?: string | ((...args: Args) => string); + spanOptions?: SpanOptions | ((...args: Args) => SpanOptions); + tracer?: Tracer; +}): (...args: Args) => Return; + +// (No @packageDocumentation comment for this package) + +``` diff --git a/package.json b/package.json index 9f26609e17f9..2023d31f0fef 100644 --- a/package.json +++ b/package.json @@ -69,11 +69,20 @@ "@crawlee/impit-client": "workspace:*", "@crawlee/jsdom": "workspace:*", "@crawlee/linkedom": "workspace:*", + "@crawlee/otel": "workspace:*", "@crawlee/playwright": "workspace:*", "@crawlee/puppeteer": "workspace:*", "@crawlee/stagehand": "workspace:*", "@crawlee/types": "workspace:*", "@crawlee/utils": "workspace:*", + "@microsoft/api-extractor": "^7.58.9", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/api-logs": "^0.210.0", + "@opentelemetry/instrumentation": "^0.210.0", + "@opentelemetry/sdk-logs": "^0.210.0", + "@opentelemetry/sdk-trace-base": "^2.0.0", + "@opentelemetry/sdk-trace-node": "^2.0.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "@oxlint/plugins": "^1.62.0", "@playwright/browser-chromium": "1.61.1", "@playwright/browser-firefox": "1.61.1", diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 618f41e302c2..d6774f247b71 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -59,7 +59,6 @@ import { getObjectType, KeyValueStore, log, - LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, @@ -1622,7 +1621,11 @@ export class BasicCrawler< setStatusMessage(message: string, options: SetStatusMessageOptions = {}) { const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined; - this.log.logWithLevel(LogLevel[(options.level as 'DEBUG') ?? 'DEBUG'], message, data); + // Each allowed level has its own method on the logger, so this goes through them rather than through + // `logWithLevel`, which is abstract and therefore cannot be instrumented. + this.log[ + ({ DEBUG: 'debug', INFO: 'info', WARNING: 'warning', ERROR: 'error' } as const)[options.level ?? 'DEBUG'] + ](message, data); // Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend. diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts index 20033b4eae21..9b075fa54a99 100644 --- a/packages/core/src/storages/key_value_store.ts +++ b/packages/core/src/storages/key_value_store.ts @@ -779,7 +779,9 @@ export class KeyValueStore { /** * Returns a file URL for the given key. * - * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`. + * The URL is derived from the key, so it is also returned for a record that does not exist (yet) — + * including one written earlier in an uncommitted storage transaction. Returns `undefined` only when + * the storage has no file URLs at all (e.g. the in-memory storage). * * @param key The key of the record to generate the public URL for. */ diff --git a/packages/fs-storage/package.json b/packages/fs-storage/package.json index 233207eaf685..35414342ff91 100644 --- a/packages/fs-storage/package.json +++ b/packages/fs-storage/package.json @@ -42,7 +42,7 @@ "access": "public" }, "dependencies": { - "@crawlee/fs-storage-native": "0.2.0", + "@crawlee/fs-storage-native": ">=0.2.1-beta.0 <0.3", "@crawlee/types": "workspace:*", "@crawlee/utils": "workspace:*", "zod": "catalog:" diff --git a/packages/fs-storage/src/resource-clients/key-value-store.ts b/packages/fs-storage/src/resource-clients/key-value-store.ts index c129c46750c5..dc17c767e35d 100644 --- a/packages/fs-storage/src/resource-clients/key-value-store.ts +++ b/packages/fs-storage/src/resource-clients/key-value-store.ts @@ -181,20 +181,17 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV /** * Generates a public `file://` URL for accessing a specific record in the key-value store. * - * Returns `undefined` if the record does not exist. + * The native `getPublicUrl` derives the URL from the key without probing bare-file extensions, so + * an `INPUT` that lives on disk as a hand-placed `INPUT.json` is resolved first. Nothing on disk + * means nothing to resolve — the requested key is used as-is, and the URL is the one the record will + * have once written. * @param key The key of the record to generate the public URL for. */ async getPublicUrl(key: string): Promise { parseArgument(key, keySchema); - // The native `getPublicUrl` stats the encoded path but does not probe bare-file extensions, - // so we resolve the on-disk key first (handling e.g. `INPUT` -> `INPUT.json`) and normalize - // the native `null`-for-missing result to the historical `undefined` contract. - const resolvedKey = await this.resolveExistingKey(key); - if (resolvedKey === undefined) { - return undefined; - } - return (await this.#nativeBackend.getPublicUrl(resolvedKey)) ?? undefined; + const resolvedKey = (await this.resolveExistingKey(key)) ?? key; + return this.#nativeBackend.getPublicUrl(resolvedKey); } /** diff --git a/packages/fs-storage/test/fs-fallback.test.ts b/packages/fs-storage/test/fs-fallback.test.ts index a3c2f743633b..4406027f8949 100644 --- a/packages/fs-storage/test/fs-fallback.test.ts +++ b/packages/fs-storage/test/fs-fallback.test.ts @@ -176,10 +176,11 @@ describe('fallback to fs for reading', () => { // `some-key.json` sits on disk with no metadata sidecar. Only `INPUT` keys probe bare files, // so this is invisible to every read path: it has no tracked record, and the `.json` extension - // probing that would resolve a bare `INPUT` is never attempted for other keys. + // probing that would resolve a bare `INPUT` is never attempted for other keys. `getPublicUrl` + // is existence-agnostic, so it still answers — with the extensionless key, not the bare file. expect(await nonInputStore.getValue('some-key')).toBeUndefined(); expect(await nonInputStore.recordExists('some-key')).toBe(false); - expect(await nonInputStore.getPublicUrl('some-key')).toBeUndefined(); + expect(await nonInputStore.getPublicUrl('some-key')).toMatch(/^file:\/\/.*\/some-key$/); // `listKeys` only surfaces bare files for the run-input keys, so `some-key` is not enumerated. const { items } = await nonInputStore.listKeys(); @@ -263,7 +264,9 @@ describe('run-input bare-file reachability (one variant per store)', () => { expect(await store.getValue(key)).toBeUndefined(); expect(await store.recordExists(key)).toBe(false); - expect(await store.getPublicUrl(key)).toBeUndefined(); + // Existence-agnostic: the URL falls back to the requested key rather than resolving to a + // sibling variant's file. + expect(await store.getPublicUrl(key)).toMatch(new RegExp(`^file://.*/${key.replace('.', '\\.')}$`)); }); }); }); diff --git a/packages/otel/.npmignore b/packages/otel/.npmignore new file mode 100644 index 000000000000..d18b05a66a6a --- /dev/null +++ b/packages/otel/.npmignore @@ -0,0 +1,5 @@ +node_modules +src +test +coverage +tsconfig.* diff --git a/packages/otel/CHANGELOG.md b/packages/otel/CHANGELOG.md new file mode 100644 index 000000000000..e9fb6ecf5930 --- /dev/null +++ b/packages/otel/CHANGELOG.md @@ -0,0 +1,4 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. \ No newline at end of file diff --git a/packages/otel/README.md b/packages/otel/README.md new file mode 100644 index 000000000000..09e3bd5eb353 --- /dev/null +++ b/packages/otel/README.md @@ -0,0 +1,81 @@ +# @crawlee/otel + +This package provides [OpenTelemetry](https://opentelemetry.io/) instrumentation for Crawlee. It traces the request +handling pipeline of the crawlers and forwards Crawlee logs to OpenTelemetry, so you can analyze crawler runs in any +OpenTelemetry-compatible backend (Jaeger, Zipkin, Signoz, ...). + +## Installation + +The OpenTelemetry API packages are peer dependencies, so install them alongside this package: + +```bash +npm install @crawlee/otel @opentelemetry/api @opentelemetry/api-logs @opentelemetry/sdk-node +``` + +## Example usage + +Crawlee is published as ECMAScript modules, so the crawler classes can only be patched through Node's module hook. +Register it in its own file, preloaded ahead of everything else: + +```typescript +// register-hook.ts +import { register } from 'node:module'; +import { pathToFileURL } from 'node:url'; + +register('@opentelemetry/instrumentation/hook.mjs', pathToFileURL('./')); +``` + +Then configure the SDK in a setup file, which also has to load before Crawlee: + +```typescript +// setup.ts +import { CrawleeInstrumentation } from '@crawlee/otel'; +import { NodeSDK } from '@opentelemetry/sdk-node'; + +export const sdk = new NodeSDK({ + instrumentations: [new CrawleeInstrumentation()], + // ... exporters and resource configuration +}); + +sdk.start(); +``` + +```bash +node --import ./register-hook.js --import ./setup.js ./main.js +``` + +Without the hook the crawler runs normally but no spans are produced. Register it with `register()` from a preloaded +file as shown - `--experimental-loader=@opentelemetry/instrumentation/hook.mjs` patches the classes but the spans do +not come out the other end. + +Spans for `BasicCrawler`, `HttpCrawler` and `BrowserCrawler` request handling are then created automatically. + +## Instrumenting your own handlers + +Use `wrapWithSpan` to put your own code on the trace. The span name and attributes can be derived from the arguments +the wrapped function receives: + +```typescript +import { CheerioCrawler } from '@crawlee/cheerio'; +import { wrapWithSpan } from '@crawlee/otel'; +import { context, trace } from '@opentelemetry/api'; + +const crawler = new CheerioCrawler({ + requestHandler: wrapWithSpan( + async ({ request, $ }) => { + trace.getSpan(context.active())?.setAttribute('page.title', $('title').text()); + }, + { spanName: ({ request }) => `scrape ${request.url}` }, + ), +}); +``` + +## Configuration + +| Option | Default | Description | +| --- | --- | --- | +| `requestHandlingInstrumentation` | `true` | Instrument the core request handling methods of the crawlers. | +| `logInstrumentation` | `true` | Forward Crawlee logs to OpenTelemetry logs. | +| `customInstrumentation` | `[]` | Additional `@crawlee/*` class methods to instrument. | + +> This package is part of the [Crawlee](https://crawlee.dev) monorepo. diff --git a/packages/otel/package.json b/packages/otel/package.json new file mode 100644 index 000000000000..2cf08a344e67 --- /dev/null +++ b/packages/otel/package.json @@ -0,0 +1,56 @@ +{ + "name": "@crawlee/otel", + "version": "4.0.0", + "description": "OpenTelemetry instrumentation for Crawlee", + "engines": { + "node": ">=22.0.0" + }, + "type": "module", + "exports": { + ".": "./dist/index.js", + "./package.json": "./package.json" + }, + "keywords": [ + "apify", + "api", + "otel", + "opentelemetry" + ], + "author": { + "name": "Apify", + "email": "support@apify.com", + "url": "https://apify.com" + }, + "contributors": [], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/apify/crawlee" + }, + "bugs": { + "url": "https://github.com/apify/crawlee/issues" + }, + "homepage": "https://crawlee.dev", + "scripts": { + "build": "pnpm clean && pnpm compile && pnpm copy", + "clean": "rimraf ./dist", + "compile": "tsc -p tsconfig.build.json", + "copy": "tsx ../../scripts/copy.ts" + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@opentelemetry/instrumentation": "^0.210.0", + "@opentelemetry/semantic-conventions": "^1.39.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0", + "@opentelemetry/api-logs": "^0.210.0" + }, + "devDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/api-logs": "^0.210.0", + "semver": "^7.7.0" + } +} diff --git a/packages/otel/src/constants.ts b/packages/otel/src/constants.ts new file mode 100644 index 000000000000..fa2dca65a703 --- /dev/null +++ b/packages/otel/src/constants.ts @@ -0,0 +1,214 @@ +import type { Attributes } from '@opentelemetry/api'; +import { SpanKind } from '@opentelemetry/api'; +import { SeverityNumber } from '@opentelemetry/api-logs'; +import { ATTR_HTTP_REQUEST_METHOD, ATTR_URL_FULL } from '@opentelemetry/semantic-conventions'; + +import type { CrawlingContextLike, LoggerMethodDefinition } from './internal-types.js'; +import type { ClassMethodToInstrument, CrawleeInstrumentationConfig } from './types.js'; + +export const PACKAGE_NAME = '@crawlee/otel'; + +/** Used when the package version cannot be determined at runtime. */ +export const UNKNOWN_PACKAGE_VERSION = '0.0.0'; + +/** + * Versions of the instrumented Crawlee packages this instrumentation knows how to patch. Methods that are missing + * in the resolved version are skipped with a warning instead of breaking the module load. + * + * Spelled out rather than written as `^4.0.0`, because a caret range never matches a prerelease and every published + * Crawlee v4 is one (`4.0.0-beta.x`, `4.0.0-rc.x`). The explicit `-0` bounds admit prereleases of 4.0.0 itself, and + * `includePrerelease` on the module definitions extends that to prereleases of later v4 minors. + * + * `@opentelemetry/instrumentation` only consults this when the loader hands it the module's base directory, which the + * ESM hook does not do - so today a wrong range here would not stop anything from being patched. It is still declared + * correctly rather than left as documentation: the range is what decides whether a module is patched as soon as a base + * directory is available, and being silently skipped is not a failure mode worth leaving armed. + */ +export const SUPPORTED_CRAWLEE_VERSIONS = ['>=4.0.0-0 <5.0.0-0']; + +export const baseConfig: CrawleeInstrumentationConfig = { + enabled: true, + requestHandlingInstrumentation: true, + logInstrumentation: true, + customInstrumentation: [], +} as const; + +/** + * Extracts span attributes from a Crawlee crawling context. Uses the stable OpenTelemetry semantic conventions where + * they exist, so that traces stay comparable with other instrumented HTTP clients. + */ +function requestAttributes(crawlingContext: CrawlingContextLike | undefined): Attributes { + const request = crawlingContext?.request; + + // The context shape depends on the installed Crawlee version, so never assume the request is there. + if (!request) { + return {}; + } + + return { + 'crawlee.request.id': request.id, + [ATTR_URL_FULL]: request.url, + [ATTR_HTTP_REQUEST_METHOD]: request.method, + 'crawlee.request.retry_count': request.retryCount, + }; +} + +export const requestHandlingInstrumentationMethods: ClassMethodToInstrument[] = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'crawlee.crawler.run', + spanOptions() { + return { + attributes: { + 'crawlee.crawler.type': this.constructor.name, + }, + }; + }, + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'handleRequest', + spanName: 'crawlee.crawler.handleRequest', + // `handleRequest(crawlingContext, requestSource, request)` + spanOptions(crawlingContext: CrawlingContextLike) { + return { attributes: requestAttributes(crawlingContext) }; + }, + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'runRequestHandler', + spanName: 'crawlee.crawler.runRequestHandler', + spanOptions(crawlingContext: CrawlingContextLike) { + return { attributes: requestAttributes(crawlingContext) }; + }, + }, + { + // `AdaptivePlaywrightCrawler` replaces `runRequestHandler` outright instead of calling `super`, so the + // `BasicCrawler` patch above never fires for it. `BrowserCrawler` does call `super`, which is why it needs + // no entry of its own. + moduleName: '@crawlee/playwright', + className: 'AdaptivePlaywrightCrawler', + methodName: 'runRequestHandler', + spanName: 'crawlee.crawler.runRequestHandler', + spanOptions(crawlingContext: CrawlingContextLike) { + return { attributes: requestAttributes(crawlingContext) }; + }, + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'requestFunctionErrorHandler', + spanName: 'crawlee.crawler.requestFunctionErrorHandler', + // `requestFunctionErrorHandler(error, crawlingContext, request, source)` + spanOptions(_error: Error, crawlingContext: CrawlingContextLike) { + return { attributes: requestAttributes(crawlingContext) }; + }, + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'handleFailedRequestHandler', + spanName: 'crawlee.crawler.handleFailedRequestHandler', + spanOptions(crawlingContext: CrawlingContextLike) { + return { attributes: requestAttributes(crawlingContext) }; + }, + }, + { + moduleName: '@crawlee/http', + className: 'HttpCrawler', + methodName: 'makeHttpRequest', + spanName: 'crawlee.http.makeHttpRequest', + spanOptions(crawlingContext: CrawlingContextLike) { + // An outbound HTTP call, so a client span rather than the default internal one. + return { kind: SpanKind.CLIENT, attributes: requestAttributes(crawlingContext) }; + }, + }, + { + moduleName: '@crawlee/browser', + className: 'BrowserCrawler', + methodName: 'navigate', + spanName: 'crawlee.browser.navigate', + spanOptions(crawlingContext: CrawlingContextLike) { + // The browser navigation is the outbound call here. + return { kind: SpanKind.CLIENT, attributes: requestAttributes(crawlingContext) }; + }, + }, +] as const; + +/** + * Maps Apify log levels to OpenTelemetry severity numbers. + * See https://github.com/apify/apify-shared-js/blob/83d46cf72a338ff671f89dcbc2b0db7dd571e29f/packages/log/src/log_consts.ts#L1 + * + * ```typescript + * export enum LogLevel { + * // Turns off logging completely + * OFF = 0, + * // For unexpected errors in Apify system + * ERROR = 1 = SeverityNumber.ERROR, + * // For situations where error is caused by user (e.g. Meteor.Error), i.e. when the error is not + * // caused by Apify system, avoid the word "ERROR" to simplify searching in log + * SOFT_FAIL = 2 = SeverityNumber.WARN, + * WARNING = 3 = SeverityNumber.WARN, + * INFO = 4 = SeverityNumber.INFO, + * DEBUG = 5 = SeverityNumber.DEBUG, + * // for performance stats + * PERF = 6 = SeverityNumber.DEBUG, + * } + * ``` + */ +export const apifyLogLevelMap: Record = { + 1: SeverityNumber.ERROR, + 2: SeverityNumber.WARN, + 3: SeverityNumber.WARN, + 4: SeverityNumber.INFO, + 5: SeverityNumber.DEBUG, + 6: SeverityNumber.DEBUG, +} as const; + +/** Human readable Apify log level names, emitted as the `severityText` of the forwarded log records. */ +export const apifyLogLevelNameMap: Record = { + 1: 'ERROR', + 2: 'SOFT_FAIL', + 3: 'WARNING', + 4: 'INFO', + 5: 'DEBUG', + 6: 'PERF', +} as const; + +/** + * The logging methods to instrument, with the level each logs at. + * + * These live on `BaseCrawleeLogger.prototype`, which every Crawlee logger derives from, so patching them forwards logs + * from any logger implementation - the default Apify one as well as a Winston, Pino or hand-written adapter. Each of + * them dispatches straight to the abstract `logWithLevel`, which the adapter implements, so a call is only seen once. + * `warningOnce` and `deprecated` are covered through `warning`. + */ +export const loggerMethods: LoggerMethodDefinition[] = [ + { methodName: 'error', level: 1, read: (args) => readMessageAndData(args) }, + { + methodName: 'exception', + level: 1, + // `exception(exception, message, data)` - the error comes first. + read: (args) => ({ + message: String(args[1] ?? ''), + data: { ...(args[2] as Record | undefined), exception: args[0] }, + }), + }, + { methodName: 'softFail', level: 2, read: (args) => readMessageAndData(args) }, + { methodName: 'warning', level: 3, read: (args) => readMessageAndData(args) }, + { methodName: 'info', level: 4, read: (args) => readMessageAndData(args) }, + { methodName: 'debug', level: 5, read: (args) => readMessageAndData(args) }, + { methodName: 'perf', level: 6, read: (args) => readMessageAndData(args) }, +]; + +/** The shape shared by every logging method except `exception`: `(message, data?)`. */ +function readMessageAndData(args: unknown[]) { + return { + message: String(args[0] ?? ''), + data: args[1] as Record | undefined, + }; +} diff --git a/packages/otel/src/index.ts b/packages/otel/src/index.ts new file mode 100644 index 000000000000..5b04f3c8abcb --- /dev/null +++ b/packages/otel/src/index.ts @@ -0,0 +1,3 @@ +export { CrawleeInstrumentation } from './instrumentation.js'; +export type * from './types.js'; +export { wrapWithSpan } from './wrapWithSpan.js'; diff --git a/packages/otel/src/instrumentation.ts b/packages/otel/src/instrumentation.ts new file mode 100644 index 000000000000..12d42548bd44 --- /dev/null +++ b/packages/otel/src/instrumentation.ts @@ -0,0 +1,227 @@ +// oxlint-disable no-underscore-dangle -- `_wrap`, `_unwrap` and `_diag` are inherited from `InstrumentationBase`. +import type { SpanOptions, TracerProvider } from '@opentelemetry/api'; +import { SeverityNumber } from '@opentelemetry/api-logs'; +import type { InstrumentationModuleDefinition } from '@opentelemetry/instrumentation'; +import { InstrumentationBase, InstrumentationNodeModuleDefinition, isWrapped } from '@opentelemetry/instrumentation'; +import { ATTR_CODE_FUNCTION_NAME } from '@opentelemetry/semantic-conventions'; + +import { + apifyLogLevelMap, + apifyLogLevelNameMap, + baseConfig, + loggerMethods, + PACKAGE_NAME, + requestHandlingInstrumentationMethods, + SUPPORTED_CRAWLEE_VERSIONS, +} from './constants.js'; +import type { ClassMethodPatchDefinition, LoggerMethodDefinition, ModuleDefinition } from './internal-types.js'; +import type { CrawleeInstrumentationConfig } from './types.js'; +import { buildLogAttributes, buildModuleDefinitions, getPackageVersion } from './utilities.js'; +import { resolveSpanName, resolveSpanOptions, setSharedTracer, wrapWithSpan } from './wrapWithSpan.js'; + +/** + * Builds a module definition for one of the instrumented Crawlee packages. + * + * `InstrumentationNodeModuleDefinition` does not take `includePrerelease` through its constructor, but + * `InstrumentationBase` reads it off the definition when it decides whether to patch a resolved module version, so it + * is set here. Without it, {@link SUPPORTED_CRAWLEE_VERSIONS} would only cover prereleases of `4.0.0` and every + * `4.x.0-beta` would go uninstrumented. + */ +function crawleeModuleDefinition( + moduleName: string, + patch: (moduleExports: any) => any, + unpatch: (moduleExports: any) => any, +): InstrumentationModuleDefinition { + const definition: InstrumentationModuleDefinition = new InstrumentationNodeModuleDefinition( + moduleName, + SUPPORTED_CRAWLEE_VERSIONS, + patch, + unpatch, + ); + definition.includePrerelease = true; + return definition; +} + +export class CrawleeInstrumentation extends InstrumentationBase { + constructor(config: CrawleeInstrumentationConfig = {}) { + // Each flag is resolved on its own rather than by spreading `config` over `baseConfig`: a spread lets an + // explicit `undefined` - which is what `{ logInstrumentation: options.logs }` produces when `options.logs` + // is not set - overwrite the default with nothing and quietly disable the feature. + super(PACKAGE_NAME, getPackageVersion(), { + ...config, + enabled: config.enabled ?? baseConfig.enabled, + requestHandlingInstrumentation: + config.requestHandlingInstrumentation ?? baseConfig.requestHandlingInstrumentation, + logInstrumentation: config.logInstrumentation ?? baseConfig.logInstrumentation, + customInstrumentation: config.customInstrumentation ?? baseConfig.customInstrumentation, + }); + } + + /** + * Shares the tracer with the exported `wrapWithSpan` helper, so that manually wrapped handlers end up in the same + * instrumentation scope as the automatic spans. + * + * This has to happen here rather than in the constructor. The constructor can only reach the tracer of the global + * API, and that is not necessarily the one this instrumentation ends up using - a `tracerProvider` passed to + * `registerInstrumentations` is never registered globally, and a duplicated `@opentelemetry/api` in the dependency + * tree has its own global. Handing over a tracer from the wrong provider silences every span, including the + * automatic ones, and because it is not `undefined` it also shadows the fallback in `wrapWithSpan`. + */ + public override setTracerProvider(tracerProvider: TracerProvider): void { + super.setTracerProvider(tracerProvider); + setSharedTracer(this.tracer); + } + + protected init(): InstrumentationModuleDefinition[] { + const methodsToInstrument = [...(this.getConfig().customInstrumentation ?? [])]; + if (this.getConfig().requestHandlingInstrumentation) { + methodsToInstrument.push(...requestHandlingInstrumentationMethods); + } + const moduleDefinitions = buildModuleDefinitions(methodsToInstrument); + const definitions = this.instantiateModuleDefinitions(moduleDefinitions); + + if (this.getConfig().logInstrumentation) { + definitions.push( + crawleeModuleDefinition( + '@crawlee/core', + (moduleExports) => { + for (const method of loggerMethods) { + const prototype = this.getPrototype( + moduleExports, + '@crawlee/core', + 'BaseCrawleeLogger', + method.methodName, + ); + if (prototype) { + this._wrap(prototype, method.methodName, this.getLogPatch(method)); + } + } + return moduleExports; + }, + (moduleExports) => { + for (const method of loggerMethods) { + this.unwrapIfWrapped(moduleExports?.BaseCrawleeLogger?.prototype, method.methodName); + } + return moduleExports; + }, + ), + ); + } + return definitions; + } + + private instantiateModuleDefinitions(moduleDefinitions: ModuleDefinition[]): InstrumentationModuleDefinition[] { + return moduleDefinitions.map((definition) => { + return crawleeModuleDefinition( + definition.moduleName, + (moduleExports) => { + for (const patch of definition.classMethodPatches) { + const prototype = this.getPrototype( + moduleExports, + definition.moduleName, + patch.className, + patch.methodName, + ); + if (prototype) { + this._wrap(prototype, patch.methodName, this.applyClassMethodPatch(patch)); + } + } + return moduleExports; + }, + (moduleExports) => { + for (const patch of definition.classMethodPatches) { + this.unwrapIfWrapped(moduleExports?.[patch.className]?.prototype, patch.methodName); + } + return moduleExports; + }, + ); + }); + } + + /** + * Resolves the prototype holding the method to patch, warning instead of throwing when the class or the method + * is not there - a missing internal method must not break loading of the instrumented module. + */ + private getPrototype(moduleExports: any, moduleName: string, className: string, methodName: string) { + const prototype = moduleExports?.[className]?.prototype; + + if (typeof prototype?.[methodName] !== 'function') { + this._diag.warn( + `Skipping instrumentation of ${moduleName}: ${className}.${methodName} was not found. ` + + `The installed version of ${moduleName} is probably not supported by ${PACKAGE_NAME}.`, + ); + return undefined; + } + + return prototype; + } + + /** Mirrors {@link getPrototype}: only methods that were actually patched are restored. */ + private unwrapIfWrapped(prototype: any, methodName: string) { + if (prototype && isWrapped(prototype[methodName])) { + this._unwrap(prototype, methodName); + } + } + + private applyClassMethodPatch(patch: ClassMethodPatchDefinition): (original: any) => any { + const { spanName, spanOptions } = patch; + const qualifiedName = `${patch.className}.${patch.methodName}`; + const codeAttributes = { [ATTR_CODE_FUNCTION_NAME]: qualifiedName }; + + // Both options are resolved through the guarded helpers, so that a throwing callback costs only what the + // caller supplied - the span itself, its name and the attributes added here all survive. + return function wrap(original: (...args: unknown[]) => any) { + return wrapWithSpan(original, { + spanName(this: unknown, ...args: unknown[]): string { + return resolveSpanName(spanName, qualifiedName, this, args); + }, + spanOptions(this: unknown, ...args: unknown[]): SpanOptions { + const resolved = resolveSpanOptions(spanOptions, this, args); + return { ...resolved, attributes: { ...codeAttributes, ...resolved.attributes } }; + }, + }); + }; + } + + private getLogPatch(method: LoggerMethodDefinition) { + // oxlint-disable-next-line no-this-alias + const instrumentation = this; + + return function wrapLog(original: (...args: any[]) => void) { + return function wrappedLog(this: unknown, ...args: any[]): void { + // The application's own logging runs first and its outcome is never affected by the forwarding, + // which matters most where Crawlee logs from inside a `catch` - `handleFailedRequestHandler`. + // These methods are synchronous and return void, so keep them that way. + try { + original.apply(this, args); + } finally { + instrumentation.forwardLogRecord(method, args); + } + }; + }; + } + + /** + * Emits one Crawlee log call as an OpenTelemetry log record. + * + * Everything here - reading the arguments, stringifying the message, handing the record to the SDK - runs inside + * the application's own call to `log.info()` and friends, so a failure anywhere in it is reported and dropped + * rather than raised. + */ + private forwardLogRecord(method: LoggerMethodDefinition, args: unknown[]): void { + try { + const { message, data } = method.read(args); + + // Crawlee leaves level filtering to the logging library, so everything is forwarded and the + // OpenTelemetry pipeline decides what to keep. + this.logger.emit({ + severityNumber: apifyLogLevelMap[method.level] ?? SeverityNumber.UNSPECIFIED, + severityText: apifyLogLevelNameMap[method.level], + body: message, + attributes: buildLogAttributes(data), + }); + } catch (err) { + this._diag.warn(`Failed to forward a Crawlee log record to OpenTelemetry: ${err}`); + } + } +} diff --git a/packages/otel/src/internal-types.ts b/packages/otel/src/internal-types.ts new file mode 100644 index 000000000000..f47d4538c011 --- /dev/null +++ b/packages/otel/src/internal-types.ts @@ -0,0 +1,41 @@ +import type { ClassMethodToInstrument } from './types.js'; + +/** The patches of one instrumented module, as produced by {@link buildModuleDefinitions}. */ +export interface ModuleDefinition { + moduleName: string; + classMethodPatches: ClassMethodPatchDefinition[]; +} + +/** + * A {@link ClassMethodToInstrument} that has already been grouped under its module, so the module name would be + * redundant. Derived from the public type rather than restated, so the fields and their documentation have one home. + */ +export type ClassMethodPatchDefinition = Omit; + +/** + * The fields of a Crawlee `Request` this instrumentation reads. + * + * Deliberately structural rather than imported from `@crawlee/core`: the instrumented Crawlee version is resolved at + * runtime and may not be the one this package was compiled against, so every field is treated as optional. + */ +export interface RequestLike { + id?: string; + url?: string; + method?: string; + retryCount?: number; +} + +/** The part of a Crawlee crawling context this instrumentation reads. */ +export interface CrawlingContextLike { + request?: RequestLike; +} + +/** One of the logging methods `BaseCrawleeLogger` provides, and how to read a log record out of a call to it. */ +export interface LoggerMethodDefinition { + /** The method on `BaseCrawleeLogger.prototype` to patch. */ + methodName: string; + /** The Crawlee log level this method logs at. */ + level: number; + /** Pulls the message and the structured data out of the call arguments, which differ per method. */ + read: (args: unknown[]) => { message: string; data?: Record }; +} diff --git a/packages/otel/src/types.ts b/packages/otel/src/types.ts new file mode 100644 index 000000000000..ec645ebb558e --- /dev/null +++ b/packages/otel/src/types.ts @@ -0,0 +1,27 @@ +import type { SpanOptions } from '@opentelemetry/api'; +import type { InstrumentationConfig } from '@opentelemetry/instrumentation'; + +export interface CrawleeInstrumentationConfig extends InstrumentationConfig { + requestHandlingInstrumentation?: boolean; + logInstrumentation?: boolean; + customInstrumentation?: ClassMethodToInstrument[]; +} + +/** One class method to wrap in a span, and the module that has to be loaded for it to exist. */ +export interface ClassMethodToInstrument { + /** The Crawlee package the class is exported from, for example `@crawlee/basic`. */ + moduleName: string; + /** The class name to patch. */ + className: string; + /** The method name to patch. */ + methodName: string; + /** + * The name of the span. Defaults to `className.methodName`. + * + * When given a function, it is called with the arguments of the patched method, and `this` is the instance the + * method was called on. The arguments are `any` because they are whatever the patched method receives. + */ + spanName?: string | ((this: any, ...args: any[]) => string); + /** The attributes of the span. Follows the same calling convention as {@link ClassMethodToInstrument.spanName}. */ + spanOptions?: SpanOptions | ((this: any, ...args: any[]) => SpanOptions); +} diff --git a/packages/otel/src/utilities.ts b/packages/otel/src/utilities.ts new file mode 100644 index 000000000000..58a9197a7fba --- /dev/null +++ b/packages/otel/src/utilities.ts @@ -0,0 +1,113 @@ +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; + +import { diag } from '@opentelemetry/api'; +import type { LogAttributes } from '@opentelemetry/api-logs'; +import { + ATTR_EXCEPTION_MESSAGE, + ATTR_EXCEPTION_STACKTRACE, + ATTR_EXCEPTION_TYPE, +} from '@opentelemetry/semantic-conventions'; + +import { PACKAGE_NAME, UNKNOWN_PACKAGE_VERSION } from './constants.js'; +import type { ModuleDefinition } from './internal-types.js'; +import type { ClassMethodToInstrument } from './types.js'; + +interface OtelPackageJson { + version: string; +} + +let packageFile: OtelPackageJson | undefined; + +function getPackageJson(): OtelPackageJson { + if (!packageFile) { + try { + // The package is ESM, so `require` has to be recreated from the module URL. + const packageFilePath = createRequire(import.meta.url).resolve(`${PACKAGE_NAME}/package.json`); + packageFile = JSON.parse(readFileSync(packageFilePath, 'utf8')) as OtelPackageJson; + } catch (err) { + // The version is only reported as the instrumentation scope version, so a failure here must not be fatal. + diag.warn(`Could not determine the ${PACKAGE_NAME} version: ${err}`); + packageFile = { version: UNKNOWN_PACKAGE_VERSION }; + } + } + return packageFile; +} + +export function getPackageVersion(): string { + return getPackageJson().version; +} + +/** + * Turns the structured `data` of a Crawlee log call into OpenTelemetry log attributes. + * + * Crawlee folds a logged error into `data`, so any `Error` value found there is mapped onto the semantic convention + * attributes - its own properties are not enumerable and would otherwise be dropped. + */ +export function buildLogAttributes(data?: unknown): LogAttributes { + const attributes: LogAttributes = {}; + + if (typeof data !== 'object' || data === null || Array.isArray(data)) { + if (data !== undefined) { + attributes['crawlee.log.data'] = String(data); + } + return attributes; + } + + for (const [key, value] of Object.entries(data)) { + if (value instanceof Error) { + attributes[ATTR_EXCEPTION_TYPE] = value.name; + attributes[ATTR_EXCEPTION_MESSAGE] = value.message; + if (value.stack) { + attributes[ATTR_EXCEPTION_STACKTRACE] = value.stack; + } + } else { + attributes[key] = value as LogAttributes[string]; + } + } + + return attributes; +} + +/** + * Groups the methods to instrument by the module that owns them, dropping duplicates. + * + * The first definition of a method wins. `CrawleeInstrumentation` passes `customInstrumentation` ahead of the built-in + * list, so configuring a method that is already instrumented overrides the built-in span rather than being ignored. + */ +export function buildModuleDefinitions(methodsToInstrument: ClassMethodToInstrument[]): ModuleDefinition[] { + const definitions: ModuleDefinition[] = []; + + for (const method of methodsToInstrument) { + let definition = definitions.find((d) => d.moduleName === method.moduleName); + if (!definition) { + if (!method.moduleName.startsWith('@crawlee/')) { + diag.warn(`Module ${method.moduleName} is not a valid Crawlee module. Skipping.`); + continue; + } + definition = { + moduleName: method.moduleName, + classMethodPatches: [], + }; + definitions.push(definition); + } + if ( + !definition.classMethodPatches.find( + (p) => p.className === method.className && p.methodName === method.methodName, + ) + ) { + definition.classMethodPatches.push({ + className: method.className, + methodName: method.methodName, + spanName: method.spanName, + spanOptions: method.spanOptions, + }); + } else { + diag.warn( + `Method ${method.className}.${method.methodName} is instrumented more than once. Keeping the first ` + + `definition, which is the \`customInstrumentation\` entry when one exists for the method.`, + ); + } + } + return definitions; +} diff --git a/packages/otel/src/wrapWithSpan.ts b/packages/otel/src/wrapWithSpan.ts new file mode 100644 index 000000000000..c96f7d932401 --- /dev/null +++ b/packages/otel/src/wrapWithSpan.ts @@ -0,0 +1,148 @@ +import type { Exception, Span, SpanOptions, Tracer } from '@opentelemetry/api'; +import { diag, SpanStatusCode, trace } from '@opentelemetry/api'; + +import { PACKAGE_NAME } from './constants.js'; +import { getPackageVersion } from './utilities.js'; + +/** + * The tracer `CrawleeInstrumentation` hands over once it knows which provider it ended up with, so that manually + * wrapped handlers share the instrumentation scope of the automatic spans. + * + * Module level state, because `wrapWithSpan` is called from user code that holds no reference to the instrumentation. + */ +let sharedTracer: Tracer | undefined; + +/** + * Points the exported {@link wrapWithSpan} at a tracer, or back at the global API when passed `undefined`. + * + * @internal + */ +export function setSharedTracer(tracer: Tracer | undefined): void { + sharedTracer = tracer; +} + +/** + * Wraps a function with OpenTelemetry span instrumentation. + * + * `Args` and `Return` are separate generics so that the argument types flow through to the `spanName` and `spanOptions` + * callbacks. They are inferred from the wrapped function, so annotate its parameters when assigning the result to an + * option whose type is a union - `requestHandler` accepts both a router and a plain handler, and TypeScript cannot + * infer parameter types through a union of function types: + * + * ```ts + * requestHandler: wrapWithSpan(async ({ request }: CheerioCrawlingContext) => { ... }) + * ``` + * + * Synchronous functions stay synchronous, asynchronous ones keep their span open until the returned promise settles. + * + * The wrapper forwards its own `this` to the wrapped function, so a method keeps working when wrapped. An arrow + * function still ignores it, as arrow functions always take `this` from where they were defined. + */ +export function wrapWithSpan( + fn: (...args: Args) => Return, + options?: { + spanName?: string | ((...args: Args) => string); + spanOptions?: SpanOptions | ((...args: Args) => SpanOptions); + tracer?: Tracer; + }, +): (...args: Args) => Return { + return function (this: unknown, ...args: Args): Return { + // Resolving the tracer lazily lets `CrawleeInstrumentation` hand over its tracer at any point. When no + // instrumentation is registered, we fall back to the global API, which returns a tracer that either + // delegates to the globally registered provider or is a no-op - so wrapping never fails on its own. + const tracer = options?.tracer ?? sharedTracer ?? trace.getTracer(PACKAGE_NAME, getPackageVersion()); + const spanName = resolveSpanName(options?.spanName, fn.name || 'anonymous', this, args); + const spanOptions = resolveSpanOptions(options?.spanOptions, this, args); + + return tracer.startActiveSpan(spanName, spanOptions, (span): Return => { + let result: Return; + + try { + result = fn.apply(this, args); + } catch (err) { + recordError(span, err); + span.end(); + throw err; + } + + // Only defer ending the span when the wrapped function is actually asynchronous, so that wrapping + // a synchronous function does not silently turn it into a promise-returning one. + if (!isPromiseLike(result)) { + span.end(); + return result; + } + + return Promise.resolve(result).then( + (value) => { + span.end(); + return value; + }, + (err) => { + recordError(span, err); + span.end(); + throw err; + }, + ) as Return; + }); + }; +} + +/** @internal Shared with `CrawleeInstrumentation`, which supplies a class qualified default. */ +export function resolveSpanName( + spanName: string | ((...args: Args) => string) | undefined, + fallback: string, + thisArg: unknown, + args: Args, +): string { + if (typeof spanName === 'function') { + return callSpanOption(spanName, fallback, thisArg, args, 'spanName'); + } + return spanName ?? fallback; +} + +/** @internal Shared with `CrawleeInstrumentation`, which merges its own attributes into the resolved options. */ +export function resolveSpanOptions( + spanOptions: SpanOptions | ((...args: Args) => SpanOptions) | undefined, + thisArg: unknown, + args: Args, +): SpanOptions { + if (typeof spanOptions === 'function') { + return callSpanOption(spanOptions, {}, thisArg, args, 'spanOptions'); + } + return spanOptions ?? {}; +} + +/** + * Calls a `spanName` or `spanOptions` callback supplied by the caller. + * + * These run inside the call path of the wrapped function, so a mistake in one must not turn into a failure of the + * function being instrumented - the span is created with the default instead. + */ +function callSpanOption( + callback: (...args: Args) => Value, + fallback: Value, + thisArg: unknown, + args: Args, + optionName: string, +): Value { + try { + return callback.apply(thisArg, args); + } catch (err) { + diag.warn(`The ${optionName} callback of a ${PACKAGE_NAME} span threw, using the default instead: ${err}`); + return fallback; + } +} + +function isPromiseLike(value: unknown): value is PromiseLike { + return typeof (value as PromiseLike | undefined)?.then === 'function'; +} + +function recordError(span: Span, err: unknown): void { + span.recordException(err as Exception); + // Per the OpenTelemetry specification, instrumentation only sets the status on failure and leaves successful + // spans `UNSET` - marking them `OK` would prevent consumers from overriding the status themselves. + span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : String(err), + }); +} diff --git a/packages/otel/test/constants.test.ts b/packages/otel/test/constants.test.ts new file mode 100644 index 000000000000..ba0ad52297ed --- /dev/null +++ b/packages/otel/test/constants.test.ts @@ -0,0 +1,33 @@ +import { SeverityNumber } from '@opentelemetry/api-logs'; + +import { apifyLogLevelMap, apifyLogLevelNameMap, loggerMethods } from '../src/constants'; + +/** + * The mapping itself is a decision worth pinning down - `SOFT_FAIL` and `WARNING` collapse onto one OpenTelemetry + * severity, and `PERF` has no counterpart at all and is reported as `DEBUG`. + */ +describe('Crawlee log levels', () => { + test.each([ + [1, 'ERROR', SeverityNumber.ERROR], + [2, 'SOFT_FAIL', SeverityNumber.WARN], + [3, 'WARNING', SeverityNumber.WARN], + [4, 'INFO', SeverityNumber.INFO], + [5, 'DEBUG', SeverityNumber.DEBUG], + [6, 'PERF', SeverityNumber.DEBUG], + ])('level %i (%s) is reported as severity %i', (level, name, severity) => { + expect(apifyLogLevelMap[level as number]).toBe(severity); + expect(apifyLogLevelNameMap[level as number]).toBe(name); + }); + + test('OFF has no severity, since nothing is ever logged at it', () => { + expect(apifyLogLevelMap[0]).toBeUndefined(); + expect(apifyLogLevelNameMap[0]).toBeUndefined(); + }); + + test('every instrumented logging method has a level that maps to a severity', () => { + for (const method of loggerMethods) { + expect(apifyLogLevelMap[method.level], `${method.methodName} has no severity`).toBeDefined(); + expect(apifyLogLevelNameMap[method.level], `${method.methodName} has no severity text`).toBeDefined(); + } + }); +}); diff --git a/packages/otel/test/instrumentation.test.ts b/packages/otel/test/instrumentation.test.ts new file mode 100644 index 000000000000..8b21faff2442 --- /dev/null +++ b/packages/otel/test/instrumentation.test.ts @@ -0,0 +1,323 @@ +import { CrawleeInstrumentation } from '@crawlee/otel'; +import { ATTR_HTTP_REQUEST_METHOD, ATTR_URL_FULL } from '@opentelemetry/semantic-conventions'; +import { isWrapped } from '@opentelemetry/instrumentation'; +import { satisfies } from 'semver'; + +import { baseConfig, requestHandlingInstrumentationMethods } from '../src/constants'; + +describe('CrawleeInstrumentation', () => { + describe('constructor and configuration', () => { + test('creates instrumentation with default config', () => { + const instrumentation = new CrawleeInstrumentation(); + + expect(instrumentation.instrumentationName).toBe('@crawlee/otel'); + expect(instrumentation.getConfig()).toMatchObject({ + enabled: true, + requestHandlingInstrumentation: true, + logInstrumentation: true, + customInstrumentation: [], + }); + }); + + test('merges provided config with defaults', () => { + const instrumentation = new CrawleeInstrumentation({ + requestHandlingInstrumentation: false, + logInstrumentation: false, + }); + + expect(instrumentation.getConfig()).toMatchObject({ + enabled: true, // default + requestHandlingInstrumentation: false, // overridden + logInstrumentation: false, // overridden + customInstrumentation: [], // default + }); + }); + + test('an explicitly undefined flag keeps its default', () => { + // What `{ logInstrumentation: options.logs }` produces when the caller has no opinion about it. + const instrumentation = new CrawleeInstrumentation({ + enabled: undefined, + requestHandlingInstrumentation: undefined, + logInstrumentation: undefined, + customInstrumentation: undefined, + }); + + expect(instrumentation.getConfig()).toMatchObject({ + enabled: true, + requestHandlingInstrumentation: true, + logInstrumentation: true, + customInstrumentation: [], + }); + }); + + test('accepts custom instrumentation config', () => { + const customMethods = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'customMethod', + spanName: 'custom.span', + }, + ]; + + const instrumentation = new CrawleeInstrumentation({ + customInstrumentation: customMethods, + }); + + expect(instrumentation.getConfig().customInstrumentation).toEqual(customMethods); + }); + + test('can disable instrumentation entirely', () => { + const instrumentation = new CrawleeInstrumentation({ + enabled: false, + }); + + expect(instrumentation.getConfig().enabled).toBe(false); + }); + }); + + describe('init method', () => { + test('returns module definitions when request handling instrumentation enabled', () => { + const instrumentation = new CrawleeInstrumentation({ + requestHandlingInstrumentation: true, + logInstrumentation: false, + }); + + // Access protected init method for testing + const definitions = (instrumentation as any).init(); + + expect(definitions.length).toBeGreaterThan(0); + }); + + test('returns fewer definitions when request handling disabled', () => { + const withHandling = new CrawleeInstrumentation({ + requestHandlingInstrumentation: true, + logInstrumentation: false, + }); + + const withoutHandling = new CrawleeInstrumentation({ + requestHandlingInstrumentation: false, + logInstrumentation: false, + }); + + const defsWithHandling = (withHandling as any).init(); + const defsWithoutHandling = (withoutHandling as any).init(); + + expect(defsWithHandling.length).toBeGreaterThan(defsWithoutHandling.length); + }); + + test('includes log instrumentation when enabled', () => { + const instrumentation = new CrawleeInstrumentation({ + requestHandlingInstrumentation: false, + logInstrumentation: true, + }); + + const definitions = (instrumentation as any).init(); + + const logDefinition = definitions.find((d: any) => d.name === '@crawlee/core'); + expect(logDefinition).toBeDefined(); + }); + + test('excludes log instrumentation when disabled', () => { + const instrumentation = new CrawleeInstrumentation({ + requestHandlingInstrumentation: false, + logInstrumentation: false, + }); + + const definitions = (instrumentation as any).init(); + + const logDefinition = definitions.find((d: any) => d.name === '@crawlee/core'); + expect(logDefinition).toBeUndefined(); + }); + + test('combines default and custom instrumentation', () => { + const instrumentation = new CrawleeInstrumentation({ + requestHandlingInstrumentation: true, + logInstrumentation: false, + customInstrumentation: [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'customMethod', + spanName: 'custom.span', + }, + ], + }); + + const definition = (instrumentation as any).init().find((d: any) => d.name === '@crawlee/basic') as { + patch: (e: any) => any; + }; + + class BasicCrawler { + run() {} + customMethod() {} + } + definition.patch({ BasicCrawler }); + + // Both the built-in method and the configured one, rather than just a non-empty definition list. + expect(isWrapped(BasicCrawler.prototype.run)).toBe(true); + expect(isWrapped(BasicCrawler.prototype.customMethod)).toBe(true); + }); + }); + + describe('setConfig', () => { + test('allows runtime config changes', () => { + const instrumentation = new CrawleeInstrumentation({ + enabled: true, + }); + + instrumentation.setConfig({ enabled: false }); + + expect(instrumentation.getConfig().enabled).toBe(false); + }); + }); +}); + +describe('baseConfig', () => { + test('has expected default values', () => { + expect(baseConfig).toEqual({ + enabled: true, + requestHandlingInstrumentation: true, + logInstrumentation: true, + customInstrumentation: [], + }); + }); +}); + +describe('requestHandlingInstrumentationMethods', () => { + test('contains expected BasicCrawler methods', () => { + const basicMethods = requestHandlingInstrumentationMethods.filter( + (m: { moduleName: string }) => m.moduleName === '@crawlee/basic', + ); + + expect(basicMethods.length).toBeGreaterThan(0); + + const methodNames = basicMethods.map((m: { methodName: any }) => m.methodName); + expect(methodNames).toContain('run'); + expect(methodNames).toContain('handleRequest'); + expect(methodNames).toContain('runRequestHandler'); + expect(methodNames).toContain('requestFunctionErrorHandler'); + expect(methodNames).toContain('handleFailedRequestHandler'); + }); + + test('contains expected BrowserCrawler methods', () => { + const browserMethods = requestHandlingInstrumentationMethods.filter( + (m: { moduleName: string }) => m.moduleName === '@crawlee/browser', + ); + + expect(browserMethods.length).toBeGreaterThan(0); + + const methodNames = browserMethods.map((m: { methodName: any }) => m.methodName); + expect(methodNames).toContain('navigate'); + }); + + test('contains expected HttpCrawler methods', () => { + const httpMethods = requestHandlingInstrumentationMethods.filter( + (m: { moduleName: string }) => m.moduleName === '@crawlee/http', + ); + + expect(httpMethods.length).toBeGreaterThan(0); + + const methodNames = httpMethods.map((m: { methodName: any }) => m.methodName); + expect(methodNames).toContain('makeHttpRequest'); + }); + + test('all methods have valid moduleName starting with @crawlee/', () => { + for (const method of requestHandlingInstrumentationMethods) { + expect(method.moduleName).toMatch(/^@crawlee\//); + } + }); + + test('all methods have required properties', () => { + for (const method of requestHandlingInstrumentationMethods) { + expect(method.moduleName).toBeDefined(); + expect(method.className).toBeDefined(); + expect(method.methodName).toBeDefined(); + expect(method.spanName).toBeDefined(); + } + }); + + test.each([ + // [methodName, index of the crawling context in the argument list] + ['handleRequest', 0], + ['runRequestHandler', 0], + ['makeHttpRequest', 0], + ['navigate', 0], + ['handleFailedRequestHandler', 0], + // `requestFunctionErrorHandler(error, crawlingContext, request, source)` + ['requestFunctionErrorHandler', 1], + ])('%s reads request attributes from argument %i', (methodName, contextArgIndex) => { + const methods = requestHandlingInstrumentationMethods.filter((m) => m.methodName === methodName); + expect(methods.length).toBeGreaterThan(0); + + const mockContext = { + request: { + id: 'test-id', + url: 'https://example.com', + method: 'GET', + retryCount: 0, + }, + }; + const args = Array.from({ length: contextArgIndex + 1 }); + args[contextArgIndex] = mockContext; + + for (const method of methods) { + expect(typeof method.spanOptions).toBe('function'); + + // oxlint-disable-next-line no-unsafe-function-type + const options = (method.spanOptions as Function)(...args); + expect(options.attributes).toEqual({ + 'crawlee.request.id': 'test-id', + [ATTR_URL_FULL]: 'https://example.com', + [ATTR_HTTP_REQUEST_METHOD]: 'GET', + 'crawlee.request.retry_count': 0, + }); + } + }); + + test('request attributes are empty when the argument is not a crawling context', () => { + const method = requestHandlingInstrumentationMethods.find((m) => m.methodName === 'runRequestHandler')!; + + // oxlint-disable-next-line no-unsafe-function-type + expect((method.spanOptions as Function)(undefined)).toEqual({ attributes: {} }); + }); +}); + +/** + * `@opentelemetry/instrumentation` decides whether to patch a module with + * `semver.satisfies(moduleVersion, supportedVersion, { includePrerelease })` and silently leaves the module alone when + * that is false, so this reads both inputs off the definitions the instrumentation actually produces rather than + * restating the literal. Crawlee v4 is published exclusively under prerelease tags, which a caret range never matches. + */ +describe('supported Crawlee versions', () => { + const definitions = () => (new CrawleeInstrumentation() as any).init() as any[]; + + const isSupported = (definition: any, version: string) => + (definition.supportedVersions as string[]).some((range) => + satisfies(version, range, { includePrerelease: definition.includePrerelease }), + ); + + test('covers every instrumented module', () => { + expect( + definitions() + .map((d) => d.name) + .sort(), + ).toEqual(['@crawlee/basic', '@crawlee/browser', '@crawlee/core', '@crawlee/http', '@crawlee/playwright']); + }); + + // The first two are the versions currently behind the `v4` and `rc` dist-tags. + test.each(['4.0.0-beta.140', '4.0.0-rc.0', '4.0.0', '4.1.0', '4.1.0-beta.3', '4.9.9'])( + 'patches Crawlee %s', + (version) => { + for (const definition of definitions()) { + expect(isSupported(definition, version), `${definition.name} rejected ${version}`).toBe(true); + } + }, + ); + + test.each(['3.13.0', '5.0.0', '5.0.0-beta.0'])('leaves Crawlee %s alone', (version) => { + for (const definition of definitions()) { + expect(isSupported(definition, version), `${definition.name} accepted ${version}`).toBe(false); + } + }); +}); diff --git a/packages/otel/test/patching.test.ts b/packages/otel/test/patching.test.ts new file mode 100644 index 000000000000..eb4a71aea8aa --- /dev/null +++ b/packages/otel/test/patching.test.ts @@ -0,0 +1,416 @@ +import { CrawleeInstrumentation } from '@crawlee/otel'; +import { SeverityNumber } from '@opentelemetry/api-logs'; +import type { LogRecord } from '@opentelemetry/sdk-logs'; +import { InMemoryLogRecordExporter, LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs'; +import type { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { + ATTR_CODE_FUNCTION_NAME, + ATTR_EXCEPTION_MESSAGE, + ATTR_EXCEPTION_STACKTRACE, + ATTR_EXCEPTION_TYPE, + ATTR_HTTP_REQUEST_METHOD, + ATTR_URL_FULL, +} from '@opentelemetry/semantic-conventions'; + +/** + * Applies the module patches produced by `init()` to a stub module, which lets us assert what the instrumentation + * actually does to the patched classes without going through the module loader hooks. + */ +function patchModule(instrumentation: CrawleeInstrumentation, moduleName: string, moduleExports: any) { + const definition = (instrumentation as any).init().find((d: any) => d.name === moduleName) as { + patch: (e: any) => any; + unpatch: (e: any) => any; + }; + + expect(definition).toBeDefined(); + definition.patch(moduleExports); + + return definition; +} + +describe('automatic instrumentation', () => { + let provider: NodeTracerProvider; + let exporter: InMemorySpanExporter; + let processor: SimpleSpanProcessor; + + beforeAll(() => { + exporter = new InMemorySpanExporter(); + processor = new SimpleSpanProcessor(exporter); + provider = new NodeTracerProvider({ spanProcessors: [processor] }); + provider.register(); + }); + + beforeEach(() => exporter.reset()); + + afterAll(async () => { + await provider.shutdown(); + }); + + test('wraps the configured method and records request attributes', async () => { + class HttpCrawler { + public seen: string[] = []; + + async makeHttpRequest(crawlingContext: { request: { url: string } }) { + this.seen.push(crawlingContext.request.url); + return 'handled'; + } + } + + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + patchModule(instrumentation, '@crawlee/http', { HttpCrawler }); + + const crawler = new HttpCrawler(); + const context = { request: { id: 'req-1', url: 'https://example.com', method: 'GET', retryCount: 2 } }; + + await expect(crawler.makeHttpRequest(context)).resolves.toBe('handled'); + // The original behaviour is preserved. + expect(crawler.seen).toEqual(['https://example.com']); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('crawlee.http.makeHttpRequest'); + expect(spans[0].attributes).toEqual({ + 'crawlee.request.id': 'req-1', + [ATTR_URL_FULL]: 'https://example.com', + [ATTR_HTTP_REQUEST_METHOD]: 'GET', + 'crawlee.request.retry_count': 2, + [ATTR_CODE_FUNCTION_NAME]: 'HttpCrawler.makeHttpRequest', + }); + }); + + test('nests the spans of nested instrumented calls', async () => { + class BasicCrawler { + async run() { + return this.handleRequest(); + } + + async handleRequest() { + return 'done'; + } + } + + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + patchModule(instrumentation, '@crawlee/basic', { BasicCrawler }); + + await new BasicCrawler().run(); + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + const run = spans.find((s: ReadableSpan) => s.name === 'crawlee.crawler.run')!; + const task = spans.find((s: ReadableSpan) => s.name === 'crawlee.crawler.handleRequest')!; + + expect(run).toBeDefined(); + expect(task).toBeDefined(); + expect(task.parentSpanContext?.spanId).toBe(run.spanContext().spanId); + expect(task.spanContext().traceId).toBe(run.spanContext().traceId); + // `run` records the concrete crawler class, resolved through `this`. + expect(run.attributes['crawlee.crawler.type']).toBe('BasicCrawler'); + }); + + test('records the exception and rethrows when the patched method fails', async () => { + class BasicCrawler { + async run(): Promise { + throw new Error('crawl failed'); + } + } + + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + patchModule(instrumentation, '@crawlee/basic', { BasicCrawler }); + + await expect(new BasicCrawler().run()).rejects.toThrow('crawl failed'); + await processor.forceFlush(); + + const span = exporter.getFinishedSpans().find((s: ReadableSpan) => s.name === 'crawlee.crawler.run')!; + expect(span.status.code).toBe(2); // SpanStatusCode.ERROR + expect(span.events.map((e) => e.name)).toContain('exception'); + }); + + test('unpatch restores the original method', async () => { + class BasicCrawler { + async run() { + return 'ok'; + } + } + + const original = BasicCrawler.prototype.run; + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + const definition = patchModule(instrumentation, '@crawlee/basic', { BasicCrawler }); + + expect(BasicCrawler.prototype.run).not.toBe(original); + + definition.unpatch({ BasicCrawler }); + + expect(BasicCrawler.prototype.run).toBe(original); + + await new BasicCrawler().run(); + await processor.forceFlush(); + expect(exporter.getFinishedSpans()).toHaveLength(0); + }); + + test('skips a method that does not exist in the installed version instead of throwing', () => { + class BasicCrawler {} + + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + + // A version of `@crawlee/basic` without the instrumented internals must still load. + expect(() => patchModule(instrumentation, '@crawlee/basic', { BasicCrawler })).not.toThrow(); + }); + + test('skips a class that is not exported by the module instead of throwing', () => { + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + + expect(() => patchModule(instrumentation, '@crawlee/basic', {})).not.toThrow(); + }); + + test('tolerates an explicitly undefined customInstrumentation', () => { + expect(() => new CrawleeInstrumentation({ customInstrumentation: undefined })).not.toThrow(); + }); +}); + +describe('log instrumentation', () => { + let loggerProvider: LoggerProvider; + let logExporter: InMemoryLogRecordExporter; + + beforeAll(() => { + logExporter = new InMemoryLogRecordExporter(); + loggerProvider = new LoggerProvider({ + processors: [new SimpleLogRecordProcessor(logExporter)], + }); + }); + + beforeEach(() => logExporter.reset()); + + afterAll(async () => { + await loggerProvider.shutdown(); + }); + + /** + * Minimal stand-in for `BaseCrawleeLogger`, mirroring how the real one derives every logging method from the + * abstract `logWithLevel` that each adapter implements. + */ + function createLogModule() { + const calls: unknown[][] = []; + + class BaseCrawleeLogger { + logWithLevel(...args: unknown[]): void { + calls.push(args); + } + + error(message: string, data?: Record) { + this.logWithLevel(1, message, data); + } + + exception(exception: Error, message: string, data?: Record) { + this.logWithLevel(1, `${message}: ${exception.message}`, { ...data, exception }); + } + + warning(message: string, data?: Record) { + this.logWithLevel(3, message, data); + } + + warningOnce(message: string) { + this.warning(message); + } + + info(message: string, data?: Record) { + this.logWithLevel(4, message, data); + } + } + + return { moduleExports: { BaseCrawleeLogger }, calls }; + } + + function patchLog(moduleExports: any) { + const instrumentation = new CrawleeInstrumentation({ requestHandlingInstrumentation: false }); + instrumentation.setLoggerProvider(loggerProvider); + + const definition = (instrumentation as any).init().find((d: any) => d.name === '@crawlee/core') as { + patch: (e: any) => any; + }; + + definition.patch(moduleExports); + } + + test('forwards log records and keeps calling the original method', () => { + const { moduleExports, calls } = createLogModule(); + patchLog(moduleExports); + + new moduleExports.BaseCrawleeLogger().info('hello', { foo: 'bar' }); + + // The original still runs, so the underlying logger keeps printing. + expect(calls).toEqual([[4, 'hello', { foo: 'bar' }]]); + + const records = logExporter.getFinishedLogRecords(); + expect(records).toHaveLength(1); + expect(records[0].body).toBe('hello'); + expect(records[0].severityNumber).toBe(SeverityNumber.INFO); + expect(records[0].severityText).toBe('INFO'); + expect(records[0].attributes).toEqual({ foo: 'bar' }); + }); + + test('returns void, so the original synchronous contract is preserved', () => { + const { moduleExports } = createLogModule(); + patchLog(moduleExports); + + expect(new moduleExports.BaseCrawleeLogger().info('hello')).toBeUndefined(); + }); + + test('forwards a log from any logger implementation, and only once per call', () => { + const { moduleExports } = createLogModule(); + patchLog(moduleExports); + + // A custom adapter only implements `logWithLevel`; the derived methods come from the base class. + class WinstonLikeAdapter extends moduleExports.BaseCrawleeLogger {} + + new WinstonLikeAdapter().warning('from a custom adapter'); + // `warningOnce` routes through `warning`, so it must not emit twice. + new WinstonLikeAdapter().warningOnce('once'); + + const records = logExporter.getFinishedLogRecords(); + expect(records.map((r) => r.body)).toEqual(['from a custom adapter', 'once']); + expect(records.every((r) => r.severityNumber === SeverityNumber.WARN)).toBe(true); + }); + + test('maps an exception onto the semantic convention attributes', () => { + const { moduleExports } = createLogModule(); + patchLog(moduleExports); + + const error = new TypeError('boom'); + new moduleExports.BaseCrawleeLogger().exception(error, 'failed'); + + const record = logExporter.getFinishedLogRecords()[0] as LogRecord; + expect(record.severityNumber).toBe(SeverityNumber.ERROR); + expect(record.attributes[ATTR_EXCEPTION_TYPE]).toBe('TypeError'); + expect(record.attributes[ATTR_EXCEPTION_MESSAGE]).toBe('boom'); + expect(record.attributes[ATTR_EXCEPTION_STACKTRACE]).toContain('TypeError: boom'); + }); +}); + +/** + * An instrumentation must not be able to change the behaviour of the code it patches. Everything the instrumentation + * itself does - reading the log arguments, emitting the record, resolving a user supplied span name or attributes - + * runs inside the application's call path, so a failure in any of it has to stay contained. + */ +describe('telemetry failures are contained', () => { + let provider: NodeTracerProvider; + let exporter: InMemorySpanExporter; + let processor: SimpleSpanProcessor; + + beforeAll(() => { + exporter = new InMemorySpanExporter(); + processor = new SimpleSpanProcessor(exporter); + provider = new NodeTracerProvider({ spanProcessors: [processor] }); + }); + + beforeEach(() => exporter.reset()); + + afterAll(async () => { + await provider.shutdown(); + }); + + /** A logger provider whose records never make it out, standing in for a throwing log record processor. */ + const explodingLoggerProvider = { + getLogger: () => ({ + emit: () => { + throw new Error('log record processor exploded'); + }, + }), + } as unknown as LoggerProvider; + + function patchLogger(loggerProvider: LoggerProvider) { + const calls: unknown[][] = []; + + class BaseCrawleeLogger { + logWithLevel(...args: unknown[]): void { + calls.push(args); + } + + info(message: unknown, data?: Record) { + this.logWithLevel(4, message, data); + } + } + + const instrumentation = new CrawleeInstrumentation({ requestHandlingInstrumentation: false }); + instrumentation.setLoggerProvider(loggerProvider); + const definition = (instrumentation as any).init().find((d: any) => d.name === '@crawlee/core') as { + patch: (e: any) => any; + }; + definition.patch({ BaseCrawleeLogger }); + + return { logger: new BaseCrawleeLogger(), calls }; + } + + test('a throwing log pipeline does not swallow the application log call', () => { + const { logger, calls } = patchLogger(explodingLoggerProvider); + + expect(() => logger.info('hello', { foo: 'bar' })).not.toThrow(); + expect(calls).toEqual([[4, 'hello', { foo: 'bar' }]]); + }); + + test('a message that cannot be stringified does not swallow the application log call', () => { + const { logger, calls } = patchLogger( + new LoggerProvider({ processors: [new SimpleLogRecordProcessor(new InMemoryLogRecordExporter())] }), + ); + const hostile = { + toString() { + throw new Error('nope'); + }, + }; + + expect(() => logger.info(hostile)).not.toThrow(); + expect(calls).toEqual([[4, hostile, undefined]]); + }); + + function patchWithFailingHook(hook: 'spanName' | 'spanOptions') { + class BasicCrawler { + async handleRequest() { + return 'done'; + } + } + + const instrumentation = new CrawleeInstrumentation({ + requestHandlingInstrumentation: false, + logInstrumentation: false, + customInstrumentation: [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'handleRequest', + spanName: 'crawlee.crawler.handleRequest', + [hook]: () => { + throw new Error(`${hook} exploded`); + }, + }, + ], + }); + instrumentation.setTracerProvider(provider); + const definition = (instrumentation as any).init().find((d: any) => d.name === '@crawlee/basic') as { + patch: (e: any) => any; + }; + definition.patch({ BasicCrawler }); + + return new BasicCrawler(); + } + + test('a throwing spanName callback does not break the patched method', async () => { + await expect(patchWithFailingHook('spanName').handleRequest()).resolves.toBe('done'); + + await processor.forceFlush(); + // The span is still recorded, under the method's default name. + expect(exporter.getFinishedSpans().map((s: ReadableSpan) => s.name)).toEqual(['BasicCrawler.handleRequest']); + }); + + test('a throwing spanOptions callback does not break the patched method', async () => { + await expect(patchWithFailingHook('spanOptions').handleRequest()).resolves.toBe('done'); + + await processor.forceFlush(); + const spans = exporter.getFinishedSpans(); + expect(spans.map((s: ReadableSpan) => s.name)).toEqual(['crawlee.crawler.handleRequest']); + // The attributes the instrumentation itself adds are unaffected by the failing callback. + expect(spans[0].attributes).toEqual({ [ATTR_CODE_FUNCTION_NAME]: 'BasicCrawler.handleRequest' }); + }); +}); diff --git a/packages/otel/test/tsconfig.json b/packages/otel/test/tsconfig.json new file mode 100644 index 000000000000..eb8cbab58123 --- /dev/null +++ b/packages/otel/test/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../../tsconfig.json", + "include": ["**/*", "../../**/*"], + "compilerOptions": { + "types": ["vitest/globals"] + } +} diff --git a/packages/otel/test/utilities.test.ts b/packages/otel/test/utilities.test.ts new file mode 100644 index 000000000000..5a4bf8383310 --- /dev/null +++ b/packages/otel/test/utilities.test.ts @@ -0,0 +1,221 @@ +import type { ClassMethodToInstrument } from '@crawlee/otel'; +import { diag } from '@opentelemetry/api'; + +import { buildModuleDefinitions } from '../src/utilities'; + +describe('buildModuleDefinitions', () => { + let diagWarnSpy: ReturnType; + + beforeEach(() => { + diagWarnSpy = vi.spyOn(diag, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + diagWarnSpy.mockRestore(); + }); + + test('builds module definitions from method list', () => { + const methods: ClassMethodToInstrument[] = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'crawlee.crawler.run', + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: '_runTaskFunction', + spanName: 'crawlee.crawler.runTaskFunction', + }, + ]; + + const definitions = buildModuleDefinitions(methods); + + expect(definitions).toHaveLength(1); + expect(definitions[0].moduleName).toBe('@crawlee/basic'); + expect(definitions[0].classMethodPatches).toHaveLength(2); + expect(definitions[0].classMethodPatches[0]).toEqual({ + className: 'BasicCrawler', + methodName: 'run', + spanName: 'crawlee.crawler.run', + spanOptions: undefined, + }); + expect(definitions[0].classMethodPatches[1]).toEqual({ + className: 'BasicCrawler', + methodName: '_runTaskFunction', + spanName: 'crawlee.crawler.runTaskFunction', + spanOptions: undefined, + }); + }); + + test('groups methods by module name', () => { + const methods: ClassMethodToInstrument[] = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'basic.run', + }, + { + moduleName: '@crawlee/browser', + className: 'BrowserCrawler', + methodName: '_handleNavigation', + spanName: 'browser.navigation', + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: '_executeHooks', + spanName: 'basic.hooks', + }, + ]; + + const definitions = buildModuleDefinitions(methods); + + expect(definitions).toHaveLength(2); + + const basicDef = definitions.find((d) => d.moduleName === '@crawlee/basic'); + const browserDef = definitions.find((d) => d.moduleName === '@crawlee/browser'); + + expect(basicDef?.classMethodPatches).toHaveLength(2); + expect(browserDef?.classMethodPatches).toHaveLength(1); + }); + + test('skips non-crawlee modules and logs warning', () => { + const methods: ClassMethodToInstrument[] = [ + { + moduleName: 'some-other-package', + className: 'SomeClass', + methodName: 'someMethod', + spanName: 'some-span', + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'basic.run', + }, + ]; + + const definitions = buildModuleDefinitions(methods); + + expect(definitions).toHaveLength(1); + expect(definitions[0].moduleName).toBe('@crawlee/basic'); + expect(diagWarnSpy).toHaveBeenCalledWith('Module some-other-package is not a valid Crawlee module. Skipping.'); + }); + + test('skips duplicate method instrumentation and logs warning', () => { + const methods: ClassMethodToInstrument[] = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'first-span', + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'duplicate-span', + }, + ]; + + const definitions = buildModuleDefinitions(methods); + + expect(definitions).toHaveLength(1); + expect(definitions[0].classMethodPatches).toHaveLength(1); + expect(definitions[0].classMethodPatches[0].spanName).toBe('first-span'); + expect(diagWarnSpy).toHaveBeenCalledWith(expect.stringContaining('Keeping the first definition')); + }); + + test('allows same method name on different classes', () => { + const methods: ClassMethodToInstrument[] = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'basic.run', + }, + { + moduleName: '@crawlee/basic', + className: 'AnotherCrawler', + methodName: 'run', + spanName: 'another.run', + }, + ]; + + const definitions = buildModuleDefinitions(methods); + + expect(definitions).toHaveLength(1); + expect(definitions[0].classMethodPatches).toHaveLength(2); + }); + + test('preserves spanOptions function', () => { + const spanOptionsFn = (ctx: any) => ({ + attributes: { 'test.attr': ctx.value }, + }); + + const methods: ClassMethodToInstrument[] = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + spanName: 'span-with-options', + spanOptions: spanOptionsFn, + }, + ]; + + const definitions = buildModuleDefinitions(methods); + + expect(definitions[0].classMethodPatches[0].spanOptions).toBe(spanOptionsFn); + }); + + test('returns empty array for empty input', () => { + const definitions = buildModuleDefinitions([]); + + expect(definitions).toEqual([]); + }); + + test('handles multiple modules with multiple classes', () => { + const methods: ClassMethodToInstrument[] = [ + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'run', + }, + { + moduleName: '@crawlee/basic', + className: 'BasicCrawler', + methodName: 'stop', + }, + { + moduleName: '@crawlee/browser', + className: 'BrowserCrawler', + methodName: 'run', + }, + { + moduleName: '@crawlee/browser', + className: 'BrowserCrawler', + methodName: '_handleNavigation', + }, + { + moduleName: '@crawlee/http', + className: 'HttpCrawler', + methodName: 'run', + }, + ]; + + const definitions = buildModuleDefinitions(methods); + + expect(definitions).toHaveLength(3); + + const basicDef = definitions.find((d) => d.moduleName === '@crawlee/basic'); + const browserDef = definitions.find((d) => d.moduleName === '@crawlee/browser'); + const httpDef = definitions.find((d) => d.moduleName === '@crawlee/http'); + + expect(basicDef?.classMethodPatches).toHaveLength(2); + expect(browserDef?.classMethodPatches).toHaveLength(2); + expect(httpDef?.classMethodPatches).toHaveLength(1); + }); +}); diff --git a/packages/otel/test/wrap-with-span.test.ts b/packages/otel/test/wrap-with-span.test.ts new file mode 100644 index 000000000000..8d8ba906609d --- /dev/null +++ b/packages/otel/test/wrap-with-span.test.ts @@ -0,0 +1,444 @@ +import { SpanStatusCode, trace } from '@opentelemetry/api'; +import type { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; + +import { setSharedTracer, wrapWithSpan } from '../src/wrapWithSpan'; + +describe('wrapWithSpan', () => { + let provider: NodeTracerProvider; + let exporter: InMemorySpanExporter; + let processor: SimpleSpanProcessor; + + beforeAll(() => { + exporter = new InMemorySpanExporter(); + processor = new SimpleSpanProcessor(exporter); + provider = new NodeTracerProvider({ + spanProcessors: [processor], + }); + provider.register(); + }); + + beforeEach(() => { + exporter.reset(); + }); + + afterAll(async () => { + await provider.shutdown(); + }); + + describe('basic functionality', () => { + test('wraps a sync function and creates a span', async () => { + const fn = vi.fn(() => 'result'); + const wrapped = wrapWithSpan(fn, { + spanName: 'test-span', + tracer: provider.getTracer('test-tracer'), + }); + + const result = await wrapped(); + + await processor.forceFlush(); + + expect(result).toBe('result'); + expect(fn).toHaveBeenCalledOnce(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('test-span'); + // Instrumentation leaves successful spans UNSET so consumers can set their own status. + expect(spans[0].status.code).toBe(SpanStatusCode.UNSET); + }); + + test('wraps an async function and creates a span', async () => { + const fn = vi.fn(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + return 'async-result'; + }); + const wrapped = wrapWithSpan(fn, { + spanName: 'async-span', + tracer: provider.getTracer('test-tracer'), + }); + + const result = await wrapped(); + + await processor.forceFlush(); + + expect(result).toBe('async-result'); + expect(fn).toHaveBeenCalledOnce(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('async-span'); + expect(spans[0].status.code).toBe(SpanStatusCode.UNSET); + }); + + test('keeps a sync function synchronous', async () => { + const fn = vi.fn(() => 'sync-result'); + const wrapped = wrapWithSpan(fn, { + spanName: 'sync-span', + tracer: provider.getTracer('test-tracer'), + }); + + const result = wrapped(); + + expect(result).toBe('sync-result'); + expect(result).not.toBeInstanceOf(Promise); + + // The span of a sync function is closed before the call returns. + await processor.forceFlush(); + expect(exporter.getFinishedSpans()).toHaveLength(1); + }); + + test('passes arguments to the wrapped function', async () => { + const fn = vi.fn((a: number, b: string) => `${a}-${b}`); + const wrapped = wrapWithSpan(fn, { + spanName: 'args-span', + tracer: provider.getTracer('test-tracer'), + }); + + const result = await wrapped(42, 'hello'); + + expect(result).toBe('42-hello'); + expect(fn).toHaveBeenCalledWith(42, 'hello'); + }); + + test('uses function name as span name when no spanName provided', async () => { + function namedFunction() { + return 'named'; + } + const wrapped = wrapWithSpan(namedFunction, { + tracer: provider.getTracer('test-tracer'), + }); + + wrapped(); + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('namedFunction'); + }); + + test('uses "anonymous" as span name for anonymous functions without spanName', async () => { + const wrapped = wrapWithSpan(() => 'anon', { + tracer: provider.getTracer('test-tracer'), + }); + + wrapped(); + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('anonymous'); + }); + }); + + describe('error handling', () => { + test('records exception and sets error status when function throws', async () => { + const error = new Error('Test error'); + const fn = vi.fn(() => { + throw error; + }); + const wrapped = wrapWithSpan(fn, { + spanName: 'error-span', + tracer: provider.getTracer('test-tracer'), + }); + + // A sync function throws synchronously - it is not turned into a rejected promise. + expect(() => wrapped()).toThrow('Test error'); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('error-span'); + expect(spans[0].status.code).toBe(SpanStatusCode.ERROR); + expect(spans[0].events).toHaveLength(1); + expect(spans[0].events[0].name).toBe('exception'); + }); + + test('records exception for async function rejection', async () => { + const fn = vi.fn(async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); + throw new Error('Async error'); + }); + const wrapped = wrapWithSpan(fn, { + spanName: 'async-error-span', + tracer: provider.getTracer('test-tracer'), + }); + + await expect(wrapped()).rejects.toThrow('Async error'); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status.code).toBe(SpanStatusCode.ERROR); + }); + }); + + describe('dynamic span name', () => { + test('supports function-based spanName', async () => { + const fn = vi.fn((url: string) => `fetched: ${url}`); + const wrapped = wrapWithSpan(fn, { + spanName: (url: string) => `fetch ${url}`, + tracer: provider.getTracer('test-tracer'), + }); + + await wrapped('https://example.com'); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('fetch https://example.com'); + }); + + test('spanName function receives all arguments', async () => { + const fn = vi.fn((a: number, b: number) => a + b); + const wrapped = wrapWithSpan(fn, { + spanName: (a: number, b: number) => `add-${a}-${b}`, + tracer: provider.getTracer('test-tracer'), + }); + + await wrapped(1, 2); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('add-1-2'); + }); + }); + + describe('span options and attributes', () => { + test('supports static spanOptions', async () => { + const fn = vi.fn(() => 'result'); + const wrapped = wrapWithSpan(fn, { + spanName: 'with-attrs', + spanOptions: { + attributes: { + 'custom.attr': 'value', + 'custom.number': 42, + }, + }, + tracer: provider.getTracer('test-tracer'), + }); + + await wrapped(); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].attributes['custom.attr']).toBe('value'); + expect(spans[0].attributes['custom.number']).toBe(42); + }); + + test('supports function-based spanOptions', async () => { + interface Request { + url: string; + method: string; + } + + const fn = vi.fn((req: Request) => `handled ${req.url}`); + const wrapped = wrapWithSpan(fn, { + spanName: 'dynamic-attrs', + spanOptions: (req: Request) => ({ + attributes: { + 'request.url': req.url, + 'request.method': req.method, + }, + }), + tracer: provider.getTracer('test-tracer'), + }); + + await wrapped({ url: 'https://example.com', method: 'GET' }); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].attributes['request.url']).toBe('https://example.com'); + expect(spans[0].attributes['request.method']).toBe('GET'); + }); + }); + + describe('context propagation', () => { + test('multiple wrapped calls create separate spans', async () => { + const fn1 = vi.fn(() => 'result1'); + const fn2 = vi.fn(() => 'result2'); + const wrapped1 = wrapWithSpan(fn1, { + spanName: 'span-1', + tracer: provider.getTracer('test-tracer'), + }); + const wrapped2 = wrapWithSpan(fn2, { + spanName: 'span-2', + tracer: provider.getTracer('test-tracer'), + }); + + await wrapped1(); + await wrapped2(); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(2); + expect(spans.map((s) => s.name).sort()).toEqual(['span-1', 'span-2']); + }); + + test('concurrent wrapped calls each create their own span', async () => { + const fn = vi.fn(async (id: number) => { + await new Promise((resolve) => setTimeout(resolve, 5)); + return `result-${id}`; + }); + + const wrapped = wrapWithSpan(fn, { + spanName: (id: number) => `concurrent-span-${id}`, + tracer: provider.getTracer('test-tracer'), + }); + + await Promise.all([wrapped(1), wrapped(2), wrapped(3)]); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(3); + expect(spans.map((s) => s.name).sort()).toEqual([ + 'concurrent-span-1', + 'concurrent-span-2', + 'concurrent-span-3', + ]); + }); + + test('nested wrapWithSpan creates multiple spans', async () => { + const innerFn = vi.fn(() => 'inner'); + const wrappedInner = wrapWithSpan(innerFn, { + spanName: 'inner-span', + tracer: provider.getTracer('test-tracer'), + }); + + const outerFn = vi.fn(async () => { + return wrappedInner(); + }); + const wrappedOuter = wrapWithSpan(outerFn, { + spanName: 'outer-span', + tracer: provider.getTracer('test-tracer'), + }); + + await wrappedOuter(); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(2); + + const innerSpan = spans.find((s: ReadableSpan) => s.name === 'inner-span'); + const outerSpan = spans.find((s: ReadableSpan) => s.name === 'outer-span'); + + expect(innerSpan).toBeDefined(); + expect(outerSpan).toBeDefined(); + }); + }); + + describe('custom tracer', () => { + test('uses custom tracer when provided', async () => { + const customTracer = trace.getTracer('custom-tracer', '1.0.0'); + const fn = vi.fn(() => 'result'); + const wrapped = wrapWithSpan(fn, { + spanName: 'custom-tracer-span', + tracer: customTracer, + }); + + await wrapped(); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + // Use instrumentationScope (newer SDK) or instrumentationLibrary (older SDK) + const span = spans[0] as any; + const scope = span.instrumentationScope ?? span.instrumentationLibrary; + expect(scope.name).toBe('custom-tracer'); + expect(scope.version).toBe('1.0.0'); + }); + + test('uses the tracer handed over by the instrumentation', async () => { + const fn = vi.fn(() => 'result'); + const tracer = trace.getTracer('crawlee'); + setSharedTracer(tracer); + const wrapped = wrapWithSpan(fn); + + await wrapped(); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + // Use instrumentationScope (newer SDK) or instrumentationLibrary (older SDK) + const span = spans[0] as any; + const scope = span.instrumentationScope ?? span.instrumentationLibrary; + expect(scope.name).toBe('crawlee'); + }); + + test('falls back to the global tracer provider when no tracer was set', async () => { + // Nothing has handed a tracer over - wrapping must still work instead of throwing. + setSharedTracer(undefined); + + const wrapped = wrapWithSpan(() => 'result', { spanName: 'fallback-span' }); + + expect(wrapped()).toBe('result'); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('fallback-span'); + const scope = (spans[0] as any).instrumentationScope ?? (spans[0] as any).instrumentationLibrary; + expect(scope.name).toBe('@crawlee/otel'); + }); + }); + + describe('this context', () => { + beforeEach(() => { + // Do not depend on a tracer set by an earlier test. + setSharedTracer(provider.getTracer('test-tracer')); + }); + + test('preserves this context for regular functions', () => { + const obj = { + value: 42, + getValue() { + return this.value; + }, + }; + + const wrapped = wrapWithSpan(obj.getValue, { spanName: 'this-span' }); + const result = wrapped.call(obj); + + expect(result).toBe(42); + }); + + test('spanName function receives this context', async () => { + const obj = { + name: 'TestObject', + doSomething() { + return 'done'; + }, + }; + + const wrapped = wrapWithSpan(obj.doSomething, { + spanName(this: typeof obj) { + return `span-for-${this.name}`; + }, + }); + + wrapped.call(obj); + + await processor.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('span-for-TestObject'); + }); + }); +}); diff --git a/packages/otel/tsconfig.build.json b/packages/otel/tsconfig.build.json new file mode 100644 index 000000000000..5f63b6d3df40 --- /dev/null +++ b/packages/otel/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/packages/otel/tsconfig.json b/packages/otel/tsconfig.json new file mode 100644 index 000000000000..66bb87a91ee7 --- /dev/null +++ b/packages/otel/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"] +} diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts index f8453cf4ac38..d57ff19efb48 100644 --- a/packages/types/src/storages.ts +++ b/packages/types/src/storages.ts @@ -206,7 +206,14 @@ export interface KeyValueStoreBackend { */ listKeys(options?: KeyValueStoreListKeysOptions): Promise; - /** Get the public URL for a record, or `undefined` if unavailable. */ + /** + * Get the public URL a record with this key has, or `undefined` if the backend exposes no public + * URLs at all. + * + * The URL is derived from the key; implementations MUST NOT check that the record exists. Callers + * legitimately ask for the URL of a record that is not on the backend yet — most notably one + * buffered in an uncommitted storage transaction — and existence is `recordExists`'s job. + */ getPublicUrl(key: string): Promise; /** Check whether a record with the given key exists. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8bdcd9e027b0..51a3987ee392 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -266,6 +266,9 @@ importers: '@crawlee/linkedom': specifier: workspace:* version: link:packages/linkedom-crawler + '@crawlee/otel': + specifier: workspace:* + version: link:packages/otel '@crawlee/playwright': specifier: workspace:* version: link:packages/playwright-crawler @@ -281,6 +284,30 @@ importers: '@crawlee/utils': specifier: workspace:* version: link:packages/utils + '@microsoft/api-extractor': + specifier: ^7.58.9 + version: 7.59.0(@types/node@24.12.2) + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/api-logs': + specifier: ^0.210.0 + version: 0.210.0 + '@opentelemetry/instrumentation': + specifier: ^0.210.0 + version: 0.210.0(@opentelemetry/api@1.9.0)(supports-color@7.2.0) + '@opentelemetry/sdk-logs': + specifier: ^0.210.0 + version: 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': + specifier: ^2.0.0 + version: 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-node': + specifier: ^2.0.0 + version: 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': + specifier: ^1.39.0 + version: 1.43.0 '@oxlint/plugins': specifier: ^1.62.0 version: 1.62.0 @@ -476,9 +503,30 @@ importers: '@crawlee/impit-client': specifier: workspace:* version: link:../packages/impit-client + '@crawlee/otel': + specifier: workspace:* + version: link:../packages/otel '@crawlee/stagehand': specifier: workspace:* version: link:../packages/stagehand-crawler + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/exporter-trace-otlp-grpc': + specifier: ^0.210.0 + version: 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': + specifier: ^2.0.0 + version: 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-node': + specifier: ^0.210.0 + version: 0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1) + '@opentelemetry/sdk-trace-base': + specifier: ^2.0.0 + version: 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': + specifier: ^1.39.0 + version: 1.43.0 apify: specifier: ^4.0.0-beta.24 version: 4.0.0-beta.24(bufferutil@4.1.0)(debug@4.4.3(supports-color@7.2.0))(supports-color@8.1.1) @@ -806,8 +854,8 @@ importers: packages/fs-storage: dependencies: '@crawlee/fs-storage-native': - specifier: 0.2.0 - version: 0.2.0 + specifier: '>=0.2.1-beta.0 <0.3' + version: 0.2.1-beta.0 '@crawlee/types': specifier: workspace:* version: link:../types @@ -962,6 +1010,25 @@ importers: specifier: ^2.8.1 version: 2.8.1 + packages/otel: + dependencies: + '@opentelemetry/instrumentation': + specifier: ^0.210.0 + version: 0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1) + '@opentelemetry/semantic-conventions': + specifier: ^1.39.0 + version: 1.43.0 + devDependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/api-logs': + specifier: ^0.210.0 + version: 0.210.0 + semver: + specifier: ^7.7.0 + version: 7.7.4 + packages/playwright-crawler: dependencies: '@apify/datastructures': @@ -1087,7 +1154,7 @@ importers: devDependencies: '@browserbasehq/stagehand': specifier: 3.0.7 - version: 3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3) + version: 3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3) playwright: specifier: ^1.60.0 version: 1.61.1 @@ -2272,60 +2339,60 @@ packages: resolution: {integrity: sha512-TzlTVpKPjaqW6qOYjQcYUDuGsLCNsvFHVBXkYGTAnf5V37jCWrE5haKNXzz0WZUtVHjrpV76L1buANjwXMfT8w==} engines: {node: '>=22'} - '@crawlee/fs-storage-native-darwin-arm64@0.2.0': - resolution: {integrity: sha512-Jg4OjQVZWqQz6QCQ939iY/EJA55zAg2ZhB5JtTy0okhkfPc8kYHN43DRP2FswDHRACaftOEy8NaI1H3+V4Z4KQ==} + '@crawlee/fs-storage-native-darwin-arm64@0.2.1-beta.0': + resolution: {integrity: sha512-Ra0bQfxAGbfhDOwZ32GJuHTGc1WrIyE23mE1sLKNnZ4ukPnDSyy//ZO23LIkZ9i6YxyLsA6X3SwmZ/TFpewVVg==} engines: {node: '>= 20'} cpu: [arm64] os: [darwin] - '@crawlee/fs-storage-native-darwin-x64@0.2.0': - resolution: {integrity: sha512-BnBgAVygRAaEBFcyTgSNr9DrquUuIe4O/jtgSPctQ/oUiZ+PkK2DKk+OEtpMXpC+wVzLsBWnSs2aFnFIf78PYA==} + '@crawlee/fs-storage-native-darwin-x64@0.2.1-beta.0': + resolution: {integrity: sha512-Uf+hMZBrKMyKrJLUAer6V4W1xkyuABzgtsh2iByJE0CgPsyOS3mY4RdQBnKgS++XMQs5w94/DbfSWA90xV54Nw==} engines: {node: '>= 20'} cpu: [x64] os: [darwin] - '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.0': - resolution: {integrity: sha512-YWJEgDHLaN11chIGAvRJOkqgdxPmzkRTeEGG1ptg0VYQbNzNdXvq6ykiq1njDwWwgUzXTqjeRFF2/U44Hc4xXw==} + '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.1-beta.0': + resolution: {integrity: sha512-P+Be5g8YxnA7f93GOnnm+5jpFb5uhI0iFXcWzm1aLpQU0fhE6hZOuiYSxKYRqn2AU6yPLjqwHm0eKuRm/UO83A==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] libc: [glibc] - '@crawlee/fs-storage-native-linux-arm64-musl@0.2.0': - resolution: {integrity: sha512-C+ED04FlzKNxECIl3miisbX9eC3pSV/yrFboCrUAycA8Vh01fRTfeREzS+hMOsCQsRXeCdQ5Ifihi1T/nJwUwQ==} + '@crawlee/fs-storage-native-linux-arm64-musl@0.2.1-beta.0': + resolution: {integrity: sha512-X8U+aS00umhQ+8gHJdQg7j40J20hLvbb9pvOP9gb0Y6PdZt/jAbivIyGP871wGViJ+fwF1QQQtFO1HP5IzE8+w==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] libc: [musl] - '@crawlee/fs-storage-native-linux-x64-gnu@0.2.0': - resolution: {integrity: sha512-NlZ3Ud9dBiqkXwFEJbEVOg6PcnYdUF+C+gR3LvD/H4FOr5noLn48Kp8ikbEMbGlPdyPtLvdsMp2MCQPlQEPYYw==} + '@crawlee/fs-storage-native-linux-x64-gnu@0.2.1-beta.0': + resolution: {integrity: sha512-u0yUDlk5rg51c4eTOyk6lnWiYaoUw1Z53VnlmXYDMFSUW7tGQHLdy0aQBL+RWBmD0+mLMRiBDJjYoxsrKYYCLg==} engines: {node: '>= 20'} cpu: [x64] os: [linux] libc: [glibc] - '@crawlee/fs-storage-native-linux-x64-musl@0.2.0': - resolution: {integrity: sha512-Jb78RXko15FIXQgyb7pdTgYVc8oLN+ltEDJt9QprJUh65kUrVmPc/d2kXJhtvkH7CGEYR6/QLjfRKvoD1eeWyg==} + '@crawlee/fs-storage-native-linux-x64-musl@0.2.1-beta.0': + resolution: {integrity: sha512-T/9ZPPt6eGm3qUwhnEy7SGbilH9Bvku+afdsXxIoDDza5Uib7PnwFTjI+stnY4Y5xQ+GAcmfwkRRM3oZ2UK6ZA==} engines: {node: '>= 20'} cpu: [x64] os: [linux] libc: [musl] - '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.0': - resolution: {integrity: sha512-UPX2/G67OmUzqAnhKnUjMwxxhJzrmUEug8gSgIGLqYAV/B6wsCU7e5a59Ekc4vroVSZxmBJ3ahNu6lXL7oMhBA==} + '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.1-beta.0': + resolution: {integrity: sha512-p8D+XSskEZ2FMGXlrzVQrkFozkS3Md6invEvTiWEPC6/Z6mvNbwJ9z8NVSRNx4Trwb1l85i0+rLJI9OqzYOa4Q==} engines: {node: '>= 20'} cpu: [arm64] os: [win32] - '@crawlee/fs-storage-native-win32-x64-msvc@0.2.0': - resolution: {integrity: sha512-jMafrxiPm/nU6rmW1PXVj5O5mnK8oz2nnf2aRaV6bpwWS9XJ7pHIJOo208o54pkNz0m7E6IKRaA248So8ei3KA==} + '@crawlee/fs-storage-native-win32-x64-msvc@0.2.1-beta.0': + resolution: {integrity: sha512-ChmVNKYjlskX9WKPsQumbF3H0MEU1MumfkZxou6FjFLtgQ14rt03qld8n1EUzQXBApyqg5RSQCKdDoTIiehmCA==} engines: {node: '>= 20'} cpu: [x64] os: [win32] - '@crawlee/fs-storage-native@0.2.0': - resolution: {integrity: sha512-fLFhUZBBvgY2KmKaOcvJK1QaiwxoDt9XGaCbgG3xGxIZJfZW1OS71FKjAdxIv9QjcRWtUxD1QwD9kFHlpb5P2g==} + '@crawlee/fs-storage-native@0.2.1-beta.0': + resolution: {integrity: sha512-SpQQj/4Jb7NT4mPL4e97loOixjg3Sb2UZPQUtZ3558I94BmKUh2+P35VKpzkMj3wnGU1QCcUOYIeNGPoH80Xgw==} engines: {node: '>= 20'} '@crawlee/types@3.16.0': @@ -3091,6 +3158,15 @@ packages: '@modelcontextprotocol/sdk': optional: true + '@grpc/grpc-js@1.14.4': + resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} + engines: {node: '>=12.10.0'} + + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + '@hapi/hoek@9.3.0': resolution: {integrity: sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==} @@ -3314,6 +3390,9 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@js-sdsl/ordered-map@4.4.2': + resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + '@jsonjoy.com/base64@1.1.2': resolution: {integrity: sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==} engines: {node: '>=10.0'} @@ -3468,6 +3547,21 @@ packages: '@mermaid-js/parser@1.2.0': resolution: {integrity: sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA==} + '@microsoft/api-extractor-model@7.33.11': + resolution: {integrity: sha512-iDu1AtuRC2Z8XVs2SieZieVavF1FXPBX1C9pMexJh8Dx/Dd/3ZZ4nRQp/PU2Vk2rUHWuBz1wOyL4DcuhVnBXwg==} + engines: {node: '>=20.9.0'} + + '@microsoft/api-extractor@7.59.0': + resolution: {integrity: sha512-KDYV8kSVjmG8JJWVg1f1aKxteNG1IQ44D07Rjhlcci8wlqrSFNhUfgv7dJhwT0W2h3NDIL5Vz2f+/DfO4OpDHw==} + engines: {node: '>=20.9.0'} + hasBin: true + + '@microsoft/tsdoc-config@0.18.1': + resolution: {integrity: sha512-9brPoVdfN9k9g0dcWkFeA7IH9bbcttzDJlXvkf8b2OBzd5MueR1V2wkKBL0abn0otvmkHJC6aapBOTJDDeMCZg==} + + '@microsoft/tsdoc@0.16.0': + resolution: {integrity: sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==} + '@modelcontextprotocol/sdk@1.29.0': resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} engines: {node: '>=18'} @@ -3746,10 +3840,210 @@ packages: '@open-draft/until@2.1.0': resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==} + '@opentelemetry/api-logs@0.210.0': + resolution: {integrity: sha512-CMtLxp+lYDriveZejpBND/2TmadrrhUfChyxzmkFtHaMDdSKfP59MAYyA0ICBvEBdm3iXwLcaj/8Ic/pnGw9Yg==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@opentelemetry/configuration@0.210.0': + resolution: {integrity: sha512-tM0ROS/hZM72kB55cSjDcghVcUXBJdGkGzpkhD7M1B/gpcvZPSGfjFgKN3dgmxNgF76NxtbUwv3ik0wS+Kz52g==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + + '@opentelemetry/context-async-hooks@2.10.0': + resolution: {integrity: sha512-bvyMcgLEkozzSzpEEEo1OMoeQ97bxj6Qs2uN3mPrSdDvObMI1myffD/BPqcLlzZO9//d1SqQA/WPw7Cz2AiqhA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/context-async-hooks@2.4.0': + resolution: {integrity: sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.10.0': + resolution: {integrity: sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.4.0': + resolution: {integrity: sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-logs-otlp-grpc@0.210.0': + resolution: {integrity: sha512-+BolenqOO6ow65go7uWRYPvvs/BBIWp1mtRn93VvGduqvMVH/IY8nXrt80a4L9hZ7lHi2Tq2/NcC3H2QzcWKag==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-logs-otlp-http@0.210.0': + resolution: {integrity: sha512-Q8/SEQtgrErbVVRg9M9iaG8m5wdPNdU0UOF7U43sAhwfmPG92ZOk/aenKhg0DXSNJHhkCDNCgS1kSoErAB3z0A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-logs-otlp-proto@0.210.0': + resolution: {integrity: sha512-Y/yPc+gDhsWB7AsNzQWxblw4ULbvhCycMaQ2aAn+HSAVbgbMiZa0SbclPVHSnpnNzKSLVavFjweAr0pQA1KKLg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-grpc@0.210.0': + resolution: {integrity: sha512-pWZ/Tjrqev9rdkqe8F6A9FGddLZrjl6iRAU5LBvvRL6I3PSgG8z1xM0cESAy1jzAF4wGohnAh8rB7hHzpUOYEA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-http@0.210.0': + resolution: {integrity: sha512-JpLThG8Hh8A/Jzdzw9i4Ftu+EzvLaX/LouN+mOOHmadL0iror0Qsi3QWzucXeiUsDDsiYgjfKyi09e6sltytgA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-proto@0.210.0': + resolution: {integrity: sha512-CFa7SOinYOVWIWJuQL7XFeyedzmFGIpHpSMNFE8Xefb6iGB4m+MukQecdssvPcJKYlfF5FpovEOLXwafAzsXWQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-prometheus@0.210.0': + resolution: {integrity: sha512-8i+7d70Hho6pcheTtbqIuS+bo+AIX/oNUTMwIEZoehUE4ZdbGmeVaE+hJS2LAErFeFaU71w164lAgYyMUEQ8zw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-grpc@0.210.0': + resolution: {integrity: sha512-1GPLOyxIfUX24WM8Oea+vx9d9TlewposUnsQXTjusxVMQ/dWvt5JIDJyTsfNDS412XRUOORgF97PwsfDY5QKGA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-http@0.210.0': + resolution: {integrity: sha512-9JkyaCl70anEtuKZdoCQmjDuz1/paEixY/DWfsvHt7PGKq3t8/nQ/6/xwxHjG+SkPAUbo1Iq4h7STe7Pk2bc5A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-proto@0.210.0': + resolution: {integrity: sha512-qVUY7Hsm/t5buGOtPcTV1Ch4W9kj2wGaQaAF5FO4XR8TMKl2GM45tUCnr0/1dF3wo4RG9khMxrddeQWdRL4fIg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-zipkin@2.4.0': + resolution: {integrity: sha512-qpiXY0TUEFjBBp9b1na9LfuVQw6W8LH+te7uv+CC+0Up78ZDtZZwOjK2M7CL7Nspnw+yS4JdgEA7oxsBu0Ctsg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/instrumentation@0.210.0': + resolution: {integrity: sha512-sLMhyHmW9katVaLUOKpfCnxSGhZq2t1ReWgwsu2cSgxmDVMB690H9TanuexanpFI94PJaokrqbp8u9KYZDUT5g==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.210.0': + resolution: {integrity: sha512-uk78DcZoBNHIm26h0oXc8Pizh4KDJ/y04N5k/UaI9J7xR7mL8QcMcYPQG9xxN7m8qotXOMDRW6qTAyptav4+3w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-grpc-exporter-base@0.210.0': + resolution: {integrity: sha512-fEJs8UhkFMrdXMOCLXyKd2uc6N209tIi8IBNqSTi83ri+MlMFrBKnOtklmv9/zzxovoN5zD1waRt6XBFGPfmIw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.210.0': + resolution: {integrity: sha512-nkHBJVSJGOwkRZl+BFIr7gikA93/U8XkL2EWaiDbj3DVjmTEZQpegIKk0lT8oqQYfP8FC6zWNjuTfkaBVqa0ZQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/propagator-b3@2.4.0': + resolution: {integrity: sha512-6VPsFiMUkJBre/86F0d+PZMaUCcuLA9DtZuC46KH8EeVEKZPEM2WlX35M/qmde8UpzoQL9qzdz54YjUYABt8Uw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/propagator-jaeger@2.4.0': + resolution: {integrity: sha512-t6muBL/3AMD++1EMF658C/KIpj3gfmTmftX3mEQql4KIxNGFvacCmmTtrQt9IZAJmQRfjQRCkv+vsGbQugeJIw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/resources@2.10.0': + resolution: {integrity: sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/resources@2.4.0': + resolution: {integrity: sha512-RWvGLj2lMDZd7M/5tjkI/2VHMpXebLgPKvBUd9LRasEWR2xAynDwEYZuLvY9P2NGG73HF07jbbgWX2C9oavcQg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.210.0': + resolution: {integrity: sha512-YuaL92Dpyk/Kc1o4e9XiaWWwiC0aBFN+4oy+6A9TP4UNJmRymPMEX10r6EMMFMD7V0hktiSig9cwWo59peeLCQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.4.0': + resolution: {integrity: sha512-qSbfq9mXbLMqmPEjijl32f3ZEmiHekebRggPdPjhHI6t1CsAQOR2Aw/SuTDftk3/l2aaPHpwP3xM2DkgBA1ANw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-node@0.210.0': + resolution: {integrity: sha512-KymqUtYvfpblDNgGxBXYqCcDjYXwjOF7Muc6ocs0rMlG/66Hcs9KiJ7hg4zLOv63JubF/vxi5WXaLrQrPKyaZQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.10.0': + resolution: {integrity: sha512-GuYQQT7QD2EeO8lcZLRQzcbOyhqAzL+6WWTKTU9mSUBYBazkEDl+VrQcXQhbB08OWM9anD1aHleVadzulpOaUQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.4.0': + resolution: {integrity: sha512-WH0xXkz/OHORDLKqaxcUZS0X+t1s7gGlumr2ebiEgNZQl2b0upK2cdoD0tatf7l8iP74woGJ/Kmxe82jdvcWRw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-node@2.10.0': + resolution: {integrity: sha512-GZK/G6oZyBLGlH1pUgeDch7D91KoHd2uotUGIkWCPi9GI5T9X0p4L7nNAMDR1BQjkRYoDqo+ddfVx9t5Uhys+Q==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace-node@2.4.0': + resolution: {integrity: sha512-MBc2l04hZPYygnWPT38UiOPy9ueutPqmJ47z0m9IKuoVQh3MblmbSgwspjhdHagZLfSfmlzhWR1xtbgVNmjX2A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace@2.10.0': + resolution: {integrity: sha512-MfQGq3GRmTh5fM/y+OjaO0vj6+luCB1XO2gfXCalKCfgKw0eHL++sm75DNweC6ohlp+aFvACqeE0fYayqdRaoQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.43.0': + resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} + engines: {node: '>=14'} + '@oxc-project/types@0.124.0': resolution: {integrity: sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==} @@ -4113,6 +4407,9 @@ packages: '@protobufjs/float@1.0.2': resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + '@protobufjs/inquire@1.1.2': + resolution: {integrity: sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==} + '@protobufjs/path@1.1.2': resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} @@ -4301,6 +4598,39 @@ packages: '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} + '@rushstack/node-core-library@5.24.0': + resolution: {integrity: sha512-g/Z47ZwARn/VkTnHcyJslrR26erRf1G5JbqPlfGZGBCrOcrEt8fTQXXDVhUDDNl/g/v3TXI1dNiAAT5FEks8BA==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + '@rushstack/problem-matcher@0.2.1': + resolution: {integrity: sha512-gulfhBs6n+I5b7DvjKRfhMGyUejtSgOHTclF/eONr8hcgF1APEDjhxIsfdUYYMzC3rvLwGluqLjbwCFZ8nxrog==} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + '@rushstack/rig-package@0.7.3': + resolution: {integrity: sha512-aAA518n6wxxjCfnTAOjQnm7ngNE0FVHxHAw2pxKlIhxrMn0XQjGcXKF0oKWpjBgJOmsaJpVob/v+zr3zxgPWuA==} + + '@rushstack/terminal@0.24.3': + resolution: {integrity: sha512-KxphDhPGC4xDrKg8O4yrWCEpPMi87aJc1iycXqMVh1dLgV0M34hiH9ZTgWjBRmcrT/OIHoOeWf48npcQFzgp+g==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + '@rushstack/ts-command-line@5.3.13': + resolution: {integrity: sha512-+jNbxhh8CkrZYxMk9X3GRYM2+Myr1xCCj+eHbJ9Vp+PCdNHS0TUl5QQFT6UbM/aTFRCzs5jiBTKYzBRpe5uYbQ==} + engines: {node: '>=20.9.0'} + '@sapphire/async-queue@1.5.5': resolution: {integrity: sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==} engines: {node: '>=v14.0.0', npm: '>=7.0.0'} @@ -4728,6 +5058,9 @@ packages: '@tybys/wasm-util@0.9.0': resolution: {integrity: sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==} + '@types/argparse@1.0.38': + resolution: {integrity: sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA==} + '@types/body-parser@1.19.6': resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} @@ -5491,6 +5824,11 @@ packages: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} + acorn-import-attributes@1.9.5: + resolution: {integrity: sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==} + peerDependencies: + acorn: ^8 + acorn-import-phases@1.0.4: resolution: {integrity: sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==} engines: {node: '>=10.13.0'} @@ -5544,6 +5882,14 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + ajv-draft-04@1.0.0: + resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} + peerDependencies: + ajv: ^8.5.0 + peerDependenciesMeta: + ajv: + optional: true + ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -5576,6 +5922,9 @@ packages: ajv@8.18.0: resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==} + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + algoliasearch-helper@3.28.1: resolution: {integrity: sha512-6iXpbkkrAI5HFpCWXlNmIDSBuoN/U1XnEvb2yJAoWfqrZ+DrybI7MQ5P5mthFaprmocq+zbi6HxnR28xnZAYBw==} peerDependencies: @@ -5645,6 +5994,9 @@ packages: arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -6169,6 +6521,9 @@ packages: resolution: {integrity: sha512-Mz9QMT5fJe7bKI7MH31UilT5cEK5EHHRCccw/YRFsRY47AuNgaV6HY3rscp0/I4Q+tTW/5zoqpSeRRI54TkDWA==} engines: {node: '>= 0.10'} + cjs-module-lexer@2.2.1: + resolution: {integrity: sha512-Ca8swihM+/4yKecYHY52kgJd300hi2lADU/a1RxNTRe+RJ9jvqQlESpbz9DnG9mowez8qwXHB8qYdIUw9e+F5Q==} + clean-css@5.3.3: resolution: {integrity: sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==} engines: {node: '>= 10.0'} @@ -7036,6 +7391,10 @@ packages: devtools-protocol@0.0.1666840: resolution: {integrity: sha512-gCcO42XCHKEs7Ag0S7aGYsnJ7hlgrO3qderYqeiY0Eqk+0GFfuvT13IA0hHreJTa2KCdDVyGMeOhdMNmrrTjVg==} + diff@8.0.4: + resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==} + engines: {node: '>=0.3.1'} + diffie-hellman@5.0.3: resolution: {integrity: sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==} @@ -8405,6 +8764,9 @@ packages: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} + import-in-the-middle@2.0.6: + resolution: {integrity: sha512-3vZV3jX0XRFW3EJDTwzWoZa+RH1b8eTTx6YOCjglrLyPuepwoBti1k3L2dKwdCUrnVEfc5CuRuGstaC/uQJJaw==} + import-lazy@4.0.0: resolution: {integrity: sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==} engines: {node: '>=8'} @@ -8830,6 +9192,9 @@ packages: resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} hasBin: true + jju@1.4.0: + resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==} + joi@17.13.4: resolution: {integrity: sha512-1RuuER6kmt8K8I3nIWvPZKi5RQCb568ZPyY4Pwjlua+yo+63ZTmIwxLZH0heBmiKN4uxjvCiarDrjaeH84xicQ==} @@ -9275,6 +9640,9 @@ packages: lodash-es@4.18.1: resolution: {integrity: sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==} + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} @@ -9807,6 +10175,9 @@ packages: resolution: {integrity: sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==} engines: {node: '>=0.10.0'} + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + mri@1.1.4: resolution: {integrity: sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==} engines: {node: '>=4'} @@ -11009,6 +11380,10 @@ packages: resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} engines: {node: '>=12.0.0'} + protobufjs@8.0.0: + resolution: {integrity: sha512-jx6+sE9h/UryaCZhsJWbJtTEy47yXoGNYI4z8ZaRncM0zBKeRqjO2JEcOUYwrYGb1WLhXM1FfMzW3annvFv0rw==} + engines: {node: '>=12.0.0'} + protocols@2.0.2: resolution: {integrity: sha512-hHVTzba3wboROl0/aWRRG9dMytgH6ow//STBZh43l/wQgmMhYhOFi0EHWAPtoCz9IAUymsyP0TSBHkhgMEGNnQ==} @@ -11415,6 +11790,10 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} + require-in-the-middle@8.0.1: + resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==} + engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'} + require-like@0.1.2: resolution: {integrity: sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==} @@ -11857,6 +12236,9 @@ packages: split@1.0.1: resolution: {integrity: sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + srcset@4.0.0: resolution: {integrity: sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw==} engines: {node: '>=12'} @@ -14269,13 +14651,13 @@ snapshots: transitivePeerDependencies: - encoding - '@browserbasehq/stagehand@3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3)': + '@browserbasehq/stagehand@3.0.7(@cfworker/json-schema@4.1.1)(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(supports-color@8.1.1)(zod@4.4.3)': dependencies: '@ai-sdk/provider': 2.0.1 '@anthropic-ai/sdk': 0.39.0(encoding@0.1.13) '@browserbasehq/sdk': 2.10.0(encoding@0.1.13) '@google/genai': 1.50.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.4.3))(bufferutil@4.1.0)(supports-color@8.1.1) - '@langchain/openai': 0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0)) + '@langchain/openai': 0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0)) '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.4.3) ai: 5.0.173(zod@4.4.3) deepmerge: 4.3.1 @@ -14302,7 +14684,7 @@ snapshots: '@ai-sdk/perplexity': 2.0.27(zod@4.4.3) '@ai-sdk/togetherai': 1.0.38(zod@4.4.3) '@ai-sdk/xai': 2.0.67(zod@4.4.3) - '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) + '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) bufferutil: 4.1.0 chrome-launcher: 1.2.1(supports-color@7.2.0) ollama-ai-provider-v2: 1.5.5(zod@4.4.3) @@ -14448,40 +14830,40 @@ snapshots: '@conventional-changelog/template@1.2.1': {} - '@crawlee/fs-storage-native-darwin-arm64@0.2.0': + '@crawlee/fs-storage-native-darwin-arm64@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native-darwin-x64@0.2.0': + '@crawlee/fs-storage-native-darwin-x64@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.0': + '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native-linux-arm64-musl@0.2.0': + '@crawlee/fs-storage-native-linux-arm64-musl@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native-linux-x64-gnu@0.2.0': + '@crawlee/fs-storage-native-linux-x64-gnu@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native-linux-x64-musl@0.2.0': + '@crawlee/fs-storage-native-linux-x64-musl@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.0': + '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native-win32-x64-msvc@0.2.0': + '@crawlee/fs-storage-native-win32-x64-msvc@0.2.1-beta.0': optional: true - '@crawlee/fs-storage-native@0.2.0': + '@crawlee/fs-storage-native@0.2.1-beta.0': optionalDependencies: - '@crawlee/fs-storage-native-darwin-arm64': 0.2.0 - '@crawlee/fs-storage-native-darwin-x64': 0.2.0 - '@crawlee/fs-storage-native-linux-arm64-gnu': 0.2.0 - '@crawlee/fs-storage-native-linux-arm64-musl': 0.2.0 - '@crawlee/fs-storage-native-linux-x64-gnu': 0.2.0 - '@crawlee/fs-storage-native-linux-x64-musl': 0.2.0 - '@crawlee/fs-storage-native-win32-arm64-msvc': 0.2.0 - '@crawlee/fs-storage-native-win32-x64-msvc': 0.2.0 + '@crawlee/fs-storage-native-darwin-arm64': 0.2.1-beta.0 + '@crawlee/fs-storage-native-darwin-x64': 0.2.1-beta.0 + '@crawlee/fs-storage-native-linux-arm64-gnu': 0.2.1-beta.0 + '@crawlee/fs-storage-native-linux-arm64-musl': 0.2.1-beta.0 + '@crawlee/fs-storage-native-linux-x64-gnu': 0.2.1-beta.0 + '@crawlee/fs-storage-native-linux-x64-musl': 0.2.1-beta.0 + '@crawlee/fs-storage-native-win32-arm64-msvc': 0.2.1-beta.0 + '@crawlee/fs-storage-native-win32-x64-msvc': 0.2.1-beta.0 '@crawlee/types@3.16.0': dependencies: @@ -15827,6 +16209,18 @@ snapshots: - supports-color - utf-8-validate + '@grpc/grpc-js@1.14.4': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.5 + yargs: 17.7.2 + '@hapi/hoek@9.3.0': {} '@hapi/topo@5.1.0': @@ -16039,6 +16433,8 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@js-sdsl/ordered-map@4.4.2': {} + '@jsonjoy.com/base64@1.1.2(tslib@2.8.1)': dependencies: tslib: 2.8.1 @@ -16168,14 +16564,14 @@ snapshots: '@keyv/serialize@1.1.1': {} - '@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3))': + '@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3))': dependencies: '@cfworker/json-schema': 4.1.1 ansi-styles: 5.2.0 camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.21 - langsmith: 0.3.87(@opentelemetry/api@1.9.0)(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) + langsmith: 0.3.87(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) mustache: 4.2.0 p-queue: 6.6.2 p-retry: 4.6.2 @@ -16188,9 +16584,9 @@ snapshots: - '@opentelemetry/sdk-trace-base' - openai - '@langchain/openai@0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))': + '@langchain/openai@0.4.9(@langchain/core@0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)))(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))': dependencies: - '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) + '@langchain/core': 0.3.80(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)) js-tiktoken: 1.0.21 openai: 4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@3.25.76) zod: 3.25.76 @@ -16247,6 +16643,41 @@ snapshots: dependencies: '@chevrotain/types': 11.1.2 + '@microsoft/api-extractor-model@7.33.11(@types/node@24.12.2)': + dependencies: + '@microsoft/tsdoc': 0.16.0 + '@microsoft/tsdoc-config': 0.18.1 + '@rushstack/node-core-library': 5.24.0(@types/node@24.12.2) + transitivePeerDependencies: + - '@types/node' + + '@microsoft/api-extractor@7.59.0(@types/node@24.12.2)': + dependencies: + '@microsoft/api-extractor-model': 7.33.11(@types/node@24.12.2) + '@microsoft/tsdoc': 0.16.0 + '@microsoft/tsdoc-config': 0.18.1 + '@rushstack/node-core-library': 5.24.0(@types/node@24.12.2) + '@rushstack/rig-package': 0.7.3 + '@rushstack/terminal': 0.24.3(@types/node@24.12.2) + '@rushstack/ts-command-line': 5.3.13(@types/node@24.12.2) + diff: 8.0.4 + minimatch: 9.0.9 + resolve: 1.22.12 + semver: 7.7.4 + source-map: 0.6.1 + typescript: 5.9.3 + transitivePeerDependencies: + - '@types/node' + + '@microsoft/tsdoc-config@0.18.1': + dependencies: + '@microsoft/tsdoc': 0.16.0 + ajv: 8.18.0 + jju: 1.4.0 + resolve: 1.22.12 + + '@microsoft/tsdoc@0.16.0': {} + '@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.4.3)': dependencies: '@hono/node-server': 1.19.17(hono@4.13.3) @@ -16613,8 +17044,287 @@ snapshots: '@open-draft/until@2.1.0': {} + '@opentelemetry/api-logs@0.210.0': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api@1.9.0': {} + '@opentelemetry/configuration@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + yaml: 2.9.0 + + '@opentelemetry/context-async-hooks@2.10.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + + '@opentelemetry/context-async-hooks@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + + '@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/core@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/exporter-logs-otlp-grpc@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-logs-otlp-http@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.210.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-logs-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.210.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-metrics-otlp-grpc@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-metrics-otlp-http@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-metrics-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-prometheus@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-trace-otlp-grpc@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-trace-otlp-http@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/exporter-zipkin@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/instrumentation@0.210.0(@opentelemetry/api@1.9.0)(supports-color@7.2.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.210.0 + import-in-the-middle: 2.0.6 + require-in-the-middle: 8.0.1(supports-color@7.2.0) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation@0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.210.0 + import-in-the-middle: 2.0.6 + require-in-the-middle: 8.0.1(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/otlp-exporter-base@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/otlp-grpc-exporter-base@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/otlp-transformer@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.210.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + protobufjs: 8.0.0 + + '@opentelemetry/propagator-b3@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/propagator-jaeger@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/resources@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-logs@0.210.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.210.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/sdk-metrics@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/sdk-node@0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.210.0 + '@opentelemetry/configuration': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/context-async-hooks': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-logs-otlp-grpc': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-logs-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-logs-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-grpc': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-prometheus': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-grpc': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-http': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-zipkin': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/instrumentation': 0.210.0(@opentelemetry/api@1.9.0)(supports-color@8.1.1) + '@opentelemetry/propagator-b3': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/propagator-jaeger': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-node': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-trace-base@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-trace-node@2.10.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/context-async-hooks': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.10.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/sdk-trace-node@2.4.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/context-async-hooks': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/sdk-trace@2.10.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/semantic-conventions@1.43.0': {} + '@oxc-project/types@0.124.0': {} '@oxfmt/binding-android-arm-eabi@0.46.0': @@ -16883,6 +17593,8 @@ snapshots: '@protobufjs/float@1.0.2': {} + '@protobufjs/inquire@1.1.2': {} + '@protobufjs/path@1.1.2': {} '@protobufjs/pool@1.1.0': {} @@ -17000,6 +17712,45 @@ snapshots: '@rtsao/scc@1.1.0': {} + '@rushstack/node-core-library@5.24.0(@types/node@24.12.2)': + dependencies: + ajv: 8.20.0 + ajv-draft-04: 1.0.0(ajv@8.20.0) + ajv-formats: 3.0.1(ajv@8.20.0) + fs-extra: 11.3.4 + import-lazy: 4.0.0 + jju: 1.4.0 + resolve: 1.22.12 + semver: 7.7.4 + optionalDependencies: + '@types/node': 24.12.2 + + '@rushstack/problem-matcher@0.2.1(@types/node@24.12.2)': + optionalDependencies: + '@types/node': 24.12.2 + + '@rushstack/rig-package@0.7.3': + dependencies: + jju: 1.4.0 + resolve: 1.22.12 + + '@rushstack/terminal@0.24.3(@types/node@24.12.2)': + dependencies: + '@rushstack/node-core-library': 5.24.0(@types/node@24.12.2) + '@rushstack/problem-matcher': 0.2.1(@types/node@24.12.2) + supports-color: 8.1.1 + optionalDependencies: + '@types/node': 24.12.2 + + '@rushstack/ts-command-line@5.3.13(@types/node@24.12.2)': + dependencies: + '@rushstack/terminal': 0.24.3(@types/node@24.12.2) + '@types/argparse': 1.0.38 + argparse: 1.0.10 + string-argv: 0.3.2 + transitivePeerDependencies: + - '@types/node' + '@sapphire/async-queue@1.5.5': {} '@sec-ant/readable-stream@0.4.1': {} @@ -17395,6 +18146,8 @@ snapshots: dependencies: tslib: 2.8.1 + '@types/argparse@1.0.38': {} + '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 @@ -18180,6 +18933,10 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 + acorn-import-attributes@1.9.5(acorn@8.16.0): + dependencies: + acorn: 8.16.0 + acorn-import-phases@1.0.4(acorn@8.16.0): dependencies: acorn: 8.16.0 @@ -18231,6 +18988,10 @@ snapshots: '@opentelemetry/api': 1.9.0 zod: 4.4.3 + ajv-draft-04@1.0.0(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + ajv-formats@2.1.1(ajv@8.18.0): optionalDependencies: ajv: 8.18.0 @@ -18239,6 +19000,10 @@ snapshots: optionalDependencies: ajv: 8.18.0 + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + ajv-keywords@3.5.2(ajv@6.14.0): dependencies: ajv: 6.14.0 @@ -18262,6 +19027,13 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.5 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + algoliasearch-helper@3.28.1(algoliasearch@5.50.1): dependencies: '@algolia/events': 4.0.1 @@ -18401,6 +19173,10 @@ snapshots: arg@5.0.2: {} + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + argparse@2.0.1: {} args@5.0.3: @@ -19111,6 +19887,8 @@ snapshots: safe-buffer: 5.2.1 to-buffer: 1.2.2 + cjs-module-lexer@2.2.1: {} + clean-css@5.3.3: dependencies: source-map: 0.6.1 @@ -20047,6 +20825,8 @@ snapshots: devtools-protocol@0.0.1666840: {} + diff@8.0.4: {} + diffie-hellman@5.0.3: dependencies: bn.js: 4.12.3 @@ -22005,6 +22785,13 @@ snapshots: parent-module: 1.0.1 resolve-from: 4.0.0 + import-in-the-middle@2.0.6: + dependencies: + acorn: 8.16.0 + acorn-import-attributes: 1.9.5(acorn@8.16.0) + cjs-module-lexer: 2.2.1 + module-details-from-path: 1.0.4 + import-lazy@4.0.0: {} import-local@3.1.0: @@ -22387,6 +23174,8 @@ snapshots: jiti@2.6.1: {} + jju@1.4.0: {} + joi@17.13.4: dependencies: '@hapi/hoek': 9.3.0 @@ -22564,7 +23353,7 @@ snapshots: kuler@2.0.0: {} - langsmith@0.3.87(@opentelemetry/api@1.9.0)(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)): + langsmith@0.3.87(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.0))(openai@4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3)): dependencies: '@types/uuid': 10.0.0 chalk: 4.1.2 @@ -22574,6 +23363,8 @@ snapshots: uuid: 10.0.0 optionalDependencies: '@opentelemetry/api': 1.9.0 + '@opentelemetry/exporter-trace-otlp-proto': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.10.0(@opentelemetry/api@1.9.0) openai: 4.104.0(encoding@0.1.13)(ws@8.21.3(bufferutil@4.1.0))(zod@4.4.3) language-subtag-registry@0.3.23: {} @@ -22896,6 +23687,8 @@ snapshots: lodash-es@4.18.1: {} + lodash.camelcase@4.3.0: {} + lodash.debounce@4.0.8: {} lodash.isequal@4.5.0: {} @@ -23772,6 +24565,8 @@ snapshots: modify-values@1.0.1: {} + module-details-from-path@1.0.4: {} + mri@1.1.4: {} mrmime@2.0.1: {} @@ -25307,6 +26102,21 @@ snapshots: '@types/node': 24.12.2 long: 5.3.2 + protobufjs@8.0.0: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 24.12.2 + long: 5.3.2 + protocols@2.0.2: {} proxy-addr@2.0.7: @@ -25882,6 +26692,20 @@ snapshots: require-from-string@2.0.2: {} + require-in-the-middle@8.0.1(supports-color@7.2.0): + dependencies: + debug: 4.4.3(supports-color@7.2.0) + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + + require-in-the-middle@8.0.1(supports-color@8.1.1): + dependencies: + debug: 4.4.3(supports-color@8.1.1) + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + require-like@0.1.2: {} requires-port@1.0.0: {} @@ -26467,6 +27291,8 @@ snapshots: dependencies: through: 2.3.8 + sprintf-js@1.0.3: {} + srcset@4.0.0: {} ssri@12.0.0: diff --git a/test/core/storages/key_value_store_public_url_transaction.test.ts b/test/core/storages/key_value_store_public_url_transaction.test.ts new file mode 100644 index 000000000000..80cffcaab733 --- /dev/null +++ b/test/core/storages/key_value_store_public_url_transaction.test.ts @@ -0,0 +1,127 @@ +import { readFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { KeyValueStore, MemoryStorageBackend, serviceLocator, withStorageTransaction } from '@crawlee/core'; +import { FileSystemStorageBackend } from '@crawlee/fs-storage'; +import { ensureDir, rm } from 'fs-extra'; + +import { cryptoRandomObjectId } from '@apify/utilities'; + +/** + * `getPublicUrl()` derives the URL from the key without checking that the record exists, so a record + * written earlier in an uncommitted transaction — buffered in the journal, absent from the backend — + * already resolves to the URL it will have after commit (apify/crawlee#4075). + * + * Each test asserts that the in-transaction URL equals the committed one *and* that its path is the + * value file on disk, which is what makes the derived URL more than a plausible-looking string. + */ +describe('KeyValueStore.getPublicUrl() inside a storage transaction (fs-storage)', () => { + // A fresh directory per test keeps the suite order-independent. + let localStorageDir: string; + + beforeEach(async () => { + serviceLocator.reset(); + localStorageDir = resolve(import.meta.dirname, '..', 'tmp', 'fs-kvs-public-url-txn', cryptoRandomObjectId(10)); + await ensureDir(localStorageDir); + serviceLocator.setStorageBackend(new FileSystemStorageBackend({ localDataDirectory: localStorageDir })); + }); + + afterEach(async () => { + serviceLocator.getStorageInstanceManager().clearCache(); + await rm(localStorageDir, { force: true, recursive: true }); + }); + + // The storage directory is spliced into the URL unencoded and the file is named by the *encoded* + // key, so the URL's path is a literal filesystem path — not a percent-decoded one. + const pathOf = (url: string) => url.slice('file://'.length); + + const writeInTransaction = async (store: KeyValueStore, key: string) => { + let urlInside: string | undefined; + await withStorageTransaction(async () => { + await store.setValue(key, { hello: 'world' }); + urlInside = await store.getPublicUrl(key); + }); + + const urlAfterCommit = await store.getPublicUrl(key); + expect(urlAfterCommit).toBeDefined(); + expect(urlInside).toBe(urlAfterCommit); + + return urlAfterCommit!; + }; + + test('returns the committed URL for a record written earlier in the same transaction', async () => { + const store = await KeyValueStore.open(); + const url = await writeInTransaction(store, 'record'); + + expect(JSON.parse(await readFile(pathOf(url), 'utf8'))).toStrictEqual({ hello: 'world' }); + }); + + // A key with characters `encodeURIComponent` leaves alone (`!'()`) exercises the extra encoding pass; + // a plain alphanumeric key would not. + test('encodes a special-character key the way the value file is named', async () => { + const store = await KeyValueStore.open(); + const url = await writeInTransaction(store, "re-cord_1.v2!'()"); + + expect(url).toContain('re-cord_1.v2%21%27%28%29'); + expect(JSON.parse(await readFile(pathOf(url), 'utf8'))).toStrictEqual({ hello: 'world' }); + }); + + // The directory path is spliced in unencoded, so a space in it must be reproduced verbatim rather + // than percent-encoded. + test('resolves to the value file when the storage directory path contains a space', async () => { + serviceLocator.reset(); + const dirWithSpace = resolve(localStorageDir, 'with space'); + await ensureDir(dirWithSpace); + serviceLocator.setStorageBackend(new FileSystemStorageBackend({ localDataDirectory: dirWithSpace })); + + const store = await KeyValueStore.open(); + const url = await writeInTransaction(store, 'record'); + + expect(pathOf(url)).toContain('with space'); + expect(JSON.parse(await readFile(pathOf(url), 'utf8'))).toStrictEqual({ hello: 'world' }); + }); + + // The flip side of deriving URLs from keys: they never imply existence, not even for a record + // tombstoned in the current transaction. Callers that need existence ask `recordExists()`. + test('returns a URL for a record that does not exist', async () => { + const store = await KeyValueStore.open(); + await store.setValue('record', { hello: 'world' }); + + const committedUrl = await store.getPublicUrl('record'); + expect(await store.getPublicUrl('never-written')).toBeDefined(); + + await withStorageTransaction(async () => { + await store.setValue('record', null); + expect(await store.getPublicUrl('record')).toBe(committedUrl); + }); + + expect(await store.recordExists('record')).toBe(false); + expect(await store.getPublicUrl('record')).toBe(committedUrl); + }); +}); + +// A URL-less backend must stay URL-less: nothing about the key-derived contract makes the in-memory +// storage fabricate a file URL for a buffered record. +describe('KeyValueStore.getPublicUrl() inside a storage transaction (memory-storage)', () => { + beforeEach(() => { + serviceLocator.reset(); + serviceLocator.setStorageBackend(new MemoryStorageBackend()); + }); + + afterEach(() => { + serviceLocator.getStorageInstanceManager().clearCache(); + }); + + test('returns undefined for a record buffered in a transaction on a URL-less backend', async () => { + const store = await KeyValueStore.open(); + + let urlInside: string | undefined = 'sentinel'; + await withStorageTransaction(async () => { + await store.setValue('record', { hello: 'world' }); + urlInside = await store.getPublicUrl('record'); + }); + + expect(urlInside).toBeUndefined(); + expect(await store.getPublicUrl('record')).toBeUndefined(); + }); +}); diff --git a/test/otel/fixtures/crawler.ts b/test/otel/fixtures/crawler.ts new file mode 100644 index 000000000000..aea983825143 --- /dev/null +++ b/test/otel/fixtures/crawler.ts @@ -0,0 +1,14 @@ +import { CheerioCrawler } from '@crawlee/cheerio'; +import { MemoryStorageBackend, serviceLocator } from '@crawlee/core'; + +import log from '@apify/log'; + +log.setLevel(log.LEVELS.OFF); +serviceLocator.setStorageBackend(new MemoryStorageBackend()); + +const crawler = new CheerioCrawler({ + maxRequestRetries: 0, + requestHandler: async () => {}, +}); + +await crawler.run([process.env.CRAWLEE_OTEL_SMOKE_URL!]); diff --git a/test/otel/fixtures/otel-setup.ts b/test/otel/fixtures/otel-setup.ts new file mode 100644 index 000000000000..35a62c3ed6c5 --- /dev/null +++ b/test/otel/fixtures/otel-setup.ts @@ -0,0 +1,23 @@ +import { writeFileSync } from 'node:fs'; + +import { CrawleeInstrumentation } from '@crawlee/otel'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; + +const exporter = new InMemorySpanExporter(); + +const sdk = new NodeSDK({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + instrumentations: [new CrawleeInstrumentation({ logInstrumentation: false })], +}); + +sdk.start(); + +// `SimpleSpanProcessor` hands every span over as it ends, so by the time the process is on its way out the exporter +// holds everything. Written from `exit` so the crawler script itself stays a plain crawler, exactly like the guide's. +process.on('exit', () => { + const output = process.env.CRAWLEE_OTEL_SMOKE_OUTPUT; + if (output) { + writeFileSync(output, JSON.stringify(exporter.getFinishedSpans().map((span) => span.name))); + } +}); diff --git a/test/otel/fixtures/register-hook.ts b/test/otel/fixtures/register-hook.ts new file mode 100644 index 000000000000..e6788e175f7a --- /dev/null +++ b/test/otel/fixtures/register-hook.ts @@ -0,0 +1,6 @@ +import { register } from 'node:module'; +import { pathToFileURL } from 'node:url'; + +// Identical to the guide's own hook file: this is the delivery mechanism the automatic instrumentation depends on, +// and the only reason this fixture exists as a separate preload is that ESM imports are hoisted. +register('@opentelemetry/instrumentation/hook.mjs', pathToFileURL('./')); diff --git a/test/otel/hook-delivery.test.ts b/test/otel/hook-delivery.test.ts new file mode 100644 index 000000000000..885f07721062 --- /dev/null +++ b/test/otel/hook-delivery.test.ts @@ -0,0 +1,68 @@ +import { execFile } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import type { Server } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; + +import { runExampleComServer } from '../shared/_helper.js'; + +/** + * Runs the instrumentation the way a user does: through Node's module hook, in a process that was never told about + * the instrumentation in its own code. + * + * The other tests in this directory apply the module patches by hand, which covers what the patches do but not + * whether they are ever delivered. Delivery has its own failure mode: the hook has to be registered before anything + * imports Crawlee, and nothing inside the crawler's own process can tell that it was not. + * + * The fixtures are deliberately the same three files the guide tells users to write, so this also fails if the guide's + * setup stops working. + */ +describe('module hook delivery', () => { + const root = resolve(__dirname, '../..'); + const fixture = (name: string) => `./${join('test/otel/fixtures', name)}`; + + let server: Server; + let serverAddress: string; + let outputDir: string; + + beforeAll(async () => { + const [startedServer, port] = await runExampleComServer(); + server = startedServer; + serverAddress = `http://localhost:${port}`; + outputDir = mkdtempSync(join(tmpdir(), 'crawlee-otel-')); + }); + + afterAll(async () => { + rmSync(outputDir, { recursive: true, force: true }); + await new Promise((done) => server.close(done)); + }); + + test('instruments a crawler that only ever imports Crawlee', async () => { + const output = join(outputDir, 'spans.json'); + + await promisify(execFile)( + join(root, 'node_modules/.bin/tsx'), + ['--import', fixture('register-hook.ts'), '--import', fixture('otel-setup.ts'), fixture('crawler.ts')], + { + cwd: root, + env: { + ...process.env, + CRAWLEE_OTEL_SMOKE_OUTPUT: output, + CRAWLEE_OTEL_SMOKE_URL: serverAddress, + }, + }, + ); + + const spans = JSON.parse(readFileSync(output, 'utf8')) as string[]; + + // One span from each patched module that a `CheerioCrawler` run reaches, so a hook that never fires + // shows up as an empty list rather than as a silently smaller trace. + expect(spans.sort()).toEqual([ + 'crawlee.crawler.handleRequest', + 'crawlee.crawler.run', + 'crawlee.crawler.runRequestHandler', + 'crawlee.http.makeHttpRequest', + ]); + }, 180_000); // A cold child process has to load and transpile the whole crawler dependency graph. +}); diff --git a/test/otel/instrumentation.test.ts b/test/otel/instrumentation.test.ts new file mode 100644 index 000000000000..04cce244324c --- /dev/null +++ b/test/otel/instrumentation.test.ts @@ -0,0 +1,192 @@ +import type { Server } from 'node:http'; + +import * as basicModule from '@crawlee/basic'; +import { CheerioCrawler } from '@crawlee/cheerio'; +import { MemoryStorageBackend, serviceLocator } from '@crawlee/core'; +import * as httpModule from '@crawlee/http'; +import { CrawleeInstrumentation } from '@crawlee/otel'; +import type { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { ATTR_CODE_FUNCTION_NAME, ATTR_HTTP_REQUEST_METHOD, ATTR_URL_FULL } from '@opentelemetry/semantic-conventions'; + +import log from '@apify/log'; + +import { runExampleComServer } from '../shared/_helper.js'; + +/** + * Drives a real crawler through the real instrumented Crawlee classes. + * + * The module patches are applied directly rather than through Node's module hook, which would otherwise have to be + * registered before Vitest loads any Crawlee module. Only the delivery of the patch is bypassed - everything the + * instrumentation itself does (which prototypes get wrapped, the span tree, the attributes, error handling, context + * propagation across the crawler's async boundaries) is exercised here. The hook path is covered by running the + * guide's own examples; see docs/guides/trace-and-monitor-crawlers.mdx. + */ +interface PatchableModuleDefinition { + name: string; + patch: (moduleExports: unknown) => unknown; + unpatch: (moduleExports: unknown) => unknown; +} + +function moduleDefinition(instrumentation: CrawleeInstrumentation, moduleName: string): PatchableModuleDefinition { + const definitions = (instrumentation as any).init() as PatchableModuleDefinition[]; + const definition = definitions.find((d) => d.name === moduleName); + + if (!definition) { + throw new Error(`No instrumentation definition for ${moduleName}`); + } + + return definition; +} + +describe('CrawleeInstrumentation against a real crawler', () => { + let server: Server; + let serverAddress: string; + let logLevel: number; + + let exporter: InMemorySpanExporter; + let provider: NodeTracerProvider; + let patched: { definition: PatchableModuleDefinition; moduleExports: unknown }[]; + + beforeAll(async () => { + logLevel = log.getLevel(); + // Two of these tests fail requests on purpose; silence the crawler's own error reporting. + log.setLevel(log.LEVELS.OFF); + + const [startedServer, port] = await runExampleComServer(); + server = startedServer; + serverAddress = `http://localhost:${port}`; + + exporter = new InMemorySpanExporter(); + provider = new NodeTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + provider.register(); + + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + instrumentation.setTracerProvider(provider); + patched = [ + { definition: moduleDefinition(instrumentation, '@crawlee/basic'), moduleExports: basicModule }, + { definition: moduleDefinition(instrumentation, '@crawlee/http'), moduleExports: httpModule }, + ]; + + for (const { definition, moduleExports } of patched) { + definition.patch(moduleExports); + } + }); + + afterAll(async () => { + for (const { definition, moduleExports } of patched) { + definition.unpatch(moduleExports); + } + + await provider.shutdown(); + server.close(); + log.setLevel(logLevel); + }); + + beforeEach(() => { + exporter.reset(); + serviceLocator.setStorageBackend(new MemoryStorageBackend()); + }); + + const byName = (spans: ReadableSpan[], name: string) => spans.filter((s) => s.name === name); + const one = (spans: ReadableSpan[], name: string) => { + const found = byName(spans, name); + expect(found, `expected exactly one ${name} span, got ${found.length}`).toHaveLength(1); + return found[0]; + }; + + test('produces a nested span tree for a successful request', async () => { + let seenUrl: string | undefined; + + const crawler = new CheerioCrawler({ + maxRequestsPerCrawl: 1, + async requestHandler({ $, request }) { + expect($('title').text()).toBe('Example Domain'); + seenUrl = request.url; + }, + }); + + await crawler.run([serverAddress]); + + const spans = exporter.getFinishedSpans(); + + expect(spans.length).toBeGreaterThan(0); + expect(new Set(spans.map((s) => (s.instrumentationScope ?? (s as any).instrumentationLibrary)?.name))).toEqual( + new Set(['@crawlee/otel']), + ); + + const run = one(spans, 'crawlee.crawler.run'); + const handleRequest = one(spans, 'crawlee.crawler.handleRequest'); + const requestHandler = one(spans, 'crawlee.crawler.runRequestHandler'); + const httpRequest = one(spans, 'crawlee.http.makeHttpRequest'); + + // A single trace, correctly nested - this only holds if the context survives the crawler's async boundaries. + expect(new Set(spans.map((s) => s.spanContext().traceId)).size).toBe(1); + expect(handleRequest.parentSpanContext?.spanId).toBe(run.spanContext().spanId); + expect(requestHandler.parentSpanContext?.spanId).toBe(handleRequest.spanContext().spanId); + + // Nothing failed, so the instrumentation leaves the status unset. + expect(requestHandler.status.code).toBe(0); + + expect(run.attributes['crawlee.crawler.type']).toBe('CheerioCrawler'); + expect(seenUrl).toBeDefined(); + expect(requestHandler.attributes).toMatchObject({ + [ATTR_URL_FULL]: seenUrl, + [ATTR_HTTP_REQUEST_METHOD]: 'GET', + [ATTR_CODE_FUNCTION_NAME]: 'BasicCrawler.runRequestHandler', + 'crawlee.request.retry_count': 0, + }); + expect(requestHandler.attributes['crawlee.request.id']).toEqual(expect.any(String)); + expect(httpRequest.attributes[ATTR_URL_FULL]).toBe(seenUrl); + }); + + test('records the failure on the request handler span and on the error handlers', async () => { + let seenUrl: string | undefined; + + const crawler = new CheerioCrawler({ + maxRequestsPerCrawl: 1, + maxRequestRetries: 0, + requestHandler({ request }) { + seenUrl = request.url; + throw new Error('handler exploded'); + }, + }); + + await crawler.run([serverAddress]); + + const spans = exporter.getFinishedSpans(); + + const requestHandler = one(spans, 'crawlee.crawler.runRequestHandler'); + expect(requestHandler.status.code).toBe(2); // SpanStatusCode.ERROR + expect(requestHandler.status.message).toBe('handler exploded'); + expect(requestHandler.events.map((e) => e.name)).toContain('exception'); + + // Both error handling methods are instrumented, and both report which request failed. + const errorHandler = one(spans, 'crawlee.crawler.requestFunctionErrorHandler'); + const failedHandler = one(spans, 'crawlee.crawler.handleFailedRequestHandler'); + expect(seenUrl).toBeDefined(); + expect(errorHandler.attributes[ATTR_URL_FULL]).toBe(seenUrl); + expect(failedHandler.attributes[ATTR_URL_FULL]).toBe(seenUrl); + + expect(requestHandler.spanContext().traceId).toBe(errorHandler.spanContext().traceId); + }); + + test('increments the retry count attribute across attempts', async () => { + const crawler = new CheerioCrawler({ + maxRequestsPerCrawl: 1, + maxRequestRetries: 1, + requestHandler() { + throw new Error('always fails'); + }, + }); + + await crawler.run([serverAddress]); + + const retryCounts = byName(exporter.getFinishedSpans(), 'crawlee.crawler.runRequestHandler') + .map((s) => s.attributes['crawlee.request.retry_count']) + .sort(); + + expect(retryCounts).toEqual([0, 1]); + }); +}); diff --git a/test/otel/log-forwarding.test.ts b/test/otel/log-forwarding.test.ts new file mode 100644 index 000000000000..cf97e8638216 --- /dev/null +++ b/test/otel/log-forwarding.test.ts @@ -0,0 +1,93 @@ +import { BasicCrawler } from '@crawlee/basic'; +import * as coreModule from '@crawlee/core'; +import { CrawleeInstrumentation } from '@crawlee/otel'; +import { SeverityNumber } from '@opentelemetry/api-logs'; +import { InMemoryLogRecordExporter, LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs'; +import { ATTR_EXCEPTION_MESSAGE, ATTR_EXCEPTION_TYPE } from '@opentelemetry/semantic-conventions'; + +/** + * Forwards logs from the real `@crawlee/core` logger, patched the same way the module hook would patch it. + * + * The patch targets `BaseCrawleeLogger`, which every Crawlee logger derives from, so this also covers a Winston, Pino + * or hand-written adapter - only `logWithLevel` differs between them. + */ +describe('log forwarding against the real Crawlee logger', () => { + let exporter: InMemoryLogRecordExporter; + let loggerProvider: LoggerProvider; + let definition: { patch: (e: unknown) => unknown; unpatch: (e: unknown) => unknown }; + + beforeAll(() => { + exporter = new InMemoryLogRecordExporter(); + loggerProvider = new LoggerProvider({ processors: [new SimpleLogRecordProcessor(exporter)] }); + + const instrumentation = new CrawleeInstrumentation({ requestHandlingInstrumentation: false }); + instrumentation.setLoggerProvider(loggerProvider); + + definition = (instrumentation as any).init().find((d: any) => d.name === '@crawlee/core'); + definition.patch(coreModule); + }); + + afterAll(async () => { + definition.unpatch(coreModule); + await loggerProvider.shutdown(); + }); + + beforeEach(() => exporter.reset()); + + test('forwards a log made through the default Crawlee logger', () => { + const log = new coreModule.ApifyLogAdapter(coreModule.log); + + log.info('hello from crawlee', { page: 3 }); + + const records = exporter.getFinishedLogRecords(); + expect(records).toHaveLength(1); + expect(records[0].body).toBe('hello from crawlee'); + expect(records[0].severityNumber).toBe(SeverityNumber.INFO); + expect(records[0].severityText).toBe('INFO'); + expect(records[0].attributes).toMatchObject({ page: 3 }); + }); + + test('records an exception on the semantic convention attributes', () => { + const log = new coreModule.ApifyLogAdapter(coreModule.log); + + log.exception(new RangeError('out of range'), 'request failed'); + + const record = exporter.getFinishedLogRecords()[0]; + expect(record.severityNumber).toBe(SeverityNumber.ERROR); + expect(record.attributes[ATTR_EXCEPTION_TYPE]).toBe('RangeError'); + expect(record.attributes[ATTR_EXCEPTION_MESSAGE]).toBe('out of range'); + }); + + test('forwards a crawler status message', () => { + coreModule.serviceLocator.setStorageBackend(new coreModule.MemoryStorageBackend()); + const crawler = new BasicCrawler({ requestHandler: async () => {} }); + exporter.reset(); + + crawler.setStatusMessage('Crawled 40/100 pages, 2 failed.', { level: 'INFO' }); + + // `setStatusMessage` is the one place in Crawlee that logs at a level chosen at runtime. It used to reach for + // `logWithLevel`, which is abstract on `BaseCrawleeLogger` and so cannot be patched - the periodic status + // messages never reached OpenTelemetry at all. + const record = exporter.getFinishedLogRecords().at(-1)!; + expect(record.body).toBe('Crawled 40/100 pages, 2 failed.'); + expect(record.severityNumber).toBe(SeverityNumber.INFO); + }); + + test('maps each level onto the matching severity', () => { + const log = new coreModule.ApifyLogAdapter(coreModule.log); + + log.error('e'); + log.softFail('s'); + log.warning('w'); + log.debug('d'); + log.perf('p'); + + expect(exporter.getFinishedLogRecords().map((r) => r.severityNumber)).toEqual([ + SeverityNumber.ERROR, + SeverityNumber.WARN, + SeverityNumber.WARN, + SeverityNumber.DEBUG, + SeverityNumber.DEBUG, + ]); + }); +}); diff --git a/test/otel/patched-methods.test.ts b/test/otel/patched-methods.test.ts new file mode 100644 index 000000000000..191002a161ff --- /dev/null +++ b/test/otel/patched-methods.test.ts @@ -0,0 +1,52 @@ +import * as basicModule from '@crawlee/basic'; +import * as browserModule from '@crawlee/browser'; +import * as coreModule from '@crawlee/core'; +import * as httpModule from '@crawlee/http'; +import * as playwrightModule from '@crawlee/playwright'; + +import { loggerMethods, requestHandlingInstrumentationMethods } from '../../packages/otel/src/constants.js'; + +/** + * Asserts that every method the automatic instrumentation patches by name still exists on the real prototype. + * + * Most of them are TypeScript `private`, so renaming one is a routine refactor - and a rename is invisible at runtime: + * the instrumentation reports the missing method through `diag`, which is a no-op unless the application installed a + * diagnostic logger, and then carries on unpatched. The span or log record simply stops being recorded. + * + * This is the tripwire. It fails in the same pull request as the rename, and names the method that moved, so the + * instrumented methods can stay private instead of being promoted to `protected` for the instrumentation's sake. + */ +const instrumentedModules: Record> = { + '@crawlee/basic': basicModule, + '@crawlee/browser': browserModule, + '@crawlee/core': coreModule, + '@crawlee/http': httpModule, + '@crawlee/playwright': playwrightModule, +}; + +describe('the patched Crawlee methods still exist', () => { + test.each( + requestHandlingInstrumentationMethods.map( + (method) => [method.moduleName, method.className, method.methodName] as const, + ), + )('%s exports %s with a %s method', (moduleName, className, methodName) => { + const moduleExports = instrumentedModules[moduleName]; + expect(moduleExports, `${moduleName} is instrumented but not covered by this test`).toBeDefined(); + + const patchedClass = moduleExports[className] as { prototype?: Record } | undefined; + expect(patchedClass?.prototype, `${moduleName} no longer exports ${className}`).toBeDefined(); + expect(typeof patchedClass!.prototype![methodName], `${className}.${methodName} is no longer a method`).toBe( + 'function', + ); + }); + + test.each(loggerMethods.map((method) => [method.methodName] as const))( + '@crawlee/core exports BaseCrawleeLogger with a %s method', + (methodName) => { + const prototype = coreModule.BaseCrawleeLogger.prototype as unknown as Record; + expect(typeof prototype[methodName], `BaseCrawleeLogger.${methodName} is no longer a method`).toBe( + 'function', + ); + }, + ); +}); diff --git a/test/otel/tracer-wiring.test.ts b/test/otel/tracer-wiring.test.ts new file mode 100644 index 000000000000..b3360d700b04 --- /dev/null +++ b/test/otel/tracer-wiring.test.ts @@ -0,0 +1,66 @@ +import { CrawleeInstrumentation, wrapWithSpan } from '@crawlee/otel'; +import { registerInstrumentations } from '@opentelemetry/instrumentation'; +import type { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; + +// Reached directly rather than through the package entry point: resetting the shared tracer is not public API. +import { setSharedTracer } from '../../packages/otel/src/wrapWithSpan.js'; + +/** + * The instrumentation is always constructed before the SDK is configured, so these tests cover that ordering for + * both ways a provider reaches it: registered globally, or handed over explicitly. The explicit case used to emit + * nothing at all, because the tracer was taken from the global API in the constructor and that provider never gets + * a delegate when the SDK does not register itself globally. + */ +describe('tracer wiring', () => { + const scopeOf = (span: ReadableSpan) => (span.instrumentationScope ?? (span as any).instrumentationLibrary)?.name; + + // `wrapWithSpan` shares one tracer per process, so a test that asserts the fallback has to start from a clean one + // no matter which of these ran first. + beforeEach(() => setSharedTracer(undefined)); + + test('emits spans when the provider is registered globally', async () => { + const exporter = new InMemorySpanExporter(); + + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + void instrumentation; + + const provider = new NodeTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + provider.register(); + + try { + wrapWithSpan(() => 'value', { spanName: 'global-provider' })(); + + const spans = exporter.getFinishedSpans(); + expect(spans.map((s) => s.name)).toEqual(['global-provider']); + expect(scopeOf(spans[0])).toBe('@crawlee/otel'); + } finally { + await provider.shutdown(); + } + }); + + test('emits spans when the provider is passed to registerInstrumentations instead of registered', async () => { + const exporter = new InMemorySpanExporter(); + + // Constructed before the provider exists, and the provider is never registered globally. + const instrumentation = new CrawleeInstrumentation({ logInstrumentation: false }); + const provider = new NodeTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + + const unregister = registerInstrumentations({ + instrumentations: [instrumentation], + tracerProvider: provider, + }); + + try { + wrapWithSpan(() => 'value', { spanName: 'explicit-provider' })(); + + const spans = exporter.getFinishedSpans(); + expect(spans.map((s) => s.name)).toEqual(['explicit-provider']); + expect(scopeOf(spans[0])).toBe('@crawlee/otel'); + } finally { + unregister(); + await provider.shutdown(); + } + }); +}); diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index b9329311ff31..07ebaeae5b7f 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -20,6 +20,7 @@ const packages = [ 'types', 'impit-client', 'got-scraping-client', + 'otel', ]; const packagesOrder = [ '@crawlee/core', @@ -39,6 +40,7 @@ const packagesOrder = [ '@crawlee/types', '@crawlee/impit-client', '@crawlee/got-scraping-client', + '@crawlee/otel', ]; /** @type {Partial} */ diff --git a/website/sidebars.js b/website/sidebars.js index 7a3025f42e93..dbed1ce8132f 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -42,6 +42,7 @@ module.exports = { 'guides/proxy-management', 'guides/session-management', 'guides/scaling-crawlers', + 'guides/trace-and-monitor-crawlers', 'guides/avoid-blocking', 'guides/cookie-modals', 'guides/jsdom-crawler-guide', diff --git a/website/static/img/jaeger-search.png b/website/static/img/jaeger-search.png new file mode 100644 index 000000000000..a78636ea84b6 Binary files /dev/null and b/website/static/img/jaeger-search.png differ