From ef0b25a6b7b4cae246648846edd33b08433746a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Wed, 2 Sep 2026 13:37:05 +0200 Subject: [PATCH 1/2] chore: pin optionalDependencies in canary and release version pinning (#4098) The canary and `pin-versions` steps in `scripts/copy.ts` only rewrote `dependencies`. `@crawlee/impit-client` is an optional dep declared as `workspace:^`, so it went out as a floating `^4.0.0-beta.N` range. Because `rc` sorts above `beta` in semver, that range resolves to `4.0.0-rc.0` today, even when the user has `@crawlee/basic` pinned to an exact beta. Both steps now share one helper that also walks `optionalDependencies`, so optional internal deps get the same pinned version as everything else. --- scripts/copy.ts | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/scripts/copy.ts b/scripts/copy.ts index 3ee28f625ab2..013bb16ae55d 100644 --- a/scripts/copy.ts +++ b/scripts/copy.ts @@ -98,6 +98,18 @@ function getNextVersion() { return `${version}-${preid}.${lastPrereleaseNumber + 1}`; } +// optionalDependencies too, otherwise `workspace:^` on @crawlee/impit-client publishes as a floating `^` range +function pinInternalDeps(pkgJson: any, version: string): void { + for (const deps of [pkgJson.dependencies, pkgJson.optionalDependencies]) { + for (const dep of Object.keys(deps ?? {})) { + if ((dep.startsWith('@crawlee/') && dep !== '@crawlee/fs-storage-native') || dep === 'crawlee') { + const prefix = deps[dep].startsWith('^') ? '^' : ''; + deps[dep] = prefix + version; + } + } + } +} + // as we publish only the dist folder, we need to copy some meta files inside (readme/license/package.json) // also changes paths inside the copied `package.json` (`dist/index.js` -> `index.js`) const root = resolve(import.meta.dirname, '..'); @@ -109,12 +121,7 @@ if (options.canary) { const nextVersion = getNextVersion(); pkgJson.version = nextVersion; - for (const dep of Object.keys(pkgJson.dependencies)) { - if ((dep.startsWith('@crawlee/') && dep !== '@crawlee/fs-storage-native') || dep === 'crawlee') { - const prefix = pkgJson.dependencies[dep].startsWith('^') ? '^' : ''; - pkgJson.dependencies[dep] = prefix + nextVersion; - } - } + pinInternalDeps(pkgJson, nextVersion); console.info(`canary: setting version to ${nextVersion}`); @@ -125,11 +132,7 @@ if (options['pin-versions']) { const pkgJson = require(pkgPath); const version = getRootVersion(false); - for (const dep of Object.keys(pkgJson.dependencies ?? {})) { - if ((dep.startsWith('@crawlee/') && dep !== '@crawlee/fs-storage-native') || dep === 'crawlee') { - pkgJson.dependencies[dep] = version; - } - } + pinInternalDeps(pkgJson, version); console.info(`pin-versions: version ${version}`, pkgJson.dependencies); From 5656fed0f54cb4ac3ec91f82270fed25f96e1773 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 2 Sep 2026 17:12:13 +0200 Subject: [PATCH 2/2] fix!: Don't empty the request queue between run() calls (#4056) --- docs/guides/result_storage.mdx | 2 +- docs/public-api/crawlee-basic.api.md | 4 +- docs/public-api/crawlee-core.api.md | 2 + docs/upgrading/upgrading_v4.md | 39 +--- .../src/internals/basic-crawler.ts | 97 ++++----- packages/core/src/storages/dataset.ts | 10 + packages/core/src/storages/key_value_store.ts | 12 ++ test/core/crawlers/basic_crawler.test.ts | 188 ++++++------------ test/core/storages/dataset.test.ts | 14 ++ test/core/storages/key_value_store.test.ts | 19 ++ .../core/storages/storage_transaction.test.ts | 6 +- test/e2e/cheerio-stop-resume-ts/actor/main.ts | 2 +- test/e2e/cheerio-stop-resume-ts/test.mjs | 2 +- 13 files changed, 176 insertions(+), 221 deletions(-) diff --git a/docs/guides/result_storage.mdx b/docs/guides/result_storage.mdx index af17763e0fbc..cb7103ce52c9 100644 --- a/docs/guides/result_storage.mdx +++ b/docs/guides/result_storage.mdx @@ -172,7 +172,7 @@ The feature is escapable at three granularities: A few operations cannot be buffered, and silently letting them through would produce storage states that no rollback can undo. They throw inside a transaction, with an error pointing at `withDirectStorageAccess()`: -- `Dataset.drop()`, `KeyValueStore.drop()`, `RequestQueue.drop()` and `RequestQueue.purge()`, +- `drop()` and `purge()` on `Dataset`, `KeyValueStore` and `RequestQueue`, - the request queue processing internals (`fetchNextRequest()`, `markRequestAsHandled()`, `reclaimRequest()`), - `KeyValueStore.setValue()` with a **stream** value — a stream can only be consumed once, so it cannot serve both a read within the handler and the commit replay. Write streams under `withDirectStorageAccess()`. diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md index 9b543bd6752d..f5dd7d1d987a 100644 --- a/docs/public-api/crawlee-basic.api.md +++ b/docs/public-api/crawlee-basic.api.md @@ -73,8 +73,7 @@ export class BasicCrawler; // (undocumented) protected getRobotsTxtFileForUrl(url: string): Promise; - // (undocumented) - hasFinishedBefore: boolean; + get hasFinishedBefore(): boolean; // (undocumented) protected readonly httpClient: BaseHttpClient; protected init(): Promise; @@ -170,7 +169,6 @@ export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult { // @public (undocumented) export interface CrawlerRunOptions extends CrawlerAddRequestsOptions { - purgeRequestQueue?: boolean; } // @public diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md index f324e51f80b7..aab338a54dee 100644 --- a/docs/public-api/crawlee-core.api.md +++ b/docs/public-api/crawlee-core.api.md @@ -347,6 +347,7 @@ export class Dataset { // (undocumented) name?: string; static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise>; + purge(): Promise; pushData(data: Data | Data[]): Promise; reduce(iteratee: DatasetReducer): Promise; reduce(iteratee: DatasetReducer, memo: undefined, options: DatasetIteratorOptions): Promise; @@ -833,6 +834,7 @@ export class KeyValueStore { // (undocumented) readonly name?: string; static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise; + purge(): Promise; recordExists(key: string): Promise; static recordExists(key: string): Promise; setValue(key: string, value: T | null, options?: RecordOptions): Promise; diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md index 1156d1d17b0f..e151e40c28ec 100644 --- a/docs/upgrading/upgrading_v4.md +++ b/docs/upgrading/upgrading_v4.md @@ -21,7 +21,7 @@ This page summarizes the breaking changes in Crawlee v4. There are many, so the - **One concurrency budget for several crawlers.** The new [`ConcurrencySystem`](#autoscaling-moved-to-concurrencysystem) can be shared between crawlers, capping their combined concurrency instead of letting each one oversubscribe the host. - **Native `fetch` types.** HTTP clients and `context.response` now use the [standard `Response`](#crawlingcontextresponse-is-now-of-type-response), and `got-scraping` is an [opt-in dependency](#http-client-packages-and-basehttpclient-reshaped) instead of a mandatory one. - **The session is the rotation unit.** A session carries its proxy, cookies and error score, and is rotated as a whole when blocked — replacing [proxy tiers](#tieredproxyurls-is-removed-from-proxyconfiguration) and [session rotation counters](#maxsessionrotations-and-requestsessionrotationcount-are-removed). -- **Crawlers stop stepping on each other.** Multiple crawlers in one process [no longer share the default request queue](#multiple-crawler-instances-use-separate-default-request-queues), and repeated `run()` calls purge the queue instead of dropping and recreating it. +- **Crawlers stop stepping on each other.** Multiple crawlers in one process [no longer share the default request queue](#multiple-crawler-instances-use-separate-default-request-queues), and repeated `run()` calls [no longer empty it](#repeated-run-calls-no-longer-empty-the-request-queue) behind your back. - **Cookies behave.** `sendRequest` finally [respects your `Cookie` header](#cookie-handling-in-httpcrawler-and-sendrequest), and browser cookies set inside the handler are [persisted to the session](#browser-cookies-are-also-persisted-after-requesthandler). - **No half-written results.** Storage writes in a request handler are [transactional](#storage-writes-in-request-handlers-are-transactional) — a handler that throws leaves nothing behind, and its retry does not duplicate data. - **Simpler storage backend contract.** A custom storage backend is now [4 classes instead of 7](#storagebackend-interface-simplified). @@ -603,45 +603,28 @@ In v4, only the **first** crawler instance uses the default request queue. Each If you explicitly pass a `requestQueue` (or `requestManager`) to the crawler, that queue is used as-is regardless of instance order. -### Repeated `run()` calls use `purge()` instead of `drop()` + recreate +### Repeated `run()` calls no longer empty the request queue -When calling `crawler.run()` multiple times on the same crawler instance, v3 would drop the default request queue and create a fresh one between runs. In v4, the crawler **purges** the queue instead — clearing all requests and resetting internal counters, but keeping the same queue object. This is more efficient and avoids edge cases around stale references. +In v3, calling `crawler.run()` again on the same instance dropped the default request queue and created a fresh one, so the same URLs were crawled again — but only for a queue actually named `default`, which the Apify platform's default queue is not, so on the platform the second run silently crawled nothing. -The new `purge()` method is available on `RequestQueue` and is also defined as an optional method on the `IRequestManager` interface. +v4 does the same thing everywhere: nothing is emptied between runs. A repeated `run()` continues with the same request manager, and requests the previous run handled — a failed request counts as handled — are not processed again. Any crawl that ends up processing nothing while its request manager holds only handled requests warns and says why, instead of finishing silently; that also covers a second crawler sharing the queue, or a queue a previous process already worked through. -By default, only queues that the crawler created itself (the "owned" queue) are purged between runs — a user-supplied queue is never touched unless you explicitly opt in. The `purgeRequestQueue` option in `CrawlerRunOptions` controls this behavior: - -| `purgeRequestQueue` value | Owned queue (auto-created) | User-supplied queue | -|---|---|---| -| omitted (default) | Purged | Not purged | -| `true` | Purged | Purged | -| `false` | Not purged | Not purged | - -One combination has no sensible default: `sameDomainDelaySecs` over a request manager you supplied that does not pace on its own. The per-domain queues that have to be emptied are the crawler's, the manager underneath them is yours, and a purge cannot respect both — so a repeated `run()` throws and asks you to pass `purgeRequestQueue` explicitly rather than guessing. A manager that takes the delay as a floor has nothing of ours underneath it, and is left alone like any other supplied manager. +The `purgeRequestQueue` option of `crawler.run()` went away with the automatic purge. To crawl the same requests again, empty the queue yourself: ```typescript -// The purge happens automatically between run() calls: const crawler = new BasicCrawler({ requestHandler: async ({ request }) => { /* ... */ } }); await crawler.run(['https://example.com/a', 'https://example.com/b']); -// Queue is purged here, so the same URLs can be processed again: -await crawler.run(['https://example.com/a', 'https://example.com/c']); -``` -You can opt out of the automatic purge by passing `purgeRequestQueue: false`: +const queue = await crawler.getRequestQueue(); +await queue.purge?.(); -```typescript -await crawler.run(urls, { purgeRequestQueue: false }); +// The same URLs are crawled again: +await crawler.run(['https://example.com/a', 'https://example.com/c']); ``` -If you supplied your own `requestQueue` and want it purged between runs, pass `purgeRequestQueue: true` explicitly: +`purge()` — empty the storage, keep its id and name — is new in v4 and available on `Dataset`, `KeyValueStore` and `RequestQueue`, as well as being an optional method on the `IRequestManager` interface. -```typescript -const queue = await RequestQueue.open('my-queue'); -const crawler = new BasicCrawler({ requestQueue: queue, requestHandler: async () => { /* ... */ } }); -await crawler.run(['https://example.com/first']); -// Explicitly purge the user-supplied queue before the second run: -await crawler.run(['https://example.com/second'], { purgeRequestQueue: true }); -``` +This has nothing to do with `purgeOnStart` / `CRAWLEE_PURGE_ON_START`, which still wipes the default storages once per process before the first run. ### Storage `.open()` now also accepts `{ id?, name? }` diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 0ec4962a643b..8c5bdfd3ae2c 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -843,9 +843,14 @@ export class BasicCrawler< } running = false; - hasFinishedBefore = false; + #hasFinishedBefore = false; #unexpectedStop = false; + /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */ + get hasFinishedBefore(): boolean { + return this.#hasFinishedBefore; + } + #log!: CrawleeLogger; get log(): CrawleeLogger { @@ -859,16 +864,6 @@ export class BasicCrawler< protected readonly internalTimeoutMillis: number; readonly #maxRequestRetries: number; readonly #maxCrawlDepth?: number; - /** - * How much of {@apilink BasicCrawler.requestManager} the crawler may empty between repeated `run()` calls. - * - * - `all` — nothing under it came from the caller, so one `purge()` on the outside covers everything. - * - `none` — the caller supplied it and the crawler put nothing of its own inside. - * - `ambiguous` — the caller supplied it, but `sameDomainDelaySecs` put the crawler's own per-domain queues - * underneath: purging empties the caller's storage too, skipping leaves ours stale. A repeated `run()` asks - * rather than guessing. - */ - readonly #purgeableExtent: 'all' | 'none' | 'ambiguous'; readonly #maxRequestsPerCrawl?: number; private get handledRequestsCount(): number { @@ -1105,14 +1100,6 @@ export class BasicCrawler< const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken; - // Our per-domain queues under a manager the caller owns is the case with no right answer; a floor - // it took leaves nothing of ours behind. See `#purgeableExtent`. - if (suppliedManager === undefined) { - this.#purgeableExtent = 'all'; - } else { - this.#purgeableExtent = pacerNeeded ? 'ambiguous' : 'none'; - } - // Built here rather than at first use so it can sit *inside* the tandem below, which is where a // loader's transferred requests pass through it. const writableManager = pacerNeeded @@ -1695,8 +1682,12 @@ export class BasicCrawler< * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for * running {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} before {@apilink BasicCrawler.run|`crawler.run()`}. * + * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous + * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one + * if that is what you want. + * * @param [requests] The requests to add. - * @param [options] Options for the request queue. + * @param [options] Options for adding the initial requests. */ async run(requests?: TypedRequestsLike, options?: CrawlerRunOptions): Promise { // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's. @@ -1708,33 +1699,7 @@ export class BasicCrawler< ); } - const { purgeRequestQueue, ...addRequestsOptions } = options ?? {}; - - if (this.hasFinishedBefore) { - // When executing the run method for the second time explicitly, - // we need to purge the RQ to allow processing the same requests again — this is important so users can - // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and - // ignored — as a failed request is still handled. - // `purgeRequestQueue` unset purges only storage the crawler opened itself (see `#purgeableExtent`); - // `true` also purges a caller-supplied manager, `false` purges nothing. - if (purgeRequestQueue === undefined && this.#purgeableExtent === 'ambiguous') { - throw new Error( - 'Cannot decide what to purge before running again: `sameDomainDelaySecs` paces the request ' + - 'manager you supplied, so the per-domain queues that have to be emptied are the ' + - "crawler's while the manager underneath them is yours. Say which you want: " + - '`run(requests, { purgeRequestQueue: true })` empties both, `false` empties neither.', - ); - } - - if ( - purgeRequestQueue !== false && - (this.#purgeableExtent === 'all' || purgeRequestQueue === true) - ) { - // One call from the outside in reaches everything the manager wraps, a pacer's per-domain queues - // included. - await this.requestManager?.purge?.(); - } - + if (this.#hasFinishedBefore) { // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built. await this.#statisticsDep.ifOwned(async (stats) => { stats.reset(); @@ -1754,7 +1719,7 @@ export class BasicCrawler< }); if (requests) { - await this.addRequests(requests, addRequestsOptions); + await this.addRequests(requests, options); } try { @@ -1813,6 +1778,26 @@ export class BasicCrawler< }; this.log.info('Final request statistics:', stats as unknown as Record); + // A crawl that did nothing while the manager holds only handled requests is a mistake whoever + // handled them - this run, another crawler on the same queue, or a previous process. Starting + // against handled requests is not: that is what resuming a crawl looks like. + if (stats.requestsFinished + stats.requestsFailed === 0) { + // Never let the diagnostic itself break the run. + const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0; + + if (alreadyHandled > 0) { + this.log.warningOnce( + 'This crawl processed no requests - the request manager holds ' + + `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` + + 'already handled, and a failed request counts as handled too. Nothing ' + + 'empties a queue between runs, so to crawl them again, purge it ' + + '(`await queue.purge()`) or use a fresh one (e.g. ' + + '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' + + 'crawler instance.', + ); + } + } + if (this.statistics.errorTracker.total !== 0) { const prettify = ([count, info]: [number, string[]]) => `${count}x: ${info.at(-1)!.trim()} (${info[0]})`; @@ -1846,7 +1831,7 @@ export class BasicCrawler< ); this.running = false; - this.hasFinishedBefore = true; + this.#hasFinishedBefore = true; } return stats; @@ -3094,19 +3079,7 @@ export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, En export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {} -export interface CrawlerRunOptions extends CrawlerAddRequestsOptions { - /** - * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance. - * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again. - * - * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged. - * A user-supplied `requestQueue` is left untouched. - * - **`true`** — the queue is always purged, even if it was supplied by the user. - * - **`false`** — nothing is purged. Only genuinely new requests will be processed; - * note that even a failed request is considered handled. - */ - purgeRequestQueue?: boolean; -} +export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {} /** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */ function hostnameOrUrl(url: string): string { diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts index b2da8b4eb825..c8b35973b65b 100644 --- a/packages/core/src/storages/dataset.ts +++ b/packages/core/src/storages/dataset.ts @@ -783,6 +783,16 @@ export class Dataset { serviceLocator.getStorageInstanceManager().removeFromCache(this); } + /** + * Removes all items from the dataset but keeps the dataset itself, along with its + * {@apilink Dataset.id|`id`} and {@apilink Dataset.name|`name`}. + */ + async purge(): Promise { + rejectOperationInTransaction('Dataset.purge()'); + + await this.backend.purge(); + } + /** * Opens a dataset and returns a promise resolving to an instance of the {@apilink Dataset} class. * diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts index 9b075fa54a99..8e3520172c86 100644 --- a/packages/core/src/storages/key_value_store.ts +++ b/packages/core/src/storages/key_value_store.ts @@ -619,6 +619,18 @@ export class KeyValueStore { serviceLocator.getStorageInstanceManager().removeFromCache(this); } + /** + * Removes all records from the store but keeps the store itself, along with its + * {@apilink KeyValueStore.id|`id`} and {@apilink KeyValueStore.name|`name`}. + */ + async purge(): Promise { + rejectOperationInTransaction('KeyValueStore.purge()'); + + await this.backend.purge(); + // The auto-saved values this cache holds are no longer in the store. + this.#cache.clear(); + } + /** @internal */ clearCache(): void { rejectOperationInTransaction('KeyValueStore.clearCache()'); diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 0a785a28cff6..86f1665c6a1c 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -339,8 +339,13 @@ describe('BasicCrawler', () => { requestHandler, }); + const queue = await basicCrawler.getRequestQueue(); + + // Nothing is emptied between runs, so the same sources are only re-crawled after a purge. await basicCrawler.run(sources); + await queue.purge?.(); await basicCrawler.run(sources); + await queue.purge?.(); await basicCrawler.run(sources); expect(processed).toHaveLength(sourcesCopy.length * 3); @@ -429,6 +434,63 @@ describe('BasicCrawler', () => { ]); }); + describe('a crawl that processes nothing', () => { + const crawlTwice = async (betweenRuns?: (crawler: BasicCrawler) => Promise) => { + const processed: string[] = []; + const crawler = new BasicCrawler({ + requestHandler: async ({ request }) => { + processed.push(request.url); + }, + }); + const warning = vitest.spyOn(crawler.log, 'warning'); + + await crawler.run(['https://example.com/only']); + await betweenRuns?.(crawler); + await crawler.run(['https://example.com/only']); + + return { processed, warning }; + }; + + test('leaves already handled requests alone and says so', async () => { + const { processed, warning } = await crawlTwice(); + + expect(processed).toEqual(['https://example.com/only']); + expect(warning).toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + + test('re-crawls the requests when the queue is purged in between', async () => { + const { processed, warning } = await crawlTwice(async (crawler) => { + await (await crawler.getRequestQueue()).purge?.(); + }); + + expect(processed).toEqual(['https://example.com/only', 'https://example.com/only']); + expect(warning).not.toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + + test('warns a crawler on its first run, over a queue another crawler exhausted', async () => { + const requestQueue = await RequestQueue.open(); + const first = new BasicCrawler({ requestQueue, requestHandler: async () => {} }); + await first.run(['https://example.com/only']); + + const second = new BasicCrawler({ requestQueue, requestHandler: async () => {} }); + const warning = vitest.spyOn(second.log, 'warning'); + + await second.run(['https://example.com/only']); + + expect(warning).toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + + test('says nothing when there was nothing to crawl in the first place', async () => { + // An empty queue is not evidence of a mistake - only requests that turn out to be handled are. + const crawler = new BasicCrawler({ requestHandler: async () => {} }); + const warning = vitest.spyOn(crawler.log, 'warning'); + + await crawler.run(); + + expect(warning).not.toHaveBeenCalledWith(expect.stringMatching(/processed no requests/)); + }); + }); + test('addRequests should respect maxCrawlDepth', async () => { const processedUrls: string[] = []; @@ -2880,107 +2942,8 @@ describe('BasicCrawler', () => { expect(visits[1].at - visits[0].at).toBeGreaterThanOrEqual(400); }); - test('a second run() crawls the same requests again', async () => { - const crawler = new BasicCrawler({ - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - await crawler.run(['http://example.com/1']); - - expect(crawler.statistics.state.requestsFinished).toBe(1); - }); - - test('a second run() over a supplied manager asks which storage to purge', async () => { - // A purge cannot respect both the crawler's per-domain queues and the caller's manager - // underneath them, so rather than pick one silently, say so. - const crawler = new BasicCrawler({ - requestQueue: await RequestQueue.open(), - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - - await expect(crawler.run(['http://example.com/1'])).rejects.toThrow( - /Cannot decide what to purge.*purgeRequestQueue/s, - ); - }); - - test.each([true, false])('a second run() with purgeRequestQueue: %s does not ask', async (purge) => { - const crawler = new BasicCrawler({ - requestQueue: await RequestQueue.open(), - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - - await expect( - crawler.run(['http://example.com/1'], { purgeRequestQueue: purge }), - ).resolves.toBeDefined(); - }); - - test('a second run() asks even when the first routed nothing by domain', async () => { - // The guard fires on the shape of the configuration, not on what the queues happen to hold: - // keying it on an existing per-domain queue would make identical code throw or not depending - // on run history. - const requestQueue = await RequestQueue.open(); - await requestQueue.addRequest({ url: 'http://example.com/pre-added-1' }); - await requestQueue.addRequest({ url: 'http://example.com/pre-added-2' }); - - const visits: number[] = []; - const crawler = new BasicCrawler({ - requestQueue, - sameDomainDelaySecs: 2, - requestHandler: async () => { - visits.push(Date.now()); - }, - }); - - await crawler.run(); - - // Precondition, established rather than assumed: no per-domain queue was opened, since one - // would have paced these 2s apart. - expect(visits).toHaveLength(2); - expect(visits[1] - visits[0]).toBeLessThan(1000); - - await expect(crawler.run()).rejects.toThrow(/Cannot decide what to purge/); - }); - - test('a second run() asks for a supplied `requestManager`, not just a `requestQueue`', async () => { - const crawler = new BasicCrawler({ - requestManager: await RequestQueue.open(), - sameDomainDelaySecs: 0.05, - requestHandler: async () => {}, - }); - - await crawler.run(['http://example.com/1']); - - await expect(crawler.run(['http://example.com/1'])).rejects.toThrow(/Cannot decide what to purge/); - }); - - test('a second run() leaves a supplied manager alone when nothing paces it', async () => { - // The contrast that makes the question above worth asking: with no `sameDomainDelaySecs` the - // crawler puts nothing of its own inside the caller's manager, so there is nothing to ask about. - let visits = 0; - const crawler = new BasicCrawler({ - requestQueue: await RequestQueue.open(), - requestHandler: async () => { - visits += 1; - }, - }); - - await crawler.run(['http://example.com/1']); - await crawler.run(['http://example.com/1']); - - expect(visits).toBe(1); - }); - - test('a second run() purges everything the crawler opened itself, without being asked', async () => { - // Nothing came from the caller, so there is nothing to ask about - and the purge has to reach - // the per-domain queues, or the second run would crawl nothing. + test('a second run() does not crawl the same requests again', async () => { + // The per-domain queues the pacer opened are not emptied between runs either. let visits = 0; const crawler = new BasicCrawler({ sameDomainDelaySecs: 0.05, @@ -2992,7 +2955,7 @@ describe('BasicCrawler', () => { await crawler.run(['http://example.com/1']); await crawler.run(['http://example.com/1']); - expect(visits).toBe(2); + expect(visits).toBe(1); }); test('wraps a user `requestManager` rather than replacing it', async () => { @@ -3068,27 +3031,6 @@ describe('BasicCrawler', () => { }), ).toThrow(/domains: 'all'/); }); - - test('a second run() leaves a manager that took the delay alone', async () => { - // The floor put no storage of ours underneath it, so there is no purge to ask about. - let visits = 0; - const crawler = new BasicCrawler({ - requestManager: new ThrottlingRequestManager({ - inner: await RequestQueue.open(), - domains: 'all', - throttleBy: 'registrableDomain', - }), - sameDomainDelaySecs: 0.05, - requestHandler: async () => { - visits += 1; - }, - }); - - await crawler.run(['http://example.com/1']); - await crawler.run(['http://example.com/1']); - - expect(visits).toBe(1); - }); }); test('enqueueLinks should respect custom user-agent robots.txt rules', async () => { diff --git a/test/core/storages/dataset.test.ts b/test/core/storages/dataset.test.ts index a210088b6368..f86c34870b4b 100644 --- a/test/core/storages/dataset.test.ts +++ b/test/core/storages/dataset.test.ts @@ -719,3 +719,17 @@ describe('dataset', () => { }); }); }); + +describe('Dataset.purge', () => { + test('empties the dataset but keeps it usable', async () => { + const dataset = await Dataset.open(); + await dataset.pushData([{ n: 1 }, { n: 2 }]); + + await dataset.purge(); + + await expect(dataset.getData()).resolves.toMatchObject({ items: [], total: 0 }); + + await dataset.pushData({ n: 3 }); + await expect(dataset.getData()).resolves.toMatchObject({ items: [{ n: 3 }] }); + }); +}); diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.ts index fff61bdfe3ee..7090dc00534d 100644 --- a/test/core/storages/key_value_store.test.ts +++ b/test/core/storages/key_value_store.test.ts @@ -721,3 +721,22 @@ describe('KeyValueStore', () => { }); }); }); + +describe('KeyValueStore.purge', () => { + test('empties the store but keeps it usable, and forgets auto-saved values', async () => { + const store = await KeyValueStore.open(); + await store.setValue('key-1', { foo: 'bar' }); + const state = await store.getAutoSavedValue('STATE', { hits: 0 }); + state.hits = 1; + + await store.purge(); + + await expect(store.getValue('key-1')).resolves.toBeNull(); + + // The auto-save cache would otherwise keep serving a value the store no longer holds. + await expect(store.getAutoSavedValue('STATE', { hits: 0 })).resolves.toEqual({ hits: 0 }); + + await store.setValue('key-2', { foo: 'baz' }); + await expect(store.getValue('key-2')).resolves.toEqual({ foo: 'baz' }); + }); +}); diff --git a/test/core/storages/storage_transaction.test.ts b/test/core/storages/storage_transaction.test.ts index 87fdd6e23fdf..945781575ee9 100644 --- a/test/core/storages/storage_transaction.test.ts +++ b/test/core/storages/storage_transaction.test.ts @@ -344,11 +344,12 @@ describe('Dataset in a transaction', () => { expect(pushDataSpy).toHaveBeenCalledWith([{ n: 1 }, { n: 2 }, { n: 3 }]); }); - test('drop is rejected inside a transaction and allowed under withDirectStorageAccess', async () => { + test('drop and purge are rejected inside a transaction and allowed under withDirectStorageAccess', async () => { const dataset = await Dataset.open(); await withStorageTransaction(async () => { await expect(dataset.drop()).rejects.toThrow(/cannot be used inside a storage transaction/); + await expect(dataset.purge()).rejects.toThrow(/cannot be used inside a storage transaction/); await withDirectStorageAccess(async () => dataset.drop()); }); }); @@ -588,11 +589,12 @@ describe('KeyValueStore in a transaction', () => { await expect(store.getValue('COMMITTED_STATE')).resolves.toEqual({ a: 2 }); }); - test('drop and clearCache are rejected inside a transaction', async () => { + test('drop, purge and clearCache are rejected inside a transaction', async () => { const store = await KeyValueStore.open(); await withStorageTransaction(async () => { await expect(store.drop()).rejects.toThrow(/cannot be used inside a storage transaction/); + await expect(store.purge()).rejects.toThrow(/cannot be used inside a storage transaction/); expect(() => store.clearCache()).toThrow(/cannot be used inside a storage transaction/); }); }); diff --git a/test/e2e/cheerio-stop-resume-ts/actor/main.ts b/test/e2e/cheerio-stop-resume-ts/actor/main.ts index 737ea8cb649c..028e140a2187 100644 --- a/test/e2e/cheerio-stop-resume-ts/actor/main.ts +++ b/test/e2e/cheerio-stop-resume-ts/actor/main.ts @@ -27,5 +27,5 @@ crawler.router.addDefaultHandler(async ({ $, enqueueLinks, request, log }) => { await crawler.run(['https://crawlee.dev/js/docs/quick-start']); requestCount = 0; -await crawler.run(['https://crawlee.dev/js/docs/quick-start'], { purgeRequestQueue: false }); +await crawler.run(['https://crawlee.dev/js/docs/quick-start']); await Actor.exit({ exit: Actor.isAtHome() }); diff --git a/test/e2e/cheerio-stop-resume-ts/test.mjs b/test/e2e/cheerio-stop-resume-ts/test.mjs index 8beaf8681c80..6611a0a4756d 100644 --- a/test/e2e/cheerio-stop-resume-ts/test.mjs +++ b/test/e2e/cheerio-stop-resume-ts/test.mjs @@ -9,4 +9,4 @@ const { stats, datasetItems } = await runActor(testActorDirname); await expect(stats.requestsFinished < 40, 'crawler.stop() works'); const visitedUrls = new Set(datasetItems.map((x) => x.url)); -await expect(visitedUrls.size === datasetItems.length, 'stateful crawler.run({ purgeRQ: false }) works'); +await expect(visitedUrls.size === datasetItems.length, 'a second crawler.run() resumes the same queue');