[pull] master from apify:master

docs/guides/result_storage.mdx

@@ -172,7 +172,7 @@ The feature is escapable at three granularities:
 
 A few operations cannot be buffered, and silently letting them through would produce storage states that no rollback can undo. They throw inside a transaction, with an error pointing at `withDirectStorageAccess()`:
 
-- `Dataset.drop()`, `KeyValueStore.drop()`, `RequestQueue.drop()` and `RequestQueue.purge()`,
+- `drop()` and `purge()` on `Dataset`, `KeyValueStore` and `RequestQueue`,
 - the request queue processing internals (`fetchNextRequest()`, `markRequestAsHandled()`, `reclaimRequest()`),
 - `KeyValueStore.setValue()` with a **stream** value — a stream can only be consumed once, so it cannot serve both a read within the handler and the commit replay. Write streams under `withDirectStorageAccess()`.
 

docs/public-api/crawlee-basic.api.md

@@ -73,8 +73,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
     getRequestQueue(): Promise<IRequestManager>;
     // (undocumented)
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
-    // (undocumented)
-    hasFinishedBefore: boolean;
+    get hasFinishedBefore(): boolean;
     // (undocumented)
     protected readonly httpClient: BaseHttpClient;
     protected init(): Promise<void>;
@@ -170,7 +169,6 @@ export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 
 // @public (undocumented)
 export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
-    purgeRequestQueue?: boolean;
 }
 
 // @public

docs/public-api/crawlee-core.api.md

@@ -347,6 +347,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
     // (undocumented)
     name?: string;
     static open<Data extends Dictionary = Dictionary>(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<Dataset<Data>>;
+    purge(): Promise<void>;
     pushData(data: Data | Data[]): Promise<void>;
     reduce(iteratee: DatasetReducer<Data, Data>): Promise<Data | undefined>;
     reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
@@ -833,6 +834,7 @@ export class KeyValueStore {
     // (undocumented)
     readonly name?: string;
     static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
+    purge(): Promise<void>;
     recordExists(key: string): Promise<boolean>;
     static recordExists(key: string): Promise<boolean>;
     setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;

docs/upgrading/upgrading_v4.md

@@ -21,7 +21,7 @@ This page summarizes the breaking changes in Crawlee v4. There are many, so the
 - **One concurrency budget for several crawlers.** The new [`ConcurrencySystem`](#autoscaling-moved-to-concurrencysystem) can be shared between crawlers, capping their combined concurrency instead of letting each one oversubscribe the host.
 - **Native `fetch` types.** HTTP clients and `context.response` now use the [standard `Response`](#crawlingcontextresponse-is-now-of-type-response), and `got-scraping` is an [opt-in dependency](#http-client-packages-and-basehttpclient-reshaped) instead of a mandatory one.
 - **The session is the rotation unit.** A session carries its proxy, cookies and error score, and is rotated as a whole when blocked — replacing [proxy tiers](#tieredproxyurls-is-removed-from-proxyconfiguration) and [session rotation counters](#maxsessionrotations-and-requestsessionrotationcount-are-removed).
-- **Crawlers stop stepping on each other.** Multiple crawlers in one process [no longer share the default request queue](#multiple-crawler-instances-use-separate-default-request-queues), and repeated `run()` calls purge the queue instead of dropping and recreating it.
+- **Crawlers stop stepping on each other.** Multiple crawlers in one process [no longer share the default request queue](#multiple-crawler-instances-use-separate-default-request-queues), and repeated `run()` calls [no longer empty it](#repeated-run-calls-no-longer-empty-the-request-queue) behind your back.
 - **Cookies behave.** `sendRequest` finally [respects your `Cookie` header](#cookie-handling-in-httpcrawler-and-sendrequest), and browser cookies set inside the handler are [persisted to the session](#browser-cookies-are-also-persisted-after-requesthandler).
 - **No half-written results.** Storage writes in a request handler are [transactional](#storage-writes-in-request-handlers-are-transactional) — a handler that throws leaves nothing behind, and its retry does not duplicate data.
 - **Simpler storage backend contract.** A custom storage backend is now [4 classes instead of 7](#storagebackend-interface-simplified).
@@ -603,45 +603,28 @@ In v4, only the **first** crawler instance uses the default request queue. Each
 
 If you explicitly pass a `requestQueue` (or `requestManager`) to the crawler, that queue is used as-is regardless of instance order.
 
-### Repeated `run()` calls use `purge()` instead of `drop()` + recreate
+### Repeated `run()` calls no longer empty the request queue
 
-When calling `crawler.run()` multiple times on the same crawler instance, v3 would drop the default request queue and create a fresh one between runs. In v4, the crawler **purges** the queue instead — clearing all requests and resetting internal counters, but keeping the same queue object. This is more efficient and avoids edge cases around stale references.
+In v3, calling `crawler.run()` again on the same instance dropped the default request queue and created a fresh one, so the same URLs were crawled again — but only for a queue actually named `default`, which the Apify platform's default queue is not, so on the platform the second run silently crawled nothing.
 
-The new `purge()` method is available on `RequestQueue` and is also defined as an optional method on the `IRequestManager` interface.
+v4 does the same thing everywhere: nothing is emptied between runs. A repeated `run()` continues with the same request manager, and requests the previous run handled — a failed request counts as handled — are not processed again. Any crawl that ends up processing nothing while its request manager holds only handled requests warns and says why, instead of finishing silently; that also covers a second crawler sharing the queue, or a queue a previous process already worked through.
 
-By default, only queues that the crawler created itself (the "owned" queue) are purged between runs — a user-supplied queue is never touched unless you explicitly opt in. The `purgeRequestQueue` option in `CrawlerRunOptions` controls this behavior:
-
-| `purgeRequestQueue` value | Owned queue (auto-created) | User-supplied queue |
-|---|---|---|
-| omitted (default) | Purged | Not purged |
-| `true` | Purged | Purged |
-| `false` | Not purged | Not purged |
-
-One combination has no sensible default: `sameDomainDelaySecs` over a request manager you supplied that does not pace on its own. The per-domain queues that have to be emptied are the crawler's, the manager underneath them is yours, and a purge cannot respect both — so a repeated `run()` throws and asks you to pass `purgeRequestQueue` explicitly rather than guessing. A manager that takes the delay as a floor has nothing of ours underneath it, and is left alone like any other supplied manager.
+The `purgeRequestQueue` option of `crawler.run()` went away with the automatic purge. To crawl the same requests again, empty the queue yourself:
 
 ```typescript
-// The purge happens automatically between run() calls:
 const crawler = new BasicCrawler({ requestHandler: async ({ request }) => { /* ... */ } });
 await crawler.run(['https://example.com/a', 'https://example.com/b']);
-// Queue is purged here, so the same URLs can be processed again:
-await crawler.run(['https://example.com/a', 'https://example.com/c']);
-```
 
-You can opt out of the automatic purge by passing `purgeRequestQueue: false`:
+const queue = await crawler.getRequestQueue();
+await queue.purge?.();
 
-```typescript
-await crawler.run(urls, { purgeRequestQueue: false });
+// The same URLs are crawled again:
+await crawler.run(['https://example.com/a', 'https://example.com/c']);
 ```
 
-If you supplied your own `requestQueue` and want it purged between runs, pass `purgeRequestQueue: true` explicitly:
+`purge()` — empty the storage, keep its id and name — is new in v4 and available on `Dataset`, `KeyValueStore` and `RequestQueue`, as well as being an optional method on the `IRequestManager` interface.
 
-```typescript
-const queue = await RequestQueue.open('my-queue');
-const crawler = new BasicCrawler({ requestQueue: queue, requestHandler: async () => { /* ... */ } });
-await crawler.run(['https://example.com/first']);
-// Explicitly purge the user-supplied queue before the second run:
-await crawler.run(['https://example.com/second'], { purgeRequestQueue: true });
-```
+This has nothing to do with `purgeOnStart` / `CRAWLEE_PURGE_ON_START`, which still wipes the default storages once per process before the first run.
 
 ### Storage `.open()` now also accepts `{ id?, name? }`
 

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -843,9 +843,14 @@ export class BasicCrawler<
     }
 
     running = false;
-    hasFinishedBefore = false;
+    #hasFinishedBefore = false;
     #unexpectedStop = false;
 
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore(): boolean {
+        return this.#hasFinishedBefore;
+    }
+
     #log!: CrawleeLogger;
 
     get log(): CrawleeLogger {
@@ -859,16 +864,6 @@ export class BasicCrawler<
     protected readonly internalTimeoutMillis: number;
     readonly #maxRequestRetries: number;
     readonly #maxCrawlDepth?: number;
-    /**
-     * How much of {@apilink BasicCrawler.requestManager} the crawler may empty between repeated `run()` calls.
-     *
-     * - `all` — nothing under it came from the caller, so one `purge()` on the outside covers everything.
-     * - `none` — the caller supplied it and the crawler put nothing of its own inside.
-     * - `ambiguous` — the caller supplied it, but `sameDomainDelaySecs` put the crawler's own per-domain queues
-     *   underneath: purging empties the caller's storage too, skipping leaves ours stale. A repeated `run()` asks
-     *   rather than guessing.
-     */
-    readonly #purgeableExtent: 'all' | 'none' | 'ambiguous';
     readonly #maxRequestsPerCrawl?: number;
 
     private get handledRequestsCount(): number {
@@ -1105,14 +1100,6 @@ export class BasicCrawler<
 
             const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
 
-            // Our per-domain queues under a manager the caller owns is the case with no right answer; a floor
-            // it took leaves nothing of ours behind. See `#purgeableExtent`.
-            if (suppliedManager === undefined) {
-                this.#purgeableExtent = 'all';
-            } else {
-                this.#purgeableExtent = pacerNeeded ? 'ambiguous' : 'none';
-            }
-
             // Built here rather than at first use so it can sit *inside* the tandem below, which is where a
             // loader's transferred requests pass through it.
             const writableManager = pacerNeeded
@@ -1695,8 +1682,12 @@ export class BasicCrawler<
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} before {@apilink BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
     async run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics> {
         // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
@@ -1708,33 +1699,7 @@ export class BasicCrawler<
                     );
                 }
 
-                const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
-
-                if (this.hasFinishedBefore) {
-                    // When executing the run method for the second time explicitly,
-                    // we need to purge the RQ to allow processing the same requests again — this is important so users can
-                    // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
-                    // ignored — as a failed request is still handled.
-                    // `purgeRequestQueue` unset purges only storage the crawler opened itself (see `#purgeableExtent`);
-                    // `true` also purges a caller-supplied manager, `false` purges nothing.
-                    if (purgeRequestQueue === undefined && this.#purgeableExtent === 'ambiguous') {
-                        throw new Error(
-                            'Cannot decide what to purge before running again: `sameDomainDelaySecs` paces the request ' +
-                                'manager you supplied, so the per-domain queues that have to be emptied are the ' +
-                                "crawler's while the manager underneath them is yours. Say which you want: " +
-                                '`run(requests, { purgeRequestQueue: true })` empties both, `false` empties neither.',
-                        );
-                    }
-
-                    if (
-                        purgeRequestQueue !== false &&
-                        (this.#purgeableExtent === 'all' || purgeRequestQueue === true)
-                    ) {
-                        // One call from the outside in reaches everything the manager wraps, a pacer's per-domain queues
-                        // included.
-                        await this.requestManager?.purge?.();
-                    }
-
+                if (this.#hasFinishedBefore) {
                     // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
                     await this.#statisticsDep.ifOwned(async (stats) => {
                         stats.reset();
@@ -1754,7 +1719,7 @@ export class BasicCrawler<
                 });
 
                 if (requests) {
-                    await this.addRequests(requests, addRequestsOptions);
+                    await this.addRequests(requests, options);
                 }
 
                 try {
@@ -1813,6 +1778,26 @@ export class BasicCrawler<
                     };
                     this.log.info('Final request statistics:', stats as unknown as Record<string, unknown>);
 
+                    // A crawl that did nothing while the manager holds only handled requests is a mistake whoever
+                    // handled them - this run, another crawler on the same queue, or a previous process. Starting
+                    // against handled requests is not: that is what resuming a crawl looks like.
+                    if (stats.requestsFinished + stats.requestsFailed === 0) {
+                        // Never let the diagnostic itself break the run.
+                        const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
+
+                        if (alreadyHandled > 0) {
+                            this.log.warningOnce(
+                                'This crawl processed no requests - the request manager holds ' +
+                                    `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
+                                    'already handled, and a failed request counts as handled too. Nothing ' +
+                                    'empties a queue between runs, so to crawl them again, purge it ' +
+                                    '(`await queue.purge()`) or use a fresh one (e.g. ' +
+                                    '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
+                                    'crawler instance.',
+                            );
+                        }
+                    }
+
                     if (this.statistics.errorTracker.total !== 0) {
                         const prettify = ([count, info]: [number, string[]]) =>
                             `${count}x: ${info.at(-1)!.trim()} (${info[0]})`;
@@ -1846,7 +1831,7 @@ export class BasicCrawler<
                     );
 
                     this.running = false;
-                    this.hasFinishedBefore = true;
+                    this.#hasFinishedBefore = true;
                 }
 
                 return stats;
@@ -3094,19 +3079,7 @@ export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, En
 
 export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {}
 
-export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
-    /**
-     * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance.
-     * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again.
-     *
-     * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged.
-     *   A user-supplied `requestQueue` is left untouched.
-     * - **`true`** — the queue is always purged, even if it was supplied by the user.
-     * - **`false`** — nothing is purged. Only genuinely new requests will be processed;
-     *   note that even a failed request is considered handled.
-     */
-    purgeRequestQueue?: boolean;
-}
+export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {}
 
 /** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
 function hostnameOrUrl(url: string): string {

packages/core/src/storages/dataset.ts

@@ -783,6 +783,16 @@ export class Dataset<Data extends Dictionary = Dictionary> {
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
 
+    /**
+     * Removes all items from the dataset but keeps the dataset itself, along with its
+     * {@apilink Dataset.id|`id`} and {@apilink Dataset.name|`name`}.
+     */
+    async purge(): Promise<void> {
+        rejectOperationInTransaction('Dataset.purge()');
+
+        await this.backend.purge();
+    }
+
     /**
      * Opens a dataset and returns a promise resolving to an instance of the {@apilink Dataset} class.
      *

packages/core/src/storages/key_value_store.ts

@@ -619,6 +619,18 @@ export class KeyValueStore {
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
 
+    /**
+     * Removes all records from the store but keeps the store itself, along with its
+     * {@apilink KeyValueStore.id|`id`} and {@apilink KeyValueStore.name|`name`}.
+     */
+    async purge(): Promise<void> {
+        rejectOperationInTransaction('KeyValueStore.purge()');
+
+        await this.backend.purge();
+        // The auto-saved values this cache holds are no longer in the store.
+        this.#cache.clear();
+    }
+
     /** @internal */
     clearCache(): void {
         rejectOperationInTransaction('KeyValueStore.clearCache()');