From 78bb3d6072f5e126a2ad4de81bd53612eb86db6a Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 19 Aug 2026 10:38:28 -0700
Subject: [PATCH 1/6] feat(knowledge): read a PDF's text layer before paying
 for OCR
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Every PDF went to OCR, an external per-document call, even though most carry an
embedded text layer that costs nothing to read. Across a real corpus of 2,693
documents, local extraction produced text for every PDF that OCR could also read,
so the great majority of those calls bought nothing.

A PDF's text layer is now read first and used when it is good enough, leaving OCR
for the documents that actually need it. Three ways a layer fails, none of which
catches the others: there is no text at all (a scan), the text is too sparse to
be the document, or there is plenty of text that is not language — a broken
encoding, or the raw character ids a CID-keyed font emits with no ToUnicode map,
which is common in exactly the contract and procurement material that reaches a
knowledge base and which a length check alone reads as healthy.

Beyond the cost, this narrows an availability dependency: an OCR outage no longer
touches every PDF, only the minority that cannot be read locally. The threshold is
env-tunable so the balance can be moved toward cost or fidelity without a deploy.

Known limitation: the judgement is per document, so a file mixing typeset pages
with scanned inserts can average above the threshold and keep its partial text.
Per-page routing would catch it and needs per-page extraction this does not have.

The opaque-input refusal now asserts against the outbound request rather than the
storage read: local parsing is not model input, so bytes are read before the
projection is checked and still never leave the worker when it refuses.
---
 ...cument-processor-secret-provenance.test.ts |  12 +-
 .../knowledge/documents/document-processor.ts |  79 ++++++++++-
 .../documents/pdf-ocr-triage.test.ts          | 127 ++++++++++++++++++
 .../documents/pdf-text-layer.test.ts          |  78 +++++++++++
 .../lib/knowledge/documents/pdf-text-layer.ts |  95 +++++++++++++
 5 files changed, 384 insertions(+), 7 deletions(-)
 create mode 100644 apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
 create mode 100644 apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts
 create mode 100644 apps/sim/lib/knowledge/documents/pdf-text-layer.ts

diff --git a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts
index 75751211211..d3386f69946 100644
--- a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts
+++ b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts
@@ -91,7 +91,17 @@ describe('knowledge document model-input provenance', () => {
     expect(fetchMock).not.toHaveBeenCalled()
   })
 
+  /**
+   * The refusal guards egress to an external model, so it is asserted against the
+   * outbound request rather than the storage read. A PDF is now parsed locally
+   * first and only reaches OCR when it has no usable text layer — local parsing is
+   * not model input, as the case above establishes — so the bytes are read before
+   * the projection is checked, and never leave the worker when it refuses.
+   */
   it('rejects secret-bearing opaque document bytes before external OCR', async () => {
+    const fetchMock = vi.fn()
+    vi.stubGlobal('fetch', fetchMock)
+
     await expect(
       runWithKnowledgeModelInputProvenance(
         undefined,
@@ -109,7 +119,7 @@ describe('knowledge document model-input provenance', () => {
       )
     ).rejects.toThrow('Knowledge model input could not be safely projected')
 
-    expect(mockDownloadFileFromUrl).not.toHaveBeenCalled()
+    expect(fetchMock).not.toHaveBeenCalled()
   })
 
   it('attaches exact-empty provenance to the internal Mistral OCR request', async () => {
diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts
index 542b84af54c..67cca11bf79 100644
--- a/apps/sim/lib/knowledge/documents/document-processor.ts
+++ b/apps/sim/lib/knowledge/documents/document-processor.ts
@@ -22,6 +22,7 @@ import {
   resolveParserExtension,
   resolveStoredArtifactExtension,
 } from '@/lib/knowledge/documents/parser-extension'
+import { assessPdfTextLayer } from '@/lib/knowledge/documents/pdf-text-layer'
 import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
 import {
   assertKnowledgeOpaqueModelInputSafe,
@@ -295,6 +296,63 @@ async function getMistralApiKey(workspaceId?: string | null): Promise<string | n
   return env.MISTRAL_API_KEY || null
 }
 
+/**
+ * Reads a PDF's embedded text layer, returning it only when it is good enough to
+ * index — otherwise `undefined`, leaving the caller to fall through to OCR.
+ *
+ * A failure to parse is not an error here: an encrypted or malformed PDF simply
+ * has no usable layer, which is precisely a case for OCR. The document is fetched
+ * again on that path, a second read from our own storage, which is a cheap price
+ * for keeping the two extraction routes independent.
+ */
+async function readEmbeddedPdfText(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string
+): Promise<
+  | {
+      content: string
+      processingMethod: 'file-parser'
+      cloudUrl?: string
+      metadata?: FileParseMetadata
+    }
+  | undefined
+> {
+  try {
+    const buffer = await downloadFileWithTimeout(fileUrl, userId)
+    const [parsed, pageCount] = await Promise.all([
+      parseBuffer(buffer, 'pdf'),
+      getPdfPageCount(buffer),
+    ])
+
+    const verdict = assessPdfTextLayer(parsed.content, pageCount)
+    if (!verdict.usable) {
+      logger.info('PDF text layer not usable, routing to OCR', {
+        filename,
+        pageCount,
+        reason: verdict.reason,
+      })
+      return undefined
+    }
+
+    logger.info('Using embedded PDF text layer', { filename, pageCount })
+    return {
+      content: parsed.content,
+      processingMethod: 'file-parser',
+      cloudUrl: undefined,
+      metadata: parsed.metadata,
+    }
+  } catch (error) {
+    logger.info('Could not read PDF text layer, routing to OCR', {
+      filename,
+      mimeType,
+      error: toError(error).message,
+    })
+    return undefined
+  }
+}
+
 async function parseDocument(
   fileUrl: string,
   filename: string,
@@ -319,14 +377,23 @@ async function parseDocument(
       MISTRAL_API_KEY: mistralApiKey,
     }).providerId
 
-    if (ocrProvider === 'azure-mistral') {
-      assertKnowledgeOpaqueModelInputSafe()
-      logger.info('Using Azure Mistral OCR')
-      return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId)
-    }
+    if (ocrProvider === 'azure-mistral' || ocrProvider === 'mistral') {
+      /**
+       * Most PDFs carry a usable text layer, and reading it costs nothing. OCR is
+       * a per-document call to an external service, so it is reserved for the
+       * documents that actually need it — which also means everything else stops
+       * depending on that service being reachable.
+       */
+      const embedded = await readEmbeddedPdfText(fileUrl, filename, mimeType, userId)
+      if (embedded) return embedded
 
-    if (ocrProvider === 'mistral') {
       assertKnowledgeOpaqueModelInputSafe()
+
+      if (ocrProvider === 'azure-mistral') {
+        logger.info('Using Azure Mistral OCR')
+        return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId)
+      }
+
       logger.info('Using Mistral OCR')
       return parseWithMistralOCR(fileUrl, filename, mimeType, userId, workspaceId, mistralApiKey)
     }
diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
new file mode 100644
index 00000000000..204873cd21f
--- /dev/null
+++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
@@ -0,0 +1,127 @@
+/**
+ * @vitest-environment node
+ *
+ * Every PDF used to be sent to OCR, an external per-document call, even though the
+ * large majority carry a usable text layer that costs nothing to read. These pin
+ * the routing: the text layer is tried first, and OCR is reached only when it is
+ * missing or unreadable.
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockParseBuffer, mockDownload, mockGetDocumentProxy, mockToken, mockBaseUrl } = vi.hoisted(
+  () => ({
+    mockParseBuffer: vi.fn(),
+    mockDownload: vi.fn(),
+    mockGetDocumentProxy: vi.fn(),
+    mockToken: vi.fn(),
+    mockBaseUrl: vi.fn(),
+  })
+)
+
+vi.mock('@/lib/auth/internal', () => ({ generateInternalToken: mockToken }))
+vi.mock('@/lib/core/utils/urls', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/core/utils/urls')>()),
+  getInternalApiBaseUrl: mockBaseUrl,
+}))
+
+vi.mock('@/lib/file-parsers', () => ({
+  parseBuffer: mockParseBuffer,
+  isSupportedFileType: (extension: string) => ['pdf'].includes(extension),
+}))
+vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadFileFromUrl: mockDownload }))
+vi.mock('unpdf', () => ({ getDocumentProxy: mockGetDocumentProxy }))
+
+import { env } from '@/lib/core/config/env'
+import { processDocument } from '@/lib/knowledge/documents/document-processor'
+import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance'
+
+/** External, so the OCR path uses the URL directly instead of re-uploading it. */
+const PDF_URL = 'https://example.com/Contract.pdf'
+const typeset = 'The Supplier shall provide the Services described herein. '.repeat(60)
+
+function parse() {
+  return runWithKnowledgeModelInputProvenance(
+    undefined,
+    () => processDocument(PDF_URL, 'Contract.pdf', 'application/pdf', 1024, 200, 1, 'user-1'),
+    { opaqueInputSafe: true }
+  )
+}
+
+describe('PDF OCR triage', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    Object.assign(env, { OCR_PROVIDER: 'mistral', MISTRAL_API_KEY: 'key' })
+    mockDownload.mockResolvedValue(Buffer.from('%PDF-1.7'))
+    mockGetDocumentProxy.mockResolvedValue({ numPages: 2 })
+    mockToken.mockResolvedValue('internal-token')
+    mockBaseUrl.mockReturnValue('http://sim.local')
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('uses the embedded text layer and never calls OCR', async () => {
+    mockParseBuffer.mockResolvedValue({ content: typeset, metadata: {} })
+    const fetchMock = vi.fn()
+    vi.stubGlobal('fetch', fetchMock)
+
+    const result = await parse()
+
+    expect(result.metadata.processingMethod).toBe('file-parser')
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+
+  it('falls through to OCR when the PDF is a scan', async () => {
+    mockParseBuffer.mockResolvedValue({ content: '', metadata: {} })
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValue(
+        new Response(JSON.stringify({ pages: [{ markdown: 'Recognised text' }], usage_info: {} }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        })
+      )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const result = await parse()
+
+    expect(result.metadata.processingMethod).toBe('mistral-ocr')
+    expect(fetchMock).toHaveBeenCalled()
+  })
+
+  /**
+   * The case a length check alone cannot see: a CID-keyed font with no Unicode map
+   * yields plenty of characters, none of them words.
+   */
+  it('falls through to OCR when the text layer is raw CID escapes', async () => {
+    mockParseBuffer.mockResolvedValue({ content: '/31 /8 /18 /12 /44 '.repeat(60), metadata: {} })
+    const fetchMock = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }], usage_info: {} }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const result = await parse()
+
+    expect(result.metadata.processingMethod).toBe('mistral-ocr')
+  })
+
+  /** An encrypted or malformed PDF has no readable layer, which is a case for OCR. */
+  it('falls through to OCR when the text layer cannot be parsed at all', async () => {
+    mockParseBuffer.mockRejectedValue(new Error('Invalid PDF structure.'))
+    const fetchMock = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }], usage_info: {} }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const result = await parse()
+
+    expect(result.metadata.processingMethod).toBe('mistral-ocr')
+  })
+})
diff --git a/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts b/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts
new file mode 100644
index 00000000000..5a69fcce7af
--- /dev/null
+++ b/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts
@@ -0,0 +1,78 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { assessPdfTextLayer } from '@/lib/knowledge/documents/pdf-text-layer'
+
+/** Roughly the character volume of a typeset page. */
+const page = (n: number) =>
+  'The Supplier shall provide the Services described in this Statement of Work. '.repeat(n)
+
+describe('assessPdfTextLayer', () => {
+  it('accepts an ordinary typeset document', () => {
+    expect(assessPdfTextLayer(page(60), 2)).toEqual({ usable: true })
+  })
+
+  it('rejects a scan, which carries no text at all', () => {
+    expect(assessPdfTextLayer('', 12)).toEqual({ usable: false, reason: 'no-text' })
+    expect(assessPdfTextLayer('   \n  ', 12)).toEqual({ usable: false, reason: 'no-text' })
+  })
+
+  /** A scan often still yields a header or a stamp — present, but not the content. */
+  it('rejects text too sparse to be the document', () => {
+    expect(assessPdfTextLayer('CONFIDENTIAL', 40)).toEqual({
+      usable: false,
+      reason: 'sparse-text',
+    })
+  })
+
+  /**
+   * A CID-keyed font with no `ToUnicode` map extracts as raw character ids. There
+   * is plenty of it, so a length check passes and the document would be indexed as
+   * gibberish — the failure mode a characters-per-page test alone cannot see.
+   */
+  it('rejects raw CID escapes from a font with no Unicode mapping', () => {
+    const cid = '/31 /8 /18 /12 /44 /9 /27 /15 /3 /62 '.repeat(40)
+
+    expect(assessPdfTextLayer(cid, 1)).toEqual({ usable: false, reason: 'cid-escapes' })
+  })
+
+  it('rejects a text layer that decoded to replacement characters', () => {
+    expect(assessPdfTextLayer('�'.repeat(500), 1)).toEqual({
+      usable: false,
+      reason: 'unreadable-encoding',
+    })
+  })
+
+  /** Real prose contains slashes and digits; only a dominant share is disqualifying. */
+  it('keeps a document that merely mentions figures and dates', () => {
+    const prose = `${page(40)} Payment of /50 net 30, effective 01/04/2026, ref /12 /9.`
+
+    expect(assessPdfTextLayer(prose, 1)).toEqual({ usable: true })
+  })
+
+  it('keeps accented and non-Latin prose, which is ordinary text', () => {
+    expect(assessPdfTextLayer('Zusammenfassung über Verträge. '.repeat(40), 1)).toEqual({
+      usable: true,
+    })
+    expect(assessPdfTextLayer('契約の概要について説明します。'.repeat(40), 1)).toEqual({
+      usable: true,
+    })
+  })
+
+  /** An unparseable page count must still apply a floor rather than divide by zero. */
+  it('treats an unknown page count as a single page', () => {
+    expect(assessPdfTextLayer('short', 0)).toEqual({ usable: false, reason: 'sparse-text' })
+    expect(assessPdfTextLayer(page(40), 0)).toEqual({ usable: true })
+  })
+
+  it('scales the threshold with length, so one good page does not carry a long scan', () => {
+    const onePageOfText = page(30)
+
+    expect(assessPdfTextLayer(onePageOfText, 1)).toEqual({ usable: true })
+    expect(assessPdfTextLayer(onePageOfText, 200)).toEqual({
+      usable: false,
+      reason: 'sparse-text',
+    })
+  })
+})
diff --git a/apps/sim/lib/knowledge/documents/pdf-text-layer.ts b/apps/sim/lib/knowledge/documents/pdf-text-layer.ts
new file mode 100644
index 00000000000..e43ded7d922
--- /dev/null
+++ b/apps/sim/lib/knowledge/documents/pdf-text-layer.ts
@@ -0,0 +1,95 @@
+import { envNumber } from '@/lib/core/config/env'
+
+/**
+ * Minimum average characters per page for an embedded text layer to be trusted.
+ *
+ * A typeset page carries roughly 1,500–3,000 characters, a scanned image carries
+ * none, so the gap is wide and the threshold does not need to be precise — it only
+ * has to sit far below real prose and far above the handful of characters a scan
+ * contributes from headers or stamps.
+ */
+const DEFAULT_MIN_CHARS_PER_PAGE = 100
+
+/**
+ * Share of characters that must be ordinary printable text.
+ *
+ * A text layer with a broken encoding extracts as mojibake or replacement
+ * characters: present in quantity, but not words.
+ */
+const MIN_PRINTABLE_RATIO = 0.8
+
+/**
+ * Share of characters that may look like CID escapes before the layer is rejected.
+ *
+ * A CID-keyed font with no `ToUnicode` map extracts as the raw character ids —
+ * `/31 /8 /18 /12` — rather than glyphs. It passes a length check comfortably while
+ * containing no readable text at all. Common in documents from older generators and
+ * in anything using subset fonts, which is much of the contract and procurement
+ * material that reaches a knowledge base.
+ */
+const MAX_CID_ESCAPE_RATIO = 0.5
+
+/** Runs of the form `/31 /8`, the raw output of a CID font with no Unicode map. */
+const CID_ESCAPE_PATTERN = /\/i?\d+/g
+
+/** Characters that count as ordinary text: printable ASCII, whitespace, and Latin-1+. */
+const PRINTABLE_PATTERN = /[\p{L}\p{N}\p{P}\p{Zs}\n\r\t]/gu
+
+export type PdfTextLayerVerdict =
+  | { usable: true }
+  | { usable: false; reason: 'no-text' | 'sparse-text' | 'unreadable-encoding' | 'cid-escapes' }
+
+function countMatches(text: string, pattern: RegExp): number {
+  let total = 0
+  for (const match of text.matchAll(pattern)) total += match[0].length
+  return total
+}
+
+/**
+ * Judges whether a PDF's embedded text layer can be indexed as-is, or whether the
+ * document has to go through OCR to be readable.
+ *
+ * Extracting a text layer costs nothing and covers the large majority of PDFs;
+ * OCR is a per-document call to an external service. Asking this question first
+ * means only the documents that actually need OCR pay for it, and it removes the
+ * dependency on that service for everything else.
+ *
+ * Three ways a text layer fails, all observed in the wild and none caught by the
+ * others: there is no text (a scan), the text is too sparse to be the document's
+ * real content, or there is plenty of text but it is not language — a broken
+ * encoding, or raw CID codes from a font with no Unicode mapping.
+ *
+ * Known limitation: this judges the document as a whole, so a file that mixes
+ * typeset pages with scanned inserts can average out above the threshold and keep
+ * its partial text. Routing per page would catch that, and needs per-page
+ * extraction this does not currently have.
+ */
+export function assessPdfTextLayer(text: string, pageCount: number): PdfTextLayerVerdict {
+  const trimmed = text.trim()
+  if (trimmed.length === 0) return { usable: false, reason: 'no-text' }
+
+  const minCharsPerPage = envNumber(
+    process.env.KB_PDF_MIN_CHARS_PER_PAGE,
+    DEFAULT_MIN_CHARS_PER_PAGE
+  )
+
+  /**
+   * An unknown page count (a PDF whose header would not parse) is treated as a
+   * single page: it still applies a floor, without inventing a page count that
+   * would scale the threshold arbitrarily.
+   */
+  const pages = pageCount > 0 ? pageCount : 1
+  if (trimmed.length / pages < minCharsPerPage) return { usable: false, reason: 'sparse-text' }
+
+  const cidChars = countMatches(trimmed, CID_ESCAPE_PATTERN)
+  if (cidChars / trimmed.length > MAX_CID_ESCAPE_RATIO) {
+    return { usable: false, reason: 'cid-escapes' }
+  }
+
+  const printableChars = countMatches(trimmed, PRINTABLE_PATTERN)
+  if (printableChars / trimmed.length < MIN_PRINTABLE_RATIO) {
+    return { usable: false, reason: 'unreadable-encoding' }
+  }
+
+  return { usable: true }
+}

From fbfe4062fe56acd1bfb2790062b244dab4b968f9 Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 19 Aug 2026 10:45:54 -0700
Subject: [PATCH 2/6] fix(knowledge): route a truncated PDF extraction to OCR,
 and drop the threshold env var

Two corrections to the text-layer triage.

A parser limit stops extraction partway and reports `truncated`. Such a result has
plenty of text by volume, so every volume-based check read it as healthy and the
document was indexed as a fragment with the remainder silently missing from
search. Truncation is now judged before anything that measures volume, and sends
the document to OCR, which reads it whole.

The characters-per-page threshold is a plain constant again. It read
`process.env` directly rather than going through the env module, and the tunable
was not worth having: a typeset page carries roughly 1,500-3,000 characters and a
scan carries none, so the value sits in a wide gap where no realistic tuning
changes an outcome. A constant is one less piece of configuration that can be set
wrong, and if the threshold is ever wrong the fix is to change it.
---
 .../knowledge/documents/document-processor.ts |  2 +-
 .../documents/pdf-ocr-triage.test.ts          | 14 +++---
 .../documents/pdf-text-layer.test.ts          |  9 ++++
 .../lib/knowledge/documents/pdf-text-layer.ts | 45 +++++++++++--------
 4 files changed, 43 insertions(+), 27 deletions(-)

diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts
index 67cca11bf79..8d6293970cd 100644
--- a/apps/sim/lib/knowledge/documents/document-processor.ts
+++ b/apps/sim/lib/knowledge/documents/document-processor.ts
@@ -326,7 +326,7 @@ async function readEmbeddedPdfText(
       getPdfPageCount(buffer),
     ])
 
-    const verdict = assessPdfTextLayer(parsed.content, pageCount)
+    const verdict = assessPdfTextLayer(parsed.content, pageCount, parsed.metadata?.truncated)
     if (!verdict.usable) {
       logger.info('PDF text layer not usable, routing to OCR', {
         filename,
diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
index 204873cd21f..bb8f1026977 100644
--- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
+++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
@@ -74,14 +74,12 @@ describe('PDF OCR triage', () => {
 
   it('falls through to OCR when the PDF is a scan', async () => {
     mockParseBuffer.mockResolvedValue({ content: '', metadata: {} })
-    const fetchMock = vi
-      .fn()
-      .mockResolvedValue(
-        new Response(JSON.stringify({ pages: [{ markdown: 'Recognised text' }], usage_info: {} }), {
-          status: 200,
-          headers: { 'Content-Type': 'application/json' },
-        })
-      )
+    const fetchMock = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ pages: [{ markdown: 'Recognised text' }], usage_info: {} }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    )
     vi.stubGlobal('fetch', fetchMock)
 
     const result = await parse()
diff --git a/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts b/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts
index 5a69fcce7af..9b759ee9b0c 100644
--- a/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts
+++ b/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts
@@ -13,6 +13,15 @@ describe('assessPdfTextLayer', () => {
     expect(assessPdfTextLayer(page(60), 2)).toEqual({ usable: true })
   })
 
+  /**
+   * A parser limit stops extraction partway, so the text that came back is plenty
+   * by volume but is only part of the document. Accepting it would index a
+   * fragment and drop the rest from search without saying so.
+   */
+  it('rejects an extraction that stopped at a parser limit', () => {
+    expect(assessPdfTextLayer(page(200), 3, true)).toEqual({ usable: false, reason: 'truncated' })
+  })
+
   it('rejects a scan, which carries no text at all', () => {
     expect(assessPdfTextLayer('', 12)).toEqual({ usable: false, reason: 'no-text' })
     expect(assessPdfTextLayer('   \n  ', 12)).toEqual({ usable: false, reason: 'no-text' })
diff --git a/apps/sim/lib/knowledge/documents/pdf-text-layer.ts b/apps/sim/lib/knowledge/documents/pdf-text-layer.ts
index e43ded7d922..ec7f42804fd 100644
--- a/apps/sim/lib/knowledge/documents/pdf-text-layer.ts
+++ b/apps/sim/lib/knowledge/documents/pdf-text-layer.ts
@@ -1,14 +1,12 @@
-import { envNumber } from '@/lib/core/config/env'
-
 /**
  * Minimum average characters per page for an embedded text layer to be trusted.
  *
- * A typeset page carries roughly 1,500–3,000 characters, a scanned image carries
- * none, so the gap is wide and the threshold does not need to be precise — it only
- * has to sit far below real prose and far above the handful of characters a scan
- * contributes from headers or stamps.
+ * A typeset page carries roughly 1,500–3,000 characters and a scanned image
+ * carries none, so this sits an order of magnitude below real prose and well above
+ * the handful of characters a scan contributes from a header or a stamp. The gap
+ * either side is wide enough that the exact value does not matter.
  */
-const DEFAULT_MIN_CHARS_PER_PAGE = 100
+const MIN_CHARS_PER_PAGE = 100
 
 /**
  * Share of characters that must be ordinary printable text.
@@ -37,7 +35,10 @@ const PRINTABLE_PATTERN = /[\p{L}\p{N}\p{P}\p{Zs}\n\r\t]/gu
 
 export type PdfTextLayerVerdict =
   | { usable: true }
-  | { usable: false; reason: 'no-text' | 'sparse-text' | 'unreadable-encoding' | 'cid-escapes' }
+  | {
+      usable: false
+      reason: 'no-text' | 'truncated' | 'sparse-text' | 'unreadable-encoding' | 'cid-escapes'
+    }
 
 function countMatches(text: string, pattern: RegExp): number {
   let total = 0
@@ -54,24 +55,32 @@ function countMatches(text: string, pattern: RegExp): number {
  * means only the documents that actually need OCR pay for it, and it removes the
  * dependency on that service for everything else.
  *
- * Three ways a text layer fails, all observed in the wild and none caught by the
- * others: there is no text (a scan), the text is too sparse to be the document's
- * real content, or there is plenty of text but it is not language — a broken
- * encoding, or raw CID codes from a font with no Unicode mapping.
+ * Four ways a text layer fails, none caught by the others: there is no text (a
+ * scan), extraction stopped at a parser limit so what came back is only part of
+ * the document, the text is too sparse to be the document's real content, or
+ * there is plenty of text but it is not language — a broken encoding, or raw CID
+ * codes from a font with no Unicode mapping.
  *
  * Known limitation: this judges the document as a whole, so a file that mixes
  * typeset pages with scanned inserts can average out above the threshold and keep
  * its partial text. Routing per page would catch that, and needs per-page
  * extraction this does not currently have.
  */
-export function assessPdfTextLayer(text: string, pageCount: number): PdfTextLayerVerdict {
+export function assessPdfTextLayer(
+  text: string,
+  pageCount: number,
+  truncated = false
+): PdfTextLayerVerdict {
   const trimmed = text.trim()
   if (trimmed.length === 0) return { usable: false, reason: 'no-text' }
 
-  const minCharsPerPage = envNumber(
-    process.env.KB_PDF_MIN_CHARS_PER_PAGE,
-    DEFAULT_MIN_CHARS_PER_PAGE
-  )
+  /**
+   * Checked before anything measuring volume, because a truncated extraction has
+   * plenty of text by definition and would otherwise read as healthy. Accepting it
+   * would index part of a document and silently drop the rest from search, so the
+   * document goes to OCR, which reads it whole.
+   */
+  if (truncated) return { usable: false, reason: 'truncated' }
 
   /**
    * An unknown page count (a PDF whose header would not parse) is treated as a
@@ -79,7 +88,7 @@ export function assessPdfTextLayer(text: string, pageCount: number): PdfTextLaye
    * would scale the threshold arbitrarily.
    */
   const pages = pageCount > 0 ? pageCount : 1
-  if (trimmed.length / pages < minCharsPerPage) return { usable: false, reason: 'sparse-text' }
+  if (trimmed.length / pages < MIN_CHARS_PER_PAGE) return { usable: false, reason: 'sparse-text' }
 
   const cidChars = countMatches(trimmed, CID_ESCAPE_PATTERN)
   if (cidChars / trimmed.length > MAX_CID_ESCAPE_RATIO) {

From 47cd6e10002a684651fea3d13128c2e8e12d81aa Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 19 Aug 2026 10:53:18 -0700
Subject: [PATCH 3/6] fix(knowledge): take the page count from the parse that
 produced the text

The density check counted pages with a second, independent read of the file. The
two could disagree: a count that failed reported no pages, the check fell back to
treating the document as a single page, and a long scan carrying only a header
looked dense enough to skip OCR and be indexed as that header.

`parseBuffer` already reports the page count from the parse that produced the
text, so the two can no longer diverge, and the redundant second open of the file
goes away with it.
---
 .../knowledge/documents/document-processor.ts | 13 +++++++----
 .../documents/pdf-ocr-triage.test.ts          | 22 +++++++++++++++++++
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts
index 8d6293970cd..01e1c5bc1b8 100644
--- a/apps/sim/lib/knowledge/documents/document-processor.ts
+++ b/apps/sim/lib/knowledge/documents/document-processor.ts
@@ -321,11 +321,16 @@ async function readEmbeddedPdfText(
 > {
   try {
     const buffer = await downloadFileWithTimeout(fileUrl, userId)
-    const [parsed, pageCount] = await Promise.all([
-      parseBuffer(buffer, 'pdf'),
-      getPdfPageCount(buffer),
-    ])
+    const parsed = await parseBuffer(buffer, 'pdf')
 
+    /**
+     * The page count comes from the same parse as the text, rather than a second
+     * independent read of the file. Counting separately lets the two disagree: a
+     * count that failed would report no pages, the density check would fall back to
+     * treating the document as a single page, and a long scan carrying only a header
+     * would look dense enough to skip OCR and be indexed as that header.
+     */
+    const pageCount = parsed.metadata?.pageCount ?? 0
     const verdict = assessPdfTextLayer(parsed.content, pageCount, parsed.metadata?.truncated)
     if (!verdict.usable) {
       logger.info('PDF text layer not usable, routing to OCR', {
diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
index bb8f1026977..f496b56ca9b 100644
--- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
+++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
@@ -72,6 +72,28 @@ describe('PDF OCR triage', () => {
     expect(fetchMock).not.toHaveBeenCalled()
   })
 
+  /**
+   * The density check reads its page count from the same parse as the text. A long
+   * scan that yields only a header must stay sparse against its real page count —
+   * counting separately allowed a failed count to present it as a single dense page.
+   */
+  it('takes the page count from the parse, so a header-only scan stays sparse', async () => {
+    // Enough to clear the floor as a single page, nowhere near enough for 80.
+    const headerOnly = 'CONFIDENTIAL - Vendor Master Agreement - Page header. '.repeat(6)
+    mockParseBuffer.mockResolvedValue({ content: headerOnly, metadata: { pageCount: 80 } })
+    const fetchMock = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }], usage_info: {} }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const result = await parse()
+
+    expect(result.metadata.processingMethod).toBe('mistral-ocr')
+  })
+
   it('falls through to OCR when the PDF is a scan', async () => {
     mockParseBuffer.mockResolvedValue({ content: '', metadata: {} })
     const fetchMock = vi.fn().mockResolvedValue(

From 58aa669eae9d24661c96a945b133b3d4401a8b55 Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 19 Aug 2026 11:01:40 -0700
Subject: [PATCH 4/6] fix(knowledge): chunk a long PDF for Azure OCR instead of
 refusing it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both OCR providers cap how many pages a single request may carry, and both were
handling that cap differently: one split the document to fit, the other rejected
any document over it. A long PDF could therefore be ingested on one provider and
not at all on the other, for a limit that belongs to a request rather than to a
document.

The splitting, concurrency, ordering and partial-failure rule now live in one
place that both providers call, so they cannot drift apart again. A chunk that
fails is dropped rather than failing the document — losing one section of a long
document beats losing all of it — and every chunk failing still throws.

Also drops the unpdf mock from the triage tests. It was masking real behaviour:
the page count now comes from the parse metadata, so the mock was no longer
needed, and while it was in place a test asserting the old page-cap refusal
passed against both the old and new code.
---
 .../knowledge/documents/document-processor.ts | 191 +++++++++++-------
 .../documents/pdf-ocr-triage.test.ts          |  57 +++++-
 2 files changed, 161 insertions(+), 87 deletions(-)

diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts
index 01e1c5bc1b8..b4611008e9f 100644
--- a/apps/sim/lib/knowledge/documents/document-processor.ts
+++ b/apps/sim/lib/knowledge/documents/document-processor.ts
@@ -594,42 +594,19 @@ async function parseWithAzureMistralOCR(
 
   const fileBuffer = await downloadFileForBase64(fileUrl, userId)
 
-  if (mimeType === 'application/pdf') {
-    const pageCount = await getPdfPageCount(fileBuffer)
-    if (pageCount > MISTRAL_MAX_PAGES) {
-      throw new Error(
-        `PDF has ${pageCount} pages, exceeding the Azure OCR limit of ${MISTRAL_MAX_PAGES}`
-      )
-    }
-    logger.info('Azure Mistral OCR: PDF page count resolved', { pageCount })
-  }
-
-  const base64Data = fileBuffer.toString('base64')
-  const dataUri = `data:${mimeType};base64,${base64Data}`
-
   try {
-    const response = await retryWithExponentialBackoff(
-      () =>
-        makeOCRRequest(
-          env.OCR_AZURE_ENDPOINT!,
-          {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${env.OCR_AZURE_API_KEY}`,
-          },
-          {
-            model: env.OCR_AZURE_MODEL_NAME!,
-            document: {
-              type: 'document_url',
-              document_url: dataUri,
-            },
-            include_image_base64: false,
-          }
-        ),
-      { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 }
-    )
-
-    const ocrResult = (await response.json()) as AzureOCRResponse
-    const content = extractPageContent(ocrResult.pages || []) || JSON.stringify(ocrResult, null, 2)
+    /**
+     * A PDF is chunked to the provider's page cap rather than refused for
+     * exceeding it, matching the other OCR provider. Refusing meant a long
+     * document could not be ingested at all, and the cap applies to a single
+     * request, not to the document.
+     */
+    const content =
+      mimeType === 'application/pdf'
+        ? await ocrPdfInChunks(fileBuffer, 'azure-mistral', (chunk) =>
+            recognizeWithAzureOCR(chunk.buffer, mimeType)
+          )
+        : await recognizeWithAzureOCR(fileBuffer, mimeType)
 
     if (!content.trim()) {
       throw new Error('Azure Mistral OCR returned empty content')
@@ -645,6 +622,34 @@ async function parseWithAzureMistralOCR(
   }
 }
 
+/** Sends one document to Azure Mistral OCR inline, as a base64 data URI. */
+async function recognizeWithAzureOCR(buffer: Buffer, mimeType: string): Promise<string> {
+  const dataUri = `data:${mimeType};base64,${buffer.toString('base64')}`
+
+  const response = await retryWithExponentialBackoff(
+    () =>
+      makeOCRRequest(
+        env.OCR_AZURE_ENDPOINT!,
+        {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${env.OCR_AZURE_API_KEY}`,
+        },
+        {
+          model: env.OCR_AZURE_MODEL_NAME!,
+          document: {
+            type: 'document_url',
+            document_url: dataUri,
+          },
+          include_image_base64: false,
+        }
+      ),
+    { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 }
+  )
+
+  const ocrResult = (await response.json()) as AzureOCRResponse
+  return extractPageContent(ocrResult.pages || []) || JSON.stringify(ocrResult, null, 2)
+}
+
 async function parseWithMistralOCR(
   fileUrl: string,
   filename: string,
@@ -812,63 +817,97 @@ async function processChunk(
   }
 }
 
-async function processMistralOCRInBatches(
-  filename: string,
-  apiKey: string,
+/**
+ * Runs a PDF through OCR a chunk at a time and stitches the pages back together.
+ *
+ * A provider that caps how many pages one request may carry needs the document
+ * split, and both providers cap at the same limit — so the splitting, the
+ * concurrency, the ordering and the partial-failure rule live here once rather
+ * than being restated per provider, where they had already drifted into one
+ * provider chunking and the other refusing anything over the cap.
+ *
+ * A chunk that fails yields `null` and is dropped: losing one section of a long
+ * document is better than losing all of it. Every chunk failing is a genuine
+ * failure and throws.
+ */
+async function ocrPdfInChunks(
   pdfBuffer: Buffer,
-  userId?: string,
-  cloudUrl?: string
-): Promise<{
-  content: string
-  processingMethod: 'mistral-ocr'
-  cloudUrl?: string
-}> {
+  provider: string,
+  recognize: (
+    chunk: { buffer: Buffer; startPage: number; endPage: number },
+    chunkIndex: number,
+    totalChunks: number
+  ) => Promise<string | null>
+): Promise<string> {
   const totalPages = await getPdfPageCount(pdfBuffer)
-  logger.info(`Splitting PDF into chunks`, { totalPages, maxPagesPerChunk: MISTRAL_MAX_PAGES })
-
   const pdfChunks = await splitPdfIntoChunks(pdfBuffer, MISTRAL_MAX_PAGES)
-  logger.info(
-    `Split into ${pdfChunks.length} chunks, processing with concurrency ${MAX_CONCURRENT_CHUNKS}`
-  )
+  logger.info('Splitting PDF for OCR', {
+    provider,
+    totalPages,
+    chunks: pdfChunks.length,
+    maxPagesPerChunk: MISTRAL_MAX_PAGES,
+    concurrency: MAX_CONCURRENT_CHUNKS,
+  })
 
   const results: { index: number; content: string | null }[] = []
 
   for (let i = 0; i < pdfChunks.length; i += MAX_CONCURRENT_CHUNKS) {
     const batch = pdfChunks.slice(i, i + MAX_CONCURRENT_CHUNKS)
-    const batchPromises = batch.map((chunk, batchIndex) =>
-      processChunk(chunk, i + batchIndex, pdfChunks.length, filename, apiKey, userId)
-    )
-
-    const batchResults = await Promise.all(batchPromises)
-    for (const result of batchResults) {
-      results.push(result)
-    }
-
-    logger.info(
-      `Completed batch ${Math.floor(i / MAX_CONCURRENT_CHUNKS) + 1}/${Math.ceil(pdfChunks.length / MAX_CONCURRENT_CHUNKS)}`
+    const batchResults = await Promise.all(
+      batch.map((chunk, batchIndex) => {
+        const index = i + batchIndex
+        return recognize(chunk, index, pdfChunks.length).then(
+          (content) => ({ index, content }),
+          (error) => {
+            logger.warn('OCR chunk failed', {
+              provider,
+              chunk: index + 1,
+              error: toError(error).message,
+            })
+            return { index, content: null }
+          }
+        )
+      })
     )
+    results.push(...batchResults)
   }
 
-  const sortedResults = results
+  const recovered = results
     .sort((a, b) => a.index - b.index)
-    .filter((r) => r.content !== null)
-    .map((r) => r.content as string)
+    .map((r) => r.content)
+    .filter((content): content is string => content !== null && content.trim().length > 0)
 
-  if (sortedResults.length === 0) {
-    throw new Error(
-      `OCR failed for all ${pdfChunks.length} chunks. ` +
-        `Large PDFs require OCR - file parser fallback would produce poor results.`
-    )
+  if (recovered.length === 0) {
+    throw new Error(`OCR failed for all ${pdfChunks.length} chunks of the document`)
   }
 
-  const combinedContent = sortedResults.join('\n\n')
-  logger.info(`Successfully processed ${sortedResults.length}/${pdfChunks.length} chunks`)
-
-  return {
-    content: combinedContent,
-    processingMethod: 'mistral-ocr',
-    cloudUrl,
+  if (recovered.length < pdfChunks.length) {
+    logger.warn('OCR recovered only part of the document', {
+      provider,
+      recovered: recovered.length,
+      chunks: pdfChunks.length,
+    })
   }
+
+  return recovered.join('\n\n')
+}
+
+async function processMistralOCRInBatches(
+  filename: string,
+  apiKey: string,
+  pdfBuffer: Buffer,
+  userId?: string,
+  cloudUrl?: string
+): Promise<{
+  content: string
+  processingMethod: 'mistral-ocr'
+  cloudUrl?: string
+}> {
+  const content = await ocrPdfInChunks(pdfBuffer, 'mistral', (chunk, index, total) =>
+    processChunk(chunk, index, total, filename, apiKey, userId).then((r) => r.content)
+  )
+
+  return { content, processingMethod: 'mistral-ocr', cloudUrl }
 }
 
 /**
diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
index f496b56ca9b..48e0bb927e3 100644
--- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
+++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
@@ -8,15 +8,12 @@
  */
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockParseBuffer, mockDownload, mockGetDocumentProxy, mockToken, mockBaseUrl } = vi.hoisted(
-  () => ({
-    mockParseBuffer: vi.fn(),
-    mockDownload: vi.fn(),
-    mockGetDocumentProxy: vi.fn(),
-    mockToken: vi.fn(),
-    mockBaseUrl: vi.fn(),
-  })
-)
+const { mockParseBuffer, mockDownload, mockToken, mockBaseUrl } = vi.hoisted(() => ({
+  mockParseBuffer: vi.fn(),
+  mockDownload: vi.fn(),
+  mockToken: vi.fn(),
+  mockBaseUrl: vi.fn(),
+}))
 
 vi.mock('@/lib/auth/internal', () => ({ generateInternalToken: mockToken }))
 vi.mock('@/lib/core/utils/urls', async (importOriginal) => ({
@@ -29,7 +26,6 @@ vi.mock('@/lib/file-parsers', () => ({
   isSupportedFileType: (extension: string) => ['pdf'].includes(extension),
 }))
 vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadFileFromUrl: mockDownload }))
-vi.mock('unpdf', () => ({ getDocumentProxy: mockGetDocumentProxy }))
 
 import { env } from '@/lib/core/config/env'
 import { processDocument } from '@/lib/knowledge/documents/document-processor'
@@ -39,6 +35,14 @@ import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-inpu
 const PDF_URL = 'https://example.com/Contract.pdf'
 const typeset = 'The Supplier shall provide the Services described herein. '.repeat(60)
 
+/** A real PDF, because splitting loads the document rather than trusting metadata. */
+async function pdfOfPages(count: number): Promise<Buffer> {
+  const { PDFDocument } = await import('pdf-lib')
+  const pdf = await PDFDocument.create()
+  for (let i = 0; i < count; i++) pdf.addPage()
+  return Buffer.from(await pdf.save())
+}
+
 function parse() {
   return runWithKnowledgeModelInputProvenance(
     undefined,
@@ -52,7 +56,6 @@ describe('PDF OCR triage', () => {
     vi.clearAllMocks()
     Object.assign(env, { OCR_PROVIDER: 'mistral', MISTRAL_API_KEY: 'key' })
     mockDownload.mockResolvedValue(Buffer.from('%PDF-1.7'))
-    mockGetDocumentProxy.mockResolvedValue({ numPages: 2 })
     mockToken.mockResolvedValue('internal-token')
     mockBaseUrl.mockReturnValue('http://sim.local')
   })
@@ -107,6 +110,7 @@ describe('PDF OCR triage', () => {
     const result = await parse()
 
     expect(result.metadata.processingMethod).toBe('mistral-ocr')
+    // 1001 pages against a 1000-page request cap: two chunks, two requests.
     expect(fetchMock).toHaveBeenCalled()
   })
 
@@ -145,3 +149,34 @@ describe('PDF OCR triage', () => {
     expect(result.metadata.processingMethod).toBe('mistral-ocr')
   })
 })
+
+describe('Azure OCR chunking', () => {
+  /**
+   * Both providers cap how many pages one OCR request may carry. Mistral split the
+   * document to fit; Azure refused anything over the cap, so a long PDF could not
+   * be ingested at all. The cap belongs to a request, not to a document.
+   */
+  it('splits a PDF past the page cap instead of refusing it', async () => {
+    Object.assign(env, {
+      OCR_PROVIDER: 'azure-mistral',
+      OCR_AZURE_API_KEY: 'key',
+      OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com',
+      OCR_AZURE_MODEL_NAME: 'mistral-ocr',
+    })
+    mockParseBuffer.mockResolvedValue({ content: '', metadata: { pageCount: 2500 } })
+    mockDownload.mockResolvedValue(await pdfOfPages(1001))
+    const fetchMock = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ pages: [{ markdown: 'Recognised page' }] }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const result = await parse()
+
+    expect(result.metadata.processingMethod).toBe('mistral-ocr')
+    // 1001 pages against a 1000-page request cap: two chunks, two requests.
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+  })
+})

From fc8af91164e4fe3261cbe0bfd7ae3fd3db2b65ec Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 19 Aug 2026 11:12:26 -0700
Subject: [PATCH 5/6] fix(knowledge): keep an unsplittable PDF and an empty OCR
 response honest
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two regressions from chunking the Azure path.

Splitting loads the document, which an encrypted or malformed PDF refuses, and
that failure was deciding whether the file reached OCR at all. Those are exactly
the documents the triage routes here — no readable text layer — and the provider
may well accept bytes a local parser will not, so a failed split now sends the
document whole and leaves the page cap to the provider, as it did before it was
chunked.

An Azure response carrying no pages fell back to the raw API payload as content.
Chunked, that payload counted as recovered text and was stitched into the
document; unchunked, it satisfied the empty-content check written to catch this.
No pages is now no content, so the chunk counts as failed and the document
reports it.
---
 .../knowledge/documents/document-processor.ts | 30 +++++++++-
 .../documents/pdf-ocr-triage.test.ts          | 55 +++++++++++++++++++
 2 files changed, 83 insertions(+), 2 deletions(-)

diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts
index b4611008e9f..94f4d6c1d53 100644
--- a/apps/sim/lib/knowledge/documents/document-processor.ts
+++ b/apps/sim/lib/knowledge/documents/document-processor.ts
@@ -647,7 +647,14 @@ async function recognizeWithAzureOCR(buffer: Buffer, mimeType: string): Promise<
   )
 
   const ocrResult = (await response.json()) as AzureOCRResponse
-  return extractPageContent(ocrResult.pages || []) || JSON.stringify(ocrResult, null, 2)
+
+  /**
+   * A response carrying no pages is no content. Returning the raw payload instead
+   * would be indexed as though it were the document: stitched into a chunked run
+   * as recovered text, and in a single-document run it would satisfy the
+   * empty-content check that exists to catch exactly this.
+   */
+  return extractPageContent(ocrResult.pages || [])
 }
 
 async function parseWithMistralOCR(
@@ -840,7 +847,26 @@ async function ocrPdfInChunks(
   ) => Promise<string | null>
 ): Promise<string> {
   const totalPages = await getPdfPageCount(pdfBuffer)
-  const pdfChunks = await splitPdfIntoChunks(pdfBuffer, MISTRAL_MAX_PAGES)
+
+  /**
+   * Splitting has to load the document, which an encrypted or malformed PDF will
+   * refuse. That must not decide whether the file reaches OCR at all: those are
+   * exactly the documents with no readable text layer, so OCR is their only route,
+   * and the provider may well accept bytes that a local parser would not. When the
+   * split fails the document is sent whole and the page cap is left to the
+   * provider — the behaviour before it was chunked.
+   */
+  let pdfChunks: { buffer: Buffer; startPage: number; endPage: number }[]
+  try {
+    pdfChunks = await splitPdfIntoChunks(pdfBuffer, MISTRAL_MAX_PAGES)
+  } catch (error) {
+    logger.info('PDF could not be split for OCR, sending it whole', {
+      provider,
+      error: toError(error).message,
+    })
+    pdfChunks = [{ buffer: pdfBuffer, startPage: 0, endPage: Math.max(0, totalPages - 1) }]
+  }
+
   logger.info('Splitting PDF for OCR', {
     provider,
     totalPages,
diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
index 48e0bb927e3..5bfb99c0760 100644
--- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
+++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
@@ -179,4 +179,59 @@ describe('Azure OCR chunking', () => {
     // 1001 pages against a 1000-page request cap: two chunks, two requests.
     expect(fetchMock).toHaveBeenCalledTimes(2)
   })
+  /**
+   * Splitting loads the document, which an encrypted or malformed PDF refuses.
+   * Those are precisely the files the triage sends here — no readable text layer —
+   * so a failed split must not decide whether they reach OCR at all.
+   */
+  it('sends a PDF that cannot be split whole rather than refusing it', async () => {
+    Object.assign(env, {
+      OCR_PROVIDER: 'azure-mistral',
+      OCR_AZURE_API_KEY: 'key',
+      OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com',
+      OCR_AZURE_MODEL_NAME: 'mistral-ocr',
+    })
+    mockParseBuffer.mockRejectedValue(new Error('Invalid PDF structure.'))
+    mockDownload.mockResolvedValue(Buffer.from('not something pdf-lib can load'))
+    const fetchMock = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }] }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const result = await parse()
+
+    expect(result.metadata.processingMethod).toBe('mistral-ocr')
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  /**
+   * A response with no pages is no content. Returning the raw payload would index
+   * the API envelope as the document and satisfy the empty-content check meant to
+   * catch it.
+   */
+  it('treats an Azure response carrying no pages as empty, not as content', async () => {
+    Object.assign(env, {
+      OCR_PROVIDER: 'azure-mistral',
+      OCR_AZURE_API_KEY: 'key',
+      OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com',
+      OCR_AZURE_MODEL_NAME: 'mistral-ocr',
+    })
+    mockParseBuffer.mockResolvedValue({ content: '', metadata: {} })
+    mockDownload.mockResolvedValue(await pdfOfPages(2))
+    vi.stubGlobal(
+      'fetch',
+      vi.fn().mockResolvedValue(
+        new Response(JSON.stringify({ pages: [], usage_info: { pages_processed: 0 } }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        })
+      )
+    )
+
+    // Counted as a failed chunk rather than stitched in as recovered text.
+    await expect(parse()).rejects.toThrow(/OCR failed for all 1 chunks/)
+  })
 })

From 0dabda6cdf4af0d0234619bcf34e46e2cbcc05ed Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 19 Aug 2026 11:26:00 -0700
Subject: [PATCH 6/6] fix(knowledge): fail a PDF whose OCR only partly came
 back
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A chunked OCR run dropped any chunk that failed and returned the rest as a
normal success, so the document was marked complete with whole page ranges
absent from search and nothing downstream could tell the difference.

That contradicted the rule this change set already applies to a truncated text
layer, which is sent to OCR precisely because indexing a fragment while
reporting success is the failure being removed. A document is now indexed whole
or not at all: any missing chunk fails it, leaving it visible with a reason and
eligible for the stuck-document sweep, which can retry and produce a complete
result. Each chunk has already exhausted its own retries, so a missing one is a
real failure rather than a blip.

The page-cap test mocked fetch with a single Response object, whose body can
only be read once — the second chunk was failing on "Body already read" and the
lenient path hid it. It now returns a fresh response per call.
---
 .../knowledge/documents/document-processor.ts | 26 +++++-----
 .../documents/pdf-ocr-triage.test.ts          | 47 ++++++++++++++++---
 2 files changed, 55 insertions(+), 18 deletions(-)

diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts
index 94f4d6c1d53..ebea0141279 100644
--- a/apps/sim/lib/knowledge/documents/document-processor.ts
+++ b/apps/sim/lib/knowledge/documents/document-processor.ts
@@ -833,9 +833,9 @@ async function processChunk(
  * than being restated per provider, where they had already drifted into one
  * provider chunking and the other refusing anything over the cap.
  *
- * A chunk that fails yields `null` and is dropped: losing one section of a long
- * document is better than losing all of it. Every chunk failing is a genuine
- * failure and throws.
+ * A document is indexed whole or not at all: if any chunk fails, the document
+ * fails, because a partial result reports success while page ranges are missing
+ * and nothing downstream can tell.
  */
 async function ocrPdfInChunks(
   pdfBuffer: Buffer,
@@ -903,16 +903,18 @@ async function ocrPdfInChunks(
     .map((r) => r.content)
     .filter((content): content is string => content !== null && content.trim().length > 0)
 
-  if (recovered.length === 0) {
-    throw new Error(`OCR failed for all ${pdfChunks.length} chunks of the document`)
-  }
-
+  /**
+   * Each chunk has already exhausted its own retries, so a missing one is a real
+   * failure rather than a blip. Failing the document leaves it visible with a
+   * reason and eligible for the stuck-document sweep, which can retry it and
+   * produce a complete result — whereas indexing what came back would be
+   * indistinguishable from a document that never had those pages.
+   */
   if (recovered.length < pdfChunks.length) {
-    logger.warn('OCR recovered only part of the document', {
-      provider,
-      recovered: recovered.length,
-      chunks: pdfChunks.length,
-    })
+    throw new Error(
+      `OCR recovered ${recovered.length} of ${pdfChunks.length} chunks; ` +
+        'indexing the document would omit the rest'
+    )
   }
 
   return recovered.join('\n\n')
diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
index 5bfb99c0760..48a29293d47 100644
--- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
+++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts
@@ -165,11 +165,13 @@ describe('Azure OCR chunking', () => {
     })
     mockParseBuffer.mockResolvedValue({ content: '', metadata: { pageCount: 2500 } })
     mockDownload.mockResolvedValue(await pdfOfPages(1001))
-    const fetchMock = vi.fn().mockResolvedValue(
-      new Response(JSON.stringify({ pages: [{ markdown: 'Recognised page' }] }), {
-        status: 200,
-        headers: { 'Content-Type': 'application/json' },
-      })
+    // A fresh Response per call: a body can only be read once.
+    const fetchMock = vi.fn().mockImplementation(
+      async () =>
+        new Response(JSON.stringify({ pages: [{ markdown: 'Recognised page' }] }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        })
     )
     vi.stubGlobal('fetch', fetchMock)
 
@@ -232,6 +234,39 @@ describe('Azure OCR chunking', () => {
     )
 
     // Counted as a failed chunk rather than stitched in as recovered text.
-    await expect(parse()).rejects.toThrow(/OCR failed for all 1 chunks/)
+    await expect(parse()).rejects.toThrow(/OCR recovered 0 of 1 chunks/)
+  })
+
+  /**
+   * A document is indexed whole or not at all. Returning the chunks that did come
+   * back would mark the document complete with whole page ranges missing from
+   * search, and nothing downstream could tell it apart from a complete one.
+   */
+  it('fails the document when one chunk of several fails', async () => {
+    Object.assign(env, {
+      OCR_PROVIDER: 'azure-mistral',
+      OCR_AZURE_API_KEY: 'key',
+      OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com',
+      OCR_AZURE_MODEL_NAME: 'mistral-ocr',
+    })
+    mockParseBuffer.mockResolvedValue({ content: '', metadata: {} })
+    mockDownload.mockResolvedValue(await pdfOfPages(1001))
+
+    let call = 0
+    vi.stubGlobal(
+      'fetch',
+      vi.fn().mockImplementation(async () => {
+        call++
+        if (call === 1) {
+          return new Response(JSON.stringify({ pages: [{ markdown: 'First half' }] }), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' },
+          })
+        }
+        return new Response('upstream failure', { status: 500 })
+      })
+    )
+
+    await expect(parse()).rejects.toThrow(/OCR recovered 1 of 2 chunks/)
   })
 })