diff --git a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts index 75751211211..d3386f69946 100644 --- a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts @@ -91,7 +91,17 @@ describe('knowledge document model-input provenance', () => { expect(fetchMock).not.toHaveBeenCalled() }) + /** + * The refusal guards egress to an external model, so it is asserted against the + * outbound request rather than the storage read. A PDF is now parsed locally + * first and only reaches OCR when it has no usable text layer — local parsing is + * not model input, as the case above establishes — so the bytes are read before + * the projection is checked, and never leave the worker when it refuses. + */ it('rejects secret-bearing opaque document bytes before external OCR', async () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + await expect( runWithKnowledgeModelInputProvenance( undefined, @@ -109,7 +119,7 @@ describe('knowledge document model-input provenance', () => { ) ).rejects.toThrow('Knowledge model input could not be safely projected') - expect(mockDownloadFileFromUrl).not.toHaveBeenCalled() + expect(fetchMock).not.toHaveBeenCalled() }) it('attaches exact-empty provenance to the internal Mistral OCR request', async () => { diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 542b84af54c..ebea0141279 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -22,6 +22,7 @@ import { resolveParserExtension, resolveStoredArtifactExtension, } from '@/lib/knowledge/documents/parser-extension' +import { assessPdfTextLayer } from '@/lib/knowledge/documents/pdf-text-layer' import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils' import { assertKnowledgeOpaqueModelInputSafe, @@ -295,6 +296,68 @@ async function getMistralApiKey(workspaceId?: string | null): Promise { + try { + const buffer = await downloadFileWithTimeout(fileUrl, userId) + const parsed = await parseBuffer(buffer, 'pdf') + + /** + * The page count comes from the same parse as the text, rather than a second + * independent read of the file. Counting separately lets the two disagree: a + * count that failed would report no pages, the density check would fall back to + * treating the document as a single page, and a long scan carrying only a header + * would look dense enough to skip OCR and be indexed as that header. + */ + const pageCount = parsed.metadata?.pageCount ?? 0 + const verdict = assessPdfTextLayer(parsed.content, pageCount, parsed.metadata?.truncated) + if (!verdict.usable) { + logger.info('PDF text layer not usable, routing to OCR', { + filename, + pageCount, + reason: verdict.reason, + }) + return undefined + } + + logger.info('Using embedded PDF text layer', { filename, pageCount }) + return { + content: parsed.content, + processingMethod: 'file-parser', + cloudUrl: undefined, + metadata: parsed.metadata, + } + } catch (error) { + logger.info('Could not read PDF text layer, routing to OCR', { + filename, + mimeType, + error: toError(error).message, + }) + return undefined + } +} + async function parseDocument( fileUrl: string, filename: string, @@ -319,14 +382,23 @@ async function parseDocument( MISTRAL_API_KEY: mistralApiKey, }).providerId - if (ocrProvider === 'azure-mistral') { - assertKnowledgeOpaqueModelInputSafe() - logger.info('Using Azure Mistral OCR') - return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId) - } + if (ocrProvider === 'azure-mistral' || ocrProvider === 'mistral') { + /** + * Most PDFs carry a usable text layer, and reading it costs nothing. OCR is + * a per-document call to an external service, so it is reserved for the + * documents that actually need it — which also means everything else stops + * depending on that service being reachable. + */ + const embedded = await readEmbeddedPdfText(fileUrl, filename, mimeType, userId) + if (embedded) return embedded - if (ocrProvider === 'mistral') { assertKnowledgeOpaqueModelInputSafe() + + if (ocrProvider === 'azure-mistral') { + logger.info('Using Azure Mistral OCR') + return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId) + } + logger.info('Using Mistral OCR') return parseWithMistralOCR(fileUrl, filename, mimeType, userId, workspaceId, mistralApiKey) } @@ -522,42 +594,19 @@ async function parseWithAzureMistralOCR( const fileBuffer = await downloadFileForBase64(fileUrl, userId) - if (mimeType === 'application/pdf') { - const pageCount = await getPdfPageCount(fileBuffer) - if (pageCount > MISTRAL_MAX_PAGES) { - throw new Error( - `PDF has ${pageCount} pages, exceeding the Azure OCR limit of ${MISTRAL_MAX_PAGES}` - ) - } - logger.info('Azure Mistral OCR: PDF page count resolved', { pageCount }) - } - - const base64Data = fileBuffer.toString('base64') - const dataUri = `data:${mimeType};base64,${base64Data}` - try { - const response = await retryWithExponentialBackoff( - () => - makeOCRRequest( - env.OCR_AZURE_ENDPOINT!, - { - 'Content-Type': 'application/json', - Authorization: `Bearer ${env.OCR_AZURE_API_KEY}`, - }, - { - model: env.OCR_AZURE_MODEL_NAME!, - document: { - type: 'document_url', - document_url: dataUri, - }, - include_image_base64: false, - } - ), - { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 } - ) - - const ocrResult = (await response.json()) as AzureOCRResponse - const content = extractPageContent(ocrResult.pages || []) || JSON.stringify(ocrResult, null, 2) + /** + * A PDF is chunked to the provider's page cap rather than refused for + * exceeding it, matching the other OCR provider. Refusing meant a long + * document could not be ingested at all, and the cap applies to a single + * request, not to the document. + */ + const content = + mimeType === 'application/pdf' + ? await ocrPdfInChunks(fileBuffer, 'azure-mistral', (chunk) => + recognizeWithAzureOCR(chunk.buffer, mimeType) + ) + : await recognizeWithAzureOCR(fileBuffer, mimeType) if (!content.trim()) { throw new Error('Azure Mistral OCR returned empty content') @@ -573,6 +622,41 @@ async function parseWithAzureMistralOCR( } } +/** Sends one document to Azure Mistral OCR inline, as a base64 data URI. */ +async function recognizeWithAzureOCR(buffer: Buffer, mimeType: string): Promise { + const dataUri = `data:${mimeType};base64,${buffer.toString('base64')}` + + const response = await retryWithExponentialBackoff( + () => + makeOCRRequest( + env.OCR_AZURE_ENDPOINT!, + { + 'Content-Type': 'application/json', + Authorization: `Bearer ${env.OCR_AZURE_API_KEY}`, + }, + { + model: env.OCR_AZURE_MODEL_NAME!, + document: { + type: 'document_url', + document_url: dataUri, + }, + include_image_base64: false, + } + ), + { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 } + ) + + const ocrResult = (await response.json()) as AzureOCRResponse + + /** + * A response carrying no pages is no content. Returning the raw payload instead + * would be indexed as though it were the document: stitched into a chunked run + * as recovered text, and in a single-document run it would satisfy the + * empty-content check that exists to catch exactly this. + */ + return extractPageContent(ocrResult.pages || []) +} + async function parseWithMistralOCR( fileUrl: string, filename: string, @@ -740,63 +824,118 @@ async function processChunk( } } -async function processMistralOCRInBatches( - filename: string, - apiKey: string, +/** + * Runs a PDF through OCR a chunk at a time and stitches the pages back together. + * + * A provider that caps how many pages one request may carry needs the document + * split, and both providers cap at the same limit — so the splitting, the + * concurrency, the ordering and the partial-failure rule live here once rather + * than being restated per provider, where they had already drifted into one + * provider chunking and the other refusing anything over the cap. + * + * A document is indexed whole or not at all: if any chunk fails, the document + * fails, because a partial result reports success while page ranges are missing + * and nothing downstream can tell. + */ +async function ocrPdfInChunks( pdfBuffer: Buffer, - userId?: string, - cloudUrl?: string -): Promise<{ - content: string - processingMethod: 'mistral-ocr' - cloudUrl?: string -}> { + provider: string, + recognize: ( + chunk: { buffer: Buffer; startPage: number; endPage: number }, + chunkIndex: number, + totalChunks: number + ) => Promise +): Promise { const totalPages = await getPdfPageCount(pdfBuffer) - logger.info(`Splitting PDF into chunks`, { totalPages, maxPagesPerChunk: MISTRAL_MAX_PAGES }) - const pdfChunks = await splitPdfIntoChunks(pdfBuffer, MISTRAL_MAX_PAGES) - logger.info( - `Split into ${pdfChunks.length} chunks, processing with concurrency ${MAX_CONCURRENT_CHUNKS}` - ) + /** + * Splitting has to load the document, which an encrypted or malformed PDF will + * refuse. That must not decide whether the file reaches OCR at all: those are + * exactly the documents with no readable text layer, so OCR is their only route, + * and the provider may well accept bytes that a local parser would not. When the + * split fails the document is sent whole and the page cap is left to the + * provider — the behaviour before it was chunked. + */ + let pdfChunks: { buffer: Buffer; startPage: number; endPage: number }[] + try { + pdfChunks = await splitPdfIntoChunks(pdfBuffer, MISTRAL_MAX_PAGES) + } catch (error) { + logger.info('PDF could not be split for OCR, sending it whole', { + provider, + error: toError(error).message, + }) + pdfChunks = [{ buffer: pdfBuffer, startPage: 0, endPage: Math.max(0, totalPages - 1) }] + } + + logger.info('Splitting PDF for OCR', { + provider, + totalPages, + chunks: pdfChunks.length, + maxPagesPerChunk: MISTRAL_MAX_PAGES, + concurrency: MAX_CONCURRENT_CHUNKS, + }) const results: { index: number; content: string | null }[] = [] for (let i = 0; i < pdfChunks.length; i += MAX_CONCURRENT_CHUNKS) { const batch = pdfChunks.slice(i, i + MAX_CONCURRENT_CHUNKS) - const batchPromises = batch.map((chunk, batchIndex) => - processChunk(chunk, i + batchIndex, pdfChunks.length, filename, apiKey, userId) - ) - - const batchResults = await Promise.all(batchPromises) - for (const result of batchResults) { - results.push(result) - } - - logger.info( - `Completed batch ${Math.floor(i / MAX_CONCURRENT_CHUNKS) + 1}/${Math.ceil(pdfChunks.length / MAX_CONCURRENT_CHUNKS)}` + const batchResults = await Promise.all( + batch.map((chunk, batchIndex) => { + const index = i + batchIndex + return recognize(chunk, index, pdfChunks.length).then( + (content) => ({ index, content }), + (error) => { + logger.warn('OCR chunk failed', { + provider, + chunk: index + 1, + error: toError(error).message, + }) + return { index, content: null } + } + ) + }) ) + results.push(...batchResults) } - const sortedResults = results + const recovered = results .sort((a, b) => a.index - b.index) - .filter((r) => r.content !== null) - .map((r) => r.content as string) - - if (sortedResults.length === 0) { + .map((r) => r.content) + .filter((content): content is string => content !== null && content.trim().length > 0) + + /** + * Each chunk has already exhausted its own retries, so a missing one is a real + * failure rather than a blip. Failing the document leaves it visible with a + * reason and eligible for the stuck-document sweep, which can retry it and + * produce a complete result — whereas indexing what came back would be + * indistinguishable from a document that never had those pages. + */ + if (recovered.length < pdfChunks.length) { throw new Error( - `OCR failed for all ${pdfChunks.length} chunks. ` + - `Large PDFs require OCR - file parser fallback would produce poor results.` + `OCR recovered ${recovered.length} of ${pdfChunks.length} chunks; ` + + 'indexing the document would omit the rest' ) } - const combinedContent = sortedResults.join('\n\n') - logger.info(`Successfully processed ${sortedResults.length}/${pdfChunks.length} chunks`) + return recovered.join('\n\n') +} - return { - content: combinedContent, - processingMethod: 'mistral-ocr', - cloudUrl, - } +async function processMistralOCRInBatches( + filename: string, + apiKey: string, + pdfBuffer: Buffer, + userId?: string, + cloudUrl?: string +): Promise<{ + content: string + processingMethod: 'mistral-ocr' + cloudUrl?: string +}> { + const content = await ocrPdfInChunks(pdfBuffer, 'mistral', (chunk, index, total) => + processChunk(chunk, index, total, filename, apiKey, userId).then((r) => r.content) + ) + + return { content, processingMethod: 'mistral-ocr', cloudUrl } } /** diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts new file mode 100644 index 00000000000..48a29293d47 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts @@ -0,0 +1,272 @@ +/** + * @vitest-environment node + * + * Every PDF used to be sent to OCR, an external per-document call, even though the + * large majority carry a usable text layer that costs nothing to read. These pin + * the routing: the text layer is tried first, and OCR is reached only when it is + * missing or unreadable. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockParseBuffer, mockDownload, mockToken, mockBaseUrl } = vi.hoisted(() => ({ + mockParseBuffer: vi.fn(), + mockDownload: vi.fn(), + mockToken: vi.fn(), + mockBaseUrl: vi.fn(), +})) + +vi.mock('@/lib/auth/internal', () => ({ generateInternalToken: mockToken })) +vi.mock('@/lib/core/utils/urls', async (importOriginal) => ({ + ...(await importOriginal()), + getInternalApiBaseUrl: mockBaseUrl, +})) + +vi.mock('@/lib/file-parsers', () => ({ + parseBuffer: mockParseBuffer, + isSupportedFileType: (extension: string) => ['pdf'].includes(extension), +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadFileFromUrl: mockDownload })) + +import { env } from '@/lib/core/config/env' +import { processDocument } from '@/lib/knowledge/documents/document-processor' +import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' + +/** External, so the OCR path uses the URL directly instead of re-uploading it. */ +const PDF_URL = 'https://example.com/Contract.pdf' +const typeset = 'The Supplier shall provide the Services described herein. '.repeat(60) + +/** A real PDF, because splitting loads the document rather than trusting metadata. */ +async function pdfOfPages(count: number): Promise { + const { PDFDocument } = await import('pdf-lib') + const pdf = await PDFDocument.create() + for (let i = 0; i < count; i++) pdf.addPage() + return Buffer.from(await pdf.save()) +} + +function parse() { + return runWithKnowledgeModelInputProvenance( + undefined, + () => processDocument(PDF_URL, 'Contract.pdf', 'application/pdf', 1024, 200, 1, 'user-1'), + { opaqueInputSafe: true } + ) +} + +describe('PDF OCR triage', () => { + beforeEach(() => { + vi.clearAllMocks() + Object.assign(env, { OCR_PROVIDER: 'mistral', MISTRAL_API_KEY: 'key' }) + mockDownload.mockResolvedValue(Buffer.from('%PDF-1.7')) + mockToken.mockResolvedValue('internal-token') + mockBaseUrl.mockReturnValue('http://sim.local') + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('uses the embedded text layer and never calls OCR', async () => { + mockParseBuffer.mockResolvedValue({ content: typeset, metadata: {} }) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + const result = await parse() + + expect(result.metadata.processingMethod).toBe('file-parser') + expect(fetchMock).not.toHaveBeenCalled() + }) + + /** + * The density check reads its page count from the same parse as the text. A long + * scan that yields only a header must stay sparse against its real page count — + * counting separately allowed a failed count to present it as a single dense page. + */ + it('takes the page count from the parse, so a header-only scan stays sparse', async () => { + // Enough to clear the floor as a single page, nowhere near enough for 80. + const headerOnly = 'CONFIDENTIAL - Vendor Master Agreement - Page header. '.repeat(6) + mockParseBuffer.mockResolvedValue({ content: headerOnly, metadata: { pageCount: 80 } }) + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }], usage_info: {} }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await parse() + + expect(result.metadata.processingMethod).toBe('mistral-ocr') + }) + + it('falls through to OCR when the PDF is a scan', async () => { + mockParseBuffer.mockResolvedValue({ content: '', metadata: {} }) + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ pages: [{ markdown: 'Recognised text' }], usage_info: {} }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await parse() + + expect(result.metadata.processingMethod).toBe('mistral-ocr') + // 1001 pages against a 1000-page request cap: two chunks, two requests. + expect(fetchMock).toHaveBeenCalled() + }) + + /** + * The case a length check alone cannot see: a CID-keyed font with no Unicode map + * yields plenty of characters, none of them words. + */ + it('falls through to OCR when the text layer is raw CID escapes', async () => { + mockParseBuffer.mockResolvedValue({ content: '/31 /8 /18 /12 /44 '.repeat(60), metadata: {} }) + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }], usage_info: {} }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await parse() + + expect(result.metadata.processingMethod).toBe('mistral-ocr') + }) + + /** An encrypted or malformed PDF has no readable layer, which is a case for OCR. */ + it('falls through to OCR when the text layer cannot be parsed at all', async () => { + mockParseBuffer.mockRejectedValue(new Error('Invalid PDF structure.')) + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }], usage_info: {} }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await parse() + + expect(result.metadata.processingMethod).toBe('mistral-ocr') + }) +}) + +describe('Azure OCR chunking', () => { + /** + * Both providers cap how many pages one OCR request may carry. Mistral split the + * document to fit; Azure refused anything over the cap, so a long PDF could not + * be ingested at all. The cap belongs to a request, not to a document. + */ + it('splits a PDF past the page cap instead of refusing it', async () => { + Object.assign(env, { + OCR_PROVIDER: 'azure-mistral', + OCR_AZURE_API_KEY: 'key', + OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com', + OCR_AZURE_MODEL_NAME: 'mistral-ocr', + }) + mockParseBuffer.mockResolvedValue({ content: '', metadata: { pageCount: 2500 } }) + mockDownload.mockResolvedValue(await pdfOfPages(1001)) + // A fresh Response per call: a body can only be read once. + const fetchMock = vi.fn().mockImplementation( + async () => + new Response(JSON.stringify({ pages: [{ markdown: 'Recognised page' }] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await parse() + + expect(result.metadata.processingMethod).toBe('mistral-ocr') + // 1001 pages against a 1000-page request cap: two chunks, two requests. + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + /** + * Splitting loads the document, which an encrypted or malformed PDF refuses. + * Those are precisely the files the triage sends here — no readable text layer — + * so a failed split must not decide whether they reach OCR at all. + */ + it('sends a PDF that cannot be split whole rather than refusing it', async () => { + Object.assign(env, { + OCR_PROVIDER: 'azure-mistral', + OCR_AZURE_API_KEY: 'key', + OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com', + OCR_AZURE_MODEL_NAME: 'mistral-ocr', + }) + mockParseBuffer.mockRejectedValue(new Error('Invalid PDF structure.')) + mockDownload.mockResolvedValue(Buffer.from('not something pdf-lib can load')) + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ pages: [{ markdown: 'Recognised' }] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await parse() + + expect(result.metadata.processingMethod).toBe('mistral-ocr') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + /** + * A response with no pages is no content. Returning the raw payload would index + * the API envelope as the document and satisfy the empty-content check meant to + * catch it. + */ + it('treats an Azure response carrying no pages as empty, not as content', async () => { + Object.assign(env, { + OCR_PROVIDER: 'azure-mistral', + OCR_AZURE_API_KEY: 'key', + OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com', + OCR_AZURE_MODEL_NAME: 'mistral-ocr', + }) + mockParseBuffer.mockResolvedValue({ content: '', metadata: {} }) + mockDownload.mockResolvedValue(await pdfOfPages(2)) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ pages: [], usage_info: { pages_processed: 0 } }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ) + ) + + // Counted as a failed chunk rather than stitched in as recovered text. + await expect(parse()).rejects.toThrow(/OCR recovered 0 of 1 chunks/) + }) + + /** + * A document is indexed whole or not at all. Returning the chunks that did come + * back would mark the document complete with whole page ranges missing from + * search, and nothing downstream could tell it apart from a complete one. + */ + it('fails the document when one chunk of several fails', async () => { + Object.assign(env, { + OCR_PROVIDER: 'azure-mistral', + OCR_AZURE_API_KEY: 'key', + OCR_AZURE_ENDPOINT: 'https://example.openai.azure.com', + OCR_AZURE_MODEL_NAME: 'mistral-ocr', + }) + mockParseBuffer.mockResolvedValue({ content: '', metadata: {} }) + mockDownload.mockResolvedValue(await pdfOfPages(1001)) + + let call = 0 + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async () => { + call++ + if (call === 1) { + return new Response(JSON.stringify({ pages: [{ markdown: 'First half' }] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + } + return new Response('upstream failure', { status: 500 }) + }) + ) + + await expect(parse()).rejects.toThrow(/OCR recovered 1 of 2 chunks/) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts b/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts new file mode 100644 index 00000000000..9b759ee9b0c --- /dev/null +++ b/apps/sim/lib/knowledge/documents/pdf-text-layer.test.ts @@ -0,0 +1,87 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { assessPdfTextLayer } from '@/lib/knowledge/documents/pdf-text-layer' + +/** Roughly the character volume of a typeset page. */ +const page = (n: number) => + 'The Supplier shall provide the Services described in this Statement of Work. '.repeat(n) + +describe('assessPdfTextLayer', () => { + it('accepts an ordinary typeset document', () => { + expect(assessPdfTextLayer(page(60), 2)).toEqual({ usable: true }) + }) + + /** + * A parser limit stops extraction partway, so the text that came back is plenty + * by volume but is only part of the document. Accepting it would index a + * fragment and drop the rest from search without saying so. + */ + it('rejects an extraction that stopped at a parser limit', () => { + expect(assessPdfTextLayer(page(200), 3, true)).toEqual({ usable: false, reason: 'truncated' }) + }) + + it('rejects a scan, which carries no text at all', () => { + expect(assessPdfTextLayer('', 12)).toEqual({ usable: false, reason: 'no-text' }) + expect(assessPdfTextLayer(' \n ', 12)).toEqual({ usable: false, reason: 'no-text' }) + }) + + /** A scan often still yields a header or a stamp — present, but not the content. */ + it('rejects text too sparse to be the document', () => { + expect(assessPdfTextLayer('CONFIDENTIAL', 40)).toEqual({ + usable: false, + reason: 'sparse-text', + }) + }) + + /** + * A CID-keyed font with no `ToUnicode` map extracts as raw character ids. There + * is plenty of it, so a length check passes and the document would be indexed as + * gibberish — the failure mode a characters-per-page test alone cannot see. + */ + it('rejects raw CID escapes from a font with no Unicode mapping', () => { + const cid = '/31 /8 /18 /12 /44 /9 /27 /15 /3 /62 '.repeat(40) + + expect(assessPdfTextLayer(cid, 1)).toEqual({ usable: false, reason: 'cid-escapes' }) + }) + + it('rejects a text layer that decoded to replacement characters', () => { + expect(assessPdfTextLayer('�'.repeat(500), 1)).toEqual({ + usable: false, + reason: 'unreadable-encoding', + }) + }) + + /** Real prose contains slashes and digits; only a dominant share is disqualifying. */ + it('keeps a document that merely mentions figures and dates', () => { + const prose = `${page(40)} Payment of /50 net 30, effective 01/04/2026, ref /12 /9.` + + expect(assessPdfTextLayer(prose, 1)).toEqual({ usable: true }) + }) + + it('keeps accented and non-Latin prose, which is ordinary text', () => { + expect(assessPdfTextLayer('Zusammenfassung über Verträge. '.repeat(40), 1)).toEqual({ + usable: true, + }) + expect(assessPdfTextLayer('契約の概要について説明します。'.repeat(40), 1)).toEqual({ + usable: true, + }) + }) + + /** An unparseable page count must still apply a floor rather than divide by zero. */ + it('treats an unknown page count as a single page', () => { + expect(assessPdfTextLayer('short', 0)).toEqual({ usable: false, reason: 'sparse-text' }) + expect(assessPdfTextLayer(page(40), 0)).toEqual({ usable: true }) + }) + + it('scales the threshold with length, so one good page does not carry a long scan', () => { + const onePageOfText = page(30) + + expect(assessPdfTextLayer(onePageOfText, 1)).toEqual({ usable: true }) + expect(assessPdfTextLayer(onePageOfText, 200)).toEqual({ + usable: false, + reason: 'sparse-text', + }) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/pdf-text-layer.ts b/apps/sim/lib/knowledge/documents/pdf-text-layer.ts new file mode 100644 index 00000000000..ec7f42804fd --- /dev/null +++ b/apps/sim/lib/knowledge/documents/pdf-text-layer.ts @@ -0,0 +1,104 @@ +/** + * Minimum average characters per page for an embedded text layer to be trusted. + * + * A typeset page carries roughly 1,500–3,000 characters and a scanned image + * carries none, so this sits an order of magnitude below real prose and well above + * the handful of characters a scan contributes from a header or a stamp. The gap + * either side is wide enough that the exact value does not matter. + */ +const MIN_CHARS_PER_PAGE = 100 + +/** + * Share of characters that must be ordinary printable text. + * + * A text layer with a broken encoding extracts as mojibake or replacement + * characters: present in quantity, but not words. + */ +const MIN_PRINTABLE_RATIO = 0.8 + +/** + * Share of characters that may look like CID escapes before the layer is rejected. + * + * A CID-keyed font with no `ToUnicode` map extracts as the raw character ids — + * `/31 /8 /18 /12` — rather than glyphs. It passes a length check comfortably while + * containing no readable text at all. Common in documents from older generators and + * in anything using subset fonts, which is much of the contract and procurement + * material that reaches a knowledge base. + */ +const MAX_CID_ESCAPE_RATIO = 0.5 + +/** Runs of the form `/31 /8`, the raw output of a CID font with no Unicode map. */ +const CID_ESCAPE_PATTERN = /\/i?\d+/g + +/** Characters that count as ordinary text: printable ASCII, whitespace, and Latin-1+. */ +const PRINTABLE_PATTERN = /[\p{L}\p{N}\p{P}\p{Zs}\n\r\t]/gu + +export type PdfTextLayerVerdict = + | { usable: true } + | { + usable: false + reason: 'no-text' | 'truncated' | 'sparse-text' | 'unreadable-encoding' | 'cid-escapes' + } + +function countMatches(text: string, pattern: RegExp): number { + let total = 0 + for (const match of text.matchAll(pattern)) total += match[0].length + return total +} + +/** + * Judges whether a PDF's embedded text layer can be indexed as-is, or whether the + * document has to go through OCR to be readable. + * + * Extracting a text layer costs nothing and covers the large majority of PDFs; + * OCR is a per-document call to an external service. Asking this question first + * means only the documents that actually need OCR pay for it, and it removes the + * dependency on that service for everything else. + * + * Four ways a text layer fails, none caught by the others: there is no text (a + * scan), extraction stopped at a parser limit so what came back is only part of + * the document, the text is too sparse to be the document's real content, or + * there is plenty of text but it is not language — a broken encoding, or raw CID + * codes from a font with no Unicode mapping. + * + * Known limitation: this judges the document as a whole, so a file that mixes + * typeset pages with scanned inserts can average out above the threshold and keep + * its partial text. Routing per page would catch that, and needs per-page + * extraction this does not currently have. + */ +export function assessPdfTextLayer( + text: string, + pageCount: number, + truncated = false +): PdfTextLayerVerdict { + const trimmed = text.trim() + if (trimmed.length === 0) return { usable: false, reason: 'no-text' } + + /** + * Checked before anything measuring volume, because a truncated extraction has + * plenty of text by definition and would otherwise read as healthy. Accepting it + * would index part of a document and silently drop the rest from search, so the + * document goes to OCR, which reads it whole. + */ + if (truncated) return { usable: false, reason: 'truncated' } + + /** + * An unknown page count (a PDF whose header would not parse) is treated as a + * single page: it still applies a floor, without inventing a page count that + * would scale the threshold arbitrarily. + */ + const pages = pageCount > 0 ? pageCount : 1 + if (trimmed.length / pages < MIN_CHARS_PER_PAGE) return { usable: false, reason: 'sparse-text' } + + const cidChars = countMatches(trimmed, CID_ESCAPE_PATTERN) + if (cidChars / trimmed.length > MAX_CID_ESCAPE_RATIO) { + return { usable: false, reason: 'cid-escapes' } + } + + const printableChars = countMatches(trimmed, PRINTABLE_PATTERN) + if (printableChars / trimmed.length < MIN_PRINTABLE_RATIO) { + return { usable: false, reason: 'unreadable-encoding' } + } + + return { usable: true } +}