feature(extract-python): implement LiteParsePipeline

.github/workflows/tests-extract-python.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Run tests
         run: |
           cd extract-python
-          uv sync --dev --extra docling --extra marker --frozen
+          uv sync --dev --extra docling --extra marker --extra liteparse --frozen
           uv run --frozen pytest -m "not miner_u" -vvv --cache-clear --show-capture=all -r A tests
 
   tests-miner-u:

extract-core/extract_core/__init__.py

@@ -56,6 +56,11 @@
 except ModuleNotFoundError:
     MinerUBackend, MinerUPipelineConfig, MinerUConfig = None, None, None
 
+try:
+    from .liteparse_ import LiteParseConfig, LiteParsePipelineConfig
+except ModuleNotFoundError:
+    LiteParsePipelineConfig, LiteParseConfig = None, None
+
 
 _pipeline_type_discriminator = make_enum_discriminator("pipeline", PipelineType)
 
@@ -81,6 +86,8 @@ def pipeline_config_discriminator(v: Any) -> str:
 
 
 __all__ = [
+    "LiteParsePipelineConfig",
+    "LiteParseConfig",
     "BaseModel",
     "BasePipelineConfig",
     "ConversionOutput",

extract-core/extract_core/configs.py

@@ -12,6 +12,7 @@
 
 class PipelineType(StrEnum):
     DOCLING = "docling"
+    LITEPARSE = "liteparse"
     MARKER = "marker"
     MINER_U = "miner_u"
 

extract-core/extract_core/default_config.py

@@ -12,6 +12,8 @@ def default_config(
             return _default_marker_config(size)
         case PipelineType.MINER_U:
             return _default_mineru_config(size)
+        case PipelineType.LITEPARSE:
+            return _default_liteparse_config()
         case _:
             raise NotImplementedError(f"unsupported pipeline type {pipeline_type}")
 
@@ -46,3 +48,9 @@ def _default_mineru_config(size: PipelineSize) -> BasePipelineConfig:
         case _:
             raise TypeError(f"unsupported pipeline size {size}")
     return MinerUPipelineConfig(config=MinerUConfig(tier=tier))
+
+
+def _default_liteparse_config() -> BasePipelineConfig:
+    from .liteparse_ import LiteParsePipelineConfig  # noqa: PLC0415
+
+    return LiteParsePipelineConfig()

extract-core/extract_core/liteparse_.py

@@ -0,0 +1,87 @@
+import os
+from copy import copy
+from datetime import timedelta
+from pathlib import Path
+from typing import ClassVar, Self
+
+from pydantic import Field
+
+from .configs import BasePipelineConfig, PipelineType, ResultBufferConfig
+from .objects import BaseModel, Device, SupportedExt
+
+_SUPPORTED_EXTS = {
+    SupportedExt.DOC,
+    SupportedExt.DOCX,
+    SupportedExt.DOCM,
+    SupportedExt.ODT,
+    SupportedExt.RTF,
+    SupportedExt.PAGES,
+    SupportedExt.PPT,
+    SupportedExt.PPTX,
+    SupportedExt.PPTM,
+    SupportedExt.ODP,
+    SupportedExt.KEY,
+    SupportedExt.XLS,
+    SupportedExt.XLSX,
+    SupportedExt.XLSM,
+    SupportedExt.ODS,
+    SupportedExt.CSV,
+    SupportedExt.TSV,
+    SupportedExt.NUMBERS,
+    SupportedExt.JPG,
+    SupportedExt.JPEG,
+    SupportedExt.PNG,
+    SupportedExt.GIF,
+    SupportedExt.BMP,
+    SupportedExt.TIFF,
+    SupportedExt.WEBP,
+    SupportedExt.SVG,
+}
+
+
+class LiteParseConfig(BaseModel):
+    # TODO: move this to extract API
+    ocr_language: str = "eng"
+    dpi: float = 300
+    preserve_very_small_text: bool = False
+    quiet: bool = False
+    extract_annotations: bool = True
+    extract_form_fields: bool = True
+    extract_xfa_packets: bool = True
+    extract_document_metadata: bool = False
+    extract_content_bounds: bool = True
+    detect_screenshot_rects: bool = False
+    render_form_fields: bool = False
+    emit_word_boxes: bool = False
+    extract_text_metadata: bool = False
+    skip_diagonal_text: bool = False
+    extract_vector_graphics: bool = False
+
+
+def _default_n_cpu() -> int:
+    return os.cpu_count() - 1
+
+
+class LiteParsePipelineConfig(BasePipelineConfig):
+    pipeline: ClassVar[PipelineType] = Field(
+        frozen=True, default=PipelineType.LITEPARSE
+    )
+    device: Device = Field(default=Device.CPU, frozen=True)
+
+    config: LiteParseConfig = Field(default_factory=LiteParseConfig)
+    tessdata_path: Path | None = None
+
+    detect_page_orientation: bool = True
+
+    n_cpus: int = Field(default_factory=_default_n_cpu)
+    page_batch_size: int = 8
+    per_page_timeout: timedelta | None = None
+
+    result_buffer: ResultBufferConfig = Field(default_factory=ResultBufferConfig)
+
+    @classmethod
+    def supported_exts(cls) -> set[SupportedExt]:
+        return copy(_SUPPORTED_EXTS)
+
+    def to_device(self) -> Self:
+        return self

extract-core/extract_core/objects.py

@@ -106,11 +106,13 @@ class SupportedExt(StrEnum):
     QMD = ".qmd"
     RMD = ".rmd"
     RTF = ".rtf"
+    SVG = ".svg"
     TEX = ".tex"
+    TEXT = ".text"
     TIF = ".tif"
     TIFF = ".tiff"
+    TSV = ".tsv"
     TXT = ".txt"
-    TEXT = ".text"
     WEBP = ".webp"
     XBRL = ".xbrl"
     XHTML = ".xhtml"
@@ -211,6 +213,12 @@ def from_docling(cls, docling_error) -> Self:  # noqa: ANN001
         )
         return cls(id=error_id, title=title, detail=detail)
 
+    @classmethod
+    def from_liteparse(cls, liteparse_error) -> Self:  # noqa: ANN001
+        title = "LiteParseConversionError"
+        error_id = f"{_id_title(title)}-{uuid.uuid4().hex}"
+        return cls(id=error_id, title=title, detail=liteparse_error.message)
+
 
 def _id_title(title: str) -> str:
     id_title = []

extract-python/.python-version

@@ -1 +1 @@
-3.14
+3.13

extract-python/extract_python/__init__.py

@@ -21,10 +21,16 @@
     logger.warning("mineru is not available, might be optional...")
     MinerUPipeline = None
 
+try:
+    from .liteparse_ import LiteParsePipeline
+except ModuleNotFoundError:
+    logger.warning("liteparse is not available, might be optional...")
+    LiteParsePipeline = None
 
 __all__ = [
     "DoclingPipeline",
     "DOCLING_DEFAULT_ARTIFACTS_PATH",
     "MarkerPipeline",
     "MinerUPipeline",
+    "LiteParsePipeline",
 ]

extract-python/extract_python/docling_.py

@@ -101,8 +101,8 @@ async def extract_content(
                     if res is sentinel:
                         break
                     pages = next(processed)
-                    buffer.add(pages, res)
-                    if buffer.is_complete(pages.doc_idx):
+                    is_complete = buffer.add(pages, res)
+                    if is_complete:
                         doc_pages = buffer.pop_complete(pages.doc_idx)
                         yield _to_result(
                             doc_pages, pages.doc, output_format, output_path=output_path