diff --git a/datashare-python/datashare_python/conftest.py b/datashare-python/datashare_python/conftest.py index 3d28dba8..40084ef7 100644 --- a/datashare-python/datashare_python/conftest.py +++ b/datashare-python/datashare_python/conftest.py @@ -48,6 +48,7 @@ _DEFAULT_AUTH = aiohttp.BasicAuth(login="guest", password="guest", encoding="utf-8") TEST_PROJECT = "test-project" +DS_URL = "https://datashare.org" _INDEX_BODY = { "mappings": { diff --git a/workers/passport-worker/entrypoints/passport_io_worker.sh b/workers/passport-worker/entrypoints/passport_io_worker.sh index bb08159b..8b4a9082 100755 --- a/workers/passport-worker/entrypoints/passport_io_worker.sh +++ b/workers/passport-worker/entrypoints/passport_io_worker.sh @@ -9,4 +9,5 @@ uv run --no-sync datashare-python worker start \ --activities passport-detection.convert-to-pdf \ --activities passport-detection.preprocess.pdfs \ --activities passport-detection.create-inference-batches \ - --activities passport-detection.aggregate-results \ No newline at end of file + --activities passport-detection.aggregate-results \ + --activities passport-detection.export-passports \ No newline at end of file diff --git a/workers/passport-worker/passport_worker/activities.py b/workers/passport-worker/passport_worker/activities.py index 57d93173..c6f10dbb 100644 --- a/workers/passport-worker/passport_worker/activities.py +++ b/workers/passport-worker/passport_worker/activities.py @@ -26,6 +26,7 @@ lifespan_image_preprocessor_cache, lifespan_pdf_converter_cache, ) +from .export import export_passports_act from .inference import ( PassportDetector, create_inference_batches_act, @@ -36,6 +37,8 @@ DocumentSearchQuery, PassportDetectionArgs, PassportDetectionResponse, + PassportExportArgs, + PassportExportResponse, PreprocessingBatches, PreprocessingConfig, ) @@ -68,9 +71,10 @@ class Activity(StrEnum): CREATE_INFERENCE_BATCHES = "passport-detection.create-inference-batches" DETECT_PASSPORTS = "passport-detection.detect-passports" AGGREGATE_RESULTS = "passport-detection.aggregate-results" + EXPORT_PASSPORTS = "passport-detection.export-passports" -class PassportDetectionActivities(ActivityWithProgress): +class SharedActivities(ActivityWithProgress): @activity_defn(name=Activity.LOAD_WORKER_CONFIG) async def worker_config( self, @@ -82,6 +86,8 @@ async def worker_config( worker_config = cast(PassportWorkerConfig, lifespan_worker_config()) return worker_config + +class PassportDetectionActivities(ActivityWithProgress): @activity_defn(name=Activity.CREATE_PREPROCESSING_BATCHES) async def create_preprocessing_batches( self, @@ -323,8 +329,38 @@ async def aggregate_results( return res +class PassportExportActivities(ActivityWithProgress): + @activity_defn(name=Activity.EXPORT_PASSPORTS) + async def export_passports( + self, + args: PassportExportArgs, + *, + progress: Annotated[ # noqa: ARG002 + AsyncProgressRateHandler | None, Weight(value=_WORKER_CONFIG_WEIGHT) + ] = None, + ) -> PassportExportResponse: + worker_config = cast(PassportWorkerConfig, lifespan_worker_config()) + paths = worker_config.paths + root = paths.artifacts + if args.root is not None: + root /= args.root + output_path = paths.workdir / args.output_path + es_client = lifespan_es_client() + output_path.parent.mkdir(parents=True, exist_ok=True) + await export_passports_act( + root, + args.format, + es_client, + output_path=output_path, + ds_base_url=worker_config.ds_url, + ) + export_path = output_path.relative_to(paths.workdir) + res = PassportExportResponse(export_path=export_path) + return res + + ACTIVITIES = [ - PassportDetectionActivities.worker_config, + SharedActivities.worker_config, PassportDetectionActivities.create_preprocessing_batches, PassportDetectionActivities.preprocess_images, PassportDetectionActivities.convert_to_pdfs, @@ -332,4 +368,5 @@ async def aggregate_results( PassportDetectionActivities.create_inference_batches, PassportDetectionActivities.detect_passports, PassportDetectionActivities.aggregate_results, + PassportExportActivities.export_passports, ] diff --git a/workers/passport-worker/passport_worker/config.py b/workers/passport-worker/passport_worker/config.py index ff39560b..1d5e2f60 100644 --- a/workers/passport-worker/passport_worker/config.py +++ b/workers/passport-worker/passport_worker/config.py @@ -117,12 +117,17 @@ class PassportTimeouts(BaseModel): result_aggregation: ActivityTimeouts = ActivityTimeouts( start_to_close=timedelta(minutes=5) ) + export: ActivityTimeouts = ActivityTimeouts( + start_to_close=timedelta(minutes=30), heartbeat=timedelta(minutes=5) + ) class PassportWorkerConfig(WorkerConfig): logging: LoggingConfig = _DEFAULT_LOGGING_CONFIG paths: WorkerPaths + ds_url: str + timeouts: PassportTimeouts = Field(default_factory=PassportTimeouts) cache: PassportWorkerCacheConfig = Field(default_factory=PassportWorkerCacheConfig) diff --git a/workers/passport-worker/passport_worker/export.py b/workers/passport-worker/passport_worker/export.py new file mode 100644 index 00000000..19181de6 --- /dev/null +++ b/workers/passport-worker/passport_worker/export.py @@ -0,0 +1,140 @@ +import json +import os +from collections import defaultdict +from collections.abc import AsyncIterable, Iterable +from io import StringIO +from itertools import batched +from pathlib import Path +from typing import Any, Never + +import pandas as pd +from aiofile import async_open +from datashare_python.constants import MANIFEST_JSON +from datashare_python.objects import ArtifactType, DocID +from icij_common.es import DOC_ROOT_ID, HITS, ID_, QUERY, SOURCE, ESClient, has_id + +from .objects import ExportFormat, PassportArtifact, Passports + + +async def export_passports_act( + root: Path, + export_format: ExportFormat, + es_client: ESClient, + *, + output_path: Path, + ds_base_url: str, +) -> None: + match export_format: + case ExportFormat.CSV: + lines = export_as_csv(root, ds_base_url=ds_base_url, es_client=es_client) + case ExportFormat.JSONL: + lines = export_as_jsonl(root) + case _: + raise ValueError(f"unsupported export format {export_format}") + async with async_open(output_path, "w") as f: + async for line in lines: + await f.write(line + "\n") + + +async def export_as_jsonl(root: Path) -> AsyncIterable[str]: + async for _, passports in _walk_passports(root): + yield passports.model_dump_json() + + +async def export_as_csv( + root: Path, ds_base_url: str, es_client: ESClient +) -> AsyncIterable[str]: + out = StringIO() + records_by_project = defaultdict(list) + async for r in _read_csv_passport_records(root): + records_by_project[r["project"]].append(r) + recs = [] + for project, records in records_by_project.items(): + root_ids = dict() + batches = batched((d["doc_id"] for d in records), 1000) + for doc_ids in batches: + root_ids |= { + k: v async for k, v in _fetch_root_ids(es_client, project, doc_ids) + } + for r in records: + doc_id = r["doc_id"] + r["root_id"] = root_ids[doc_id] + r["url"] = _make_url( + base_url=ds_base_url, + doc_id=doc_id, + root_id=r["root_id"], + project=project, + ) + recs.append(r) + df = pd.DataFrame.from_records(recs).sort_values(by=["doc_id"]) + df.to_csv(out, index=False) + out.flush() + for line in out.getvalue().split("\n"): + if line.strip(): + yield line + + +async def _read_csv_passport_records(root: Path) -> AsyncIterable[dict[str, Any]]: + async for (project, doc_id), passports in _walk_passports(root): + for doc_page in passports.pages: + page_i = doc_page.page_number + for passport_page in doc_page.passports: + as_record = {"page": page_i, "doc_id": doc_id, "project": project} + as_record["confidence"] = passport_page.confidence + mrz = passport_page.mrz + if mrz is not None: + as_record["mzr_country"] = mrz.country + for k, v in mrz.metadata.items(): + as_record[f"mrz_{k}"] = v + yield as_record + + +def _make_url(*, base_url: str, doc_id: str, root_id: str | None, project: str) -> str: + if root_id is None: + root_id = doc_id + return f"{base_url}/#/ds/{project}/{doc_id}/{root_id}?tab=viewer" + + +async def _walk_passports( + root: Path, +) -> AsyncIterable[tuple[tuple[str, DocID], Passports]]: + # For efficiency, we first check if there's a passport file then open the manifest + # to see if it's referenced + for artifact_root, _, files in os.walk(root, onerror=_raise): + artifact_root = Path(artifact_root) # noqa: PLW2901 + for f in files: + if f != PassportArtifact.filename: + continue + manifest_path = artifact_root / MANIFEST_JSON + async with async_open(manifest_path) as m_f: + manifest_json = await m_f.read() + manifest = json.loads(manifest_json) + if ArtifactType.PASSPORTS.value not in manifest: + continue + passports_path = artifact_root / f + async with async_open(passports_path) as p_f: + passports_json = await p_f.read() + passports = Passports.model_validate_json(passports_json) + doc_id = manifest_path.parent.name + project = manifest_path.parents[3].name + yield (project, doc_id), passports + + +def _raise(e: OSError) -> Never: + raise e + + +_DOC_ID_SOURCES = [DOC_ROOT_ID] + + +async def _fetch_root_ids( + es_client: ESClient, project: str, batch: Iterable[str] +) -> AsyncIterable[tuple[str, str]]: + query = {QUERY: has_id(list(batch))} + # We can't use mget because we would need the root id which is precisely what we're + # looking for + async for page in es_client.poll_search_pages( + index=project, body=query, _source_includes=_DOC_ID_SOURCES + ): + for hit in page[HITS][HITS]: + yield hit[ID_], hit[SOURCE][DOC_ROOT_ID] diff --git a/workers/passport-worker/passport_worker/objects.py b/workers/passport-worker/passport_worker/objects.py index 005ce6ac..d2c1d402 100644 --- a/workers/passport-worker/passport_worker/objects.py +++ b/workers/passport-worker/passport_worker/objects.py @@ -38,6 +38,11 @@ OnnxProvider = Literal["CUDAExecutionProvider", "CPUExecutionProvider"] +class ExportFormat(StrEnum): + CSV = "csv" + JSONL = "jsonl" + + @cache def default_country_codes() -> list[str]: import passport_service # noqa: PLC0415 @@ -133,6 +138,13 @@ def as_manifest_task_input(self) -> dict[str, Any]: return as_entry +class PassportExportArgs(TaskArgs): + project: str + output_path: Path + format: ExportFormat = ExportFormat.CSV + root: Path | None = None + + class PreprocessingBatches(BaseModel): to_pdf: Batches images: Batches @@ -210,3 +222,7 @@ def aggregate( (r.successes for r in inference_results), start=ProcessingReport() ) return cls(processed=processed, successes=successes, errors=errors) + + +class PassportExportResponse(DatashareModel): + export_path: Path diff --git a/workers/passport-worker/passport_worker/workflows.py b/workers/passport-worker/passport_worker/workflows.py index d47050de..ed2ebe1f 100644 --- a/workers/passport-worker/passport_worker/workflows.py +++ b/workers/passport-worker/passport_worker/workflows.py @@ -13,12 +13,18 @@ from datashare_python.config import ActivityTimeouts from datashare_python.utils import WorkflowWithProgress, execute_activity - from .activities import PassportDetectionActivities + from .activities import ( + PassportDetectionActivities, + PassportExportActivities, + SharedActivities, + ) from .config import PassportWorkerConfig from .objects import ( Batches, PassportDetectionArgs, PassportDetectionResponse, + PassportExportArgs, + PassportExportResponse, PreprocessingBatches, PreprocessingConfig, ) @@ -44,16 +50,15 @@ class PassportDetectionWorkflow(WorkflowWithProgress): @workflow.run async def run(self, args: PassportDetectionArgs) -> PassportDetectionResponse: logger.info("creating preprocessing batches...") - batch_args = [args.docs, args.project] # Fetch worker config worker_config = await execute_activity( - PassportDetectionActivities.worker_config, - args=batch_args, + SharedActivities.worker_config, task_queue=TaskQueue.IO, timeouts=ActivityTimeouts(start_to_close=timedelta(minutes=5)), ) worker_config = cast(PassportWorkerConfig, worker_config) # Create preprocessing batches + batch_args = [args.docs, args.project] preprocessing_batches = await execute_activity( PassportDetectionActivities.create_preprocessing_batches, args=batch_args, @@ -97,6 +102,25 @@ async def run(self, args: PassportDetectionArgs) -> PassportDetectionResponse: return response +@workflow.defn(name="passport-detection.export-passports") +class ExportWorkflow(WorkflowWithProgress): + @workflow.run + async def run(self, args: PassportExportArgs) -> PassportExportResponse: + logger.info("creating preprocessing batches...") + worker_config = await execute_activity( + SharedActivities.worker_config, + task_queue=TaskQueue.IO, + timeouts=ActivityTimeouts(start_to_close=timedelta(minutes=5)), + ) + res = await execute_activity( + PassportExportActivities.export_passports, + args=(args,), + task_queue=TaskQueue.IO, + timeouts=worker_config.timeouts.export, + ) + return res + + async def preprocess( args: PassportDetectionArgs, preprocessing_batches: PreprocessingBatches, @@ -219,4 +243,4 @@ def _process_pdfs_tasks( return all_tasks -WORKFLOWS = [PassportDetectionWorkflow] +WORKFLOWS = [PassportDetectionWorkflow, ExportWorkflow] diff --git a/workers/passport-worker/pyproject.toml b/workers/passport-worker/pyproject.toml index bd95146b..f342b354 100644 --- a/workers/passport-worker/pyproject.toml +++ b/workers/passport-worker/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ [project.optional-dependencies] preprocessing = [ "icij-passport-core[preprocessing]==0.11.8", + "pandas~=3.0", ] inference-gpu = [ "icij-passport-core[inference,gpu]==0.11.8" diff --git a/workers/passport-worker/tests/conftest.py b/workers/passport-worker/tests/conftest.py index ad0cfc76..f29edf8c 100644 --- a/workers/passport-worker/tests/conftest.py +++ b/workers/passport-worker/tests/conftest.py @@ -11,6 +11,7 @@ TemporalClientConfig, ) from datashare_python.conftest import ( # noqa: F401 + DS_URL, TEST_PROJECT, clear_dirs, dev_worker_context, @@ -70,6 +71,7 @@ def test_worker_config(tmp_path_factory: TempPathFactory) -> PassportWorkerConfi logging_config = LoggingConfig(format=LogFormat.DEFAULT, loggers=loggers) return PassportWorkerConfig( + ds_url=DS_URL, logging=logging_config, datashare=DatashareClientConfig(url="http://localhost:8080"), temporal=TemporalClientConfig(host="localhost:7233"), @@ -334,10 +336,14 @@ async def workflows_worker( ) -> AsyncGenerator[None, None]: client = test_temporal_client_session task_queue = TaskQueue.WORKFLOWS + workflows = [ + "passport-detection.detect-passports", + "passport-detection.export-passports", + ] worker_ctx = dev_worker_context( "test-passport-worklow-worker", is_async=True, - workflows=["passport-detection.detect-passports"], + workflows=workflows, worker_config=test_worker_config, client=client, task_queue=task_queue, @@ -359,6 +365,7 @@ async def io_worker( Activity.PREPROCESS_PDFS, Activity.CREATE_INFERENCE_BATCHES, Activity.AGGREGATE_RESULTS, + Activity.EXPORT_PASSPORTS, ] task_queue = TaskQueue.IO dependencies = "passport-detection.io" diff --git a/workers/passport-worker/tests/test_export.py b/workers/passport-worker/tests/test_export.py new file mode 100644 index 00000000..91b0cac8 --- /dev/null +++ b/workers/passport-worker/tests/test_export.py @@ -0,0 +1,94 @@ +from pathlib import Path + +import pytest +from datashare_python.conftest import TEST_PROJECT +from datashare_python.objects import Document, WorkerPaths +from datashare_python.utils import write_artifact +from icij_common.es import ESClient +from passport_service.objects import Passport +from passport_worker.config import PassportWorkerConfig +from passport_worker.export import export_as_csv, export_as_jsonl +from passport_worker.objects import ( + PagePassports, + PassportArtifact, + PassportDetectionArgs, + PassportDetectionConfig, + PassportInferenceConfig, + PassportManifestEntry, + Passports, + YOLOPassportDetectorConfig, +) + + +@pytest.fixture +def artifact_dir_with_passports( + populate_es: list[Document], + test_worker_config: PassportWorkerConfig, + test_model_path: Path, +) -> WorkerPaths: + passport_detector_config = YOLOPassportDetectorConfig( + model_path=test_model_path, onnx_provider="CPUExecutionProvider" + ) + args = PassportDetectionArgs( + project=TEST_PROJECT, + docs=None, + config=PassportDetectionConfig( + inference=PassportInferenceConfig( + passport_detector=passport_detector_config + ) + ), + ) + manifest_entry = PassportManifestEntry.complete(args) + paths = test_worker_config.paths + passports = [Passport(class_id="passport", confidence=1.0, box=(0, 0, 100, 100))] + for d in populate_es: + if "not" in d.path.name: + continue + passport_pages = [] + for i in range(d.metadata.get("tika_metadata_xmptpg_npages", 1)): + passport_pages.append(PagePassports(page_number=i + 1, passports=passports)) + passport_artifact = PassportArtifact( + project=d.project, + doc_id=d.doc_id, + manifest_entry=manifest_entry, + artifact=Passports(pages=passport_pages).model_dump_json().encode(), + ) + write_artifact(paths.artifacts, passport_artifact) + return paths + + +async def test_export_as_csv( + artifact_dir_with_passports: WorkerPaths, test_es_client: ESClient +) -> None: + # When + csv = [ + csv_l + async for csv_l in export_as_csv( + root=artifact_dir_with_passports.artifacts, + ds_base_url="https://my-datashare", + es_client=test_es_client, + ) + ] + + # Then + assert csv[0] == "page,doc_id,project,confidence,root_id,url" + expected_l = ( + "1,doc-2,test-project,1.0,doc-2," + "https://my-datashare/#/ds/test-project/doc-2/doc-2?tab=viewer" + ) + assert csv[1] == expected_l + assert len(csv) == 1 + 5 # header + 5 docs with a single passport page in them + + +async def test_export_as_jsonl(artifact_dir_with_passports: WorkerPaths) -> None: + # When + jsonl = [ + csv_l + async for csv_l in export_as_jsonl( + root=artifact_dir_with_passports.artifacts, + ) + ] + + # Then + assert len(jsonl) == 5 + assert isinstance(Passports.model_validate_json(jsonl[0]), Passports) diff --git a/workers/passport-worker/tests/test_objects.py b/workers/passport-worker/tests/test_objects.py index 25772030..c2175a18 100644 --- a/workers/passport-worker/tests/test_objects.py +++ b/workers/passport-worker/tests/test_objects.py @@ -1,5 +1,5 @@ import pytest -from objects import PassportProcessingError +from passport_worker.objects import PassportProcessingError @pytest.mark.regression diff --git a/workers/passport-worker/tests/test_workflows.py b/workers/passport-worker/tests/test_workflows.py index d583e806..0353d103 100644 --- a/workers/passport-worker/tests/test_workflows.py +++ b/workers/passport-worker/tests/test_workflows.py @@ -10,15 +10,22 @@ from icij_common.pydantic_utils import safe_copy from passport_worker.config import PassportWorkerConfig from passport_worker.objects import ( + ExportFormat, PassportDetectionArgs, PassportDetectionConfig, PassportDetectionResponse, + PassportExportArgs, + PassportExportResponse, PassportInferenceConfig, PassportManifestEntry, Passports, YOLOPassportDetectorConfig, ) -from passport_worker.workflows import PassportDetectionWorkflow, TaskQueue +from passport_worker.workflows import ( + ExportWorkflow, + PassportDetectionWorkflow, + TaskQueue, +) from temporalio.client import Client as TemporalClient from temporalio.worker import Worker @@ -116,3 +123,21 @@ async def test_passport_detection_workflow( # noqa: PLR0917 with_mrz = with_mrz[0] assert with_mrz.mrz is not None assert with_mrz.mrz.metadata["names"] == "JANE" + + # Given + wf_id = f"export-passports-{uuid.uuid4()}" + output_path = Path("export.csv") + args = PassportExportArgs( + project=TEST_PROJECT, format=ExportFormat.CSV, output_path=output_path + ) + res = await temporal_client.execute_workflow( + ExportWorkflow, + args, + id=wf_id, + task_queue=TaskQueue.WORKFLOWS, + ) + assert res == PassportExportResponse(export_path=output_path) + export_path = worker_paths.workdir / res.export_path + assert export_path.exists() + export_csv = export_path.read_text().splitlines() + assert len(export_csv) >= 3 diff --git a/workers/passport-worker/uv.dist.lock b/workers/passport-worker/uv.dist.lock index 79c7edd2..a3de0120 100644 --- a/workers/passport-worker/uv.dist.lock +++ b/workers/passport-worker/uv.dist.lock @@ -4,7 +4,9 @@ requires-python = ">=3.12, <3.14" resolution-markers = [ "sys_platform == 'darwin'", "platform_machine == 'aarch64' and sys_platform == 'linux'", - "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')", + "sys_platform == 'win32'", + "sys_platform == 'emscripten'", + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] [[package]] @@ -306,6 +308,7 @@ inference-gpu = [ ] preprocessing = [ { name = "icij-passport-core", extra = ["preprocessing"] }, + { name = "pandas" }, ] [package.dev-dependencies] @@ -325,6 +328,7 @@ requires-dist = [ { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.8" }, { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.8" }, { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.8" }, + { name = "pandas", marker = "extra == 'preprocessing'", specifier = "~=3.0" }, ] provides-extras = ["inference-cpu", "inference-gpu", "preprocessing"] @@ -976,6 +980,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/17/d7b106e05bfa642e8694451e7d3d759c6a241c5386a5d962e4f66c047e06/pandas-3.0.6.tar.gz", hash = "sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10", size = 4667686, upload-time = "2026-09-17T23:23:18.345Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/4c/597d588c055d4373cff19cbbc32d4dd046c7be8fadee957585b5ba9e5b24/pandas-3.0.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7dac2d65e9087e8e7b5a45fe15c4920911a221df061ab629943ce016489145c7", size = 10397411, upload-time = "2026-09-17T23:20:49.465Z" }, + { url = "https://files.pythonhosted.org/packages/18/8f/48907c7c707b61a8e5018c32e1a3f70623209bfb59020a2f6196159d3aa7/pandas-3.0.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9dab635a549e58a053c7b0fa054dc0bd7be22f0ed9a720f4a85d5fb993276172", size = 10051279, upload-time = "2026-09-17T23:20:52.409Z" }, + { url = "https://files.pythonhosted.org/packages/67/fa/613d867c3d9554a61bafdec6f79565c8a3e73235feb52cc4a72ad2e0fa6a/pandas-3.0.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3dccb584123b399c07562ac4d62543e90ede49ddf8ce3c13ffc64cbe828c281", size = 10303429, upload-time = "2026-09-17T23:20:55.597Z" }, + { url = "https://files.pythonhosted.org/packages/cb/67/0c0f18e38d7f2d2af8c24b3315bc4046e73bbdd4a5540405506671ad0c0d/pandas-3.0.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d", size = 10788193, upload-time = "2026-09-17T23:20:58.617Z" }, + { url = "https://files.pythonhosted.org/packages/39/53/1b57f3162501fe36687e4870e1b918a6458ca7386b173af75663ace95857/pandas-3.0.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e7c1905ef02c3d6d43d9dbd5b6ccb4da4870a0b0c821bbc103fbdb6f3ad2707b", size = 11323501, upload-time = "2026-09-17T23:21:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d2/b1182e8d39100369d7f13f4a125a3fb6b096fef112c46d0566c25781ff68/pandas-3.0.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:569e114072b24fc4970c12e2b4bab252671668a40b324318903380cab0254c0c", size = 11838173, upload-time = "2026-09-17T23:21:04.85Z" }, + { url = "https://files.pythonhosted.org/packages/c7/33/5b717af24d2f27995e51e0875a269dddd373045216e34cf62e3aa764eaa1/pandas-3.0.6-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:2a8fc94be2ee5f1d86f97aacd8cc566f81680b6498e76f3007421bb5d98151bf", size = 7118499, upload-time = "2026-09-17T23:21:07.661Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/14b3b17cd75cef4a1ee1af4234eb41cc1afe4103b98c2d80e8916abfd42b/pandas-3.0.6-cp312-cp312-win_amd64.whl", hash = "sha256:3ef908d28590b3f42d7070e7ad8f9b34b442b260b7f3c1afb57e0040c58cdb1b", size = 9661439, upload-time = "2026-09-17T23:21:10.959Z" }, + { url = "https://files.pythonhosted.org/packages/3b/11/3d580a604a1e35d69f6676847bd12db7d14344bc677b717f4413e79c5d0d/pandas-3.0.6-cp312-cp312-win_arm64.whl", hash = "sha256:f4e7c52eb108d752e7592268108fd3e98efd76d83a3125cdd06c621c2e44359b", size = 8959806, upload-time = "2026-09-17T23:21:13.851Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/143605a1f6443ad50ebda78a31e5a3a10147fec2590e931584aaa5ff0a09/pandas-3.0.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ae8073aed8e21d1a7fe263dcdc6840743549722a6738198a0a46000fa9476f2", size = 10418900, upload-time = "2026-09-17T23:21:16.594Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ca/87f8548f73d452aab35e4a90f8b39ae303295e0f2ef0b4055c44d6b3f1be/pandas-3.0.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:60d81f9e1799b36f3739e7fff44d1fbb2e8fd5a271b3863e03de9715fccda0fa", size = 10064785, upload-time = "2026-09-17T23:21:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/43/1a/d951442e5607c6e3b2462eff8f420797d428aa74b87c6ecfe4f48553626e/pandas-3.0.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:097090508a1dd335013d39106fc10b20f4fd4a171638e47b77d55798ed9dab6c", size = 10245290, upload-time = "2026-09-17T23:21:22.797Z" }, + { url = "https://files.pythonhosted.org/packages/50/fa/96d50e1e6cd0b08b5e2b7c838f65ae644940f75a124063380b5ef73b6866/pandas-3.0.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e92d9fa834c7d877130027cddc0cad8dcff97c1f6cca26bd6310f847228b658", size = 10757657, upload-time = "2026-09-17T23:21:25.673Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/f82d13a2cb703e1a8acee7e01fdc2b898d9cd0c00f07d1dfce63af43e350/pandas-3.0.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b27c8d890e4aa2171437ae2a39de1d215e674158e4865c4023a8b31c932513b2", size = 11249114, upload-time = "2026-09-17T23:21:28.898Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ce/8aef2e561a2f2c8b38c913c67373c65ba6748174e763d27c80271b24bd17/pandas-3.0.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8029ec0f1f89e4f985929ce1f6626dabf3140d61a4e9c1215afdab34eaf9a5d", size = 11820511, upload-time = "2026-09-17T23:21:32.11Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bd/63cb67e6903ef6d9c2871916dbcbc09d254da0fe8b870cf62e16b21945f2/pandas-3.0.6-cp313-cp313-win_amd64.whl", hash = "sha256:f3ce8a6968045481e91a3990e797e348ce13db45ee164a7095bbc824e26c09dd", size = 9638092, upload-time = "2026-09-17T23:21:34.883Z" }, + { url = "https://files.pythonhosted.org/packages/75/2e/e7b35b712edb068d382ddc8b2bea8a04974100515ba2daa22b478b265842/pandas-3.0.6-cp313-cp313-win_arm64.whl", hash = "sha256:cc39303913e2ea129915670de5d1c9fbd647f543bb72e5543bac8baa94e9e42f", size = 8952032, upload-time = "2026-09-17T23:21:37.729Z" }, +] + [[package]] name = "passporteye" version = "2.2.2" @@ -1760,6 +1794,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" diff --git a/workers/passport-worker/uv.lock b/workers/passport-worker/uv.lock index 89770e2a..36eed6aa 100644 --- a/workers/passport-worker/uv.lock +++ b/workers/passport-worker/uv.lock @@ -4,7 +4,9 @@ requires-python = ">=3.12, <3.14" resolution-markers = [ "sys_platform == 'darwin'", "platform_machine == 'aarch64' and sys_platform == 'linux'", - "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')", + "sys_platform == 'win32'", + "sys_platform == 'emscripten'", + "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] [[package]] @@ -306,6 +308,7 @@ inference-gpu = [ ] preprocessing = [ { name = "icij-passport-core", extra = ["preprocessing"] }, + { name = "pandas" }, ] [package.dev-dependencies] @@ -325,6 +328,7 @@ requires-dist = [ { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.8" }, { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.8" }, { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.8" }, + { name = "pandas", marker = "extra == 'preprocessing'", specifier = "~=3.0" }, ] provides-extras = ["inference-cpu", "inference-gpu", "preprocessing"] @@ -1004,6 +1008,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/17/d7b106e05bfa642e8694451e7d3d759c6a241c5386a5d962e4f66c047e06/pandas-3.0.6.tar.gz", hash = "sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10", size = 4667686, upload-time = "2026-09-17T23:23:18.345Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/4c/597d588c055d4373cff19cbbc32d4dd046c7be8fadee957585b5ba9e5b24/pandas-3.0.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7dac2d65e9087e8e7b5a45fe15c4920911a221df061ab629943ce016489145c7", size = 10397411, upload-time = "2026-09-17T23:20:49.465Z" }, + { url = "https://files.pythonhosted.org/packages/18/8f/48907c7c707b61a8e5018c32e1a3f70623209bfb59020a2f6196159d3aa7/pandas-3.0.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9dab635a549e58a053c7b0fa054dc0bd7be22f0ed9a720f4a85d5fb993276172", size = 10051279, upload-time = "2026-09-17T23:20:52.409Z" }, + { url = "https://files.pythonhosted.org/packages/67/fa/613d867c3d9554a61bafdec6f79565c8a3e73235feb52cc4a72ad2e0fa6a/pandas-3.0.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3dccb584123b399c07562ac4d62543e90ede49ddf8ce3c13ffc64cbe828c281", size = 10303429, upload-time = "2026-09-17T23:20:55.597Z" }, + { url = "https://files.pythonhosted.org/packages/cb/67/0c0f18e38d7f2d2af8c24b3315bc4046e73bbdd4a5540405506671ad0c0d/pandas-3.0.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d", size = 10788193, upload-time = "2026-09-17T23:20:58.617Z" }, + { url = "https://files.pythonhosted.org/packages/39/53/1b57f3162501fe36687e4870e1b918a6458ca7386b173af75663ace95857/pandas-3.0.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e7c1905ef02c3d6d43d9dbd5b6ccb4da4870a0b0c821bbc103fbdb6f3ad2707b", size = 11323501, upload-time = "2026-09-17T23:21:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d2/b1182e8d39100369d7f13f4a125a3fb6b096fef112c46d0566c25781ff68/pandas-3.0.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:569e114072b24fc4970c12e2b4bab252671668a40b324318903380cab0254c0c", size = 11838173, upload-time = "2026-09-17T23:21:04.85Z" }, + { url = "https://files.pythonhosted.org/packages/c7/33/5b717af24d2f27995e51e0875a269dddd373045216e34cf62e3aa764eaa1/pandas-3.0.6-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:2a8fc94be2ee5f1d86f97aacd8cc566f81680b6498e76f3007421bb5d98151bf", size = 7118499, upload-time = "2026-09-17T23:21:07.661Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/14b3b17cd75cef4a1ee1af4234eb41cc1afe4103b98c2d80e8916abfd42b/pandas-3.0.6-cp312-cp312-win_amd64.whl", hash = "sha256:3ef908d28590b3f42d7070e7ad8f9b34b442b260b7f3c1afb57e0040c58cdb1b", size = 9661439, upload-time = "2026-09-17T23:21:10.959Z" }, + { url = "https://files.pythonhosted.org/packages/3b/11/3d580a604a1e35d69f6676847bd12db7d14344bc677b717f4413e79c5d0d/pandas-3.0.6-cp312-cp312-win_arm64.whl", hash = "sha256:f4e7c52eb108d752e7592268108fd3e98efd76d83a3125cdd06c621c2e44359b", size = 8959806, upload-time = "2026-09-17T23:21:13.851Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/143605a1f6443ad50ebda78a31e5a3a10147fec2590e931584aaa5ff0a09/pandas-3.0.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ae8073aed8e21d1a7fe263dcdc6840743549722a6738198a0a46000fa9476f2", size = 10418900, upload-time = "2026-09-17T23:21:16.594Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ca/87f8548f73d452aab35e4a90f8b39ae303295e0f2ef0b4055c44d6b3f1be/pandas-3.0.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:60d81f9e1799b36f3739e7fff44d1fbb2e8fd5a271b3863e03de9715fccda0fa", size = 10064785, upload-time = "2026-09-17T23:21:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/43/1a/d951442e5607c6e3b2462eff8f420797d428aa74b87c6ecfe4f48553626e/pandas-3.0.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:097090508a1dd335013d39106fc10b20f4fd4a171638e47b77d55798ed9dab6c", size = 10245290, upload-time = "2026-09-17T23:21:22.797Z" }, + { url = "https://files.pythonhosted.org/packages/50/fa/96d50e1e6cd0b08b5e2b7c838f65ae644940f75a124063380b5ef73b6866/pandas-3.0.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e92d9fa834c7d877130027cddc0cad8dcff97c1f6cca26bd6310f847228b658", size = 10757657, upload-time = "2026-09-17T23:21:25.673Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/f82d13a2cb703e1a8acee7e01fdc2b898d9cd0c00f07d1dfce63af43e350/pandas-3.0.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b27c8d890e4aa2171437ae2a39de1d215e674158e4865c4023a8b31c932513b2", size = 11249114, upload-time = "2026-09-17T23:21:28.898Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ce/8aef2e561a2f2c8b38c913c67373c65ba6748174e763d27c80271b24bd17/pandas-3.0.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8029ec0f1f89e4f985929ce1f6626dabf3140d61a4e9c1215afdab34eaf9a5d", size = 11820511, upload-time = "2026-09-17T23:21:32.11Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bd/63cb67e6903ef6d9c2871916dbcbc09d254da0fe8b870cf62e16b21945f2/pandas-3.0.6-cp313-cp313-win_amd64.whl", hash = "sha256:f3ce8a6968045481e91a3990e797e348ce13db45ee164a7095bbc824e26c09dd", size = 9638092, upload-time = "2026-09-17T23:21:34.883Z" }, + { url = "https://files.pythonhosted.org/packages/75/2e/e7b35b712edb068d382ddc8b2bea8a04974100515ba2daa22b478b265842/pandas-3.0.6-cp313-cp313-win_arm64.whl", hash = "sha256:cc39303913e2ea129915670de5d1c9fbd647f543bb72e5543bac8baa94e9e42f", size = 8952032, upload-time = "2026-09-17T23:21:37.729Z" }, +] + [[package]] name = "passporteye" version = "2.2.2" @@ -1788,6 +1822,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" diff --git a/workers/workflows-worker/all_workflows_worker_entrypoint.sh b/workers/workflows-worker/all_workflows_worker_entrypoint.sh index 8bc92ddc..99836fe1 100755 --- a/workers/workflows-worker/all_workflows_worker_entrypoint.sh +++ b/workers/workflows-worker/all_workflows_worker_entrypoint.sh @@ -7,4 +7,5 @@ uv run --no-sync datashare-python worker start \ --workflow asr.transcription \ --workflow extract.markdown \ --workflow passport-detection.detect-passports \ + --workflow passport-detection.export-passports \ --workflow translation