diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4fff926..6596c127 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,6 +97,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Pull Tesseract LFS libraries + # Only the OCR libraries, so the processing spec tests can run OCR + # cases; a full LFS checkout would also fetch the large LibreOffice + # bundle, which the tests do not need + run: git lfs pull --include "documentcloud/documents/processing/ocr/tesseract/*" + - name: Cache OCR language data + uses: actions/cache@v4 + with: + path: documentcloud/documents/processing/tests/spec/.cache + key: processing-spec-tessdata-7d4322bd - name: Install Python uses: actions/setup-python@v5 with: @@ -112,3 +122,6 @@ jobs: PG_USER: test PG_PASSWORD: ${{ secrets.PG_PASSWORD }} DATABASE_URL: postgres://test:postgres@127.0.0.1:5432/test + # OCR is expected to work in CI: fail the processing spec's OCR + # cases instead of skipping them if the runtime is unavailable + DC_SPEC_REQUIRE_OCR: "1" diff --git a/documentcloud/documents/processing/tests/spec/.gitignore b/documentcloud/documents/processing/tests/spec/.gitignore new file mode 100644 index 00000000..ceddaa37 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/.gitignore @@ -0,0 +1 @@ +.cache/ diff --git a/documentcloud/documents/processing/tests/spec/README.md b/documentcloud/documents/processing/tests/spec/README.md new file mode 100644 index 00000000..a1f5cac3 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/README.md @@ -0,0 +1,216 @@ +# Processing Pipeline Test Spec + +Golden-output tests for the document processing pipeline. For each test +document in `documents/`, the `expected/` directory contains **every file the +current pipeline produces** for that document, plus the database-facing +metadata sent back to the API. The goal is to be able to change processing +code and validate that the outputs (or exactly the outputs you intended to +change) stay the same. + +This builds on the output-contract spec in +[MuckRock/research `processing-pipeline-spec`](https://github.com/MuckRock/research/tree/main/processing-pipeline-spec), +which documents *what* the pipeline must produce. This directory makes that +contract executable against the real implementation. + +## Layout + +``` +spec/ +├── harness.py # runs the real pipeline in-process (no mocks) +├── compare.py # comparison / normalization rules +├── generate.py # regenerates the golden outputs +├── test_spec.py # pytest: fresh run must match expected/ +├── make_corpus.py # builds the input PDFs (deterministic) +└── documents/ + └── / + ├── case.json # doc_id, slug, processing settings, redactions + ├── input/ # the "uploaded" document + └── expected/ # everything the pipeline produced + ├── .pdf # processed PDF (OCR text grafted in) + ├── .txt # concatenated text, pages joined by \n\n + ├── .txt.json # per-page text + OCR metadata + ├── .index # I/O cache (implementation artifact) + ├── pages/ + │ ├── -p-{xlarge,large,normal,small,thumbnail}.gif + │ ├── -p.txt + │ └── -p.position.json + └── metadata.json # captured API callbacks: page_count, + # page_spec, file_hash, status +``` + +`metadata.json` is not a pipeline storage file — it records what the pipeline +PATCHed back to the API (the `database` key is the merged final state, and +`callbacks` is the raw sequence of requests). + +## How the harness works + +`harness.PipelineRunner` runs the actual serverless functions +(`info_and_image/main.py`, `ocr/main.py`) with: + +- the `local` storage environment writing under a temporary `MEDIA_ROOT` +- a real Redis (`REDIS_PROCESSING_URL`, defaults to `localhost:6379` — the + same requirement the existing pipeline tests have in CI) +- pubsub topics dispatched through an in-process FIFO queue, so each function + runs to completion before the next starts (in production every message runs + in its own Lambda; pdfium cannot be nested inside one process) +- API callbacks (`serverless.utils.request`) captured instead of sent + +Nothing inside the pipeline is mocked: pdfium rendering and text extraction, +Tesseract OCR, PDF grafting, pdfplumber text positions, and page_spec +crunching all run for real. + +## Test corpus + +| Case | Pages | What it validates | +| --- | --- | --- | +| `text-1page` | 1 | Baseline: embedded text, no OCR, full output file set | +| `text-3page` | 3 | All pages get all artifacts; concatenation order | +| `text-4page` | 4 | More pages than `TEXT_POSITION_BATCH` (3): multi-batch flush path | +| `scan-2page` | 2 | Image-only pages: OCR text, grafting, OCR positions | +| `mixed-2page` | 2 | Per-page routing: page 1 `ocr: null`, page 2 `ocr: "tess4"` | +| `force-ocr-1page` | 1 | `force_ocr` overrides embedded text (`ocr: "tess4_force"`) | +| `mixed-sizes-2page` | 2 | Differing page dimensions; page_spec encoding | +| `blank-1page` | 1 | Blank page is OCR'd; produces `"\f"` text and `[]` positions | +| `redact-2page` | 2 | Redaction path: only the dirty page is reprocessed; page_spec is not resent | +| `corrupt-1page` | — | Error path: an unparseable PDF produces an error POST to the API and no output files | + +Inputs are built deterministically by `make_corpus.py` and committed, so the +goldens always correspond to known input bytes. + +## Running the validation + +```bash +pytest documentcloud/documents/processing/tests/spec/test_spec.py +``` + +This runs in CI alongside the other tests (it needs the Redis service the +existing pipeline tests already use). The CI workflow pulls the Tesseract +LFS libraries so the OCR cases run there too; in a local checkout without +them, OCR cases are skipped and the embedded-text cases still run. The +pinned eng.traineddata is downloaded automatically on first use. + +To check outside of pytest: + +```bash +python documentcloud/documents/processing/tests/spec/generate.py --check +``` + +## Regenerating the goldens + +After an *intentional* change to pipeline behavior: + +```bash +python documentcloud/documents/processing/tests/spec/generate.py [case ...] +``` + +then review the diff — it is a precise inventory of what your change did to +the pipeline's output — and commit it. + +### OCR requirements + +The OCR cases run the bundled Tesseract, which is stored in Git LFS: + +```bash +git lfs pull --include "documentcloud/documents/processing/ocr/tesseract/*" +``` + +OCR output is only comparable when produced with the same language data, so +the harness pins `eng.traineddata` by content hash +(`tessdata_fast` 4.1.0; `generate.py` downloads it into `.cache/` on first +use). If you want goldens that match production OCR exactly, place the +production `eng.traineddata` (from the `ocr-languages` storage folder) in +`.cache/` instead and regenerate — but note the committed goldens must then +be produced with that same file. + +## Validating a modified or replacement pipeline + +The corpus and comparison logic are deliberately independent of the current +implementation: `documents/` is a set of input → expected-output pairs, and +`compare.compare_case(expected_dir, actual_dir)` only looks at two +directories. `harness.py` is the *current* pipeline's runner; to validate a +replacement, write a runner for it that stages each case's input, produces a +document directory in the same layout, and records the final database +metadata (plus any error reports) in `metadata.json` — then compare with +`strictness="equivalent"`. + +Two strictness modes: + +- **`exact`** (default, used by `test_spec.py` and CI) answers "is the + current pipeline unchanged?" — byte-exact text and images, full API + callback sequence. This intentionally pins implementation details, + including quirks like pdfium's trailing NUL and Tesseract's trailing form + feed, so any change is visible. +- **`equivalent`** (`generate.py --check --equivalent`, or + `strictness="equivalent"`) answers "does a different implementation + produce equivalent final output?" — trailing control characters are + normalized away, OCR text is held to a similarity threshold instead of + equality, images to a mean pixel delta instead of identical bytes, the + `ocr` field to OCR'd-or-not instead of an engine name, and the callback + sequence is not compared (though error reports must still be sent). The + thresholds at the top of `compare.py` are a first cut — tune them against + real replacement candidates. + +Open question for the team: which of the pinned quirks are actually +depended on downstream (search indexing, the frontend, add-ons)? The +equivalence rules currently treat the trailing NUL/form-feed as accidents a +replacement is free to fix; if a consumer turns out to rely on one, move it +into the contract and tighten the rule. + +## Comparison rules (exact mode) + +Some pipeline output is intentionally or unavoidably non-reproducible, so +`compare.py` normalizes even in exact mode: + +| Output | Rule | +| --- | --- | +| `*.txt` (full and per-page) | exact bytes | +| `*.txt.json` | JSON equality; `updated` timestamps ignored (wall clock) | +| `*.position.json` | JSON equality; coordinate floats within `1e-4` | +| `*.gif` | exact bytes; on mismatch, pixels are diffed and reported | +| `*.pdf` | semantic: page count, page dimensions, per-page text layer. Bytes are not compared — the OCR grafter writes uuid-named XObjects and pikepdf regenerates the document ID | +| `*.index` | presence only (gzip bytes embed a timestamp; not part of the contract) | +| `metadata.json` | the merged database fields **and** the full API callback sequence (order, methods, URLs, payloads). `page_count` and `status` exact; `page_spec` compared decoded (segment order follows Redis set iteration and is not deterministic); `file_hash` exact, except for redaction cases where the final PDF is rewritten non-reproducibly — there the callbacks still assert that a well-formed SHA-1 was sent, just not its value. Storage bucket prefixes in error messages are stripped (the bucket name varies by environment) | + +Behaviors of the current pipeline that the goldens capture and that any +reimplementation would need to match (or knowingly change): + +- Text extracted by pdfium ends with a trailing NUL (`\x00`) character +- Text produced by Tesseract ends with a form feed (`\f`); a blank page + yields `"\f"`, not the empty string +- Pages in `.txt` are joined with `\n\n` +- `ocr` values in `txt.json`: `null` (embedded text), `"tess4"`, + `"tess4_force"` (when `force_ocr` is set) +- The `updated` timestamps are milliseconds; each page carries its own +- `file_hash` is the SHA-1 of the PDF as hashed during page-cache processing + (before OCR grafting rewrites it) +- Redaction re-sends `file_hash` and `status` but not `page_spec` + +## Not covered yet + +- **Textract OCR** (`ocr_engine: "textract"`) — requires AWS and an AI-credit + API; the golden harness only runs Tesseract +- **Non-PDF uploads** — document conversion needs the LibreOffice bundle + (`document_conversion/libreoffice/lo.tar.gz`, a large LFS object) +- **Page modifications** (reorder / rotate / insert) — the modify path needs + `storage.async_download`, which the local storage backend does not implement +- **Bulk import**, **set_page_text**, revision control copies +- **Non-English OCR** — needs additional pinned traineddata files +- **Large documents** — every case fits in a single `IMAGE_BATCH` (55 pages); + multi-batch image extraction is not exercised (text-position batching is, + via `text-4page`, and OCR batching via `scan-2page` since `OCR_BATCH` is 1) +- **Access levels** — all cases run with `access: private`; local storage + ignores ACLs, so a regression in how `access` propagates between pipeline + messages would not be caught +- **Error paths beyond an unparseable input** (`corrupt-1page`) — e.g. + OCR failures, storage failures, and the retry queue +- **Real-world documents** — the corpus is synthetic (built with PyMuPDF), + so rotated pages, non-Latin/RTL text, ligatures, skewed scans, forms, + broken xref tables, JPEG2000 images, and unusual encodings are not + represented. Adding a handful of small, redistributable real documents is + the highest-value corpus improvement for replacement confidence + +Platform note: the GIF goldens are byte-compared and therefore pinned to the +Linux builds of pdfium/Pillow used in CI. Run the suite on Linux (or in the +dev container); a macOS run will report pixel deltas, and the bundled +Tesseract libraries only load on Linux x86-64 (OCR cases skip elsewhere — +or fail if `DC_SPEC_REQUIRE_OCR` is set, as it is in CI). diff --git a/documentcloud/documents/processing/tests/spec/__init__.py b/documentcloud/documents/processing/tests/spec/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/documentcloud/documents/processing/tests/spec/compare.py b/documentcloud/documents/processing/tests/spec/compare.py new file mode 100644 index 00000000..4d8d89fa --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/compare.py @@ -0,0 +1,595 @@ +""" +Compare a freshly-processed document directory against a committed golden +(`expected/`) directory. + +Two strictness modes serve two different questions: + +``exact`` (default) — "is the current pipeline unchanged?" Used by +test_spec.py for regression testing. Rules, per file type: + +- ``pages/*.txt``, ``{slug}.txt`` exact bytes +- ``{slug}.txt.json`` JSON equality with all ``updated`` + timestamps normalized (they are wall + clock times) +- ``pages/*.position.json`` JSON equality with a small float + tolerance on coordinates +- ``pages/*.gif`` exact bytes; on mismatch the images are + decoded and the pixel difference is + reported to distinguish real rendering + changes from metadata drift +- ``{slug}.pdf`` semantic comparison (page count, page + dimensions, per-page extracted text). + Byte comparison is skipped: the OCR + grafter writes uuid-named XObjects and + non-reproducible ids +- ``{slug}.index``, ``*.pagesize`` presence only (implementation artifacts, + not part of the output contract; the + index is a gzip whose bytes embed a + timestamp) +- ``metadata.json`` the merged database fields and the full + API callback sequence (order, methods, + URLs, payloads), with page_spec decoded, + file_hash loosened to presence+shape for + redaction cases, and bucket prefixes + stripped from error messages + +``equivalent`` — "does a different implementation produce equivalent final +output?" For validating a modified or replacement pipeline (different +renderer, OCR engine, or PDF library) against the same goldens: + +- text is compared with trailing control characters normalized away (the + current pipeline's trailing NUL from pdfium and trailing form feed from + Tesseract are treated as implementation accidents, not contract); + OCR-produced text must reach a similarity ratio rather than match exactly +- images must have the same dimensions and a small mean pixel difference + rather than identical bytes +- positions must parse, stay within the 0-1 coordinate range, agree on + emptiness, and carry approximately the same text +- the PDF must have the same page count, dimensions within tolerance, and + a similar text layer +- metadata: page_count and status exact; page_spec decoded with a small + dimension tolerance; file_hash present and well-formed but not compared + by value; the callback *sequence* is not compared (message choreography + is not the contract), but error reports must still be sent +- txt.json: page structure and lang exact; the ``ocr`` field is compared + as OCR'd-or-not rather than by engine name + +The equivalence thresholds are a first cut — tune them as real replacement +candidates are evaluated. +""" + +# Standard Library +import difflib +import io +import json +import re +from pathlib import Path + +POSITION_TOLERANCE = 1e-4 + +PRESENCE_ONLY_SUFFIXES = (".index", ".pagesize") + +# Equivalence-mode thresholds +OCR_TEXT_SIMILARITY = 0.90 +POSITION_TEXT_SIMILARITY = 0.80 +PDF_TEXT_SIMILARITY = 0.90 +IMAGE_MEAN_DELTA = 4.0 # mean absolute per-channel difference, 0-255 +DIMENSION_TOLERANCE = 0.5 # PDF points + +# Trailing characters the current implementation emits but which are not +# treated as contract in equivalence mode: pdfium terminates extracted text +# with NUL, Tesseract with a form feed +CONTROL_CHARS = "\x00\x0c" + +PAGE_FILE = re.compile(r"-p(\d+)\.(txt|position\.json)$") +SHA1_HEX = re.compile(r"^[0-9a-f]{40}$") +# Error messages may embed storage paths whose bucket prefix varies by +# environment; compare from the documents/ segment on +BUCKET_PREFIX = re.compile(r"[^\s']*documents/") + + +def compare_case( + expected_dir, actual_dir, ignore_metadata_fields=(), strictness="exact" +): + """Compare two document directories. Returns a list of human-readable + problems; an empty list means the directories match. + + ignore_metadata_fields lists metadata.json database fields excluded from + comparison — used for file_hash on redaction cases, where the final PDF + is rewritten with non-reproducible ids so its hash differs run to run. + + strictness is "exact" (regression testing the current pipeline) or + "equivalent" (validating a replacement implementation); see the module + docstring.""" + assert strictness in ("exact", "equivalent") + expected_dir = Path(expected_dir) + actual_dir = Path(actual_dir) + context = { + "strictness": strictness, + "ignore_fields": ignore_metadata_fields, + # 0-indexed page numbers the golden marks as OCR'd, for text + # similarity rules in equivalence mode + "ocr_pages": _expected_ocr_pages(expected_dir), + } + problems = [] + + expected_files = _relative_files(expected_dir) + actual_files = _relative_files(actual_dir) + + for missing in sorted(expected_files - actual_files): + problems.append(f"missing output file: {missing}") + for extra in sorted(actual_files - expected_files): + problems.append(f"unexpected output file: {extra}") + + for name in sorted(expected_files & actual_files): + problems.extend( + _compare_file(expected_dir / name, actual_dir / name, name, context) + ) + + return problems + + +def _relative_files(directory): + return { + str(file_path.relative_to(directory)) + for file_path in directory.rglob("*") + if file_path.is_file() + } + + +def _expected_ocr_pages(expected_dir): + """The set of 0-indexed pages the golden txt.json marks as OCR'd.""" + for json_text_path in Path(expected_dir).glob("*.txt.json"): + document = json.loads(json_text_path.read_text()) + return {page["page"] for page in document.get("pages", []) if page.get("ocr")} + return set() + + +def _page_number(name): + """0-indexed page number of a per-page file, or None.""" + match = PAGE_FILE.search(name) + return int(match.group(1)) - 1 if match else None + + +def _normalize_text(text): + return text.strip(CONTROL_CHARS + " \t\r\n") + + +def _similarity(expected_text, actual_text): + return difflib.SequenceMatcher( + None, _normalize_text(expected_text), _normalize_text(actual_text) + ).ratio() + + +def _compare_file(expected, actual, name, context): + # pylint: disable=too-many-return-statements + if name == "metadata.json": + return _compare_metadata(expected, actual, name, context) + if name.endswith(PRESENCE_ONLY_SUFFIXES): + return [] # presence already checked + if name.endswith(".position.json"): + return _compare_positions(expected, actual, name, context) + if name.endswith(".txt.json"): + return _compare_json_text(expected, actual, name, context) + if name.endswith(".txt"): + return _compare_text(expected, actual, name, context) + if name.endswith(".gif"): + return _compare_images(expected, actual, name, context) + if name.endswith(".pdf"): + return _compare_pdfs(expected, actual, name, context) + return _compare_bytes(expected, actual, name) + + +def _compare_bytes(expected, actual, name): + if expected.read_bytes() != actual.read_bytes(): + return [f"{name}: contents differ"] + return [] + + +def _compare_text(expected, actual, name, context): + if context["strictness"] == "exact": + return _compare_bytes(expected, actual, name) + expected_text = expected.read_text(encoding="utf-8") + actual_text = actual.read_text(encoding="utf-8") + page = _page_number(name) + # A page is held to the similarity threshold if the golden marks it as + # OCR'd; the concatenated document text is if any page was OCR'd + ocr = ( + page in context["ocr_pages"] if page is not None else bool(context["ocr_pages"]) + ) + if ocr: + ratio = _similarity(expected_text, actual_text) + if ratio < OCR_TEXT_SIMILARITY: + return [ + f"{name}: OCR text similarity {ratio:.3f} below " + f"{OCR_TEXT_SIMILARITY}" + ] + return [] + if _normalize_text(expected_text) != _normalize_text(actual_text): + return [f"{name}: text differs (after trailing-control normalization)"] + return [] + + +def _compare_metadata(expected, actual, name, context): + expected_metadata = json.loads(expected.read_text()) + actual_metadata = json.loads(actual.read_text()) + problems = _compare_database( + expected_metadata["database"], + actual_metadata["database"], + name, + context, + ) + if context["strictness"] == "exact": + problems.extend( + _compare_callbacks( + expected_metadata.get("callbacks", []), + actual_metadata.get("callbacks", []), + name, + loose_file_hash="file_hash" in context["ignore_fields"], + ) + ) + else: + # Message choreography is not the contract for a replacement, but + # error reports are: the same error endpoints must be hit + expected_errors = [ + (callback["method"], callback["url"]) + for callback in expected_metadata.get("errors", []) + ] + actual_errors = [ + (callback["method"], callback["url"]) + for callback in actual_metadata.get("errors", []) + ] + if expected_errors != actual_errors: + problems.append( + f"{name}: error reports differ: expected {expected_errors!r}, " + f"got {actual_errors!r}" + ) + return problems + + +def _compare_database(expected_database, actual_database, name, context): + equivalent = context["strictness"] == "equivalent" + problems = [] + for field in sorted(set(expected_database) | set(actual_database)): + if field in context["ignore_fields"]: + continue + expected_value = expected_database.get(field) + actual_value = actual_database.get(field) + if field == "page_spec": + # The segment order within a page_spec depends on Redis set + # iteration order and is not deterministic; compare the decoded + # per-page dimensions instead + tolerance = DIMENSION_TOLERANCE if equivalent else 0.0 + if not _page_specs_match(expected_value, actual_value, tolerance): + problems.append( + f"{name}: page_spec decodes differently: " + f"expected {expected_value!r}, got {actual_value!r}" + ) + elif field == "file_hash" and equivalent: + # A replacement produces different PDF bytes; require a + # well-formed hash, not the same one + if not (isinstance(actual_value, str) and SHA1_HEX.match(actual_value)): + problems.append( + f"{name}: file_hash is not a well-formed SHA-1: " + f"{actual_value!r}" + ) + elif expected_value != actual_value: + problems.append( + f"{name}: database field {field!r} differs: " + f"expected {expected_value!r}, got {actual_value!r}" + ) + return problems + + +def _normalize_callback(callback, loose_file_hash): + """Normalize one captured API callback for comparison. + + - page_spec values are decoded (segment order is not deterministic) + - file_hash values are replaced by a placeholder when loose_file_hash is + set (redaction rewrites the PDF non-reproducibly), so the sequence + still asserts that a well-formed hash was sent, just not its value + - storage bucket prefixes in error messages are stripped + """ + json_ = dict(callback.get("json", {})) + if "page_spec" in json_: + json_["page_spec"] = _decode_page_spec(json_["page_spec"]) + if ( + loose_file_hash + and isinstance(json_.get("file_hash"), str) + and SHA1_HEX.match(json_["file_hash"]) + ): + json_["file_hash"] = "" + if isinstance(json_.get("message"), str): + json_["message"] = BUCKET_PREFIX.sub("documents/", json_["message"]) + return { + "method": callback.get("method"), + "url": callback.get("url"), + "json": json_, + } + + +def _compare_callbacks(expected_callbacks, actual_callbacks, name, loose_file_hash): + expected_normalized = [ + _normalize_callback(callback, loose_file_hash) + for callback in expected_callbacks + ] + actual_normalized = [ + _normalize_callback(callback, loose_file_hash) for callback in actual_callbacks + ] + if expected_normalized == actual_normalized: + return [] + problems = [] + if len(expected_normalized) != len(actual_normalized): + problems.append( + f"{name}: callback count differs: expected " + f"{len(expected_normalized)}, got {len(actual_normalized)}" + ) + for index, (expected_callback, actual_callback) in enumerate( + zip(expected_normalized, actual_normalized) + ): + if expected_callback != actual_callback: + problems.append( + f"{name}: callback {index} differs: " + f"expected {expected_callback!r}, got {actual_callback!r}" + ) + return problems or [f"{name}: callback sequences differ"] + + +def _decode_page_spec(page_spec): + """Decode a crunched page_spec string into a page number to dimension + mapping (e.g. "612.00x792.00:0-1" -> {0: "612.00x792.00", 1: ...}).""" + if not isinstance(page_spec, str): + return page_spec + pages = {} + for segment in page_spec.split(";"): + dimension, _, page_ranges = segment.partition(":") + for page_range in page_ranges.split(","): + start, _, end = page_range.partition("-") + for page in range(int(start), int(end or start) + 1): + pages[page] = dimension + return pages + + +def _page_specs_match(expected_spec, actual_spec, tolerance): + expected_pages = _decode_page_spec(expected_spec) + actual_pages = _decode_page_spec(actual_spec) + if not isinstance(expected_pages, dict) or not isinstance(actual_pages, dict): + return expected_pages == actual_pages + if set(expected_pages) != set(actual_pages): + return False + for page, expected_dimension in expected_pages.items(): + actual_dimension = actual_pages[page] + if expected_dimension == actual_dimension: + continue + if tolerance <= 0: + return False + try: + expected_w, expected_h = map(float, expected_dimension.split("x")) + actual_w, actual_h = map(float, actual_dimension.split("x")) + except ValueError: + return False + if ( + abs(expected_w - actual_w) > tolerance + or abs(expected_h - actual_h) > tolerance + ): + return False + return True + + +def _normalize_updated(document): + document = dict(document) + document["updated"] = 0 + document["pages"] = [{**page, "updated": 0} for page in document.get("pages", [])] + return document + + +def _normalize_page_equivalent(page): + """In equivalence mode a page's ocr field is compared as OCR'd-or-not + (a replacement may use a different engine label) and its contents are + compared separately with text rules.""" + page = dict(page) + page["ocr"] = bool(page.get("ocr")) + page.pop("contents", None) + return page + + +def _compare_json_text(expected, actual, name, context): + # pylint: disable=too-many-locals + equivalent = context["strictness"] == "equivalent" + expected_json = _normalize_updated(json.loads(expected.read_text())) + actual_json = _normalize_updated(json.loads(actual.read_text())) + problems = [] + expected_pages = expected_json.get("pages", []) + actual_pages = actual_json.get("pages", []) + if len(expected_pages) != len(actual_pages): + return [ + f"{name}: page count differs: expected {len(expected_pages)}, " + f"got {len(actual_pages)}" + ] + for index, (expected_page, actual_page) in enumerate( + zip(expected_pages, actual_pages) + ): + if equivalent: + expected_contents = expected_page.get("contents", "") + actual_contents = actual_page.get("contents", "") + if expected_page.get("ocr"): + ratio = _similarity(expected_contents, actual_contents) + if ratio < OCR_TEXT_SIMILARITY: + problems.append( + f"{name}: page {index} OCR text similarity " + f"{ratio:.3f} below {OCR_TEXT_SIMILARITY}" + ) + elif _normalize_text(expected_contents) != _normalize_text(actual_contents): + problems.append(f"{name}: page {index} contents differ") + expected_page = _normalize_page_equivalent(expected_page) + actual_page = _normalize_page_equivalent(actual_page) + if expected_page != actual_page: + for key in sorted(set(expected_page) | set(actual_page)): + if expected_page.get(key) != actual_page.get(key): + problems.append( + f"{name}: page {index} field {key!r} differs: " + f"expected {expected_page.get(key)!r}, " + f"got {actual_page.get(key)!r}" + ) + top_expected = {k: v for k, v in expected_json.items() if k != "pages"} + top_actual = {k: v for k, v in actual_json.items() if k != "pages"} + if top_expected != top_actual: + problems.append( + f"{name}: top-level fields differ: expected {top_expected!r}, " + f"got {top_actual!r}" + ) + return problems + + +def _compare_positions(expected, actual, name, context): + expected_words = json.loads(expected.read_text()) + actual_words = json.loads(actual.read_text()) + if context["strictness"] == "equivalent": + return _compare_positions_equivalent(expected_words, actual_words, name) + if len(expected_words) != len(actual_words): + return [ + f"{name}: word count differs: expected {len(expected_words)}, " + f"got {len(actual_words)}" + ] + problems = [] + for index, (expected_word, actual_word) in enumerate( + zip(expected_words, actual_words) + ): + for key in sorted(set(expected_word) | set(actual_word)): + expected_value = expected_word.get(key) + actual_value = actual_word.get(key) + if isinstance(expected_value, float) and isinstance( + actual_value, (int, float) + ): + if abs(expected_value - actual_value) > POSITION_TOLERANCE: + problems.append( + f"{name}: word {index} {key!r} differs beyond " + f"tolerance: expected {expected_value!r}, " + f"got {actual_value!r}" + ) + elif expected_value != actual_value: + problems.append( + f"{name}: word {index} {key!r} differs: " + f"expected {expected_value!r}, got {actual_value!r}" + ) + return problems + + +def _compare_positions_equivalent(expected_words, actual_words, name): + """A replacement extractor may segment words differently; require valid + structure, in-range coordinates, agreement on emptiness, and + approximately the same text.""" + problems = [] + if bool(expected_words) != bool(actual_words): + return [ + f"{name}: emptiness differs: expected {len(expected_words)} " + f"words, got {len(actual_words)}" + ] + for index, word in enumerate(actual_words): + for key in ("x1", "x2", "y1", "y2"): + value = word.get(key) + if not isinstance(value, (int, float)) or not 0 <= value <= 1: + problems.append(f"{name}: word {index} {key!r} out of range: {value!r}") + if not isinstance(word.get("text"), str): + problems.append(f"{name}: word {index} has no text") + expected_text = " ".join(word.get("text", "") for word in expected_words) + actual_text = " ".join(word.get("text", "") for word in actual_words) + ratio = _similarity(expected_text, actual_text) + if expected_words and ratio < POSITION_TEXT_SIMILARITY: + problems.append( + f"{name}: position text similarity {ratio:.3f} below " + f"{POSITION_TEXT_SIMILARITY}" + ) + return problems + + +def _compare_images(expected, actual, name, context): + # pylint: disable=too-many-locals, too-many-return-statements + expected_bytes = expected.read_bytes() + actual_bytes = actual.read_bytes() + if expected_bytes == actual_bytes: + return [] + equivalent = context["strictness"] == "equivalent" + + # Third Party + from PIL import Image, ImageChops, ImageStat + + expected_image = Image.open(io.BytesIO(expected_bytes)).convert("RGB") + actual_image = Image.open(io.BytesIO(actual_bytes)).convert("RGB") + if expected_image.size != actual_image.size: + return [ + f"{name}: image size differs: expected {expected_image.size}, " + f"got {actual_image.size}" + ] + difference = ImageChops.difference(expected_image, actual_image) + bounding_box = difference.getbbox() + if bounding_box is None: + if equivalent: + return [] + return [f"{name}: bytes differ but pixels are identical"] + mean_delta = max(ImageStat.Stat(difference).mean) + if equivalent: + if mean_delta <= IMAGE_MEAN_DELTA: + return [] + return [ + f"{name}: mean pixel delta {mean_delta:.2f} above " f"{IMAGE_MEAN_DELTA}" + ] + max_channel_difference = max(band.getextrema()[1] for band in difference.split()) + return [ + f"{name}: pixels differ in region {bounding_box} " + f"(max channel delta {max_channel_difference}, mean {mean_delta:.2f})" + ] + + +def _pdf_summary(pdf_path): + # Third Party + import pymupdf + + summary = [] + with pymupdf.open(str(pdf_path)) as pdf: + for page in pdf: + summary.append( + { + "width": round(page.rect.width, 2), + "height": round(page.rect.height, 2), + "text": page.get_text(), + } + ) + return summary + + +def _compare_pdfs(expected, actual, name, context): + equivalent = context["strictness"] == "equivalent" + try: + expected_summary = _pdf_summary(expected) + actual_summary = _pdf_summary(actual) + except Exception: # pylint: disable=broad-except + # Unparseable PDF (e.g. the corrupt-input error case): fall back to + # comparing the raw bytes + return _compare_bytes(expected, actual, name) + problems = [] + if len(expected_summary) != len(actual_summary): + return [ + f"{name}: page count differs: expected {len(expected_summary)}, " + f"got {len(actual_summary)}" + ] + for index, (expected_page, actual_page) in enumerate( + zip(expected_summary, actual_summary) + ): + for key in ("width", "height"): + delta = abs(expected_page[key] - actual_page[key]) + if delta > (DIMENSION_TOLERANCE if equivalent else 0): + problems.append( + f"{name}: page {index + 1} {key} differs: " + f"expected {expected_page[key]}, got {actual_page[key]}" + ) + if equivalent: + ratio = _similarity(expected_page["text"], actual_page["text"]) + if ratio < PDF_TEXT_SIMILARITY: + problems.append( + f"{name}: page {index + 1} text layer similarity " + f"{ratio:.3f} below {PDF_TEXT_SIMILARITY}" + ) + elif expected_page["text"] != actual_page["text"]: + problems.append(f"{name}: page {index + 1} text layer differs") + return problems diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/case.json b/documentcloud/documents/processing/tests/spec/documents/blank-1page/case.json new file mode 100644 index 00000000..1e67e00d --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/blank-1page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9107, + "slug": "blank-1page", + "why": "Edge case: a blank page has no text layer so it is OCR'd, producing empty text and an empty positions array.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": true +} diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.index b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.index new file mode 100644 index 00000000..e05f80b6 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.pdf new file mode 100644 index 00000000..632811a7 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.txt b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.txt new file mode 100644 index 00000000..8214d0ee --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.txt @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.txt.json b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.txt.json new file mode 100644 index 00000000..31522840 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/blank-1page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974762854, "pages": [{"page": 0, "contents": "\f", "ocr": "tess4", "lang": "eng", "updated": 1783974762854}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/metadata.json new file mode 100644 index 00000000..732712ab --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 1 + }, + "method": "patch", + "url": "documents/-9107/" + }, + { + "json": { + "file_hash": "f1bdb2dfd659ac357ff4c08cd1b19db0376df2dc", + "page_spec": "612.00x792.00:0" + }, + "method": "patch", + "url": "documents/-9107/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9107/" + } + ], + "database": { + "file_hash": "f1bdb2dfd659ac357ff4c08cd1b19db0376df2dc", + "page_count": 1, + "page_spec": "612.00x792.00:0", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-large.gif new file mode 100644 index 00000000..a0e40383 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-normal.gif new file mode 100644 index 00000000..7b459ac8 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-small.gif new file mode 100644 index 00000000..d223deec Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-thumbnail.gif new file mode 100644 index 00000000..428b62d0 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-xlarge.gif new file mode 100644 index 00000000..ddff8779 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1.position.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1.position.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1.txt new file mode 100644 index 00000000..8214d0ee --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/blank-1page/expected/pages/blank-1page-p1.txt @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/blank-1page/input/blank-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/blank-1page/input/blank-1page.pdf new file mode 100644 index 00000000..4189865f Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/blank-1page/input/blank-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/case.json b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/case.json new file mode 100644 index 00000000..8cabf85d --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/case.json @@ -0,0 +1,8 @@ +{ + "doc_id": -9110, + "slug": "corrupt-1page", + "why": "Error path: an unparseable PDF must produce an error POST to the API and no output files.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": false, + "expects_error": true +} diff --git a/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/expected/corrupt-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/expected/corrupt-1page.pdf new file mode 100644 index 00000000..bb64917b Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/expected/corrupt-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/expected/metadata.json new file mode 100644 index 00000000..4b3bce5e --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/expected/metadata.json @@ -0,0 +1,21 @@ +{ + "callbacks": [ + { + "json": { + "message": "ERROR 3: unable to load 'spec-bucket/documents/-9110/corrupt-1page.pdf'" + }, + "method": "post", + "url": "documents/-9110/errors/" + } + ], + "database": {}, + "errors": [ + { + "json": { + "message": "ERROR 3: unable to load 'spec-bucket/documents/-9110/corrupt-1page.pdf'" + }, + "method": "post", + "url": "documents/-9110/errors/" + } + ] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/input/corrupt-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/input/corrupt-1page.pdf new file mode 100644 index 00000000..bb64917b Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/corrupt-1page/input/corrupt-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/case.json b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/case.json new file mode 100644 index 00000000..03e87bd1 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9105, + "slug": "force-ocr-1page", + "why": "force_ocr overrides embedded text detection: the page has a text layer but must be OCR'd anyway (ocr: tess4_force).", + "settings": {"ocr_engine": "tess4", "force_ocr": true, "ocr_code": "eng"}, + "needs_ocr": true +} diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.index b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.index new file mode 100644 index 00000000..0be3c7c5 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.pdf new file mode 100644 index 00000000..98c98361 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.txt b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.txt new file mode 100644 index 00000000..cda9f220 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.txt @@ -0,0 +1,4 @@ +FORCE OCR TEST DOCUMENT +This page has an embedded text layer, but the case settings +use force_ocr, so the pipeline must OCR it anyway. + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.txt.json b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.txt.json new file mode 100644 index 00000000..39cac7f3 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/force-ocr-1page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974764228, "pages": [{"page": 0, "contents": "FORCE OCR TEST DOCUMENT\nThis page has an embedded text layer, but the case settings\nuse force_ocr, so the pipeline must OCR it anyway.\n\f", "ocr": "tess4_force", "lang": "eng", "updated": 1783974764228}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/metadata.json new file mode 100644 index 00000000..b2036f84 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 1 + }, + "method": "patch", + "url": "documents/-9105/" + }, + { + "json": { + "file_hash": "29342815d1963995d88cdbc5905e62b33fa28c93", + "page_spec": "612.00x792.00:0" + }, + "method": "patch", + "url": "documents/-9105/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9105/" + } + ], + "database": { + "file_hash": "29342815d1963995d88cdbc5905e62b33fa28c93", + "page_count": 1, + "page_spec": "612.00x792.00:0", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-large.gif new file mode 100644 index 00000000..96c26335 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-normal.gif new file mode 100644 index 00000000..55e8d184 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-small.gif new file mode 100644 index 00000000..fc7620c5 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-thumbnail.gif new file mode 100644 index 00000000..da7afe55 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-xlarge.gif new file mode 100644 index 00000000..fe9ce3b9 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1.position.json new file mode 100644 index 00000000..2dc4942d --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1.position.json @@ -0,0 +1 @@ +[{"text": "FORCE", "x1": 0.11857083333333332, "x2": 0.19571319444444443, "y1": 0.08784083958407103, "y2": 0.11466804021933914, "upright": true, "direction": "ltr"}, {"text": "OCR", "x1": 0.2042847222222222, "x2": 0.2542847222222222, "y1": 0.08784083958407103, "y2": 0.11466804021933914, "upright": true, "direction": "ltr"}, {"text": "TEST", "x1": 0.26142777777777776, "x2": 0.3199986111111111, "y1": 0.08784083958407103, "y2": 0.11466804021933914, "upright": true, "direction": "ltr"}, {"text": "DOCUMENT", "x1": 0.3271416666666666, "x2": 0.4585694444444445, "y1": 0.08784083958407103, "y2": 0.11466804021933914, "upright": true, "direction": "ltr"}, {"text": "This", "x1": 0.11714166666666664, "x2": 0.15999861111111108, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.1685708333333333, "x2": 0.21714166666666665, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "has", "x1": 0.22571388888888885, "x2": 0.2599993055555555, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "an", "x1": 0.2685708333333333, "x2": 0.29142777777777773, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "embedded", "x1": 0.29999999999999993, "x2": 0.40714166666666657, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.414286111111111, "x2": 0.4499999999999999, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "layer,", "x1": 0.45857222222222216, "x2": 0.5114284722222222, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "but", "x1": 0.5200013888888888, "x2": 0.5500013888888888, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.5585722222222221, "x2": 0.5885722222222222, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "case", "x1": 0.5957152777777778, "x2": 0.6428583333333333, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "settings", "x1": 0.6514291666666666, "x2": 0.7285708333333334, "y1": 0.11984998229404747, "y2": 0.14667718292931559, "upright": true, "direction": "ltr"}, {"text": "use", "x1": 0.11857222222222216, "x2": 0.1542868055555555, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "force_ocr,", "x1": 0.1614291666666666, "x2": 0.2614395833333333, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "so", "x1": 0.26999999999999996, "x2": 0.2928569444444444, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.29999999999999993, "x2": 0.32999999999999996, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "pipeline", "x1": 0.33857083333333327, "x2": 0.41571249999999993, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "must", "x1": 0.4242847222222222, "x2": 0.47142777777777767, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "OCR", "x1": 0.47999861111111103, "x2": 0.528570486111111, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "it", "x1": 0.5371416666666666, "x2": 0.5471416666666666, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}, {"text": "anyway.", "x1": 0.5542847222222222, "x2": 0.6357131944444444, "y1": 0.15185912500402393, "y2": 0.17868632563929204, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1.txt new file mode 100644 index 00000000..cda9f220 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/expected/pages/force-ocr-1page-p1.txt @@ -0,0 +1,4 @@ +FORCE OCR TEST DOCUMENT +This page has an embedded text layer, but the case settings +use force_ocr, so the pipeline must OCR it anyway. + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/input/force-ocr-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/input/force-ocr-1page.pdf new file mode 100644 index 00000000..f3ccb002 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/force-ocr-1page/input/force-ocr-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/case.json b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/case.json new file mode 100644 index 00000000..44db47b0 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9104, + "slug": "mixed-2page", + "why": "Per-page routing: page 1 has embedded text (ocr: null), page 2 is scanned (ocr: tess4).", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": true +} diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/metadata.json new file mode 100644 index 00000000..b72d94b1 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 2 + }, + "method": "patch", + "url": "documents/-9104/" + }, + { + "json": { + "file_hash": "bbcbdc9c25a909aee87716cdb16fc13b5a47789e", + "page_spec": "612.00x792.00:0-1" + }, + "method": "patch", + "url": "documents/-9104/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9104/" + } + ], + "database": { + "file_hash": "bbcbdc9c25a909aee87716cdb16fc13b5a47789e", + "page_count": 2, + "page_spec": "612.00x792.00:0-1", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.index b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.index new file mode 100644 index 00000000..46942fc9 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.pdf new file mode 100644 index 00000000..0906e257 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.txt b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.txt new file mode 100644 index 00000000..47baeb6c Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.txt.json b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.txt.json new file mode 100644 index 00000000..30ccd992 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/mixed-2page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974766168, "pages": [{"page": 0, "contents": "Mixed document, page one has an embedded text layer.\nOnly the second page goes through OCR.\u0000", "ocr": null, "lang": "eng", "updated": 1783974766168}, {"page": 1, "contents": "MIXED PAGE TWO\nTHIS PAGE IS SCANNED\n\f", "ocr": "tess4", "lang": "eng", "updated": 1783974766168}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-large.gif new file mode 100644 index 00000000..6b112a40 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-normal.gif new file mode 100644 index 00000000..c75c3fcc Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-small.gif new file mode 100644 index 00000000..d5c38f24 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-thumbnail.gif new file mode 100644 index 00000000..d884af1b Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-xlarge.gif new file mode 100644 index 00000000..02bd6e1a Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1.position.json new file mode 100644 index 00000000..312cb5d8 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1.position.json @@ -0,0 +1 @@ +[{"text": "Mixed", "x1": 0.11764705882352941, "x2": 0.17429901960784314, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document,", "x1": 0.18020424836601306, "x2": 0.27938235294117647, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.2852875816993464, "x2": 0.33252941176470585, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "one", "x1": 0.3384346405228758, "x2": 0.37386601307189543, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "has", "x1": 0.3797712418300654, "x2": 0.4140130718954249, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "an", "x1": 0.4199183006535948, "x2": 0.44353921568627463, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "embedded", "x1": 0.44944444444444454, "x2": 0.5498120915032682, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.555717320261438, "x2": 0.5899591503267975, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "layer.", "x1": 0.5958643790849674, "x2": 0.6478006535947713, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "Only", "x1": 0.11764705882352941, "x2": 0.1613202614379085, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.16722549019607844, "x2": 0.1967516339869281, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "second", "x1": 0.20265686274509806, "x2": 0.27114052287581697, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.2770457516339869, "x2": 0.3242875816993464, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "goes", "x1": 0.3301928104575163, "x2": 0.3762450980392157, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "through", "x1": 0.38215032679738564, "x2": 0.4541813725490197, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "OCR.", "x1": 0.4600866013071897, "x2": 0.5131911764705883, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1.txt new file mode 100644 index 00000000..55b702d3 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p1.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-large.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-large.gif new file mode 100644 index 00000000..59e852d4 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-normal.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-normal.gif new file mode 100644 index 00000000..a97e730d Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-small.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-small.gif new file mode 100644 index 00000000..740b2205 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-thumbnail.gif new file mode 100644 index 00000000..0b45c997 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-xlarge.gif new file mode 100644 index 00000000..cec9aa23 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2.position.json b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2.position.json new file mode 100644 index 00000000..05025e0a --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2.position.json @@ -0,0 +1 @@ +[{"text": "MIXED", "x1": 0.12142916666666667, "x2": 0.2600016666666667, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "PAGE", "x1": 0.28, "x2": 0.40000083333333336, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "TWO", "x1": 0.41571388888888894, "x2": 0.5199993888888889, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "THIS", "x1": 0.11857083333333335, "x2": 0.22, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "PAGE", "x1": 0.23857083333333334, "x2": 0.3585716666666667, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "IS", "x1": 0.37857083333333336, "x2": 0.41571325, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "SCANNED", "x1": 0.4328569444444445, "x2": 0.6514278611111113, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2.txt b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2.txt new file mode 100644 index 00000000..2745dade --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/expected/pages/mixed-2page-p2.txt @@ -0,0 +1,3 @@ +MIXED PAGE TWO +THIS PAGE IS SCANNED + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-2page/input/mixed-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/input/mixed-2page.pdf new file mode 100644 index 00000000..7a715d2d Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-2page/input/mixed-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/case.json b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/case.json new file mode 100644 index 00000000..7c176795 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9106, + "slug": "mixed-sizes-2page", + "why": "Pages with different dimensions: page_spec encoding and per-size aspect ratios.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": false +} diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/metadata.json new file mode 100644 index 00000000..efcf5d33 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 2 + }, + "method": "patch", + "url": "documents/-9106/" + }, + { + "json": { + "file_hash": "059caa46f2593b7861c190a800196f0ef8885419", + "page_spec": "612.00x792.00:0;595.00x420.00:1" + }, + "method": "patch", + "url": "documents/-9106/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9106/" + } + ], + "database": { + "file_hash": "059caa46f2593b7861c190a800196f0ef8885419", + "page_count": 2, + "page_spec": "612.00x792.00:0;595.00x420.00:1", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.index b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.index new file mode 100644 index 00000000..cd1a6699 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.pdf new file mode 100644 index 00000000..929b8071 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.txt b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.txt new file mode 100644 index 00000000..16a086b0 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.txt.json b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.txt.json new file mode 100644 index 00000000..8f6ef5fe --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/mixed-sizes-2page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974767263, "pages": [{"page": 0, "contents": "Letter sized page.\n612 by 792 points.\u0000", "ocr": null, "lang": "eng", "updated": 1783974767263}, {"page": 1, "contents": "A5 landscape page.\n595 by 420 points.\u0000", "ocr": null, "lang": "eng", "updated": 1783974767263}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-large.gif new file mode 100644 index 00000000..64c1a653 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-normal.gif new file mode 100644 index 00000000..e9f76ad5 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-small.gif new file mode 100644 index 00000000..115b158e Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-thumbnail.gif new file mode 100644 index 00000000..54ceb59a Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-xlarge.gif new file mode 100644 index 00000000..2689bd31 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1.position.json new file mode 100644 index 00000000..a2b5a991 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1.position.json @@ -0,0 +1 @@ +[{"text": "Letter", "x1": 0.11764705882352941, "x2": 0.17196241830065362, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "sized", "x1": 0.17786764705882352, "x2": 0.22744607843137254, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "page.", "x1": 0.2333513071895425, "x2": 0.2864983660130719, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "612", "x1": 0.11764705882352941, "x2": 0.153078431372549, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "by", "x1": 0.15898366013071896, "x2": 0.18141503267973857, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "792", "x1": 0.1873202614379085, "x2": 0.22275163398692813, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "points.", "x1": 0.22865686274509803, "x2": 0.29123529411764704, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1.txt new file mode 100644 index 00000000..4bd48ad6 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p1.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-large.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-large.gif new file mode 100644 index 00000000..9278274c Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-normal.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-normal.gif new file mode 100644 index 00000000..09c76ed9 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-small.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-small.gif new file mode 100644 index 00000000..a0bb9e18 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-thumbnail.gif new file mode 100644 index 00000000..75950ae1 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-xlarge.gif new file mode 100644 index 00000000..0b158589 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2.position.json b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2.position.json new file mode 100644 index 00000000..d3345564 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2.position.json @@ -0,0 +1 @@ +[{"text": "A5", "x1": 0.12100840336134454, "x2": 0.1477294117647059, "y1": 0.18974047619047613, "y2": 0.22069285714285708, "upright": true, "direction": "ltr"}, {"text": "landscape", "x1": 0.15380336134453781, "x2": 0.25338991596638655, "y1": 0.18974047619047613, "y2": 0.22069285714285708, "upright": true, "direction": "ltr"}, {"text": "page.", "x1": 0.2594638655462185, "x2": 0.3141294117647059, "y1": 0.18974047619047613, "y2": 0.22069285714285708, "upright": true, "direction": "ltr"}, {"text": "595", "x1": 0.12100840336134454, "x2": 0.15745210084033612, "y1": 0.2454547619047618, "y2": 0.27640714285714274, "upright": true, "direction": "ltr"}, {"text": "by", "x1": 0.16352605042016807, "x2": 0.1865983193277311, "y1": 0.2454547619047618, "y2": 0.27640714285714274, "upright": true, "direction": "ltr"}, {"text": "420", "x1": 0.19267226890756303, "x2": 0.22911596638655465, "y1": 0.2454547619047618, "y2": 0.27640714285714274, "upright": true, "direction": "ltr"}, {"text": "points.", "x1": 0.23518991596638653, "x2": 0.2995563025210084, "y1": 0.2454547619047618, "y2": 0.27640714285714274, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2.txt b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2.txt new file mode 100644 index 00000000..1d4651b0 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/expected/pages/mixed-sizes-2page-p2.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/input/mixed-sizes-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/input/mixed-sizes-2page.pdf new file mode 100644 index 00000000..49bec0ef Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/mixed-sizes-2page/input/mixed-sizes-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/case.json b/documentcloud/documents/processing/tests/spec/documents/redact-2page/case.json new file mode 100644 index 00000000..808e533c --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/redact-2page/case.json @@ -0,0 +1,10 @@ +{ + "doc_id": -9108, + "slug": "redact-2page", + "why": "Redaction path: after standard processing, page 2 is redacted and reprocessed. The redacted page loses its text layer and is OCR'd; page_spec is not resent.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": true, + "redactions": [ + {"page_number": 1, "x1": 0.1, "x2": 0.9, "y1": 0.12, "y2": 0.18} + ] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/metadata.json new file mode 100644 index 00000000..ed093692 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/metadata.json @@ -0,0 +1,41 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 2 + }, + "method": "patch", + "url": "documents/-9108/" + }, + { + "json": { + "file_hash": "f8b1b87e9eb3ff4401a1990c0591d1f1213c9c62", + "page_spec": "612.00x792.00:0-1" + }, + "method": "patch", + "url": "documents/-9108/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9108/" + }, + { + "json": { + "file_hash": "b5592df535f96e2b75a4840d9a26afecc9c3f7b1", + "status": "success" + }, + "method": "patch", + "url": "documents/-9108/" + } + ], + "database": { + "file_hash": "b5592df535f96e2b75a4840d9a26afecc9c3f7b1", + "page_count": 2, + "page_spec": "612.00x792.00:0-1", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-large.gif new file mode 100644 index 00000000..990b43b1 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-normal.gif new file mode 100644 index 00000000..86619e40 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-small.gif new file mode 100644 index 00000000..dfd0c0af Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-thumbnail.gif new file mode 100644 index 00000000..8ae4778a Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-xlarge.gif new file mode 100644 index 00000000..8079a1fe Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1.position.json new file mode 100644 index 00000000..ea9a83c2 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1.position.json @@ -0,0 +1 @@ +[{"text": "Redaction", "x1": 0.11764705882352941, "x2": 0.2132777777777778, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "test", "x1": 0.21918300653594774, "x2": 0.2534248366013072, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document,", "x1": 0.2593300653594771, "x2": 0.3585081699346405, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.3644133986928105, "x2": 0.41165522875817, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "one", "x1": 0.41756045751633997, "x2": 0.45299183006535954, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "is", "x1": 0.45889705882352955, "x2": 0.4742336601307191, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "untouched.", "x1": 0.480138888888889, "x2": 0.5852434640522878, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "Only", "x1": 0.11764705882352941, "x2": 0.1613202614379085, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.16722549019607844, "x2": 0.21446732026143794, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "two", "x1": 0.22037254901960784, "x2": 0.2534248366013072, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "receives", "x1": 0.2593300653594771, "x2": 0.33841339869281045, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "a", "x1": 0.34431862745098035, "x2": 0.35612908496732026, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "redaction.", "x1": 0.3620343137254902, "x2": 0.4553071895424837, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1.txt new file mode 100644 index 00000000..929df8ea Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p1.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-large.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-large.gif new file mode 100644 index 00000000..4eb71911 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-normal.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-normal.gif new file mode 100644 index 00000000..fcf1fdb7 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-small.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-small.gif new file mode 100644 index 00000000..86e62a45 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-thumbnail.gif new file mode 100644 index 00000000..6e804ec0 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-xlarge.gif new file mode 100644 index 00000000..9db7e5ab Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2.position.json b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2.position.json new file mode 100644 index 00000000..a812d0a5 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.12000000000000001, "x2": 0.1657137777777778, "y1": 0.08998486945884171, "y2": 0.11466589404328836, "upright": true, "direction": "ltr"}, {"text": "two", "x1": 0.1742861111111111, "x2": 0.20571465277777776, "y1": 0.08998486945884171, "y2": 0.11466589404328836, "upright": true, "direction": "ltr"}, {"text": "before", "x1": 0.21285694444444445, "x2": 0.2714283611111111, "y1": 0.08998486945884171, "y2": 0.11466589404328836, "upright": true, "direction": "ltr"}, {"text": "redaction.", "x1": 0.28, "x2": 0.37000027777777783, "y1": 0.08998486945884171, "y2": 0.11466589404328836, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2.txt b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2.txt new file mode 100644 index 00000000..2accd433 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/pages/redact-2page-p2.txt @@ -0,0 +1,2 @@ +Page two before redaction. + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.index b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.index new file mode 100644 index 00000000..ac3f89de Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.pdf new file mode 100644 index 00000000..cdb33994 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.txt b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.txt new file mode 100644 index 00000000..95be8ac6 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.txt.json b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.txt.json new file mode 100644 index 00000000..3f05f289 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/redact-2page/expected/redact-2page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974770034, "pages": [{"page": 0, "contents": "Redaction test document, page one is untouched.\nOnly page two receives a redaction.\u0000", "ocr": null, "lang": "eng", "updated": 1783974768941}, {"page": 1, "contents": "Page two before redaction.\n\f", "ocr": "tess4", "lang": "eng", "updated": 1783974770034}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/redact-2page/input/redact-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/redact-2page/input/redact-2page.pdf new file mode 100644 index 00000000..220e0a5c Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/redact-2page/input/redact-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/case.json b/documentcloud/documents/processing/tests/spec/documents/scan-2page/case.json new file mode 100644 index 00000000..face967e --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9103, + "slug": "scan-2page", + "why": "Image-only (scanned) document. Every page goes through Tesseract OCR: text, grafted PDF layer, and positions come from OCR.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": true +} diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/metadata.json new file mode 100644 index 00000000..29a1775d --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 2 + }, + "method": "patch", + "url": "documents/-9103/" + }, + { + "json": { + "file_hash": "e2f2abf28789abe84804fe3e1acb9d7ffe36d727", + "page_spec": "612.00x792.00:0-1" + }, + "method": "patch", + "url": "documents/-9103/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9103/" + } + ], + "database": { + "file_hash": "e2f2abf28789abe84804fe3e1acb9d7ffe36d727", + "page_count": 2, + "page_spec": "612.00x792.00:0-1", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-large.gif new file mode 100644 index 00000000..0589e943 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-normal.gif new file mode 100644 index 00000000..3d7a4611 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-small.gif new file mode 100644 index 00000000..e1187b52 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-thumbnail.gif new file mode 100644 index 00000000..ee55a28a Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-xlarge.gif new file mode 100644 index 00000000..2f92b4d0 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1.position.json new file mode 100644 index 00000000..fb81a21e --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1.position.json @@ -0,0 +1 @@ +[{"text": "SCANNED", "x1": 0.12000000000000001, "x2": 0.3385709166666667, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "PAGE", "x1": 0.3571430555555556, "x2": 0.4785720555555556, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "ONE", "x1": 0.49571388888888895, "x2": 0.589999013888889, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "NO", "x1": 0.12142777777777786, "x2": 0.18428452777777787, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "TEXT", "x1": 0.1999986111111112, "x2": 0.31571311111111117, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "LAYER", "x1": 0.3314277777777778, "x2": 0.47571340277777785, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "HERE", "x1": 0.49571388888888895, "x2": 0.6171428888888889, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1.txt new file mode 100644 index 00000000..b4bf200f --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p1.txt @@ -0,0 +1,3 @@ +SCANNED PAGE ONE +NO TEXT LAYER HERE + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-large.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-large.gif new file mode 100644 index 00000000..135f720b Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-normal.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-normal.gif new file mode 100644 index 00000000..0eefd4ce Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-small.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-small.gif new file mode 100644 index 00000000..a6108e5d Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-thumbnail.gif new file mode 100644 index 00000000..445c8845 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-xlarge.gif new file mode 100644 index 00000000..0863280e Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2.position.json b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2.position.json new file mode 100644 index 00000000..81362727 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2.position.json @@ -0,0 +1 @@ +[{"text": "SCANNED", "x1": 0.12000000000000001, "x2": 0.3385709166666667, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "PAGE", "x1": 0.3571430555555556, "x2": 0.4785720555555556, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "TWO", "x1": 0.49428611111111115, "x2": 0.5985716111111111, "y1": 0.116791681421627, "y2": 0.1522035862601809, "upright": true, "direction": "ltr"}, {"text": "EVERY", "x1": 0.12142916666666671, "x2": 0.27143020833333337, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "PAGE", "x1": 0.29000000000000004, "x2": 0.41000083333333337, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "IS", "x1": 0.43000000000000005, "x2": 0.46714241666666667, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}, {"text": "OCRD", "x1": 0.4828569444444445, "x2": 0.6128567777777779, "y1": 0.18853512753651178, "y2": 0.2239470323750657, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2.txt b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2.txt new file mode 100644 index 00000000..be10f756 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/pages/scan-2page-p2.txt @@ -0,0 +1,3 @@ +SCANNED PAGE TWO +EVERY PAGE IS OCRD + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.index b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.index new file mode 100644 index 00000000..84611b23 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.pdf new file mode 100644 index 00000000..4ba3f273 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.txt b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.txt new file mode 100644 index 00000000..4a741b73 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.txt @@ -0,0 +1,7 @@ +SCANNED PAGE ONE +NO TEXT LAYER HERE + + +SCANNED PAGE TWO +EVERY PAGE IS OCRD + \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.txt.json b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.txt.json new file mode 100644 index 00000000..ee659d2a --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/scan-2page/expected/scan-2page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974772384, "pages": [{"page": 0, "contents": "SCANNED PAGE ONE\nNO TEXT LAYER HERE\n\f", "ocr": "tess4", "lang": "eng", "updated": 1783974772384}, {"page": 1, "contents": "SCANNED PAGE TWO\nEVERY PAGE IS OCRD\n\f", "ocr": "tess4", "lang": "eng", "updated": 1783974772384}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/scan-2page/input/scan-2page.pdf b/documentcloud/documents/processing/tests/spec/documents/scan-2page/input/scan-2page.pdf new file mode 100644 index 00000000..7fac71e4 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/scan-2page/input/scan-2page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/case.json b/documentcloud/documents/processing/tests/spec/documents/text-1page/case.json new file mode 100644 index 00000000..58c9b0d5 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-1page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9101, + "slug": "text-1page", + "why": "Simplest case: one page with an embedded text layer. No OCR. Validates the baseline output file set.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": false +} diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/metadata.json new file mode 100644 index 00000000..7cdef1cb --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 1 + }, + "method": "patch", + "url": "documents/-9101/" + }, + { + "json": { + "file_hash": "f2486b2d0778989fa9ee60500310ca90703e080a", + "page_spec": "612.00x792.00:0" + }, + "method": "patch", + "url": "documents/-9101/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9101/" + } + ], + "database": { + "file_hash": "f2486b2d0778989fa9ee60500310ca90703e080a", + "page_count": 1, + "page_spec": "612.00x792.00:0", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-large.gif new file mode 100644 index 00000000..6bad130c Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-normal.gif new file mode 100644 index 00000000..145215a5 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-small.gif new file mode 100644 index 00000000..2494fc11 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-thumbnail.gif new file mode 100644 index 00000000..ec50610d Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-xlarge.gif new file mode 100644 index 00000000..55d77f4e Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1.position.json new file mode 100644 index 00000000..27cc8963 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1.position.json @@ -0,0 +1 @@ +[{"text": "PROCESSING", "x1": 0.11764705882352941, "x2": 0.2592875816993464, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "PIPELINE", "x1": 0.26519281045751636, "x2": 0.3608235294117647, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "TEST", "x1": 0.36672875816993467, "x2": 0.42102287581699344, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "SPEC", "x1": 0.42692810457516345, "x2": 0.48476960784313733, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "Case:", "x1": 0.11764705882352941, "x2": 0.17313071895424836, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "single", "x1": 0.1790359477124183, "x2": 0.23451960784313727, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.24042483660130717, "x2": 0.2876666666666667, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "with", "x1": 0.2935718954248366, "x2": 0.3313398692810457, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "an", "x1": 0.33724509803921565, "x2": 0.3608660130718954, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "embedded", "x1": 0.3667712418300654, "x2": 0.467138888888889, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.4730441176470589, "x2": 0.5072859477124184, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "layer.", "x1": 0.5131911764705884, "x2": 0.5651274509803923, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "This", "x1": 0.11764705882352941, "x2": 0.15777287581699348, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.16367810457516338, "x2": 0.21091993464052286, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "must", "x1": 0.21682516339869282, "x2": 0.2628562091503268, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "not", "x1": 0.2687614379084967, "x2": 0.2982875816993464, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "be", "x1": 0.30419281045751634, "x2": 0.3278137254901961, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "OCR'd;", "x1": 0.333718954248366, "x2": 0.4026911764705882, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "its", "x1": 0.4085964052287582, "x2": 0.4298382352941177, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.4357434640522876, "x2": 0.4699852941176471, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "is", "x1": 0.475890522875817, "x2": 0.49122712418300657, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "extracted", "x1": 0.49713235294117647, "x2": 0.5845000000000001, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "directly", "x1": 0.59040522875817, "x2": 0.6576781045751635, "y1": 0.15971085858585857, "y2": 0.17612499999999998, "upright": true, "direction": "ltr"}, {"text": "from", "x1": 0.11764705882352941, "x2": 0.16013071895424835, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.1660359477124183, "x2": 0.19556209150326798, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "PDF", "x1": 0.2014673202614379, "x2": 0.2439509803921569, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "and", "x1": 0.2498562091503268, "x2": 0.2852875816993464, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.2911928104575163, "x2": 0.320718954248366, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "output", "x1": 0.32662418300653595, "x2": 0.38567647058823534, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "PDF", "x1": 0.39158169934640524, "x2": 0.4340653594771242, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "keeps", "x1": 0.4399705882352942, "x2": 0.49664379084967325, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.5025490196078433, "x2": 0.5320751633986929, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "original", "x1": 0.5379803921568629, "x2": 0.6064428104575167, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.6123480392156866, "x2": 0.646589869281046, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}, {"text": "layer.", "x1": 0.6524950980392159, "x2": 0.7044313725490199, "y1": 0.18925631313131322, "y2": 0.20567045454545463, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1.txt new file mode 100644 index 00000000..1c1e92fc Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/pages/text-1page-p1.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.index b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.index new file mode 100644 index 00000000..d04ffaaf Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.pdf new file mode 100644 index 00000000..e8454813 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.txt b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.txt new file mode 100644 index 00000000..1c1e92fc Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.txt.json b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.txt.json new file mode 100644 index 00000000..83667daf --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-1page/expected/text-1page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974773155, "pages": [{"page": 0, "contents": "PROCESSING PIPELINE TEST SPEC\nCase: single page with an embedded text layer.\nThis page must not be OCR'd; its text is extracted directly\nfrom the PDF and the output PDF keeps the original text layer.\u0000", "ocr": null, "lang": "eng", "updated": 1783974773155}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-1page/input/text-1page.pdf b/documentcloud/documents/processing/tests/spec/documents/text-1page/input/text-1page.pdf new file mode 100644 index 00000000..6ffeeece Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-1page/input/text-1page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/case.json b/documentcloud/documents/processing/tests/spec/documents/text-3page/case.json new file mode 100644 index 00000000..88512b7b --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-3page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9102, + "slug": "text-3page", + "why": "Multi-page embedded text. Every page gets all artifacts; text concatenation order is correct.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": false +} diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/metadata.json new file mode 100644 index 00000000..eef87750 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 3 + }, + "method": "patch", + "url": "documents/-9102/" + }, + { + "json": { + "file_hash": "ae8d812a33b3910d9c8b1242039cf1ddf36c3dca", + "page_spec": "612.00x792.00:0-2" + }, + "method": "patch", + "url": "documents/-9102/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9102/" + } + ], + "database": { + "file_hash": "ae8d812a33b3910d9c8b1242039cf1ddf36c3dca", + "page_count": 3, + "page_spec": "612.00x792.00:0-2", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-large.gif new file mode 100644 index 00000000..ab0bd50b Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-normal.gif new file mode 100644 index 00000000..fefd3ff0 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-small.gif new file mode 100644 index 00000000..b6879b7b Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-thumbnail.gif new file mode 100644 index 00000000..a2882181 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-xlarge.gif new file mode 100644 index 00000000..1604a811 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1.position.json new file mode 100644 index 00000000..120272c1 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.11764705882352941, "x2": 0.1672467320261438, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "one", "x1": 0.17315196078431372, "x2": 0.20858333333333334, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "of", "x1": 0.21448856209150327, "x2": 0.23220424836601308, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.23810947712418304, "x2": 0.2676356209150327, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "multi-page", "x1": 0.27354084967320264, "x2": 0.3726977124183007, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.37860294117647064, "x2": 0.41284477124183017, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document.", "x1": 0.4187500000000001, "x2": 0.5179281045751636, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "The", "x1": 0.11764705882352941, "x2": 0.15424673202614378, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.16015196078431373, "x2": 0.2073937908496732, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.21329901960784312, "x2": 0.2475408496732026, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "files", "x1": 0.25344607843137257, "x2": 0.29121405228758174, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "must", "x1": 0.29711928104575164, "x2": 0.34315032679738566, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "contain", "x1": 0.34905555555555556, "x2": 0.4175392156862746, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "this", "x1": 0.4234444444444445, "x2": 0.45649673202614394, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.46240196078431384, "x2": 0.5096437908496734, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "1", "x1": 0.5155490196078433, "x2": 0.5273594771241832, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "marker.", "x1": 0.5332647058823531, "x2": 0.6052532679738565, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1.txt new file mode 100644 index 00000000..ef267a90 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p1.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-large.gif new file mode 100644 index 00000000..c6e400e2 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-normal.gif new file mode 100644 index 00000000..d9fd0f60 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-small.gif new file mode 100644 index 00000000..0c5edcbf Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-thumbnail.gif new file mode 100644 index 00000000..fc6ca149 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-xlarge.gif new file mode 100644 index 00000000..fefa2e25 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2.position.json b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2.position.json new file mode 100644 index 00000000..4ebb1b32 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.11764705882352941, "x2": 0.1672467320261438, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "two", "x1": 0.17315196078431372, "x2": 0.20620424836601306, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "of", "x1": 0.21210947712418296, "x2": 0.2298251633986928, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.23573039215686273, "x2": 0.2652565359477124, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "multi-page", "x1": 0.2711617647058823, "x2": 0.37031862745098043, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.3762238562091504, "x2": 0.41046568627450986, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document.", "x1": 0.4163709150326798, "x2": 0.5155490196078433, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "The", "x1": 0.11764705882352941, "x2": 0.15424673202614378, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.16015196078431373, "x2": 0.2073937908496732, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.21329901960784312, "x2": 0.2475408496732026, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "files", "x1": 0.25344607843137257, "x2": 0.29121405228758174, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "must", "x1": 0.29711928104575164, "x2": 0.34315032679738566, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "contain", "x1": 0.34905555555555556, "x2": 0.4175392156862746, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "this", "x1": 0.4234444444444445, "x2": 0.45649673202614394, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.46240196078431384, "x2": 0.5096437908496734, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "2", "x1": 0.5155490196078433, "x2": 0.5273594771241832, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "marker.", "x1": 0.5332647058823531, "x2": 0.6052532679738565, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2.txt b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2.txt new file mode 100644 index 00000000..2240b2e2 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p2.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-large.gif new file mode 100644 index 00000000..e65002bc Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-normal.gif new file mode 100644 index 00000000..6ff46bcc Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-small.gif new file mode 100644 index 00000000..a02a035f Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-thumbnail.gif new file mode 100644 index 00000000..b00fb246 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-xlarge.gif new file mode 100644 index 00000000..cf053cbe Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3.position.json b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3.position.json new file mode 100644 index 00000000..32a75f4b --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.11764705882352941, "x2": 0.1672467320261438, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "three", "x1": 0.17315196078431372, "x2": 0.22156209150326797, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "of", "x1": 0.22746732026143793, "x2": 0.2451830065359477, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.25108823529411767, "x2": 0.28061437908496734, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "multi-page", "x1": 0.2865196078431373, "x2": 0.3856764705882354, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.3915816993464053, "x2": 0.42582352941176477, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document.", "x1": 0.4317287581699348, "x2": 0.5309068627450982, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "The", "x1": 0.11764705882352941, "x2": 0.15424673202614378, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.16015196078431373, "x2": 0.2073937908496732, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.21329901960784312, "x2": 0.2475408496732026, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "files", "x1": 0.25344607843137257, "x2": 0.29121405228758174, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "must", "x1": 0.29711928104575164, "x2": 0.34315032679738566, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "contain", "x1": 0.34905555555555556, "x2": 0.4175392156862746, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "this", "x1": 0.4234444444444445, "x2": 0.45649673202614394, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.46240196078431384, "x2": 0.5096437908496734, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "3", "x1": 0.5155490196078433, "x2": 0.5273594771241832, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "marker.", "x1": 0.5332647058823531, "x2": 0.6052532679738565, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3.txt b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3.txt new file mode 100644 index 00000000..683bef95 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/pages/text-3page-p3.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.index b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.index new file mode 100644 index 00000000..73a8adbe Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.pdf b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.pdf new file mode 100644 index 00000000..5c28eb6a Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.txt b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.txt new file mode 100644 index 00000000..78c20bca Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.txt.json b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.txt.json new file mode 100644 index 00000000..78ccb1bd --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-3page/expected/text-3page.txt.json @@ -0,0 +1 @@ +{"updated": 1783974775645, "pages": [{"page": 0, "contents": "Page one of the multi-page text document.\nThe page text files must contain this page 1 marker.\u0000", "ocr": null, "lang": "eng", "updated": 1783974775645}, {"page": 1, "contents": "Page two of the multi-page text document.\nThe page text files must contain this page 2 marker.\u0000", "ocr": null, "lang": "eng", "updated": 1783974775645}, {"page": 2, "contents": "Page three of the multi-page text document.\nThe page text files must contain this page 3 marker.\u0000", "ocr": null, "lang": "eng", "updated": 1783974775645}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-3page/input/text-3page.pdf b/documentcloud/documents/processing/tests/spec/documents/text-3page/input/text-3page.pdf new file mode 100644 index 00000000..60069809 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-3page/input/text-3page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/case.json b/documentcloud/documents/processing/tests/spec/documents/text-4page/case.json new file mode 100644 index 00000000..f083b5b9 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-4page/case.json @@ -0,0 +1,7 @@ +{ + "doc_id": -9109, + "slug": "text-4page", + "why": "Four pages exceed TEXT_POSITION_BATCH (3), exercising the multi-batch flush path for text position extraction.", + "settings": {"ocr_engine": "tess4", "force_ocr": false, "ocr_code": "eng"}, + "needs_ocr": false +} diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/metadata.json b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/metadata.json new file mode 100644 index 00000000..1e4aad27 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/metadata.json @@ -0,0 +1,33 @@ +{ + "callbacks": [ + { + "json": { + "page_count": 4 + }, + "method": "patch", + "url": "documents/-9109/" + }, + { + "json": { + "file_hash": "9e014cf2f85174c664ccf169a1a673905b7bcbdf", + "page_spec": "612.00x792.00:0-3" + }, + "method": "patch", + "url": "documents/-9109/" + }, + { + "json": { + "status": "success" + }, + "method": "patch", + "url": "documents/-9109/" + } + ], + "database": { + "file_hash": "9e014cf2f85174c664ccf169a1a673905b7bcbdf", + "page_count": 4, + "page_spec": "612.00x792.00:0-3", + "status": "success" + }, + "errors": [] +} diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-large.gif new file mode 100644 index 00000000..02024057 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-normal.gif new file mode 100644 index 00000000..1e0a6056 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-small.gif new file mode 100644 index 00000000..340cea9c Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-thumbnail.gif new file mode 100644 index 00000000..0aa32b0f Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-xlarge.gif new file mode 100644 index 00000000..fb519608 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1.position.json b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1.position.json new file mode 100644 index 00000000..fd37cbaa --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.11764705882352941, "x2": 0.1672467320261438, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "one", "x1": 0.17315196078431372, "x2": 0.20858333333333334, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "of", "x1": 0.21448856209150327, "x2": 0.23220424836601308, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.23810947712418304, "x2": 0.2676356209150327, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "four", "x1": 0.27354084967320264, "x2": 0.310140522875817, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.31604575163398696, "x2": 0.3632875816993465, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.36919281045751645, "x2": 0.4034346405228759, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document.", "x1": 0.4093398692810459, "x2": 0.5085179738562093, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "Text", "x1": 0.11764705882352941, "x2": 0.15896241830065358, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "position", "x1": 0.16486764705882354, "x2": 0.2380669934640523, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "extraction", "x1": 0.24397222222222223, "x2": 0.33605555555555555, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "happens", "x1": 0.3419607843137255, "x2": 0.4234444444444445, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "in", "x1": 0.42934967320261447, "x2": 0.4458758169934641, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "batches;", "x1": 0.4517810457516341, "x2": 0.5320751633986929, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "marker", "x1": 0.5379803921568629, "x2": 0.6040637254901963, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "1.", "x1": 0.6099689542483662, "x2": 0.627684640522876, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1.txt b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1.txt new file mode 100644 index 00000000..2c832ea0 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p1.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-large.gif new file mode 100644 index 00000000..8fbd38ac Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-normal.gif new file mode 100644 index 00000000..c3d93262 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-small.gif new file mode 100644 index 00000000..734e7fcd Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-thumbnail.gif new file mode 100644 index 00000000..18c5964e Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-xlarge.gif new file mode 100644 index 00000000..5d1d9df7 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2.position.json b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2.position.json new file mode 100644 index 00000000..0b2ff2c0 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.11764705882352941, "x2": 0.1672467320261438, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "two", "x1": 0.17315196078431372, "x2": 0.20620424836601306, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "of", "x1": 0.21210947712418296, "x2": 0.2298251633986928, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.23573039215686273, "x2": 0.2652565359477124, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "four", "x1": 0.2711617647058823, "x2": 0.30776143790849675, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.31366666666666665, "x2": 0.36090849673202613, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.3668137254901961, "x2": 0.4010555555555556, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document.", "x1": 0.40696078431372557, "x2": 0.506138888888889, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "Text", "x1": 0.11764705882352941, "x2": 0.15896241830065358, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "position", "x1": 0.16486764705882354, "x2": 0.2380669934640523, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "extraction", "x1": 0.24397222222222223, "x2": 0.33605555555555555, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "happens", "x1": 0.3419607843137255, "x2": 0.4234444444444445, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "in", "x1": 0.42934967320261447, "x2": 0.4458758169934641, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "batches;", "x1": 0.4517810457516341, "x2": 0.5320751633986929, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "marker", "x1": 0.5379803921568629, "x2": 0.6040637254901963, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "2.", "x1": 0.6099689542483662, "x2": 0.627684640522876, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2.txt b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2.txt new file mode 100644 index 00000000..b261308f Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p2.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-large.gif new file mode 100644 index 00000000..376847e7 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-normal.gif new file mode 100644 index 00000000..746b120e Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-small.gif new file mode 100644 index 00000000..d4949e00 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-thumbnail.gif new file mode 100644 index 00000000..cbfd6832 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-xlarge.gif new file mode 100644 index 00000000..85220c86 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3.position.json b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3.position.json new file mode 100644 index 00000000..197cb21a --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.11764705882352941, "x2": 0.1672467320261438, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "three", "x1": 0.17315196078431372, "x2": 0.22156209150326797, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "of", "x1": 0.22746732026143793, "x2": 0.2451830065359477, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.25108823529411767, "x2": 0.28061437908496734, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "four", "x1": 0.2865196078431373, "x2": 0.32311928104575166, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.32902450980392156, "x2": 0.3762663398692811, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.38217156862745105, "x2": 0.4164133986928105, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document.", "x1": 0.4223186274509805, "x2": 0.521496732026144, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "Text", "x1": 0.11764705882352941, "x2": 0.15896241830065358, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "position", "x1": 0.16486764705882354, "x2": 0.2380669934640523, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "extraction", "x1": 0.24397222222222223, "x2": 0.33605555555555555, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "happens", "x1": 0.3419607843137255, "x2": 0.4234444444444445, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "in", "x1": 0.42934967320261447, "x2": 0.4458758169934641, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "batches;", "x1": 0.4517810457516341, "x2": 0.5320751633986929, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "marker", "x1": 0.5379803921568629, "x2": 0.6040637254901963, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "3.", "x1": 0.6099689542483662, "x2": 0.627684640522876, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3.txt b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3.txt new file mode 100644 index 00000000..9531d36a Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p3.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-large.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-large.gif new file mode 100644 index 00000000..6861cfbc Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-large.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-normal.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-normal.gif new file mode 100644 index 00000000..259f2340 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-normal.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-small.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-small.gif new file mode 100644 index 00000000..821f4cc3 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-small.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-thumbnail.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-thumbnail.gif new file mode 100644 index 00000000..8b5c14d5 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-thumbnail.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-xlarge.gif b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-xlarge.gif new file mode 100644 index 00000000..5bc271e1 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4-xlarge.gif differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4.position.json b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4.position.json new file mode 100644 index 00000000..affce008 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4.position.json @@ -0,0 +1 @@ +[{"text": "Page", "x1": 0.11764705882352941, "x2": 0.1672467320261438, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "four", "x1": 0.17315196078431372, "x2": 0.2097516339869281, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "of", "x1": 0.21565686274509804, "x2": 0.23337254901960783, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "the", "x1": 0.23927777777777776, "x2": 0.2688039215686275, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "four", "x1": 0.27470915032679744, "x2": 0.3113088235294118, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "page", "x1": 0.3172140522875817, "x2": 0.36445588235294124, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "text", "x1": 0.37036111111111114, "x2": 0.40460294117647067, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "document.", "x1": 0.4105081699346406, "x2": 0.509686274509804, "y1": 0.10061994949494954, "y2": 0.11703409090909095, "upright": true, "direction": "ltr"}, {"text": "Text", "x1": 0.11764705882352941, "x2": 0.15896241830065358, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "position", "x1": 0.16486764705882354, "x2": 0.2380669934640523, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "extraction", "x1": 0.24397222222222223, "x2": 0.33605555555555555, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "happens", "x1": 0.3419607843137255, "x2": 0.4234444444444445, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "in", "x1": 0.42934967320261447, "x2": 0.4458758169934641, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "batches;", "x1": 0.4517810457516341, "x2": 0.5320751633986929, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "marker", "x1": 0.5379803921568629, "x2": 0.6040637254901963, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}, {"text": "4.", "x1": 0.6099689542483662, "x2": 0.627684640522876, "y1": 0.13016540404040405, "y2": 0.14657954545454546, "upright": true, "direction": "ltr"}] \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4.txt b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4.txt new file mode 100644 index 00000000..25ca47d9 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/pages/text-4page-p4.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.index b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.index new file mode 100644 index 00000000..fbbde451 Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.index differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.pdf b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.pdf new file mode 100644 index 00000000..69ce274e Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.txt b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.txt new file mode 100644 index 00000000..f217919e Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.txt differ diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.txt.json b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.txt.json new file mode 100644 index 00000000..5b9cd68c --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/documents/text-4page/expected/text-4page.txt.json @@ -0,0 +1 @@ +{"updated": 1784385686382, "pages": [{"page": 0, "contents": "Page one of the four page text document.\nText position extraction happens in batches; marker 1.\u0000", "ocr": null, "lang": "eng", "updated": 1784385686382}, {"page": 1, "contents": "Page two of the four page text document.\nText position extraction happens in batches; marker 2.\u0000", "ocr": null, "lang": "eng", "updated": 1784385686382}, {"page": 2, "contents": "Page three of the four page text document.\nText position extraction happens in batches; marker 3.\u0000", "ocr": null, "lang": "eng", "updated": 1784385686382}, {"page": 3, "contents": "Page four of the four page text document.\nText position extraction happens in batches; marker 4.\u0000", "ocr": null, "lang": "eng", "updated": 1784385686382}]} \ No newline at end of file diff --git a/documentcloud/documents/processing/tests/spec/documents/text-4page/input/text-4page.pdf b/documentcloud/documents/processing/tests/spec/documents/text-4page/input/text-4page.pdf new file mode 100644 index 00000000..568a17df Binary files /dev/null and b/documentcloud/documents/processing/tests/spec/documents/text-4page/input/text-4page.pdf differ diff --git a/documentcloud/documents/processing/tests/spec/generate.py b/documentcloud/documents/processing/tests/spec/generate.py new file mode 100644 index 00000000..412e94a4 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/generate.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +""" +(Re)generate the golden outputs for the processing pipeline test spec. + +For every test case under `documents/`, this runs the current pipeline +against the case's input document and snapshots everything it produced — +the storage files and the database-facing metadata — into the case's +`expected/` directory. + +Run this after intentionally changing pipeline behavior, review the diff, +and commit the result. `test_spec.py` then validates that the pipeline +still produces these outputs. + +Requirements: + - a running Redis (REDIS_PROCESSING_URL, defaults to localhost:6379) + - for OCR cases: the Git LFS Tesseract libraries + (git lfs pull --include "documentcloud/documents/processing/ocr/tesseract/*") + and the pinned eng.traineddata (downloaded automatically on first run) + +Usage: + python generate.py [case ...] # default: all cases + python generate.py --check [case ...] # regenerate into a scratch dir and + # compare against expected/ instead + # of overwriting it +""" + +# Standard Library +import argparse +import shutil +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).parents[5] +sys.path.insert(0, str(REPO_ROOT)) + +# DocumentCloud +from documentcloud.documents.processing.tests.spec import compare, harness + + +def snapshot(runner, case, metadata, destination): + """Copy a processed document directory plus its captured metadata into + the destination directory.""" + # Standard Library + import json + + destination = Path(destination) + shutil.rmtree(destination, ignore_errors=True) + shutil.copytree(runner.doc_directory(case), destination) + with open(destination / "metadata.json", "w", encoding="utf-8") as meta_file: + json.dump(metadata, meta_file, indent=2, sort_keys=True) + meta_file.write("\n") + + +def main(argv): + # pylint: disable=too-many-locals + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("cases", nargs="*", help="case names (default: all)") + parser.add_argument( + "--check", + action="store_true", + help="compare a fresh run against expected/ instead of overwriting", + ) + parser.add_argument( + "--equivalent", + action="store_true", + help="with --check: use the looser equivalence rules meant for " + "validating a modified or replacement pipeline (see README)", + ) + args = parser.parse_args(argv) + + harness.ensure_environment() + + cases = harness.all_cases() + if args.cases: + cases = [case for case in cases if case["name"] in args.cases] + missing = set(args.cases) - {case["name"] for case in cases} + if missing: + parser.error(f"unknown cases: {', '.join(sorted(missing))}") + + ocr_ready = harness.ocr_ready(download=True) + skipped = [case["name"] for case in cases if case["needs_ocr"] and not ocr_ready] + cases = [case for case in cases if not case["needs_ocr"] or ocr_ready] + if skipped: + print( + "OCR unavailable (git lfs pull the tesseract libraries and ensure " + f"the traineddata is present); skipping: {', '.join(skipped)}" + ) + + failures = [] + media_root = Path(tempfile.mkdtemp(prefix="dc-processing-spec-")) + try: + with harness.PipelineRunner(media_root, use_ocr=ocr_ready) as runner: + for case in cases: + print(f"processing {case['name']} ...", flush=True) + metadata = runner.run_case(case) + if bool(metadata["errors"]) != case["expects_error"]: + failures.append( + f"{case['name']}: expected errors={case['expects_error']}, " + f"pipeline reported: {metadata['errors']}" + ) + continue + if args.check: + scratch = media_root / "check" / case["name"] + snapshot(runner, case, metadata, scratch) + problems = compare.compare_case( + case["dir"] / "expected", + scratch, + ignore_metadata_fields=( + ("file_hash",) if case.get("redactions") else () + ), + strictness="equivalent" if args.equivalent else "exact", + ) + for problem in problems: + failures.append(f"{case['name']}: {problem}") + status = "ok" if not problems else f"{len(problems)} problem(s)" + print(f" checked against expected/: {status}") + else: + snapshot(runner, case, metadata, case["dir"] / "expected") + print(f" wrote {case['dir'] / 'expected'}") + finally: + shutil.rmtree(media_root, ignore_errors=True) + + if failures: + print() + for failure in failures: + print(f"FAIL: {failure}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/documentcloud/documents/processing/tests/spec/harness.py b/documentcloud/documents/processing/tests/spec/harness.py new file mode 100644 index 00000000..cbe15a73 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/harness.py @@ -0,0 +1,371 @@ +""" +Run the real processing pipeline in-process against local file storage. + +This drives the actual serverless functions (info_and_image, ocr) with: + +- the `local` environment (files written under MEDIA_ROOT on disk) +- a real Redis instance (`REDIS_PROCESSING_URL`, defaults to localhost:6379) +- pubsub topics wired to an in-process message queue: each function runs to + completion before the next message is dispatched, mirroring production + where every message runs in its own Lambda (pdfium cannot be nested + within one process) +- API callbacks captured instead of sent, so the database-facing side of the + output contract (page_count, page_spec, file_hash, status) is recorded + +Nothing in the pipeline itself is mocked: pdfium rendering, text extraction, +Tesseract OCR, grafting, and text position extraction all run for real. + +OCR requirements: the bundled Tesseract shared libraries are stored in Git +LFS (`git lfs pull --include "documentcloud/documents/processing/ocr/tesseract/*"`) +and an `eng.traineddata` must be available (see `fetch_traineddata`). +""" + +# Standard Library +import ctypes +import hashlib +import json +import os +import shutil +import urllib.request +from collections import deque +from pathlib import Path + +SPEC_DIR = Path(__file__).parent +PROCESSING_DIR = SPEC_DIR.parent.parent +CACHE_DIR = SPEC_DIR / ".cache" +DOCUMENTS_DIR = SPEC_DIR / "documents" + +TESSERACT_DIR = PROCESSING_DIR / "ocr" / "tesseract" +# Dependency order matters: each library must be loadable when the next one +# needs it (in production, Lambda's LD_LIBRARY_PATH handles this) +TESSERACT_LIBS = [ + "libjbig.so.0", + "libpng16.so.16.37.0", + "libjpeg.so.8.2.2", + "libwebp.so.6.0.2", + "libtiff.so.5.5.0", + "liblept.so.5", +] + +# The OCR language data used to generate the golden outputs. OCR output is +# only comparable when produced with the same traineddata, so the file is +# pinned by content hash. Production language packs live in the +# `ocr-languages` storage folder; if you want goldens that match production +# OCR exactly, drop the production eng.traineddata into .cache/ instead. +TRAINEDDATA_URL = ( + "https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/4.1.0/" + "eng.traineddata" +) +TRAINEDDATA_SHA256 = "7d4322bd2a7749724879683fc3912cb542f19906c83bcc1a52132556427170b2" + +DEFAULT_ENV = { + "ENVIRONMENT": "local", + "DOCUMENT_BUCKET": "spec-bucket", + "API_CALLBACK": "http://api.processing-spec.invalid/api/", + "PROCESSING_TOKEN": "processing-spec-token", + "REDIS_PROCESSING_URL": "redis://localhost:6379", + "REDIS_PROCESSING_PASSWORD": "", + "USE_TIMEOUT": "false", + "TIMEOUTS": "15,30,60,120", + # Required by the local httpsub mock; never actually called by the runner + "DOC_PROCESSING_URL": "mock://process", + "PROGRESS_URL": "mock://progress", + "IMPORT_URL": "mock://import", + "SIDEKICK_PROCESSING_URL": "mock://sidekick", +} + + +def ensure_environment(): + """Set the environment variables and Django settings the pipeline modules + need at import time. Values already present (e.g. in CI or a dev shell) + are left untouched.""" + for key, value in DEFAULT_ENV.items(): + os.environ.setdefault(key, value) + + # Django + from django.conf import settings + + if not settings.configured: + settings.configure(MEDIA_ROOT="") + + +def load_case(case_dir): + """Load a test case definition from its directory.""" + case_dir = Path(case_dir) + with open(case_dir / "case.json", encoding="utf-8") as case_file: + case = json.load(case_file) + case.setdefault("needs_ocr", False) + case.setdefault("expects_error", False) + case["dir"] = case_dir + case["name"] = case_dir.name + case["input"] = case_dir / "input" / f"{case['slug']}.pdf" + return case + + +def all_cases(): + """All test case definitions, sorted by name.""" + return [ + load_case(case_dir) + for case_dir in sorted(DOCUMENTS_DIR.iterdir()) + if (case_dir / "case.json").exists() + ] + + +def _is_lfs_pointer(file_path): + try: + with open(file_path, "rb") as pointer_file: + return pointer_file.read(24).startswith(b"version https://git-lfs") + except OSError: + return True + + +def ocr_libraries_available(): + """Whether the bundled Tesseract shared libraries are usable: real files + (they are stored in Git LFS and may be un-pulled pointers) that actually + load on this platform (they are Linux x86-64 builds).""" + if any( + _is_lfs_pointer(TESSERACT_DIR / lib) + for lib in TESSERACT_LIBS + ["libtesseract.so.5"] + ): + return False + try: + preload_tesseract_libraries() + ctypes.CDLL(str(TESSERACT_DIR / "libtesseract.so.5"), mode=ctypes.RTLD_GLOBAL) + except OSError: + return False + return True + + +def preload_tesseract_libraries(): + """Load libtesseract's dependencies into the process so ctypes can + resolve them (Lambda does this via LD_LIBRARY_PATH).""" + for lib in TESSERACT_LIBS: + ctypes.CDLL(str(TESSERACT_DIR / lib), mode=ctypes.RTLD_GLOBAL) + + +def traineddata_path(): + """Where the pinned eng.traineddata lives (override dir with + DC_SPEC_TESSDATA_DIR).""" + data_dir = Path(os.environ.get("DC_SPEC_TESSDATA_DIR", CACHE_DIR)) + return data_dir / "eng.traineddata" + + +def fetch_traineddata(): + """Download the pinned eng.traineddata if it is not cached yet. + Returns its path, or None if it cannot be obtained.""" + data_path = traineddata_path() + if data_path.exists(): + return data_path + data_path.parent.mkdir(parents=True, exist_ok=True) + try: + with urllib.request.urlopen(TRAINEDDATA_URL, timeout=120) as response: + contents = response.read() + except OSError: + return None + if hashlib.sha256(contents).hexdigest() != TRAINEDDATA_SHA256: + raise RuntimeError( + f"Downloaded traineddata does not match pinned hash " + f"{TRAINEDDATA_SHA256}" + ) + data_path.write_bytes(contents) + return data_path + + +def ocr_ready(download=False): + """Whether OCR-dependent cases can run in this checkout.""" + if not ocr_libraries_available(): + return False + if traineddata_path().exists(): + return True + return download and fetch_traineddata() is not None + + +class FakeResponse: + status_code = 200 + text = "" + + +class PipelineRunner: + """Context manager that wires the pipeline up against a MEDIA_ROOT + directory and runs test cases through it.""" + + # pylint: disable=too-many-instance-attributes + + def __init__(self, media_root, use_ocr=True): + ensure_environment() + self.media_root = Path(media_root) + self.use_ocr = use_ocr + self.callbacks = [] + self._overrides = [] + + def __enter__(self): + # Imports deferred: pipeline modules read env vars at import time + # Django + from django.test.utils import override_settings + + # DocumentCloud + from documentcloud.common.environment import publisher + from documentcloud.common.serverless import error_handling, utils + from documentcloud.documents.processing.info_and_image import main as ii_main + from documentcloud.documents.processing.ocr import main as ocr_main + + self.publisher = publisher + self.utils = utils + self.ii_main = ii_main + self.ocr_main = ocr_main + + self.media_root.mkdir(parents=True, exist_ok=True) + self._settings_override = override_settings(MEDIA_ROOT=str(self.media_root)) + self._settings_override.enable() + + # Capture API callbacks instead of performing HTTP requests + self._original_request = utils.request + utils.request = self._record_callback + + # The local-environment sentry stub re-raises exceptions "for + # debugging"; neutralize it so the error path behaves as in + # production (error is sent via the API callback and the function + # returns) and error cases can be snapshotted + self._original_capture_exception = utils.capture_exception + utils.capture_exception = lambda exc: None + + # Never use pebble subprocesses: the queue and the callback recorder + # live in this process + self._original_use_timeout = error_handling.USE_TIMEOUT + error_handling.USE_TIMEOUT = False + self._error_handling = error_handling + + # Dispatch every topic through the in-process message queue + self.queue = deque() + self._original_tasks = dict(publisher.tasks) + for topic, function in [ + (ii_main.PDF_PROCESS_TOPIC, ii_main.process_pdf), + (ii_main.PAGE_CACHE_TOPIC, ii_main.process_page_cache), + (ii_main.IMAGE_EXTRACT_TOPIC, ii_main.extract_image), + (ii_main.OCR_TOPIC, ocr_main.run_tesseract), + (ii_main.ASSEMBLE_TEXT_TOPIC, ii_main.assemble_page_text), + (ii_main.TEXT_POSITION_EXTRACT_TOPIC, ii_main.extract_text_position), + (ii_main.REDACT_TOPIC, ii_main.redact_doc), + ]: + publisher.register_internal_callback(topic, self._enqueue(function)) + + if self.use_ocr: + preload_tesseract_libraries() + self._stage_ocr_data() + + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.publisher.tasks = self._original_tasks + self.utils.request = self._original_request + self.utils.capture_exception = self._original_capture_exception + self._error_handling.USE_TIMEOUT = self._original_use_timeout + self._settings_override.disable() + + def _record_callback(self, _redis, method, url, json_): + self.callbacks.append({"method": method, "url": url, "json": json_}) + return FakeResponse() + + def _enqueue(self, function): + def callback(data): + self.queue.append((function, data)) + + return callback + + def _drain(self): + while self.queue: + function, data = self.queue.popleft() + function(data) + + def _stage_ocr_data(self): + """Place the OCR language data where the pipeline downloads it from + (the `ocr-languages` folder of local storage).""" + languages_dir = self.media_root / self.ocr_main.OCR_DATA_DIRECTORY + languages_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(TESSERACT_DIR / "tessdata" / "pdf.ttf", languages_dir / "pdf.ttf") + data_path = traineddata_path() + if data_path.exists(): + shutil.copy(data_path, languages_dir / data_path.name) + # The pipeline caches language data in TMP_DIRECTORY across + # invocations; clear it so each run uses the staged data + shutil.rmtree(self.ocr_main.TMP_DIRECTORY, ignore_errors=True) + + def doc_directory(self, case): + """The storage directory holding all of a document's files.""" + # DocumentCloud + from documentcloud.common import path + + return self.media_root / path.path(case["doc_id"]) + + def run_case(self, case): + """Run one test case through the pipeline. Returns the merged + database-facing metadata captured from the API callbacks.""" + # DocumentCloud + from documentcloud.common import access_choices, path + from documentcloud.common.environment import encode_pubsub_data + + doc_id = case["doc_id"] + slug = case["slug"] + settings = case.get("settings", {}) + + # Stage the input document where an upload would put it + doc_path = self.media_root / path.doc_path(doc_id, slug) + shutil.rmtree(self.doc_directory(case), ignore_errors=True) + doc_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(case["input"], doc_path) + + self.callbacks.clear() + redis = self.utils.get_redis() + self.utils.clean_up(redis, doc_id) + + # Standard processing, as triggered by an upload + self.utils.initialize(redis, doc_id) + self.publisher.publish( + self.ii_main.PDF_PROCESS_TOPIC, + encode_pubsub_data( + { + "doc_id": doc_id, + "slug": slug, + "access": access_choices.PRIVATE, + "ocr_code": settings.get("ocr_code", "eng"), + "force_ocr": settings.get("force_ocr", False), + "ocr_engine": settings.get("ocr_engine", "tess4"), + } + ), + ) + self._drain() + + # Optional second phase: redaction of the processed document + if case.get("redactions"): + self.utils.initialize(redis, doc_id) + self.publisher.publish( + self.ii_main.REDACT_TOPIC, + encode_pubsub_data( + { + "doc_id": doc_id, + "slug": slug, + "access": access_choices.PRIVATE, + "ocr_code": settings.get("ocr_code", "eng"), + "redactions": case["redactions"], + } + ), + ) + self._drain() + + return self.collect_metadata() + + def collect_metadata(self): + """Merge the captured API callbacks into the final database-facing + metadata, keeping the raw callback sequence for reference.""" + database = {} + errors = [] + for callback in self.callbacks: + if callback["method"] == "patch": + database.update(callback["json"]) + else: + errors.append(callback) + return { + "database": database, + "callbacks": self.callbacks + [], + "errors": errors, + } diff --git a/documentcloud/documents/processing/tests/spec/make_corpus.py b/documentcloud/documents/processing/tests/spec/make_corpus.py new file mode 100644 index 00000000..b9801533 --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/make_corpus.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +""" +Build the input documents for the processing pipeline test spec. + +Each test case directory under `documents/` gets an `input/` PDF built +deterministically with PyMuPDF. This script only needs to be re-run when the +corpus itself changes — the inputs are committed to the repository so that the +golden outputs always correspond to a known set of input bytes. + +Usage: + python make_corpus.py [case ...] # default: all cases +""" + +# Standard Library +import sys +from pathlib import Path + +# Third Party +import pymupdf + +SPEC_DIR = Path(__file__).parent +DOCUMENTS_DIR = SPEC_DIR / "documents" + +LETTER = (612, 792) # US Letter in PDF points +A5_LANDSCAPE = (595, 420) + +# Text sized and spaced generously so the scanned pages OCR cleanly +SCAN_FONT_SIZE = 28 +TEXT_FONT_SIZE = 13 + + +def add_text_page(pdf, lines, size=LETTER, font_size=TEXT_FONT_SIZE): + """Add a page with an embedded text layer.""" + page = pdf.new_page(width=size[0], height=size[1]) + y = 90 + for line in lines: + page.insert_text((72, y), line, fontname="helv", fontsize=font_size) + y += font_size * 1.8 + return page + + +def add_scanned_page(pdf, lines, size=LETTER, dpi=150): + """Add an image-only page: the text is rasterized so there is no text + layer, forcing the page through OCR.""" + # Render the text onto a scratch page, then insert it as an image + scratch = pymupdf.open() + page = scratch.new_page(width=size[0], height=size[1]) + y = 120 + for line in lines: + page.insert_text((72, y), line, fontname="helv", fontsize=SCAN_FONT_SIZE) + y += SCAN_FONT_SIZE * 2 + png_bytes = page.get_pixmap(dpi=dpi).tobytes("png") + scratch.close() + + image_page = pdf.new_page(width=size[0], height=size[1]) + image_page.insert_image(image_page.rect, stream=png_bytes) + return image_page + + +def build_text_1page(pdf): + add_text_page( + pdf, + [ + "PROCESSING PIPELINE TEST SPEC", + "Case: single page with an embedded text layer.", + "This page must not be OCR'd; its text is extracted directly", + "from the PDF and the output PDF keeps the original text layer.", + ], + ) + + +def build_text_3page(pdf): + for number, word in enumerate(["one", "two", "three"], start=1): + add_text_page( + pdf, + [ + f"Page {word} of the multi-page text document.", + f"The page text files must contain this page {number} marker.", + ], + ) + + +def build_text_4page(pdf): + # Four pages fill more than one TEXT_POSITION_BATCH (3), exercising the + # multi-batch flush path in extract_image + for number, word in enumerate(["one", "two", "three", "four"], start=1): + add_text_page( + pdf, + [ + f"Page {word} of the four page text document.", + f"Text position extraction happens in batches; marker {number}.", + ], + ) + + +def build_scan_2page(pdf): + add_scanned_page(pdf, ["SCANNED PAGE ONE", "NO TEXT LAYER HERE"]) + add_scanned_page(pdf, ["SCANNED PAGE TWO", "EVERY PAGE IS OCRD"]) + + +def build_mixed_2page(pdf): + add_text_page( + pdf, + [ + "Mixed document, page one has an embedded text layer.", + "Only the second page goes through OCR.", + ], + ) + add_scanned_page(pdf, ["MIXED PAGE TWO", "THIS PAGE IS SCANNED"]) + + +def build_force_ocr_1page(pdf): + add_text_page( + pdf, + [ + "FORCE OCR TEST DOCUMENT", + "This page has an embedded text layer, but the case settings", + "use force_ocr, so the pipeline must OCR it anyway.", + ], + font_size=SCAN_FONT_SIZE // 2, + ) + + +def build_mixed_sizes_2page(pdf): + add_text_page(pdf, ["Letter sized page.", "612 by 792 points."], size=LETTER) + add_text_page( + pdf, + ["A5 landscape page.", "595 by 420 points."], + size=A5_LANDSCAPE, + ) + + +def build_blank_1page(pdf): + pdf.new_page(width=LETTER[0], height=LETTER[1]) + + +def build_redact_2page(pdf): + add_text_page( + pdf, + [ + "Redaction test document, page one is untouched.", + "Only page two receives a redaction.", + ], + ) + add_text_page( + pdf, + [ + "Page two before redaction.", + "SECRET LINE THAT GETS REDACTED", + "Visible line under the redaction.", + ], + ) + + +BUILDERS = { + "text-1page": build_text_1page, + "text-3page": build_text_3page, + "text-4page": build_text_4page, + "scan-2page": build_scan_2page, + "mixed-2page": build_mixed_2page, + "force-ocr-1page": build_force_ocr_1page, + "mixed-sizes-2page": build_mixed_sizes_2page, + "blank-1page": build_blank_1page, + "redact-2page": build_redact_2page, +} + + +def build_corrupt_1page(): + """A file with a PDF header but truncated, unparseable structure — the + pipeline must report an error through the API callback.""" + return b"%PDF-1.6\n1 0 obj\n<< /Type /Catalog" + b"\x00garbage" * 8 + + +RAW_BUILDERS = { + "corrupt-1page": build_corrupt_1page, +} + + +def main(argv): + names = argv or sorted(BUILDERS) + sorted(RAW_BUILDERS) + for name in names: + input_dir = DOCUMENTS_DIR / name / "input" + input_dir.mkdir(parents=True, exist_ok=True) + output = input_dir / f"{name}.pdf" + if name in RAW_BUILDERS: + output.write_bytes(RAW_BUILDERS[name]()) + else: + pdf = pymupdf.open() + BUILDERS[name](pdf) + pdf.save(str(output), garbage=4, deflate=True, deflate_images=True) + pdf.close() + print(f"wrote {output}") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/documentcloud/documents/processing/tests/spec/test_spec.py b/documentcloud/documents/processing/tests/spec/test_spec.py new file mode 100644 index 00000000..e555e38b --- /dev/null +++ b/documentcloud/documents/processing/tests/spec/test_spec.py @@ -0,0 +1,86 @@ +""" +Validate the processing pipeline against the committed golden outputs. + +For every case under `documents/` that has an `expected/` directory, run the +real pipeline on the case's input document and compare everything it +produces — storage files and database-facing metadata — against the golden +copies. See README.md for the comparison rules and how to regenerate the +goldens after an intentional behavior change (`python generate.py`). + +Requires a reachable Redis (as the pipeline tests in CI already do). Cases +that need OCR are skipped unless the Git LFS Tesseract libraries and the +pinned eng.traineddata are present; set DC_SPEC_REQUIRE_OCR=1 (as CI does) +to make an unavailable OCR runtime a failure instead of a skip. +""" + +# Standard Library +import json +import os +import shutil +import tempfile +from pathlib import Path + +# Third Party +import pytest + +# DocumentCloud +from documentcloud.documents.processing.tests.spec import compare, harness + +# Pytest fixtures are injected by parameter name +# pylint: disable=redefined-outer-name + + +CASES = [case for case in harness.all_cases() if (case["dir"] / "expected").exists()] + + +def _case_id(case): + return case["name"] + + +@pytest.fixture(scope="module") +def ocr_ready(): + # Downloads the hash-pinned eng.traineddata on first use (a few MB); + # once cached under spec/.cache/ no network is needed + return harness.ocr_ready(download=True) + + +@pytest.mark.slow +@pytest.mark.parametrize("case", CASES, ids=_case_id) +def test_pipeline_output_contract(case, ocr_ready): + if case["needs_ocr"] and not ocr_ready: + message = ( + "OCR unavailable: run git lfs pull --include " + '"documentcloud/documents/processing/ocr/tesseract/*" and ' + "provide eng.traineddata (see spec/README.md)" + ) + if os.environ.get("DC_SPEC_REQUIRE_OCR"): + pytest.fail(message) + pytest.skip(message) + + media_root = Path(tempfile.mkdtemp(prefix="dc-processing-spec-")) + with harness.PipelineRunner(media_root, use_ocr=ocr_ready) as runner: + metadata = runner.run_case(case) + if not case["expects_error"]: + assert not metadata[ + "errors" + ], f"pipeline reported errors: {metadata['errors']}" + + actual_dir = media_root / "actual" + shutil.copytree(runner.doc_directory(case), actual_dir) + with open(actual_dir / "metadata.json", "w", encoding="utf-8") as meta_file: + json.dump(metadata, meta_file) + + problems = compare.compare_case( + case["dir"] / "expected", + actual_dir, + ignore_metadata_fields=(("file_hash",) if case.get("redactions") else ()), + ) + # Keep the output tree on failure so it can be diffed against expected/ + assert not problems, "\n".join( + [ + f"{case['name']} does not match its golden outputs " + f"(actual output kept at {actual_dir}):" + ] + + problems + ) + shutil.rmtree(media_root, ignore_errors=True)