A Developer’s Guide to OmniParseBench: Leaderboard Benchmarks and OCR Testing

import os
import re
import sys
import json
import time
import html
import shutil
import platform
import traceback
import subprocess
import collections
import urllib.request
from pathlib import Path

RESULTS = {}          # step -> one-line result, for the summary table
CLAIMS = []           # (step, claim, held)
CURRENT = ["1"]


def banner(title):
    print("\n" + "=" * 78)
    print(title)
    print("=" * 78)


def section(name):
    def wrap(fn):
        def run(*a, **kw):
            CURRENT[0] = name.split(".")[0]
            banner(name)
            t0 = time.time()
            try:
                out = fn(*a, **kw)
                RESULTS[name] = out if isinstance(out, str) else "ok"
                return out
            except Exception as e:
                RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
                print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
                traceback.print_exc(limit=3)
                return None
            finally:
                print(f"\n[{time.time() - t0:.1f} s]")
        return run
    return wrap


def claim(text, held):
    CLAIMS.append((CURRENT[0], text, bool(held)))
    return held


banner("1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision")
SYSTEM = platform.system()
OPB_VERSION = "1.0.0rc1"        # a pre-release: a bare `pip install omni-parse-bench` resolves to 0.2.0, whose API differs
REPO = "datalab-to/omni_parse_bench"
REV = "961c0f89a2da701f5ce53afbd5e80aa469b0369a"   # the dataset with the published verdicts of 27 parsers
INDRA = ["indra-ocr==0.3.2", "numpy==2.5.4", "onnxruntime==1.31.0", "opencv-python-headless==5.0.0.93",
         "pypdfium2==5.14.0", "huggingface_hub==2.2.0"]   # the pins of the package's own Indra server image
WORK = Path(os.environ.get("OPB_WORK") or ("/content/opb" if os.path.isdir("/content") else "opb_work")).resolve()
WORK.mkdir(parents=True, exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", f"omni-parse-bench[benchmark]=={OPB_VERSION}", "uv"], check=True)
if SYSTEM == "Linux" and shutil.which("tesseract") is None:
    scripts = ("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang hang-vert hans hans-vert hant "
               "hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split()
    subprocess.run(["apt-get", "-qq", "update"], check=True)
    subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"]
                   + [f"tesseract-ocr-script-{s}" for s in scripts], check=True, stdout=subprocess.DEVNULL)

import numpy as np
import pandas as pd
import importlib.metadata as md
from huggingface_hub import snapshot_download
from omni_parse_bench import benchmark, data, harness, markdown, vocab
from omni_parse_bench.scorers import unit_tests

print(f"omni-parse-bench {md.version('omni-parse-bench')}, Python {platform.python_version()}, {SYSTEM} {platform.machine()}, "
      f"{os.cpu_count()} CPU cores")
if SYSTEM == "Linux":
    print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
tess = subprocess.run(["tesseract", "--version"], capture_output=True, text=True)
print("tesseract:", (tess.stdout or tess.stderr).splitlines()[0])
TESS_MAJOR = int((tess.stdout or tess.stderr).split()[1].split(".")[0])
print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())})
print(f"dataset {REPO} @ {REV[:7]}; work folder {WORK}")
RESULTS["1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision"] = (
    f"omni-parse-bench {md.version('omni-parse-bench')}, tesseract {(tess.stdout or tess.stderr).split()[1]}, dataset @ {REV[:7]}")

        

@section("2. How the scorer decides: one hand-built page, five test types, one change at a time")

def scorer_anatomy():

T = lambda i, tt, out, tags, args, st=None: {"id": i, "test_type": tt, "output_type": out, "derivation": "authored",

"tags": tags, "args": args, "settings": st or {}}

exact = lambda s: {"text": s, "max_diffs": 0}

gold = [

T("title", "present", "html", ["heading"], {"content": exact("Quarterly Report")}),

T("total", "present", "html", ["eng"], {"content": exact("Total due")}),

T("growth", "present", "html", ["eng"], {"content": {"number": "-4.8%"}}),

T("amount", "present", "html", ["eng"], {"content": {"number": "1,240.00"}}),

T("eq_ratio", "present", "html", ["math"], {"content": {"latex": r"\alpha_{1} + \beta"}}),

T("eq_half", "present", "html", ["math"], {"content": {"latex": r"\frac{1}{2} \to x"}}),

T("order", "order", "html", ["eng"], {"before": exact("Introduction: this report covers the second quarter."),

"after": exact("Conclusion: the plan holds.")}),

T("repeat", "repeat", "html", ["eng"], {"content": exact("Revenue")}, {"expected_count": 2}),

T("cell_top", "table_cell", "html
Source Read original →
Related reading
A Developer’s Guide to Laya: Zero-Shot Decisions and Calibration
The Developer’s Guide to NeMo Guardrails for Enterprise AI Safety
A Coding Guide to Cactus Whistle: Does a 16.9 MB Speech Model Hold Up Against Whisper Base? Word Error Rate, Speed, Streaming, Timestamps, Keywords, and Speech-to-Tool Calls on a CPU