From 86b48d2c96818f4dd7897627b04cdaf2c1dc1172 Mon Sep 17 00:00:00 2001 From: SWHL Date: Fri, 12 Jun 2026 17:19:53 +0800 Subject: [PATCH] feat: optimize the wheel packaging process (#687) * chore: add pyproject toml configuration * build: migrate RapidOCR wheel packaging to pyproject.toml - add PEP 517 setuptools build configuration - derive package version from git tags via setuptools-scm - move wheel model preparation into an explicit tools script - generate MANIFEST.in for default model assets before building - avoid importing rapidocr during package metadata generation * ci: build release wheels from validated git tags * build: align package runtime support with Python 3.8 - set Requires-Python to >=3.8 - remove Python 3.6 and 3.7 classifiers - make wheel asset preparation script type hints Python 3.8 compatible - clarify split between build-time and wheel runtime support --- .../workflows/gen_whl_to_pypi_rapidocr.yml | 32 +- .gitignore | 2 + python/README.md | 2 +- python/pyproject.toml | 67 ++++ python/setup.py | 124 ------- python/tools/prepare_wheel_assets.py | 351 ++++++++++++++++++ 6 files changed, 445 insertions(+), 133 deletions(-) create mode 100644 python/pyproject.toml delete mode 100644 python/setup.py create mode 100644 python/tools/prepare_wheel_assets.py diff --git a/.github/workflows/gen_whl_to_pypi_rapidocr.yml b/.github/workflows/gen_whl_to_pypi_rapidocr.yml index 4c47bd0..a4c6de9 100644 --- a/.github/workflows/gen_whl_to_pypi_rapidocr.yml +++ b/.github/workflows/gen_whl_to_pypi_rapidocr.yml @@ -3,20 +3,23 @@ name: Push rapidocr to pypi on: push: tags: - - v* + - "v*" jobs: TestAndPublish: runs-on: ubuntu-latest + steps: - name: Pull latest code uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: '3.10' - architecture: 'x64' + python-version: "3.10" + architecture: "x64" - name: Display Python version run: python -c "import sys; print(sys.version)" @@ -25,9 +28,9 @@ jobs: run: | cd python python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest wheel setuptools get_pypi_latest_version openvino==2023.3.0 onnxruntime python-bidi MNN - pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + python -m pip install -r requirements.txt + python -m pip install pytest build setuptools wheel setuptools-scm openvino==2023.3.0 onnxruntime python-bidi MNN + python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu python -m pip install paddlepaddle==3.1.1 -i https://www.paddlepaddle.org.cn/packages/stable/cpu/ - name: Unit testings @@ -35,13 +38,26 @@ jobs: cd python pytest tests/test_*.py + - name: Verify release tag + run: | + test -n "$GITHUB_REF_NAME" + git tag --points-at HEAD | grep -Fx "$GITHUB_REF_NAME" + VERSION="${GITHUB_REF_NAME#v}" + echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RAPIDOCR=$VERSION" >> "$GITHUB_ENV" + echo "Building rapidocr version: $VERSION" + + - name: Prepare wheel assets + run: | + cd python + python tools/prepare_wheel_assets.py + - name: Build wheel package run: | cd python - python setup.py bdist_wheel ${{ github.ref_name }} + python -m build --wheel mv dist ../ - - name: Publish distribution 📦 to PyPI + - name: Publish distribution to PyPI uses: pypa/gh-action-pypi-publish@v1.5.0 with: password: ${{ secrets.RAPIDOCR }} diff --git a/.gitignore b/.gitignore index 4c21ff5..79b5601 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +python/rapidocr/_version.py + # Created by .ignore support plugin (hsz.mobi) ### Python template # Byte-compiled / optimized / DLL files diff --git a/python/README.md b/python/README.md index e20c2bb..0248abd 100755 --- a/python/README.md +++ b/python/README.md @@ -1 +1 @@ -### See [Documentation](https://rapidai.github.io/RapidOCRDocs/main/install_usage/rapidocr/install/) +### See [Documentation](https://rapidai.github.io/RapidOCRDocs) diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..695cb11 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,67 @@ +[build-system] +requires = [ + "setuptools>=77", + "wheel", + "setuptools-scm>=8", +] +build-backend = "setuptools.build_meta" + +[project] +name = "rapidocr" +dynamic = ["version", "dependencies"] +description = "Awesome OCR Library" +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.8,<4" +license = "Apache-2.0" +authors = [ + { name = "SWHL", email = "liekkaskono@163.com" }, +] +keywords = [ + "ocr", + "text_detection", + "text_recognition", + "db", + "onnxruntime", + "paddleocr", + "openvino", + "rapidocr", +] +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] + +[project.urls] +Documentation = "https://rapidai.github.io/RapidOCRDocs" +Changelog = "https://github.com/RapidAI/RapidOCR/releases" + +[project.scripts] +rapidocr = "rapidocr.main:main" + +[tool.setuptools] +include-package-data = true +platforms = ["Any"] + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements.txt"] } + +[tool.setuptools.packages.find] +where = ["."] +include = ["rapidocr*"] +exclude = ["tests*"] +namespaces = false + +[tool.setuptools.package-data] +rapidocr = [ + "**/*.yaml", +] + +[tool.setuptools_scm] +root = ".." +tag_regex = "^v?(?P[0-9]+(?:\\.[0-9]+)*.*)$" +version_file = "rapidocr/_version.py" +local_scheme = "no-local-version" diff --git a/python/setup.py b/python/setup.py deleted file mode 100644 index cd015de..0000000 --- a/python/setup.py +++ /dev/null @@ -1,124 +0,0 @@ -# -*- encoding: utf-8 -*- -# @Author: SWHL -# @Contact: liekkaskono@163.com -import shutil -import sys -from pathlib import Path -from typing import List, Union - -import setuptools -from get_pypi_latest_version import GetPyPiLatestVersion - -from rapidocr.utils.download_models import download_models - - -def read_txt(txt_path: Union[Path, str]) -> List[str]: - with open(txt_path, "r", encoding="utf-8") as f: - data = [v.rstrip("\n") for v in f] - return data - - -def get_readme(): - root_dir = Path(__file__).resolve().parent.parent - readme_path = str(root_dir / "docs" / "doc_whl_rapidocr.md") - print(readme_path) - with open(readme_path, "r", encoding="utf-8") as f: - readme = f.read() - return readme - - -def collect_package_files(package_root: Path, pattern: str) -> List[str]: - return sorted( - path.relative_to(package_root).as_posix() - for path in package_root.rglob(pattern) - if path.is_file() - ) - - -download_models() - -MODULE_NAME = "rapidocr" -PACKAGE_DIR = Path(MODULE_NAME) -PACKAGE_ROOT = PACKAGE_DIR -WRAP_DIR = Path(f"wrap_temp/{MODULE_NAME}") -NEED_WRAP = "bdist_wheel" in sys.argv - -if NEED_WRAP: - if WRAP_DIR.exists(): - shutil.rmtree(WRAP_DIR) - WRAP_DIR.mkdir(parents=True) - - dest_dir = WRAP_DIR / MODULE_NAME - shutil.copytree(PACKAGE_DIR, dest_dir) - - with open(WRAP_DIR / "__init__.py", "w", encoding="utf-8") as f: - f.write("from .rapidocr.main import RapidOCR, VisRes\n") - - PACKAGE_DIR = WRAP_DIR - PACKAGE_ROOT = PACKAGE_DIR / MODULE_NAME - -PACKAGE_DATA = collect_package_files(PACKAGE_ROOT, "*.yaml") -PACKAGE_DATA.extend( - [ - "models/ch_PP-OCRv4_det_mobile.onnx", - "models/ch_PP-OCRv4_rec_mobile.onnx", - "models/ch_ppocr_mobile_v2.0_cls_mobile.onnx", - ] -) - -obtainer = GetPyPiLatestVersion() -try: - latest_version = obtainer(MODULE_NAME) -except Exception as e: - latest_version = "0.0.0" -VERSION_NUM = obtainer.version_add_one(latest_version, add_patch=True) - -if len(sys.argv) > 2: - match_str = " ".join(sys.argv[2:]) - matched_versions = obtainer.extract_version(match_str) - if matched_versions: - VERSION_NUM = matched_versions -sys.argv = sys.argv[:2] - -project_urls = { - "Documentation": "https://rapidai.github.io/RapidOCRDocs", - "Changelog": "https://github.com/RapidAI/RapidOCR/releases", -} - -setuptools.setup( - name=MODULE_NAME, - version=VERSION_NUM, - platforms="Any", - description="Awesome OCR Library", - long_description=get_readme(), - long_description_content_type="text/markdown", - author="SWHL", - author_email="liekkaskono@163.com", - url="https://github.com/RapidAI/RapidOCR", - project_urls=project_urls, - license="Apache-2.0", - include_package_data=True, - install_requires=read_txt("requirements.txt"), - package_dir={"": str(PACKAGE_DIR)}, - packages=setuptools.find_namespace_packages(where=str(PACKAGE_DIR)), - package_data={MODULE_NAME: PACKAGE_DATA}, - keywords=[ - "ocr,text_detection,text_recognition,db,onnxruntime,paddleocr,openvino,rapidocr" - ], - classifiers=[ - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - ], - python_requires=">=3.6,<4", - entry_points={ - "console_scripts": [f"{MODULE_NAME}={MODULE_NAME}.main:main"], - }, -) - -if NEED_WRAP and Path("wrap_temp").exists(): - shutil.rmtree("wrap_temp") diff --git a/python/tools/prepare_wheel_assets.py b/python/tools/prepare_wheel_assets.py new file mode 100644 index 0000000..eb1f7ac --- /dev/null +++ b/python/tools/prepare_wheel_assets.py @@ -0,0 +1,351 @@ +# -*- encoding: utf-8 -*- +# @Author: SWHL +# @Contact: liekkaskono@163.com +import argparse +import hashlib +import sys +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Optional + +PYTHON_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_CONFIG_YAML = PYTHON_ROOT / "rapidocr" / "config.yaml" +DEFAULT_MANIFEST_IN = PYTHON_ROOT / "MANIFEST.in" +DEFAULT_MODEL_DIR = PYTHON_ROOT / "rapidocr" / "models" +DEFAULT_MODELS_YAML = PYTHON_ROOT / "rapidocr" / "default_models.yaml" +CHUNK_SIZE = 1024 * 1024 +REQUEST_TIMEOUT = 60 +TASK_SECTIONS = ("Det", "Cls", "Rec") + + +@dataclass(frozen=True) +class ModelSpec: + engine: str + ocr_version: str + task: str + lang: str + model_type: str + + +@dataclass(frozen=True) +class WheelAsset: + relative_path: Path + url: str + sha256: Optional[str] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Prepare model assets that should be bundled into the RapidOCR wheel." + ) + parser.add_argument( + "--config-yaml", + type=Path, + default=DEFAULT_CONFIG_YAML, + help=f"RapidOCR default config used to select wheel models. Default: {DEFAULT_CONFIG_YAML}", + ) + parser.add_argument( + "--models-yaml", + type=Path, + default=DEFAULT_MODELS_YAML, + help=f"Model registry YAML used as the source of URLs and checksums. Default: {DEFAULT_MODELS_YAML}", + ) + parser.add_argument( + "--model-dir", + type=Path, + default=DEFAULT_MODEL_DIR, + help=f"Directory used to store wheel model assets. Default: {DEFAULT_MODEL_DIR}", + ) + parser.add_argument( + "--manifest-in", + type=Path, + default=DEFAULT_MANIFEST_IN, + help=f"MANIFEST.in file to write with exact wheel asset includes. Default: {DEFAULT_MANIFEST_IN}", + ) + parser.add_argument( + "--skip-manifest-in", + action="store_true", + help="Do not write MANIFEST.in after preparing assets.", + ) + parser.add_argument( + "--check", + action="store_true", + help="Only verify that required assets exist and match their checksums.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Download assets even when an existing file passes checksum validation.", + ) + return parser.parse_args() + + +def load_yaml(path: Path) -> dict: + try: + import yaml + except ImportError as e: + raise RuntimeError( + "PyYAML is required to read RapidOCR YAML config files. " + "Install it with `python -m pip install PyYAML` or " + "`python -m pip install -r requirements.txt`." + ) from e + + with path.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + if not isinstance(data, dict): + raise RuntimeError(f"Invalid YAML mapping: {path}") + + return data + + +def resolve_model_specs(config: dict) -> List[ModelSpec]: + global_cfg = config.get("Global", {}) + specs = [] + + for section in TASK_SECTIONS: + task_cfg = config.get(section) + if not isinstance(task_cfg, dict): + raise RuntimeError(f"Missing config section: {section}") + + task = _required_str(task_cfg, "task_type", section) + if not global_cfg.get(f"use_{task}", True): + continue + + specs.append( + ModelSpec( + engine=_required_str(task_cfg, "engine_type", section), + ocr_version=_required_str(task_cfg, "ocr_version", section), + task=task, + lang=_required_str(task_cfg, "lang_type", section), + model_type=_required_str(task_cfg, "model_type", section), + ) + ) + + if not specs: + raise RuntimeError("No enabled OCR tasks found in config.") + + return specs + + +def resolve_assets(specs: Iterable[ModelSpec], registry: dict) -> List[WheelAsset]: + assets = [] + for spec in specs: + model_info = select_model_info(spec, registry) + assets.extend(model_info_to_assets(spec, model_info)) + + return _dedupe_assets(assets) + + +def select_model_info(spec: ModelSpec, registry: dict) -> dict: + try: + task_models = registry[spec.engine][spec.ocr_version][spec.task] + except KeyError as e: + raise RuntimeError( + "Missing model registry section: " + f"{spec.engine}.{spec.ocr_version}.{spec.task}" + ) from e + + for model_name, model_info in task_models.items(): + if model_name.startswith(spec.lang) and spec.model_type in model_name: + if not isinstance(model_info, dict): + raise RuntimeError(f"Invalid model registry entry: {model_name}") + return model_info + + raise RuntimeError( + "No model registry entry matches default config: " + f"{spec.engine}.{spec.ocr_version}.{spec.task} " + f"lang={spec.lang} model_type={spec.model_type}" + ) + + +def model_info_to_assets(spec: ModelSpec, model_info: dict) -> List[WheelAsset]: + model_url = _required_str(model_info, "model_dir", "model registry") + + if spec.engine == "paddle": + assets = [ + WheelAsset( + relative_path=Path(Path(model_url).name) / filename, + url=f"{model_url.rstrip('/')}/{filename}", + sha256=sha256, + ) + for filename, sha256 in model_info.items() + if filename not in ("model_dir", "dict_url") + ] + else: + assets = [ + WheelAsset( + relative_path=Path(Path(model_url).name), + url=model_url, + sha256=model_info.get("SHA256"), + ) + ] + + dict_url = model_info.get("dict_url") + if spec.task == "rec" and dict_url: + assets.append( + WheelAsset( + relative_path=Path(Path(dict_url).name), + url=dict_url, + sha256=None, + ) + ) + + return assets + + +def prepare_assets( + assets: Iterable[WheelAsset], model_dir: Path, check_only: bool, force: bool +) -> int: + model_dir = model_dir.expanduser().resolve() + missing_or_invalid = [] + + for asset in assets: + save_path = model_dir / asset.relative_path + if not force and validate_asset(save_path, asset.sha256): + print(f"OK {save_path}") + continue + + if check_only: + status = "missing" if not save_path.exists() else "invalid checksum" + print(f"FAIL {save_path} ({status})") + missing_or_invalid.append(save_path) + continue + + download_asset(asset, save_path) + + if missing_or_invalid: + print( + "\nRequired wheel assets are missing or invalid. " + "Run without --check to download them.", + file=sys.stderr, + ) + return 1 + + print("\nWheel assets are ready.") + return 0 + + +def write_manifest_in(manifest_path: Path, assets: Iterable[WheelAsset]) -> None: + manifest_path = manifest_path.expanduser().resolve() + lines = [ + "# Generated by tools/prepare_wheel_assets.py.", + "# Re-run that script after changing rapidocr/config.yaml defaults.", + ] + for asset in assets: + package_path = Path("rapidocr") / "models" / asset.relative_path + lines.append(f"include {package_path.as_posix()}") + + manifest_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"Wrote {manifest_path}") + + +def file_sha256(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(CHUNK_SIZE), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def validate_asset(path: Path, expected_sha256: Optional[str]) -> bool: + if not path.is_file(): + return False + if expected_sha256 is None: + return True + return file_sha256(path) == expected_sha256 + + +def download_asset(asset: WheelAsset, save_path: Path) -> None: + save_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = save_path.with_suffix(f"{save_path.suffix}.part") + + request = urllib.request.Request( + asset.url, + headers={"User-Agent": "RapidOCR wheel asset preparer"}, + ) + print(f"Downloading {asset.relative_path.as_posix()}") + print(f" from {asset.url}") + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response: + total_size = _get_content_length(response.headers.get("Content-Length")) + downloaded = 0 + with tmp_path.open("wb") as f: + while True: + chunk = response.read(CHUNK_SIZE) + if not chunk: + break + f.write(chunk) + downloaded += len(chunk) + _print_progress(downloaded, total_size) + + if total_size is not None: + print() + + if asset.sha256 is not None: + actual_sha256 = file_sha256(tmp_path) + if actual_sha256 != asset.sha256: + tmp_path.unlink(missing_ok=True) + raise RuntimeError( + f"Checksum mismatch for {asset.relative_path}: " + f"expected {asset.sha256}, got {actual_sha256}" + ) + + tmp_path.replace(save_path) + print(f"Saved {save_path}") + + +def _dedupe_assets(assets: Iterable[WheelAsset]) -> List[WheelAsset]: + deduped = {} + for asset in assets: + existing = deduped.get(asset.relative_path) + if existing and existing != asset: + raise RuntimeError( + f"Conflicting wheel asset definitions: {asset.relative_path}" + ) + deduped[asset.relative_path] = asset + return list(deduped.values()) + + +def _get_content_length(value: Optional[str]) -> Optional[int]: + if value is None: + return None + try: + return int(value) + except ValueError: + return None + + +def _print_progress(downloaded: int, total_size: Optional[int]) -> None: + if not total_size: + return + percent = downloaded / total_size * 100 + print( + f"\r {downloaded / 1024 / 1024:.1f}MB / " + f"{total_size / 1024 / 1024:.1f}MB ({percent:.1f}%)", + end="", + ) + + +def _required_str(data: dict, key: str, section: str) -> str: + value = data.get(key) + if not isinstance(value, str) or not value: + raise RuntimeError(f"Missing required string field {section}.{key}") + return value + + +def main() -> int: + args = parse_args() + config = load_yaml(args.config_yaml) + registry = load_yaml(args.models_yaml) + specs = resolve_model_specs(config) + assets = resolve_assets(specs, registry) + + status = prepare_assets(assets, args.model_dir, args.check, args.force) + if status == 0 and not args.check and not args.skip_manifest_in: + write_manifest_in(args.manifest_in, assets) + return status + + +if __name__ == "__main__": + raise SystemExit(main())