mirror of
https://github.com/RapidAI/RapidOCR.git
synced 2026-11-10 21:10:55 +08:00
feat: optimize the wheel packaging process (#687)
* chore: add pyproject toml configuration * build: migrate RapidOCR wheel packaging to pyproject.toml - add PEP 517 setuptools build configuration - derive package version from git tags via setuptools-scm - move wheel model preparation into an explicit tools script - generate MANIFEST.in for default model assets before building - avoid importing rapidocr during package metadata generation * ci: build release wheels from validated git tags * build: align package runtime support with Python 3.8 - set Requires-Python to >=3.8 - remove Python 3.6 and 3.7 classifiers - make wheel asset preparation script type hints Python 3.8 compatible - clarify split between build-time and wheel runtime support
This commit is contained in:
6 files changed
+445
-133
No files matched your search
+1
-1
@@ -1 +1 @@
|
||||
### See [Documentation](https://rapidai.github.io/RapidOCRDocs/main/install_usage/rapidocr/install/)
|
||||
### See [Documentation](https://rapidai.github.io/RapidOCRDocs)
|
||||
@@ -0,0 +1,67 @@
|
||||
[build-system]
|
||||
requires = [
|
||||
"setuptools>=77",
|
||||
"wheel",
|
||||
"setuptools-scm>=8",
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "rapidocr"
|
||||
dynamic = ["version", "dependencies"]
|
||||
description = "Awesome OCR Library"
|
||||
readme = { file = "README.md", content-type = "text/markdown" }
|
||||
requires-python = ">=3.8,<4"
|
||||
license = "Apache-2.0"
|
||||
authors = [
|
||||
{ name = "SWHL", email = "liekkaskono@163.com" },
|
||||
]
|
||||
keywords = [
|
||||
"ocr",
|
||||
"text_detection",
|
||||
"text_recognition",
|
||||
"db",
|
||||
"onnxruntime",
|
||||
"paddleocr",
|
||||
"openvino",
|
||||
"rapidocr",
|
||||
]
|
||||
classifiers = [
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Documentation = "https://rapidai.github.io/RapidOCRDocs"
|
||||
Changelog = "https://github.com/RapidAI/RapidOCR/releases"
|
||||
|
||||
[project.scripts]
|
||||
rapidocr = "rapidocr.main:main"
|
||||
|
||||
[tool.setuptools]
|
||||
include-package-data = true
|
||||
platforms = ["Any"]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
dependencies = { file = ["requirements.txt"] }
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["rapidocr*"]
|
||||
exclude = ["tests*"]
|
||||
namespaces = false
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
rapidocr = [
|
||||
"**/*.yaml",
|
||||
]
|
||||
|
||||
[tool.setuptools_scm]
|
||||
root = ".."
|
||||
tag_regex = "^v?(?P<version>[0-9]+(?:\\.[0-9]+)*.*)$"
|
||||
version_file = "rapidocr/_version.py"
|
||||
local_scheme = "no-local-version"
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
# @Author: SWHL
|
||||
# @Contact: liekkaskono@163.com
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Union
|
||||
|
||||
import setuptools
|
||||
from get_pypi_latest_version import GetPyPiLatestVersion
|
||||
|
||||
from rapidocr.utils.download_models import download_models
|
||||
|
||||
|
||||
def read_txt(txt_path: Union[Path, str]) -> List[str]:
|
||||
with open(txt_path, "r", encoding="utf-8") as f:
|
||||
data = [v.rstrip("\n") for v in f]
|
||||
return data
|
||||
|
||||
|
||||
def get_readme():
|
||||
root_dir = Path(__file__).resolve().parent.parent
|
||||
readme_path = str(root_dir / "docs" / "doc_whl_rapidocr.md")
|
||||
print(readme_path)
|
||||
with open(readme_path, "r", encoding="utf-8") as f:
|
||||
readme = f.read()
|
||||
return readme
|
||||
|
||||
|
||||
def collect_package_files(package_root: Path, pattern: str) -> List[str]:
|
||||
return sorted(
|
||||
path.relative_to(package_root).as_posix()
|
||||
for path in package_root.rglob(pattern)
|
||||
if path.is_file()
|
||||
)
|
||||
|
||||
|
||||
download_models()
|
||||
|
||||
MODULE_NAME = "rapidocr"
|
||||
PACKAGE_DIR = Path(MODULE_NAME)
|
||||
PACKAGE_ROOT = PACKAGE_DIR
|
||||
WRAP_DIR = Path(f"wrap_temp/{MODULE_NAME}")
|
||||
NEED_WRAP = "bdist_wheel" in sys.argv
|
||||
|
||||
if NEED_WRAP:
|
||||
if WRAP_DIR.exists():
|
||||
shutil.rmtree(WRAP_DIR)
|
||||
WRAP_DIR.mkdir(parents=True)
|
||||
|
||||
dest_dir = WRAP_DIR / MODULE_NAME
|
||||
shutil.copytree(PACKAGE_DIR, dest_dir)
|
||||
|
||||
with open(WRAP_DIR / "__init__.py", "w", encoding="utf-8") as f:
|
||||
f.write("from .rapidocr.main import RapidOCR, VisRes\n")
|
||||
|
||||
PACKAGE_DIR = WRAP_DIR
|
||||
PACKAGE_ROOT = PACKAGE_DIR / MODULE_NAME
|
||||
|
||||
PACKAGE_DATA = collect_package_files(PACKAGE_ROOT, "*.yaml")
|
||||
PACKAGE_DATA.extend(
|
||||
[
|
||||
"models/ch_PP-OCRv4_det_mobile.onnx",
|
||||
"models/ch_PP-OCRv4_rec_mobile.onnx",
|
||||
"models/ch_ppocr_mobile_v2.0_cls_mobile.onnx",
|
||||
]
|
||||
)
|
||||
|
||||
obtainer = GetPyPiLatestVersion()
|
||||
try:
|
||||
latest_version = obtainer(MODULE_NAME)
|
||||
except Exception as e:
|
||||
latest_version = "0.0.0"
|
||||
VERSION_NUM = obtainer.version_add_one(latest_version, add_patch=True)
|
||||
|
||||
if len(sys.argv) > 2:
|
||||
match_str = " ".join(sys.argv[2:])
|
||||
matched_versions = obtainer.extract_version(match_str)
|
||||
if matched_versions:
|
||||
VERSION_NUM = matched_versions
|
||||
sys.argv = sys.argv[:2]
|
||||
|
||||
project_urls = {
|
||||
"Documentation": "https://rapidai.github.io/RapidOCRDocs",
|
||||
"Changelog": "https://github.com/RapidAI/RapidOCR/releases",
|
||||
}
|
||||
|
||||
setuptools.setup(
|
||||
name=MODULE_NAME,
|
||||
version=VERSION_NUM,
|
||||
platforms="Any",
|
||||
description="Awesome OCR Library",
|
||||
long_description=get_readme(),
|
||||
long_description_content_type="text/markdown",
|
||||
author="SWHL",
|
||||
author_email="liekkaskono@163.com",
|
||||
url="https://github.com/RapidAI/RapidOCR",
|
||||
project_urls=project_urls,
|
||||
license="Apache-2.0",
|
||||
include_package_data=True,
|
||||
install_requires=read_txt("requirements.txt"),
|
||||
package_dir={"": str(PACKAGE_DIR)},
|
||||
packages=setuptools.find_namespace_packages(where=str(PACKAGE_DIR)),
|
||||
package_data={MODULE_NAME: PACKAGE_DATA},
|
||||
keywords=[
|
||||
"ocr,text_detection,text_recognition,db,onnxruntime,paddleocr,openvino,rapidocr"
|
||||
],
|
||||
classifiers=[
|
||||
"Programming Language :: Python :: 3.6",
|
||||
"Programming Language :: Python :: 3.7",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
],
|
||||
python_requires=">=3.6,<4",
|
||||
entry_points={
|
||||
"console_scripts": [f"{MODULE_NAME}={MODULE_NAME}.main:main"],
|
||||
},
|
||||
)
|
||||
|
||||
if NEED_WRAP and Path("wrap_temp").exists():
|
||||
shutil.rmtree("wrap_temp")
|
||||
@@ -0,0 +1,351 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
# @Author: SWHL
|
||||
# @Contact: liekkaskono@163.com
|
||||
import argparse
|
||||
import hashlib
|
||||
import sys
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
PYTHON_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_CONFIG_YAML = PYTHON_ROOT / "rapidocr" / "config.yaml"
|
||||
DEFAULT_MANIFEST_IN = PYTHON_ROOT / "MANIFEST.in"
|
||||
DEFAULT_MODEL_DIR = PYTHON_ROOT / "rapidocr" / "models"
|
||||
DEFAULT_MODELS_YAML = PYTHON_ROOT / "rapidocr" / "default_models.yaml"
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
REQUEST_TIMEOUT = 60
|
||||
TASK_SECTIONS = ("Det", "Cls", "Rec")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelSpec:
|
||||
engine: str
|
||||
ocr_version: str
|
||||
task: str
|
||||
lang: str
|
||||
model_type: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WheelAsset:
|
||||
relative_path: Path
|
||||
url: str
|
||||
sha256: Optional[str]
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Prepare model assets that should be bundled into the RapidOCR wheel."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config-yaml",
|
||||
type=Path,
|
||||
default=DEFAULT_CONFIG_YAML,
|
||||
help=f"RapidOCR default config used to select wheel models. Default: {DEFAULT_CONFIG_YAML}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--models-yaml",
|
||||
type=Path,
|
||||
default=DEFAULT_MODELS_YAML,
|
||||
help=f"Model registry YAML used as the source of URLs and checksums. Default: {DEFAULT_MODELS_YAML}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_MODEL_DIR,
|
||||
help=f"Directory used to store wheel model assets. Default: {DEFAULT_MODEL_DIR}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--manifest-in",
|
||||
type=Path,
|
||||
default=DEFAULT_MANIFEST_IN,
|
||||
help=f"MANIFEST.in file to write with exact wheel asset includes. Default: {DEFAULT_MANIFEST_IN}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-manifest-in",
|
||||
action="store_true",
|
||||
help="Do not write MANIFEST.in after preparing assets.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="Only verify that required assets exist and match their checksums.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Download assets even when an existing file passes checksum validation.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict:
|
||||
try:
|
||||
import yaml
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"PyYAML is required to read RapidOCR YAML config files. "
|
||||
"Install it with `python -m pip install PyYAML` or "
|
||||
"`python -m pip install -r requirements.txt`."
|
||||
) from e
|
||||
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise RuntimeError(f"Invalid YAML mapping: {path}")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def resolve_model_specs(config: dict) -> List[ModelSpec]:
|
||||
global_cfg = config.get("Global", {})
|
||||
specs = []
|
||||
|
||||
for section in TASK_SECTIONS:
|
||||
task_cfg = config.get(section)
|
||||
if not isinstance(task_cfg, dict):
|
||||
raise RuntimeError(f"Missing config section: {section}")
|
||||
|
||||
task = _required_str(task_cfg, "task_type", section)
|
||||
if not global_cfg.get(f"use_{task}", True):
|
||||
continue
|
||||
|
||||
specs.append(
|
||||
ModelSpec(
|
||||
engine=_required_str(task_cfg, "engine_type", section),
|
||||
ocr_version=_required_str(task_cfg, "ocr_version", section),
|
||||
task=task,
|
||||
lang=_required_str(task_cfg, "lang_type", section),
|
||||
model_type=_required_str(task_cfg, "model_type", section),
|
||||
)
|
||||
)
|
||||
|
||||
if not specs:
|
||||
raise RuntimeError("No enabled OCR tasks found in config.")
|
||||
|
||||
return specs
|
||||
|
||||
|
||||
def resolve_assets(specs: Iterable[ModelSpec], registry: dict) -> List[WheelAsset]:
|
||||
assets = []
|
||||
for spec in specs:
|
||||
model_info = select_model_info(spec, registry)
|
||||
assets.extend(model_info_to_assets(spec, model_info))
|
||||
|
||||
return _dedupe_assets(assets)
|
||||
|
||||
|
||||
def select_model_info(spec: ModelSpec, registry: dict) -> dict:
|
||||
try:
|
||||
task_models = registry[spec.engine][spec.ocr_version][spec.task]
|
||||
except KeyError as e:
|
||||
raise RuntimeError(
|
||||
"Missing model registry section: "
|
||||
f"{spec.engine}.{spec.ocr_version}.{spec.task}"
|
||||
) from e
|
||||
|
||||
for model_name, model_info in task_models.items():
|
||||
if model_name.startswith(spec.lang) and spec.model_type in model_name:
|
||||
if not isinstance(model_info, dict):
|
||||
raise RuntimeError(f"Invalid model registry entry: {model_name}")
|
||||
return model_info
|
||||
|
||||
raise RuntimeError(
|
||||
"No model registry entry matches default config: "
|
||||
f"{spec.engine}.{spec.ocr_version}.{spec.task} "
|
||||
f"lang={spec.lang} model_type={spec.model_type}"
|
||||
)
|
||||
|
||||
|
||||
def model_info_to_assets(spec: ModelSpec, model_info: dict) -> List[WheelAsset]:
|
||||
model_url = _required_str(model_info, "model_dir", "model registry")
|
||||
|
||||
if spec.engine == "paddle":
|
||||
assets = [
|
||||
WheelAsset(
|
||||
relative_path=Path(Path(model_url).name) / filename,
|
||||
url=f"{model_url.rstrip('/')}/{filename}",
|
||||
sha256=sha256,
|
||||
)
|
||||
for filename, sha256 in model_info.items()
|
||||
if filename not in ("model_dir", "dict_url")
|
||||
]
|
||||
else:
|
||||
assets = [
|
||||
WheelAsset(
|
||||
relative_path=Path(Path(model_url).name),
|
||||
url=model_url,
|
||||
sha256=model_info.get("SHA256"),
|
||||
)
|
||||
]
|
||||
|
||||
dict_url = model_info.get("dict_url")
|
||||
if spec.task == "rec" and dict_url:
|
||||
assets.append(
|
||||
WheelAsset(
|
||||
relative_path=Path(Path(dict_url).name),
|
||||
url=dict_url,
|
||||
sha256=None,
|
||||
)
|
||||
)
|
||||
|
||||
return assets
|
||||
|
||||
|
||||
def prepare_assets(
|
||||
assets: Iterable[WheelAsset], model_dir: Path, check_only: bool, force: bool
|
||||
) -> int:
|
||||
model_dir = model_dir.expanduser().resolve()
|
||||
missing_or_invalid = []
|
||||
|
||||
for asset in assets:
|
||||
save_path = model_dir / asset.relative_path
|
||||
if not force and validate_asset(save_path, asset.sha256):
|
||||
print(f"OK {save_path}")
|
||||
continue
|
||||
|
||||
if check_only:
|
||||
status = "missing" if not save_path.exists() else "invalid checksum"
|
||||
print(f"FAIL {save_path} ({status})")
|
||||
missing_or_invalid.append(save_path)
|
||||
continue
|
||||
|
||||
download_asset(asset, save_path)
|
||||
|
||||
if missing_or_invalid:
|
||||
print(
|
||||
"\nRequired wheel assets are missing or invalid. "
|
||||
"Run without --check to download them.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
print("\nWheel assets are ready.")
|
||||
return 0
|
||||
|
||||
|
||||
def write_manifest_in(manifest_path: Path, assets: Iterable[WheelAsset]) -> None:
|
||||
manifest_path = manifest_path.expanduser().resolve()
|
||||
lines = [
|
||||
"# Generated by tools/prepare_wheel_assets.py.",
|
||||
"# Re-run that script after changing rapidocr/config.yaml defaults.",
|
||||
]
|
||||
for asset in assets:
|
||||
package_path = Path("rapidocr") / "models" / asset.relative_path
|
||||
lines.append(f"include {package_path.as_posix()}")
|
||||
|
||||
manifest_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print(f"Wrote {manifest_path}")
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
hasher = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(CHUNK_SIZE), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def validate_asset(path: Path, expected_sha256: Optional[str]) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
if expected_sha256 is None:
|
||||
return True
|
||||
return file_sha256(path) == expected_sha256
|
||||
|
||||
|
||||
def download_asset(asset: WheelAsset, save_path: Path) -> None:
|
||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = save_path.with_suffix(f"{save_path.suffix}.part")
|
||||
|
||||
request = urllib.request.Request(
|
||||
asset.url,
|
||||
headers={"User-Agent": "RapidOCR wheel asset preparer"},
|
||||
)
|
||||
print(f"Downloading {asset.relative_path.as_posix()}")
|
||||
print(f" from {asset.url}")
|
||||
with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response:
|
||||
total_size = _get_content_length(response.headers.get("Content-Length"))
|
||||
downloaded = 0
|
||||
with tmp_path.open("wb") as f:
|
||||
while True:
|
||||
chunk = response.read(CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
_print_progress(downloaded, total_size)
|
||||
|
||||
if total_size is not None:
|
||||
print()
|
||||
|
||||
if asset.sha256 is not None:
|
||||
actual_sha256 = file_sha256(tmp_path)
|
||||
if actual_sha256 != asset.sha256:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
raise RuntimeError(
|
||||
f"Checksum mismatch for {asset.relative_path}: "
|
||||
f"expected {asset.sha256}, got {actual_sha256}"
|
||||
)
|
||||
|
||||
tmp_path.replace(save_path)
|
||||
print(f"Saved {save_path}")
|
||||
|
||||
|
||||
def _dedupe_assets(assets: Iterable[WheelAsset]) -> List[WheelAsset]:
|
||||
deduped = {}
|
||||
for asset in assets:
|
||||
existing = deduped.get(asset.relative_path)
|
||||
if existing and existing != asset:
|
||||
raise RuntimeError(
|
||||
f"Conflicting wheel asset definitions: {asset.relative_path}"
|
||||
)
|
||||
deduped[asset.relative_path] = asset
|
||||
return list(deduped.values())
|
||||
|
||||
|
||||
def _get_content_length(value: Optional[str]) -> Optional[int]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _print_progress(downloaded: int, total_size: Optional[int]) -> None:
|
||||
if not total_size:
|
||||
return
|
||||
percent = downloaded / total_size * 100
|
||||
print(
|
||||
f"\r {downloaded / 1024 / 1024:.1f}MB / "
|
||||
f"{total_size / 1024 / 1024:.1f}MB ({percent:.1f}%)",
|
||||
end="",
|
||||
)
|
||||
|
||||
|
||||
def _required_str(data: dict, key: str, section: str) -> str:
|
||||
value = data.get(key)
|
||||
if not isinstance(value, str) or not value:
|
||||
raise RuntimeError(f"Missing required string field {section}.{key}")
|
||||
return value
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
config = load_yaml(args.config_yaml)
|
||||
registry = load_yaml(args.models_yaml)
|
||||
specs = resolve_model_specs(config)
|
||||
assets = resolve_assets(specs, registry)
|
||||
|
||||
status = prepare_assets(assets, args.model_dir, args.check, args.force)
|
||||
if status == 0 and not args.check and not args.skip_manifest_in:
|
||||
write_manifest_in(args.manifest_in, assets)
|
||||
return status
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user