diff --git a/Dockerfile b/Dockerfile index c4820f08f2..0d15c07785 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,6 +85,14 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co ENV DEBIAN_FRONTEND=noninteractive +# The Noto Sans CJK SC font is now pre-baked into infiniflow/ragflow_deps:latest +# (downloaded + verified by deps/download_deps.py) instead of being curl'd from +# GitHub at main-image build time. gopdf reads it at runtime from +# /usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf (pdf_writer.go). +RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/NotoSansCJKsc-VF.ttf,target=/tmp/NotoSansCJKsc-VF.ttf \ + mkdir -p /usr/local/share/fonts/truetype/noto && \ + cp /tmp/NotoSansCJKsc-VF.ttf /usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf + # Setup apt RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ if [ "$NEED_MIRROR" == "1" ]; then \ @@ -99,10 +107,6 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ chmod 1777 /tmp && \ apt update && \ apt --no-install-recommends install -y ca-certificates curl vim unzip iproute2 fonts-dejavu fontconfig && \ - mkdir -p /usr/local/share/fonts/truetype/noto && \ - curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL \ - -o /usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf \ - https://github.com/notofonts/noto-cjk/raw/main/Sans/Variable/TTF/NotoSansCJKsc-VF.ttf && \ fc-cache -f && \ rm -rf /var/lib/apt/lists/* && apt-get clean diff --git a/deps/Dockerfile b/deps/Dockerfile index e2ed64e5b7..e45e64158e 100644 --- a/deps/Dockerfile +++ b/deps/Dockerfile @@ -2,6 +2,6 @@ # docker build -f Dockerfile -t infiniflow/ragflow_deps:latest . FROM scratch -COPY cl100k_base.tiktoken stagehand-server-v3-linux-x64 stagehand-server-v3-linux-arm64 / +COPY cl100k_base.tiktoken stagehand-server-v3-linux-x64 stagehand-server-v3-linux-arm64 NotoSansCJKsc-VF.ttf / COPY huggingface.co /huggingface.co diff --git a/deps/download_deps.py b/deps/download_deps.py index 69f3c2c97f..2cfbe621de 100755 --- a/deps/download_deps.py +++ b/deps/download_deps.py @@ -45,6 +45,7 @@ # tokenizer cannot be ingested (see internal/tokenizer/embedding_token_limits.md). import argparse +import hashlib import os import platform import re @@ -71,6 +72,16 @@ REPO_ROOT = os.path.dirname(SCRIPT_DIR) STAGEHAND_SERVER_VERSION = "v3.7.2" +# Noto Sans CJK SC variable TTF, rendered into PDFs by the Go backend +# (internal/agent/component/io/pdf_writer.go). When non-empty, it is the pinned +# SHA-256 every downloaded/kept font must match. When empty, the hash recorded in +# deps/NotoSansCJKsc-VF.ttf.sha256 is the baseline: a present non-empty sidecar is +# verified (drift rejected), while a missing/empty sidecar is bootstrapped by +# recording the downloaded hash so the next run detects changes. Both first and +# later runs therefore run the download directly without manual setup. +NOTO_CJK_FONT_SHA256 = "" + + def gh_proxy() -> str: """GitHub release proxy prefix (gh-proxy.com behind the GFW), else empty.""" return os.environ.get("GH_PROXY") or "" @@ -101,6 +112,13 @@ def stagehand_url(arch: str) -> str: return f"{gh_proxy()}https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/{STAGEHAND_SERVER_VERSION}/stagehand-server-v3-linux-{arch}" +def noto_cjk_font_url() -> str: + """Noto Sans CJK SC variable TTF, rendered into PDFs by the Go backend + (internal/agent/component/io/pdf_writer.go). It is a GitHub raw URL, so the + gh-proxy prefix applies behind the GFW.""" + return f"{gh_proxy()}https://github.com/notofonts/noto-cjk/raw/main/Sans/Variable/TTF/NotoSansCJKsc-VF.ttf" + + # Mirrors internal/common.DeepDocModelFiles (Go in-process DeepDoc backend). # These are the ONLY weights the Go backend loads; the full InfiniFlow/deepdoc # repo also ships .onnx (Python-only), which this Go-only script deliberately @@ -167,6 +185,59 @@ def _valid_stagehand_binary(path): return file.read(4) == b"\x7fELF" +def _valid_font(path): + """True when PATH is a TrueType font with the sfnt signature b"\\x00\\x01\\x00\\x00". + + gopdf's AddTTFFont accepts ONLY this signature; OpenType (OTTO), TrueType + collections (ttcf), and WOFF/WOFF2 are rejected, so we must not let them + through. A raw GitHub non-font response (e.g. a 404 HTML page) also fails this + check, so we never copy a non-font into the dependency image.""" + if not os.path.isfile(path) or os.path.getsize(path) == 0: + return False + with open(path, "rb") as file: + return file.read(4) == b"\x00\x01\x00\x00" + + +def _sha256_of(path): + """SHA-256 of PATH, streamed in chunks so large files don't blow memory.""" + h = hashlib.sha256() + with open(path, "rb") as file: + for chunk in iter(lambda: file.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def _verify_font_checksum(font): + """Validate FONT's SHA-256, removing it and failing on any mismatch. + + Precedence: a non-empty NOTO_CJK_FONT_SHA256 pin wins and is verified strictly. + Otherwise the hash recorded in deps/NotoSansCJKsc-VF.ttf.sha256 is the baseline: + a present non-empty sidecar is verified (drift is rejected), while a missing or + empty sidecar is bootstrapped by recording the downloaded hash so subsequent runs + detect changes. The caller runs the magic-number (_valid_font) check beforehand.""" + actual = _sha256_of(font) + sidecar = os.path.join(SCRIPT_DIR, "NotoSansCJKsc-VF.ttf.sha256") + if NOTO_CJK_FONT_SHA256: + if actual != NOTO_CJK_FONT_SHA256: + os.unlink(font) + raise RuntimeError(f"NotoSansCJKsc-VF.ttf checksum mismatch (got {actual}, pinned {NOTO_CJK_FONT_SHA256})") + return + # No pinned constant: verify against the sidecar baseline, or bootstrap it. + if os.path.isfile(sidecar): + with open(sidecar) as file: + content = file.read().strip() + expected = content.split()[0] if content else "" + if expected: + if actual != expected: + os.unlink(font) + raise RuntimeError(f"NotoSansCJKsc-VF.ttf checksum changed vs recorded {expected} (got {actual})") + return + # Empty sidecar: fall through to (re)record the baseline below. + with open(sidecar, "w") as file: + file.write(actual + "\n") + print(f" >>> PIN THIS HASH in download_deps.py NOTO_CJK_FONT_SHA256: {actual}") + + def load_tokenizer_assets(assets=None): """Validate and return the inlined tokenizer asset list as (repo, file, kind) tuples. When ASSETS is None, uses the module-level TOKENIZER_ASSETS. @@ -361,6 +432,29 @@ def main(): raise RuntimeError(f"Invalid downloaded dependency: {filename}") _install_stagehand() + # ── Noto Sans CJK SC font (rendered into PDFs by the Go backend) ───────── + # Verified by a font magic-number check (only the sfnt signature + # b"\x00\x01\x00\x00" that gopdf's AddTTFFont accepts) plus a SHA-256 check + # (_verify_font_checksum). A present file is NEVER reused unverified; a bad + # download is removed and the build fails loudly instead of shipping a PDF + # backend that silently loses CJK rendering. + print("== NotoSansCJKsc-VF.ttf ==") + font = os.path.join(SCRIPT_DIR, "NotoSansCJKsc-VF.ttf") + if os.path.isfile(font) and os.path.getsize(font) > 0: + if not _valid_font(font): + print(" WARNING: existing NotoSansCJKsc-VF.ttf is not a valid font; re-downloading", file=sys.stderr) + os.unlink(font) + else: + _verify_font_checksum(font) + if not (os.path.isfile(font) and os.path.getsize(font) > 0): + print(f"Downloading NotoSansCJKsc-VF.ttf from {noto_cjk_font_url()}...") + download_with_progress(noto_cjk_font_url(), font) + if not _valid_font(font): + os.unlink(font) + raise RuntimeError("Downloaded NotoSansCJKsc-VF.ttf is not a valid font; refusing to keep it") + _verify_font_checksum(font) + print(" ✓ NotoSansCJKsc-VF.ttf ready") + # Download the Go DeepDoc `.ort` weights so this script is a one-stop for a # Go dev's runtime resources: models (below) from a single invocation. print("== DeepDoc models ==")