diff --git a/.dockerignore b/.dockerignore index 941513916c..55700b6609 100644 --- a/.dockerignore +++ b/.dockerignore @@ -38,10 +38,10 @@ target/ # regardless of which resources are prepared for the dependency image. # The deps image is built from this directory with: # cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps . -# The Go downloader is re-included because build.sh reads its ORT version pin. -# Re-including requires NOT excluding the directory itself. +# The native downloader tools/scripts/download_dev_deps.py is copied by the +# main Dockerfile (not excluded here) so build.sh can read its ORT version pin +# when running `./build.sh --go`. ragflow_deps/* -!ragflow_deps/download_deps.py # ── IDE and editor config ────────────────────────────────────────────────── .idea/ diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index 29961eb3c8..d6994a9992 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -900,18 +900,18 @@ jobs: fi - name: Run ragflow_deps regression guards - # Pure-Python, local, needs no API keys or model downloads; guards (a) the - # ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build - # depends on (notably the version-bump path C2 hardened) and (b) the - # tokenizer-asset chain: the downloader's asset list, the three runtime + # Local, needs no API keys or model downloads; guards (a) the native + # archive / ONNX Runtime extraction logic the in-process (Go) DeepDoc + # backend build depends on (notably the version-bump path), and (b) the + # tokenizer-asset chain: the downloader's manifest, the three runtime # Dockerfiles' copy loops and the loaders' SHA-1 pins have to agree, or the - # Go counters silently degrade to the calibrated cl100k estimate. Run them - # in isolation so the test/testcases conftest (which requires + # Go counters silently degrade to the calibrated cl100k estimate. Run the + # pytest in isolation so the test/testcases conftest (which requires # SILICONFLOW_API_KEY) cannot fail the collection. run: | set -euo pipefail uv sync --python 3.13 --group test --frozen - uv run pytest ragflow_deps/test_download_deps.py ragflow_deps/test_tokenizer_assets.py -q --durations=100 + uv run pytest tools/scripts/test_download_dev_deps.py -q --durations=100 - name: Build ragflow:nightly run: | diff --git a/AGENTS.md b/AGENTS.md index 24b67c9e84..1d0f682f04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Use this file as the local operating guide for the current codebase. Prefer the - `web/`: frontend application. - `docker/`: local and production compose files. - `sdk/python/`: legacy Python client SDK pending deletion. -- `ragflow_deps/download_deps.py`: Python helper for Go native libraries, DeepDoc models, and tokenizer assets. Replace it with non-Python preparation before removing Python dependencies. +- `tools/scripts/download_dev_deps.py` and `ragflow_deps/download_deps.py`: Python downloaders for the Go build's dependencies. `download_dev_deps.py` pulls the native static libraries (pdfium, pdf_oxide, office_oxide, onnxruntime) that `build.sh` links; `download_deps.py` pulls the runtime resources (cl100k BPE table, DeepDoc models, embedding tokenizer assets) and installs the stagehand server into the SDK cache. Versions and URLs are defined inline in each downloader (`tools/scripts/download_dev_deps.py` for native static libraries, `ragflow_deps/download_deps.py` for runtime resources); the tokenizer assets are inlined as `TOKENIZER_ASSETS` in each downloader. Replace these with non-Python preparation before removing Python dependencies. - `test/`: contains Python tests, including tests that import the legacy Python implementation directly. Migrate needed coverage to Go tests or frontend tests, then remove the Python tests; keep only tests for supported code. ## Go-Specific Rules @@ -129,7 +129,7 @@ bash build.sh --test-manual # local opt-in only bash build.sh --test-all # integration + e2e; excludes manual ``` -Native libraries, Go DeepDoc models, tokenizer assets, and `ragflow_deps/cl100k_base.tiktoken` come from `ragflow_deps/download_deps.py`; see `README_zh.md` for its isolated setup. The downloader installs stagehand into the SDK cache using the version in `go.mod`. Build the Go resource image with `cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .` after downloading the resources. +Native libraries, Go DeepDoc models, tokenizer assets, and `ragflow_deps/cl100k_base.tiktoken` come from `tools/scripts/download_dev_deps.py` (native static libs) and `ragflow_deps/download_deps.py` (models, cl100k, tokenizer assets); run them with `uv run`. Build the Go resource image with `cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .` after downloading the resources. ### Frontend ```bash diff --git a/Dockerfile b/Dockerfile index 88bb8400bb..485ae5332d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,10 +194,6 @@ RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod,sharing=locked COPY internal internal COPY cmd cmd COPY build.sh ./ -# build.sh's check_ort_version_consistency (run via `./build.sh --go`) greps the -# ORT version pin from this file; without it the --go build fails with -# "could not parse the ONNX Runtime version from one of the pinned locations". -COPY ragflow_deps/download_deps.py ./ragflow_deps/ COPY Dockerfile ./ # ONNX Runtime static archives: build.sh's _seed_from_system looks for the ORT diff --git a/README.md b/README.md index 707cd19ba6..2994aa59cf 100644 --- a/README.md +++ b/README.md @@ -256,15 +256,15 @@ For document-engine changes, configuration updates, restarting services, and ret cd ragflow ``` -2. Run the [dependency download script](./ragflow_deps/download_deps.py) to download the static libraries and model files: +2. Run the dependency download script once to prepare everything the local Go server needs to build, run, parse documents and count tokens (`download_dev_deps.py` fetches the native static libraries and the runtime resources — cl100k vocabulary, DeepDoc `.ort` weights, embedding tokenizer assets): ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py ``` - The script prepares native libraries and model resources required for the Go build and needs `requests` and `huggingface-hub`. Skip this step if you have prepared the same resources by other means. When started from the repository root, Go services automatically find `internal/rag/res/deepdoc`; to start from another directory, set `DEEPDOC_MODEL_DIR` to its absolute path. + This is the local-dev one-stop: after one run the environment is ready, and already-present files are skipped so re-running never re-downloads. The image-facing counterpart `ragflow_deps/download_deps.py` fetches the same runtime resources PLUS the stagehand server binary (used by the Browser component) and skips the native `.a` files — run it only when building the dependency image or in CI, not for local development. + + `download_dev_deps.py` pulls and unpacks pdfium, pdf_oxide, office_oxide, and onnxruntime under `~/ragflow-native-libs`, and writes the cl100k vocabulary, DeepDoc models, and embedding tokenizer assets under `ragflow_deps/`. It is a Python script; `uv run` installs its declared dependencies (`requests` and `huggingface-hub`) automatically. Skip this step if you have prepared the same resources by other means. When started from the repository root, Go services automatically find `internal/rag/res/deepdoc`; to start from another directory, set `DEEPDOC_MODEL_DIR` to its absolute path. 3. Build the services: diff --git a/README_ar.md b/README_ar.md index 585236a006..c14efce706 100644 --- a/README_ar.md +++ b/README_ar.md @@ -212,14 +212,15 @@ cd ragflow ``` -2. شغّل [برنامج تنزيل التبعيات](./ragflow_deps/download_deps.py) لإعداد المكتبات الأصلية وملفات النماذج: +2. شغّل برامج تنزيل التبعيات لتحضير المكتبات الأصلية الثابتة التي يربطها بناء Go (`download_dev_deps.py`) وموارد النماذج/المحلّلات التي يحمّلها الخلفية أثناء التشغيل (`download_deps.py`): ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + يقوم `download_dev_deps.py` بتنزيل وفك pdfium وpdf_oxide وoffice_oxide وonnxruntime تحت `~/ragflow-native-libs`؛ ويقوم `download_deps.py` بتنزيل جدول cl100k ونماذج DeepDoc وأصول محلّلات التضمين تحت `ragflow_deps/`. كلاهما برنامج Python؛ ويثبّت `uv run` تبعياته المعلنة تلقائيًا (`requests`، بالإضافة إلى `huggingface-hub` لتنزيل النماذج والمحلّلات). تجاوز هذه الخطوة إذا كنت قد جهّزت الموارد نفسها بوسيلة أخرى. عند التشغيل من جذر المستودع تجد خدمات Go مجلد `internal/rag/res/deepdoc` تلقائيًا؛ وللتشغيل من مجلد آخر عيّن `DEEPDOC_MODEL_DIR` على مساره المطلق. + 3. ابنِ الخدمات: ```bash @@ -273,7 +274,7 @@ راجع [تشغيل الخدمة من المصدر](./docs/develop/launch_ragflow_from_source.md) لمزيد من التفاصيل. -> **ملاحظات التشغيل:** لجعل إعداد Elasticsearch دائمًا، أضف `vm.max_map_count=262144` إلى `/etc/sysctl.conf`. في إعداد MySQL الافتراضي، ينفذ مدخل صورة Docker ترحيل قاعدة البيانات قبل تشغيل Admin وSyncer وIngestor وAPI. يستخدم DeepDoc في الإصدار المفتوح 1.0 استدلال CPU لتحليل التخطيط وOCR والجداول. بعد تغيير إعداد Docker، أعد تشغيل الخدمات واتبع [دليل Docker](./docker/README.md) للاحتفاظ بالبيانات أو حذفها. يجهز `download_deps.py` المكتبات الأصلية وموارد النماذج، ويتطلب `requests` و`huggingface-hub`. ينفذ `--migrate` الترحيل ثم يخرج؛ ويشغّل `--admin` خدمة الإدارة، ويشغّل `--ingestor` خدمة الاستيعاب، ويشغّل `--syncer` خدمة مزامنة مصادر البيانات الخارجية، ويشغّل `--api` خدمة API لواجهة الويب وحزم SDK. تعني استجابة HTTP 200 أن API جاهز. +> **ملاحظات التشغيل:** لجعل إعداد Elasticsearch دائمًا، أضف `vm.max_map_count=262144` إلى `/etc/sysctl.conf`. في إعداد MySQL الافتراضي، ينفذ مدخل صورة Docker ترحيل قاعدة البيانات قبل تشغيل Admin وSyncer وIngestor وAPI. يستخدم DeepDoc في الإصدار المفتوح 1.0 استدلال CPU لتحليل التخطيط وOCR والجداول. بعد تغيير إعداد Docker، أعد تشغيل الخدمات واتبع [دليل Docker](./docker/README.md) للاحتفاظ بالبيانات أو حذفها. يجهز `download_dev_deps.py` المكتبات الأصلية لبناء Go، ويجهز `download_deps.py` النماذج وموارد المحلّلات وجدول cl100k؛ ويعمل كلاهما عبر `uv run`. ينفذ `--migrate` الترحيل ثم يخرج؛ ويشغّل `--admin` خدمة الإدارة، ويشغّل `--ingestor` خدمة الاستيعاب، ويشغّل `--syncer` خدمة مزامنة مصادر البيانات الخارجية، ويشغّل `--api` خدمة API لواجهة الويب وحزم SDK. تعني استجابة HTTP 200 أن API جاهز. ## 📚 التوثيق diff --git a/README_fr.md b/README_fr.md index 9a1f5f3d66..29a5aea873 100644 --- a/README_fr.md +++ b/README_fr.md @@ -209,14 +209,15 @@ Docker utilise `docker/.env` et `docker/docker-compose.yml`. Suivez le [guide de cd ragflow ``` -2. Exécutez le [script de téléchargement des dépendances](./ragflow_deps/download_deps.py) pour préparer les bibliothèques natives et les fichiers de modèles : +2. Exécutez les scripts de téléchargement des dépendances pour préparer les bibliothèques natives statiques liées par la compilation Go (`download_dev_deps.py`) et les ressources de modèles / tokeniseurs chargées à l'exécution par le backend (`download_deps.py`) : ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + `download_dev_deps.py` télécharge et décompresse pdfium, pdf_oxide, office_oxide et onnxruntime sous `~/ragflow-native-libs` ; `download_deps.py` télécharge le vocabulaire cl100k, les modèles DeepDoc et les ressources de tokeniseurs d'embedding sous `ragflow_deps/`. Les deux sont des scripts Python ; `uv run` installe automatiquement leurs dépendances déclarées (`requests`, plus `huggingface-hub` pour les téléchargements de modèles et de tokeniseurs). Ignorez cette étape si vous avez déjà préparé les mêmes ressources autrement. Lors du démarrage depuis la racine du dépôt, les services Go trouvent automatiquement `internal/rag/res/deepdoc` ; pour démarrer depuis un autre répertoire, définissez `DEEPDOC_MODEL_DIR` sur son chemin absolu. + 3. Compilez les services : ```bash @@ -270,7 +271,7 @@ Docker utilise `docker/.env` et `docker/docker-compose.yml`. Suivez le [guide de Consultez [Lancer le service depuis les sources](./docs/develop/launch_ragflow_from_source.md) pour plus de détails. -> **Notes d’exploitation :** pour rendre le réglage Elasticsearch permanent, ajoutez `vm.max_map_count=262144` à `/etc/sysctl.conf`. Avec MySQL par défaut, l’entrée de l’image Docker exécute la migration avant de démarrer Admin, Syncer, Ingestor et API. Dans la version open source 1.0, DeepDoc utilise le CPU pour l’analyse de mise en page, l’OCR et les tableaux. Après une modification Docker, redémarrez les services et suivez le [guide Docker](./docker/README.md) pour conserver ou supprimer les données. `download_deps.py` prépare les bibliothèques natives et les modèles et nécessite `requests` et `huggingface-hub`. `--migrate` migre puis quitte ; `--admin` démarre le service d’administration, `--ingestor` le service d’ingestion, `--syncer` le service de synchronisation des sources de données externes et `--api` le service API pour l’interface Web et les SDK. Une réponse HTTP 200 indique que l’API est prête. +> **Notes d’exploitation :** pour rendre le réglage Elasticsearch permanent, ajoutez `vm.max_map_count=262144` à `/etc/sysctl.conf`. Avec MySQL par défaut, l’entrée de l’image Docker exécute la migration avant de démarrer Admin, Syncer, Ingestor et API. Dans la version open source 1.0, DeepDoc utilise le CPU pour l’analyse de mise en page, l’OCR et les tableaux. Après une modification Docker, redémarrez les services et suivez le [guide Docker](./docker/README.md) pour conserver ou supprimer les données. `download_dev_deps.py` prépare les bibliothèques natives et `download_deps.py` les modèles et ressources de tokeniseurs ; les deux s'exécutent via `uv run`. `--migrate` migre puis quitte ; `--admin` démarre le service d’administration, `--ingestor` le service d’ingestion, `--syncer` le service de synchronisation des sources de données externes et `--api` le service API pour l’interface Web et les SDK. Une réponse HTTP 200 indique que l’API est prête. ## 📚 Documentation diff --git a/README_id.md b/README_id.md index 29d6986574..429c58f976 100644 --- a/README_id.md +++ b/README_id.md @@ -209,14 +209,15 @@ Docker menggunakan `docker/.env` dan `docker/docker-compose.yml`. Ikuti [panduan cd ragflow ``` -2. Jalankan [skrip pengunduh dependensi](./ragflow_deps/download_deps.py) untuk menyiapkan pustaka native dan file model: +2. Jalankan skrip pengunduh dependensi untuk menyiapkan pustaka native statis yang ditautkan oleh build Go (`download_dev_deps.py`) dan sumber daya model / tokenizer yang dimuat backend saat runtime (`download_deps.py`): ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + `download_dev_deps.py` mengunduh dan mengekstrak pdfium, pdf_oxide, office_oxide, dan onnxruntime ke `~/ragflow-native-libs`; `download_deps.py` mengunduh kosakata cl100k, model DeepDoc, dan aset tokenizer embedding ke `ragflow_deps/`. Keduanya adalah skrip Python; `uv run` memasang dependensi yang dideklarasikan secara otomatis (`requests`, plus `huggingface-hub` untuk unduhan model dan tokenizer). Lewati langkah ini jika Anda sudah menyiapkan sumber daya yang sama dengan cara lain. Saat dijalankan dari akar repositori, layanan Go secara otomatis menemukan `internal/rag/res/deepdoc`; untuk menjalankan dari direktori lain, tetapkan `DEEPDOC_MODEL_DIR` ke jalur absolutnya. + 3. Build layanan: ```bash @@ -270,7 +271,7 @@ Docker menggunakan `docker/.env` dan `docker/docker-compose.yml`. Ikuti [panduan Lihat [Menjalankan Layanan dari Sumber](./docs/develop/launch_ragflow_from_source.md) untuk detailnya. -> **Catatan operasional:** agar pengaturan Elasticsearch permanen, tambahkan `vm.max_map_count=262144` ke `/etc/sysctl.conf`. Dengan MySQL default, entrypoint image Docker menjalankan migrasi sebelum memulai Admin, Syncer, Ingestor, dan API. DeepDoc open-source 1.0 memakai inferensi CPU untuk layout, OCR, dan tabel. Setelah konfigurasi Docker berubah, mulai ulang layanan dan ikuti [panduan Docker](./docker/README.md) untuk mempertahankan atau menghapus data. `download_deps.py` menyiapkan pustaka native dan model serta memerlukan `requests` dan `huggingface-hub`. `--migrate` menjalankan migrasi lalu keluar; `--admin` memulai layanan administrasi, `--ingestor` layanan ingestion, `--syncer` layanan sinkronisasi sumber data eksternal, dan `--api` layanan API untuk Web UI dan SDK. HTTP 200 berarti API siap. +> **Catatan operasional:** agar pengaturan Elasticsearch permanen, tambahkan `vm.max_map_count=262144` ke `/etc/sysctl.conf`. Dengan MySQL default, entrypoint image Docker menjalankan migrasi sebelum memulai Admin, Syncer, Ingestor, dan API. DeepDoc open-source 1.0 memakai inferensi CPU untuk layout, OCR, dan tabel. Setelah konfigurasi Docker berubah, mulai ulang layanan dan ikuti [panduan Docker](./docker/README.md) untuk mempertahankan atau menghapus data. `download_dev_deps.py` menyiapkan pustaka native untuk build Go, dan `download_deps.py` menyiapkan model, aset tokenizer, serta tabel cl100k; keduanya dijalankan melalui `uv run`. `--migrate` menjalankan migrasi lalu keluar; `--admin` memulai layanan administrasi, `--ingestor` layanan ingestion, `--syncer` layanan sinkronisasi sumber data eksternal, dan `--api` layanan API untuk Web UI dan SDK. HTTP 200 berarti API siap. ## 📚 Dokumentasi diff --git a/README_ja.md b/README_ja.md index 4edfaac808..31b48e13cc 100644 --- a/README_ja.md +++ b/README_ja.md @@ -219,14 +219,15 @@ Docker は迅速な評価、統合テスト、本番環境に、ソース起動 cd ragflow ``` -2. 依存関係を準備します: +2. Go ビルドがリンクするネイティブ静的ライブラリ（`download_dev_deps.py`）と、バックエンドが実行時に読み込むモデル / トークナイザーリソース（`download_deps.py`）を、依存関係ダウンロードスクリプトで用意します: ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + `download_dev_deps.py` は pdfium、pdf_oxide、office_oxide、onnxruntime を `~/ragflow-native-libs` にダウンロードして展開します。`download_deps.py` は cl100k 語彙、DeepDoc モデル、embedding トークナイザーリソースを `ragflow_deps/` にダウンロードします。どちらも Python スクリプトで、`uv run` が宣言された依存関係（`requests`、モデルとトークナイザーのダウンロードには `huggingface-hub`）を自動的にインストールします。同等のリソースを他の方法で用意済みの場合はこの手順をスキップしてください。リポジトリルートから起動する場合、Go サービスは `internal/rag/res/deepdoc` を自動的に見つけます。他のディレクトリから起動する場合は、`DEEPDOC_MODEL_DIR` にその絶対パスを設定してください。 + 3. サービスをビルドします: ```bash @@ -280,7 +281,7 @@ Docker は迅速な評価、統合テスト、本番環境に、ソース起動 詳細は[ソースコードからサービスを起動](./docs/develop/launch_ragflow_from_source.md)を参照してください。 -> **運用上の注意:** Elasticsearch の設定を永続化するには、`/etc/sysctl.conf` に `vm.max_map_count=262144` を追加します。既定の MySQL 構成では、Docker エントリポイントが移行後に Admin、Syncer、Ingestor、API を起動します。オープンソース 1.0 の DeepDoc はレイアウト解析、OCR、表認識に CPU 推論を使用します。Docker 設定変更後の再起動とデータ保持・削除は [Docker ガイド](./docker/README.md)に従ってください。`download_deps.py` はネイティブライブラリとモデルを準備し、`requests` と `huggingface-hub` を使用します。`--migrate` は移行後に終了し、`--admin` は管理サービス、`--ingestor` は取り込みサービス、`--syncer` は外部データソース同期サービス、`--api` は Web UI と SDK 向けの API サービスを起動します。HTTP 200 は API の準備完了を示します。 +> **運用上の注意:** Elasticsearch の設定を永続化するには、`/etc/sysctl.conf` に `vm.max_map_count=262144` を追加します。既定の MySQL 構成では、Docker エントリポイントが移行後に Admin、Syncer、Ingestor、API を起動します。オープンソース 1.0 の DeepDoc はレイアウト解析、OCR、表認識に CPU 推論を使用します。Docker 設定変更後の再起動とデータ保持・削除は [Docker ガイド](./docker/README.md)に従ってください。`download_dev_deps.py` はネイティブライブラリを、`download_deps.py` はモデルとトークナイザーリソースを準備します。どちらも `uv run` で実行します。`--migrate` は移行後に終了し、`--admin` は管理サービス、`--ingestor` は取り込みサービス、`--syncer` は外部データソース同期サービス、`--api` は Web UI と SDK 向けの API サービスを起動します。HTTP 200 は API の準備完了を示します。 ## 📚 ドキュメンテーション diff --git a/README_ko.md b/README_ko.md index ed2a7404af..d14acc4d64 100644 --- a/README_ko.md +++ b/README_ko.md @@ -219,14 +219,15 @@ Docker는 빠른 평가, 통합 테스트 및 프로덕션에 적합하고 소 cd ragflow ``` -2. 의존성을 준비합니다: +2. Go 빌드가 링크하는 네이티브 정적 라이브러리(`download_dev_deps.py`)와 백엔드가 런타임에 로드하는 모델/토크나이저 리소스(`download_deps.py`)를 의존성 다운로드 스크립트로 준비합니다: ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + `download_dev_deps.py`는 pdfium, pdf_oxide, office_oxide, onnxruntime을 `~/ragflow-native-libs`에 내려받아 압축 해제합니다. `download_deps.py`는 cl100k 어휘집, DeepDoc 모델, 임베딩 토크나이저 리소스를 `ragflow_deps/`에 내려받습니다. 두 스크립트 모두 Python 스크립트이며, `uv run`이 선언된 종속성(`requests`, 모델 및 토크나이저 다운로드에는 `huggingface-hub` 추가)을 자동으로 설치합니다. 같은 리소스를 다른 방법으로 준비했다면 이 단계를 건너뛰세요. 저장소 루트에서 시작하면 Go 서비스가 `internal/rag/res/deepdoc`을 자동으로 찾습니다. 다른 디렉터리에서 시작하려면 `DEEPDOC_MODEL_DIR`을 해당 디렉터리의 절대 경로로 설정하세요. + 3. 서비스를 빌드합니다: ```bash @@ -281,7 +282,7 @@ Docker는 빠른 평가, 통합 테스트 및 프로덕션에 적합하고 소 자세한 내용은 [소스에서 서비스 시작](./docs/develop/launch_ragflow_from_source.md)을 참조하세요. -> **운영 참고:** Elasticsearch 설정을 영구 적용하려면 `/etc/sysctl.conf`에 `vm.max_map_count=262144`를 추가하세요. 기본 MySQL 구성에서는 Docker 엔트리포인트가 마이그레이션 후 Admin, Syncer, Ingestor, API를 시작합니다. 오픈 소스 1.0의 DeepDoc은 레이아웃, OCR, 표 인식에 CPU 추론을 사용합니다. Docker 설정 변경 후 재시작과 데이터 유지·삭제는 [Docker 가이드](./docker/README.md)를 따르세요. `download_deps.py`는 네이티브 라이브러리와 모델을 준비하며 `requests`와 `huggingface-hub`를 사용합니다. `--migrate`는 마이그레이션 후 종료하고, `--admin`은 관리 서비스, `--ingestor`는 수집 서비스, `--syncer`는 외부 데이터 소스 동기화 서비스, `--api`는 Web UI와 SDK용 API 서비스를 시작합니다. HTTP 200은 API 준비 완료를 뜻합니다. +> **운영 참고:** Elasticsearch 설정을 영구 적용하려면 `/etc/sysctl.conf`에 `vm.max_map_count=262144`를 추가하세요. 기본 MySQL 구성에서는 Docker 엔트리포인트가 마이그레이션 후 Admin, Syncer, Ingestor, API를 시작합니다. 오픈 소스 1.0의 DeepDoc은 레이아웃, OCR, 표 인식에 CPU 추론을 사용합니다. Docker 설정 변경 후 재시작과 데이터 유지·삭제는 [Docker 가이드](./docker/README.md)를 따르세요. `download_dev_deps.py`는 네이티브 라이브러리를, `download_deps.py`는 모델과 토크나이저 리소스를 준비합니다. 둘 다 `uv run`으로 실행합니다. `--migrate`는 마이그레이션 후 종료하고, `--admin`은 관리 서비스, `--ingestor`는 수집 서비스, `--syncer`는 외부 데이터 소스 동기화 서비스, `--api`는 Web UI와 SDK용 API 서비스를 시작합니다. HTTP 200은 API 준비 완료를 뜻합니다. ## 📚 문서 diff --git a/README_pt_br.md b/README_pt_br.md index 303fd2843a..63697dd5c5 100644 --- a/README_pt_br.md +++ b/README_pt_br.md @@ -210,14 +210,15 @@ Use `docker/.env` e `docker/docker-compose.yml`. Consulte a [configuração do D cd ragflow ``` -2. Prepare as dependências: +2. Execute os scripts de download de dependências para preparar as bibliotecas nativas estáticas que a compilação do Go utiliza (`download_dev_deps.py`) e os recursos de modelo / tokenizador carregados em tempo de execução pelo backend (`download_deps.py`): ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + `download_dev_deps.py` baixa e descompacta pdfium, pdf_oxide, office_oxide e onnxruntime em `~/ragflow-native-libs`; `download_deps.py` baixa o vocabulário cl100k, os modelos DeepDoc e os recursos de tokenizadores de embedding em `ragflow_deps/`. Ambos são scripts Python; o `uv run` instala automaticamente as dependências declaradas (`requests`, além de `huggingface-hub` para os downloads de modelos e tokenizadores). Pule esta etapa se você já preparou os mesmos recursos por outros meios. Quando iniciado a partir da raiz do repositório, os serviços Go encontram `internal/rag/res/deepdoc` automaticamente; para iniciar de outro diretório, defina `DEEPDOC_MODEL_DIR` com seu caminho absoluto. + 3. Compile os serviços: ```bash @@ -271,7 +272,7 @@ Use `docker/.env` e `docker/docker-compose.yml`. Consulte a [configuração do D Consulte [Iniciar pelo código-fonte](./docs/develop/launch_ragflow_from_source.md) para detalhes. -> **Notas operacionais:** para tornar o ajuste do Elasticsearch permanente, adicione `vm.max_map_count=262144` a `/etc/sysctl.conf`. Com o MySQL padrão, o entrypoint Docker executa a migração antes de iniciar Admin, Syncer, Ingestor e API. No open source 1.0, o DeepDoc usa CPU para layout, OCR e tabelas. Após alterar o Docker, reinicie os serviços e siga o [guia Docker](./docker/README.md) para manter ou remover dados. `download_deps.py` prepara bibliotecas nativas e modelos e requer `requests` e `huggingface-hub`. `--migrate` migra e encerra; `--admin` inicia o serviço de administração, `--ingestor` o serviço de ingestão, `--syncer` o serviço de sincronização de fontes de dados externas e `--api` o serviço de API para a interface Web e os SDKs. HTTP 200 indica que a API está pronta. +> **Notas operacionais:** para tornar o ajuste do Elasticsearch permanente, adicione `vm.max_map_count=262144` a `/etc/sysctl.conf`. Com o MySQL padrão, o entrypoint Docker executa a migração antes de iniciar Admin, Syncer, Ingestor e API. No open source 1.0, o DeepDoc usa CPU para layout, OCR e tabelas. Após alterar o Docker, reinicie os serviços e siga o [guia Docker](./docker/README.md) para manter ou remover dados. `download_dev_deps.py` prepara as bibliotecas nativas e `download_deps.py` os modelos e recursos de tokenizadores; ambos são executados via `uv run`. `--migrate` migra e encerra; `--admin` inicia o serviço de administração, `--ingestor` o serviço de ingestão, `--syncer` o serviço de sincronização de fontes de dados externas e `--api` o serviço de API para a interface Web e os SDKs. HTTP 200 indica que a API está pronta. ## 📚 Documentação diff --git a/README_ru.md b/README_ru.md index 84b8e3632c..33b1530f4f 100644 --- a/README_ru.md +++ b/README_ru.md @@ -216,14 +216,15 @@ Docker подходит для быстрой оценки, интеграцио cd ragflow ``` -2. Подготовьте зависимости: +2. Выполните скрипты загрузки зависимостей, чтобы подготовить нативные статические библиотеки, которые линкует сборка Go (`download_dev_deps.py`), и ресурсы моделей / токенизаторов, которые бэкенд загружает во время выполнения (`download_deps.py`): ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + `download_dev_deps.py` загружает и распаковывает pdfium, pdf_oxide, office_oxide и onnxruntime в `~/ragflow-native-libs`; `download_deps.py` загружает словарь cl100k, модели DeepDoc и ресурсы токенизаторов эмбеддингов в `ragflow_deps/`. Оба скрипта — Python; `uv run` автоматически установит объявленные зависимости (`requests`, а для загрузки моделей и токенизаторов ещё и `huggingface-hub`). Пропустите этот шаг, если те же ресурсы уже подготовлены иным способом. При запуске из корня репозитория сервисы Go автоматически находят `internal/rag/res/deepdoc`; при запуске из другого каталога задайте `DEEPDOC_MODEL_DIR` с его абсолютным путём. + 3. Соберите сервисы: ```bash @@ -277,7 +278,7 @@ Docker подходит для быстрой оценки, интеграцио Подробнее см. [Запуск из исходников](./docs/develop/launch_ragflow_from_source.md). -> **Эксплуатационные примечания:** для постоянной настройки Elasticsearch добавьте `vm.max_map_count=262144` в `/etc/sysctl.conf`. При стандартном MySQL точка входа Docker выполняет миграцию перед запуском Admin, Syncer, Ingestor и API. В open-source 1.0 DeepDoc использует CPU для анализа макета, OCR и таблиц. После изменения Docker перезапустите сервисы и следуйте [руководству Docker](./docker/README.md) для сохранения или удаления данных. `download_deps.py` подготавливает нативные библиотеки и модели и требует `requests` и `huggingface-hub`. `--migrate` выполняет миграцию и завершается; `--admin` запускает службу администрирования, `--ingestor` — службу загрузки данных, `--syncer` — службу синхронизации внешних источников данных, а `--api` — службу API для Web UI и SDK. HTTP 200 означает готовность API. +> **Эксплуатационные примечания:** для постоянной настройки Elasticsearch добавьте `vm.max_map_count=262144` в `/etc/sysctl.conf`. При стандартном MySQL точка входа Docker выполняет миграцию перед запуском Admin, Syncer, Ingestor и API. В open-source 1.0 DeepDoc использует CPU для анализа макета, OCR и таблиц. После изменения Docker перезапустите сервисы и следуйте [руководству Docker](./docker/README.md) для сохранения или удаления данных. `download_dev_deps.py` подготавливает нативные библиотеки, а `download_deps.py` — модели и ресурсы токенизаторов; оба запускаются через `uv run`. `--migrate` выполняет миграцию и завершается; `--admin` запускает службу администрирования, `--ingestor` — службу загрузки данных, `--syncer` — службу синхронизации внешних источников данных, а `--api` — службу API для Web UI и SDK. HTTP 200 означает готовность API. ## 📚 Документация diff --git a/README_tr.md b/README_tr.md index 4fbb222f42..ab3106eb19 100644 --- a/README_tr.md +++ b/README_tr.md @@ -219,14 +219,15 @@ Ayrıntılar için [hızlı başlangıç kılavuzuna](./docs/quickstart.mdx) bak cd ragflow ``` -2. Bağımlılıkları hazırlayın: +2. Go derlemesinin bağladığı yerel statik kitaplıkları (`download_dev_deps.py`) ve backend'in çalışma zamanında yüklediği model / tokenizer kaynaklarını (`download_deps.py`) hazırlamak için bağımlılık indirme betiklerini çalıştırın: ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` + `download_dev_deps.py`, pdfium, pdf_oxide, office_oxide ve onnxruntime'ı `~/ragflow-native-libs` altına indirir ve açar; `download_deps.py`, cl100k sözlüğünü, DeepDoc modellerini ve gömme tokenizer varlıklarını `ragflow_deps/` altına indirir. İkisi de Python betiğidir; `uv run`, bildirilen bağımlılıkları (`requests` ve model ile tokenizer indirmeleri için `huggingface-hub`) otomatik olarak kurar. Aynı kaynakları başka şekilde hazırladıysanız bu adımı atlayın. Depo kökünden başlatıldığında Go hizmetleri `internal/rag/res/deepdoc` dizinini otomatik bulur; başka bir dizinden başlatmak için `DEEPDOC_MODEL_DIR` değişkenini o dizinin mutlak yoluna ayarlayın. + 3. Hizmetleri derleyin: ```bash @@ -280,7 +281,7 @@ Ayrıntılar için [hızlı başlangıç kılavuzuna](./docs/quickstart.mdx) bak Ayrıntılar için [Kaynaktan Başlatma](./docs/develop/launch_ragflow_from_source.md) belgesine bakın. -> **İşletim notları:** Elasticsearch ayarını kalıcı yapmak için `/etc/sysctl.conf` dosyasına `vm.max_map_count=262144` ekleyin. Varsayılan MySQL yapılandırmasında Docker giriş noktası Admin, Syncer, Ingestor ve API'den önce geçişi çalıştırır. Açık kaynak 1.0'da DeepDoc düzen, OCR ve tablo tanıma için CPU kullanır. Docker değişikliklerinden sonra hizmetleri yeniden başlatın ve verileri korumak veya kaldırmak için [Docker kılavuzunu](./docker/README.md) izleyin. `download_deps.py` yerel kitaplıkları ve modelleri hazırlar; `requests` ve `huggingface-hub` gerektirir. `--migrate` geçişi çalıştırıp çıkar; `--admin` yönetim hizmetini, `--ingestor` veri alma hizmetini, `--syncer` harici veri kaynağı eşitleme hizmetini ve `--api` Web UI ile SDK'lara yönelik API hizmetini başlatır. HTTP 200 API'nin hazır olduğunu gösterir. +> **İşletim notları:** Elasticsearch ayarını kalıcı yapmak için `/etc/sysctl.conf` dosyasına `vm.max_map_count=262144` ekleyin. Varsayılan MySQL yapılandırmasında Docker giriş noktası Admin, Syncer, Ingestor ve API'den önce geçişi çalıştırır. Açık kaynak 1.0'da DeepDoc düzen, OCR ve tablo tanıma için CPU kullanır. Docker değişikliklerinden sonra hizmetleri yeniden başlatın ve verileri korumak veya kaldırmak için [Docker kılavuzunu](./docker/README.md) izleyin. `download_dev_deps.py` yerel kitaplıkları, `download_deps.py` modelleri ve tokenizer kaynaklarını hazırlar; ikisi de `uv run` ile çalıştırılır. `--migrate` geçişi çalıştırıp çıkar; `--admin` yönetim hizmetini, `--ingestor` veri alma hizmetini, `--syncer` harici veri kaynağı eşitleme hizmetini ve `--api` Web UI ile SDK'lara yönelik API hizmetini başlatır. HTTP 200 API'nin hazır olduğunu gösterir. ## 📚 Dokümantasyon diff --git a/README_tzh.md b/README_tzh.md index 44e1270342..a988979663 100644 --- a/README_tzh.md +++ b/README_tzh.md @@ -263,15 +263,14 @@ Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`。鏡像、端 cd ragflow ``` -2. 運行 [依賴下載腳本](./ragflow_deps/download_deps.py) 下載靜態庫和模型文件： +2. 執行依賴下載腳本，準備 Go 建置所需連結的原生靜態庫（`download_dev_deps.py`）與後端執行時載入的模型/分詞器資源（`download_deps.py`）： ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` - 下載腳本準備構建所需的靜態庫和模型資源，並需要 `requests` 和 `huggingface-hub`。如果已通過其他方式準備好相同資源，可跳過這一步。Go 服務從倉庫根目錄啓動時會自動查找 `internal/rag/res/deepdoc`；如需從其他目錄啓動，請將 `DEEPDOC_MODEL_DIR` 設為該目錄的絕對路徑。 + `download_dev_deps.py` 拉取並解壓 pdfium、pdf_oxide、office_oxide、onnxruntime 等到 `~/ragflow-native-libs`；`download_deps.py` 拉取 cl100k 詞表、DeepDoc 模型與 embedding 分詞器資源到 `ragflow_deps/`。兩個腳本均為 Python 腳本，`uv run` 會自動安裝其聲明的依賴（`requests`，模型與分詞器下載另需 `huggingface-hub`）。如果已透過其他方式準備好相同資源，可跳過這一步。Go 服務從倉庫根目錄啓動時會自動查找 `internal/rag/res/deepdoc`；如需從其他目錄啓動，請將 `DEEPDOC_MODEL_DIR` 設為該目錄的絕對路徑。 3. 編譯： diff --git a/README_zh.md b/README_zh.md index 49011c8c35..19ac502e0f 100644 --- a/README_zh.md +++ b/README_zh.md @@ -263,15 +263,14 @@ Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`。镜像、端 cd ragflow ``` -2. 运行 [依赖下载脚本](./ragflow_deps/download_deps.py) 下载静态库和模型文件： +2. 运行依赖下载脚本，准备 Go 构建所需链接的原生静态库（`download_dev_deps.py`）与后端运行时加载的模型/分词器资源（`download_deps.py`）： ```bash - python3 -m venv /tmp/ragflow-go-download-venv - /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py + uv run tools/scripts/download_dev_deps.py + uv run ragflow_deps/download_deps.py ``` - 下载脚本准备构建所需的静态库和模型资源，并需要 `requests` 和 `huggingface-hub`。如果已通过其他方式准备好相同资源，可跳过这一步。Go 服务从仓库根目录启动时会自动查找 `internal/rag/res/deepdoc`；如需从其他目录启动，请将 `DEEPDOC_MODEL_DIR` 设为该目录的绝对路径。 + `download_dev_deps.py` 拉取并解压 pdfium、pdf_oxide、office_oxide、onnxruntime 等到 `~/ragflow-native-libs`；`download_deps.py` 拉取 cl100k 词表、DeepDoc 模型与 embedding 分词器资源到 `ragflow_deps/`。两个脚本均为 Python 脚本，`uv run` 会自动安装其声明的依赖（`requests`，模型与分词器下载还需 `huggingface-hub`）。如果已通过其他方式准备好相同资源，可跳过这一步。Go 服务从仓库根目录启动时会自动查找 `internal/rag/res/deepdoc`；如需从其他目录启动，请将 `DEEPDOC_MODEL_DIR` 设为该目录的绝对路径。 3. 编译： diff --git a/build.sh b/build.sh index 1dd09bc8cd..01455d877f 100755 --- a/build.sh +++ b/build.sh @@ -22,14 +22,24 @@ STRIP_SYMBOLS="" # Native static library settings. These are the user-cache paths (~/ragflow-native-libs/). # If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority -# and skips the network (download_deps.py) fallback. +# and skips the network (download_dev_deps.py) fallback. SYSTEM_DEPS="/opt/ragflow-native-libs" # office_oxide native library settings — static linking OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide" -OFFICE_OXIDE_VERSION="0.1.12" +# Derived from go.mod (single source of truth: the Go module version). Matches +# tools/scripts/download_dev_deps.py's office_oxide_url derivation. The pipeline exits 0 +# with an empty value when go.mod has no matching entry (no pipefail here), so +# the empty case needs its own check: an empty version would slip past the +# stale-lib comparison below and produce a broken .../lib/v link path. +OFFICE_OXIDE_VERSION="$(grep -m1 -E '^[[:space:]]*github.com/yfedoseev/office_oxide/go[[:space:]]+v' "$PROJECT_ROOT/go.mod" | sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')" +if [ -z "$OFFICE_OXIDE_VERSION" ]; then + echo -e "${RED}Error: could not read office_oxide version from go.mod${NC}" >&2; exit 1 +fi # pdfium native library settings — static linking (kognitos/pdfium-static) +# Pinned here (not grepped from the dev downloader) so build.sh stays tied to the +# checked-out source tree and the Docker build needs no extra COPY of the script. PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static" PDFIUM_STATIC_VERSION="7809" @@ -42,7 +52,7 @@ PDF_OXIDE_VERSION="0.3.73" # (--undefined=OrtGetApiBase + --dynamic-list, no --whole-archive); OrtGetApiBase # is then resolved via dlopen(NULL) (the process-global symbol table, not the # executable's own path), so no libonnxruntime.so is needed at runtime. -# Downloaded by ragflow_deps/download_deps.py into onnxruntime/static_lib. +# Downloaded by tools/scripts/download_dev_deps.py into onnxruntime/static_lib. ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib" # Copy a dependency from the system pre-seed directory to the user cache. @@ -222,20 +232,18 @@ check_go_deps() { check_ort_version_consistency() { print_section "Checking ONNX Runtime version consistency" - local env_go deps dockerfile + local env_go dockerfile env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')" - deps="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')" dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')" - if [ -z "$env_go" ] || [ -z "$deps" ] || [ -z "$dockerfile" ]; then + if [ -z "$env_go" ] || [ -z "$dockerfile" ]; then echo -e "${RED}Error: could not parse the ONNX Runtime version from one of the pinned locations${NC}" >&2 exit 1 fi - if [ "$env_go" != "$deps" ] || [ "$env_go" != "$dockerfile" ]; then + if [ "$env_go" != "$dockerfile" ]; then echo -e "${RED}Error: ONNX Runtime version is inconsistent — fix before building:${NC}" >&2 printf ' %-10s %s\n' "$env_go" "internal/common/environments.go:DeepDocORTVersion" - printf ' %-10s %s\n' "$deps" "ragflow_deps/download_deps.py:ORT_VERSION" printf ' %-10s %s\n' "$dockerfile" "dockerfile:ARG ORT_VERSION" exit 1 fi @@ -299,7 +307,7 @@ check_onnxruntime_deps() { echo -e "${RED}Error: no ONNX Runtime ${ort_version} static lib under $ONNXRUNTIME_STATIC_PREFIX${NC}" >&2 echo " available: ${avail:-}" >&2 echo " DeepDocORTVersion=${ort_version}; fetch it with:" >&2 - echo " uv run python3 ragflow_deps/download_deps.py" >&2 + echo " uv run tools/scripts/download_dev_deps.py" >&2 echo " or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2 return 1 fi @@ -314,7 +322,7 @@ check_onnxruntime_deps() { # # A mismatch means the local copy is corrupt or stale; we only WARN (not # fail) so a one-off re-issue cannot break the build. The downloader - # ragflow_deps/download_deps.py still verifies against + # tools/scripts/download_dev_deps.py still verifies against # the published sidecar at download time, so a re-issued archive is picked # up on the next download. local asset="onnxruntime-v${ort_version}-linux-x86_64.zip" @@ -329,7 +337,7 @@ check_onnxruntime_deps() { echo " local sha256: $actual" >&2 echo " The local copy may be corrupt or stale. Refresh it with:" >&2 echo " rm -f ${zip_path}" >&2 - echo " uv run python3 ragflow_deps/download_deps.py" >&2 + echo " uv run tools/scripts/download_dev_deps.py" >&2 else echo " ✓ ${asset} sha256 matches pinned digest" fi @@ -354,7 +362,7 @@ check_office_oxide_deps() { if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then echo -e "${RED}Error: office_oxide native library not found${NC}" echo " Expected: ${lib_path}" - echo " Run: uv run python3 ragflow_deps/download_deps.py" + echo " Run: uv run tools/scripts/download_dev_deps.py" echo " Or manually download: https://github.com/yfedoseev/office_oxide/releases/download/v${OFFICE_OXIDE_VERSION}/native-linux-x86_64.tar.gz" exit 1 fi @@ -368,12 +376,12 @@ check_office_oxide_deps() { found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1) echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}" rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/office_oxide-linux-x86_64.tar.gz" - if ! (cd "${PROJECT_ROOT}" && uv run python3 ragflow_deps/download_deps.py); then + if ! (cd "${PROJECT_ROOT}" && uv run tools/scripts/download_dev_deps.py); then echo -e "${RED}Error: office_oxide native lib version mismatch${NC}" echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}" echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:" echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz" - echo " uv run python3 ragflow_deps/download_deps.py" + echo " uv run tools/scripts/download_dev_deps.py" exit 1 fi lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}" @@ -406,7 +414,7 @@ check_pdfium_deps() { echo " pdfium (static) not found" echo " Expected: ${lib_path}" - echo " Run: uv run python3 ragflow_deps/download_deps.py" + echo " Run: uv run tools/scripts/download_dev_deps.py" echo " Or: curl -fsSL https://github.com/kognitos/pdfium-static/releases/download/chromium%2F${PDFIUM_STATIC_VERSION}/pdfium-linux-x64-static.tgz | tar xz -C ${PDFIUM_STATIC_PREFIX}" return 1 } @@ -512,7 +520,7 @@ check_pdf_oxide_deps() { echo " Required: v${PDF_OXIDE_VERSION}; found: ${found_version:-unknown}" echo " A stale lib silently reverts PDF parsing fixes. Refresh:" echo " rm -rf ${PDF_OXIDE_PREFIX} ragflow_deps/pdf_oxide-go-ffi-linux-amd64.tar.gz" - echo " uv run python3 ragflow_deps/download_deps.py" + echo " uv run tools/scripts/download_dev_deps.py" return 1 ;; esac @@ -520,7 +528,7 @@ check_pdf_oxide_deps() { echo " pdf_oxide (static) not found" echo " Expected: ${lib_path}" - echo " Run: uv run python3 ragflow_deps/download_deps.py" + echo " Run: uv run tools/scripts/download_dev_deps.py" echo " Or: curl -fsSL https://github.com/yfedoseev/pdf_oxide/releases/download/v${PDF_OXIDE_VERSION}/pdf_oxide-go-ffi-linux-amd64.tar.gz | tar xz -C ${PDF_OXIDE_PREFIX}" return 1 } @@ -650,12 +658,12 @@ build_go() { echo " -Wl,--dynamic-list). Without it the binary compiles but dies" >&2 echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2 echo " Fetch the static libs with:" >&2 - echo " uv run python3 ragflow_deps/download_deps.py" >&2 + echo " uv run tools/scripts/download_dev_deps.py" >&2 echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2 echo " This production binary must statically include ORT — there is no" >&2 echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2 echo " - the ORT static_lib dir was never seeded: run" >&2 - echo " 'uv run python3 ragflow_deps/download_deps.py', or pre-seed" >&2 + echo " 'uv run tools/scripts/download_dev_deps.py', or pre-seed" >&2 echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2 echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2 return 1 @@ -1093,7 +1101,7 @@ DEPENDENCIES: - cmake >= 4.0 - go >= 1.26.4 - clang++ with C++20 support - - office_oxide native library (download with: uv run python3 ragflow_deps/download_deps.py) + - office_oxide native library (download with: uv run tools/scripts/download_dev_deps.py) - lld (Linux only): sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld - pcre2 development files - Debian/Ubuntu: libpcre2-dev diff --git a/docs/develop/launch_ragflow_from_source.md b/docs/develop/launch_ragflow_from_source.md index 815a1de055..bde892bb32 100644 --- a/docs/develop/launch_ragflow_from_source.md +++ b/docs/develop/launch_ragflow_from_source.md @@ -27,7 +27,7 @@ All long-running RAGFlow processes started below use `bin/ragflow_server`. - Go 1.27 or later, as declared in `go.mod` (check `go version`). - CMake 4.0 or later, Clang 20, LLD 20, and PCRE2 development headers. - Node.js 18.20.4 or later and npm for the frontend. -- Python 3.10 or later only for `ragflow_deps/download_deps.py`, which downloads the native libraries and model resources required by the Go build. +- Python 3.10 or later for `tools/scripts/download_dev_deps.py` and `ragflow_deps/download_deps.py`, which download the native libraries and model resources required by the Go build. See the [Docker installation guide](https://docs.docker.com/engine/install/) if Docker is not installed. Use the Go version declared in `go.mod` and the compiler versions listed above when preparing the build environment. @@ -42,7 +42,7 @@ clang++ --version ld.lld --version ``` -Install Go 1.27 or later if needed; make sure the Go executable selected by your shell meets the requirement in `go.mod`. On Ubuntu, install the compiler, linker, PCRE2 headers, Python virtual-environment support, and command-line tools used by the build with: +Install Go 1.27 or later if needed; make sure the Go executable selected by your shell meets the requirement in `go.mod`. Install `uv` as well; the dependency downloaders run through it. On Ubuntu, install the compiler, linker, PCRE2 headers, Python 3, and command-line tools used by the build with: ```bash sudo apt update @@ -54,19 +54,15 @@ Ubuntu's enabled repositories may not provide `clang-20` and `lld-20`. If APT ca Make `clang++` resolve to Clang 20 and `ld.lld` to LLD 20 before building. Installing `lld-20` alone does not necessarily replace an older system default `ld.lld`. Check both versions above: an older LLD can produce a binary that builds successfully but fails before the Go server starts. Install CMake 4.0 or later separately if your distribution package is older; the Go development guide includes the Ubuntu 24.04 Kitware repository setup. The `binutils` package supplies `objcopy` and `nm`; do not ignore a missing-`objcopy` warning because skipping the RE2 symbol-renaming step can cause a runtime crash. -Download the native libraries and Go DeepDoc model weights with a small, isolated Python environment: +Download the native static libraries and the runtime model / tokenizer resources with the two dependency downloaders (`uv run` installs each script's declared Python dependencies): ```bash -python3 -m venv /tmp/ragflow-go-download-venv -/tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub -/tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py +uv run tools/scripts/download_dev_deps.py # local-dev one-stop: native .a + cl100k, DeepDoc .ort, tokenizer assets ``` -The downloader fetches the static libraries needed by `build.sh`, plus `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, and `ocr.res` into `internal/rag/res/deepdoc/`. These files are required by the in-process Go DeepDoc backend. Keep the server's working directory at the repository root so it can find them automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `internal/rag/res/deepdoc`. +`download_dev_deps.py` is the local-dev one-stop: it fetches the static libraries needed by `build.sh` (pdfium / pdf_oxide / office_oxide / onnxruntime → `~/ragflow-native-libs`) AND the runtime resources the backend loads at runtime — `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, `ocr.res` into `internal/rag/res/deepdoc/` (required by the in-process Go DeepDoc backend) and `ragflow_deps/cl100k_base.tiktoken` and the embedding tokenizer assets under `ragflow_deps/huggingface.co/`. Run it once and the local environment is ready; already-present files are skipped on re-run. The image-facing `ragflow_deps/download_deps.py` fetches the same runtime resources PLUS the stagehand server (installed into `${XDG_CACHE_HOME:-$HOME/.cache}/stagehand/lib/go_/`) and skips the native `.a`; run it only when building the dependency image or in CI. Native archives support Linux x86_64. Keep the server's working directory at the repository root so it can find the DeepDoc models automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `internal/rag/res/deepdoc`. -It also downloads `ragflow_deps/cl100k_base.tiktoken` and the embedding tokenizer assets, and installs the stagehand driver into `${XDG_CACHE_HOME:-$HOME/.cache}/stagehand/lib/go_/`. Native archives support Linux x86_64. - -The isolated download environment can be removed after the resources have been prepared. +Skip either script if you have prepared the same resources by other means. Build the C++ bindings and Go binaries: @@ -90,7 +86,7 @@ It should print API usage information and may exit with status 1. This command o The Go/C++ build requires LLD 20. Confirm that `ld.lld --version` reports LLD 20 before building; merely installing `lld-20` is insufficient when an older `ld.lld` remains the system default. If the binary builds successfully but exits or crashes before reaching Go `main`, verify the selected linker and rebuild with LLD 20. -To prepare the resource image used by the Go Dockerfile, run the downloader above, then build from `ragflow_deps/`: +To prepare the resource image used by the Go Dockerfile, first run `uv run ragflow_deps/download_deps.py` (above) to populate `ragflow_deps/`, then build from `ragflow_deps/`: ```bash cd ragflow_deps diff --git a/internal/common/environments.go b/internal/common/environments.go index c12bf00c48..baefb2f367 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -324,9 +324,8 @@ func HasModelFiles(dir string) bool { // DeepDocORTVersion is the onnxruntime native release the in-process (Go) // DeepDoc backend is built and tested against (e.g. "1.29.0"). It is ONE OF -// THREE raw version declarations that must stay equal (the other two are -// ORT_VERSION in ragflow_deps/download_deps.py and ARG ORT_VERSION in -// Dockerfile) — NOT a single source of truth. The +// TWO raw version declarations that must stay equal (the other is +// ARG ORT_VERSION in Dockerfile) — NOT a single source of truth. The // download URL and extracted dir name are built from those ORT_VERSION // constants, not from this one. The Go binding // (github.com/infiniflow/onnxruntime_go, the org mirror of yalue/onnxruntime_go) @@ -336,12 +335,17 @@ func HasModelFiles(dir string) bool { // the same minor line. ONNX Runtime is linked statically (libonnxruntime.a), // so there is no .so / SONAME at runtime. // -// To bump ORT, the three Go-side native pins above must change together +// To bump ORT, the two Go-side native pins above must change together // (drift breaks the static link or the runtime OrtGetApiBase lookup): // - DeepDocORTVersion (here, Go) -// - ORT_VERSION in ragflow_deps/download_deps.py // - ARG ORT_VERSION in Dockerfile // +// The dev downloader tools/scripts/download_dev_deps.py carries its own copy of +// ORT_VERSION used only to fetch the matching infiniflow/ragflow-build release; +// build.sh no longer greps it, so the Docker build needs no extra COPY of the +// script. Keep all three (this, Dockerfile ARG, the downloader) on the same +// version when bumping. +// // Separately, keep these on the same ORT minor line but version them // independently of the Go native lib (see pyproject.toml / development.md): // - the onnxruntime== / onnxruntime-gpu== pins in pyproject.toml (Python side) diff --git a/internal/development.md b/internal/development.md index f12ea68cd9..e1e1df36d5 100644 --- a/internal/development.md +++ b/internal/development.md @@ -39,12 +39,12 @@ go version ### 1.4 Install dependent library ```shell sudo apt install libpcre2-dev -uv run ragflow_deps/download_deps.py +uv run tools/scripts/download_dev_deps.py # local-dev one-stop: native .a + cl100k, DeepDoc .ort, tokenizer assets ``` -Install `uv` before running the downloader; it installs the script's declared Python dependencies. +Install `uv` before running the downloaders; it installs each script's declared Python dependencies. -This downloader provisions Go native libraries, `.ort` weights, model tokenizer assets, and `cl100k_base.tiktoken`. It installs the local stagehand driver into the SDK's cache, using the SDK version from `go.mod`. Native archives support Linux x86_64. +`download_dev_deps.py` is the local-dev one-stop: it provisions the Go native libraries under `~/ragflow-native-libs` AND the `.ort` weights, embedding tokenizer assets, and `cl100k_base.tiktoken` into `internal/rag/res/deepdoc/` and `ragflow_deps/huggingface.co/`. Run it once and the local environment is ready. The image-facing `ragflow_deps/download_deps.py` provisions the same runtime resources plus the stagehand driver (installed into the SDK's cache, using the SDK version from `go.mod`) and skips the native `.a`; run it only when building the dependency image or in CI. Native archives support Linux x86_64. Build the Go resource image from `ragflow_deps/` after preparing these files: @@ -58,9 +58,9 @@ The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stag > **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables: > > ```bash -> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_deps.py +> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_dev_deps.py > PLATFORM="linux_amd64" # macOS (darwin_*) is temporarily unsupported; linux_arm64 is not a supported native build target -> # NOTE: the ONNX Runtime static lib fetched by download_deps.py is +> # NOTE: the ONNX Runtime static lib fetched by download_dev_deps.py is > # linux-x64 ONLY (onnxruntime-linux-x64-static_lib-*). On darwin_* / non-amd64 > # PLATFORM values the production DeepDoc backend cannot be linked, so those > # PLATFORM examples cover office_oxide/pdfium/pdf_oxide only — ORT is a @@ -111,18 +111,18 @@ The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stag > Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from > `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If > you see `Error: ONNX Runtime static libraries are not linked`, run -> `uv run ragflow_deps/download_deps.py` (or pre-seed +> `uv run tools/scripts/download_dev_deps.py` (or pre-seed > `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is > no ORT-free production build path — if ORT is absent the binary fails at > startup, so the remedy is always to seed the static lib above, never to build > without it. -> **Note**: The ONNX Runtime native version is pinned in several Go-side places -> that must stay in sync. Bumping it in one spot and not the others fails the -> build with `Error: ONNX Runtime version is inconsistent`: +> **Note**: The ONNX Runtime native version is pinned in a few places that must +> stay in sync. Bumping it in one spot and not the others fails the build with +> `Error: ONNX Runtime version is inconsistent`: > - `internal/common/environments.go` — `DeepDocORTVersion` > - `Dockerfile` — `ARG ORT_VERSION` -> - `ragflow_deps/download_deps.py` — `ORT_VERSION` +> - `tools/scripts/download_dev_deps.py` — `ORT_VERSION` (used by the dev downloader to fetch the matching release; build.sh does not grep it, but it must match what gets linked) > > `build.sh` runs this consistency check automatically before the Go build > (through `check_go_deps`) and fails fast on any mismatch. Run it on demand @@ -138,7 +138,7 @@ The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stag > `build.sh`, without a network request. The downloader separately checks the > published `onnxruntime-v-linux-x86_64.zip.sha256` sidecar. A local ZIP > mismatch only emits a warning with the expected/actual SHA-256 and the -> refresh command (`rm -f ` + `uv run ragflow_deps/download_deps.py`); +> refresh command (`rm -f ` + `uv run tools/scripts/download_dev_deps.py`); > the build continues checking and linking the extracted static libraries. CI seeds > ORT from `/opt` and has no local zip, so the check is skipped there (the bake > is authoritative). This matters because the `onnxruntime_go` binding reaches @@ -152,7 +152,7 @@ The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stag > would otherwise silently reuse a stale linked binary. Stamping the path with > the archive's sha256 changes the flag string whenever the content changes → > automatic relink. If you ever see a stale-`.a` crash after an ORT re-issue, -> re-run `uv run ragflow_deps/download_deps.py` +> re-run `uv run tools/scripts/download_dev_deps.py` > so the stamp moves; a plain `go clean -cache` also forces it. ### 1.5 Build RAGFlow diff --git a/internal/tokenizer/embedding_token_limits.md b/internal/tokenizer/embedding_token_limits.md index a9ef2f098e..afe13a0278 100644 --- a/internal/tokenizer/embedding_token_limits.md +++ b/internal/tokenizer/embedding_token_limits.md @@ -347,7 +347,8 @@ an unrelated `vocab.txt` there silently replaces the verified artifact; and audi 1. **Identify the architecture** from the model's own artifact (`tokenizer.json`, or `vocab.txt` / `sentencepiece.bpe.model`). If the vocabulary is byte-identical to a family fixture, only step 4 is needed; if it differs, the model needs its own fixture. -2. **Add the artifact** to `ragflow_deps/download_deps.py`'s `TOKENIZER_ASSETS` list - +2. **Add the artifact** to the `TOKENIZER_ASSETS` list inlined in + `ragflow_deps/download_deps.py` (the downloader fetches individual files from) - individual files, not `snapshot_download` of a multi-GB repo - pin its SHA-1 in the loader map so a mismatch fails at load instead of counting with the wrong table, and add it to the copy loop in `Dockerfile` / `Dockerfile_base` so every runtime image ships it @@ -388,9 +389,11 @@ property tier green, `gofmt` clean. ## Assets and the oracle -Assets are fetched, never committed: `ragflow_deps/download_deps.py` has a `TOKENIZER_ASSETS` -list that downloads **individual files** (not `snapshot_download` of a repo that also -carries multi-GB weights) into `ragflow_deps/huggingface.co//`, which +Assets are fetched, never committed: the `TOKENIZER_ASSETS` list inlined in +`ragflow_deps/download_deps.py` is the source the downloader reads to download +**individual files** (not `snapshot_download` of a repo that also carries multi-GB +weights) from, into +`ragflow_deps/huggingface.co//`, which `.gitignore` already excludes (`huggingface.co/`). Loaders read them from disk with a SHA-1 pin, walk up from the working and executable directories like `bpe_loader.go`, and perform no network I/O. A missing asset fails loudly at every layer that can report one: the @@ -412,10 +415,12 @@ embedder whose model declares a tokenizer it cannot load **fails the ingest** wi `embedding tokenizer "" is declared for but its asset is unavailable …: refusing to count with the calibrated estimate`, instead of counting differently. -The three places are held together by `ragflow_deps/test_tokenizer_assets.py`, which -fails when a `"runtime"` asset is missing from either Dockerfile, when an `"oracle"` -asset is shipped, or when a runtime asset has no 40-hex SHA-1 pin in -`internal/tokenizer/*.go`. It is there because the first version of this chain silently +The three places are held together by the inlined `TOKENIZER_ASSETS` list (read by +`download_deps.py` and `download_dev_deps.py`) and the Dockerfile copy loops plus +the Go loader SHA-1 pins, which must stay in sync: a `"runtime"` asset must be copied +into both `Dockerfile` and `Dockerfile_base` and pinned with a 40-hex SHA-1 in +`internal/tokenizer/*.go`, while an `"oracle"` asset must never be shipped. The first +version of this chain silently skipped step three: the counters loaded fine in a developer checkout and quietly degraded to the calibrated path in the image. @@ -784,8 +789,8 @@ bash scripts/restart_ingestor.sh # how much of the catalog the tags cover (regenerate the tables in the strategy section) python3 scripts/tokenizer_coverage.py --markdown -# asset plumbing: download list vs runtime images vs SHA-1 pins -python3 -m pytest ragflow_deps/test_tokenizer_assets.py -q +# asset plumbing: download manifest vs runtime images vs SHA-1 pins +./build.sh --test ./internal/tokenizer/ -run TestTokenizerAssetsAreWellFormed ``` Expected output today: all packages `ok`; the oracle test logs diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.py index 0a7f3970ab..5689e84420 100755 --- a/ragflow_deps/download_deps.py +++ b/ragflow_deps/download_deps.py @@ -9,26 +9,31 @@ # ] # /// -# This script prepares Go dependencies on Linux x86_64: the native static -# libraries (pdfium / pdf_oxide / office_oxide / onnxruntime) for `build.sh`, -# and the Go DeepDoc `.ort` weights (det/layout/tsr/rec.ort + ocr.res) so a Go -# dev can run the in-process backend locally. It also downloads the cl100k BPE -# table and installs the local stagehand driver in the Go SDK's cache. -# Archives and tokenizer assets land under ragflow_deps/; DeepDoc weights land -# under internal/rag/res/deepdoc/, regardless of the caller's working directory. +# This script prepares the runtime / image resources that the dependency image +# ships on Linux x86_64: the cl100k BPE table, the stagehand-server binaries +# (installed into the Go SDK's cache), the Go DeepDoc `.ort` weights +# (det/layout/tsr/rec.ort + ocr.res), and the embedding tokenizer assets the Go +# counters load. # -# Downloaded archives and tokenizer assets land under `ragflow_deps/`. +# This is the CI / image-build one-stop. It fetches the runtime resources PLUS +# the stagehand server binary, and deliberately does NOT fetch the native static +# libraries the Go build links — those come from tools/scripts/download_dev_deps.py +# and are baked into the server binary by the build stage. Run this once to ready +# a dependency image; for local development, run the dev downloader instead. +# All version numbers and URLs are defined in the block below; bump a version +# there, not at the call sites. # # Typical workflow: # # uv run ragflow_deps/download_deps.py -# Go DeepDoc weights: in addition to the native libs, this script downloads the -# five Go model files (internal/common.DeepDocModelFiles) from InfiniFlow/deepdoc -# straight into the repo's canonical model directory `internal/rag/res/deepdoc/` (one level -# up from this script). The Go server auto-discovers that directory via -# resolveDeepDocModelDir(), and build.sh --test-native / internal/deepdoc/native/run.sh -# default MODEL_DIR there too — so after running this script NO MODEL_DIR env needs -# to be set: +# +# Go DeepDoc weights: in addition to the resources above, this script downloads +# the five Go model files (internal/common.DeepDocModelFiles) from +# InfiniFlow/deepdoc straight into the repo's canonical model directory +# `internal/rag/res/deepdoc/` (one level up from this script). The Go server +# auto-discovers that directory via resolveDeepDocModelDir(), and build.sh +# --test-native / internal/deepdoc/native/run.sh default MODEL_DIR there too — +# so after running this script NO MODEL_DIR env needs to be set: # # bash build.sh --test-native # or: cd internal/deepdoc/native && bash run.sh # @@ -36,235 +41,99 @@ # tokenizer file per family (XLM-R SentencePiece, BERT WordPiece, two byte-level BPE # families) from `huggingface.co//` at the top of ragflow_deps/ - the path # the loaders search and the one ragflow_deps/Dockerfile ships. This script fetches -# them too. Models that declare a missing tokenizer cannot be ingested -# (see internal/tokenizer/embedding_token_limits.md). +# them too, per the inlined TOKENIZER_ASSETS list. Models that declare a missing +# tokenizer cannot be ingested (see internal/tokenizer/embedding_token_limits.md). import argparse -import hashlib import os import platform import re import shutil import sys -import tarfile import tempfile -import zipfile import requests -# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF -# THREE places (with that Go constant and ARG ORT_VERSION in Dockerfile) -# that must carry the same ONNX Runtime -# native release for the statically-linked Go DeepDoc backend. There is no -# single source of truth — keep all three equal. build.sh --check-ort-version -# greps this file (and the other two) to fail fast on drift. (The Python pip -# onnxruntime== pin in pyproject.toml is versioned independently and is not -# part of this check.) -# -# Source of the native static archives: infiniflow/ragflow-build (our own -# ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs -# account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is -# `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`. The archive is occasionally -# re-issued under this SAME tag/asset name with patched content; this script -# detects that via a `{asset}.sha256` -# sidecar and re-download/re-extract, so a stale local copy never silently -# lingers. -ORT_VERSION = "1.29.0" +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(SCRIPT_DIR) + +# ── Dependency versions & URLs (edit here) ──────────────────────────────── +# Single source of truth for the runtime / image-resource versions and URLs +# this script fetches. build.sh greps none of these (they are image-facing); +# keep the `NAME = "value"` form greppable for consistency with +# tools/scripts/download_dev_deps.py. + +# stagehand-server release binaries (used by the Browser component in local +# mode). They are LOOSELY matched to the stagehand-go Go module: both stay on +# the v3.x line and remain protocol-compatible, but the two version numbers do +# NOT track each other, so there is no cross-reference to keep in sync here. +# Kept with the "v" prefix because it is used verbatim in the release tag path. +STAGEHAND_SERVER_VERSION = "v3.7.2" -def _ort_asset_name(version): - """Release asset filename under infiniflow/ragflow-build tag onnxruntime-v{version}.""" - return f"onnxruntime-v{version}-linux-x86_64.zip" +def gh_proxy() -> str: + """GitHub release proxy prefix (gh-proxy.com behind the GFW), else empty.""" + return os.environ.get("GH_PROXY") or "" -def _ort_extracted_dir(version): - """Top-level directory name INSIDE the release zip (what extractall creates).""" - return f"onnxruntime-v{version}-linux-x86_64" +def go_module_version(module: str, repo_root: str) -> str: + """Read a Go module's pinned version (the "vX.Y.Z" after its require line) + from `repo_root`/go.mod. Raises RuntimeError when go.mod or the module is + missing — the caller cannot link the right library without it.""" + go_mod = os.path.join(repo_root, "go.mod") + if not os.path.isfile(go_mod): + raise RuntimeError(f"go.mod not found at {go_mod}") + with open(go_mod, encoding="utf-8") as file: + match = re.search(rf"{re.escape(module)}\s+v([^\s]+)", file.read()) + if match is None: + raise RuntimeError(f"{module} version not found in {go_mod}") + return match.group(1) -def _ort_normalized_dir(version): - """Directory name build.sh's `find ... -name '*.a'` glob expects under static_lib.""" - return f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" +def cl100k_url() -> str: + """cl100k BPE table. Hosted on Azure blob storage (NOT a GitHub release), + so the GitHub mirror prefix does not apply and is deliberately omitted.""" + return "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" -def _sha256_of(path): - """sha256 of a file, streamed in chunks so large archives don't blow memory.""" - h = hashlib.sha256() - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(1 << 20), b""): - h.update(chunk) - return h.hexdigest() +def stagehand_url(arch: str) -> str: + """stagehand-server release binary: stagehand_url("x64" | "arm64").""" + return f"{gh_proxy()}https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/{STAGEHAND_SERVER_VERSION}/stagehand-server-v3-linux-{arch}" # Mirrors internal/common.DeepDocModelFiles (Go in-process DeepDoc backend). # These are the ONLY weights the Go backend loads; the full InfiniFlow/deepdoc # repo also ships .onnx (Python-only), which this Go-only script deliberately -# skips to keep the download lean. +# skips to keep the download lean. The files are fetched one by one, so this +# list MUST be edited by hand when the Go slice changes. DEEPDOC_REPO = "InfiniFlow/deepdoc" DEEPDOC_MODEL_FILES = ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.res"] -# Embedding tokenizer assets for the Go counters (internal/tokenizer). Each entry is -# (repo, file, kind): -# -# "runtime" - the counters load it in production, so the runtime images must ship it -# (the copy loop in Dockerfile / Dockerfile_base) and the loader must pin -# its SHA-1; -# "oracle" - only scripts/gen_tokenizer_oracle.py needs it, to regenerate the test -# fixtures, so it is deliberately NOT shipped to the image. -# -# ragflow_deps/test_tokenizer_assets.py checks these download, packaging, and pin lists. -# -# Fetched per file, not by snapshot: these repos also carry multi-GB weights we do not -# want. Missing tokenizer assets fail dependency preparation. + +# Embedding tokenizer assets the Go counters load. Inlined here (no separate +# manifest file) as the single source of truth this script reads directly; the +# dev script tools/scripts/download_dev_deps.py carries its own copy (no shared +# module, by design). One entry per tuple: (repo, file, kind). +# runtime - the Go counters load it in production; the runtime images must ship +# it and the loader must pin its SHA-1. +# oracle - only scripts/gen_tokenizer_oracle.py needs it to regenerate the +# test fixtures, so it is deliberately NOT shipped to the image. TOKENIZER_ASSETS = [ - # XLM-R SentencePiece (Unigram) - the BAAI bge / multilingual-e5 / m3e / - # gte-multilingual / jina-v3 family shares this vocabulary. ("BAAI/bge-m3", "sentencepiece.bpe.model", "runtime"), - # BERT WordPiece vocab for the bge-*-en / e5-* / gte-base family. ("BAAI/bge-large-en-v1.5", "vocab.txt", "runtime"), - # Byte-level BPE families (Qwen3-Embedding, Mistral/Llama embeddings). ("Qwen/Qwen3-Embedding-0.6B", "tokenizer.json", "runtime"), ("intfloat/e5-mistral-7b-instruct", "tokenizer.json", "runtime"), - # Cross-check oracles only: the HF tokenizer.json files the same families convert to. ("BAAI/bge-m3", "tokenizer.json", "oracle"), ("BAAI/bge-large-en-v1.5", "tokenizer.json", "oracle"), ] -def prune_stale_onnxruntime(static_lib_dir, version): - """Remove ONNX Runtime version dirs under static_lib that do NOT match - `version`. Without this, a version bump leaves the stale dir next to - the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate - symbols / wrong version, silently).""" - if not os.path.isdir(static_lib_dir): - return - expected = _ort_normalized_dir(version) - for name in os.listdir(static_lib_dir): - if not name.startswith("onnxruntime-linux-x64-static_lib-"): - continue - if name == expected: - continue - stale = os.path.join(static_lib_dir, name) - print(f" Removing stale ONNX Runtime dir: {stale}") - shutil.rmtree(stale) - - -def has_static_archives(directory): - """True when `directory` holds at least one static archive (.a).""" - return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files) - - -def extract_onnxruntime(static_lib_dir, archive_path, version): - """Ensure the ONNX Runtime static archives for `version` sit under - `static_lib_dir`. Returns True when that version is available afterwards - (extracted now or already present), False when the archive is missing. - - The infiniflow/ragflow-build release zip carries a top-level dir named - onnxruntime-v{version}-linux-x86_64, so a present - `static_lib_dir` is NOT evidence that THIS version is extracted: after a - version bump the stale dir is pruned and the new one must be extracted. - """ - if not os.path.isfile(archive_path): - print(f" Skipping extraction: {os.path.basename(archive_path)} not found") - return False - prune_stale_onnxruntime(static_lib_dir, version) - version_dir = os.path.join(static_lib_dir, _ort_normalized_dir(version)) - if os.path.isdir(version_dir) and has_static_archives(version_dir): - print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}") - return True - os.makedirs(static_lib_dir, exist_ok=True) - print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}") - with zipfile.ZipFile(archive_path) as zf: - zf.extractall(static_lib_dir) - # The infiniflow/ragflow-build release zip carries a top-level dir named - # onnxruntime-v{version}-linux-x86_64, but build.sh's glob and the stale - # checks above all expect onnxruntime-linux-x64-static_lib-{version}-glibc2_28. - # Rename it so every consumer shares one name convention (driven by - # ORT_VERSION). - extracted = os.path.join(static_lib_dir, _ort_extracted_dir(version)) - normalized = os.path.join(static_lib_dir, _ort_normalized_dir(version)) - if os.path.isdir(extracted) and extracted != normalized: - if os.path.exists(normalized): - shutil.rmtree(normalized) - print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}") - os.rename(extracted, normalized) - return True - - -def _go_module_version(module): - repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - with open(os.path.join(repo_root, "go.mod")) as file: - version = re.search(rf"{re.escape(module)}\s+v([^\s]+)", file.read()) - if version is None: - raise RuntimeError(f"{module} version not found in go.mod") - return version.group(1) - - -def get_urls(use_china_mirrors=False) -> list[str | list[str]]: - office_version = _go_module_version("github.com/yfedoseev/office_oxide/go") - if use_china_mirrors: - return [ - "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", - # stagehand-server-v3 Node.js SEA binaries (used by Browser - # component in local mode). - # - # The stagehand-go Go module (pinned in go.mod) and the - # stagehand-server binary (this release) are LOOSELY - # MATCHED — both stay on the v3.x line and remain - # protocol-compatible. The two version numbers do NOT - # track each other: the Go SDK is at v3.21.0 while the - # current latest server release is v3.7.2. - # - # On every go.mod bump, refresh this URL to the current - # latest server release. There is no version - # correspondence to maintain; "both on v3.x" is the - # compatibility contract. - "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", - "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, - # office_oxide, onnxruntime). Used by build.sh's check_*_deps - # functions — pre-downloaded to avoid network access during CI. - ["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], - ["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], - [f"https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v{office_version}/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], - [ - f"https://gh-proxy.com/https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", - _ort_asset_name(ORT_VERSION), - ], - ] - else: - return [ - "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", - # stagehand-server-v3 Node.js SEA binaries (used by Browser - # component in local mode). - # - # The stagehand-go Go module (pinned in go.mod) and the - # stagehand-server binary (this release) are LOOSELY - # MATCHED — both stay on the v3.x line and remain - # protocol-compatible. The two version numbers do NOT - # track each other: the Go SDK is at v3.21.0 while the - # current latest server release is v3.7.2. - # - # On every go.mod bump, refresh this URL to the current - # latest server release. There is no version - # correspondence to maintain; "both on v3.x" is the - # compatibility contract. - "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", - "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, - # office_oxide, onnxruntime). Used by build.sh's check_*_deps - # functions — pre-downloaded to avoid network access during CI. - ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], - ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], - [f"https://github.com/yfedoseev/office_oxide/releases/download/v{office_version}/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], - [ - f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", - _ort_asset_name(ORT_VERSION), - ], - ] - - def download_with_progress(url, filename): + """Download URL to FILENAME atomically (temp file, then move on success). + + A failed or empty download leaves any previous valid file untouched, so an + interrupted run never destroys a working dependency tree. Mirrors + download_dev_deps.download_with_progress.""" filename = os.fspath(filename) temporary = None try: @@ -290,35 +159,33 @@ def download_with_progress(url, filename): print() -def _extract_native_archive(archive_path, target, required_files, version=None): - def complete(directory): - for name in required_files: - path = os.path.join(directory, name) - if not os.path.isfile(path) or os.path.getsize(path) == 0: - return False - if version is not None and name.endswith(".a"): - with open(path, "rb") as file: - if b"\x00" + version.encode() + b"\x00" not in file.read(): - return False - return True +def _valid_stagehand_binary(path): + """True when PATH is a non-empty ELF executable (the stagehand-server SEA).""" + if not os.path.isfile(path) or os.path.getsize(path) == 0: + return False + with open(path, "rb") as file: + return file.read(4) == b"\x7fELF" - if complete(target): - print(f" ✓ {os.path.basename(target)} already extracted to {target}") - return - os.makedirs(os.path.dirname(target), exist_ok=True) - with tempfile.TemporaryDirectory(dir=os.path.dirname(target)) as staging: - with tarfile.open(archive_path) as archive: - members = archive.getmembers() - for member in members: - if os.path.isabs(member.name) or ".." in member.name.split("/") or not (member.isfile() or member.isdir()): - raise tarfile.TarError(f"Unsafe native archive member: {member.name}") - archive.extractall(staging, members=members) - if not complete(staging): - raise RuntimeError(f"{archive_path} is missing required native files or version {version}") - if os.path.isdir(target): - shutil.rmtree(target) - os.replace(staging, target) - print(f" ✓ extracted {os.path.basename(archive_path)} to {target}") + +def load_tokenizer_assets(assets=None): + """Validate and return the inlined tokenizer asset list as (repo, file, kind) + tuples. When ASSETS is None, uses the module-level TOKENIZER_ASSETS. + + Fetched per file, not by snapshot: these repos also carry multi-GB weights + we do not want. A malformed entry (bad kind, malformed repo, or an empty + list) fails dependency preparation loudly instead of handing a broken tree + to the image build.""" + items = TOKENIZER_ASSETS if assets is None else assets + result = [] + for repo, filename, kind in items: + if kind not in ("runtime", "oracle"): + raise RuntimeError(f"unknown tokenizer asset kind {kind!r} for {repo}/{filename}") + if "/" not in repo or not filename: + raise RuntimeError(f"malformed tokenizer asset repo/file {repo}/{filename}") + result.append((repo, filename, kind)) + if not result: + raise RuntimeError("no tokenizer assets listed") + return result def _install_stagehand(): @@ -327,12 +194,12 @@ def _install_stagehand(): architecture = {"x86_64": "x64", "aarch64": "arm64"}.get(platform.machine()) if architecture is None: raise RuntimeError(f"Unsupported stagehand architecture: {platform.machine()}") - version = _go_module_version("github.com/browserbase/stagehand-go/v3") + version = go_module_version("github.com/browserbase/stagehand-go/v3", REPO_ROOT) cache_root = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") target_dir = os.path.join(cache_root, "stagehand", "lib", f"go_{version}") os.makedirs(target_dir, exist_ok=True) filename = f"stagehand-server-v3-linux-{architecture}" - source = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) + source = os.path.join(SCRIPT_DIR, filename) target = os.path.join(target_dir, filename) with tempfile.TemporaryDirectory(dir=target_dir) as staging: prepared = os.path.join(staging, filename) @@ -342,49 +209,16 @@ def _install_stagehand(): print(f" ✓ stagehand installed to {target}") -def _valid_download(filename): - if not os.path.isfile(filename) or os.path.getsize(filename) == 0: - return False - if filename.endswith((".tar.gz", ".tgz")): - if not tarfile.is_tarfile(filename): - return False - if os.path.basename(filename) == "office_oxide-linux-x86_64.tar.gz": - version = _go_module_version("github.com/yfedoseev/office_oxide/go") - with tarfile.open(filename) as archive: - try: - member = next((member for member in archive.getmembers() if os.path.normpath(member.name) == "lib/liboffice_oxide.a"), None) - if member is None: - return False - library = archive.extractfile(member) - if library is None: - return False - with library: - return b"\x00" + version.encode() + b"\x00" in library.read() - except (KeyError, tarfile.TarError, EOFError): - return False - return True - if filename.endswith(".zip"): - return zipfile.is_zipfile(filename) - if os.path.basename(filename).startswith("stagehand-server-"): - with open(filename, "rb") as file: - return file.read(4) == b"\x7fELF" - return True - - def download_go_models(use_china_mirrors=False): """Download the Go DeepDoc `.ort` weights so a Go dev can run the in-process backend with no further setup. The files are written into the repo's canonical model directory - `internal/rag/res/deepdoc/` (relative to the repo root, one level up from this - script). The Go server auto-discovers that directory via - resolveDeepDocModelDir(), and build.sh --test-native / run.sh default - MODEL_DIR there too — so after this script runs, no MODEL_DIR env needs to - be set. - - Uses hf_hub_download per-file (not snapshot_download) to fetch only the - five Go model files; the `.onnx` siblings are Python-only and skipped. - On China mirrors, route through hf-mirror.com via HF_ENDPOINT. + `internal/rag/res/deepdoc/` (relative to the repo root, one level up from + this script). The Go server auto-discovers that directory, and build.sh + --test-native defaults MODEL_DIR there too. A copy also lands under + ragflow_deps/huggingface.co/InfiniFlow/deepdoc/ to keep the dependency + image's existing build-context layout. """ if use_china_mirrors: os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" @@ -392,11 +226,7 @@ def download_go_models(use_china_mirrors=False): # huggingface-free) without the huggingface_hub dependency installed. from huggingface_hub import hf_hub_download - # Canonical local-dev model dir the Go backend auto-discovers. - repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - target_dir = os.path.join(repo_root, "internal", "rag", "res", "deepdoc") - os.makedirs(target_dir, exist_ok=True) - + target_dir = os.path.join(REPO_ROOT, "internal", "rag", "res", "deepdoc") missing = [] for fname in DEEPDOC_MODEL_FILES: dest = os.path.join(target_dir, fname) @@ -430,7 +260,7 @@ def download_go_models(use_china_mirrors=False): sys.exit(1) # Keep the dependency image's existing huggingface.co build-context layout. - image_dir = os.path.join(repo_root, "ragflow_deps", "huggingface.co", DEEPDOC_REPO) + image_dir = os.path.join(SCRIPT_DIR, "huggingface.co", DEEPDOC_REPO) os.makedirs(image_dir, exist_ok=True) for fname in DEEPDOC_MODEL_FILES: shutil.copyfile(os.path.join(target_dir, fname), os.path.join(image_dir, fname)) @@ -456,22 +286,27 @@ def download_tokenizer_assets(use_china_mirrors=False): # huggingface-free) without the huggingface_hub dependency installed. from huggingface_hub import hf_hub_download - base = os.path.join(os.path.dirname(os.path.abspath(__file__)), "huggingface.co") + base = os.path.join(SCRIPT_DIR, "huggingface.co") missing = [] - for repo_id, filename, _kind in TOKENIZER_ASSETS: - target_dir = os.path.join(base, repo_id) - target_file = os.path.join(target_dir, filename) - if os.path.isfile(target_file) and os.path.getsize(target_file) > 0: - print(f" ✓ {repo_id}/{filename} already present") - continue - os.makedirs(target_dir, exist_ok=True) - print(f"Downloading tokenizer asset {repo_id}/{filename}...") - try: - hf_hub_download(repo_id=repo_id, filename=filename, local_dir=target_dir) - if not os.path.isfile(target_file) or os.path.getsize(target_file) == 0: - raise RuntimeError(f"Empty or missing tokenizer asset: {target_file}") - except Exception as e: # noqa: BLE001 - collected and surfaced below - missing.append((repo_id, filename, e)) + try: + entries = load_tokenizer_assets() + except Exception as e: # noqa: BLE001 - manifest missing/malformed is fatal + missing.append(("", "", e)) + else: + for repo_id, filename, _kind in entries: + target_dir = os.path.join(base, repo_id) + target_file = os.path.join(target_dir, filename) + if os.path.isfile(target_file) and os.path.getsize(target_file) > 0: + print(f" ✓ {repo_id}/{filename} already present") + continue + os.makedirs(target_dir, exist_ok=True) + print(f"Downloading tokenizer asset {repo_id}/{filename}...") + try: + hf_hub_download(repo_id=repo_id, filename=filename, local_dir=target_dir) + if not os.path.isfile(target_file) or os.path.getsize(target_file) == 0: + raise RuntimeError(f"Empty or missing tokenizer asset: {target_file}") + except Exception as e: # noqa: BLE001 - collected and surfaced below + missing.append((repo_id, filename, e)) if missing: for repo_id, filename, e in missing: @@ -494,129 +329,53 @@ def download_tokenizer_assets(use_china_mirrors=False): print(" ✓ embedding tokenizer assets ready") -if __name__ == "__main__": - # Anchor archives, the BPE table, and tokenizer assets to ragflow_deps/. - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support") +def main(): + parser = argparse.ArgumentParser(description="Download the runtime / image resources with optional China mirror support") parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads") args = parser.parse_args() - urls = get_urls(args.china_mirrors) + # Anchor archives, the BPE table, and tokenizer assets to ragflow_deps/. + os.chdir(SCRIPT_DIR) - for url in urls: - download_url = url[0] if isinstance(url, list) else url - filename = url[1] if isinstance(url, list) else url.split("/")[-1] - print(f"Downloading {filename} from {download_url}...") + # ── cl100k_base BPE table ────────────────────────────────────────────── + print("== cl100k_base.tiktoken ==") + cl100k = os.path.join(SCRIPT_DIR, "cl100k_base.tiktoken") + if os.path.isfile(cl100k) and os.path.getsize(cl100k) > 0: + print(" ✓ cl100k_base.tiktoken already present") + else: + print(f"Downloading cl100k_base.tiktoken from {cl100k_url()}...") + download_with_progress(cl100k_url(), cl100k) - # The ONNX Runtime archive is re-issued under the SAME release tag and - # asset name whenever its content changes (e.g. the patched build that - # exports SessionGetInitializer*). A pure existence check would then - # keep a colleague's stale local copy and fail to link onnxruntime_go. - # Verify against the published .sha256 sidecar so a re-issued archive - # is always re-downloaded and re-extracted. - is_ort = filename == _ort_asset_name(ORT_VERSION) - expected_sha = None - if is_ort: - sidecar_url = download_url + ".sha256" - try: - resp = requests.get(sidecar_url, timeout=30) - resp.raise_for_status() - expected_sha = resp.text.split()[0] - except Exception as exc: # noqa: BLE001 - best-effort; fall back to legacy - print(f" WARNING: could not fetch {sidecar_url} ({exc}); skipping checksum for {filename}") - - needs_download = True - if _valid_download(filename): - if expected_sha is not None: - actual = _sha256_of(filename) - if actual == expected_sha: - print(f" ✓ {filename} checksum matches released {expected_sha}; skipping download") - needs_download = False - else: - print(f" {filename} checksum mismatch (local {actual} != released {expected_sha}); re-downloading") - else: - needs_download = False - - if needs_download: - download_with_progress(download_url, filename) - if expected_sha is not None: - actual = _sha256_of(filename) - if actual != expected_sha: - os.unlink(filename) - print(f" ERROR: {filename} checksum mismatch after download (got {actual}, expected {expected_sha})", file=sys.stderr) - sys.exit(1) - print(f" ✓ {filename} checksum verified ({actual})") - if not _valid_download(filename): - os.unlink(filename) - raise RuntimeError(f"Invalid downloaded dependency: {filename}") - # Force re-extract below: drop any previously extracted version dir - # so the same-named re-issued archive actually refreshes the .a files. - if is_ort: - native_libs = os.path.expanduser("~/ragflow-native-libs") - version_dir = os.path.join(native_libs, "onnxruntime", "static_lib", _ort_normalized_dir(ORT_VERSION)) - if os.path.isdir(version_dir): - print(f" Removing stale extracted ONNX Runtime dir: {version_dir}") - shutil.rmtree(version_dir) - - # Extract native static libraries to ~/ragflow-native-libs for Go build. - # Ensures build.sh can find them without network access. - native_deps_dir = os.path.expanduser("~/ragflow-native-libs") - office_version = _go_module_version("github.com/yfedoseev/office_oxide/go") - extractions = [ - ("pdfium-linux-x64-static.tgz", "pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None), - ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide", ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"], None), - ("office_oxide-linux-x86_64.tar.gz", "office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version), - ] - - for archive, subdir, required_files, version in extractions: - archive_path = os.path.join(os.getcwd(), archive) - if not os.path.isfile(archive_path): - print(f" Skipping extraction: {archive} not found") + # ── stagehand-server binaries (image root + local cache) ──────────────── + print("== stagehand-server ==") + for arch in ("x64", "arm64"): + filename = f"stagehand-server-v3-linux-{arch}" + target = os.path.join(SCRIPT_DIR, filename) + if os.path.isfile(target) and os.path.getsize(target) > 0: + print(f" ✓ {filename} already present") continue - target = os.path.join(native_deps_dir, subdir) - try: - _extract_native_archive(archive_path, target, required_files, version) - except (tarfile.TarError, EOFError, RuntimeError): - os.unlink(archive_path) - raise - + print(f"Downloading {filename} from {stagehand_url(arch)}...") + download_with_progress(stagehand_url(arch), target) + if not _valid_stagehand_binary(target): + os.unlink(target) + raise RuntimeError(f"Invalid downloaded dependency: {filename}") _install_stagehand() - if not extract_onnxruntime( - os.path.join(native_deps_dir, "onnxruntime", "static_lib"), - os.path.join(os.getcwd(), _ort_asset_name(ORT_VERSION)), - ORT_VERSION, - ): - # The archive was not downloaded or failed to extract, so no .a landed. - # Fail loud instead of exiting 0: build.sh's ORT guard would otherwise - # reject the build later with a less actionable message, and a missing - # .a left here is exactly the "silent green" this PR is meant to prevent. - print( - f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not " - f"extracted to {os.path.join(native_deps_dir, 'onnxruntime', 'static_lib')}. " - f"Check the download above; build.sh will refuse to link without them.", - file=sys.stderr, - ) - sys.exit(1) - - # ONNX Runtime is statically linked into the server binary, so there is no - # runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will - # find the archives — the .a files live under - # ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend - # resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback. - ort_static_dir = os.path.join(native_deps_dir, "onnxruntime", "static_lib") - ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")] - if ort_a_files: - print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}") - else: - print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr) - sys.exit(1) - - # Download the Go DeepDoc `.ort` weights so this script is a one-stop for Go - # dev: native libs (above) + model files (below) from a single invocation. + # Download the Go DeepDoc `.ort` weights so this script is a one-stop for a + # Go dev's runtime resources: models (below) from a single invocation. + print("== DeepDoc models ==") download_go_models(args.china_mirrors) # Same one-stop idea for the token counters: their tokenizer files, in the layout # the loaders search and ragflow_deps/Dockerfile ships. + print("== tokenizer assets ==") download_tokenizer_assets(args.china_mirrors) + + print() + print(f"✓ Runtime/image resources ready under {SCRIPT_DIR}/") + print(" Build the dependency image with:") + print(f" cd {SCRIPT_DIR} && docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .") + + +if __name__ == "__main__": + main() diff --git a/ragflow_deps/test_download_deps.py b/ragflow_deps/test_download_deps.py deleted file mode 100644 index 6eb44c0e23..0000000000 --- a/ragflow_deps/test_download_deps.py +++ /dev/null @@ -1,253 +0,0 @@ -# Tests for ragflow_deps/download_deps.py ONNX Runtime extraction. -# -# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these -# tests must guarantee the archive is really landed on disk — above all after an -# ORT version bump, where the zip's version-stamped top-level dir changes but -# static_lib/ still exists from the previous run. -# -# The release zip from infiniflow/ragflow-build carries a top-level dir named -# onnxruntime-v{version}-linux-x86_64; extract_onnxruntime() must rename it to -# the build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28 so -# every consumer shares one name convention. These tests pin that rename. - -import http.server -import io -import os -import runpy -import shutil -import sys -import tarfile -import threading -import types -import zipfile -from pathlib import Path - -import download_deps as deps -import pytest -import requests -from download_deps import ( - _ort_asset_name, - _ort_extracted_dir, - _ort_normalized_dir, - extract_onnxruntime, - has_static_archives, -) - - -def make_ort_zip(path, version): - """Build a zip shaped like the infiniflow/ragflow-build ORT release: a - top-level dir named onnxruntime-v{version}-linux-x86_64 holding - lib/libonnxruntime.a. extract_onnxruntime() must rename it to the - build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28.""" - member = f"{_ort_extracted_dir(version)}/lib/libonnxruntime.a" - with zipfile.ZipFile(path, "w") as zf: - zf.writestr(member, b"!\nort-payload") - return path - - -def test_extracts_on_first_run(tmp_path): - # static_lib/ does not exist yet: the plain first-run path. - static_lib = tmp_path / "onnxruntime" / "static_lib" - version = "1.29.0" - archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) - - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - - extracted = static_lib / _ort_extracted_dir(version) - assert not extracted.exists(), "release zip top-level dir must be renamed away" - version_dir = static_lib / _ort_normalized_dir(version) - assert version_dir.is_dir() - assert has_static_archives(str(version_dir)) - - -def test_extracts_after_version_bump_when_static_lib_exists(tmp_path): - """Regression: a static_lib/ left over from a previous version must not - suppress extraction of the bumped version. Pruning the stale dir leaves - static_lib/ in place; if extraction is then skipped no .a is ever landed and - build.sh's ORT guard rejects the build.""" - static_lib = tmp_path / "onnxruntime" / "static_lib" - static_lib.mkdir(parents=True) - stale = static_lib / _ort_normalized_dir("1.28.0") - (stale / "lib").mkdir(parents=True) - (stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort") - - version = "1.29.0" - archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) - - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - - assert not stale.exists(), "stale ORT version should be pruned" - extracted = static_lib / _ort_extracted_dir(version) - assert not extracted.exists(), "release zip top-level dir must be renamed away" - version_dir = static_lib / _ort_normalized_dir(version) - assert version_dir.is_dir(), "bumped ORT version was silently not extracted" - assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a" - - -def test_skips_when_archive_missing(tmp_path): - static_lib = tmp_path / "onnxruntime" / "static_lib" - assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.29.0") is False - - -def test_idempotent_when_matching_version_already_present(tmp_path): - static_lib = tmp_path / "onnxruntime" / "static_lib" - version = "1.29.0" - archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) - - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - before = sorted(os.listdir(static_lib)) - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - assert sorted(os.listdir(static_lib)) == before - - -@pytest.fixture -def download_server(): - class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self): - if self.path == "/missing": - self.send_response(404) - self.end_headers() - self.wfile.write(b"not an artifact") - else: - self.send_response(200) - self.send_header("Content-Length", "100" if self.path == "/interrupted" else "7") - self.end_headers() - self.wfile.write(b"payload") - - def log_message(self, *args): - pass - - server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=server.serve_forever) - thread.start() - try: - yield f"http://127.0.0.1:{server.server_port}" - finally: - server.shutdown() - thread.join() - server.server_close() - - -@pytest.mark.parametrize("endpoint", ["missing", "interrupted"]) -def test_failed_download_preserves_previous_file(tmp_path, download_server, endpoint): - dest = tmp_path / "artifact" - dest.write_bytes(b"previous valid artifact") - with pytest.raises(requests.RequestException): - deps.download_with_progress(f"{download_server}/{endpoint}", dest) - assert dest.read_bytes() == b"previous valid artifact" - assert list(tmp_path.iterdir()) == [dest] - - -def test_successful_download_replaces_previous_file(tmp_path, download_server): - dest = tmp_path / "artifact" - dest.write_bytes(b"old") - deps.download_with_progress(f"{download_server}/complete", dest) - assert dest.read_bytes() == b"payload" - assert list(tmp_path.iterdir()) == [dest] - - -def test_dependency_preparation_recovers_empty_native_directory(tmp_path, monkeypatch): - root = Path(__file__).resolve().parents[1] - context = tmp_path / "ragflow_deps" - context.mkdir() - script = context / "download_deps.py" - shutil.copy(root / "ragflow_deps/download_deps.py", script) - (tmp_path / "go.mod").write_text("""module fixture - -require ( - github.com/yfedoseev/office_oxide/go v0.1.12 - github.com/browserbase/stagehand-go/v3 v3.22.0 -) -""") - native = tmp_path / "native" - required = { - "pdfium-linux-x64-static.tgz": ("pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"]), - "pdf_oxide-go-ffi-linux-amd64.tar.gz": ("pdf_oxide", ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"]), - "office_oxide-linux-x86_64.tar.gz": ("office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"]), - } - for archive_name, (directory, files) in required.items(): - (native / directory).mkdir(parents=True) - with tarfile.open(context / archive_name, "w:gz") as archive: - for name in files: - item = tarfile.TarInfo("./" + name) - payload = b"payload\x000.1.12\x00" if name.endswith("liboffice_oxide.a") else b"payload" - item.size = len(payload) - archive.addfile(item, io.BytesIO(payload)) - (native / "office_oxide/lib").mkdir() - (native / "office_oxide/include/office_oxide_c").mkdir(parents=True) - (native / "office_oxide/lib/liboffice_oxide.a").write_bytes(b"old\x000.1.9\x00") - (native / "office_oxide/include/office_oxide_c/office_oxide.h").write_bytes(b"old") - make_ort_zip(context / _ort_asset_name("1.29.0"), "1.29.0") - for arch in ["x64", "arm64"]: - (context / f"stagehand-server-v3-linux-{arch}").write_bytes(b"\x7fELFpayload") - (context / "cl100k_base.tiktoken").write_bytes(b"configured table") - - def offline(*args, **kwargs): - raise requests.ConnectionError("offline sidecar") - - def hf_download(*, repo_id, filename, local_dir): - Path(local_dir).mkdir(parents=True, exist_ok=True) - (Path(local_dir) / filename).write_bytes(b"model") - - monkeypatch.setitem(sys.modules, "huggingface_hub", types.SimpleNamespace(hf_hub_download=hf_download)) - monkeypatch.setattr(requests, "get", offline) - original_expanduser = os.path.expanduser - monkeypatch.setattr(os.path, "expanduser", lambda path: str(native) if path == "~/ragflow-native-libs" else original_expanduser(path)) - monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) - monkeypatch.setattr(sys, "argv", [str(script)]) - monkeypatch.chdir(tmp_path) - runpy.run_path(str(script), run_name="__main__") - for directory, files in required.values(): - for name in files: - expected = b"payload\x000.1.12\x00" if name.endswith("liboffice_oxide.a") else b"payload" - assert (native / directory / name).read_bytes() == expected - stagehand = tmp_path / "cache/stagehand/lib/go_3.22.0/stagehand-server-v3-linux-x64" - assert stagehand.read_bytes() == b"\x7fELFpayload" - assert os.access(stagehand, os.X_OK) - for filename in ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.res"]: - assert (context / "huggingface.co/InfiniFlow/deepdoc" / filename).read_bytes() == b"model" - - -def test_incomplete_native_archive_preserves_existing_directory(tmp_path): - target = tmp_path / "native" - target.mkdir() - (target / "existing").write_bytes(b"keep") - archive = tmp_path / "incomplete.tar.gz" - with tarfile.open(archive, "w:gz"): - pass - with pytest.raises(RuntimeError, match="missing required native file"): - deps._extract_native_archive(archive, target, ["lib/library.a"]) - assert (target / "existing").read_bytes() == b"keep" - - -@pytest.mark.parametrize("kind", ["traversal", "absolute", "symlink", "hardlink", "fifo", "device"]) -def test_native_archive_rejects_unsafe_members(tmp_path, kind): - target = tmp_path / "native" - target.mkdir() - (target / "existing").write_bytes(b"keep") - outside = tmp_path / "outside" - outside.write_bytes(b"untouched") - archive_path = tmp_path / "unsafe.tar.gz" - with tarfile.open(archive_path, "w:gz") as archive: - library = tarfile.TarInfo("./lib/library.a") - library.size = len(b"payload") - archive.addfile(library, io.BytesIO(b"payload")) - member = tarfile.TarInfo("unexpected") - if kind == "traversal": - member.name = "../outside" - elif kind == "absolute": - member.name = str(outside) - else: - member.type = {"symlink": tarfile.SYMTYPE, "hardlink": tarfile.LNKTYPE, "fifo": tarfile.FIFOTYPE, "device": tarfile.CHRTYPE}[kind] - member.linkname = str(outside) - payload = b"unsafe" if member.isfile() else b"" - member.size = len(payload) - archive.addfile(member, io.BytesIO(payload)) - - with pytest.raises(tarfile.TarError, match="Unsafe native archive member"): - deps._extract_native_archive(archive_path, str(target), ["lib/library.a"]) - - assert outside.read_bytes() == b"untouched" - assert (target / "existing").read_bytes() == b"keep" - assert not (target / "lib/library.a").exists() - assert sorted(path.name for path in tmp_path.iterdir()) == ["native", "outside", "unsafe.tar.gz"] diff --git a/ragflow_deps/test_tokenizer_assets.py b/ragflow_deps/test_tokenizer_assets.py deleted file mode 100644 index cc3ed9c214..0000000000 --- a/ragflow_deps/test_tokenizer_assets.py +++ /dev/null @@ -1,85 +0,0 @@ -# Guard: the tokenizer assets production loads must actually be shipped and pinned. -# -# The Go counters in internal/tokenizer read their vocabulary from -# ragflow_deps/huggingface.co//. Three places have to agree: -# -# 1. ragflow_deps/download_deps.py - fetches the file, and says whether it is -# "runtime" (the counters load it) or "oracle" -# (only scripts/gen_tokenizer_oracle.py does); -# 2. Dockerfile / Dockerfile_base -# - copy the runtime ones into the image at the -# path the counters search; -# 3. internal/tokenizer/*.go - pin the SHA-1 of the runtime ones. -# -# When they drift, the layer that can see it fails loudly: the runtime image does not -# build without a runtime asset, and an embedder whose model declares a counter that -# cannot load refuses to ingest rather than substituting the calibrated cl100k count -# (cl100k under-counts XLM-R on some content, and an under-count is what makes a provider -# answer 400). Those failures are still late - a broken image gets built, or the first -# document fails - which is why the drift is checked here, cheaply, instead. -# See internal/tokenizer/embedding_token_limits.md. - -import ast -import re -from pathlib import Path - -HERE = Path(__file__).resolve().parent -ROOT = HERE.parent -RUNTIME_IMAGES = ("Dockerfile_base", "Dockerfile") -GO_SOURCES = ("spm.go", "wordpiece.go", "bpe.go") -KINDS = ("runtime", "oracle") -DOWNLOADER = "download_deps.py" - - -def load_tokenizer_assets(): - """Read TOKENIZER_ASSETS out of the Go-side downloader without importing it (that - module pulls in huggingface_hub, which a unit test run does not need).""" - source = (HERE / DOWNLOADER).read_text(encoding="utf-8") - for node in ast.parse(source).body: - if isinstance(node, ast.Assign) and any(getattr(target, "id", None) == "TOKENIZER_ASSETS" for target in node.targets): - return [tuple(ast.literal_eval(element)) for element in node.value.elts] - raise AssertionError(f"TOKENIZER_ASSETS not found in {DOWNLOADER}") - - -def image_text(name): - return (ROOT / name).read_text(encoding="utf-8") - - -def test_assets_are_well_formed(): - assets = load_tokenizer_assets() - assert assets, "tokenizer_assets is empty" - for repo, filename, kind in assets: - assert kind in KINDS, f"{repo}/{filename}: unknown kind {kind!r}" - assert "/" in repo and filename, f"{repo}/{filename}: malformed entry" - runtime = [a for a in assets if a[2] == "runtime"] - assert runtime, "no runtime asset: at least one counter would always be unavailable" - - -def test_every_runtime_asset_is_copied_into_every_runtime_image(): - assets = [a for a in load_tokenizer_assets() if a[2] == "runtime"] - for image in RUNTIME_IMAGES: - text = image_text(image) - for repo, filename, _kind in assets: - assert f"{repo}/{filename}" in text, ( - f"{image} does not copy {repo}/{filename}; the matching counter would be unavailable in that image and ingest would silently fall back to the calibrated count" - ) - - -def test_every_runtime_asset_is_pinned_by_the_loader(): - sources = {name: (ROOT / "internal" / "tokenizer" / name).read_text(encoding="utf-8") for name in GO_SOURCES} - for repo, filename, kind in load_tokenizer_assets(): - if kind != "runtime": - continue - key = f"{repo}/{filename}" - pinned = [match.group(1) for text in sources.values() for match in re.finditer(rf'"{re.escape(key)}":\s*"([0-9a-f]{{40}})"', text)] - assert pinned, f"{key} has no 40-hex SHA-1 pin in internal/tokenizer/*.go" - for digest in pinned: - assert len(set(digest)) > 4, f"{key}: pin {digest!r} does not look like a digest" - - -def test_oracle_only_assets_are_not_shipped(): - assets = [a for a in load_tokenizer_assets() if a[2] == "oracle"] - for image in RUNTIME_IMAGES: - text = image_text(image) - for repo, filename, _kind in assets: - assert f"{repo}/{filename}" not in text, f"{image} ships {repo}/{filename}, which only the fixture generator needs" diff --git a/tools/scripts/download_dev_deps.py b/tools/scripts/download_dev_deps.py new file mode 100755 index 0000000000..a8c91a27a0 --- /dev/null +++ b/tools/scripts/download_dev_deps.py @@ -0,0 +1,671 @@ +#!/usr/bin/env python3 + +# PEP 723 metadata +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "requests", +# "huggingface-hub", +# ] +# /// + +# Download the native static libraries the Go build links (pdfium / pdf_oxide / +# office_oxide / onnxruntime) and unpack them into ~/ragflow-native-libs, so +# build.sh can compile the Go server without network access. +# +# This is the LOCAL-DEV one-stop: it also fetches the runtime resources the Go +# server needs to actually RUN, parse documents and count tokens (the cl100k +# BPE table, the DeepDoc `.ort` weights for the in-process PDF/table backend, +# and the embedding tokenizer assets). Run it ONCE and the local environment is +# ready — you do NOT also need download_deps.py for local development. +# +# The image-facing counterpart ragflow_deps/download_deps.py fetches the same +# runtime resources PLUS the stagehand server binary (used by the Browser +# component) and skips the native `.a` files, because the build stage bakes +# those into the server binary. Both scripts land the shared runtime resources +# at the SAME paths, so running one after the other never re-downloads an +# already-present asset. +# +# All version numbers and URLs are defined in the block below; bump a version +# there, not at the call sites. +# +# Typical workflow: +# +# uv run tools/scripts/download_dev_deps.py +# +# Behind the GFW, route GitHub release downloads through a mirror with +# --china-mirrors (or export GH_PROXY=https://gh-proxy.com/). + +import argparse +import hashlib +import os +import re +import shutil +import sys +import tarfile +import tempfile +import zipfile + +import requests + +# ── Dependency versions & URLs (edit here) ──────────────────────────────── +# This script's own copy of the native static-library versions and download URLs +# it fetches. build.sh does NOT grep these (it pins PDFIUM/PDF_OXIDE itself and +# reads ORT from internal/common/environments.go + the Dockerfile ARG, so it +# stays tied to the checked-out source tree and the Docker build needs no COPY +# of this script) — keep the two in sync when bumping a version. + +# ONNX Runtime (statically linked into the Go in-process DeepDoc backend). +# +# Source of the native static archives: infiniflow/ragflow-build (our own +# ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs +# account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is +# `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`. The archive is occasionally +# re-issued under this SAME tag/asset name with patched content (e.g. the +# build that exports SessionGetInitializer*); the downloader detects that via +# a `{asset}.sha256` sidecar and re-downloads/re-extracts, so a stale local +# copy never silently lingers. +ORT_VERSION = "1.29.0" + +# pdf_oxide go-ffi release. Bare module version (the "v" is added by +# pdf_oxide_url); matches the value build.sh uses to validate the embedded +# .a stamp. +PDF_OXIDE_VERSION = "0.3.73" + +# pdfium release, as the numeric build number after "chromium%2F" (the "v" and +# the URL-encoded slash are added by pdfium_url). Matches build.sh's +# PDFIUM_STATIC_VERSION. +PDFIUM_VERSION = "7809" + + +def gh_proxy() -> str: + """GitHub release proxy prefix (gh-proxy.com behind the GFW), else empty.""" + return os.environ.get("GH_PROXY") or "" + + +def go_module_version(module: str, repo_root: str) -> str: + """Read a Go module's pinned version (the "vX.Y.Z" after its require line) + from `repo_root`/go.mod. Raises RuntimeError when go.mod or the module is + missing — the caller cannot link the right library without it.""" + go_mod = os.path.join(repo_root, "go.mod") + if not os.path.isfile(go_mod): + raise RuntimeError(f"go.mod not found at {go_mod}") + with open(go_mod, encoding="utf-8") as file: + match = re.search(rf"{re.escape(module)}\s+v([^\s]+)", file.read()) + if match is None: + raise RuntimeError(f"{module} version not found in {go_mod}") + return match.group(1) + + +# ── URL builders ────────────────────────────────────────────────────────── +# Every download URL is assembled from the constants above via one of these +# helpers, so a version bump touches exactly one constant. + + +def pdf_oxide_url() -> str: + """pdf_oxide go-ffi tarball.""" + return f"{gh_proxy()}https://github.com/yfedoseev/pdf_oxide/releases/download/v{PDF_OXIDE_VERSION}/pdf_oxide-go-ffi-linux-amd64.tar.gz" + + +def pdfium_url() -> str: + """pdfium static library tarball.""" + return f"{gh_proxy()}https://github.com/kognitos/pdfium-static/releases/download/chromium%2F{PDFIUM_VERSION}/pdfium-linux-x64-static.tgz" + + +def office_oxide_url(repo_root: str) -> str: + """office_oxide native tarball. The version is derived from go.mod so it + always matches what the Go build actually links (no second hardcoded copy + to drift).""" + version = go_module_version("github.com/yfedoseev/office_oxide/go", repo_root) + return f"{gh_proxy()}https://github.com/yfedoseev/office_oxide/releases/download/v{version}/native-linux-x86_64.tar.gz" + + +# ── ONNX Runtime release asset helpers ───────────────────────────────────── + + +def ort_asset_name(version: str = ORT_VERSION) -> str: + """Release asset filename under infiniflow/ragflow-build tag onnxruntime-v{version}.""" + return f"onnxruntime-v{version}-linux-x86_64.zip" + + +def ort_url(version: str = ORT_VERSION) -> str: + return f"{gh_proxy()}https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{version}/{ort_asset_name(version)}" + + +def ort_extracted_dir(version: str = ORT_VERSION) -> str: + """Top-level directory name INSIDE the release zip (what extractall creates).""" + return f"onnxruntime-v{version}-linux-x86_64" + + +def ort_normalized_dir(version: str = ORT_VERSION) -> str: + """Directory name build.sh's `find ... -name '*.a'` glob expects under static_lib.""" + return f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + + +def _repo_root() -> str: + """Walk up from this file to the repo root (the directory holding go.mod). + + The script lives at tools/scripts/, two levels below the repo root, so a + fixed parent-count would break if it is ever moved again; searching for + go.mod keeps REPO_ROOT correct regardless of nesting.""" + d = os.path.dirname(os.path.abspath(__file__)) + while d and d != os.path.dirname(d): + if os.path.isfile(os.path.join(d, "go.mod")): + return d + d = os.path.dirname(d) + raise RuntimeError("go.mod not found: cannot locate repo root") + + +REPO_ROOT = _repo_root() +# Downloaded archives are anchored under ragflow_deps/ (ignored by .gitignore +# via *.tgz / ragflow_deps/*.zip) regardless of the caller's working directory, +# so a move of this script does not scatter large files into tools/scripts/. +RAGFLOW_DEPS_DIR = os.path.join(REPO_ROOT, "ragflow_deps") +NATIVE_LIBS_DIR = os.path.expanduser("~/ragflow-native-libs") + + +def _sha256_of(path): + """sha256 of a file, streamed in chunks so large archives don't blow memory.""" + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def download_with_progress(url, filename): + """Download URL to FILENAME atomically (temp file, then move on success). + + A failed or empty download leaves any previous valid file untouched, so an + interrupted run never destroys a working dependency tree.""" + filename = os.fspath(filename) + temporary = None + try: + with requests.get(url, stream=True, timeout=(15, 60)) as response: + response.raise_for_status() + total_size = int(response.headers.get("content-length", 0)) + with tempfile.NamedTemporaryFile(dir=os.path.dirname(os.path.abspath(filename)), delete=False) as file: + temporary = file.name + downloaded = 0 + for data in response.iter_content(1 << 20): + file.write(data) + downloaded += len(data) + if total_size > 0: + progress = (downloaded / total_size) * 100 + sys.stdout.write(f"\rProgress: {progress:.1f}% ({downloaded}/{total_size} bytes)") + sys.stdout.flush() + if downloaded == 0: + raise requests.RequestException(f"Empty dependency download: {url}") + os.replace(temporary, filename) + finally: + if temporary is not None and os.path.exists(temporary): + os.unlink(temporary) + print() + + +def _valid_download(filename): + """True when FILENAME is a non-empty, structurally valid archive of its kind.""" + if not os.path.isfile(filename) or os.path.getsize(filename) == 0: + return False + if filename.endswith((".tar.gz", ".tgz")): + if not tarfile.is_tarfile(filename): + return False + if os.path.basename(filename) == "office_oxide-linux-x86_64.tar.gz": + version = go_module_version("github.com/yfedoseev/office_oxide/go", REPO_ROOT) + with tarfile.open(filename) as archive: + try: + member = next((m for m in archive.getmembers() if os.path.normpath(m.name) == "lib/liboffice_oxide.a"), None) + if member is None: + return False + library = archive.extractfile(member) + if library is None: + return False + with library: + return b"\x00" + version.encode() + b"\x00" in library.read() + except (KeyError, tarfile.TarError, EOFError): + return False + return True + if filename.endswith(".zip"): + return zipfile.is_zipfile(filename) + return True + + +def _extract_native_archive(archive_path, target, required_files, version=None): + """Extract ARCHIVE_PATH into TARGET, verifying REQUIRED_FILES (and, for .a + files, an optional null-delimited VERSION stamp) before promoting the + staged tree. + + Path-traversal / absolute / non-regular members are rejected before + extraction, and an incomplete archive raises instead of replacing a + previously good directory.""" + + def complete(directory): + for name in required_files: + path = os.path.join(directory, name) + if not os.path.isfile(path) or os.path.getsize(path) == 0: + return False + if version is not None and name.endswith(".a"): + with open(path, "rb") as file: + if b"\x00" + version.encode() + b"\x00" not in file.read(): + return False + return True + + if complete(target): + print(f" ✓ {os.path.basename(target)} already extracted to {target}") + return + os.makedirs(os.path.dirname(target), exist_ok=True) + staging = tempfile.mkdtemp(dir=os.path.dirname(target)) + try: + with tarfile.open(archive_path) as archive: + members = archive.getmembers() + for member in members: + if os.path.isabs(member.name) or ".." in member.name.split("/") or not (member.isfile() or member.isdir()): + raise tarfile.TarError(f"Unsafe native archive member: {member.name}") + archive.extractall(staging, members=members) + if not complete(staging): + raise RuntimeError(f"{archive_path} is missing required native files or version {version}") + if os.path.isdir(target): + shutil.rmtree(target) + os.replace(staging, target) + staging = None + finally: + # No-op after a successful replace (the dir was renamed away); cleans + # up the partial tree when extraction or validation failed. + if staging is not None: + shutil.rmtree(staging, ignore_errors=True) + print(f" ✓ extracted {os.path.basename(archive_path)} to {target}") + + +def prune_stale_onnxruntime(static_lib_dir, version): + """Remove ONNX Runtime version dirs under static_lib that do NOT match + `version`. Without this, a version bump leaves the stale dir next to + the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate + symbols / wrong version, silently).""" + if not os.path.isdir(static_lib_dir): + return + expected = ort_normalized_dir(version) + for name in os.listdir(static_lib_dir): + if not name.startswith("onnxruntime-linux-x64-static_lib-"): + continue + if name == expected: + continue + stale = os.path.join(static_lib_dir, name) + print(f" Removing stale ONNX Runtime dir: {stale}") + shutil.rmtree(stale) + + +def has_static_archives(directory): + """True when `directory` holds at least one static archive (.a).""" + return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files) + + +def extract_onnxruntime(static_lib_dir, archive_path, version): + """Ensure the ONNX Runtime static archives for `version` sit under + `static_lib_dir`. Returns True when that version is available afterwards + (extracted now or already present), False when the archive is missing. + + The infiniflow/ragflow-build release zip carries a top-level dir named + onnxruntime-v{version}-linux-x86_64, so a present + `static_lib_dir` is NOT evidence that THIS version is extracted: after a + version bump the stale dir is pruned and the new one must be extracted. + """ + if not os.path.isfile(archive_path): + print(f" Skipping extraction: {os.path.basename(archive_path)} not found") + return False + prune_stale_onnxruntime(static_lib_dir, version) + version_dir = os.path.join(static_lib_dir, ort_normalized_dir(version)) + if os.path.isdir(version_dir) and has_static_archives(version_dir): + print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}") + return True + os.makedirs(static_lib_dir, exist_ok=True) + print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}") + with zipfile.ZipFile(archive_path) as zf: + zf.extractall(static_lib_dir) + # The release zip's top-level dir is onnxruntime-v{version}-linux-x86_64, but + # build.sh's glob and the stale checks above all expect + # onnxruntime-linux-x64-static_lib-{version}-glibc2_28. Rename it so every + # consumer shares one name convention (driven by ORT_VERSION). + extracted = os.path.join(static_lib_dir, ort_extracted_dir(version)) + normalized = os.path.join(static_lib_dir, ort_normalized_dir(version)) + if os.path.isdir(extracted) and extracted != normalized: + if os.path.exists(normalized): + shutil.rmtree(normalized) + print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}") + os.rename(extracted, normalized) + return True + + +def _fetch_ort_sidecar_sha(url): + """Fetch the published `.sha256` sidecar for the ORT archive. + + The infiniflow/ragflow-build zip is occasionally re-issued under the SAME + tag/asset name with patched content, so a stale local copy would link the + wrong build. Returns the expected digest, or None when the sidecar cannot + be fetched (best-effort; the downloader then falls back to the legacy + existence check).""" + try: + resp = requests.get(url, timeout=30) + resp.raise_for_status() + return resp.text.split()[0] + except Exception as exc: # noqa: BLE001 - best-effort; fall back to legacy + print(f" WARNING: could not fetch {url} ({exc}); skipping checksum") + return None + + +def _download_ort_archive(archive_path): + """Download the ORT release zip, verifying it against the published + .sha256 sidecar so a re-issued archive is always refreshed. + + A (re-)download drops any previously extracted version dir, so the + same-named re-issued archive actually refreshes the .a files instead of + being masked by the extraction-time idempotence check.""" + url = ort_url() + expected_sha = _fetch_ort_sidecar_sha(url + ".sha256") + if os.path.isfile(archive_path) and os.path.getsize(archive_path) > 0: + if expected_sha is not None: + actual = _sha256_of(archive_path) + if actual == expected_sha: + print(f" ✓ {os.path.basename(archive_path)} checksum matches released {expected_sha}; skipping download") + return + print(f" {os.path.basename(archive_path)} checksum mismatch (local {actual} != released {expected_sha}); re-downloading") + elif _valid_download(archive_path): + return + download_with_progress(url, archive_path) + if expected_sha is not None: + actual = _sha256_of(archive_path) + if actual != expected_sha: + os.unlink(archive_path) + print(f" ERROR: {os.path.basename(archive_path)} checksum mismatch after download (got {actual}, expected {expected_sha})", file=sys.stderr) + sys.exit(1) + print(f" ✓ {os.path.basename(archive_path)} checksum verified ({actual})") + if not _valid_download(archive_path): + os.unlink(archive_path) + raise RuntimeError(f"Invalid downloaded dependency: {os.path.basename(archive_path)}") + version_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib", ort_normalized_dir()) + if os.path.isdir(version_dir): + print(f" Removing stale extracted ONNX Runtime dir: {version_dir}") + shutil.rmtree(version_dir) + + +# ── Runtime resources the local Go dev server also needs ─────────────────── +# The build-facing native `.a` libs above let the server LINK; these let it +# RUN, parse documents and count tokens. Fetched here too so a developer runs +# THIS script once and the environment is ready. Landing paths match +# ragflow_deps/download_deps.py, so re-running that image script never +# re-downloads an already-present asset. stagehand is deliberately NOT fetched +# here (Browser component only) — that is download_deps.py's job for images. + + +# cl100k BPE table. Hosted on Azure blob storage (NOT a GitHub release), so the +# GitHub mirror prefix does not apply and is deliberately omitted. +def cl100k_url() -> str: + return "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" + + +HF_DEEPDOC_REPO = "InfiniFlow/deepdoc" +HF_DEEPDOC_MODEL_FILES = ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.res"] + + +# Embedding tokenizer assets the Go counters load. Inlined here (no separate +# manifest file) as the single source of truth this script reads directly; the +# image script ragflow_deps/download_deps.py carries its own copy (no shared +# module, by design). One entry per tuple: (repo, file, kind). +# runtime - the Go counters load it in production; the runtime images must ship +# it and the loader must pin its SHA-1. +# oracle - only scripts/gen_tokenizer_oracle.py needs it to regenerate the +# test fixtures, so it is deliberately NOT shipped to the image. +TOKENIZER_ASSETS = [ + ("BAAI/bge-m3", "sentencepiece.bpe.model", "runtime"), + ("BAAI/bge-large-en-v1.5", "vocab.txt", "runtime"), + ("Qwen/Qwen3-Embedding-0.6B", "tokenizer.json", "runtime"), + ("intfloat/e5-mistral-7b-instruct", "tokenizer.json", "runtime"), + ("BAAI/bge-m3", "tokenizer.json", "oracle"), + ("BAAI/bge-large-en-v1.5", "tokenizer.json", "oracle"), +] + + +# Single huggingface_hub import seam, so the module stays importable (and its +# unit tests stay huggingface-free) without the dep installed. +def _hf_download(repo_id, filename, local_dir): + """Lazy huggingface_hub import + download into LOCAL_DIR.""" + from huggingface_hub import hf_hub_download + + return hf_hub_download(repo_id=repo_id, filename=filename, local_dir=local_dir) + + +def download_cl100k(): + """Fetch the cl100k BPE table into ragflow_deps/ (skip if already present).""" + target = os.path.join(RAGFLOW_DEPS_DIR, "cl100k_base.tiktoken") + if os.path.isfile(target) and os.path.getsize(target) > 0: + print(" ✓ cl100k_base.tiktoken already present") + return + print(f"Downloading cl100k_base.tiktoken from {cl100k_url()}...") + download_with_progress(cl100k_url(), target) + + +def load_tokenizer_assets(assets=None): + """Validate and return the inlined tokenizer asset list as (repo, file, kind) + tuples. When ASSETS is None, uses the module-level TOKENIZER_ASSETS. + + Fetched per file, not by snapshot: these repos also carry multi-GB weights + we do not want. A malformed entry (bad kind, malformed repo, or an empty + list) fails dependency preparation loudly instead of handing a broken tree + to the image build.""" + items = TOKENIZER_ASSETS if assets is None else assets + result = [] + for repo, filename, kind in items: + if kind not in ("runtime", "oracle"): + raise RuntimeError(f"unknown tokenizer asset kind {kind!r} for {repo}/{filename}") + if "/" not in repo or not filename: + raise RuntimeError(f"malformed tokenizer asset repo/file {repo}/{filename}") + result.append((repo, filename, kind)) + if not result: + raise RuntimeError("no tokenizer assets listed") + return result + + +def download_go_models(use_china_mirrors=False): + """Download the Go DeepDoc `.ort` weights so a Go dev can run the in-process + backend with no further setup. + + Written into the repo's canonical model directory + internal/rag/res/deepdoc/ (one level up from ragflow_deps/). The Go server + auto-discovers that directory, and build.sh --test-native defaults MODEL_DIR + there too. A copy also lands under ragflow_deps/huggingface.co/InfiniFlow/deepdoc/ + to keep the dependency image's existing build-context layout.""" + if use_china_mirrors: + os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" + target_dir = os.path.join(REPO_ROOT, "internal", "rag", "res", "deepdoc") + missing = [] + for fname in HF_DEEPDOC_MODEL_FILES: + dest = os.path.join(target_dir, fname) + if os.path.isfile(dest) and os.path.getsize(dest) > 0: + print(f" ✓ {fname} already present") + continue + print(f"Downloading deepdoc model {fname}...") + try: + _hf_download(repo_id=HF_DEEPDOC_REPO, filename=fname, local_dir=target_dir) + if not os.path.isfile(dest) or os.path.getsize(dest) == 0: + raise RuntimeError(f"Empty or missing model: {dest}") + except Exception as e: # noqa: BLE001 - collected and surfaced below + missing.append((fname, e)) + if missing: + for fname, e in missing: + print(f" ERROR: failed to download {fname}: {e}", file=sys.stderr) + print( + "\nThe Go in-process DeepDoc backend loads ONLY the .ort weights listed in\n" + "HF_DEEPDOC_MODEL_FILES. They are fetched from " + f"{HF_DEEPDOC_REPO} into:\n {target_dir}\n" + "Without them the backend cannot serve — the server exits with a fatal\n" + '"no in-process DeepDoc backend serving". To recover:\n' + " - re-run this script (a transient HF/network error usually clears);\n" + " - behind the GFW, re-run with --china-mirrors (routes via hf-mirror.com);\n" + " - or copy the missing files into that directory by hand.", + file=sys.stderr, + ) + sys.exit(1) + # Keep the dependency image's existing huggingface.co build-context layout. + image_dir = os.path.join(RAGFLOW_DEPS_DIR, "huggingface.co", HF_DEEPDOC_REPO) + os.makedirs(image_dir, exist_ok=True) + for fname in HF_DEEPDOC_MODEL_FILES: + shutil.copyfile(os.path.join(target_dir, fname), os.path.join(image_dir, fname)) + print(f" ✓ Go DeepDoc models ready under {target_dir}") + + +def download_tokenizer_assets(use_china_mirrors=False): + """Download the embedding tokenizer assets the Go counters load, into + ragflow_deps/huggingface.co// (the path internal/tokenizer + searches and the one ragflow_deps/Dockerfile builds its image from). + + A missing asset is fatal: models declaring its tokenizer refuse ingestion.""" + if use_china_mirrors: + os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" + base = os.path.join(RAGFLOW_DEPS_DIR, "huggingface.co") + missing = [] + try: + entries = load_tokenizer_assets() + except Exception as e: # noqa: BLE001 - manifest missing/malformed is fatal + missing.append(("", "", e)) + else: + for repo_id, filename, _kind in entries: + target_dir = os.path.join(base, repo_id) + target_file = os.path.join(target_dir, filename) + if os.path.isfile(target_file) and os.path.getsize(target_file) > 0: + print(f" ✓ {repo_id}/{filename} already present") + continue + os.makedirs(target_dir, exist_ok=True) + print(f"Downloading tokenizer asset {repo_id}/{filename}...") + try: + _hf_download(repo_id=repo_id, filename=filename, local_dir=target_dir) + if not os.path.isfile(target_file) or os.path.getsize(target_file) == 0: + raise RuntimeError(f"Empty or missing tokenizer asset: {target_file}") + except Exception as e: # noqa: BLE001 - collected and surfaced below + missing.append((repo_id, filename, e)) + if missing: + for repo_id, filename, e in missing: + print(f" ERROR: could not fetch {repo_id}/{filename}: {e}", file=sys.stderr) + print( + "\nThe embedding tokenizer counters in internal/tokenizer load these files from\n" + f" {base}\n" + "Without one, models that declare its tokenizer cannot be ingested.\n" + "The runtime images refuse to build on the same condition, so this script fails\n" + "instead of handing a broken tree to the image build. To recover:\n" + " - re-run this script (a transient HF/network error usually clears);\n" + " - behind the GFW, re-run with --china-mirrors (routes via hf-mirror.com);\n" + " - or point MODEL_ASSETS_DIR at a directory holding the same layout.\n", + file=sys.stderr, + ) + sys.exit(1) + print(" ✓ embedding tokenizer assets ready") + + +def main(): + """Download the native static libraries the Go build links, then the runtime + resources (cl100k BPE table, DeepDoc .ort weights, embedding tokenizer assets) + the local Go dev server needs to run, parse and count tokens. + + This is the local-dev one-stop: one invocation readies the environment. The + Browser component's server binary is intentionally NOT fetched here — that + is the image script's job. Run `uv run ragflow_deps/download_deps.py` for the + full image resources (it adds that binary and skips the native .a, which the + build stage bakes into the binary).""" + parser = argparse.ArgumentParser(description="Download the native static libraries and runtime resources the local Go dev server needs") + parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads") + args = parser.parse_args() + if args.china_mirrors: + os.environ["GH_PROXY"] = "https://gh-proxy.com/" + + # Anchor the downloaded archives to ragflow_deps/ regardless of the + # caller's working directory (build.sh invokes this from the repo root). + os.chdir(RAGFLOW_DEPS_DIR) + + # ONNX Runtime archive (+ checksum sidecar), then the other native archives. + _download_ort_archive(os.path.join(RAGFLOW_DEPS_DIR, ort_asset_name())) + archives = [ + (pdfium_url(), "pdfium-linux-x64-static.tgz"), + (pdf_oxide_url(), "pdf_oxide-go-ffi-linux-amd64.tar.gz"), + (office_oxide_url(REPO_ROOT), "office_oxide-linux-x86_64.tar.gz"), + ] + for url, filename in archives: + print(f"Downloading {filename} from {url}...") + if _valid_download(filename): + print(f" ✓ {filename} already present") + continue + download_with_progress(url, filename) + if not _valid_download(filename): + os.unlink(filename) + raise RuntimeError(f"Invalid downloaded dependency: {filename}") + + # Extract native static libraries to ~/ragflow-native-libs for the Go build. + # Ensures build.sh can find them without network access. + office_version = go_module_version("github.com/yfedoseev/office_oxide/go", REPO_ROOT) + extractions = [ + ("pdfium-linux-x64-static.tgz", "pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None), + ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide", ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"], None), + ("office_oxide-linux-x86_64.tar.gz", "office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version), + ] + + for archive, subdir, required_files, version in extractions: + archive_path = os.path.join(RAGFLOW_DEPS_DIR, archive) + if not os.path.isfile(archive_path): + print(f" Skipping extraction: {archive} not found") + continue + target = os.path.join(NATIVE_LIBS_DIR, subdir) + try: + _extract_native_archive(archive_path, target, required_files, version) + except (tarfile.TarError, EOFError, RuntimeError): + os.unlink(archive_path) + raise + + if not extract_onnxruntime( + os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib"), + os.path.join(RAGFLOW_DEPS_DIR, ort_asset_name()), + ORT_VERSION, + ): + # The archive was not downloaded or failed to extract, so no .a landed. + # Fail loud instead of exiting 0: build.sh's ORT guard would otherwise + # reject the build later with a less actionable message, and a missing + # .a left here is exactly the "silent green" this split must prevent. + print( + f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not " + f"extracted to {os.path.join(NATIVE_LIBS_DIR, 'onnxruntime', 'static_lib')}. " + f"Check the download above; build.sh will refuse to link without them.", + file=sys.stderr, + ) + sys.exit(1) + + # ONNX Runtime is statically linked into the server binary, so there is no + # runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will + # find the archives — the .a files live under + # ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend + # resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback. + ort_static_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib") + ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")] + if ort_a_files: + print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}") + else: + print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr) + sys.exit(1) + + # The Go server also needs the runtime resources (cl100k BPE table, the + # DeepDoc .ort weights for in-process PDF/table parsing, the embedding + # tokenizer assets for token counting) to actually RUN and ingest — not + # just link. Fetch them here too so a local developer runs THIS script + # once and the environment is ready. The Browser component's server binary + # is fetched by the image script, not here. + print("== cl100k_base.tiktoken ==") + download_cl100k() + print("== DeepDoc models ==") + download_go_models(args.china_mirrors) + print("== tokenizer assets ==") + download_tokenizer_assets(args.china_mirrors) + + print() + print(f"✓ Native build libraries ready under {NATIVE_LIBS_DIR}/") + print(" Run: bash build.sh (it will find them without network access).") + print(" Runtime resources (cl100k / DeepDoc .ort / tokenizer) are also in place,") + print(" so the local Go server is ready to build, run, parse and count tokens.") + + +if __name__ == "__main__": + main() diff --git a/tools/scripts/test_download_dev_deps.py b/tools/scripts/test_download_dev_deps.py new file mode 100644 index 0000000000..9c3047d415 --- /dev/null +++ b/tools/scripts/test_download_dev_deps.py @@ -0,0 +1,369 @@ +# Tests for the local-dev dependency downloader (tools/scripts/download_dev_deps.py): +# the native static libraries (pdfium / pdf_oxide / office_oxide / onnxruntime) +# the Go build links, PLUS the runtime resources (cl100k / DeepDoc .ort / embedding +# tokenizer assets) the local Go server needs to run, parse and count tokens. +# +# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these +# tests must guarantee the archives are really landed on disk — above all after +# an ORT version bump, where the zip's version-stamped top-level dir changes but +# static_lib/ still exists from the previous run. +# +# The release zip from infiniflow/ragflow-build carries a top-level dir named +# onnxruntime-v{version}-linux-x86_64; extract_onnxruntime() must rename it to the +# build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28 so every +# consumer shares one name convention. These tests pin that rename, the stale-dir +# pruning, the archive member sanitization, and the atomic download behaviour. + +import importlib.util +import io +import os +import tarfile +import threading +import zipfile +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +import download_dev_deps as native +import pytest +import requests + + +def make_ort_zip(path, version): + """Build a zip shaped like the infiniflow/ragflow-build ORT release: a + top-level dir named onnxruntime-v{version}-linux-x86_64 holding + lib/libonnxruntime.a. extract_onnxruntime() must rename it to the + build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28.""" + member = f"{native.ort_extracted_dir(version)}/lib/libonnxruntime.a" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(member, b"!\nort-payload") + return path + + +def test_extracts_on_first_run(tmp_path): + # static_lib/ does not exist yet: the plain first-run path. + static_lib = tmp_path / "onnxruntime" / "static_lib" + version = "1.29.0" + archive = make_ort_zip(tmp_path / native.ort_asset_name(version), version) + + assert native.extract_onnxruntime(str(static_lib), str(archive), version) is True + + extracted = static_lib / native.ort_extracted_dir(version) + assert not extracted.exists(), "release zip top-level dir must be renamed away" + version_dir = static_lib / native.ort_normalized_dir(version) + assert version_dir.is_dir() + assert native.has_static_archives(str(version_dir)) + + +def test_extracts_after_version_bump_when_static_lib_exists(tmp_path): + """Regression: a static_lib/ left over from a previous version must not + suppress extraction of the bumped version. Pruning the stale dir leaves + static_lib/ in place; if extraction is then skipped no .a is ever landed and + build.sh's ORT guard rejects the build.""" + static_lib = tmp_path / "onnxruntime" / "static_lib" + static_lib.mkdir(parents=True) + stale = static_lib / native.ort_normalized_dir("1.28.0") + (stale / "lib").mkdir(parents=True) + (stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort") + + version = "1.29.0" + archive = make_ort_zip(tmp_path / native.ort_asset_name(version), version) + + assert native.extract_onnxruntime(str(static_lib), str(archive), version) is True + + assert not stale.exists(), "stale ORT version should be pruned" + extracted = static_lib / native.ort_extracted_dir(version) + assert not extracted.exists(), "release zip top-level dir must be renamed away" + version_dir = static_lib / native.ort_normalized_dir(version) + assert version_dir.is_dir(), "bumped ORT version was silently not extracted" + assert native.has_static_archives(str(version_dir)), "bumped ORT version landed no .a" + + +def test_skips_when_archive_missing(tmp_path): + static_lib = tmp_path / "onnxruntime" / "static_lib" + assert native.extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.29.0") is False + + +def test_idempotent_when_matching_version_already_present(tmp_path): + static_lib = tmp_path / "onnxruntime" / "static_lib" + version = "1.29.0" + archive = make_ort_zip(tmp_path / native.ort_asset_name(version), version) + + assert native.extract_onnxruntime(str(static_lib), str(archive), version) is True + before = sorted(os.listdir(static_lib)) + assert native.extract_onnxruntime(str(static_lib), str(archive), version) is True + assert sorted(os.listdir(static_lib)) == before + + +@pytest.fixture +def download_server(): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/missing": + self.send_response(404) + self.end_headers() + self.wfile.write(b"not an artifact") + else: + self.send_response(200) + self.send_header("Content-Length", "100" if self.path == "/interrupted" else "7") + self.end_headers() + self.wfile.write(b"payload") + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join() + server.server_close() + + +@pytest.mark.parametrize("endpoint", ["missing", "interrupted"]) +def test_failed_download_preserves_previous_file(tmp_path, download_server, endpoint): + dest = tmp_path / "artifact" + dest.write_bytes(b"previous valid artifact") + with pytest.raises(requests.RequestException): + native.download_with_progress(f"{download_server}/{endpoint}", dest) + assert dest.read_bytes() == b"previous valid artifact" + assert list(tmp_path.iterdir()) == [dest] + + +def test_successful_download_replaces_previous_file(tmp_path, download_server): + dest = tmp_path / "artifact" + dest.write_bytes(b"old") + native.download_with_progress(f"{download_server}/complete", dest) + assert dest.read_bytes() == b"payload" + assert list(tmp_path.iterdir()) == [dest] + + +def test_incomplete_native_archive_preserves_existing_directory(tmp_path): + target = tmp_path / "native" + target.mkdir() + (target / "existing").write_bytes(b"keep") + archive = tmp_path / "incomplete.tar.gz" + with tarfile.open(archive, "w:gz"): + pass + with pytest.raises(RuntimeError, match="missing required native file"): + native._extract_native_archive(archive, target, ["lib/library.a"]) + assert (target / "existing").read_bytes() == b"keep" + + +@pytest.mark.parametrize("kind", ["traversal", "absolute", "symlink", "hardlink", "fifo", "device"]) +def test_native_archive_rejects_unsafe_members(tmp_path, kind): + target = tmp_path / "native" + target.mkdir() + (target / "existing").write_bytes(b"keep") + outside = tmp_path / "outside" + outside.write_bytes(b"untouched") + archive_path = tmp_path / "unsafe.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + library = tarfile.TarInfo("./lib/library.a") + library.size = len(b"payload") + archive.addfile(library, io.BytesIO(b"payload")) + member = tarfile.TarInfo("unexpected") + if kind == "traversal": + member.name = "../outside" + elif kind == "absolute": + member.name = str(outside) + else: + member.type = {"symlink": tarfile.SYMTYPE, "hardlink": tarfile.LNKTYPE, "fifo": tarfile.FIFOTYPE, "device": tarfile.CHRTYPE}[kind] + member.linkname = str(outside) + payload = b"unsafe" if member.isfile() else b"" + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + + with pytest.raises(tarfile.TarError, match="Unsafe native archive member"): + native._extract_native_archive(archive_path, str(target), ["lib/library.a"]) + + assert outside.read_bytes() == b"untouched" + assert (target / "existing").read_bytes() == b"keep" + assert not (target / "lib/library.a").exists() + assert sorted(path.name for path in tmp_path.iterdir()) == ["native", "outside", "unsafe.tar.gz"] + + +# ── Local-dev one-stop: the dev downloader must also fetch the runtime +# resources (cl100k / DeepDoc .ort / tokenizer assets) so a developer runs THIS +# script ONCE and the Go server is ready to build, run, parse and count — but it +# must NOT fetch stagehand (that is the image script's job). ────────────────── + + +@pytest.fixture +def native_dirs(tmp_path, monkeypatch): + """Point the dev script's repo-anchored paths at a temp tree so the + runtime-resource downloads written by tests never touch the real repo.""" + deps = tmp_path / "ragflow_deps" + deps.mkdir() + root = tmp_path / "repo" + root.mkdir() + monkeypatch.setattr(native, "RAGFLOW_DEPS_DIR", str(deps)) + monkeypatch.setattr(native, "REPO_ROOT", str(root)) + return deps, root + + +def _fake_hf(repo_id, filename, local_dir): + """Stand-in for huggingface_hub.hf_hub_download that writes a real file so + downstream copy/validate steps behave like the real downloader.""" + os.makedirs(local_dir, exist_ok=True) + with open(os.path.join(local_dir, filename), "wb") as f: + f.write(b"hf-payload") + + +def test_download_cl100k_skips_when_present(native_dirs, download_server, monkeypatch): + deps, _ = native_dirs + existing = deps / "cl100k_base.tiktoken" + existing.write_bytes(b"previous cl100k") + monkeypatch.setattr(native, "cl100k_url", lambda: f"{download_server}/complete") + native.download_cl100k() + assert existing.read_bytes() == b"previous cl100k" + + +def test_download_cl100k_downloads_when_missing(native_dirs, download_server, monkeypatch): + deps, _ = native_dirs + monkeypatch.setattr(native, "cl100k_url", lambda: f"{download_server}/complete") + native.download_cl100k() + assert (deps / "cl100k_base.tiktoken").read_bytes() == b"payload" + + +def test_download_go_models_skips_when_present(native_dirs, monkeypatch): + _, root = native_dirs + deepdoc_dir = root / "internal" / "rag" / "res" / "deepdoc" + deepdoc_dir.mkdir(parents=True) + for fname in native.HF_DEEPDOC_MODEL_FILES: + (deepdoc_dir / fname).write_bytes(b"already here") + fake = mock.MagicMock(side_effect=_fake_hf) + monkeypatch.setattr(native, "_hf_download", fake) + native.download_go_models() + # Already-present files means hf_hub_download is never called. + assert fake.call_count == 0 + + +def test_download_go_models_downloads_when_missing(native_dirs, monkeypatch): + deps, root = native_dirs + monkeypatch.setattr(native, "_hf_download", _fake_hf) + native.download_go_models() + deepdoc_dir = root / "internal" / "rag" / "res" / "deepdoc" + for fname in native.HF_DEEPDOC_MODEL_FILES: + assert (deepdoc_dir / fname).is_file(), f"missing DeepDoc model {fname}" + # The image-layout copy under ragflow_deps/huggingface.co must also exist. + for fname in native.HF_DEEPDOC_MODEL_FILES: + assert (deps / "huggingface.co" / native.HF_DEEPDOC_REPO / fname).is_file() + + +def test_download_go_models_fails_loud(native_dirs, monkeypatch): + monkeypatch.setattr( + native, + "_hf_download", + lambda repo_id, filename, local_dir: (_ for _ in ()).throw(RuntimeError("hf down")), + ) + with pytest.raises(SystemExit) as exc: + native.download_go_models() + assert exc.value.code == 1 + + +def _seed_tokenizer_targets(deps, present): + """Create (present=True) or leave absent the on-disk targets the inlined + TOKENIZER_ASSETS list resolves to under ragflow_deps/huggingface.co//.""" + for repo, filename, _kind in native.TOKENIZER_ASSETS: + target = deps / "huggingface.co" / repo / filename + if present: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"already here") + + +def test_download_tokenizer_assets_skips_when_present(native_dirs, monkeypatch): + deps, _ = native_dirs + _seed_tokenizer_targets(deps, present=True) + fake = mock.MagicMock(side_effect=_fake_hf) + monkeypatch.setattr(native, "_hf_download", fake) + native.download_tokenizer_assets() + assert fake.call_count == 0 + + +def test_download_tokenizer_assets_downloads_when_missing(native_dirs, monkeypatch): + deps, _ = native_dirs + monkeypatch.setattr(native, "_hf_download", _fake_hf) + native.download_tokenizer_assets() + for repo, filename, _kind in native.TOKENIZER_ASSETS: + assert (deps / "huggingface.co" / repo / filename).is_file(), f"missing {repo}/{filename}" + + +def test_download_tokenizer_assets_fails_loud(native_dirs, monkeypatch): + monkeypatch.setattr( + native, + "_hf_download", + lambda repo_id, filename, local_dir: (_ for _ in ()).throw(RuntimeError("hf down")), + ) + with pytest.raises(SystemExit) as exc: + native.download_tokenizer_assets() + assert exc.value.code == 1 + + +def test_download_tokenizer_assets_fails_loud_on_bad_list(native_dirs, monkeypatch): + # load_tokenizer_assets raises on a malformed inlined entry (bad kind); + # download_tokenizer_assets must surface it as SystemExit 1 rather than + # leaking a raw traceback or handing a broken tree to the image build. + monkeypatch.setattr( + native, + "load_tokenizer_assets", + lambda: (_ for _ in ()).throw(RuntimeError("unknown tokenizer asset kind 'bogus'")), + ) + with pytest.raises(SystemExit) as exc: + native.download_tokenizer_assets() + assert exc.value.code == 1 + + +def test_load_tokenizer_assets_returns_inlined_list(): + # The asset list is inlined (no separate manifest file); it must match the + # six entries the runtime images ship and the Go loader pins. + assert native.load_tokenizer_assets() == [ + ("BAAI/bge-m3", "sentencepiece.bpe.model", "runtime"), + ("BAAI/bge-large-en-v1.5", "vocab.txt", "runtime"), + ("Qwen/Qwen3-Embedding-0.6B", "tokenizer.json", "runtime"), + ("intfloat/e5-mistral-7b-instruct", "tokenizer.json", "runtime"), + ("BAAI/bge-m3", "tokenizer.json", "oracle"), + ("BAAI/bge-large-en-v1.5", "tokenizer.json", "oracle"), + ] + + +def test_load_tokenizer_assets_rejects_bad_kind(): + with pytest.raises(RuntimeError, match="kind"): + native.load_tokenizer_assets([("X/y.bin", "f.bin", "bogus")]) + + +def test_load_tokenizer_assets_rejects_empty(): + with pytest.raises(RuntimeError, match="no tokenizer assets"): + native.load_tokenizer_assets([]) + + +def test_load_tokenizer_assets_rejects_malformed_repo(): + with pytest.raises(RuntimeError, match="malformed"): + native.load_tokenizer_assets([("nobadrepo", "f.bin", "runtime")]) + + +def test_native_script_does_not_fetch_stagehand(): + # Partition contract: the local-dev one-stop must not pull the Browser + # component's stagehand binary — that is download_deps.py's job for images. + assert not hasattr(native, "stagehand_url") + assert not hasattr(native, "_install_stagehand") + assert "stagehand" not in native.main.__doc__.lower() + + +# ── Image script (download_deps.py) must stay runtime-resources only: it must +# NOT handle the native .a archives (the build stage bakes those into the +# binary). Imported by path inside the test so the import lives next to the +# native script and never touches module import timing. ────────────────────── + + +def test_image_script_does_not_handle_native_archives(): + deps_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "ragflow_deps", "download_deps.py") + assert os.path.isfile(deps_path), f"download_deps.py not found at {deps_path}" + spec = importlib.util.spec_from_file_location("download_deps_img", deps_path) + img = importlib.util.module_from_spec(spec) + spec.loader.exec_module(img) + assert not hasattr(img, "extract_onnxruntime") + assert not hasattr(img, "_extract_native_archive") + assert "onnxruntime" not in (img.__doc__ or "").lower()