mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 21:32:43 +08:00
refactor(deps): use Go dependency downloader as sole entrypoint (#20360)
This commit is contained in:
1 parent
435da41959
commit
4071451ea4
40 files changed
+838
-984
No files matched your search
@@ -180,7 +180,7 @@ ORACLES = {
|
||||
# the served behaviour comes from tokenizer.json above.
|
||||
SPM_MODEL = "huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model"
|
||||
|
||||
# The cl100k table as shipped in ragflow_deps (the OpenAI blob download_deps.py fetches).
|
||||
# The cl100k table as provisioned by download_deps.py or build.sh in ragflow_deps.
|
||||
# The Go counter loads this file; tiktoken reads its own cached copy of the same blob, so
|
||||
# the check in main() is what keeps the check-out honest about what it ships.
|
||||
CL100K_TABLE = "cl100k_base.tiktoken"
|
||||
@@ -337,7 +337,7 @@ def main() -> int:
|
||||
else:
|
||||
tokenizer_path = os.path.join(args.deps, rel)
|
||||
if not os.path.isfile(tokenizer_path):
|
||||
print(f"missing oracle tokenizer: {tokenizer_path}\nrun `uv run ragflow_deps/download_go_deps.py`", file=sys.stderr)
|
||||
print(f"missing oracle tokenizer: {tokenizer_path}\nrun `uv run ragflow_deps/download_deps.py`", file=sys.stderr)
|
||||
return 1
|
||||
tokenizer = Tokenizer.from_file(tokenizer_path)
|
||||
vocab = tokenizer.get_vocab()
|
||||
@@ -379,7 +379,7 @@ def main() -> int:
|
||||
# --spm-crosscheck was asked for, so the asset it reads has to be there:
|
||||
# skipping quietly would turn "the second opinion agrees" into a claim
|
||||
# nobody made. Assets are not optional in this script - they are checked.
|
||||
print(f"missing sentencepiece model: {model_file}\nrun `uv run ragflow_deps/download_go_deps.py`", file=sys.stderr)
|
||||
print(f"missing sentencepiece model: {model_file}\nrun `uv run ragflow_deps/download_deps.py`", file=sys.stderr)
|
||||
return 1
|
||||
processor = spm.SentencePieceProcessor(model_file=model_file)
|
||||
divergent = 0
|
||||
|
||||
Reference in new issue
Block a user