diff --git a/internal/deepdoc/native/testdata_dir_test.go b/internal/deepdoc/native/testdata_dir_test.go new file mode 100644 index 0000000000..8cc82afd48 --- /dev/null +++ b/internal/deepdoc/native/testdata_dir_test.go @@ -0,0 +1,79 @@ +//go:build fetch_testdata + +// This package's TestMain skips everything unless the fixtures were fetched, so +// these tests only run under the same tag — which is how CI executes them. +package native + +import ( + "os" + "path/filepath" + "testing" +) + +// packageDirName must match the real package dir name: resolveTestdataDir +// derives the pre-seeded subtree from basename(pkgDir). +const packageDirName = "native" + +// newPackageDir creates an empty /native directory to stand in for the +// package directory. +func newPackageDir(t *testing.T) string { + t.Helper() + dir := filepath.Join(t.TempDir(), packageDirName) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + return dir +} + +// seedTestdata creates /testdata with one file so it counts as populated. +func seedTestdata(t *testing.T, dir string) { + t.Helper() + target := filepath.Join(dir, "testdata") + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", target, err) + } + if err := os.WriteFile(filepath.Join(target, "fixture.json"), []byte("{}"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } +} + +// TestResolveTestdataDir_KeepsLocalCopy verifies that an inline testdata +// directory always wins: it is what a developer fetched locally, and it must +// keep working unchanged. +func TestResolveTestdataDir_KeepsLocalCopy(t *testing.T) { + local := newPackageDir(t) + seedTestdata(t, local) + // A pre-seeded tree is configured too, but the local copy takes priority. + t.Setenv("RAGFLOW_TESTDATA_DIR", t.TempDir()) + + if got := resolveTestdataDir(local); got != "" { + t.Fatalf("want empty (stay in the package dir), got %q", got) + } +} + +// TestResolveTestdataDir_FallsBackToPreseeded covers the read-only workspace +// case: the fixtures cannot be linked into the package dir, and the pre-seeded +// copy must be used from where it is instead of failing the package. +func TestResolveTestdataDir_FallsBackToPreseeded(t *testing.T) { + local := newPackageDir(t) // no inline testdata + root := t.TempDir() + seedTestdata(t, filepath.Join(root, "deepdoc", packageDirName)) + t.Setenv("RAGFLOW_TESTDATA_DIR", root) + + want := filepath.Join(root, "deepdoc", packageDirName) + if got := resolveTestdataDir(local); got != want { + t.Fatalf("want %q, got %q", want, got) + } +} + +// TestResolveTestdataDir_EmptyWhenNothingAvailable makes sure we do not chdir +// into a directory that has no fixtures: that would turn a clear "missing +// testdata" failure into a confusing file-not-found one. +func TestResolveTestdataDir_EmptyWhenNothingAvailable(t *testing.T) { + local := newPackageDir(t) + t.Setenv("RAGFLOW_TESTDATA_DIR", t.TempDir()) // pre-seeded tree is empty + + if got := resolveTestdataDir(local); got != "" { + t.Fatalf("want empty, got %q", got) + } +} diff --git a/internal/deepdoc/native/testdata_skip_test.go b/internal/deepdoc/native/testdata_skip_test.go index 7126ec9c2f..50df278fb3 100644 --- a/internal/deepdoc/native/testdata_skip_test.go +++ b/internal/deepdoc/native/testdata_skip_test.go @@ -10,9 +10,51 @@ package native import ( "fmt" "os" + "path/filepath" + "runtime" "testing" ) +// packageDir returns this package's directory (the one holding testdata). +func packageDir() string { + _, file, _, ok := runtime.Caller(0) + if !ok { + return "" + } + return filepath.Dir(file) +} + +// dirHasFiles reports whether dir exists and holds at least one entry. +func dirHasFiles(dir string) bool { + entries, err := os.ReadDir(dir) + return err == nil && len(entries) > 0 +} + +// resolveTestdataDir decides which directory the fixtures should be read from. +// +// Tests reach fixtures through the relative path "testdata/...", so the process +// must either hold them inline or run from a directory that does. CI runners +// pre-seed the assets at RAGFLOW_TESTDATA_DIR and mount the workspace +// read-only, so linking the fixtures into the package dir — what the fetch +// script used to do — fails there even though the data is present and readable. +// Running from the pre-seeded directory instead keeps the workspace untouched. +// +// It returns "" when the package's own testdata should be used as-is. +func resolveTestdataDir(pkgDir string) string { + if pkgDir == "" || dirHasFiles(filepath.Join(pkgDir, "testdata")) { + return "" + } + root := os.Getenv("RAGFLOW_TESTDATA_DIR") + if root == "" { + return "" + } + preseeded := filepath.Join(root, "deepdoc", filepath.Base(pkgDir)) + if dirHasFiles(filepath.Join(preseeded, "testdata")) { + return preseeded + } + return "" +} + // TestMain skips the entire package unless the testdata was fetched via the // fetch_testdata build tag. In the default build (and any build without that // tag) it stays false, so the tests are skipped rather than failing on absent @@ -27,5 +69,13 @@ func TestMain(m *testing.M) { fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it") os.Exit(0) } + // The fetch only guarantees the fixtures exist somewhere; with a pre-seeded + // runner they live outside the package, so run from their directory. + if dir := resolveTestdataDir(packageDir()); dir != "" { + if err := os.Chdir(dir); err != nil { + fmt.Fprintf(os.Stderr, "FAIL: cannot chdir to pre-seeded testdata %s: %v\n", dir, err) + os.Exit(1) + } + } os.Exit(m.Run()) } diff --git a/scripts/fetch_deepdoc_testdata.sh b/scripts/fetch_deepdoc_testdata.sh index e1209f5af5..54c80ce821 100644 --- a/scripts/fetch_deepdoc_testdata.sh +++ b/scripts/fetch_deepdoc_testdata.sh @@ -62,7 +62,8 @@ mkdir -p "$LOCKDIR" exec 9>"$LOCKDIR/$PKG.lock" flock 9 -# Determine whether we need a writable copy (regeneration) or a symlink. +# Determine whether we need a writable copy (regeneration) or can use the +# pre-seeded copy directly. NEED_WRITE=0 for v in "${!GEN_@}"; do if [ -n "${!v:-}" ]; then NEED_WRITE=1; break; fi @@ -75,42 +76,23 @@ done if [ -n "${RAGFLOW_TESTDATA_DIR:-}" ]; then PRESET="$RAGFLOW_TESTDATA_DIR/deepdoc/$PKG/testdata" if [ -d "$PRESET" ] && [ -n "$(ls -A "$PRESET" 2>/dev/null)" ]; then - if [ -L "$TARGET" ] && [ "$(readlink -f "$TARGET")" = "$(readlink -f "$PRESET")" ] && [ -n "$(ls -A "$PRESET" 2>/dev/null)" ]; then - echo "fetch_deepdoc_testdata: $PKG already linked to pre-seeded $PRESET" - exit 0 - fi - if [ -d "$TARGET" ] && [ ! -L "$TARGET" ] && [ -n "$(ls -A "$TARGET" 2>/dev/null)" ]; then - echo "fetch_deepdoc_testdata: $PKG testdata already present inline at $TARGET" - exit 0 - fi - # Remove any stale TARGET before linking. TARGET may be a wrong symlink, an - # empty directory, or a leftover real directory from a prior GEN_* copy or a - # dereferenced symlink on a persistent runner. `ln -sfn` below also unlinks - # TARGET first, but removing it here keeps intent explicit. A partial - # removal failure is non-fatal: ln -sfn still replaces TARGET, and the copy - # fallback below covers the rare case where even that is blocked. - # - # Use `ln -sfn`, never `ln -s`: when TARGET already exists as a directory, - # plain `ln -s` descends INTO it and creates /testdata (the exact - # "Permission denied" failure this job hit), because the existing dir is not - # writable. `-f` unlinks TARGET first; `-n`/`--no-dereference` stops ln from - # treating TARGET as a container. - rm -rf -- "$TARGET" 2>/dev/null || true if [ "$NEED_WRITE" -eq 1 ]; then - echo "fetch_deepdoc_testdata: copying writable pre-seeded testdata for regeneration ($PKG)" - rm -rf -- "$TARGET" 2>/dev/null || true - cp -r "$PRESET" "$TARGET" - elif ln -sfn "$PRESET" "$TARGET" 2>/dev/null; then - echo "fetch_deepdoc_testdata: linked $TARGET -> $PRESET (pre-seeded)" - else - # Symlink creation failed (e.g. the parent directory is not writable on a - # shared runner). The pre-seeded fixtures are authoritative and readable, - # so copy them in place instead of reddening CI over a filesystem quirk. - echo "fetch_deepdoc_testdata: symlink failed; copying pre-seeded testdata for $PKG" >&2 - mkdir -p "$TARGET" - cp -r "$PRESET"/. "$TARGET"/. + # Regeneration rewrites the fixtures, so it needs a writable copy next to + # the tests — which a pre-seeded runner cannot provide. Say so plainly + # instead of failing later with a confusing permission error. + if ! (rm -rf -- "$TARGET" && cp -r "$PRESET" "$TARGET") 2>/dev/null; then + echo "fetch_deepdoc_testdata: GEN_* regeneration needs a writable $TARGET, but the workspace is read-only. Run it where $TARGET is writable, or drop RAGFLOW_TESTDATA_DIR to clone the fixtures." >&2 + exit 1 + fi + echo "fetch_deepdoc_testdata: copied writable pre-seeded testdata for regeneration ($PKG)" + exit 0 fi - echo "fetch_deepdoc_testdata: done ($PKG, pre-seeded)" + # Leave the workspace untouched: the consuming test binary reads the + # fixtures from this directory directly (see testdata_skip_test.go). + # Linking or copying them into $TARGET used to be required, and failed on + # runners that mount the workspace read-only — which was then reported as a + # missing fixture even though the data was right here. + echo "fetch_deepdoc_testdata: using pre-seeded testdata at $PRESET (workspace untouched)" exit 0 fi echo "fetch_deepdoc_testdata: RAGFLOW_TESTDATA_DIR set but deepdoc/$PKG/testdata is missing; falling back to clone" >&2