fix(rag): strip URL query/fragment before deriving file suffix in load_from_url (#43338)

api/core/rag/extractor/extract_processor.py

@@ -2,7 +2,7 @@ import re
 import tempfile
 from pathlib import Path
 from typing import Literal, overload
-from urllib.parse import unquote
+from urllib.parse import unquote, urlsplit
 
 from sqlalchemy.orm import Session
 
@@ -83,7 +83,10 @@ class ExtractProcessor:
         response = remote_fetcher.make_request("GET", url, headers={"User-Agent": USER_AGENT})
 
         with tempfile.TemporaryDirectory() as temp_dir:
-            suffix = Path(url).suffix
+            # Derive the suffix from the URL path only: query strings and fragments
+            # (e.g. "?token=..." / "#page=2") are not part of the file name and
+            # would otherwise leak into the suffix and misroute the extractor.
+            suffix = Path(urlsplit(url).path).suffix
             if not suffix and suffix != ".":
                 # get content-type
                 content_type = response.headers.get("Content-Type")

api/tests/unit_tests/core/rag/extractor/test_extract_processor.py

@@ -121,6 +121,35 @@ class TestExtractProcessorLoaders:
                 "",
             ),
             ("https://example.com/no_suffix", {}, ""),
+            # Query strings and fragments are not part of the file name and must
+            # not leak into the derived suffix (see issue #43333).
+            (
+                "https://example.com/file.pdf?token=abc",
+                {"Content-Type": "application/pdf"},
+                ".pdf",
+            ),
+            (
+                "https://example.com/file.pdf#page=2",
+                {"Content-Type": "application/pdf"},
+                ".pdf",
+            ),
+            (
+                "https://example.com/file.pdf?token=abc#page=2",
+                {"Content-Type": "application/pdf"},
+                ".pdf",
+            ),
+            # A misleading extension inside the query string or fragment must
+            # not win over the response headers either.
+            (
+                "https://example.com/no_suffix?filename=report.txt",
+                {"Content-Type": "application/pdf"},
+                ".pdf",
+            ),
+            (
+                "https://example.com/no_suffix#report.txt",
+                {"Content-Disposition": 'attachment; filename="report.md"'},
+                ".md",
+            ),
         ],
     )
     def test_load_from_url_builds_temp_file_with_correct_suffix(
@@ -139,10 +168,10 @@ class TestExtractProcessorLoaders:
         monkeypatch.setattr(ExtractProcessor, "extract", fake_extract)
 
         docs = ExtractProcessor.load_from_url(url, return_text=False)
-        assert captured["file_path_docs"].endswith(expected_suffix)
+        assert Path(captured["file_path_docs"]).suffix == expected_suffix
 
         text = ExtractProcessor.load_from_url(url, return_text=True)
-        assert captured["file_path_text"].endswith(expected_suffix)
+        assert Path(captured["file_path_text"]).suffix == expected_suffix
 
         assert len(docs) == 2
         assert text == "u1\nu2"