mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-13 23:01:11 +08:00
VincentLambert
371cfabd95
Add Sitemap data source — ingests web pages from sitemap.xml with incremental sync. Supports HTML→Markdown extraction, PDF discovery, URL filtering, SSRF protection.
616 lines
25 KiB
Python
616 lines
25 KiB
Python
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import queue
|
|
import re
|
|
import threading
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
from urllib.parse import urljoin, urlparse
|
|
from xml.etree import ElementTree as ET
|
|
|
|
import regex
|
|
import requests
|
|
|
|
from common.data_source.config import (
|
|
INDEX_BATCH_SIZE,
|
|
REQUEST_TIMEOUT_SECONDS,
|
|
DocumentSource,
|
|
)
|
|
from common.data_source.html_utils import web_html_to_markdown
|
|
from common.data_source.interfaces import LoadConnector, PollConnector, SlimConnectorWithPermSync
|
|
from common.data_source.models import (
|
|
Document,
|
|
GenerateDocumentsOutput,
|
|
GenerateSlimDocumentOutput,
|
|
SecondsSinceUnixEpoch,
|
|
SlimDocument,
|
|
)
|
|
from common.ssrf_guard import assert_url_is_safe
|
|
from common.ssrf_guard import pin_dns as _pin_dns
|
|
|
|
_SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
|
|
_MAX_REDIRECTS = 10
|
|
_MAX_SITEMAP_DEPTH = 5
|
|
_MAX_SITEMAP_FETCHES = 1000
|
|
_MAX_RESPONSE_BYTES = 64 * 1024 * 1024
|
|
_READ_CHUNK_BYTES = 64 * 1024
|
|
_MAX_URL_FILTER_LENGTH = 512
|
|
_MAX_MATCHED_URL_LENGTH = 4096
|
|
_URL_FILTER_TIMEOUT_SECONDS = 1.0
|
|
# A quantified group that itself contains a quantifier, e.g. "(a+)+" or "(a*b)*":
|
|
# the classic shape behind catastrophic backtracking in Python's regex engine.
|
|
_NESTED_QUANTIFIER_RE = re.compile(r"\([^()]*[+*}][^()]*\)\s*[+*{?]")
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_QUEUE_END = object()
|
|
_QUEUE_POLL_SECONDS = 0.5
|
|
|
|
|
|
class _WorkerThreadIterator:
|
|
"""Iterator over items produced by ``source`` in a daemon worker thread.
|
|
|
|
See :func:`iter_in_worker_thread`. ``offload_next`` tells the sync worker that
|
|
``next()`` may block and should be awaited through ``asyncio.to_thread``.
|
|
"""
|
|
|
|
offload_next = True
|
|
|
|
def __init__(self, source: Iterator[Any], maxsize: int, on_close: Callable[[], None] | None):
|
|
self._source = source
|
|
self._queue: queue.Queue = queue.Queue(maxsize=maxsize)
|
|
self._stop = threading.Event()
|
|
self._closed = False
|
|
self._on_close = on_close
|
|
self._worker = threading.Thread(target=self._produce, name="sitemap-batch-producer", daemon=True)
|
|
self._worker.start()
|
|
|
|
def _put(self, item: Any) -> bool:
|
|
"""Enqueue ``item`` unless the consumer went away; never blocks forever."""
|
|
while not self._stop.is_set():
|
|
try:
|
|
self._queue.put(item, timeout=_QUEUE_POLL_SECONDS)
|
|
return True
|
|
except queue.Full:
|
|
continue
|
|
return False
|
|
|
|
def _produce(self) -> None:
|
|
try:
|
|
for item in self._source:
|
|
if not self._put(item):
|
|
return
|
|
self._put(_QUEUE_END)
|
|
except BaseException as exc: # noqa: BLE001 - forwarded to the consumer
|
|
self._put(exc)
|
|
finally:
|
|
close = getattr(self._source, "close", None)
|
|
if close is not None:
|
|
try:
|
|
close()
|
|
except Exception:
|
|
logger.debug("worker-thread generator close() failed", exc_info=True)
|
|
|
|
def __iter__(self) -> "_WorkerThreadIterator":
|
|
return self
|
|
|
|
def __next__(self) -> Any:
|
|
while True:
|
|
if self._closed:
|
|
raise StopIteration
|
|
try:
|
|
item = self._queue.get(timeout=_QUEUE_POLL_SECONDS)
|
|
except queue.Empty:
|
|
continue
|
|
if item is _QUEUE_END:
|
|
self.close()
|
|
raise StopIteration
|
|
if isinstance(item, BaseException):
|
|
self.close()
|
|
raise item
|
|
return item
|
|
|
|
def close(self) -> None:
|
|
"""Stop the producer; also invoked when the consumer abandons iteration."""
|
|
if self._closed:
|
|
return
|
|
self._closed = True
|
|
self._stop.set()
|
|
if self._on_close is not None:
|
|
try:
|
|
self._on_close()
|
|
except Exception:
|
|
logger.debug("worker-thread on_close callback failed", exc_info=True)
|
|
|
|
def join(self, timeout: float | None = None) -> None:
|
|
self._worker.join(timeout)
|
|
|
|
@property
|
|
def producer_alive(self) -> bool:
|
|
return self._worker.is_alive()
|
|
|
|
|
|
async def validate_connector_in_thread(connector: "SitemapConnector") -> None:
|
|
"""Run ``validate_connector_settings`` off the event loop and propagate cancellation.
|
|
|
|
The sync worker wraps the whole task in ``asyncio.wait_for``. A thread started by
|
|
``asyncio.to_thread`` cannot be interrupted, so when the awaiting coroutine is
|
|
cancelled (task timeout) the connector is told to stop at its next cancellation
|
|
check instead of finishing its fetches in the background.
|
|
"""
|
|
try:
|
|
await asyncio.to_thread(connector.validate_connector_settings)
|
|
except asyncio.CancelledError:
|
|
logger.warning("Sitemap validation cancelled (task timeout) for %s; stopping the connector", connector.sitemap_url)
|
|
connector.cancel()
|
|
raise
|
|
|
|
|
|
def iter_in_worker_thread(source: Iterator[Any], maxsize: int = 2, on_close: Callable[[], None] | None = None) -> _WorkerThreadIterator:
|
|
"""Consume ``source`` in a daemon worker thread and hand its items over through a bounded queue.
|
|
|
|
The sync worker iterates connector generators on the asyncio thread. Running the
|
|
generator (and therefore all its blocking HTTP calls) in a worker thread keeps that
|
|
blocking off the event-loop thread and lets the next batch be downloaded while the
|
|
previous one is indexed. Exceptions raised by the source are re-raised in the
|
|
consumer. Closing the consumer stops the producer: queue writes are abandoned,
|
|
and ``on_close`` (typically the connector's ``cancel``) is invoked so that an
|
|
in-progress traversal gives up at its next check instead of finishing its fetches.
|
|
"""
|
|
return _WorkerThreadIterator(source, maxsize, on_close)
|
|
|
|
|
|
class SitemapConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
|
|
"""Connector that ingests web pages listed in a sitemap.xml.
|
|
|
|
Supports:
|
|
- Standard urlset sitemaps
|
|
- Sitemap index files (recursive, up to _MAX_SITEMAP_DEPTH levels)
|
|
- Incremental polling via <lastmod>
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
sitemap_url: str,
|
|
batch_size: int = 10,
|
|
user_agent: str = "RAGFlow-SitemapConnector/1.0",
|
|
url_filter: str | None = None,
|
|
follow_pdf_links: bool = False,
|
|
restrict_pdf_to_domain: bool = True,
|
|
) -> None:
|
|
self.sitemap_url = sitemap_url.strip()
|
|
self.batch_size = batch_size
|
|
self.user_agent = user_agent
|
|
self._url_filter: regex.Pattern | None = self._compile_url_filter(url_filter) if url_filter else None
|
|
# Dedicated session: no ambient proxies / .netrc credentials from the worker
|
|
# environment, so every request goes straight to the DNS-pinned address.
|
|
self._session = requests.Session()
|
|
self._session.trust_env = False
|
|
self._cancel = threading.Event()
|
|
self.follow_pdf_links = follow_pdf_links
|
|
self.restrict_pdf_to_domain = restrict_pdf_to_domain
|
|
self.credentials: dict[str, Any] = {}
|
|
|
|
@classmethod
|
|
def build_connector(cls, config: dict[str, Any]) -> "SitemapConnector":
|
|
"""Build a connector from the connector config stored by the UI/API."""
|
|
connector = cls(
|
|
sitemap_url=config["sitemap_url"],
|
|
batch_size=int(config.get("batch_size") or INDEX_BATCH_SIZE),
|
|
user_agent=(config.get("user_agent") or "").strip() or "RAGFlow-SitemapConnector/1.0",
|
|
url_filter=(config.get("url_filter") or "").strip() or None,
|
|
follow_pdf_links=bool(config.get("follow_pdf_links", False)),
|
|
restrict_pdf_to_domain=bool(config.get("restrict_pdf_to_domain", True)),
|
|
)
|
|
connector.load_credentials(config.get("credentials") or {})
|
|
return connector
|
|
|
|
def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
|
|
self.credentials = credentials or {}
|
|
return None
|
|
|
|
def cancel(self) -> None:
|
|
"""Ask an in-progress traversal to stop at its next check (next URL or next body chunk)."""
|
|
self._cancel.set()
|
|
|
|
@property
|
|
def cancelled(self) -> bool:
|
|
return self._cancel.is_set()
|
|
|
|
@staticmethod
|
|
def _compile_url_filter(pattern: str) -> regex.Pattern:
|
|
"""Compile the user-provided url_filter with bounded evaluation.
|
|
|
|
The pattern is matched against sitemap-controlled URLs. It is compiled with the
|
|
``regex`` module (already a transitive dependency through tiktoken / nltk) so
|
|
that every match runs under a timeout, which is the only robust defence against
|
|
catastrophic backtracking (``(a|aa)+`` and friends cannot be detected syntactically).
|
|
The obvious nested-quantifier shapes are still refused up front to give users an
|
|
immediate error, and the pattern length is bounded.
|
|
"""
|
|
if len(pattern) > _MAX_URL_FILTER_LENGTH:
|
|
raise ValueError(f"url_filter is too long (max {_MAX_URL_FILTER_LENGTH} characters)")
|
|
if _NESTED_QUANTIFIER_RE.search(pattern):
|
|
raise ValueError("url_filter must not contain nested quantifiers such as '(a+)+', which can cause catastrophic backtracking")
|
|
try:
|
|
return regex.compile(pattern)
|
|
except regex.error as exc:
|
|
raise ValueError(f"url_filter is not a valid regex: {exc}") from exc
|
|
|
|
def validate_connector_settings(self) -> None:
|
|
self._validate_url(self.sitemap_url)
|
|
if self.batch_size < 1:
|
|
raise ValueError("batch_size must be greater than 0")
|
|
if not any(self._url_matches(u[0]) for u in self._iter_sitemap_urls(self.sitemap_url, depth=0)):
|
|
raise ValueError("Sitemap contains no URLs matching the filter")
|
|
|
|
def load_from_state(self) -> GenerateDocumentsOutput:
|
|
yield from self._load_urls()
|
|
|
|
def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:
|
|
yield from self._load_urls(start=start, end=end)
|
|
|
|
def retrieve_all_slim_docs_perm_sync(self, callback: Any = None) -> GenerateSlimDocumentOutput:
|
|
"""Yield the complete set of document IDs the connector currently exposes.
|
|
|
|
When ``follow_pdf_links`` is enabled the HTML pages are fetched so that
|
|
the PDFs discovered in them are part of the retained set; otherwise a
|
|
prune run would delete them right after they were indexed.
|
|
"""
|
|
del callback
|
|
seen: set[str] = set()
|
|
sitemap_domain = urlparse(self.sitemap_url).netloc
|
|
batch: list[SlimDocument] = []
|
|
for url, _lastmod in self._iter_sitemap_urls(self.sitemap_url, depth=0):
|
|
if url in seen or not self._url_matches(url):
|
|
continue
|
|
seen.add(url)
|
|
batch.append(SlimDocument(id=self._build_document_id(url)))
|
|
if len(batch) >= self.batch_size:
|
|
yield batch
|
|
batch = []
|
|
|
|
if not self.follow_pdf_links:
|
|
continue
|
|
try:
|
|
raw, content_type = self._fetch_raw(url)
|
|
except (requests.RequestException, ValueError, OSError) as exc:
|
|
logger.warning("Failed to fetch page %s for PDF discovery: %s", url, exc)
|
|
continue
|
|
if "application/pdf" in content_type:
|
|
continue
|
|
for pdf_url in self._extract_pdf_links(raw, url):
|
|
if pdf_url in seen:
|
|
continue
|
|
if self.restrict_pdf_to_domain and urlparse(pdf_url).netloc != sitemap_domain:
|
|
continue
|
|
seen.add(pdf_url)
|
|
batch.append(SlimDocument(id=self._build_document_id(pdf_url)))
|
|
if len(batch) >= self.batch_size:
|
|
yield batch
|
|
batch = []
|
|
if batch:
|
|
yield batch
|
|
|
|
# ------------------------------------------------------------------
|
|
# Internal helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _load_urls(
|
|
self,
|
|
start: SecondsSinceUnixEpoch | None = None,
|
|
end: SecondsSinceUnixEpoch | None = None,
|
|
) -> GenerateDocumentsOutput:
|
|
seen: set[str] = set()
|
|
pending_pdfs: list[tuple[str, str]] = [] # (pdf_url, parent_url)
|
|
sitemap_domain = urlparse(self.sitemap_url).netloc
|
|
|
|
# --- Pass 1: sitemap URLs ---
|
|
batch: list[Document] = []
|
|
for url, lastmod in self._iter_sitemap_urls(self.sitemap_url, depth=0):
|
|
if self._cancel.is_set():
|
|
logger.info("Sitemap traversal cancelled")
|
|
return
|
|
if url in seen or not self._url_matches(url):
|
|
continue
|
|
seen.add(url)
|
|
|
|
if start is not None or end is not None:
|
|
if lastmod is None:
|
|
if start is not None:
|
|
continue
|
|
else:
|
|
ts = lastmod.timestamp()
|
|
if start is not None and ts <= start:
|
|
continue
|
|
if end is not None and ts > end:
|
|
continue
|
|
|
|
doc = self._fetch_and_build_document(url, lastmod, seen, pending_pdfs, sitemap_domain)
|
|
if doc is None:
|
|
continue
|
|
|
|
batch.append(doc)
|
|
if len(batch) >= self.batch_size:
|
|
yield batch
|
|
batch = []
|
|
|
|
if batch:
|
|
yield batch
|
|
|
|
# --- Pass 2: PDF links discovered from HTML pages ---
|
|
if not self.follow_pdf_links or not pending_pdfs:
|
|
return
|
|
|
|
batch = []
|
|
for pdf_url, parent_url in pending_pdfs:
|
|
if self._cancel.is_set():
|
|
logger.info("Sitemap traversal cancelled during PDF pass")
|
|
return
|
|
doc = self._fetch_and_build_document(pdf_url, None, seen, [], sitemap_domain, parent_url)
|
|
if doc is None:
|
|
continue
|
|
batch.append(doc)
|
|
if len(batch) >= self.batch_size:
|
|
yield batch
|
|
batch = []
|
|
|
|
if batch:
|
|
yield batch
|
|
|
|
def _iter_sitemap_urls(self, sitemap_url: str, depth: int, _walk: dict[str, Any] | None = None) -> Iterator[tuple[str, datetime | None]]:
|
|
"""Recursively yield (url, lastmod) pairs from a sitemap or sitemap index.
|
|
|
|
Each sitemap URL is fetched at most once (an index referencing itself or an
|
|
ancestor is skipped) and the walk stops after ``_MAX_SITEMAP_FETCHES`` sitemap
|
|
documents, so one connector cannot trigger unbounded egress.
|
|
"""
|
|
if _walk is None:
|
|
_walk = {"visited": set(), "fetched": 0}
|
|
if depth > _MAX_SITEMAP_DEPTH:
|
|
logger.warning("Max sitemap depth reached, stopping at %s", sitemap_url)
|
|
return
|
|
if sitemap_url in _walk["visited"]:
|
|
logger.warning("Sitemap %s already visited, skipping", sitemap_url)
|
|
return
|
|
_walk["visited"].add(sitemap_url)
|
|
if _walk["fetched"] >= _MAX_SITEMAP_FETCHES:
|
|
logger.warning("Maximum number of sitemap fetches (%s) reached, stopping at %s", _MAX_SITEMAP_FETCHES, sitemap_url)
|
|
return
|
|
_walk["fetched"] += 1
|
|
if self._cancel.is_set():
|
|
return
|
|
|
|
try:
|
|
self._validate_url(sitemap_url)
|
|
content, _ = self._fetch_raw(sitemap_url)
|
|
except (requests.RequestException, ValueError, OSError) as exc:
|
|
logger.warning("Failed to fetch sitemap %s: %s", sitemap_url, exc)
|
|
return
|
|
|
|
try:
|
|
root = ET.fromstring(content)
|
|
except ET.ParseError as exc:
|
|
raise ValueError(f"Failed to parse sitemap XML from {sitemap_url!r}: {exc}") from exc
|
|
|
|
tag = root.tag.lower()
|
|
|
|
# Sitemap index → recurse into child sitemaps
|
|
if "sitemapindex" in tag:
|
|
for sitemap_el in root.iter(f"{{{_SITEMAP_NS}}}sitemap"):
|
|
loc_el = sitemap_el.find(f"{{{_SITEMAP_NS}}}loc")
|
|
if loc_el is None or not (loc_el.text or "").strip():
|
|
continue
|
|
child_url = loc_el.text.strip()
|
|
yield from self._iter_sitemap_urls(child_url, depth + 1, _walk)
|
|
return
|
|
|
|
# Standard urlset
|
|
for url_el in root.iter(f"{{{_SITEMAP_NS}}}url"):
|
|
loc_el = url_el.find(f"{{{_SITEMAP_NS}}}loc")
|
|
if loc_el is None or not (loc_el.text or "").strip():
|
|
continue
|
|
loc = loc_el.text.strip()
|
|
|
|
lastmod: datetime | None = None
|
|
lastmod_el = url_el.find(f"{{{_SITEMAP_NS}}}lastmod")
|
|
if lastmod_el is not None and (lastmod_el.text or "").strip():
|
|
lastmod = self._parse_lastmod(lastmod_el.text.strip())
|
|
|
|
yield loc, lastmod
|
|
|
|
def _fetch_and_build_document(
|
|
self,
|
|
url: str,
|
|
lastmod: datetime | None,
|
|
seen: set[str] | None = None,
|
|
pending_pdfs: list[tuple[str, str]] | None = None,
|
|
sitemap_domain: str | None = None,
|
|
parent_url: str | None = None,
|
|
) -> Document | None:
|
|
try:
|
|
self._validate_url(url)
|
|
raw, content_type = self._fetch_raw(url)
|
|
except (requests.RequestException, ValueError, OSError) as exc:
|
|
logger.warning("Failed to fetch page %s: %s", url, exc)
|
|
return None
|
|
|
|
updated_at = lastmod or datetime.now(UTC)
|
|
|
|
if "application/pdf" in content_type:
|
|
if not raw:
|
|
logger.debug("Empty PDF for %s, skipping", url)
|
|
return None
|
|
blob = raw
|
|
extension = ".pdf"
|
|
else:
|
|
if self.follow_pdf_links and seen is not None and pending_pdfs is not None:
|
|
for pdf_url in self._extract_pdf_links(raw, url):
|
|
if pdf_url in seen:
|
|
continue
|
|
if not self.restrict_pdf_to_domain or urlparse(pdf_url).netloc == sitemap_domain:
|
|
pending_pdfs.append((pdf_url, url))
|
|
seen.add(pdf_url)
|
|
|
|
try:
|
|
text = web_html_to_markdown(raw.decode("utf-8", errors="replace")).cleaned_text
|
|
except Exception as exc: # noqa: BLE001 - third-party extractor, any failure just skips the page
|
|
logger.warning("Failed to parse page %s: %s", url, exc)
|
|
return None
|
|
|
|
if not text or not text.strip():
|
|
logger.debug("Empty content for %s, skipping", url)
|
|
return None
|
|
|
|
blob = text.encode("utf-8")
|
|
extension = ".md"
|
|
|
|
return Document(
|
|
id=self._build_document_id(url),
|
|
source=DocumentSource.SITEMAP,
|
|
semantic_identifier=self._url_to_identifier(url),
|
|
extension=extension,
|
|
blob=blob,
|
|
doc_updated_at=updated_at,
|
|
size_bytes=len(blob),
|
|
metadata={
|
|
"sitemap_url": self.sitemap_url,
|
|
"url": url,
|
|
**({"parent_url": parent_url} if parent_url else {}),
|
|
},
|
|
)
|
|
|
|
@staticmethod
|
|
def _extract_pdf_links(html_bytes: bytes, base_url: str) -> list[str]:
|
|
"""Extract absolute PDF href links from an HTML page."""
|
|
from html.parser import HTMLParser
|
|
|
|
class _Extractor(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.links: list[str] = []
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == "a":
|
|
href = (dict(attrs).get("href", "") or "").strip()
|
|
if href:
|
|
self.links.append(href)
|
|
|
|
ex = _Extractor()
|
|
ex.feed(html_bytes.decode("utf-8", errors="replace"))
|
|
pdf_links: list[str] = []
|
|
for href in ex.links:
|
|
absolute = urljoin(base_url, href)
|
|
# Match on the path only, so "/manual.pdf?download=1#page=2" counts.
|
|
if urlparse(absolute).path.lower().endswith(".pdf"):
|
|
pdf_links.append(absolute)
|
|
return pdf_links
|
|
|
|
def _fetch_raw(self, url: str) -> tuple[bytes, str]:
|
|
"""Fetch a URL with redirect following, SSRF protection and a response size cap.
|
|
|
|
Every hop is validated by ``assert_url_is_safe`` and the resolved address is
|
|
pinned for the duration of the request (DNS rebinding protection). The body
|
|
is streamed and rejected once it exceeds ``_MAX_RESPONSE_BYTES``.
|
|
|
|
Returns (content, content_type).
|
|
"""
|
|
current_url = url
|
|
current_hostname, current_ip = assert_url_is_safe(current_url)
|
|
|
|
response: requests.Response | None = None
|
|
for _ in range(_MAX_REDIRECTS + 1):
|
|
with _pin_dns(current_hostname, current_ip):
|
|
response = self._session.get(
|
|
current_url,
|
|
timeout=REQUEST_TIMEOUT_SECONDS,
|
|
allow_redirects=False,
|
|
stream=True,
|
|
headers={"User-Agent": self.user_agent},
|
|
)
|
|
if response.status_code not in (301, 302, 303, 307, 308):
|
|
response.raise_for_status()
|
|
content_type = response.headers.get("Content-Type", "").lower()
|
|
return self._read_capped(response, current_url), content_type
|
|
response.close()
|
|
location = response.headers.get("Location")
|
|
if not location:
|
|
response.raise_for_status()
|
|
raise ValueError(f"Redirect without Location header fetching {url!r}")
|
|
redirect_url = urljoin(current_url, location)
|
|
current_hostname, current_ip = assert_url_is_safe(redirect_url)
|
|
current_url = redirect_url
|
|
raise ValueError(f"Exceeded {_MAX_REDIRECTS} redirects fetching {url!r}")
|
|
|
|
def _read_capped(self, response: requests.Response, url: str) -> bytes:
|
|
"""Read a streamed response body, refusing anything larger than _MAX_RESPONSE_BYTES.
|
|
|
|
Stops early (raising ``ValueError``) when :meth:`cancel` was called.
|
|
"""
|
|
declared = response.headers.get("Content-Length")
|
|
if declared and declared.isdigit() and int(declared) > _MAX_RESPONSE_BYTES:
|
|
response.close()
|
|
raise ValueError(f"Response for {url!r} exceeds the maximum allowed size of {_MAX_RESPONSE_BYTES} bytes")
|
|
chunks: list[bytes] = []
|
|
total = 0
|
|
try:
|
|
for chunk in response.iter_content(chunk_size=_READ_CHUNK_BYTES):
|
|
if self._cancel.is_set():
|
|
raise ValueError(f"Fetch of {url!r} cancelled")
|
|
if not chunk:
|
|
continue
|
|
total += len(chunk)
|
|
if total > _MAX_RESPONSE_BYTES:
|
|
raise ValueError(f"Response for {url!r} exceeds the maximum allowed size of {_MAX_RESPONSE_BYTES} bytes")
|
|
chunks.append(chunk)
|
|
finally:
|
|
response.close()
|
|
return b"".join(chunks)
|
|
|
|
@staticmethod
|
|
def _validate_url(url: str) -> tuple[str, str]:
|
|
if not url:
|
|
raise ValueError("URL is required")
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
raise ValueError(f"URL must be a valid http or https URL: {url!r}")
|
|
return assert_url_is_safe(url)
|
|
|
|
def _url_matches(self, url: str) -> bool:
|
|
if self._url_filter is None:
|
|
return True
|
|
try:
|
|
return bool(self._url_filter.search(url[:_MAX_MATCHED_URL_LENGTH], timeout=_URL_FILTER_TIMEOUT_SECONDS))
|
|
except TimeoutError:
|
|
logger.warning("url_filter evaluation timed out after %ss on %s; URL skipped", _URL_FILTER_TIMEOUT_SECONDS, url[:200])
|
|
return False
|
|
|
|
@staticmethod
|
|
def _url_to_identifier(url: str) -> str:
|
|
parsed = urlparse(url)
|
|
identifier = f"{parsed.netloc}{parsed.path}"
|
|
if parsed.query:
|
|
identifier += f"?{parsed.query}"
|
|
if parsed.fragment:
|
|
identifier += f"#{parsed.fragment}"
|
|
return identifier.strip("/") or url
|
|
|
|
@staticmethod
|
|
def _build_document_id(url: str) -> str:
|
|
return f"sitemap:{hashlib.md5(url.encode('utf-8')).hexdigest()}"
|
|
|
|
@staticmethod
|
|
def _parse_lastmod(value: str) -> datetime | None:
|
|
for fmt in ("%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d"):
|
|
try:
|
|
dt = datetime.strptime(value, fmt) # noqa: DTZ007 - tz set explicitly below
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=UTC)
|
|
return dt.astimezone(UTC)
|
|
except ValueError:
|
|
continue
|
|
logger.debug("Unrecognised lastmod format: %r", value)
|
|
return None
|