ragflow/api/apps/restful_apis/dataset_api.py

#
#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import logging

from peewee import OperationalError
from quart import make_response, request

from api.apps import current_user, login_required
from api.apps.services import dataset_api_service
from api.utils.api_utils import add_tenant_id_to_kwargs, get_error_argument_result, get_error_data_result, get_json_result, get_result
from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_size
from api.utils.validation_utils import (
    CreateDatasetReq,
    DeleteDatasetReq,
    ListDatasetReq,
    UpdateDatasetReq,
    validate_and_parse_json_request,
    validate_and_parse_request_args,
)
from common.constants import RetCode


@manager.route("/datasets/tags/aggregation", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def aggregate_tags(tenant_id):
    dataset_ids = request.args.get("dataset_ids", "").split(",")
    dataset_ids = [d for d in dataset_ids if d]
    if not dataset_ids:
        return get_error_data_result(message="Lack of dataset_ids in query parameters")

    try:
        validate_rest_api_ids(dataset_ids, "dataset_ids")
        success, result = dataset_api_service.aggregate_tags(dataset_ids, tenant_id)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/metadata/flattened", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def get_flattened_metadata(tenant_id):
    dataset_ids = request.args.get("dataset_ids", "").split(",")
    dataset_ids = [d for d in dataset_ids if d]
    if not dataset_ids:
        return get_error_data_result(message="Lack of dataset_ids in query parameters")

    try:
        validate_rest_api_ids(dataset_ids, "dataset_ids")
        success, result = dataset_api_service.get_flattened_metadata(dataset_ids, tenant_id)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets", methods=["POST"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def create(tenant_id: str = None):
    """
    Create a new dataset.
    ---
    tags:
      - Datasets
    security:
      - ApiKeyAuth: []
    parameters:
      - in: header
        name: Authorization
        type: string
        required: true
        description: Bearer token for authentication.
      - in: body
        name: body
        description: Dataset creation parameters.
        required: true
        schema:
          type: object
          required:
            - name
          properties:
            name:
              type: string
              description: Dataset name (required).
            avatar:
              type: string
              description: Optional base64-encoded avatar image.
            description:
              type: string
              description: Optional dataset description.
            language:
              type: string
              minLength: 1
              maxLength: 32
              description: Optional document language (e.g. "English", "Chinese"); leading/trailing whitespace is stripped and the trimmed value must contain 1 to 32 characters. If omitted, the server/database default is used.
            embedding_model:
              type: string
              description: Optional embedding model name; if omitted, the tenant's default embedding model is used.
            permission:
              type: string
              enum: ['me', 'team']
              description: Visibility of the dataset (private to me or shared with team).
            chunk_method:
              type: string
              enum: ["naive", "book", "email", "laws", "manual", "one", "paper",
                     "picture", "presentation", "qa", "table", "tag"]
              description: Chunking method; if omitted, defaults to "naive".
            parser_config:
              type: object
              description: Optional parser configuration; server-side defaults will be applied.
    responses:
      200:
        description: Successful operation.
        schema:
          type: object
          properties:
            data:
              type: object
    """
    req, err = await validate_and_parse_json_request(request, CreateDatasetReq)
    if err is not None:
        return get_error_argument_result(err)

    try:
        if not tenant_id:
            tenant_id = current_user.id
        success, result = await dataset_api_service.create_dataset(tenant_id, req)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except LookupError as e:
        return get_error_argument_result(str(e))
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def delete(tenant_id):
    """
    Delete datasets.
    ---
    tags:
      - Datasets
    security:
      - ApiKeyAuth: []
    parameters:
      - in: header
        name: Authorization
        type: string
        required: true
        description: Bearer token for authentication.
      - in: body
        name: body
        description: Dataset deletion parameters.
        required: true
        schema:
          type: object
          required:
            - ids
          properties:
            ids:
              type: array or null
              items:
                type: string
              description: |
                Specifies the datasets to delete:
                - If `null`, all datasets will be deleted.
                - If an array of IDs, only the specified datasets will be deleted.
                - If an empty array, no datasets will be deleted.
    responses:
      200:
        description: Successful operation.
        schema:
          type: object
    """
    req, err = await validate_and_parse_json_request(request, DeleteDatasetReq)
    if err is not None:
        return get_error_argument_result(err)

    try:
        success, result = await dataset_api_service.delete_datasets(tenant_id, req.get("ids"), req.get("delete_all", False))
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except OperationalError as e:
        logging.exception(e)
        return get_error_data_result(message="Database operation failed")
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>", methods=["PUT"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def update(tenant_id, dataset_id):
    """
    Update a dataset.
    ---
    tags:
      - Datasets
    security:
      - ApiKeyAuth: []
    parameters:
      - in: path
        name: dataset_id
        type: string
        required: true
        description: ID of the dataset to update.
      - in: header
        name: Authorization
        type: string
        required: true
        description: Bearer token for authentication.
      - in: body
        name: body
        description: Dataset update parameters.
        required: true
        schema:
          type: object
          properties:
            name:
              type: string
              description: New name of the dataset.
            avatar:
              type: string
              description: Updated base64 encoding of the avatar.
            description:
              type: string
              description: Updated description of the dataset.
            language:
              type: string
              minLength: 1
              maxLength: 32
              description: Optional document language (e.g. "English", "Chinese"); leading/trailing whitespace is stripped and the trimmed value must contain 1 to 32 characters. If omitted, the existing language is unchanged.
            embedding_model:
              type: string
              description: Updated embedding model Name.
            permission:
              type: string
              enum: ['me', 'team']
              description: Updated dataset permission.
            chunk_method:
              type: string
              enum: ["naive", "book", "email", "laws", "manual", "one", "paper",
                     "picture", "presentation", "qa", "table", "tag"
                     ]
              description: Updated chunking method.
            pagerank:
              type: integer
              description: Updated page rank.
            parser_config:
              type: object
              description: Updated parser configuration.
    responses:
      200:
        description: Successful operation.
        schema:
          type: object
    """
    # Field name transformations during model dump:
    # | Original       | Dump Output  |
    # |----------------|-------------|
    # | embedding_model| embd_id     |
    # | chunk_method   | parser_id   |
    extras = {"dataset_id": dataset_id}
    req, err = await validate_and_parse_json_request(request, UpdateDatasetReq, extras=extras, exclude_unset=True)
    if err is not None:
        return get_error_argument_result(err)

    try:
        success, result = await dataset_api_service.update_dataset(tenant_id, dataset_id, req)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except OperationalError as e:
        logging.exception(e)
        return get_error_data_result(message="Database operation failed")
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def list_datasets(tenant_id):
    """
    List datasets.
    ---
    tags:
      - Datasets
    security:
      - ApiKeyAuth: []
    parameters:
      - in: query
        name: id
        type: string
        required: false
        description: Dataset ID to filter.
      - in: query
        name: ids
        type: array
        required: false
        items:
          type: string
        collectionFormat: multi
        description: Dataset IDs to filter.
      - in: query
        name: name
        type: string
        required: false
        description: Dataset name to filter.
      - in: query
        name: page
        type: integer
        required: false
        default: 1
        description: Page number.
      - in: query
        name: page_size
        type: integer
        required: false
        default: 30
        description: Number of items per page.
      - in: query
        name: orderby
        type: string
        required: false
        default: "create_time"
        description: Field to order by.
      - in: query
        name: desc
        type: boolean
        required: false
        default: true
        description: Order in descending.
      - in: header
        name: Authorization
        type: string
        required: true
        description: Bearer token for authentication.
    responses:
      200:
        description: Successful operation.
        schema:
          type: array
          items:
            type: object
    """
    if request.args.get("type") == "filter":
        try:
            success, result = dataset_api_service.list_dataset_filters(tenant_id)
            if success:
                return get_result(data=result)
            else:
                return get_error_data_result(message=result)
        except OperationalError as e:
            logging.exception(e)
            return get_error_data_result(message="Database operation failed")
        except Exception as e:
            logging.exception(e)
            return get_error_data_result(message="Internal server error")

    args, err = validate_and_parse_request_args(request, ListDatasetReq)
    if err is not None:
        return get_error_argument_result(err)

    try:
        success, result = dataset_api_service.list_datasets(tenant_id, args)
        if success:
            return get_result(data=result.get("data"), total=result.get("total"))
        else:
            return get_error_data_result(message=result)
    except OperationalError as e:
        logging.exception(e)
        return get_error_data_result(message="Database operation failed")
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def get_dataset(tenant_id, dataset_id):
    try:
        success, result = dataset_api_service.get_dataset(dataset_id, tenant_id)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/ingestions/summary", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def get_ingestion_summary(tenant_id, dataset_id):
    try:
        success, result = dataset_api_service.get_ingestion_summary(dataset_id, tenant_id)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/tags", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def list_tags(tenant_id, dataset_id):
    try:
        success, result = dataset_api_service.list_tags(dataset_id, tenant_id)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/tags", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def delete_tags(tenant_id, dataset_id):
    req = await request.get_json()
    if not req or "tags" not in req:
        return get_error_data_result(message="Lack of tags in request body")
    if not isinstance(req["tags"], list) or not all(isinstance(t, str) for t in req["tags"]):
        return get_error_argument_result("tags must be a list of strings")

    try:
        success, result = dataset_api_service.delete_tags(dataset_id, tenant_id, req["tags"])
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/tags", methods=["PUT"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def rename_tag(tenant_id, dataset_id):
    req = await request.get_json()
    if not req or "from_tag" not in req or "to_tag" not in req:
        return get_error_data_result(message="Lack of from_tag or to_tag in request body")
    if not isinstance(req["from_tag"], str) or not isinstance(req["to_tag"], str):
        return get_error_argument_result("from_tag and to_tag must be strings")

    if not req["from_tag"].strip() or not req["to_tag"].strip():
        return get_error_argument_result("from_tag and to_tag must not be empty")

    try:
        success, result = dataset_api_service.rename_tag(dataset_id, tenant_id, req["from_tag"], req["to_tag"])
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/graph", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def get_knowledge_graph(tenant_id, dataset_id):
    """Get the knowledge graph of a dataset.

    GET /api/v1/datasets/<dataset_id>/graph
    Query params: optional filter params.
    Success: {"code": 0, "data": {...}}
    Errors: AUTHENTICATION_ERROR for access denied; DATA_ERROR for internal errors.
    """
    try:
        success, result = await dataset_api_service.get_knowledge_graph(dataset_id, tenant_id)
        if success:
            return get_result(data=result)
        else:
            return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts", methods=["HEAD"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def has_any_wiki(tenant_id, dataset_id):
    """Probe whether this dataset has any compiled artifact pages.

    The frontend uses this to decide whether to surface the Artifact tab
    in the dataset sidebar.
    """
    try:
        success, result = await dataset_api_service.has_any_wiki(dataset_id, tenant_id)
        response = await make_response("", 200 if success and result["has"] else 404)
        return response
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def list_wiki_pages(tenant_id, dataset_id):
    """List artifact pages for the dataset Artifact tab.

    GET /api/v1/datasets/<dataset_id>/artifacts?page=1&page_size=100&page_type=entity&topic=topic&keywords=query
    Success: {"code": 0, "data": {"total": int, "items": [{slug, title, page_type}]}}
    """
    try:
        page = validate_rest_api_page(request.args.get("page", DEFAULT_PAGE))
        page_size = validate_rest_api_page_size(request.args.get("page_size", DEFAULT_PAGE_SIZE))
    except ValueError as e:
        return get_error_argument_result(str(e))
    page_type = (request.args.get("page_type") or "").strip() or None
    topic = (request.args.get("topic") or "").strip() or None
    keywords = (request.args.get("keywords") or "").strip()

    try:
        success, result = await dataset_api_service.list_wiki_pages(
            dataset_id,
            tenant_id,
            page=page,
            page_size=page_size,
            page_type=page_type,
            topic=topic,
            keywords=keywords,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts/topics", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def list_wiki_topics(tenant_id, dataset_id):
    """List wiki topics for the dataset Artifact tab.

    GET /api/v1/datasets/<dataset_id>/artifacts/topics?page=1&page_size=100&keywords=query
    Success: {"code": 0, "data": {"total": int, "items": [{topic, title, slug}]}}
    """
    try:
        page = validate_rest_api_page(request.args.get("page", DEFAULT_PAGE))
        page_size = validate_rest_api_page_size(request.args.get("page_size", DEFAULT_PAGE_SIZE))
    except ValueError as e:
        return get_error_argument_result(str(e))
    keywords = (request.args.get("keywords") or "").strip()

    try:
        success, result = await dataset_api_service.list_wiki_topics(
            dataset_id,
            tenant_id,
            page=page,
            page_size=page_size,
            keywords=keywords,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts/graph", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def get_wiki_graph(tenant_id, dataset_id):
    """Return an incremental slice of the canvas graph for this dataset.

    GET /api/v1/datasets/<dataset_id>/artifacts/graph[?node=<slug>][&keywords=<q>][&top_n=<n>]
    - ``node`` omitted: overview centred on the heaviest-weighted
      entities, expanded outward until the entity budget is hit.
    - ``node`` provided: subgraph centred on that entity, including all
      outgoing relations and their ``to`` targets (also capped).
    - ``keywords`` (overview only; ignored with ``node``): seed the overview
      from the best BM25 matches instead of the heaviest-weighted entities.
    - ``top_n`` (a.k.a. ``topN``): override the entity budget (default 128).

    Only entities referenced by at least one relation are returned.
    Success: ``{"code": 0, "data": {"entities":[…],"relations":[…],
    "total_entities":int,"total_relations":int,
    "returned_entities":int,"returned_relations":int}}``.
    """
    try:
        node = request.args.get("node", None)
        if isinstance(node, str):
            node = node.strip() or None
        keywords = request.args.get("keywords", "")
        top_n_arg = request.args.get("top_n") or request.args.get("topN")
        top_n = None
        if top_n_arg is not None:
            try:
                top_n = int(top_n_arg)
            except (TypeError, ValueError):
                top_n = None
        success, result = await dataset_api_service.get_wiki_graph(
            dataset_id,
            tenant_id,
            node=node,
            keywords=keywords,
            top_n=top_n,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts/structure", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def get_dataset_structure(tenant_id, dataset_id):
    """Return the dataset-scope (KB-wide) structure graph for one kind.

    GET /api/v1/datasets/<dataset_id>/artifacts/structure?kind=<kind>
    where ``kind`` is one of:
      graph | mindmap | timeline | session_essence | session_graph

    These are the non-tree structured artifacts merged across the whole
    dataset (written when a template has ``dataset_merge`` enabled). Response
    mirrors the per-document structure graph so the frontend reuses its view::

        {"code": 0, "data": {
            "kind": "<kind>",
            "total_entities": 100,
            "total_relations": 200,
            "returned_entities": 80,
            "returned_relations": 150,
            "templates": [{
                "template_id": "<template_id>", "template_name": "<template_name>",
                "kind": "<kind>", "entities": [], "relations": [],
            }],
        }}
    """
    try:
        kind = request.args.get("kind", "")
        if isinstance(kind, str):
            kind = kind.strip()
        if not kind:
            return get_error_data_result(
                message="`kind` is required (one of: graph, mindmap, timeline, session_essence, session_graph).",
                code=RetCode.ARGUMENT_ERROR,
            )
        if dataset_api_service._resolve_dataset_structure_kind(kind) is None:
            return get_error_data_result(
                message=f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, session_essence, session_graph.",
                code=RetCode.ARGUMENT_ERROR,
            )
        keywords = request.args.get("keywords", "")
        success, result = await dataset_api_service.get_dataset_structure(
            dataset_id,
            tenant_id,
            kind,
            keywords=keywords,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts/structure", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def delete_dataset_structure(tenant_id, dataset_id):
    """Delete the dataset-scope (KB-wide) structure graph for one kind.

    DELETE /api/v1/datasets/<dataset_id>/artifacts/structure?kind=<kind>
    Optional query param: wipe=false cancels the task without deleting stored rows.
    """
    kind = request.args.get("kind", "")
    if isinstance(kind, str):
        kind = kind.strip()
    if not kind:
        return get_error_data_result(
            message="`kind` is required (one of: graph, mindmap, timeline, session_essence, session_graph).",
            code=RetCode.ARGUMENT_ERROR,
        )
    if dataset_api_service._resolve_dataset_structure_kind(kind) is None:
        return get_error_data_result(
            message=f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, session_essence, session_graph.",
            code=RetCode.ARGUMENT_ERROR,
        )
    wipe_arg = (request.args.get("wipe", "true") or "true").strip().lower()
    wipe = wipe_arg not in ("false", "0", "no", "off")
    try:
        success, result = dataset_api_service.delete_dataset_structure(
            dataset_id,
            tenant_id,
            kind,
            wipe=wipe,
        )
        if success:
            return get_result(data=result)
        if result == "no authorization":
            return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)
        return get_error_data_result(message=result)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts/alteration", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def get_wiki_alteration(tenant_id, dataset_id):
    """Return document drift for a compiled dataset product.

    GET /api/v1/datasets/<dataset_id>/artifacts/alteration?kind=<kind>
    ``kind`` (default ``wiki``) is one of: wiki | graph | mindmap | timeline |
    tree (tree covers both ``tree`` and ``page_index``).
    Success: {"code": 0, "data": {"removed": int, "newly_uploaded": int, ...}}.
    Wiki responses also include ``changed`` and ``changed_doc_ids`` based on
    source chunk hash drift.
    """
    kind = (request.args.get("kind") or "wiki").strip().lower()
    if kind not in {"wiki", "graph", "mindmap", "timeline", "tree"}:
        return get_error_data_result(message=f"Unsupported kind: {kind!r}. Expected one of: wiki, graph, mindmap, timeline, tree.")
    try:
        if kind == "wiki":
            success, result = await dataset_api_service.get_wiki_alteration(dataset_id, tenant_id)
        else:
            success, result = await dataset_api_service.get_structure_alteration(dataset_id, tenant_id, kind)
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def clear_wiki(tenant_id, dataset_id):
    """Wipe every artifact-related row from ES for this KB.

    DELETE /api/v1/datasets/<dataset_id>/artifacts
    Removes the artifact ``compile_kwd`` row types written by the artifact
    pipeline (MAP extracts / REDUCE results / PLAN / drafts / pages / topics / graph rows).
    Success: {"code": 0, "data": {"deleted": {kwd: result}}}
    """
    try:
        success, result = await dataset_api_service.clear_wiki(
            dataset_id,
            tenant_id,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/artifacts/<page_type>/<path:slug>", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def get_wiki_page(tenant_id, dataset_id, page_type, slug):
    """Fetch one artifact page by (page_type, slug).

    GET /api/v1/datasets/<dataset_id>/artifacts/<page_type>/<slug>
    ``slug`` is the tail after the page type — the same form that markdown
    links in ``content_md_rendered`` carry as
    ``artifact/<kb_id>/<page_type>/<slug>``.
    Success: {"code": 0, "data": page_dict | null}
    """
    try:
        success, result = await dataset_api_service.get_wiki_page(
            dataset_id,
            tenant_id,
            page_type,
            slug,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/skills", methods=["HEAD"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def has_any_skill(tenant_id, dataset_id):
    """Probe whether this dataset has a compiled Corpus2Skill tree."""
    try:
        success, result = await dataset_api_service.has_any_skill(dataset_id, tenant_id)
        response = await make_response("", 200 if success and result["has"] else 404)
        return response
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/skills", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def get_skill_tree(tenant_id, dataset_id):
    """Fetch the aggregate recursive Corpus2Skill tree for this dataset.

    GET /api/v1/datasets/<dataset_id>/skills
    Success: {"code": 0, "data": skill_all_row | null}
    """
    try:
        success, result = await dataset_api_service.get_skill_tree(
            dataset_id,
            tenant_id,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/skills", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def delete_all_skills(tenant_id, dataset_id):
    """Delete every compiled skill for this dataset.

    DELETE /api/v1/datasets/<dataset_id>/skills
    Success: {"code": 0, "data": {"deleted": <n>}}
    """
    try:
        success, result = await dataset_api_service.delete_skills(
            dataset_id,
            tenant_id,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/skills/<path:skill_kwd>", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def get_skill_page(tenant_id, dataset_id, skill_kwd):
    """Fetch full markdown for one Corpus2Skill node by skill_kwd.

    GET /api/v1/datasets/<dataset_id>/skills/<skill_kwd>
    Success: {"code": 0, "data": skill_row | null}
    """
    try:
        success, result = await dataset_api_service.get_skill_page(
            dataset_id,
            tenant_id,
            skill_kwd,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/navigation", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def list_dataset_nav(tenant_id, dataset_id):
    """First level of the dataset navigation tree — the top-level clusters.

    GET /api/v1/datasets/<dataset_id>/navigation
    GET /api/v1/datasets/<dataset_id>/navigation?keywords=<query>&top_k=<n>
    Success: {"code": 0, "data": {"total": <n>, "items": [{name, description, doc_count, type, has_children, ...}, ...]}}
    """
    q = (request.args.get("keywords") or "").strip() or None
    top_k_raw = request.args.get("top_k")
    top_k = None
    if top_k_raw:
        try:
            top_k = max(1, int(top_k_raw))
        except (ValueError, TypeError):
            return get_error_data_result(message="top_k must be a positive integer")
    try:
        success, result = await dataset_api_service.list_nav_clusters(
            dataset_id,
            tenant_id,
            q=q,
            top_k=top_k,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/navigation/search", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def search_dataset_nav(tenant_id, dataset_id):
    """Unified navigation search across different knowledge layers.

    GET /api/v1/datasets/<dataset_id>/navigation/search?q=<query>&mode=<mode>&top_k=20&doc_ids=<id>,<id>

    Modes:
      - nav_doc:         navigation tree document leaves (default)
      - nav_cluster:     navigation tree cluster nodes
      - navigation_tree: tree-structured BFS beam descent
      - chunk:           raw document chunks (deduplicated by doc_id)
      - all:             union of all modes above

    ``doc_ids`` (comma-separated, optional) restricts the search to those
    documents; omitted → all documents of the dataset.

    Success: {"code": 0, "data": {"mode": <mode>, "total": <n>,
        "items": [{name, description, doc_count, type, has_children, doc_id, score, ...}, ...]}}
    """
    q = (request.args.get("q") or "").strip()
    if not q:
        return get_result(data={"mode": request.args.get("mode", "nav_doc"), "total": 0, "items": []})
    mode = (request.args.get("mode") or "nav_doc").strip()
    top_k_raw = request.args.get("top_k")
    top_k = None
    if top_k_raw:
        try:
            top_k = max(1, int(top_k_raw))
        except (ValueError, TypeError):
            return get_error_data_result(message="top_k must be a positive integer")
    doc_scope = [d.strip() for d in request.args.get("doc_ids", "").split(",") if d.strip()] or None
    try:
        success, result = await dataset_api_service.search_dataset_layers(
            dataset_id,
            tenant_id,
            q,
            mode,
            top_k=top_k,
            doc_scope=doc_scope,
        )
        if success:
            result["items"] = await dataset_api_service._enrich_nav_items(dataset_id, tenant_id, result.get("items", []))
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/navigation/<path:name>/children", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def list_dataset_nav_children(tenant_id, dataset_id, name):
    """Direct children of a navigation node (hierarchical, one level per call).

    GET /api/v1/datasets/<dataset_id>/navigation/<name>/children
    Success: {"code": 0, "data": {"total": <n>, "items": [{name, description, doc_count, type, doc_id, has_children}, ...]}}
    """
    try:
        success, result = await dataset_api_service.list_nav_children(
            dataset_id,
            tenant_id,
            name,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/navigation", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def delete_dataset_nav(tenant_id, dataset_id):
    """Delete the entire dataset navigation tree.

    DELETE /api/v1/datasets/<dataset_id>/navigation
    Success: {"code": 0, "data": {"deleted": <n>}}
    """
    try:
        success, result = await dataset_api_service.delete_nav(
            dataset_id,
            tenant_id,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/navigation/<path:name>", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def delete_dataset_nav_node(tenant_id, dataset_id, name):
    """Delete one navigation node and its whole subtree.

    DELETE /api/v1/datasets/<dataset_id>/navigation/<name>
    Success: {"code": 0, "data": {"deleted": <n>}}
    """
    try:
        success, result = await dataset_api_service.delete_nav_node(
            dataset_id,
            tenant_id,
            name,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/navigation", methods=["POST"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def generate_dataset_nav(tenant_id, dataset_id):
    """Create the entire navigation tree from all dataset documents.

    Deletes any existing navigation tree first, then rebuilds it from
    scratch. When ``documents`` is provided, only those docs are used;
    otherwise all docs in the dataset are auto-discovered.

    POST /api/v1/datasets/<dataset_id>/navigation
    Body (optional): {"documents": [{"doc_id": "...", "summary": "...",
           "doc_title": "... (optional)", "source_type": "... (optional)"}, ...]}
    Success: {"code": 0, "data": {"deleted": <n>, "upserted": <n>}}
    """
    try:
        req = await request.json or {}
        documents = req.get("documents")

        success, result = await dataset_api_service.generate_nav(
            dataset_id,
            tenant_id,
            documents,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/skills/<path:skill_kwd>", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def delete_skill_page(tenant_id, dataset_id, skill_kwd):
    """Delete one compiled skill node by skill_kwd.

    DELETE /api/v1/datasets/<dataset_id>/skills/<skill_kwd>
    Success: {"code": 0, "data": {"deleted": <n>}}
    """
    try:
        success, result = await dataset_api_service.delete_skill(
            dataset_id,
            tenant_id,
            skill_kwd,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


# The two artifact-commit endpoints
#   GET /datasets/<dataset_id>/artifacts/<page_type>/<path:slug>/commits
#   GET /datasets/<dataset_id>/artifacts/commits/<commit_id>
# were retired here — their functionality is now served by the generic
# file-commit routes:
#   GET /datasets/<dataset_id>/commits?slug=<page_type>/<name>
#   GET /datasets/<dataset_id>/commits/<commit_id>
# See ``api/apps/restful_apis/file_commit_api.py`` and
# ``api/db/services/file_commit_service.py`` (record_page_edit /
# list_page_commits / get_page_commit_detail).


@manager.route("/datasets/<dataset_id>/artifacts/<page_type>/<path:slug>", methods=["PUT"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def update_wiki_page(tenant_id, dataset_id, page_type, slug):
    """Edit one artifact page in place.

    PUT /api/v1/datasets/<dataset_id>/artifacts/<page_type>/<slug>
    Body: {"content_md": "<markdown>"}

    Only the page row is updated — canvas / entity / relation rows stay
    stale until the next artifact compile. Success returns the
    re-fetched page dict so the dialog can refresh its preview cleanly.
    """
    try:
        req = await request.get_json()
        if not isinstance(req, dict):
            return get_error_argument_result("Body must be a JSON object.")
        content_md = req.get("content_md")
        if not isinstance(content_md, str):
            return get_error_argument_result("'content_md' must be a string.")
        # Commit metadata — both optional. Title defaults server-side to
        # "Edit <slug>" inside record_edit when missing.
        title = req.get("title")
        comments = req.get("comments")
        if title is not None and not isinstance(title, str):
            return get_error_argument_result("'title' must be a string.")
        if comments is not None and not isinstance(comments, str):
            return get_error_argument_result("'comments' must be a string.")
        success, result = await dataset_api_service.update_wiki_page(
            dataset_id,
            tenant_id,
            page_type,
            slug,
            content_md,
            user_id=getattr(current_user, "id", None),
            title=title,
            comments=comments,
        )
        if success:
            return get_result(data=result)
        if isinstance(result, dict):
            msg = result.get("error", result)
            code = result.get("code", RetCode.SERVER_ERROR)
        else:
            msg = result
            code = RetCode.SERVER_ERROR
        return get_result(data=False, message=msg, code=code)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/index", methods=["POST"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def run_index(tenant_id, dataset_id):
    index_type = request.args.get("type", "")
    index_type = index_type.lower()
    try:
        success, result = dataset_api_service.run_index(dataset_id, tenant_id, index_type)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/index", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def trace_index(tenant_id, dataset_id):
    index_type = request.args.get("type", "")
    index_type = index_type.lower()
    try:
        success, result = dataset_api_service.trace_index(dataset_id, tenant_id, index_type)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/<index_type>", methods=["DELETE"])  # noqa: F821
@manager.route("/datasets/<dataset_id>/index", methods=["DELETE"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def delete_index(tenant_id, dataset_id, index_type=None):
    index_type = (index_type or request.args.get("type", "")).lower()
    if index_type not in dataset_api_service._VALID_INDEX_TYPES:
        return get_error_argument_result(f"Invalid index type '{index_type}'")
    # `wipe` controls whether the persisted index artefacts (graph rows /
    # raptor summaries) are removed. Default true preserves historical
    # behaviour; pass wipe=false to cancel the running task while keeping
    # prior progress so it can be resumed later.
    wipe_arg = (request.args.get("wipe", "true") or "true").strip().lower()
    wipe = wipe_arg not in ("false", "0", "no", "off")
    try:
        success, result = dataset_api_service.delete_index(dataset_id, tenant_id, index_type, wipe=wipe)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/embedding/check", methods=["POST"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def check_embedding(tenant_id, dataset_id):
    try:
        req = await request.get_json()
        if not req or not req.get("embd_id"):
            return get_error_data_result(message="`embd_id` is required.")
        status, result = dataset_api_service.check_embedding(dataset_id, tenant_id, req)
        if status is True:
            return get_result(data=result)
        elif status == "not_effective":
            return get_json_result(code=result["code"], message=result["message"], data=result["data"])
        else:
            return get_error_data_result(message=result)
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/ingestions", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def list_ingestion_logs(tenant_id, dataset_id):
    try:
        page = validate_rest_api_page(request.args.get("page", DEFAULT_PAGE))
        page_size = validate_rest_api_page_size(request.args.get("page_size", DEFAULT_PAGE_SIZE))
        orderby = request.args.get("orderby", "create_time")
        desc = request.args.get("desc", "true").lower() != "false"
        operation_status = request.args.getlist("operation_status")
        create_date_from = request.args.get("create_date_from", None)
        create_date_to = request.args.get("create_date_to", None)
        log_type = request.args.get("log_type", "dataset")
        keywords = request.args.get("keywords", None)
        success, result = dataset_api_service.list_ingestion_logs(dataset_id, tenant_id, page, page_size, orderby, desc, operation_status, create_date_from, create_date_to, log_type, keywords)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/ingestions/<log_id>", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def get_ingestion_log(tenant_id, dataset_id, log_id):
    try:
        success, result = dataset_api_service.get_ingestion_log(dataset_id, tenant_id, log_id)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/metadata/config", methods=["GET"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
def get_auto_metadata(tenant_id, dataset_id):
    """
    Get auto-metadata configuration for a dataset.
    ---
    tags:
      - Datasets
    security:
      - ApiKeyAuth: []
    parameters:
      - in: path
        name: dataset_id
        type: string
        required: true
        description: ID of the dataset.
      - in: header
        name: Authorization
        type: string
        required: true
        description: Bearer token for authentication.
    responses:
      200:
        description: Successful operation.
        schema:
          type: object
    """
    try:
        success, result = dataset_api_service.get_auto_metadata(dataset_id, tenant_id)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")


@manager.route("/datasets/<dataset_id>/metadata/config", methods=["PUT"])  # noqa: F821
@login_required
@add_tenant_id_to_kwargs
async def update_auto_metadata(tenant_id, dataset_id):
    """
    Update auto-metadata configuration for a dataset.
    ---
    tags:
      - Datasets
    security:
      - ApiKeyAuth: []
    parameters:
      - in: path
        name: dataset_id
        type: string
        required: true
        description: ID of the dataset.
      - in: header
        name: Authorization
        type: string
        required: true
        description: Bearer token for authentication.
      - in: body
        name: body
        description: Auto-metadata configuration.
        required: true
        schema:
          type: object
    responses:
      200:
        description: Successful operation.
        schema:
          type: object
    """
    from api.utils.validation_utils import AutoMetadataConfig

    cfg, err = await validate_and_parse_json_request(request, AutoMetadataConfig)
    if err is not None:
        return get_error_argument_result(err)

    try:
        success, result = await dataset_api_service.update_auto_metadata(dataset_id, tenant_id, cfg)
        if success:
            return get_result(data=result)
        else:
            return get_error_data_result(message=result)
    except ValueError as e:
        return get_error_argument_result(str(e))
    except Exception as e:
        logging.exception(e)
        return get_error_data_result(message="Internal server error")