refactor(tracing): one trace pipeline behind a minimal provider interface (#44673)

.devcontainer/post_create_command.sh

@@ -7,7 +7,7 @@ cd web && pnpm install
 pipx install uv
 
 echo "alias start-api=\"cd $WORKSPACE_ROOT/api && uv run python -m flask run --host 0.0.0.0 --port=5001 --debug\"" >> ~/.bashrc
-echo "alias start-worker=\"cd $WORKSPACE_ROOT/api && uv run python -m celery -A app.celery worker -P threads -c 1 --loglevel INFO -Q dataset,dataset_summary,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_publisher,trigger_refresh_executor,retention\"" >> ~/.bashrc
+echo "alias start-worker=\"$WORKSPACE_ROOT/dev/start-worker --pool threads\"" >> ~/.bashrc
 echo "alias start-web=\"cd $WORKSPACE_ROOT/web && pnpm dev\"" >> ~/.bashrc
 echo "alias start-web-prod=\"cd $WORKSPACE_ROOT/web && pnpm build && pnpm start\"" >> ~/.bashrc
 echo "alias start-containers=\"cd $WORKSPACE_ROOT/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env up -d\"" >> ~/.bashrc

AGENTS.md

@@ -8,4 +8,4 @@ Before starting backend development, read the [architecture](docs/BACKEND_ARCHIT
 
 - Run backend commands through `uv run --project api <command>`.
 - Backend integration tests are CI-only and are not expected to run locally.
-- Keep `docker/.env.example` limited to variables required for a default Docker Compose deployment to start. Put optional and provider-specific settings in the matching `docker/envs/*.env.example` file; `docker/.env` overrides those service-specific env files.
+- Keep `docker/.env.example` limited to settings that require manual user configuration. Omit settings whose built-in defaults work unchanged. Put optional and provider-specific settings in the matching `docker/envs/*.env.example` file; `docker/.env` overrides those service-specific env files.

api/.env.example

@@ -118,10 +118,11 @@ CELERY_BROKER_URL=redis://:difyai123456@localhost:${REDIS_PORT}/1
 CELERY_BACKEND=redis
 
 # Ops trace configuration
-OPS_TRACE_UNIFIED_ENABLED=false
-OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES=780
-OPS_TRACE_RETRYABLE_DISPATCH_DELAY_SECONDS=5
-OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS=3900
+# Keep new workers separate from legacy ops_trace workers during upgrades.
+OPS_TRACE_QUEUE=ops_trace_v2
+OPS_TRACE_EXPORT_MAX_ATTEMPTS=6
+# Daily cleanup of trace payloads an interrupted delivery left in storage.
+ENABLE_TRACE_PAYLOAD_SWEEP_TASK=true
 
 # Database configuration
 DB_TYPE=postgresql

api/.importlinter

@@ -6,6 +6,7 @@ root_packages =
     context
     configs
     controllers
+    domain
     extensions
     factories
     fields
@@ -17,6 +18,26 @@ root_packages =
     services
 include_external_packages = True
 
+[importlinter:contract:domain-boundary]
+name = Domain code depends on no application, persistence, framework, configuration, queue, transport, or legacy kernel code
+type = forbidden
+source_modules =
+    domain
+forbidden_modules =
+    celery
+    configs
+    controllers
+    core
+    enterprise
+    extensions
+    flask
+    models
+    repositories
+    services
+    sqlalchemy
+    tasks
+    werkzeug
+
 [importlinter:contract:backend-layers]
 name = Backend modules follow the transport, application, execution-kernel, and shared-library layers
 type = layers
@@ -1328,3 +1349,81 @@ forbidden_modules =
     sqlalchemy
     tasks
     werkzeug
+
+[importlinter:contract:tracing-application-boundary]
+name = Trace pipeline and Agent trace collection services and contracts are framework and persistence neutral
+type = forbidden
+source_modules =
+    services.tracing.canonical
+    services.tracing.collection
+    services.tracing.collection_ports
+    services.tracing.delivery
+    services.tracing.errors
+    services.tracing.hierarchy
+    services.tracing.provider_configs
+    services.tracing.providers
+    services.tracing.records
+    services.tracing.recovery
+    services.tracing.scopes
+    services.tracing.service
+forbidden_modules =
+    celery
+    configs
+    controllers
+    core
+    enterprise
+    extensions
+    flask
+    models
+    repositories
+    sqlalchemy
+    tasks
+    werkzeug
+    services.tracing.adapters
+    services.tracing.egress
+    services.tracing.registry
+    services.tracing.request_log_protection
+    services.tracing.runtime_bridge
+
+[importlinter:contract:tracing-controller-boundary]
+name = Tracing handlers reach persistence and tracing adapters only through the composition root
+type = forbidden
+source_modules =
+    controllers.console.agent.tracing
+    controllers.console.agent.tracing_runs
+    controllers.console.app.ops_trace
+    controllers.inner_api.agent.tracing
+forbidden_modules =
+    extensions.ext_database
+    models.application_trace
+    models.model
+    repositories
+    sqlalchemy
+    services.tracing.adapters
+    services.tracing.registry
+    services.tracing.runtime_bridge
+    services.tracing.service
+    domain
+# The composition root (extensions.ext_application_services) wires repositories and
+# adapters, so only direct imports are checked here.
+allow_indirect_imports = True
+
+[importlinter:contract:tracing-repository-boundary]
+name = Tracing persistence adapters only depend on the tracing contracts they implement
+type = forbidden
+source_modules =
+    repositories.app.tracing_config_repository
+    repositories.application_trace_repository
+    repositories.trace_record_repository
+    repositories.tracing_store_errors
+forbidden_modules =
+    controllers
+    extensions
+    flask
+    tasks
+    werkzeug
+    services.tracing.adapters
+    services.tracing.registry
+    services.tracing.runtime_bridge
+    services.tracing.service
+allow_indirect_imports = True

api/app_factory.py

@@ -186,7 +186,6 @@ def initialize_extensions(app: DifyApp):
         ext_database,
         ext_enterprise_telemetry,
         ext_fastopenapi,
-        ext_forward_refs,
         ext_hosting_provider,
         ext_import_modules,
         ext_key_provider,
@@ -218,7 +217,6 @@ def initialize_extensions(app: DifyApp):
         ext_warnings,
         ext_import_modules,
         ext_orjson,
-        ext_forward_refs,
         ext_compress,
         ext_code_based_extension,
         ext_database,

api/configs/feature/__init__.py

@@ -13,7 +13,6 @@ from pydantic import (
     PositiveInt,
     computed_field,
     field_validator,
-    model_validator,
 )
 from pydantic_settings import BaseSettings
 
@@ -1438,6 +1437,11 @@ class MultiModalTransferConfig(BaseSettings):
 
 
 class OpsTraceConfig(BaseSettings):
+    OPS_TRACE_QUEUE: str = Field(
+        default="ops_trace_v2",
+        pattern=r"^[A-Za-z0-9_.-]+$",
+        description="Celery queue for tracing and sandbox metering; isolate new workers during rolling upgrades.",
+    )
     OPS_AGENT_TRACE_ENABLED: bool = Field(
         default=False, description="Enable application-scoped Agent tracing collection and delivery."
     )
@@ -1463,37 +1467,11 @@ class OpsTraceConfig(BaseSettings):
         default=7, description="Retention of trace payloads and encrypted destination snapshots."
     )
 
-    OPS_TRACE_UNIFIED_ENABLED: bool = Field(
-        description="Enable unified ops tracing for providers registered in the unified registry.",
-        default=False,
+    OPS_TRACE_EXPORT_MAX_ATTEMPTS: PositiveInt = Field(
+        default=6,
+        description="Attempts to send one trace to its provider when the provider is temporarily unavailable.",
     )
 
-    # Include scheduling and export grace after the parent workflow's maximum execution time.
-    # Recommended: max_retries >= ceil((WORKFLOW_MAX_EXECUTION_TIME + grace_seconds) / delay_seconds).
-    OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES: PositiveInt = Field(
-        description="Maximum retry attempts for transient ops trace provider dispatch failures.",
-        default=780,
-    )
-
-    OPS_TRACE_RETRYABLE_DISPATCH_DELAY_SECONDS: PositiveInt = Field(
-        description="Delay in seconds between transient ops trace provider dispatch retry attempts.",
-        default=5,
-    )
-
-    OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS: PositiveInt = Field(
-        description="Retention in seconds for unified tracing parent contexts.",
-        default=3900,
-    )
-
-    @model_validator(mode="after")
-    def validate_parent_context_retention(self) -> "OpsTraceConfig":
-        if not self.OPS_TRACE_UNIFIED_ENABLED:
-            return self
-        retry_window = self.OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES * self.OPS_TRACE_RETRYABLE_DISPATCH_DELAY_SECONDS
-        if retry_window > self.OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS:
-            raise ValueError("OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS must cover the retry window")
-        return self
-
 
 class CeleryBeatConfig(BaseSettings):
     CELERY_BEAT_SCHEDULER_TIME: int = Field(
@@ -1503,6 +1481,10 @@ class CeleryBeatConfig(BaseSettings):
 
 
 class CeleryScheduleTasksConfig(BaseSettings):
+    ENABLE_TRACE_PAYLOAD_SWEEP_TASK: bool = Field(
+        description="Enable the daily cleanup of trace payloads that an interrupted delivery left in storage",
+        default=True,
+    )
     ENABLE_APPLICATION_TRACE_RECONCILE_TASK: bool = Field(
         description="Enable periodic application trace recovery and retention cleanup when Agent tracing is enabled",
         default=True,

api/controllers/console/agent/tracing.py

@@ -1,6 +1,7 @@
 """Agent tracing settings address Agent resources, never hidden inline Apps."""
 
 from collections.abc import Callable
+from dataclasses import asdict
 from typing import Literal
 from uuid import UUID
 
@@ -9,7 +10,6 @@ from flask_restx import Resource
 from pydantic import Field
 from werkzeug.exceptions import NotFound
 
-from configs import dify_config
 from controllers.common.rbac import AgentId, PlainApp, RBACCheck
 from controllers.common.schema import query_params_from_model, register_response_schema_models
 from controllers.console import console_ns
@@ -206,22 +206,22 @@ class AgentTraceConfigApi(Resource):
 
 
 def _provider_capabilities():
-    from services.tracing.provider_dispatch import provider_capabilities
-
     query = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))
     try:
-        capabilities = provider_capabilities(query.tracing_provider)
-    except ValueError as error:
+        capabilities = application_services().tracing.provider_capabilities.get(query.tracing_provider)
+    except AppTracingConfigInvalidProviderError as error:
         raise UnsupportedTracingProviderError() from error
+    except AppTracingConfigProviderUnavailableError as error:
+        raise TracingProviderUnavailableError() from error
     return dump_response(
         TraceProviderCapabilitiesResponse,
-        {**capabilities, "api_collection_enabled": dify_config.OPS_AGENT_TRACE_ENABLED},
+        {**asdict(capabilities.support), "api_collection_enabled": capabilities.api_collection_enabled},
     )
 
 
 @console_ns.route("/agent/<uuid:agent_id>/trace-capabilities")
 class AgentTraceCapabilitiesApi(Resource):
-    @error_responses(UnsupportedTracingProviderError)
+    @error_responses(TracingProviderUnavailableError, UnsupportedTracingProviderError)
     @console_ns.doc("get_agent_trace_capabilities", params=query_params_from_model(TraceProviderQuery))
     @console_ns.response(
         200, "Provider support declarations", console_ns.models[TraceProviderCapabilitiesResponse.__name__]
@@ -234,7 +234,7 @@ class AgentTraceCapabilitiesApi(Resource):
 
 @console_ns.route("/apps/<uuid:app_id>/trace-capabilities")
 class AppTraceCapabilitiesApi(Resource):
-    @error_responses(UnsupportedTracingProviderError)
+    @error_responses(TracingProviderUnavailableError, UnsupportedTracingProviderError)
     @console_ns.doc("get_app_trace_capabilities", params=query_params_from_model(TraceProviderQuery))
     @console_ns.response(
         200, "Provider support declarations", console_ns.models[TraceProviderCapabilitiesResponse.__name__]

api/controllers/console/agent/tracing_runs.py

@@ -23,9 +23,8 @@ from libs.external_api import error_responses
 from libs.helper import dump_response
 from machinery.context import RequestContext
 from models.account import TenantAccountRole
-from repositories.application_trace_repository import TraceCollectionError
 from services.app.agent_app_contracts import AgentAppNotFoundError
-from services.tracing.service import application_tracing_service
+from services.tracing.errors import TraceCollectionError
 
 type TraceSourceKind = Literal["message", "workflow", "node"]
 _source_kind_adapter = TypeAdapter(TraceSourceKind)
@@ -155,7 +154,7 @@ def _agent_app_id(context: RequestContext, agent_id: UUID) -> str:
 def _status(context: RequestContext, app_id: str, kind: TraceSourceKind, source_id: str):
     node_id = _query_node_id()
     result = _run_call(
-        lambda: application_tracing_service().status(
+        lambda: application_services().tracing.agent_traces.status(
             tenant_id=context.active_workspace_id,
             app_id=app_id,
             source_kind=kind,
@@ -170,7 +169,7 @@ def _retry(context: RequestContext, app_id: str, kind: TraceSourceKind, source_i
     node_id = _query_node_id()
     payload = TraceRunRetryPayload.model_validate(console_ns.payload or {})
     result = _run_call(
-        lambda: application_tracing_service().retry(
+        lambda: application_services().tracing.agent_traces.retry(
             tenant_id=context.active_workspace_id,
             app_id=app_id,
             source_kind=kind,

api/controllers/console/app/ops_trace.py

@@ -73,7 +73,12 @@ class TraceAppConfigResponse(ResponseModel):
     id: str | None = None
     app_id: str | None = None
     tracing_provider: str | None = None
-    tracing_config: dict[str, str | None] | None = Field(default=None)
+    tracing_config: dict[str, str | None] | None = Field(
+        default=None, description="Non-secret fields; secrets are write-only and never returned"
+    )
+    configured_secrets: list[str] | None = Field(
+        default=None, description="Secret fields that hold a value; submit them empty to keep that value"
+    )
     is_active: bool | None = None
     created_at: str | None = None
     updated_at: str | None = None
@@ -85,6 +90,7 @@ class TraceAppConfigResponse(ResponseModel):
             app_id=record.app_id,
             tracing_provider=record.tracing_provider,
             tracing_config=record.tracing_config,
+            configured_secrets=list(record.configured_secrets),
             is_active=record.is_active,
             created_at=str(record.created_at),
             updated_at=str(record.updated_at),

api/controllers/inner_api/agent/tracing.py

@@ -13,9 +13,14 @@ from configs import dify_config
 from controllers.common.schema import register_response_schema_models, register_schema_models
 from controllers.inner_api import inner_api_ns
 from controllers.inner_api.wraps import agent_inner_api_only
+from extensions.ext_application_services import application_services
 from fields.base import ResponseModel
-from repositories.application_trace_repository import TraceCollectionError
-from services.tracing.service import application_tracing_service
+from services.tracing.errors import (
+    TraceCollectionConflictError,
+    TraceCollectionError,
+    TraceCollectionLimitError,
+    TraceCollectionNotFoundError,
+)
 
 
 class AgentTraceBatchResponse(ResponseModel):
@@ -38,12 +43,17 @@ class AgentTraceBatchesApi(Resource):
         request.max_content_length = dify_config.OPS_AGENT_TRACE_BATCH_MAX_BYTES
         try:
             payload = AgentTraceBatch.model_validate(inner_api_ns.payload or {})
-            accepted = application_tracing_service().ingest(payload)
+            accepted = application_services().tracing.agent_traces.ingest(payload)
         except ValidationError as error:
             raise BadRequest("Invalid Agent trace batch") from error
+        except TraceCollectionNotFoundError as error:
+            raise NotFound(error.code) from error
+        except TraceCollectionConflictError as error:
+            raise Conflict(error.code) from error
+        except TraceCollectionLimitError as error:
+            raise RequestEntityTooLarge(error.code) from error
         except TraceCollectionError as error:
-            error_type = {404: NotFound, 409: Conflict, 413: RequestEntityTooLarge}.get(error.status, BadRequest)
-            raise error_type(error.code) from error
+            raise BadRequest(error.code) from error
         return AgentTraceBatchResponse(result="success", batch_id=payload.batch_id, accepted_spans=accepted).model_dump(
             mode="json"
         )

api/core/agent/cot_agent_runner.py

@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
 from collections.abc import Generator, Mapping, Sequence
 from typing import Any, TypedDict
 
+from dify_trace_sdk.emitter import TraceScope
 from sqlalchemy.orm import Session
 
 from core.agent.base_agent_runner import BaseAgentRunner
@@ -15,7 +16,6 @@ from core.app.apps.base_app_queue_manager import PublishFrom
 from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueMessageEndEvent, QueueMessageFileEvent
 from core.credit_usage import CreditUsageAppType, CreditUsageCreatedBy
 from core.model_context import use_credit_usage_metadata
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
 from core.tools.__base.tool import Tool
 from core.tools.entities.tool_entities import ToolInvokeMeta
@@ -64,7 +64,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         self._repack_app_generate_entity(app_generate_entity)
         self._init_react_state(query)
 
-        trace_manager = app_generate_entity.trace_manager
+        trace_scope = app_generate_entity.trace_scope
 
         # check model mode
         if "Observation" not in app_generate_entity.model_conf.stop:
@@ -247,7 +247,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                         action=scratchpad.action,
                         tool_instances=tool_instances,
                         message_file_ids=message_file_ids,
-                        trace_manager=trace_manager,
+                        trace_scope=trace_scope,
                     )
                     scratchpad.observation = tool_invoke_response
                     scratchpad.agent_response = tool_invoke_response
@@ -314,14 +314,14 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         action: AgentScratchpadUnit.Action,
         tool_instances: Mapping[str, Tool],
         message_file_ids: list[str],
-        trace_manager: TraceQueueManager | None = None,
+        trace_scope: TraceScope | None,
     ) -> tuple[str, ToolInvokeMeta]:
         """
         handle invoke action
         :param action: action
         :param tool_instances: tool instances
         :param message_file_ids: message file ids
-        :param trace_manager: trace manager
+        :param trace_scope: where the tool call is reported, if anywhere
         :return: observation, meta
         """
         # action is tool call, invoke tool
@@ -348,7 +348,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 message=self.message,
                 invoke_from=self.application_generate_entity.invoke_from,
                 agent_tool_callback=self.agent_callback,
-                trace_manager=trace_manager,
+                trace_scope=trace_scope,
             )
         session.commit()
         session.close()

api/core/agent/fc_agent_runner.py

@@ -125,8 +125,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         final_answer = ""
         prompt_messages: list = []  # Initialize prompt_messages
 
-        # get tracing instance
-        trace_manager = app_generate_entity.trace_manager
+        trace_scope = app_generate_entity.trace_scope
 
         def increase_usage(final_llm_usage_dict: dict[str, LLMUsage | None], usage: LLMUsage):
             if not final_llm_usage_dict["usage"]:
@@ -331,7 +330,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
                             message=self.message,
                             invoke_from=self.application_generate_entity.invoke_from,
                             agent_tool_callback=self.agent_callback,
-                            trace_manager=trace_manager,
+                            trace_scope=trace_scope,
                             app_id=self.application_generate_entity.app_config.app_id,
                             message_id=self.message.id,
                             conversation_id=self.conversation.id,

api/core/app/apps/advanced_chat/app_generator.py

@@ -43,11 +43,11 @@ from core.app.entities.task_entities import (
 )
 from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig
 from core.helper.trace_id_helper import extract_external_trace_id_from_args, extract_trace_session_id_from_args
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.prompt.utils.get_thread_messages_length import get_thread_messages_length
 from core.repositories import DifyCoreRepositoryFactory
 from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository
 from extensions.ext_database import db
+from extensions.tracing import create_trace_scope
 from factories import file_factory
 from graphon.engine.filter import ResponseStreamFilter
 from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
@@ -200,9 +200,12 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             # convert to app config
             app_config = AdvancedChatAppConfigManager.get_app_config(app_model=app_model, workflow=workflow)
 
-            # get tracing instance
-            trace_manager = TraceQueueManager(
-                app_id=app_model.id, user_id=user.id if isinstance(user, Account) else user.session_id
+            trace_scope = create_trace_scope(
+                tenant_id=app_model.tenant_id,
+                app_id=app_model.id,
+                app_tracing=app_model.tracing,
+                user_id=user.id if isinstance(user, Account) else user.session_id,
+                extras=extras,
             )
 
             if invoke_from == InvokeFrom.DEBUGGER:
@@ -229,7 +232,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
                 stream=streaming,
                 invoke_from=invoke_from,
                 extras=extras,
-                trace_manager=trace_manager,
+                trace_scope=trace_scope,
                 workflow_run_id=str(workflow_run_id),
             )
             contexts.plugin_tool_providers.set({})
@@ -292,15 +295,18 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         """
         Resume a paused advanced chat execution.
 
-        ``trace_manager`` is transient and excluded from generate-entity serialization,
-        so resumed executions rebuild it here before persistence layers receive the entity.
+        ``trace_scope`` is transient and excluded from generate-entity serialization,
+        so resumed executions create it again before persistence layers receive the entity.
         """
-        if application_generate_entity.trace_manager is None:
+        if application_generate_entity.trace_scope is None:
             application_generate_entity = application_generate_entity.model_copy(
                 update={
-                    "trace_manager": TraceQueueManager(
+                    "trace_scope": create_trace_scope(
+                        tenant_id=app_model.tenant_id,
                         app_id=app_model.id,
+                        app_tracing=app_model.tracing,
                         user_id=user.id if isinstance(user, Account) else user.session_id,
+                        extras=application_generate_entity.extras,
                     )
                 }
             )

api/core/app/apps/advanced_chat/app_runner.py

@@ -360,7 +360,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
             inputs=dict(inputs),
             query=query or "",
             message_id=message_id,
-            trace_manager=app_generate_entity.trace_manager,
+            trace_scope=app_generate_entity.trace_scope,
         )
 
     def _initialize_conversation_variables(self) -> list[Variable]:

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -8,6 +8,7 @@ from datetime import datetime
 from threading import Thread
 from typing import Any
 
+from dify_trace_sdk.events import MessageCompleted
 from sqlalchemy import select, update
 from sqlalchemy.orm import Session
 
@@ -72,8 +73,6 @@ from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTas
 from core.app.task_pipeline.message_cycle_manager import MessageCycleManager
 from core.base.tts import AppGeneratorTTSPublisher
 from core.db.session_factory import session_factory
-from core.ops.entities.trace_entity import TraceTaskName
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.workflow.file_reference import resolve_file_record_id
 from core.workflow.nodes.human_input.pause_reason import HumanInputRequired
 from core.workflow.system_variables import build_system_variables
@@ -229,7 +228,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             message_id=self._message_id,
         )
 
-        generator = self._wrapper_process_stream_response(trace_manager=self._application_generate_entity.trace_manager)
+        generator = self._wrapper_process_stream_response()
 
         if self._base_task_pipeline.stream:
             return self._to_stream_response(generator)
@@ -350,9 +349,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             return None
         raise RuntimeError(f"TTS publisher returned an unknown status: {audio_msg.status}")
 
-    def _wrapper_process_stream_response(
-        self, trace_manager: TraceQueueManager | None = None
-    ) -> Generator[StreamResponse]:
+    def _wrapper_process_stream_response(self) -> Generator[StreamResponse]:
         tts_publisher = None
         task_id = self._application_generate_entity.task_id
         tenant_id = self._application_generate_entity.app_config.tenant_id
@@ -372,7 +369,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             )
 
         try:
-            for response in self._process_stream_response(tts_publisher=tts_publisher, trace_manager=trace_manager):
+            for response in self._process_stream_response(tts_publisher=tts_publisher):
                 while audio_response := self._listen_audio_msg(publisher=tts_publisher, task_id=task_id):
                     yield audio_response
                 if tts_publisher and isinstance(response, ErrorStreamResponse):
@@ -651,12 +648,9 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _handle_workflow_succeeded_event(
         self,
         event: QueueWorkflowSucceededEvent,
-        *,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse]:
         """Handle workflow succeeded events."""
-        _ = trace_manager
         self._ensure_workflow_initialized()
         validated_state = self._ensure_graph_runtime_initialized()
         workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
@@ -674,12 +668,9 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _handle_workflow_partial_success_event(
         self,
         event: QueueWorkflowPartialSuccessEvent,
-        *,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse]:
         """Handle workflow partial success events."""
-        _ = trace_manager
         self._ensure_workflow_initialized()
         validated_state = self._ensure_graph_runtime_initialized()
         workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
@@ -738,12 +729,9 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _handle_workflow_failed_event(
         self,
         event: QueueWorkflowFailedEvent,
-        *,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse]:
         """Handle workflow failed events."""
-        _ = trace_manager
         self._ensure_workflow_initialized()
         validated_state = self._ensure_graph_runtime_initialized()
 
@@ -768,11 +756,9 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         event: QueueStopEvent,
         *,
         graph_runtime_state: RuntimeState | None = None,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse]:
         """Handle stop events."""
-        _ = trace_manager
         resolved_state = None
         if self._workflow_run_id:
             resolved_state = self._resolve_graph_runtime_state(graph_runtime_state)
@@ -935,7 +921,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         event: Any,
         *,
         tts_publisher: AppGeneratorTTSPublisher | None = None,
-        trace_manager: TraceQueueManager | None = None,
         queue_message: WorkflowQueueMessage | MessageQueueMessage | None = None,
     ) -> Generator[StreamResponse]:
         """Dispatch events using elegant pattern matching."""
@@ -947,7 +932,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             yield from handler(
                 event,
                 tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
                 queue_message=queue_message,
             )
             return
@@ -963,7 +947,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             yield from self._handle_node_failed_events(
                 event,
                 tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
                 queue_message=queue_message,
             )
             return
@@ -974,7 +957,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _process_stream_response(
         self,
         tts_publisher: AppGeneratorTTSPublisher | None = None,
-        trace_manager: TraceQueueManager | None = None,
     ) -> Generator[StreamResponse]:
         """
         Process stream response using elegant Fluent Python patterns.
@@ -993,22 +975,22 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
                     break
 
                 case QueueWorkflowFailedEvent():
-                    yield from self._handle_workflow_failed_event(event, trace_manager=trace_manager)
+                    yield from self._handle_workflow_failed_event(event)
                     break
                 case QueueWorkflowPausedEvent():
                     yield from self._handle_workflow_paused_event(event)
                     break
 
                 case QueueWorkflowSucceededEvent():
-                    yield from self._handle_workflow_succeeded_event(event, trace_manager=trace_manager)
+                    yield from self._handle_workflow_succeeded_event(event)
                     break
 
                 case QueueWorkflowPartialSuccessEvent():
-                    yield from self._handle_workflow_partial_success_event(event, trace_manager=trace_manager)
+                    yield from self._handle_workflow_partial_success_event(event)
                     break
 
                 case QueueStopEvent():
-                    yield from self._handle_stop_event(event, graph_runtime_state=None, trace_manager=trace_manager)
+                    yield from self._handle_stop_event(event, graph_runtime_state=None)
                     break
 
                 # Handle all other events through elegant dispatch
@@ -1017,7 +999,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
                         self._dispatch_event(
                             event,
                             tts_publisher=tts_publisher,
-                            trace_manager=trace_manager,
                             queue_message=queue_message,
                         )
                     ):
@@ -1086,16 +1067,10 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         session.add_all(message_files)
 
     def _emit_message_trace(self) -> None:
-        trace_manager = self._application_generate_entity.trace_manager
-        if trace_manager:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.MESSAGE_TRACE,
-                    conversation_id=self._conversation_id,
-                    message_id=self._message_id,
-                    trace_session_id=self._application_generate_entity.extras.get("trace_session_id"),
-                )
-            )
+        """Report the finished message once its records are committed."""
+        trace_scope = self._application_generate_entity.trace_scope
+        if trace_scope is not None:
+            trace_scope.emit(MessageCompleted(message_id=self._message_id, conversation_id=self._conversation_id))
 
     def _seed_graph_runtime_state_from_queue_manager(self) -> None:
         """Bootstrap the cached runtime state from the queue manager when present."""

api/core/app/apps/agent_app/app_generator.py

@@ -49,8 +49,8 @@ from core.app.entities.app_invoke_entities import (
 )
 from core.credit_usage import CreditUsageAppType
 from core.db.session_factory import session_factory
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
+from extensions.tracing import application_tracing_hooks, create_trace_scope
 from factories import file_factory
 from models import Account, App, AppModelConfig, Conversation, EndUser, Message, MessageAnnotation
 from models.agent import (
@@ -142,7 +142,14 @@ class AgentAppGenerator(MessageBasedAppGenerator):
                 else []
             )
 
-        trace_manager = TraceQueueManager(app_model.id, user.id if isinstance(user, Account) else user.session_id)
+        extras = {"auto_generate_conversation_name": args.get("auto_generate_name", True)}
+        trace_scope = create_trace_scope(
+            tenant_id=app_model.tenant_id,
+            app_id=app_model.id,
+            app_tracing=app_model.tracing,
+            user_id=user.id if isinstance(user, Account) else user.session_id,
+            extras=extras,
+        )
         application_generate_entity = AgentAppGenerateEntity(
             task_id=str(uuid.uuid4()),
             app_config=app_config,
@@ -162,11 +169,9 @@ class AgentAppGenerator(MessageBasedAppGenerator):
             user_id=user.id,
             stream=streaming,
             invoke_from=invoke_from,
-            extras={
-                "auto_generate_conversation_name": args.get("auto_generate_name", True),
-            },
+            extras=extras,
             call_depth=0,
-            trace_manager=trace_manager,
+            trace_scope=trace_scope,
             agent_id=agent.id,
             agent_config_snapshot_id=agent_config_id,
             agent_config_version_kind=agent_config_version_kind,
@@ -314,8 +319,6 @@ class AgentAppGenerator(MessageBasedAppGenerator):
                 # A local stop or failed cancellation wait is still just intent.
                 pass
             except Exception as e:
-                from core.ops.base_trace_instance import application_tracing_hooks
-
                 tracing_hooks = application_tracing_hooks()
                 if tracing_hooks is not None:
                     tracing_hooks.capture_agent_app_terminal(
@@ -387,7 +390,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):
                 inputs=dict(application_generate_entity.inputs),
                 query=query or "",
                 message_id=message.id,
-                trace_manager=application_generate_entity.trace_manager,
+                trace_scope=application_generate_entity.trace_scope,
             )
         except ModerationError as e:
             publish_text_answer(queue_manager=queue_manager, model_name=model_name, answer=str(e), user_query=query)

api/core/app/apps/agent_app/app_runner.py

@@ -50,6 +50,7 @@ from core.app.entities.queue_entities import (
     QueueRetrieverResourcesEvent,
 )
 from extensions.ext_database import db
+from extensions.tracing import application_tracing_hooks
 from graphon.file import File
 from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from graphon.model_runtime.entities.message_entities import (
@@ -765,8 +766,6 @@ class AgentAppRunner:
             binding_id=runtime.binding_id,
             snapshot=terminal.session_snapshot,
         )
-        from core.ops.base_trace_instance import application_tracing_hooks
-
         tracing_hooks = application_tracing_hooks()
         if tracing_hooks is not None:
             tracing_hooks.capture_agent_app_terminal(
@@ -1059,8 +1058,6 @@ class AgentAppRunner:
             # snapshot/usage, but let the original business failure own the root.
             # Check after cleanup so a Stop observed during its wait still wins.
             return
-        from core.ops.base_trace_instance import application_tracing_hooks
-
         tracing_hooks = application_tracing_hooks()
         if tracing_hooks is not None:
             outcome: Literal["succeeded", "failed", "cancelled"]

api/core/app/apps/agent_app/runtime_request_builder.py

@@ -37,7 +37,6 @@ from clients.agent_backend import (
 from configs import dify_config
 from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
 from core.app.llm.model_access import resolve_model_context_window, resolve_model_supports_vision
-from core.ops.base_trace_instance import application_tracing_hooks
 from core.plugin.provider_identity import normalize_plugin_daemon_provider_identity
 from core.workflow.file_reference import build_file_reference, is_canonical_file_reference
 from core.workflow.nodes.agent_v2.dify_tools_builder import (
@@ -54,6 +53,7 @@ from core.workflow.nodes.agent_v2.runtime_request_builder import (
     build_shell_layer_config,
     load_runtime_agent_skill_configs,
 )
+from extensions.tracing import application_tracing_hooks
 from graphon.file import File, FileTransferMethod, FileType, file_manager
 from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent
 from models.agent_config_entities import AgentKnowledgeSpaceConfig, AgentSoulConfig, AgentSoulToolsConfig

api/core/app/apps/agent_chat/app_generator.py

@@ -23,7 +23,7 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, InvokeFrom
 from core.db.session_factory import session_factory
 from core.helper.trace_id_helper import extract_trace_session_id_from_args
-from core.ops.ops_trace_manager import TraceQueueManager
+from extensions.tracing import create_trace_scope
 from factories import file_factory
 from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from libs.flask_utils import preserve_flask_contexts
@@ -179,8 +179,13 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
                 annotation_reply=annotation_reply,
             )
 
-            # get tracing instance
-            trace_manager = TraceQueueManager(app_model.id, user.id if isinstance(user, Account) else user.session_id)
+            trace_scope = create_trace_scope(
+                tenant_id=app_model.tenant_id,
+                app_id=app_model.id,
+                app_tracing=app_model.tracing,
+                user_id=user.id if isinstance(user, Account) else user.session_id,
+                extras=extras,
+            )
 
             # init application generate entity
             application_generate_entity = AgentChatAppGenerateEntity(
@@ -204,7 +209,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
                 invoke_from=invoke_from,
                 extras=extras,
                 call_depth=0,
-                trace_manager=trace_manager,
+                trace_scope=trace_scope,
             )
 
             # init generate records

api/core/app/apps/base_app_runner.py

@@ -482,7 +482,7 @@ class AppRunner:
             inputs=dict(inputs),
             query=query or "",
             message_id=message_id,
-            trace_manager=app_generate_entity.trace_manager,
+            trace_scope=app_generate_entity.trace_scope,
         )
 
     def check_hosting_moderation(

api/core/app/apps/chat/app_generator.py

@@ -23,8 +23,8 @@ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueMa
 from core.app.entities.app_invoke_entities import ChatAppGenerateEntity, InvokeFrom
 from core.db.session_factory import session_factory
 from core.helper.trace_id_helper import extract_trace_session_id_from_args
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
+from extensions.tracing import create_trace_scope
 from factories import file_factory
 from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from models import Account
@@ -168,9 +168,12 @@ class ChatAppGenerator(MessageBasedAppGenerator):
                 annotation_reply=annotation_reply,
             )
 
-            # get tracing instance
-            trace_manager = TraceQueueManager(
-                app_id=app_model.id, user_id=user.id if isinstance(user, Account) else user.session_id
+            trace_scope = create_trace_scope(
+                tenant_id=app_model.tenant_id,
+                app_id=app_model.id,
+                app_tracing=app_model.tracing,
+                user_id=user.id if isinstance(user, Account) else user.session_id,
+                extras=extras,
             )
 
             # init application generate entity
@@ -193,7 +196,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
                 user_id=user.id,
                 invoke_from=invoke_from,
                 extras=extras,
-                trace_manager=trace_manager,
+                trace_scope=trace_scope,
                 stream=streaming,
             )
 

api/core/app/apps/completion/app_generator.py

@@ -23,8 +23,8 @@ from core.app.entities.app_invoke_entities import CompletionAppGenerateEntity, I
 from core.app.entities.task_entities import AppBlockingResponse, AppStreamResponse
 from core.db.session_factory import session_factory
 from core.helper.trace_id_helper import extract_trace_session_id_from_args
-from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
+from extensions.tracing import create_trace_scope
 from factories import file_factory
 from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
 from models import Account, App, Conversation, EndUser, Message
@@ -151,9 +151,13 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
                 annotation_reply=annotation_reply,
             )
 
-            # get tracing instance
-            trace_manager = TraceQueueManager(
-                app_id=app_model.id, user_id=user.id if isinstance(user, Account) else user.session_id
+            extras = extract_trace_session_id_from_args(args)
+            trace_scope = create_trace_scope(
+                tenant_id=app_model.tenant_id,
+                app_id=app_model.id,
+                app_tracing=app_model.tracing,
+                user_id=user.id if isinstance(user, Account) else user.session_id,
+                extras=extras,
             )
 
             # init application generate entity
@@ -170,10 +174,8 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
                 user_id=user.id,
                 stream=streaming,
                 invoke_from=invoke_from,
-                extras={
-                    **extract_trace_session_id_from_args(args),
-                },
-                trace_manager=trace_manager,
+                extras=extras,
+                trace_scope=trace_scope,
             )
 
             # init generate records

api/core/app/apps/workflow/app_generator.py

@@ -38,12 +38,12 @@ from core.helper.trace_id_helper import (
     extract_parent_trace_context_from_args,
     extract_trace_session_id_from_args,
 )
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.repositories import DifyCoreRepositoryFactory
 from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository
 from core.trigger.constants import is_trigger_node_type
 from core.workflow.node_factory import get_default_root_node_id
 from extensions.ext_database import db
+from extensions.tracing import create_trace_scope
 from factories import file_factory
 from graphon.engine.filter import ResponseStreamFilter
 from graphon.engine.layer import Layer
@@ -197,12 +197,6 @@ class WorkflowAppGenerator(BaseAppGenerator):
                 workflow=workflow,
             )
 
-            # get tracing instance
-            trace_manager = TraceQueueManager(
-                app_id=app_model.id,
-                user_id=user.id if isinstance(user, Account) else user.session_id,
-            )
-
             inputs: Mapping[str, Any] = args["inputs"]
 
             extras = {
@@ -210,6 +204,13 @@ class WorkflowAppGenerator(BaseAppGenerator):
                 **extract_parent_trace_context_from_args(args),
                 **extract_trace_session_id_from_args(args),
             }
+            trace_scope = create_trace_scope(
+                tenant_id=app_model.tenant_id,
+                app_id=app_model.id,
+                app_tracing=app_model.tracing,
+                user_id=user.id if isinstance(user, Account) else user.session_id,
+                extras=extras,
+            )
             workflow_run_id = str(workflow_run_id or uuid.uuid4())
             root_node_id = root_node_id or get_default_root_node_id(workflow.graph_dict)
             root_node_config = workflow.get_node_config_by_id(root_node_id)
@@ -235,7 +236,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
                 invoke_from=invoke_from,
                 allow_human_input=allow_human_input,
                 call_depth=call_depth,
-                trace_manager=trace_manager,
+                trace_scope=trace_scope,
                 workflow_execution_id=workflow_run_id,
                 extras=extras,
             )
@@ -303,15 +304,18 @@ class WorkflowAppGenerator(BaseAppGenerator):
         """
         Resume a paused workflow execution using the persisted runtime state.
 
-        ``trace_manager`` is transient and excluded from generate-entity serialization,
-        so resumed executions rebuild it here before persistence layers receive the entity.
+        ``trace_scope`` is transient and excluded from generate-entity serialization,
+        so resumed executions create it again before persistence layers receive the entity.
         """
-        if application_generate_entity.trace_manager is None:
+        if application_generate_entity.trace_scope is None:
             application_generate_entity = application_generate_entity.model_copy(
                 update={
-                    "trace_manager": TraceQueueManager(
+                    "trace_scope": create_trace_scope(
+                        tenant_id=app_model.tenant_id,
                         app_id=app_model.id,
+                        app_tracing=app_model.tracing,
                         user_id=user.id if isinstance(user, Account) else user.session_id,
+                        extras=application_generate_entity.extras,
                     )
                 }
             )

api/core/app/apps/workflow/generate_task_pipeline.py

@@ -57,7 +57,6 @@ from core.app.entities.task_entities import (
 )
 from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline
 from core.base.tts import AppGeneratorTTSPublisher
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.workflow.system_variables import build_system_variables
 from extensions.ext_database import db
 from graphon.entities import WorkflowStartReason
@@ -127,7 +126,7 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         Process generate task pipeline.
         :return:
         """
-        generator = self._wrapper_process_stream_response(trace_manager=self._application_generate_entity.trace_manager)
+        generator = self._wrapper_process_stream_response()
         if self._base_task_pipeline.stream:
             return self._to_stream_response(generator)
         else:
@@ -247,9 +246,7 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             return None
         raise RuntimeError(f"TTS publisher returned an unknown status: {audio_msg.status}")
 
-    def _wrapper_process_stream_response(
-        self, trace_manager: TraceQueueManager | None = None
-    ) -> Generator[StreamResponse]:
+    def _wrapper_process_stream_response(self) -> Generator[StreamResponse]:
         tts_publisher = None
         task_id = self._application_generate_entity.task_id
         tenant_id = self._application_generate_entity.app_config.tenant_id
@@ -269,7 +266,7 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             )
 
         try:
-            for response in self._process_stream_response(tts_publisher=tts_publisher, trace_manager=trace_manager):
+            for response in self._process_stream_response(tts_publisher=tts_publisher):
                 while audio_response := self._listen_audio_msg(publisher=tts_publisher, task_id=task_id):
                     yield audio_response
                 if tts_publisher and isinstance(response, ErrorStreamResponse):
@@ -468,12 +465,9 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _handle_workflow_succeeded_event(
         self,
         event: QueueWorkflowSucceededEvent,
-        *,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse]:
         """Handle workflow succeeded events."""
-        _ = trace_manager
         self._ensure_workflow_initialized()
         validated_state = self._ensure_graph_runtime_initialized()
         workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
@@ -488,12 +482,9 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _handle_workflow_partial_success_event(
         self,
         event: QueueWorkflowPartialSuccessEvent,
-        *,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse]:
         """Handle workflow partial success events."""
-        _ = trace_manager
         self._ensure_workflow_initialized()
         validated_state = self._ensure_graph_runtime_initialized()
         workflow_finish_resp = self._workflow_response_converter.workflow_finish_to_stream_response(
@@ -523,12 +514,9 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _handle_workflow_failed_and_stop_events(
         self,
         event: QueueWorkflowFailedEvent | QueueStopEvent,
-        *,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse]:
         """Handle workflow failed and stop events."""
-        _ = trace_manager
         self._ensure_workflow_initialized()
         validated_state = self._ensure_graph_runtime_initialized()
 
@@ -641,7 +629,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         event: AppQueueEvent,
         *,
         tts_publisher: AppGeneratorTTSPublisher | None = None,
-        trace_manager: TraceQueueManager | None = None,
         queue_message: WorkflowQueueMessage | MessageQueueMessage | None = None,
     ) -> Generator[StreamResponse]:
         """Dispatch events using elegant pattern matching."""
@@ -653,7 +640,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             yield from handler(
                 event,
                 tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
                 queue_message=queue_message,
             )
             return
@@ -669,7 +655,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             yield from self._handle_node_failed_events(
                 event,
                 tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
                 queue_message=queue_message,
             )
             return
@@ -679,7 +664,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             yield from self._handle_workflow_failed_and_stop_events(
                 event,
                 tts_publisher=tts_publisher,
-                trace_manager=trace_manager,
                 queue_message=queue_message,
             )
             return
@@ -690,7 +674,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
     def _process_stream_response(
         self,
         tts_publisher: AppGeneratorTTSPublisher | None = None,
-        trace_manager: TraceQueueManager | None = None,
     ) -> Generator[StreamResponse]:
         """
         Process stream response using elegant Fluent Python patterns.
@@ -730,7 +713,6 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
                         self._dispatch_event(
                             event,
                             tts_publisher=tts_publisher,
-                            trace_manager=trace_manager,
                             queue_message=queue_message,
                         )
                     ):

api/core/app/entities/app_invoke_entities.py

@@ -1,7 +1,8 @@
 from collections.abc import Mapping, Sequence
 from enum import StrEnum
-from typing import TYPE_CHECKING, Any, Literal
+from typing import Any, Literal
 
+from dify_trace_sdk.emitter import TraceScope
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
 
 from constants import UUID_NIL
@@ -21,10 +22,6 @@ from graphon.model_runtime.entities.model_entities import AIModelEntity
 from models.model import AppMode
 from repositories.entities.workflow_execution import WorkflowExecutionTiming
 
-if TYPE_CHECKING:
-    from core.ops.ops_trace_manager import TraceQueueManager
-
-
 DIFY_RUN_CONTEXT_KEY = "_dify"
 
 
@@ -204,8 +201,8 @@ class AppGenerateEntity(BaseModel):
     # extra parameters, like: auto_generate_conversation_name
     extras: dict[str, Any] = Field(default_factory=dict)
 
-    # tracing instance
-    trace_manager: "TraceQueueManager | None" = Field(default=None, exclude=True, repr=False)
+    # Process-local: a resumed execution creates its scope again.
+    trace_scope: TraceScope | None = Field(default=None, exclude=True, repr=False)
 
 
 class EasyUIBasedAppGenerateEntity(AppGenerateEntity):
@@ -387,17 +384,3 @@ class RagPipelineGenerateEntity(WorkflowAppGenerateEntity):
     document_id: str | None = None
     original_document_id: str | None = None
     start_node_id: str | None = None
-
-
-# Resolve circular model references after model definitions.
-from core.ops.ops_trace_manager import TraceQueueManager  # noqa: E402
-
-AppGenerateEntity.model_rebuild()
-EasyUIBasedAppGenerateEntity.model_rebuild()
-ConversationAppGenerateEntity.model_rebuild()
-ChatAppGenerateEntity.model_rebuild()
-CompletionAppGenerateEntity.model_rebuild()
-AgentChatAppGenerateEntity.model_rebuild()
-AdvancedChatAppGenerateEntity.model_rebuild()
-WorkflowAppGenerateEntity.model_rebuild()
-RagPipelineGenerateEntity.model_rebuild()

api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py

@@ -5,6 +5,7 @@ from collections.abc import Generator, Mapping, Sequence
 from threading import Thread
 from typing import Any, cast
 
+from dify_trace_sdk.events import MessageCompleted
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -49,8 +50,6 @@ from core.app.task_pipeline.message_file_utils import prepare_file_dict
 from core.base.tts import AppGeneratorTTSPublisher
 from core.db.session_factory import session_factory
 from core.model_manager import ModelInstance
-from core.ops.entities.trace_entity import TraceTaskName
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.prompt.utils.prompt_message_util import PromptMessageUtil
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
 from events.message_event import message_was_created
@@ -130,7 +129,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
                 message_id=self._message_id,
             )
 
-        generator = self._wrapper_process_stream_response(trace_manager=self._application_generate_entity.trace_manager)
+        generator = self._wrapper_process_stream_response()
         if self.stream:
             return self._to_stream_response(generator)
         else:
@@ -218,9 +217,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
             return None
         raise RuntimeError(f"TTS publisher returned an unknown status: {audio_msg.status}")
 
-    def _wrapper_process_stream_response(
-        self, trace_manager: TraceQueueManager | None = None
-    ) -> Generator[StreamResponse]:
+    def _wrapper_process_stream_response(self) -> Generator[StreamResponse]:
         tenant_id = self._application_generate_entity.app_config.tenant_id
         task_id = self._application_generate_entity.task_id
         publisher = None
@@ -238,7 +235,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
                 get_credit_usage_app_type(self._app_config.app_mode),
             )
         try:
-            for response in self._process_stream_response(publisher=publisher, trace_manager=trace_manager):
+            for response in self._process_stream_response(publisher=publisher):
                 while audio_response := self._listen_audio_msg(publisher, task_id):
                     yield audio_response
                 if publisher and isinstance(response, ErrorStreamResponse):
@@ -271,9 +268,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
             if publisher:
                 publisher.cancel()
 
-    def _process_stream_response(
-        self, publisher: AppGeneratorTTSPublisher | None, trace_manager: TraceQueueManager | None = None
-    ) -> Generator[StreamResponse]:
+    def _process_stream_response(self, publisher: AppGeneratorTTSPublisher | None) -> Generator[StreamResponse]:
         """
         Process stream response.
         :return:
@@ -288,16 +283,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
                     with session_factory.create_session() as session:
                         err = self.handle_error(event=event, session=session, message_id=self._message_id)
                         session.commit()
-
-                    if trace_manager:
-                        trace_manager.add_trace_task(
-                            TraceTask(
-                                TraceTaskName.MESSAGE_TRACE,
-                                conversation_id=self._conversation_id,
-                                message_id=self._message_id,
-                                trace_session_id=self._application_generate_entity.extras.get("trace_session_id"),
-                            )
-                        )
+                    self._emit_message_trace()
 
                     yield self.error_to_stream_response(err)
                     break
@@ -323,15 +309,11 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
                         # A stopped Agent run may persist provider-reported usage after
                         # cancellation completes. Do not replace it with local token estimates.
                         if isinstance(event, QueueStopEvent):
-                            saved_message = self._save_message(
-                                session=session,
-                                trace_manager=trace_manager,
-                                preserve_existing_usage=True,
-                            )
+                            saved_message = self._save_message(session=session, preserve_existing_usage=True)
                             if self._app_config.app_mode == AppMode.AGENT:
                                 stopped_agent_message = saved_message
                         else:
-                            self._save_message(session=session, trace_manager=trace_manager)
+                            self._save_message(session=session)
                         session.commit()
                     if stopped_agent_message is not None:
                         # Provider/billing subscribers may perform external I/O.
@@ -340,6 +322,9 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
                             stopped_agent_message,
                             application_generate_entity=self._application_generate_entity,
                         )
+                    else:
+                        # The Agent worker reports a stopped Agent run's terminal itself.
+                        self._emit_message_trace()
                     message_end_resp = self._message_end_to_stream_response()
                     yield message_end_resp
                 case QueueRetrieverResourcesEvent():
@@ -436,7 +421,6 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
         self,
         *,
         session: Session,
-        trace_manager: TraceQueueManager | None = None,
         preserve_existing_usage: bool = False,
     ) -> Message:
         """
@@ -500,16 +484,6 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
             metadata["agent_stop_requested"] = True
         message.message_metadata = json.dumps(metadata, ensure_ascii=False)
 
-        if trace_manager and not agent_stop_requested:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.MESSAGE_TRACE,
-                    conversation_id=self._conversation_id,
-                    message_id=self._message_id,
-                    trace_session_id=self._application_generate_entity.extras.get("trace_session_id"),
-                )
-            )
-
         if not agent_stop_requested:
             message_was_created.send(
                 message,
@@ -517,6 +491,12 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
             )
         return message
 
+    def _emit_message_trace(self) -> None:
+        """Report the finished message once its records are committed."""
+        trace_scope = self._application_generate_entity.trace_scope
+        if trace_scope is not None:
+            trace_scope.emit(MessageCompleted(message_id=self._message_id, conversation_id=self._conversation_id))
+
     def _handle_stop(self, event: QueueStopEvent):
         """
         Handle stop.

api/core/app/task_pipeline/message_cycle_manager.py

@@ -145,6 +145,8 @@ class MessageCycleManager:
                                 conversation_id,
                                 conversation.app_id,
                                 message_id=message_id,
+                                # The request's scope keeps the caller's trace ID with the message.
+                                trace_scope=self._application_generate_entity.trace_scope,
                             )
                             redis_client.setex(cache_key, 3600, name)
                         except Exception:

api/core/callback_handler/agent_tool_callback_handler.py

@@ -4,8 +4,6 @@ from typing import Any, TextIO
 from pydantic import BaseModel
 
 from configs import dify_config
-from core.ops.entities.trace_entity import TraceTaskName
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.tools.entities.tool_entities import ToolInvokeMessage
 
 _TEXT_COLOR_MAPPING = {
@@ -58,9 +56,6 @@ class DifyAgentCallbackHandler(BaseModel):
         tool_name: str,
         tool_inputs: Mapping[str, Any],
         tool_outputs: Iterable[ToolInvokeMessage] | str,
-        message_id: str | None = None,
-        timer: Any | None = None,
-        trace_manager: TraceQueueManager | None = None,
     ):
         """If not the final action, print out observation."""
         if dify_config.DEBUG:
@@ -70,18 +65,6 @@ class DifyAgentCallbackHandler(BaseModel):
             print_text("Outputs: " + str(tool_outputs)[:1000] + "\n", color=self.color)
             print_text("\n")
 
-        if trace_manager:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.TOOL_TRACE,
-                    message_id=message_id,
-                    tool_name=tool_name,
-                    tool_inputs=tool_inputs,
-                    tool_outputs=tool_outputs,
-                    timer=timer,
-                )
-            )
-
     def on_tool_error(self, error: Exception | KeyboardInterrupt, **kwargs: Any):
         """Do nothing."""
         if dify_config.DEBUG:

api/core/callback_handler/workflow_tool_callback_handler.py

@@ -3,7 +3,6 @@ from typing import Any
 
 from configs import dify_config
 from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler, print_text
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.tools.entities.tool_entities import ToolInvokeMessage
 
 
@@ -15,9 +14,6 @@ class DifyWorkflowCallbackHandler(DifyAgentCallbackHandler):
         tool_name: str,
         tool_inputs: Mapping[str, Any],
         tool_outputs: Iterable[ToolInvokeMessage],
-        message_id: str | None = None,
-        timer: Any | None = None,
-        trace_manager: TraceQueueManager | None = None,
     ) -> Generator[ToolInvokeMessage]:
         for tool_output in tool_outputs:
             if dify_config.DEBUG:

api/core/llm_generator/llm_generator.py

@@ -5,6 +5,8 @@ from collections.abc import Sequence
 from typing import Any, Literal, Protocol, TypedDict, cast
 
 import json_repair
+from dify_trace_sdk.emitter import TraceScope
+from dify_trace_sdk.events import ConversationNamed
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -24,9 +26,6 @@ from core.llm_generator.prompts import (
 )
 from core.model_context import with_credit_usage_created_by
 from core.model_manager import ModelInstance, ModelManager
-from core.ops.entities.trace_entity import TraceTaskName
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
-from core.ops.utils import measure_time
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
 from core.telemetry import PromptGenerationEvent, TelemetryContext
 from core.telemetry import emit as telemetry_emit
@@ -37,6 +36,7 @@ from graphon.model_runtime.entities.llm_entities import LLMResult
 from graphon.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage
 from graphon.model_runtime.entities.model_entities import ModelType
 from graphon.model_runtime.errors.invoke import InvokeError
+from libs.datetime_utils import Timing, measure_time
 from models import App, Message, WorkflowNodeExecutionModel
 from models.workflow import Workflow
 
@@ -141,7 +141,7 @@ class LLMGenerator:
         generated_output: str,
         model_provider: str,
         model_name: str,
-        timer: dict,
+        timer: Timing,
         result: LLMResult | None = None,
         error: str | None = None,
     ) -> None:
@@ -165,11 +165,12 @@ class LLMGenerator:
                         "prompt_tokens": usage.prompt_tokens if usage else 0,
                         "completion_tokens": usage.completion_tokens if usage else 0,
                         "total_tokens": usage.total_tokens if usage else 0,
-                        "latency": (timer["end"] - timer["start"]).total_seconds() if timer.get("end") else 0.0,
+                        "latency": (timer.ended_at - timer.started_at).total_seconds(),
                         "total_price": float(usage.total_price) if usage and usage.total_price else None,
                         "currency": usage.currency if usage else None,
-                        "timer": timer,
                         "error": error,
+                        "started_at": timer.started_at,
+                        "ended_at": timer.ended_at,
                     },
                 )
             )
@@ -187,6 +188,7 @@ class LLMGenerator:
         message_id: str | None = None,
         *,
         model_instance: ModelInstance | None = None,
+        trace_scope: TraceScope | None,
     ) -> str:
         """Generate with a resolved model, or resolve it for legacy background callers.
 
@@ -234,19 +236,17 @@ class LLMGenerator:
         if len(name) > 75:
             name = name[:75] + "..."
 
-        # get tracing instance
-        trace_manager = TraceQueueManager(app_id=app_id)
-        trace_manager.add_trace_task(
-            TraceTask(
-                TraceTaskName.GENERATE_NAME_TRACE,
-                conversation_id=conversation_id,
-                message_id=message_id,
-                generate_conversation_name=name,
-                inputs=prompt,
-                timer=timer,
-                tenant_id=tenant_id,
+        if trace_scope is not None:
+            trace_scope.emit(
+                ConversationNamed(
+                    conversation_id=conversation_id,
+                    message_id=message_id,
+                    inputs=prompt,
+                    name=name,
+                    started_at=timer.started_at,
+                    ended_at=timer.ended_at,
+                )
             )
-        )
 
         return name
 

api/core/moderation/input_moderation.py

@@ -2,14 +2,15 @@ import logging
 from collections.abc import Mapping
 from typing import Any
 
+from dify_trace_sdk.emitter import TraceScope
+from dify_trace_sdk.events import ModerationChecked, json_safe
+
 from core.app.app_config.entities import AppConfig
 from core.app.entities.app_invoke_entities import get_credit_usage_app_type
 from core.model_context import use_credit_usage_metadata
 from core.moderation.base import ModerationAction, ModerationError
 from core.moderation.factory import ModerationFactory
-from core.ops.entities.trace_entity import TraceTaskName
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
-from core.ops.utils import measure_time
+from libs.datetime_utils import measure_time
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +24,7 @@ class InputModeration:
         inputs: Mapping[str, Any],
         query: str,
         message_id: str,
-        trace_manager: TraceQueueManager | None = None,
+        trace_scope: TraceScope | None,
     ) -> tuple[bool, Mapping[str, Any], str]:
         """
         Process sensitive_word_avoidance.
@@ -33,7 +34,7 @@ class InputModeration:
         :param inputs: inputs
         :param query: query
         :param message_id: message id
-        :param trace_manager: trace manager
+        :param trace_scope: where the moderation verdict is reported, if anywhere
         :return:
         """
         inputs = dict(inputs)
@@ -51,14 +52,17 @@ class InputModeration:
             with measure_time() as timer:
                 moderation_result = moderation_factory.moderation_for_inputs(inputs, query)
 
-        if trace_manager:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.MODERATION_TRACE,
+        if trace_scope is not None:
+            trace_scope.emit(
+                ModerationChecked(
                     message_id=message_id,
-                    moderation_result=moderation_result,
-                    inputs=inputs,
-                    timer=timer,
+                    inputs={key: json_safe(value) for key, value in inputs.items()},
+                    query=moderation_result.query,
+                    flagged=moderation_result.flagged,
+                    action=moderation_result.action.value,
+                    preset_response=moderation_result.preset_response,
+                    started_at=timer.started_at,
+                    ended_at=timer.ended_at,
                 )
             )
 

api/core/ops/base_trace_instance.py

@@ -1,144 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Literal, Protocol, cast
-
-from flask import Flask, current_app, has_app_context
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from configs import dify_config
-from core.ops.entities.config_entity import BaseTracingConfig
-from core.ops.entities.trace_entity import BaseTraceInfo
-from core.ops.exceptions import RetryableTraceDispatchError
-from extensions.ext_database import db
-from models import Account, App
-
-if TYPE_CHECKING:
-    from dify_agent.layers.execution_context import ApplicationTracingContext
-
-    from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
-    from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
-    from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
-
-
-class ApplicationTracingHooks(Protocol):
-    """Execution-owned port for the application tracing lifecycle.
-
-    The composition root supplies an app-scoped implementation. Core callers
-    carry trusted execution context and do not construct services or vendor SDKs.
-    """
-
-    def prepare_agent_app(self, context: AgentAppRuntimeBuildContext) -> ApplicationTracingContext | None: ...
-
-    def prepare_workflow_agent(
-        self, context: WorkflowAgentRuntimeBuildContext, conversation_id: str | None
-    ) -> ApplicationTracingContext | None: ...
-
-    def freeze_workflow_run(
-        self,
-        entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity,
-        *,
-        workflow_id: str,
-        workflow_run_id: str,
-        conversation_id: object,
-        graph_data: object,
-    ) -> None: ...
-
-    def queue_enabled_at_start(self, app_id: object) -> bool: ...
-
-    def capture_terminal(self, info: BaseTraceInfo) -> bool: ...
-
-    def capture_agent_app_terminal(
-        self,
-        *,
-        tenant_id: str,
-        app_id: str,
-        message_id: str,
-        outcome: Literal["succeeded", "failed", "cancelled"],
-        output: str | None,
-    ) -> None: ...
-
-
-class ApplicationTracingHooksNotRegisteredError(RetryableTraceDispatchError):
-    """Enabled tracing must not silently fall back to a different export path."""
-
-    def __init__(self) -> None:
-        super().__init__("application_tracing_hooks_not_registered")
-
-
-_APPLICATION_TRACING_HOOKS_KEY = "application_tracing_hooks"
-
-
-def bind_application_tracing_hooks(app: Flask, hooks: ApplicationTracingHooks) -> None:
-    """Bind the execution port to one Flask application, never a process singleton."""
-    app.extensions[_APPLICATION_TRACING_HOOKS_KEY] = hooks
-
-
-def application_tracing_hooks() -> ApplicationTracingHooks | None:
-    """Resolve the current app's port; a disabled deployment needs no Flask context."""
-    if not dify_config.OPS_AGENT_TRACE_ENABLED:
-        return None
-    if not has_app_context():
-        raise ApplicationTracingHooksNotRegisteredError()
-    hooks = current_app.extensions.get(_APPLICATION_TRACING_HOOKS_KEY)
-    if hooks is None:
-        raise ApplicationTracingHooksNotRegisteredError()
-    return cast(ApplicationTracingHooks, hooks)
-
-
-class BaseTraceInstance(ABC):
-    """
-    Base trace instance for ops trace services
-    """
-
-    def __init__(self, trace_config: BaseTracingConfig):
-        """
-        Initializer for the trace instance.
-        Distribute trace tasks by matching entities
-        """
-        self.trace_config = trace_config
-
-    @abstractmethod
-    def trace(self, trace_info: BaseTraceInfo):
-        """
-        Abstract method to trace activities.
-        Subclasses must implement specific tracing logic for activities.
-        """
-        ...
-
-    def get_service_account_with_tenant(self, app_id: str) -> Account:
-        """
-        Get the app creator with the app's tenant and membership role.
-
-        Trace tasks may run after the creator switches workspaces. Bind only the
-        returned account's context without changing the creator's current workspace.
-
-        Args:
-            app_id: The ID of the app
-
-        Returns:
-            Account: The service account with the app's tenant set up
-
-        Raises:
-            ValueError: If the app, creator account, app tenant or membership cannot be found
-        """
-        with Session(db.engine, expire_on_commit=False) as session:
-            # Get the app to find its creator
-            app_stmt = select(App).where(App.id == app_id)
-            app = session.scalar(app_stmt)
-            if not app:
-                raise ValueError(f"App with id {app_id} not found")
-
-            if not app.created_by:
-                raise ValueError(f"App with id {app_id} has no creator (created_by is None)")
-            account_stmt = select(Account).where(Account.id == app.created_by)
-            service_account = session.scalar(account_stmt)
-            if not service_account:
-                raise ValueError(f"Creator account with id {app.created_by} not found for app {app_id}")
-
-            service_account.set_tenant_id_with_session(app.tenant_id, session=session)
-            if service_account.current_tenant_id != app.tenant_id:
-                raise ValueError(f"App tenant {app.tenant_id} not found for account {service_account.id}")
-
-            return service_account

api/core/ops/entities/config_entity.py

@@ -1,64 +0,0 @@
-from enum import StrEnum
-
-from pydantic import BaseModel
-
-from core.ops.utils import validate_project_name, validate_url
-
-
-class TracingProviderEnum(StrEnum):
-    ARIZE = "arize"
-    PHOENIX = "phoenix"
-    LANGFUSE = "langfuse"
-    LANGSMITH = "langsmith"
-    OPIK = "opik"
-    WEAVE = "weave"
-    ALIYUN = "aliyun"
-    MLFLOW = "mlflow"
-    DATABRICKS = "databricks"
-    TENCENT = "tencent"
-
-
-class BaseTracingConfig(BaseModel):
-    """
-    Base model class for tracing configurations
-    """
-
-    @classmethod
-    def validate_endpoint_url(cls, v: str, default_url: str) -> str:
-        """
-        Common endpoint URL validation logic
-
-        Args:
-            v: URL value to validate
-            default_url: Default URL to use if input is None or empty
-
-        Returns:
-            Validated and normalized URL
-        """
-        return validate_url(v, default_url)
-
-    @classmethod
-    def validate_project_field(cls, v: str, default_name: str) -> str:
-        """
-        Common project name validation logic
-
-        Args:
-            v: Project name to validate
-            default_name: Default name to use if input is None or empty
-
-        Returns:
-            Validated project name
-        """
-        return validate_project_name(v, default_name)
-
-
-OPS_FILE_PATH = "ops_trace/"
-OPS_TRACE_FAILED_KEY = "FAILED_OPS_TRACE"
-
-
-def ops_trace_payload_path(app_id: str, file_id: str) -> str:
-    return f"{OPS_FILE_PATH}{app_id}/{file_id}.json"
-
-
-def workflow_final_trace_file_id(workflow_run_id: str) -> str:
-    return f"workflow-final-{workflow_run_id}"

api/core/ops/entities/trace_entity.py

@@ -1,292 +0,0 @@
-from collections.abc import Mapping
-from datetime import datetime
-from enum import StrEnum
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict, field_serializer, field_validator
-
-from core.helper.trace_id_helper import ParentTraceContext
-
-
-class BaseTraceInfo(BaseModel):
-    operation_id: str | None = None
-    message_id: str | None = None
-    message_data: Any | None = None
-    inputs: str | dict[str, Any] | list[Any] | None = None
-    outputs: str | dict[str, Any] | list[Any] | None = None
-    start_time: datetime | None = None
-    end_time: datetime | None = None
-    metadata: dict[str, Any]
-    trace_id: str | None = None
-
-    @field_validator("inputs", "outputs")
-    @classmethod
-    def ensure_type(cls, v: str | dict[str, Any] | list[Any] | None) -> str | dict[str, Any] | list[Any] | None:
-        if v is None:
-            return None
-        if isinstance(v, str | dict | list):
-            return v
-        return ""
-
-    model_config = ConfigDict(protected_namespaces=())
-
-    @property
-    def resolved_trace_id(self) -> str | None:
-        """Get trace_id with intelligent fallback.
-
-        Priority:
-        1. External trace_id (from X-Trace-Id header)
-        2. workflow_run_id (if this trace type has it)
-        3. message_id (as final fallback)
-        """
-        if self.trace_id:
-            return self.trace_id
-
-        # Try workflow_run_id (only exists on workflow-related traces)
-        workflow_run_id = getattr(self, "workflow_run_id", None)
-        if workflow_run_id:
-            return workflow_run_id
-
-        # Final fallback to message_id
-        return str(self.message_id) if self.message_id else None
-
-    @property
-    def resolved_parent_context(self) -> tuple[str | None, str | None]:
-        """Resolve cross-workflow parent linking from metadata.
-
-        Extracts typed parent IDs from the ``parent_trace_context`` metadata
-        payload (set by tool_node when invoking nested workflows).
-
-        Returns:
-            (trace_correlation_override, parent_span_id_source) where
-            trace_correlation_override is the outer workflow_run_id and
-            parent_span_id_source is the outer node_execution_id.
-        """
-        parent_ctx = self.metadata.get("parent_trace_context")
-        match parent_ctx:
-            case ParentTraceContext():
-                context = parent_ctx
-            case Mapping():
-                try:
-                    context = ParentTraceContext.model_validate(parent_ctx)
-                except ValueError:
-                    return None, None
-            case _:
-                return None, None
-        return (
-            context.parent_workflow_run_id,
-            context.parent_node_execution_id,
-        )
-
-    @field_serializer("start_time", "end_time")
-    def serialize_datetime(self, dt: datetime | None) -> str | None:
-        if dt is None:
-            return None
-        return dt.isoformat()
-
-
-class WorkflowTraceInfo(BaseTraceInfo):
-    workflow_data: Any = None
-    conversation_id: str | None = None
-    workflow_app_log_id: str | None = None
-    workflow_id: str
-    tenant_id: str
-    workflow_run_id: str
-    workflow_run_elapsed_time: int | float
-    workflow_run_status: str
-    workflow_run_inputs: Mapping[str, Any]
-    workflow_run_outputs: Mapping[str, Any]
-    workflow_run_version: str
-    error: str | None = None
-    total_tokens: int
-    prompt_tokens: int | None = None
-    completion_tokens: int | None = None
-    file_list: list[str]
-    invoked_by: str | None = None
-    query: str
-    metadata: dict[str, Any]
-
-
-class MessageTraceInfo(BaseTraceInfo):
-    conversation_model: str
-    message_tokens: int
-    answer_tokens: int
-    total_tokens: int
-    error: str | None = None
-    file_list: str | dict[str, Any] | list[Any] | None = None
-    message_file_data: Any | None = None
-    conversation_mode: str
-    gen_ai_server_time_to_first_token: float | None = None
-    llm_streaming_time_to_generate: float | None = None
-    is_streaming_request: bool = False
-
-
-class ModerationTraceInfo(BaseTraceInfo):
-    flagged: bool
-    action: str
-    preset_response: str
-    query: str
-
-
-class SuggestedQuestionTraceInfo(BaseTraceInfo):
-    total_tokens: int
-    status: str | None = None
-    error: str | None = None
-    from_account_id: str | None = None
-    agent_based: bool | None = None
-    from_source: str | None = None
-    model_provider: str | None = None
-    model_id: str | None = None
-    suggested_question: list[str]
-    level: str
-    status_message: str | None = None
-    workflow_run_id: str | None = None
-
-    model_config = ConfigDict(protected_namespaces=())
-
-
-class DatasetRetrievalTraceInfo(BaseTraceInfo):
-    documents: Any = None
-    error: str | None = None
-
-
-class ToolTraceInfo(BaseTraceInfo):
-    tool_name: str
-    tool_inputs: dict[str, Any]
-    tool_outputs: str
-    metadata: dict[str, Any]
-    message_file_data: Any = None
-    error: str | None = None
-    tool_config: dict[str, Any]
-    time_cost: int | float
-    tool_parameters: dict[str, Any]
-    file_url: str | None | list[str] = None
-
-
-class GenerateNameTraceInfo(BaseTraceInfo):
-    conversation_id: str | None = None
-    tenant_id: str
-
-
-class PromptGenerationTraceInfo(BaseTraceInfo):
-    """Trace information for prompt generation operations (rule-generate, code-generate, etc.)."""
-
-    tenant_id: str
-    user_id: str
-    app_id: str | None = None
-
-    operation_type: str
-    instruction: str
-
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-    model_provider: str
-    model_name: str
-
-    latency: float
-
-    total_price: float | None = None
-    currency: str | None = None
-
-    error: str | None = None
-
-    model_config = ConfigDict(protected_namespaces=())
-
-
-class WorkflowNodeTraceInfo(BaseTraceInfo):
-    workflow_id: str
-    workflow_run_id: str
-    tenant_id: str
-    node_execution_id: str
-    node_id: str
-    node_type: str
-    title: str
-
-    status: str
-    error: str | None = None
-    elapsed_time: float
-
-    index: int
-    predecessor_node_id: str | None = None
-
-    total_tokens: int = 0
-    total_price: float = 0.0
-    currency: str | None = None
-
-    model_provider: str | None = None
-    model_name: str | None = None
-    prompt_tokens: int | None = None
-    completion_tokens: int | None = None
-
-    tool_name: str | None = None
-
-    iteration_id: str | None = None
-    iteration_index: int | None = None
-    loop_id: str | None = None
-    loop_index: int | None = None
-    parallel_id: str | None = None
-
-    node_inputs: Mapping[str, Any] | None = None
-    node_outputs: Mapping[str, Any] | None = None
-    process_data: Mapping[str, Any] | None = None
-
-    invoked_by: str | None = None
-
-    model_config = ConfigDict(protected_namespaces=())
-
-
-class DraftNodeExecutionTrace(WorkflowNodeTraceInfo):
-    pass
-
-
-class TaskData(BaseModel):
-    app_id: str
-    trace_info_type: str
-    trace_info: Any = None
-
-
-trace_info_info_map = {
-    "WorkflowTraceInfo": WorkflowTraceInfo,
-    "MessageTraceInfo": MessageTraceInfo,
-    "ModerationTraceInfo": ModerationTraceInfo,
-    "SuggestedQuestionTraceInfo": SuggestedQuestionTraceInfo,
-    "DatasetRetrievalTraceInfo": DatasetRetrievalTraceInfo,
-    "ToolTraceInfo": ToolTraceInfo,
-    "GenerateNameTraceInfo": GenerateNameTraceInfo,
-    "PromptGenerationTraceInfo": PromptGenerationTraceInfo,
-    "WorkflowNodeTraceInfo": WorkflowNodeTraceInfo,
-    "DraftNodeExecutionTrace": DraftNodeExecutionTrace,
-}
-
-
-class OperationType(StrEnum):
-    """Operation type for token metric labels.
-
-    Used as a metric attribute on ``dify.tokens.input`` / ``dify.tokens.output``
-    counters so consumers can break down token usage by operation.
-    """
-
-    WORKFLOW = "workflow"
-    NODE_EXECUTION = "node_execution"
-    MESSAGE = "message"
-    RULE_GENERATE = "rule_generate"
-    CODE_GENERATE = "code_generate"
-    STRUCTURED_OUTPUT = "structured_output"
-    INSTRUCTION_MODIFY = "instruction_modify"
-
-
-class TraceTaskName(StrEnum):
-    CONVERSATION_TRACE = "conversation"
-    DRAFT_NODE_EXECUTION_TRACE = "draft_node_execution"
-    WORKFLOW_TRACE = "workflow"
-    MESSAGE_TRACE = "message"
-    MODERATION_TRACE = "moderation"
-    SUGGESTED_QUESTION_TRACE = "suggested_question"
-    DATASET_RETRIEVAL_TRACE = "dataset_retrieval"
-    TOOL_TRACE = "tool"
-    GENERATE_NAME_TRACE = "generate_conversation_name"
-    PROMPT_GENERATION_TRACE = "prompt_generation"
-    NODE_EXECUTION_TRACE = "node_execution"
-    DATASOURCE_TRACE = "datasource"

api/core/ops/exceptions.py

@@ -1,37 +0,0 @@
-"""Core exceptions shared by ops trace dispatchers and trace providers.
-
-Provider packages may raise these types to request generic task behavior, but
-generic Celery tasks should not import provider-specific exception classes.
-"""
-
-
-class TraceProviderNotInstalledError(ImportError):
-    def __init__(self, tracing_provider: str, module_name: str) -> None:
-        super().__init__(f"Tracing provider {tracing_provider} requires the missing module {module_name}.")
-
-
-class RetryableTraceDispatchError(RuntimeError):
-    """Base class for transient trace dispatch failures that Celery may retry."""
-
-    acceptance_unknown: bool = False
-
-
-class TraceParentContextAccessError(RetryableTraceDispatchError):
-    """Raised when unified parent context storage is temporarily unavailable."""
-
-
-class InvalidTraceParentContextError(RuntimeError):
-    """Raised when stored unified parent context cannot be safely restored."""
-
-
-class PendingTraceParentContextError(RetryableTraceDispatchError):
-    """Raised when a nested trace arrives before its parent span context is available."""
-
-    parent_node_execution_id: str
-
-    def __init__(self, parent_node_execution_id: str) -> None:
-        self.parent_node_execution_id = parent_node_execution_id
-        super().__init__(
-            "Pending trace parent context for parent_node_execution_id="
-            f"{parent_node_execution_id}. Retry after the parent span context is published."
-        )

api/core/ops/unified_trace/__init__.py

@@ -1 +0,0 @@
-"""Provider-neutral trace construction and dispatch for opt-in unified tracing."""

api/core/ops/unified_trace/entities.py

@@ -1,79 +0,0 @@
-"""Provider-independent trace entities emitted by the unified trace builder."""
-
-from datetime import datetime
-from enum import StrEnum
-from typing import Any, Self
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from core.helper.trace_id_helper import ParentTraceContext
-
-
-class CanonicalSpanKind(StrEnum):
-    CHAIN = "chain"
-    LLM = "llm"
-    RETRIEVER = "retriever"
-    TOOL = "tool"
-    AGENT = "agent"
-
-
-class CanonicalSpanStatus(StrEnum):
-    OK = "ok"
-    ERROR = "error"
-
-
-class CanonicalSpan(BaseModel):
-    """One provider-neutral operation with an explicit parent."""
-
-    id: str
-    parent_id: str | None
-    name: str
-    kind: CanonicalSpanKind
-    start_time: datetime
-    end_time: datetime | None
-    inputs: Any = None
-    outputs: Any = None
-    status: CanonicalSpanStatus
-    error: str | None = None
-    metadata: dict[str, Any] = Field(default_factory=dict)
-    synthetic: bool = False
-    can_parent_workflow: bool = False
-    publishes_parent_context: bool = False
-    links: tuple[str, ...] = ()
-
-    model_config = ConfigDict(extra="forbid", frozen=True)
-
-
-class CanonicalTrace(BaseModel):
-    """A parent-before-child span tree ready for a provider adapter."""
-
-    trace_id: str
-    session_id: str
-    root_span_id: str
-    spans: tuple[CanonicalSpan, ...]
-    external_parent: ParentTraceContext | None = None
-    required_parent_context_id: str | None = None
-
-    model_config = ConfigDict(extra="forbid", frozen=True)
-
-    @model_validator(mode="after")
-    def validate_fragment(self) -> Self:
-        if self.external_parent is not None and self.required_parent_context_id is not None:
-            raise ValueError("canonical trace cannot require two external parent modes")
-
-        seen: set[str] = set()
-        root_seen = False
-        for span in self.spans:
-            if span.id in seen:
-                raise ValueError(f"duplicate canonical span id: {span.id}")
-            if span.id == self.root_span_id:
-                root_seen = True
-                if span.parent_id is not None:
-                    raise ValueError("canonical trace root cannot have a local parent")
-            elif span.parent_id is None or span.parent_id not in seen:
-                raise ValueError(f"canonical span parent must appear first: {span.id}")
-            seen.add(span.id)
-
-        if not root_seen:
-            raise ValueError("canonical trace root is missing")
-        return self

api/core/ops/unified_trace/parent_context.py

@@ -1,249 +0,0 @@
-"""Coordinate nested-workflow provider parent contexts through Redis.
-
-The coordinator owns storage, compatibility decisions, validation, and retry
-signals. Provider adapters only create and consume their opaque context fields.
-"""
-
-import hashlib
-import json
-from collections.abc import Callable, Mapping
-from dataclasses import dataclass
-from enum import StrEnum
-from typing import Literal, Protocol
-
-from pydantic import BaseModel, ConfigDict, ValidationError
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from configs import dify_config
-from core.helper.trace_id_helper import ParentTraceContext
-from core.ops.exceptions import (
-    InvalidTraceParentContextError,
-    PendingTraceParentContextError,
-    TraceParentContextAccessError,
-)
-from extensions.ext_database import db
-from models.model import App, TraceAppConfig
-from models.workflow import WorkflowRun
-
-_PARENT_CONTEXT_KEY_PREFIX = "trace:unified:parent:"
-
-
-class RedisParentContextStore(Protocol):
-    def setex(self, name: str, time: int, value: str) -> object:
-        raise NotImplementedError
-
-    def get(self, name: str) -> bytes | str | None:
-        raise NotImplementedError
-
-
-class ProviderParentContext(BaseModel):
-    """Versioned envelope containing the minimum provider restoration state."""
-
-    version: Literal[1] = 1
-    provider: str
-    scope: str
-    trace_id: str
-    parent_id: str
-    provider_context: dict[str, str]
-
-    model_config = ConfigDict(extra="forbid", frozen=True)
-
-
-@dataclass(frozen=True)
-class ParentDestination:
-    provider: str
-    scope: str
-    unified: bool
-
-
-class ParentResolutionKind(StrEnum):
-    RESTORED = "restored"
-    LINKED_ROOT = "linked_root"
-
-
-@dataclass(frozen=True)
-class ParentResolution:
-    kind: ParentResolutionKind
-    context: ProviderParentContext | None = None
-    linked_parent: ParentTraceContext | None = None
-
-    @classmethod
-    def restored(cls, context: ProviderParentContext) -> "ParentResolution":
-        return cls(kind=ParentResolutionKind.RESTORED, context=context)
-
-    @classmethod
-    def linked_root(cls, parent: ParentTraceContext) -> "ParentResolution":
-        return cls(kind=ParentResolutionKind.LINKED_ROOT, linked_parent=parent)
-
-
-ParentDestinationResolver = Callable[[str], ParentDestination | None]
-
-
-def destination_scope(provider: str, endpoint: str, project: str) -> str:
-    """Return a stable non-secret fingerprint for a provider destination."""
-    value = f"{provider}\0{endpoint.rstrip('/')}\0{project}"
-    return hashlib.sha256(value.encode()).hexdigest()
-
-
-def parent_destination_from_config(
-    provider: str,
-    tracing_config: Mapping[str, object],
-    *,
-    unified: bool,
-) -> ParentDestination:
-    """Build destination compatibility from non-secret persisted fields."""
-    endpoint = tracing_config.get("endpoint")
-    project = tracing_config.get("project")
-    return ParentDestination(
-        provider=provider,
-        scope=destination_scope(
-            provider,
-            endpoint if isinstance(endpoint, str) else "",
-            project if isinstance(project, str) else "",
-        ),
-        unified=unified,
-    )
-
-
-def resolve_parent_destination(parent_workflow_run_id: str) -> ParentDestination | None:
-    """Resolve whether a parent workflow can publish compatible unified context."""
-    with Session(db.engine) as session:
-        workflow_run = session.get(WorkflowRun, parent_workflow_run_id)
-        if workflow_run is None:
-            return None
-        app = session.get(App, workflow_run.app_id)
-        if app is None or not app.tracing:
-            return None
-        try:
-            app_tracing = json.loads(app.tracing)
-        except (TypeError, json.JSONDecodeError):
-            return None
-        if not isinstance(app_tracing, Mapping) or not app_tracing.get("enabled"):
-            return None
-        provider = app_tracing.get("tracing_provider")
-        if not isinstance(provider, str):
-            return None
-        trace_config = session.scalar(
-            select(TraceAppConfig)
-            .where(TraceAppConfig.app_id == app.id, TraceAppConfig.tracing_provider == provider)
-            .limit(1)
-        )
-        if trace_config is None or not isinstance(trace_config.tracing_config, Mapping):
-            return None
-
-        unified = False
-        if dify_config.OPS_TRACE_UNIFIED_ENABLED:
-            from core.ops.unified_trace.registry import unified_provider_config_map
-
-            try:
-                unified_provider_config_map[provider]
-                unified = True
-            except KeyError:
-                pass
-        return parent_destination_from_config(provider, trace_config.tracing_config, unified=unified)
-
-
-class ParentContextCoordinator:
-    """Publish and resolve cross-task parent contexts for unified providers."""
-
-    def __init__(
-        self,
-        store: RedisParentContextStore,
-        resolve_parent_destination: ParentDestinationResolver,
-    ) -> None:
-        self._store = store
-        self._resolve_parent_destination = resolve_parent_destination
-
-    @staticmethod
-    def _key(parent_node_execution_id: str) -> str:
-        return f"{_PARENT_CONTEXT_KEY_PREFIX}{parent_node_execution_id}"
-
-    def publish(self, parent_node_execution_id: str, context: ProviderParentContext) -> None:
-        """Persist an accepted provider parent so a nested task can restore it."""
-        try:
-            self._store.setex(
-                self._key(parent_node_execution_id),
-                dify_config.OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS,
-                context.model_dump_json(),
-            )
-        except Exception as error:
-            raise TraceParentContextAccessError(
-                f"Could not publish unified parent context for node_execution_id={parent_node_execution_id}"
-            ) from error
-
-    def resolve(
-        self,
-        parent: ParentTraceContext,
-        *,
-        expected_provider: str,
-        expected_scope: str,
-    ) -> ParentResolution:
-        """Restore compatible context or explicitly select a linked new root."""
-        destination = self._resolve_parent_destination(parent.parent_workflow_run_id)
-        if (
-            destination is None
-            or not destination.unified
-            or destination.provider != expected_provider
-            or destination.scope != expected_scope
-        ):
-            return ParentResolution.linked_root(parent)
-
-        parent_node_execution_id = parent.parent_node_execution_id
-        if not parent_node_execution_id:
-            raise InvalidTraceParentContextError("Nested workflow parent context has no node execution ID")
-
-        return ParentResolution.restored(
-            self._restore(
-                parent_node_execution_id,
-                expected_provider=expected_provider,
-                expected_scope=expected_scope,
-            )
-        )
-
-    def resolve_required(
-        self,
-        parent_context_id: str,
-        *,
-        expected_provider: str,
-        expected_scope: str,
-    ) -> ParentResolution:
-        """Restore a parent context that must exist for an asynchronous child."""
-        return ParentResolution.restored(
-            self._restore(
-                parent_context_id,
-                expected_provider=expected_provider,
-                expected_scope=expected_scope,
-            )
-        )
-
-    def _restore(
-        self,
-        parent_context_id: str,
-        *,
-        expected_provider: str,
-        expected_scope: str,
-    ) -> ProviderParentContext:
-        try:
-            raw_context = self._store.get(self._key(parent_context_id))
-        except Exception as error:
-            raise TraceParentContextAccessError(
-                f"Could not read unified parent context for parent_context_id={parent_context_id}"
-            ) from error
-
-        if raw_context is None:
-            raise PendingTraceParentContextError(parent_context_id)
-
-        try:
-            context = ProviderParentContext.model_validate_json(raw_context)
-        except (ValidationError, ValueError, TypeError) as error:
-            raise InvalidTraceParentContextError(
-                f"Invalid unified parent context for parent_context_id={parent_context_id}"
-            ) from error
-
-        if context.provider != expected_provider or context.scope != expected_scope:
-            raise InvalidTraceParentContextError(
-                "Stored unified parent context does not match the expected provider destination: "
-                f"parent_context_id={parent_context_id}"
-            )
-        return context

api/core/ops/unified_trace/provider.py

@@ -1,82 +0,0 @@
-"""Unified trace runtime and provider adapter contract."""
-
-from collections.abc import Callable
-from typing import Protocol, override
-
-from core.ops.base_trace_instance import BaseTraceInstance
-from core.ops.entities.config_entity import BaseTracingConfig
-from core.ops.entities.trace_entity import BaseTraceInfo
-from core.ops.unified_trace.entities import CanonicalTrace
-from core.ops.unified_trace.parent_context import (
-    ParentContextCoordinator,
-    ParentResolution,
-    ProviderParentContext,
-)
-from core.ops.unified_trace.trace_builder import CanonicalTraceBuilder
-
-ParentContextPublisher = Callable[[str, ProviderParentContext], None]
-
-
-class UnifiedTraceAdapter(Protocol):
-    @property
-    def provider_name(self) -> str:
-        raise NotImplementedError
-
-    @property
-    def scope(self) -> str:
-        raise NotImplementedError
-
-    def emit(
-        self,
-        trace: CanonicalTrace,
-        parent: ParentResolution | None,
-        publish_parent_context: ParentContextPublisher,
-    ) -> None:
-        """Emit one Core-resolved, parent-first canonical fragment.
-
-        Return only after the provider-specific synchronous acceptance step succeeds.
-        Raise RetryableTraceDispatchError when acceptance is unconfirmed because of a
-        recoverable provider or transport failure. Raise another exception for a
-        terminal failure. Publish parent context only after the corresponding provider
-        parent span has been accepted.
-        """
-        raise NotImplementedError
-
-
-class UnifiedTraceInstance(BaseTraceInstance):
-    """Build, coordinate, and emit a trace without a legacy fallback path."""
-
-    def __init__(
-        self,
-        trace_config: BaseTracingConfig,
-        *,
-        builder: CanonicalTraceBuilder,
-        adapter: UnifiedTraceAdapter,
-        coordinator: ParentContextCoordinator,
-    ) -> None:
-        super().__init__(trace_config)
-        self._builder = builder
-        self._adapter = adapter
-        self._coordinator = coordinator
-
-    @override
-    def trace(self, trace_info: BaseTraceInfo) -> None:
-        canonical_trace = self._builder.build(trace_info)
-        if canonical_trace is None:
-            return
-
-        parent_resolution = None
-        if canonical_trace.external_parent is not None:
-            parent_resolution = self._coordinator.resolve(
-                canonical_trace.external_parent,
-                expected_provider=self._adapter.provider_name,
-                expected_scope=self._adapter.scope,
-            )
-        elif canonical_trace.required_parent_context_id is not None:
-            parent_resolution = self._coordinator.resolve_required(
-                canonical_trace.required_parent_context_id,
-                expected_provider=self._adapter.provider_name,
-                expected_scope=self._adapter.scope,
-            )
-
-        self._adapter.emit(canonical_trace, parent_resolution, self._coordinator.publish)

api/core/ops/unified_trace/registry.py

@@ -1,35 +0,0 @@
-"""Lazy registry for providers implemented by the unified tracing path."""
-
-import collections
-from typing import TypedDict, override
-
-from core.ops.base_trace_instance import BaseTraceInstance
-from core.ops.entities.config_entity import BaseTracingConfig, TracingProviderEnum
-
-
-class UnifiedProviderConfigEntry(TypedDict):
-    config_class: type[BaseTracingConfig]
-    trace_instance: type[BaseTraceInstance]
-
-
-class UnifiedTraceProviderConfigMap(collections.UserDict[str, UnifiedProviderConfigEntry]):
-    """Resolve unified providers without importing their SDKs until selected."""
-
-    @override
-    def __getitem__(self, key: str) -> UnifiedProviderConfigEntry:
-        match key:
-            case TracingProviderEnum.PHOENIX:
-                from dify_trace_arize_phoenix.config import PhoenixConfig
-                from dify_trace_arize_phoenix.unified_trace import UnifiedPhoenixTrace
-
-                return {"config_class": PhoenixConfig, "trace_instance": UnifiedPhoenixTrace}
-            case TracingProviderEnum.LANGSMITH:
-                from dify_trace_langsmith.config import LangSmithConfig
-                from dify_trace_langsmith.unified_trace import UnifiedLangSmithTrace
-
-                return {"config_class": LangSmithConfig, "trace_instance": UnifiedLangSmithTrace}
-            case _:
-                raise KeyError(f"Unified tracing provider is not registered: {key}")
-
-
-unified_provider_config_map = UnifiedTraceProviderConfigMap()

api/core/ops/unified_trace/trace_builder.py

@@ -1,452 +0,0 @@
-"""Build provider-neutral traces from core ops trace entities.
-
-Workflow persistence is accessed through an injected loader. Provider adapters
-therefore receive a complete parent-first tree and never query Dify models.
-"""
-
-from collections.abc import Callable, Mapping, Sequence
-from datetime import datetime, timedelta
-from typing import Any
-from uuid import uuid4
-
-from pydantic import ValidationError
-from sqlalchemy.orm import sessionmaker
-
-from core.app.workflow.retry_history import RETRY_HISTORY_PROCESS_DATA_KEY
-from core.helper.trace_id_helper import ParentTraceContext
-from core.ops.entities.trace_entity import (
-    BaseTraceInfo,
-    DatasetRetrievalTraceInfo,
-    GenerateNameTraceInfo,
-    MessageTraceInfo,
-    ModerationTraceInfo,
-    SuggestedQuestionTraceInfo,
-    ToolTraceInfo,
-    WorkflowTraceInfo,
-)
-from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
-from core.ops.unified_trace.hierarchy import (
-    WorkflowExecutionLike,
-    build_workflow_hierarchy,
-    execution_id,
-    execution_metadata,
-)
-from core.repositories import DifyCoreRepositoryFactory
-from extensions.ext_database import db
-from models import Account
-from models.workflow import WorkflowNodeExecutionTriggeredFrom
-
-WorkflowExecutionLoader = Callable[[WorkflowTraceInfo], Sequence[WorkflowExecutionLike]]
-ServiceAccountResolver = Callable[[str], Account]
-
-
-class RepositoryWorkflowExecutionLoader:
-    """Export caller executions; Workflow Tool source internals stay in authorized console traces."""
-
-    def __init__(self, get_service_account: ServiceAccountResolver) -> None:
-        self._get_service_account = get_service_account
-
-    def __call__(self, trace_info: WorkflowTraceInfo) -> Sequence[WorkflowExecutionLike]:
-        app_id = trace_info.metadata.get("app_id")
-        if not isinstance(app_id, str) or not app_id:
-            raise ValueError("No app_id found in workflow trace metadata")
-        repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
-            session_factory=sessionmaker(bind=db.engine),
-            tenant_id=trace_info.tenant_id,
-            user=self._get_service_account(app_id),
-            app_id=app_id,
-            triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
-        )
-        return repository.get_by_workflow_execution(workflow_execution_id=trace_info.workflow_run_id)
-
-
-_NODE_KIND: dict[str, CanonicalSpanKind] = {
-    "llm": CanonicalSpanKind.LLM,
-    "knowledge-retrieval": CanonicalSpanKind.RETRIEVER,
-    "tool": CanonicalSpanKind.TOOL,
-    "agent": CanonicalSpanKind.AGENT,
-}
-_RETRY_SUMMARY_FIELDS = ("retry_index", "error", "elapsed_time", "created_at", "finished_at")
-
-
-def _read_attribute(value: object, name: str, default: Any = None) -> Any:
-    """Read fields shared by persisted trace models and legacy trace objects."""
-    return getattr(value, name, default)  # guard-ignore: no-new-getattr -- supports legacy trace objects
-
-
-def _retry_metadata(process_data: Mapping[str, Any]) -> dict[str, Any]:
-    raw_history = process_data.get(RETRY_HISTORY_PROCESS_DATA_KEY)
-    if not isinstance(raw_history, list):
-        return {}
-
-    attempts: list[dict[str, Any]] = []
-    for raw_attempt in raw_history:
-        if not isinstance(raw_attempt, Mapping):
-            continue
-        retry_index = raw_attempt.get("retry_index")
-        if isinstance(retry_index, bool) or not isinstance(retry_index, int) or retry_index <= 0:
-            continue
-        attempts.append({field: raw_attempt.get(field) for field in _RETRY_SUMMARY_FIELDS})
-
-    return {"retry_count": len(attempts), "retry_attempts": attempts} if attempts else {}
-
-
-def resolve_session_id(trace_info: WorkflowTraceInfo | MessageTraceInfo) -> str:
-    """Resolve an explicit trace session before stable Dify fallbacks."""
-    custom_session_id = trace_info.metadata.get("trace_session_id")
-    if isinstance(custom_session_id, str) and custom_session_id:
-        return custom_session_id
-
-    if isinstance(trace_info, WorkflowTraceInfo):
-        if trace_info.conversation_id:
-            return trace_info.conversation_id
-        parent_workflow_run_id, _ = trace_info.resolved_parent_context
-        return parent_workflow_run_id or trace_info.workflow_run_id
-
-    if trace_info.message_data is None:
-        return ""
-    conversation_id = _read_attribute(trace_info.message_data, "conversation_id")
-    return conversation_id if isinstance(conversation_id, str) else ""
-
-
-def _status(error: str | None, status: Any = None) -> CanonicalSpanStatus:
-    status_value = _read_attribute(status, "value", status)
-    return CanonicalSpanStatus.ERROR if error or status_value in {"failed", "exception"} else CanonicalSpanStatus.OK
-
-
-def _external_parent(trace_info: WorkflowTraceInfo) -> ParentTraceContext | None:
-    value = trace_info.metadata.get("parent_trace_context")
-    if isinstance(value, ParentTraceContext):
-        return value
-    if isinstance(value, Mapping):
-        try:
-            return ParentTraceContext.model_validate(value)
-        except ValidationError:
-            return None
-    return None
-
-
-def _started_at(value: datetime | None) -> datetime:
-    return value or datetime.now()
-
-
-def _single_session_id(trace_info: BaseTraceInfo) -> str:
-    value = trace_info.metadata.get("trace_session_id")
-    return value if isinstance(value, str) else ""
-
-
-class CanonicalTraceBuilder:
-    """Convert supported ops trace entities into canonical parent-first trees."""
-
-    def __init__(self, load_workflow_executions: WorkflowExecutionLoader) -> None:
-        self._load_workflow_executions = load_workflow_executions
-
-    def build(self, trace_info: BaseTraceInfo) -> CanonicalTrace | None:
-        match trace_info:
-            case WorkflowTraceInfo():
-                return self._build_workflow(trace_info)
-            case MessageTraceInfo():
-                return self._build_message(trace_info)
-            case ModerationTraceInfo():
-                return self._build_moderation(trace_info)
-            case SuggestedQuestionTraceInfo():
-                return self._build_suggested_question(trace_info)
-            case DatasetRetrievalTraceInfo():
-                return self._build_dataset_retrieval(trace_info)
-            case ToolTraceInfo():
-                return self._build_tool(trace_info)
-            case GenerateNameTraceInfo():
-                return self._build_generate_name(trace_info)
-            case _:
-                return None
-
-    def _build_workflow(self, trace_info: WorkflowTraceInfo) -> CanonicalTrace:
-        executions = self._load_workflow_executions(trace_info)
-        hierarchy = build_workflow_hierarchy(executions)
-        workflow_data = trace_info.workflow_data
-        workflow_start = _started_at(_read_attribute(workflow_data, "created_at") or trace_info.start_time)
-        workflow_end = _read_attribute(workflow_data, "finished_at") or trace_info.end_time
-        message_span_id = trace_info.message_id
-        workflow_span_id = trace_info.workflow_run_id
-        spans: dict[str, CanonicalSpan] = {}
-
-        if message_span_id:
-            spans[message_span_id] = CanonicalSpan(
-                id=message_span_id,
-                parent_id=None,
-                name=f"chatflow_{trace_info.workflow_run_id}",
-                kind=CanonicalSpanKind.CHAIN,
-                start_time=_started_at(trace_info.start_time or workflow_start),
-                end_time=trace_info.end_time or workflow_end,
-                inputs=trace_info.query or dict(trace_info.workflow_run_inputs),
-                outputs=dict(trace_info.workflow_run_outputs),
-                status=_status(trace_info.error),
-                error=trace_info.error or None,
-                metadata={**trace_info.metadata, "trace_entity_type": "message"},
-                publishes_parent_context=True,
-            )
-            root_span_id = message_span_id
-            workflow_parent_id: str | None = message_span_id
-        else:
-            root_span_id = workflow_span_id
-            workflow_parent_id = None
-
-        spans[workflow_span_id] = CanonicalSpan(
-            id=workflow_span_id,
-            parent_id=workflow_parent_id,
-            name=f"workflow_{workflow_span_id}",
-            kind=CanonicalSpanKind.CHAIN,
-            start_time=workflow_start,
-            end_time=workflow_end,
-            inputs=dict(trace_info.workflow_run_inputs),
-            outputs=dict(trace_info.workflow_run_outputs),
-            status=_status(trace_info.error, trace_info.workflow_run_status),
-            error=trace_info.error or None,
-            metadata={
-                **trace_info.metadata,
-                "workflow_id": trace_info.workflow_id,
-                "workflow_run_id": workflow_span_id,
-                "workflow_app_log_id": trace_info.workflow_app_log_id,
-                "total_tokens": trace_info.total_tokens,
-            },
-        )
-
-        execution_by_id = {execution_id(item): item for item in executions}
-        for wrapper in hierarchy.wrappers:
-            spans[wrapper.id] = CanonicalSpan(
-                id=wrapper.id,
-                parent_id=wrapper.parent_execution_id,
-                name=f"{wrapper.key.kind}[{wrapper.key.index}]",
-                kind=CanonicalSpanKind.CHAIN,
-                start_time=wrapper.start_time,
-                end_time=wrapper.end_time,
-                status=CanonicalSpanStatus.ERROR if wrapper.has_error else CanonicalSpanStatus.OK,
-                error="wrapper child failed" if wrapper.has_error else None,
-                metadata={
-                    "wrapper_type": wrapper.key.kind,
-                    "wrapper_index": wrapper.key.index,
-                    "container_execution_id": wrapper.parent_execution_id,
-                },
-                synthetic=True,
-            )
-
-        for item_execution_id, item in execution_by_id.items():
-            process_data = _read_attribute(item, "process_data") or {}
-            outputs = _read_attribute(item, "outputs") or {}
-            node_type = str(_read_attribute(item, "node_type", ""))
-            error = _read_attribute(item, "error")
-            started_at = _started_at(_read_attribute(item, "created_at"))
-            elapsed_time = _read_attribute(item, "elapsed_time") or 0
-            metadata = dict(execution_metadata(item))
-            metadata.update(
-                {
-                    "node_id": _read_attribute(item, "node_id", ""),
-                    "node_execution_id": item_execution_id,
-                    "node_type": node_type,
-                    "status": _read_attribute(item, "status", ""),
-                    "model_provider": process_data.get("model_provider"),
-                    "model_name": process_data.get("model_name"),
-                }
-            )
-            usage = process_data.get("usage") or (outputs.get("usage") if isinstance(outputs, Mapping) else None) or {}
-            if isinstance(usage, Mapping):
-                metadata.update(
-                    {
-                        "prompt_tokens": usage.get("prompt_tokens", 0),
-                        "completion_tokens": usage.get("completion_tokens", 0),
-                        "total_tokens": usage.get("total_tokens", 0),
-                    }
-                )
-            metadata.update(_retry_metadata(process_data))
-            title = _read_attribute(item, "title")
-            name = f"{node_type}_{title}" if isinstance(title, str) and title else node_type
-            spans[item_execution_id] = CanonicalSpan(
-                id=item_execution_id,
-                parent_id=hierarchy.parent_by_execution_id.get(item_execution_id, workflow_span_id),
-                name=name,
-                kind=_NODE_KIND.get(node_type, CanonicalSpanKind.CHAIN),
-                start_time=started_at,
-                end_time=started_at + timedelta(seconds=elapsed_time),
-                inputs=process_data.get("prompts", []) if node_type == "llm" else _read_attribute(item, "inputs") or {},
-                outputs=outputs,
-                status=_status(error, _read_attribute(item, "status")),
-                error=error,
-                metadata=metadata,
-                can_parent_workflow=node_type == "tool",
-            )
-
-        ordered: list[CanonicalSpan] = []
-        emitted: set[str] = set()
-
-        def emit(span_id: str) -> None:
-            if span_id in emitted:
-                return
-            span = spans[span_id]
-            if span.parent_id in spans:
-                emit(span.parent_id)
-            emitted.add(span_id)
-            ordered.append(span)
-
-        for span_id in sorted(spans):
-            emit(span_id)
-
-        return CanonicalTrace(
-            trace_id=trace_info.resolved_trace_id or root_span_id,
-            session_id=resolve_session_id(trace_info),
-            root_span_id=root_span_id,
-            spans=tuple(ordered),
-            external_parent=_external_parent(trace_info),
-        )
-
-    def _single_trace(
-        self,
-        trace_info: BaseTraceInfo,
-        *,
-        name: str,
-        kind: CanonicalSpanKind,
-        inputs: Any,
-        outputs: Any,
-        error: str | None = None,
-        start_time: datetime | None = None,
-        end_time: datetime | None = None,
-        parent_context_id: str | None = None,
-        span_id: str | None = None,
-        session_id: str | None = None,
-    ) -> CanonicalTrace:
-        operation_id = span_id or trace_info.operation_id or str(uuid4())
-        trace_id = trace_info.resolved_trace_id or parent_context_id or operation_id
-        span = CanonicalSpan(
-            id=operation_id,
-            parent_id=None,
-            name=name,
-            kind=kind,
-            start_time=_started_at(start_time or trace_info.start_time),
-            end_time=end_time or trace_info.end_time,
-            inputs=inputs,
-            outputs=outputs,
-            status=_status(error),
-            error=error,
-            metadata=dict(trace_info.metadata),
-        )
-        return CanonicalTrace(
-            trace_id=trace_id,
-            session_id=session_id if session_id is not None else _single_session_id(trace_info),
-            root_span_id=operation_id,
-            spans=(span,),
-            required_parent_context_id=parent_context_id,
-        )
-
-    def _build_message(self, trace_info: MessageTraceInfo) -> CanonicalTrace | None:
-        message = trace_info.message_data
-        if message is None:
-            return None
-        message_id = trace_info.message_id or str(_read_attribute(message, "id", "")) or str(uuid4())
-        started_at = _started_at(trace_info.start_time or _read_attribute(message, "created_at"))
-        ended_at = trace_info.end_time or _read_attribute(message, "updated_at")
-        answer = _read_attribute(message, "answer", trace_info.outputs)
-        message_error = trace_info.error or _read_attribute(message, "error")
-        metadata = {
-            **trace_info.metadata,
-            "trace_entity_type": "message",
-            "model_provider": _read_attribute(message, "model_provider"),
-            "model_name": _read_attribute(message, "model_id"),
-            "prompt_tokens": trace_info.message_tokens,
-            "completion_tokens": trace_info.answer_tokens,
-            "total_tokens": trace_info.total_tokens,
-        }
-        spans: list[CanonicalSpan] = [
-            CanonicalSpan(
-                id=message_id,
-                parent_id=None,
-                name="message",
-                kind=CanonicalSpanKind.CHAIN,
-                start_time=started_at,
-                end_time=ended_at,
-                inputs=trace_info.inputs,
-                outputs=answer,
-                status=_status(message_error),
-                error=message_error,
-                metadata=metadata,
-                publishes_parent_context=True,
-            ),
-            CanonicalSpan(
-                id=f"{message_id}:llm",
-                parent_id=message_id,
-                name="llm",
-                kind=CanonicalSpanKind.LLM,
-                start_time=started_at,
-                end_time=ended_at,
-                inputs=trace_info.inputs,
-                outputs=trace_info.outputs if trace_info.outputs is not None else answer,
-                status=_status(message_error),
-                error=message_error,
-                metadata=metadata,
-                synthetic=True,
-            ),
-        ]
-        return CanonicalTrace(
-            trace_id=trace_info.resolved_trace_id or message_id,
-            session_id=resolve_session_id(trace_info),
-            root_span_id=message_id,
-            spans=tuple(spans),
-        )
-
-    def _build_moderation(self, trace_info: ModerationTraceInfo) -> CanonicalTrace | None:
-        if trace_info.message_data is None:
-            return None
-        return self._single_trace(
-            trace_info,
-            name="moderation",
-            kind=CanonicalSpanKind.TOOL,
-            inputs=trace_info.inputs,
-            outputs={"action": trace_info.action, "flagged": trace_info.flagged},
-            parent_context_id=trace_info.message_id,
-        )
-
-    def _build_suggested_question(self, trace_info: SuggestedQuestionTraceInfo) -> CanonicalTrace | None:
-        if trace_info.message_data is None:
-            return None
-        return self._single_trace(
-            trace_info,
-            name="suggested_question",
-            kind=CanonicalSpanKind.TOOL,
-            inputs=trace_info.inputs,
-            outputs=trace_info.suggested_question,
-            error=trace_info.error,
-            parent_context_id=trace_info.message_id,
-        )
-
-    def _build_dataset_retrieval(self, trace_info: DatasetRetrievalTraceInfo) -> CanonicalTrace | None:
-        if trace_info.message_data is None:
-            return None
-        return self._single_trace(
-            trace_info,
-            name="dataset_retrieval",
-            kind=CanonicalSpanKind.RETRIEVER,
-            inputs=trace_info.inputs,
-            outputs={"documents": trace_info.documents},
-            error=trace_info.error,
-            parent_context_id=trace_info.message_id,
-        )
-
-    def _build_tool(self, trace_info: ToolTraceInfo) -> CanonicalTrace:
-        return self._single_trace(
-            trace_info,
-            name=trace_info.tool_name,
-            kind=CanonicalSpanKind.TOOL,
-            inputs=trace_info.tool_inputs,
-            outputs=trace_info.tool_outputs,
-            error=trace_info.error,
-            parent_context_id=trace_info.message_id,
-        )
-
-    def _build_generate_name(self, trace_info: GenerateNameTraceInfo) -> CanonicalTrace:
-        return self._single_trace(
-            trace_info,
-            name="generate_name",
-            kind=CanonicalSpanKind.TOOL,
-            inputs=trace_info.inputs,
-            outputs=trace_info.outputs,
-            parent_context_id=trace_info.message_id,
-            session_id=_single_session_id(trace_info) or trace_info.conversation_id or "",
-        )

api/core/ops/utils.py

@@ -1,175 +0,0 @@
-from contextlib import contextmanager
-from datetime import datetime
-from typing import Any
-from urllib.parse import urlparse
-
-from pydantic import TypeAdapter
-from sqlalchemy import select
-from sqlalchemy.orm import Session, scoped_session
-
-from models.model import Message
-
-JSON_DICT_ADAPTER: TypeAdapter[dict[str, Any]] = TypeAdapter(dict[str, Any])
-
-
-def filter_none_values(data: dict[str, Any]) -> dict[str, Any]:
-    new_data = {}
-    for key, value in data.items():
-        if value is None:
-            continue
-        if isinstance(value, datetime):
-            new_data[key] = value.isoformat()
-        else:
-            new_data[key] = value
-    return new_data
-
-
-def get_message_data(message_id: str, session: Session | scoped_session):
-    return session.scalar(select(Message).where(Message.id == message_id))
-
-
-@contextmanager
-def measure_time():
-    timing_info = {"start": datetime.now(), "end": None}
-    try:
-        yield timing_info
-    finally:
-        # pyrefly: ignore [bad-assignment]
-        timing_info["end"] = datetime.now()
-
-
-def replace_text_with_content(data):
-    match data:
-        case dict():
-            new_data = {}
-            for key, value in data.items():
-                if key == "text":
-                    new_data["content"] = value
-                else:
-                    new_data[key] = replace_text_with_content(value)
-            return new_data
-        case list():
-            return [replace_text_with_content(item) for item in data]
-        case _:
-            return data
-
-
-def generate_dotted_order(run_id: str, start_time: str | datetime, parent_dotted_order: str | None = None) -> str:
-    """
-    generate dotted_order for langsmith
-    """
-    start_time = datetime.fromisoformat(start_time) if isinstance(start_time, str) else start_time
-    timestamp = start_time.strftime("%Y%m%dT%H%M%S%f") + "Z"
-    current_segment = f"{timestamp}{run_id}"
-
-    if parent_dotted_order is None:
-        return current_segment
-
-    return f"{parent_dotted_order}.{current_segment}"
-
-
-def validate_url(url: str, default_url: str, allowed_schemes: tuple = ("https", "http")) -> str:
-    """
-    Validate and normalize URL with proper error handling.
-
-    NOTE: This function does not retain the `path` component of the provided URL.
-    In most cases, it is recommended to use `validate_url_with_path` instead.
-
-    This function is deprecated and retained only for compatibility purposes.
-    New implementations should use `validate_url_with_path`.
-
-    Args:
-        url: The URL to validate
-        default_url: Default URL to use if input is None or empty
-        allowed_schemes: Tuple of allowed URL schemes (default: https, http)
-
-    Returns:
-        Normalized URL string
-
-    Raises:
-        ValueError: If URL format is invalid or scheme not allowed
-    """
-    if not url or url.strip() == "":
-        return default_url
-
-    # Parse URL to validate format
-    parsed = urlparse(url)
-
-    # Check if scheme is allowed
-    if parsed.scheme not in allowed_schemes:
-        raise ValueError(f"URL scheme must be one of: {', '.join(allowed_schemes)}")
-
-    # Reconstruct URL with only scheme, netloc (removing path, query, fragment)
-    normalized_url = f"{parsed.scheme}://{parsed.netloc}"
-
-    return normalized_url
-
-
-def validate_url_with_path(
-    url: str,
-    default_url: str,
-    required_suffix: str | None = None,
-    *,
-    allowed_schemes: tuple[str, ...] = ("https", "http"),
-) -> str:
-    """
-    Validate URL that may include path components
-
-    Args:
-        url: The URL to validate
-        default_url: Default URL to use if input is None or empty
-        required_suffix: Optional suffix that URL must end with
-        allowed_schemes: Tuple of allowed URL schemes (default: https, http)
-
-    Returns:
-        Validated URL string, returned verbatim so path, query and trailing
-        separators survive — `required_suffix` consumers depend on that
-
-    Raises:
-        ValueError: If URL format is invalid or doesn't match required suffix
-    """
-    if not url or url.strip() == "":
-        return default_url
-    url = url.strip()
-
-    # Parse URL to validate format
-    parsed = urlparse(url)
-
-    # Check if scheme is allowed
-    if parsed.scheme not in allowed_schemes:
-        expected = " or ".join(f"{scheme}://" for scheme in allowed_schemes)
-        raise ValueError(f"URL must start with {expected}")
-
-    # Check required suffix if specified
-    if required_suffix and not url.endswith(required_suffix):
-        raise ValueError(f"URL should end with {required_suffix}")
-
-    return url
-
-
-def validate_project_name(project: str, default_name: str) -> str:
-    """
-    Validate and normalize project name
-
-    Args:
-        project: Project name to validate
-        default_name: Default name to use if input is None or empty
-
-    Returns:
-        Normalized project name
-    """
-    if not project or project.strip() == "":
-        return default_name
-
-    return project.strip()
-
-
-def validate_integer_id(id_str: str) -> str:
-    """
-    Validate and normalize integer ID
-    """
-    id_str = id_str.strip()
-    if not id_str.isdigit():
-        raise ValueError("ID must be a valid integer")
-
-    return id_str

api/core/rag/retrieval/dataset_retrieval.py

@@ -8,6 +8,7 @@ from collections import Counter, defaultdict
 from collections.abc import Generator, Mapping
 from typing import Any, cast
 
+from dify_trace_sdk.events import DatasetRetrieved
 from flask import Flask, current_app
 from opentelemetry.trace import get_current_span
 from sqlalchemy import and_, func, literal, or_, select, update
@@ -35,9 +36,6 @@ from core.entities.model_entities import ModelStatus
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_context import with_credit_usage_created_by, with_credit_usage_metadata
 from core.model_manager import ModelInstance, ModelManager
-from core.ops.entities.trace_entity import TraceTaskName
-from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
-from core.ops.utils import measure_time
 from core.prompt.advanced_prompt_transform import AdvancedPromptTransform
 from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate
 from core.prompt.simple_prompt_transform import ModelMode
@@ -80,6 +78,7 @@ from graphon.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMU
 from graphon.model_runtime.entities.message_entities import PromptMessage, PromptMessageRole, PromptMessageTool
 from graphon.model_runtime.entities.model_entities import ModelFeature, ModelType
 from graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModel
+from libs.datetime_utils import Timing, measure_time
 from libs.helper import parse_uuid_str_or_none
 from libs.json_in_md_parser import parse_and_check_json_markdown
 from models.dataset import (
@@ -949,7 +948,7 @@ class DatasetRetrieval:
         flask_app: Flask,
         documents: list[Document],
         message_id: str | None = None,
-        timer: dict[str, Any] | None = None,
+        timer: Timing | None = None,
     ):
         """Handle retrieval end."""
         with flask_app.app_context():
@@ -1062,17 +1061,29 @@ class DatasetRetrieval:
 
             self._send_trace_task(message_id, documents, timer)
 
-    def _send_trace_task(self, message_id: str | None, documents: list[Document], timer: dict[str, Any] | None):
-        """Send trace task if trace manager is available."""
-        trace_manager: TraceQueueManager | None = (
-            self.application_generate_entity.trace_manager if self.application_generate_entity else None
-        )
-        if trace_manager:
-            trace_manager.add_trace_task(
-                TraceTask(
-                    TraceTaskName.DATASET_RETRIEVAL_TRACE, message_id=message_id, documents=documents, timer=timer
-                )
+    def _send_trace_task(self, message_id: str | None, documents: list[Document], timer: Timing | None):
+        trace_scope = self.application_generate_entity.trace_scope if self.application_generate_entity else None
+        if trace_scope is None or timer is None:
+            return
+        trace_scope.emit(
+            DatasetRetrieved(
+                message_id=message_id,
+                # Embeddings only bloat traces.
+                documents=[
+                    document.model_dump(
+                        mode="json",
+                        exclude={
+                            "vector": True,
+                            "children": {"__all__": {"vector"}},
+                            "attachments": {"__all__": {"vector"}},
+                        },
+                    )
+                    for document in documents
+                ],
+                started_at=timer.started_at,
+                ended_at=timer.ended_at,
             )
+        )
 
     @staticmethod
     def _resolve_creator_user_role(user_from: str) -> CreatorUserRole | None:

api/core/telemetry/__init__.py

@@ -6,7 +6,6 @@ lives in ``core.telemetry.gateway`` which is shared by both CE and EE.
 
 from __future__ import annotations
 
-from core.ops.entities.trace_entity import TraceTaskName
 from core.telemetry.events import (
     AppCreatedEvent,
     AppDeletedEvent,
@@ -30,6 +29,5 @@ __all__ = [
     "PromptGenerationPayload",
     "TelemetryContext",
     "TelemetryEvent",
-    "TraceTaskName",
     "emit",
 ]

api/core/telemetry/events.py

@@ -2,9 +2,11 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Any, NotRequired, Protocol, TypedDict, runtime_checkable
 
-from core.ops.entities.trace_entity import TraceTaskName
+from dify_trace_sdk.events import DraftNodeExecuted, PromptGenerated, TracePayload, json_safe
+
 from enterprise.telemetry.contracts import SignalType, TelemetryCase
 
 # ---------------------------------------------------------------------------
@@ -70,10 +72,11 @@ class PromptGenerationPayload(TypedDict):
     completion_tokens: int
     total_tokens: int
     latency: float
-    total_price: NotRequired[float | None]
-    currency: NotRequired[str | None]
-    timer: NotRequired[dict]
-    error: NotRequired[str | None]
+    total_price: float | None
+    currency: str | None
+    error: str | None
+    started_at: datetime
+    ended_at: datetime
 
 
 class FeedbackCreatedPayload(TypedDict):
@@ -126,8 +129,13 @@ class TelemetryEvent(Protocol):
     def signal_type(self) -> SignalType: ...
     @property
     def ce_eligible(self) -> bool: ...
-    @property
-    def trace_task_name(self) -> TraceTaskName | None: ...
+
+
+@runtime_checkable
+class TraceSignalEvent(TelemetryEvent, Protocol):
+    """A ``SignalType.TRACE`` event, delivered through the trace pipeline."""
+
+    def trace_payload(self) -> TracePayload: ...
 
 
 # ---------------------------------------------------------------------------
@@ -142,7 +150,10 @@ class DraftNodeExecutionTraceEvent:
     case: TelemetryCase = TelemetryCase.DRAFT_NODE_EXECUTION
     signal_type: SignalType = SignalType.TRACE
     ce_eligible: bool = False
-    trace_task_name: TraceTaskName | None = TraceTaskName.DRAFT_NODE_EXECUTION_TRACE
+
+    def trace_payload(self) -> TracePayload:
+        node_execution = self.payload["node_execution_data"]
+        return DraftNodeExecuted(node_execution={key: json_safe(value) for key, value in node_execution.items()})
 
 
 @dataclass(frozen=True)
@@ -152,7 +163,25 @@ class PromptGenerationEvent:
     case: TelemetryCase = TelemetryCase.PROMPT_GENERATION
     signal_type: SignalType = SignalType.TRACE
     ce_eligible: bool = False
-    trace_task_name: TraceTaskName | None = TraceTaskName.PROMPT_GENERATION_TRACE
+
+    def trace_payload(self) -> TracePayload:
+        payload = self.payload
+        return PromptGenerated(
+            operation_type=payload["operation_type"],
+            instruction=payload["instruction"],
+            generated_output=payload["generated_output"],
+            model_provider=payload["model_provider"],
+            model_name=payload["model_name"],
+            prompt_tokens=payload["prompt_tokens"],
+            completion_tokens=payload["completion_tokens"],
+            total_tokens=payload["total_tokens"],
+            latency=payload["latency"],
+            total_price=payload["total_price"],
+            currency=payload["currency"],
+            error=payload["error"],
+            started_at=payload["started_at"],
+            ended_at=payload["ended_at"],
+        )
 
 
 @dataclass(frozen=True)
@@ -162,7 +191,6 @@ class AppCreatedEvent:
     case: TelemetryCase = TelemetryCase.APP_CREATED
     signal_type: SignalType = SignalType.METRIC_LOG
     ce_eligible: bool = False
-    trace_task_name: TraceTaskName | None = None
 
 
 @dataclass(frozen=True)
@@ -172,7 +200,6 @@ class AppUpdatedEvent:
     case: TelemetryCase = TelemetryCase.APP_UPDATED
     signal_type: SignalType = SignalType.METRIC_LOG
     ce_eligible: bool = False
-    trace_task_name: TraceTaskName | None = None
 
 
 @dataclass(frozen=True)
@@ -182,7 +209,6 @@ class AppDeletedEvent:
     case: TelemetryCase = TelemetryCase.APP_DELETED
     signal_type: SignalType = SignalType.METRIC_LOG
     ce_eligible: bool = False
-    trace_task_name: TraceTaskName | None = None
 
 
 @dataclass(frozen=True)
@@ -192,4 +218,3 @@ class FeedbackCreatedEvent:
     case: TelemetryCase = TelemetryCase.FEEDBACK_CREATED
     signal_type: SignalType = SignalType.METRIC_LOG
     ce_eligible: bool = False
-    trace_task_name: TraceTaskName | None = None

api/core/telemetry/gateway.py

@@ -1,10 +1,10 @@
 """Telemetry gateway — single routing layer for all editions.
 
-Dispatches ``TelemetryEvent`` instances to either the CE/EE trace
-pipeline (``TraceQueueManager``) or the enterprise-only metric/log
-Celery queue.  Each event class carries its own routing metadata
-(``signal_type``, ``ce_eligible``, ``trace_task_name``), so this
-module contains no per-case mapping tables.
+Dispatches ``TelemetryEvent`` instances to either the trace pipeline
+or the enterprise-only metric/log Celery queue.  Each event class
+carries its own routing metadata (``signal_type``, ``ce_eligible``, and
+``trace_payload`` for trace signals), so this module contains no
+per-case mapping tables.
 
 This module lives in ``core/`` so both CE and EE share one
 ``emit()`` entry point.  Enterprise-specific dispatch (Celery task,
@@ -17,14 +17,12 @@ from __future__ import annotations
 import json
 import logging
 import uuid
-from typing import TYPE_CHECKING, Any, cast
+from typing import Any, cast
 
-from core.telemetry.events import TelemetryEvent
+from core.telemetry.events import TelemetryEvent, TraceSignalEvent
 from enterprise.telemetry.contracts import SignalType
 from extensions.ext_storage import storage
-
-if TYPE_CHECKING:
-    from core.ops.ops_trace_manager import TraceQueueManager
+from extensions.tracing import create_trace_scope_for_app
 
 logger = logging.getLogger(__name__)
 
@@ -82,14 +80,14 @@ def _handle_payload_sizing(
 # ---------------------------------------------------------------------------
 
 
-def emit(event: TelemetryEvent, trace_manager: TraceQueueManager | None = None) -> None:
+def emit(event: TelemetryEvent) -> None:
     """Emit a telemetry event."""
     if not event.ce_eligible and not is_enterprise_telemetry_enabled():
         logger.debug("Dropping EE-only event: case=%s (EE disabled)", event.case)
         return
 
     if event.signal_type == SignalType.TRACE:
-        _emit_trace(event, trace_manager)
+        _emit_trace(event)
     else:
         _emit_metric_log(event)
 
@@ -99,21 +97,17 @@ def emit(event: TelemetryEvent, trace_manager: TraceQueueManager | None = None)
 # ---------------------------------------------------------------------------
 
 
-def _emit_trace(event: TelemetryEvent, trace_manager: TraceQueueManager | None) -> None:
-    from core.ops.ops_trace_manager import TraceQueueManager as LocalTraceQueueManager
-    from core.ops.ops_trace_manager import TraceTask
-
-    if event.trace_task_name is None:
-        logger.warning("No trace_task_name on event: case=%s", event.case)
+def _emit_trace(event: TelemetryEvent) -> None:
+    if not isinstance(event, TraceSignalEvent):
+        logger.warning("Trace signal without a trace payload: case=%s", event.case)
         return
-
     ctx = event.context
-    queue_manager: TraceQueueManager = trace_manager or LocalTraceQueueManager(
-        app_id=ctx.app_id,
-        user_id=ctx.user_id,
-    )
-    queue_manager.add_trace_task(TraceTask(event.trace_task_name, user_id=ctx.user_id, **event.payload))
-    logger.debug("Enqueued trace task: case=%s, app_id=%s", event.case, ctx.app_id)
+    if ctx.tenant_id is None:
+        logger.warning("Trace signal without a workspace: case=%s", event.case)
+        return
+    scope = create_trace_scope_for_app(tenant_id=ctx.tenant_id, app_id=ctx.app_id, user_id=ctx.user_id, extras={})
+    if scope is not None:
+        scope.emit(event.trace_payload())
 
 
 def _emit_metric_log(event: TelemetryEvent) -> None:

api/core/tools/tool_engine.py

@@ -7,13 +7,14 @@ from datetime import UTC, datetime
 from mimetypes import guess_type
 from typing import Any, cast
 
+from dify_trace_sdk.emitter import TraceScope
+from dify_trace_sdk.events import ToolInvoked, json_safe
 from sqlalchemy.orm import Session, sessionmaker
 from yarl import URL
 
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
 from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
-from core.ops.ops_trace_manager import TraceQueueManager
 from core.tools.__base.tool import Tool
 from core.tools.entities.tool_entities import (
     ToolInvokeMessage,
@@ -34,6 +35,7 @@ from core.tools.utils.message_transformer import ToolFileMessageTransformer, saf
 from core.tools.workflow_as_tool.tool import WorkflowTool
 from extensions.ext_database import db
 from graphon.file import FileTransferMethod, FileType
+from libs.datetime_utils import utc_now
 from models.enums import CreatorUserRole, MessageFileBelongsTo
 from models.model import Message, MessageFile
 
@@ -55,7 +57,7 @@ class ToolEngine:
         message: Message,
         invoke_from: InvokeFrom,
         agent_tool_callback: DifyAgentCallbackHandler,
-        trace_manager: TraceQueueManager | None = None,
+        trace_scope: TraceScope | None,
         conversation_id: str | None = None,
         app_id: str | None = None,
         message_id: str | None = None,
@@ -83,6 +85,7 @@ class ToolEngine:
             # hit the callback handler
             agent_tool_callback.on_tool_start(tool_name=tool.entity.identity.name, tool_inputs=tool_parameters)
 
+            started_at = utc_now()
             messages = ToolEngine._invoke(session, tool, tool_parameters, user_id, conversation_id, app_id, message_id)
             invocation_meta_dict: dict[str, ToolInvokeMeta] = {}
 
@@ -121,9 +124,21 @@ class ToolEngine:
                 tool_name=tool.entity.identity.name,
                 tool_inputs=tool_parameters,
                 tool_outputs=plain_text,
-                message_id=message.id,
-                trace_manager=trace_manager,
             )
+            if trace_scope is not None:
+                trace_scope.emit(
+                    ToolInvoked(
+                        message_id=message.id,
+                        tool_name=tool.entity.identity.name,
+                        tool_inputs={key: json_safe(value) for key, value in tool_parameters.items()},
+                        tool_outputs=plain_text,
+                        tool_config={key: json_safe(value) for key, value in (meta.tool_config or {}).items()},
+                        error=meta.error,
+                        time_cost=meta.time_cost,
+                        started_at=started_at,
+                        ended_at=utc_now(),
+                    )
+                )
 
             # transform tool invoke message to get LLM friendly message
             return plain_text, message_files, meta

api/core/workflow/node_factory.py

@@ -50,7 +50,6 @@ from core.workflow.nodes.agent.plugin_strategy_adapter import (
 from core.workflow.nodes.agent.runtime_support import AgentRuntimeSupport
 from core.workflow.nodes.agent_v2 import DifyAgentNode
 from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingResolver
-from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data
 from core.workflow.nodes.agent_v2.output_adapter import WorkflowAgentOutputAdapter
 from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeRequestBuilder
 from core.workflow.nodes.human_input.callback import DifyHITLCallback
@@ -59,6 +58,7 @@ from core.workflow.nodes.human_input_v2.node import HumanInputNode as HumanInput
 from core.workflow.system_variables import SystemVariableKey, get_system_text, system_variable_selector
 from core.workflow.template_rendering import CodeExecutorJinja2TemplateRenderer
 from core.workflow.workflow_tool_node import DifyWorkflowToolNode
+from domain.workflow.agent_node import is_dify_agent_node_data
 from extensions.application_services.data_sources import build_data_source_credentials
 from graphon.entities.base_node_data import BaseNodeData
 from graphon.entities.graph_config import NodeConfigDict, NodeConfigDictAdapter

api/core/workflow/nodes/agent_v2/runtime_request_builder.py

@@ -49,9 +49,9 @@ from clients.agent_backend import (
 from configs import dify_config
 from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
 from core.app.llm.model_access import resolve_model_context_window
-from core.ops.base_trace_instance import application_tracing_hooks
 from core.plugin.provider_identity import normalize_plugin_daemon_provider_identity
 from core.workflow.system_variables import SystemVariableKey, get_system_text, get_system_value
+from extensions.tracing import application_tracing_hooks
 from graphon.file import File, FileTransferMethod
 from graphon.variables.segments import Segment
 from graphon.variables.template_resolution import convert_template

api/core/workflow/nodes/agent_v2/validators.py

@@ -12,6 +12,7 @@ from core.agent.workflow_references import (
     workflow_previous_node_output_refs_from_selectors,
 )
 from core.workflow.graph_topology import WorkflowGraphTopology
+from domain.workflow.agent_node import is_dify_agent_node_data
 from graphon.enums import BuiltinNodeTypes, ErrorStrategy
 from models.agent import (
     Agent,
@@ -32,7 +33,6 @@ from models.model import UploadFile
 from models.workflow import Workflow
 from services.agent.knowledge_datasets import list_missing_tenant_knowledge_dataset_ids
 
-from .discriminator import is_dify_agent_node_data
 from .entities import DifyAgentNodeData
 
 

api/docker/entrypoint.sh

@@ -35,10 +35,10 @@ if [[ "${MODE}" == "worker" ]]; then
   if [[ -z "${CELERY_QUEUES}" ]]; then
     if [[ "${DEPLOYMENT_EDITION:-COMMUNITY}" == "CLOUD" ]]; then
       # Cloud edition: separate queues for dataset and trigger tasks
-      DEFAULT_QUEUES="api_token,dataset,dataset_summary,priority_dataset,priority_pipeline,pipeline,mail,human_input_delivery,human_input_contact_sync,ops_trace,app_deletion,app_rbac,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_publisher,trigger_refresh_executor,retention,workflow_based_app_execution,knowledge_fs_lifecycle"
+      DEFAULT_QUEUES="api_token,dataset,dataset_summary,priority_dataset,priority_pipeline,pipeline,mail,human_input_delivery,human_input_contact_sync,${OPS_TRACE_QUEUE:-ops_trace_v2},app_deletion,app_rbac,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_publisher,trigger_refresh_executor,retention,workflow_based_app_execution,knowledge_fs_lifecycle"
     else
       # Self-hosted editions: dataset, pipeline and workflow have separate queues
-      DEFAULT_QUEUES="api_token,dataset,dataset_summary,priority_dataset,priority_pipeline,pipeline,mail,human_input_delivery,human_input_contact_sync,ops_trace,app_deletion,app_rbac,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_publisher,trigger_refresh_executor,retention,workflow_based_app_execution,knowledge_fs_lifecycle"
+      DEFAULT_QUEUES="api_token,dataset,dataset_summary,priority_dataset,priority_pipeline,pipeline,mail,human_input_delivery,human_input_contact_sync,${OPS_TRACE_QUEUE:-ops_trace_v2},app_deletion,app_rbac,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_publisher,trigger_refresh_executor,retention,workflow_based_app_execution,knowledge_fs_lifecycle"
     fi
   else
     DEFAULT_QUEUES="${CELERY_QUEUES}"

api/domain/tracing/errors.py

@@ -0,0 +1,19 @@
+"""Business failures of trace runs; ``code`` names the reason clients and run records show."""
+
+
+class TraceRunError(Exception):
+    def __init__(self, code: str) -> None:
+        self.code = code
+        super().__init__(code)
+
+
+class TraceRunNotFoundError(TraceRunError):
+    """The run collected nothing, or a batch names an invocation the run does not have."""
+
+
+class TraceRunConflictError(TraceRunError):
+    """The operation conflicts with the run's current state."""
+
+
+class TraceRunLimitError(TraceRunError):
+    """A runtime batch exceeds the run's collection limits."""

api/domain/tracing/normalization.py

@@ -1,12 +1,12 @@
-"""Normalize Agent SDK observations without conflating business and provider IDs."""
+"""Normalize Agent runtime observations and outer traces for the Agent trace collector."""
 
-import hashlib
 import json
+import math
 from collections.abc import Mapping, Sequence
 from datetime import UTC, datetime
 from typing import Any
 
-from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from dify_trace_sdk.model import Cost, Span, SpanKind, SpanStatus, TokenUsage, Trace
 
 MAX_VALUE_CHARS = 16000
 _SECRET_KEYS = frozenset(
@@ -41,12 +41,6 @@ _CONTENT_ATTRIBUTES = frozenset(
 )
 
 
-def span_id(logical_id: str) -> str:
-    """Stable nonzero OTel span identity for Dify's persisted logical ID."""
-    value = hashlib.sha256(f"dify-application-span:{logical_id}".encode()).hexdigest()[:16]
-    return value if int(value, 16) else "0000000000000001"
-
-
 def sanitize(value: Any, *, depth: int = 0) -> Any:
     if depth > 12:
         return "[truncated]"
@@ -67,6 +61,8 @@ def sanitize(value: Any, *, depth: int = 0) -> Any:
                 if isinstance(decoded, (dict, list)):
                     return sanitize(decoded, depth=depth + 1)
         return value[:MAX_VALUE_CHARS] + ("[truncated]" if len(value) > MAX_VALUE_CHARS else "")
+    if isinstance(value, float) and not math.isfinite(value):
+        return str(value)
     if value is None or isinstance(value, (int, float, bool)):
         return value
     return str(value)[:MAX_VALUE_CHARS]
@@ -143,48 +139,68 @@ def _first(attributes: Mapping[str, Any], keys: Sequence[str]) -> Any:
     return None
 
 
-def normalize_agent_span(data: Mapping[str, Any], *, capture_content: bool) -> CanonicalSpan:
-    attributes = safe_attributes(data.get("attributes") or {}, capture_content=capture_content)
+def _count(value: object) -> int | None:
+    return int(value) if isinstance(value, int | float) and not isinstance(value, bool) and value >= 0 else None
+
+
+def _amount(value: object) -> float | None:
+    """A finite, non-negative price; the Agent runtime reports its decimal price as a string."""
+    if isinstance(value, bool) or not isinstance(value, int | float | str):
+        return None
+    try:
+        amount = float(value)
+    except ValueError:
+        return None
+    return amount if math.isfinite(amount) and amount >= 0 else None
+
+
+def _agent_kind(attributes: Mapping[str, Any]) -> SpanKind:
     operation = str(attributes.get("gen_ai.operation.name", ""))
     declared = str(attributes.get("openinference.span.kind", "")).lower()
     if operation in {"invoke_agent", "agent"} or declared == "agent":
-        kind = CanonicalSpanKind.AGENT
-    elif operation in {"execute_tool", "tool"} or "gen_ai.tool.name" in attributes or declared == "tool":
-        kind = CanonicalSpanKind.TOOL
-    elif (
+        return SpanKind.AGENT
+    if operation in {"execute_tool", "tool"} or "gen_ai.tool.name" in attributes or declared == "tool":
+        return SpanKind.TOOL
+    if (
         operation in {"chat", "text_completion", "generate_content", "embeddings"}
         or "gen_ai.request.model" in attributes
         or declared == "llm"
     ):
-        kind = CanonicalSpanKind.LLM
-    elif declared == "retriever":
-        kind = CanonicalSpanKind.RETRIEVER
+        return SpanKind.LLM
+    if declared == "retriever":
+        return SpanKind.RETRIEVER
+    return SpanKind.CHAIN
+
+
+def normalize_agent_span(data: Mapping[str, Any], *, capture_content: bool) -> Span:
+    attributes = safe_attributes(data.get("attributes") or {}, capture_content=capture_content)
+    kind = _agent_kind(attributes)
+    metadata: dict[str, Any] = {key: value for key, value in attributes.items() if key not in _CONTENT_ATTRIBUTES}
+    model_name = _first(attributes, ("gen_ai.response.model", "gen_ai.request.model", "llm.model_name"))
+    model_provider = _first(attributes, ("gen_ai.provider.name", "gen_ai.system"))
+    prompt_tokens = _count(_first(attributes, ("gen_ai.usage.input_tokens", "llm.token_count.prompt")))
+    completion_tokens = _count(_first(attributes, ("gen_ai.usage.output_tokens", "llm.token_count.completion")))
+    cache_read_tokens = _first(attributes, ("gen_ai.usage.cache_read.input_tokens", "gen_ai.usage.input_tokens.cached"))
+    if cache_read_tokens is not None:
+        metadata["cache_read_tokens"] = cache_read_tokens
+    price = _amount(_first(attributes, ("dify.llm.total_price",)))
+    currency = _first(attributes, ("dify.llm.currency",))
+    time_to_first_token = _first(attributes, ("gen_ai.response.time_to_first_token",))
+    usage = cost = None
+    if kind is SpanKind.LLM:
+        if prompt_tokens is not None or completion_tokens is not None:
+            usage = TokenUsage(
+                prompt_tokens=prompt_tokens or 0,
+                completion_tokens=completion_tokens or 0,
+                total_tokens=(prompt_tokens or 0) + (completion_tokens or 0),
+            )
+        if price is not None:
+            cost = Cost(amount=price, currency=str(currency or "USD"))
     else:
-        kind = CanonicalSpanKind.CHAIN
-    metadata = {key: value for key, value in attributes.items() if key not in _CONTENT_ATTRIBUTES}
-    mappings = {
-        "model_name": ("gen_ai.response.model", "gen_ai.request.model", "llm.model_name"),
-        "model_provider": ("gen_ai.provider.name", "gen_ai.system"),
-        "prompt_tokens": ("gen_ai.usage.input_tokens", "llm.token_count.prompt"),
-        "completion_tokens": ("gen_ai.usage.output_tokens", "llm.token_count.completion"),
-        "cache_read_tokens": ("gen_ai.usage.cache_read.input_tokens", "gen_ai.usage.input_tokens.cached"),
-        "total_price": ("dify.llm.total_price",),
-        "currency": ("dify.llm.currency",),
-        "time_to_first_token": ("gen_ai.response.time_to_first_token",),
-    }
-    for key, candidates in mappings.items():
-        value = _first(attributes, candidates)
-        if value is not None:
-            metadata[key] = value
-    if kind == CanonicalSpanKind.LLM:
-        input_tokens, output_tokens = metadata.get("prompt_tokens"), metadata.get("completion_tokens")
-        if isinstance(input_tokens, (int, float)) and isinstance(output_tokens, (int, float)):
-            metadata["total_tokens"] = input_tokens + output_tokens
-    else:
-        # Agent/chain aggregate usage is useful context, never another billed generation.
-        for key in ("prompt_tokens", "completion_tokens", "total_tokens", "total_price"):
-            if key in metadata:
-                metadata[f"aggregate_{key}"] = metadata.pop(key)
+        # Agent and chain aggregates are context, never another billed generation.
+        for key, value in (("prompt_tokens", prompt_tokens), ("completion_tokens", completion_tokens)):
+            if value is not None:
+                metadata[f"aggregate_{key}"] = value
     error = str(data.get("status_message") or "Agent operation failed") if data.get("status") == "error" else None
     if error and not capture_content:
         error = "Agent operation failed"
@@ -218,26 +234,30 @@ def normalize_agent_span(data: Mapping[str, Any], *, capture_content: bool) -> C
         if capture_content
         else None
     )
-    return CanonicalSpan(
-        id=str(data["span_id"]),
-        parent_id=data.get("parent_span_id"),
+    return Span(
+        span_id=str(data["span_id"]),
+        parent_span_id=data.get("parent_span_id"),
         name=str(data["name"]),
         kind=kind,
         start_time=datetime.fromtimestamp(int(data["start_time_ns"]) / 1e9, UTC),
         end_time=datetime.fromtimestamp(int(data["end_time_ns"]) / 1e9, UTC),
+        status=SpanStatus.ERROR if error else SpanStatus.OK,
+        error=error,
         inputs=sanitize(_decoded(inputs)),
         outputs=sanitize(_decoded(outputs)),
-        status=CanonicalSpanStatus.ERROR if error else CanonicalSpanStatus.OK,
-        error=error,
-        metadata=metadata,
+        model_name=str(model_name) if kind is SpanKind.LLM and model_name else None,
+        model_provider=str(model_provider) if kind is SpanKind.LLM and model_provider else None,
+        usage=usage,
+        cost=cost,
+        time_to_first_token=float(time_to_first_token)
+        if kind is SpanKind.LLM and isinstance(time_to_first_token, int | float) and time_to_first_token >= 0
+        else None,
+        attributes=sanitize(metadata),
     )
 
 
-def remap_outer_trace(
-    trace: CanonicalTrace, *, trace_id: str, capture_content: bool, agent_message: bool
-) -> CanonicalTrace:
-    """Use the same IDs allocated before runtime and suppress synthetic Agent usage."""
-    safe_metadata = {
+_SAFE_OUTER_ATTRIBUTES = frozenset(
+    {
         "workflow_id",
         "workflow_run_id",
         "workflow_app_log_id",
@@ -245,13 +265,6 @@ def remap_outer_trace(
         "node_execution_id",
         "node_type",
         "status",
-        "model_provider",
-        "model_name",
-        "prompt_tokens",
-        "completion_tokens",
-        "total_tokens",
-        "total_price",
-        "currency",
         "trace_entity_type",
         "wrapper_type",
         "wrapper_index",
@@ -263,64 +276,107 @@ def remap_outer_trace(
         "conversation_id",
         "external_trace_id",
         "message_execution_state",
+        "parent_workflow_run_id",
+        "parent_node_execution_id",
         "linked_parent_workflow_run_id",
         "linked_parent_node_execution_id",
         "linked_parent_application_trace_id",
         "linked_parent_trace_id",
     }
-    spans = []
+)
+
+
+def remap_outer_trace(
+    trace: Trace, *, trace_id: str, capture_content: bool, synthetic_span_ids: frozenset[str]
+) -> Trace:
+    """Adopt the run's trace ID and capture policy; drop spans the Agent runtime reports itself."""
+    spans: list[Span] = []
     for original in trace.spans:
-        if agent_message and original.synthetic and original.kind == CanonicalSpanKind.LLM:
+        if original.span_id in synthetic_span_ids:
             continue
-        metadata = (
-            sanitize(original.metadata)
-            if capture_content
-            else {k: sanitize(v) for k, v in original.metadata.items() if k in safe_metadata}
-        )
-        if trace.trace_id != trace_id:
-            metadata.setdefault("external_trace_id", trace.trace_id)
-        if trace.external_parent is not None:
-            # This pipeline exports a complete local tree. Cross-workflow
-            # correlation is explicit until a durable cross-run merge exists.
-            metadata["linked_parent_workflow_run_id"] = trace.external_parent.parent_workflow_run_id
-            if trace.external_parent.parent_node_execution_id:
-                metadata["linked_parent_node_execution_id"] = trace.external_parent.parent_node_execution_id
-            metadata.pop("parent_trace_context", None)
-        if original.kind != CanonicalSpanKind.LLM:
-            for key in ("prompt_tokens", "completion_tokens", "total_tokens", "total_price"):
-                if key in metadata:
-                    metadata[f"aggregate_{key}"] = metadata.pop(key)
+        attributes = dict(original.attributes)
+        if trace.external_trace_id:
+            attributes.setdefault("external_trace_id", trace.external_trace_id)
+        if not capture_content:
+            attributes = {key: value for key, value in attributes.items() if key in _SAFE_OUTER_ATTRIBUTES}
         spans.append(
             original.model_copy(
                 update={
-                    "id": span_id(original.id),
-                    "parent_id": span_id(original.parent_id) if original.parent_id else None,
                     "inputs": sanitize(original.inputs) if capture_content else None,
                     "outputs": sanitize(original.outputs) if capture_content else None,
-                    "metadata": metadata,
+                    "attributes": sanitize(attributes),
                     "error": original.error
                     if capture_content or not original.error
                     else "Application execution failed",
-                    "can_parent_workflow": False,
-                    "publishes_parent_context": False,
                 }
             )
         )
-    return CanonicalTrace(
-        trace_id=trace_id, session_id=trace.session_id, root_span_id=span_id(trace.root_span_id), spans=tuple(spans)
+    return Trace(
+        trace_id=trace_id,
+        external_trace_id=None,
+        session_id=trace.session_id,
+        user_id=trace.user_id,
+        parent=None,
+        spans=tuple(spans),
     )
 
 
-def order_spans(root_id: str, spans: Sequence[CanonicalSpan]) -> tuple[CanonicalSpan, ...]:
-    by_id: dict[str, CanonicalSpan] = {}
+def legacy_outer_trace(data: Mapping[str, Any]) -> Trace:
+    """Read an outer trace in the shape the previous release stored, so runs captured before an
+    upgrade still export. Its spans already carry this ledger's identifiers and capture policy."""
+    spans: list[Span] = []
+    for raw in data["spans"]:
+        metadata = raw.get("metadata") or {}
+        llm = raw["kind"] == SpanKind.LLM.value
+        prompt, completion = _count(metadata.get("prompt_tokens")), _count(metadata.get("completion_tokens"))
+        amount, currency = _amount(metadata.get("total_price")), metadata.get("currency")
+        spans.append(
+            Span(
+                span_id=raw["id"],
+                parent_span_id=raw["parent_id"],
+                name=raw["name"],
+                kind=SpanKind(raw["kind"]),
+                start_time=raw["start_time"],
+                end_time=raw["end_time"] or raw["start_time"],
+                status=SpanStatus(raw["status"]),
+                error=raw.get("error"),
+                inputs=sanitize(raw.get("inputs")),
+                outputs=sanitize(raw.get("outputs")),
+                model_name=metadata.get("model_name") if llm else None,
+                model_provider=metadata.get("model_provider") if llm else None,
+                usage=TokenUsage(
+                    prompt_tokens=prompt or 0,
+                    completion_tokens=completion or 0,
+                    total_tokens=_count(metadata.get("total_tokens")) or (prompt or 0) + (completion or 0),
+                )
+                if llm and (prompt is not None or completion is not None)
+                else None,
+                cost=Cost(amount=amount, currency=currency)
+                if llm and amount is not None and isinstance(currency, str) and currency
+                else None,
+                attributes=sanitize(metadata),
+            )
+        )
+    return Trace(
+        trace_id=data["trace_id"],
+        external_trace_id=None,
+        session_id=data.get("session_id") or None,
+        user_id=None,
+        parent=None,
+        spans=tuple(spans),
+    )
+
+
+def order_spans(root_id: str, spans: Sequence[Span]) -> tuple[Span, ...]:
+    by_id: dict[str, Span] = {}
     for span in spans:
-        prior = by_id.get(span.id)
+        prior = by_id.get(span.span_id)
         if prior is not None and prior != span:
             raise ValueError("Conflicting trace span identity")
-        by_id[span.id] = span
+        by_id[span.span_id] = span
     if root_id not in by_id:
         raise ValueError("Trace root is missing")
-    ordered: list[CanonicalSpan] = []
+    ordered: list[Span] = []
     complete: set[str] = set()
     visiting: set[str] = set()
 
@@ -333,14 +389,15 @@ def order_spans(root_id: str, spans: Sequence[CanonicalSpan]) -> tuple[Canonical
         if span is None:
             raise ValueError("Trace parent is missing")
         visiting.add(identifier)
-        if span.parent_id is not None:
-            visit(span.parent_id)
-        elif span.id != root_id:
+        if span.parent_span_id is not None:
+            visit(span.parent_span_id)
+        elif span.span_id != root_id:
             raise ValueError("Unexpected trace root")
         visiting.remove(identifier)
         complete.add(identifier)
         ordered.append(span)
 
+    visit(root_id)
     for span in spans:
-        visit(span.id)
+        visit(span.span_id)
     return tuple(ordered)

api/domain/tracing/sources.py

@@ -0,0 +1,49 @@
+"""Committed facts about the records a trace run is collected for."""
+
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass(frozen=True, slots=True)
+class SourceWorkflowRun:
+    id: str
+    workflow_id: str
+    graph: str | None
+    """The graph snapshot of the run; ``None`` when the backend does not store it."""
+    status: str
+    finished_at: datetime | None
+    """``None`` while the run has not finished."""
+    created_by: str
+
+
+@dataclass(frozen=True, slots=True)
+class SourceNodeExecution:
+    id: str
+    node_id: str
+    workflow_id: str | None
+    """``None`` when the backend does not store the workflow."""
+    workflow_run_id: str | None
+    """``None`` for a single-node preview."""
+    title: str
+    status: str
+    error: str | None
+    inputs: str | None
+    outputs: str | None
+    created_at: datetime
+    elapsed_time: float
+
+
+@dataclass(frozen=True, slots=True)
+class MessageProgress:
+    """What recovery may infer about a message from its committed row."""
+
+    message_id: str
+    conversation_id: str
+    agent_app: bool
+    paused: bool
+    explicit_error: bool
+    """The message failed and recorded why."""
+    answer_saved: bool
+    """The final answer transaction committed, not just an early usage flush."""
+    agent_stop_requested: bool
+    updated_at: datetime

api/domain/tracing/terminal_recovery.py

@@ -0,0 +1,65 @@
+"""Decide when a run's dropped terminal event can be recovered from its source's committed state.
+
+The decision combines the run's invocations with what its source committed.
+``now`` is naive UTC, like the run's timestamps.
+"""
+
+from datetime import datetime, timedelta
+
+from domain.tracing.errors import TraceRunError
+from domain.tracing.sources import MessageProgress, SourceWorkflowRun
+from domain.tracing.trace_run_agg import TraceRunAgg
+from graphon.enums import WorkflowExecutionStatus
+
+_SETTLE_SECONDS = 5
+_WORKFLOW_TERMINAL = {
+    WorkflowExecutionStatus.SUCCEEDED,
+    WorkflowExecutionStatus.PARTIAL_SUCCEEDED,
+    WorkflowExecutionStatus.FAILED,
+    WorkflowExecutionStatus.STOPPED,
+}
+
+
+def message_terminal_due(
+    run: TraceRunAgg, progress: MessageProgress, *, now: datetime, collection_timeout: int
+) -> bool:
+    """Whether the message's committed row is the run's business terminal now.
+
+    Raises:
+        TraceRunError: The terminal can no longer arrive.
+    """
+    invocations = list(run.invocations.values())
+    pending_agent_stop = progress.agent_app and progress.agent_stop_requested
+    if progress.paused and not pending_agent_stop:
+        return False
+    if any(not invocation.finalized or invocation.finalized_at is None for invocation in invocations):
+        if (progress.explicit_error or pending_agent_stop) and progress.updated_at + timedelta(
+            seconds=collection_timeout
+        ) < now:
+            raise TraceRunError("agent_trace_collection_incomplete")
+        return False
+    finalized = [invocation.finalized_at for invocation in invocations if invocation.finalized_at is not None]
+    anchor = max([progress.updated_at, *finalized])
+    if pending_agent_stop:
+        # Even a fully saved partial answer/usage cannot establish which
+        # runtime terminal won the stop race. Wait for the existing worker
+        # handoff, or converge to incomplete delivery after its deadline.
+        if anchor + timedelta(seconds=collection_timeout) < now:
+            raise TraceRunError("business_terminal_not_persisted")
+        return False
+    if run.capture_agent and not invocations:
+        return False
+    if anchor + timedelta(seconds=_SETTLE_SECONDS) > now:
+        return False
+    if not (progress.explicit_error or progress.answer_saved):
+        if finalized and max(finalized) + timedelta(seconds=collection_timeout) < now:
+            raise TraceRunError("business_terminal_not_persisted")
+        return False
+    return True
+
+
+def workflow_terminal_due(workflow: SourceWorkflowRun, *, now: datetime) -> bool:
+    """Whether the workflow run finished long enough ago for its node executions to be saved."""
+    if workflow.status not in _WORKFLOW_TERMINAL or workflow.finished_at is None:
+        return False
+    return workflow.finished_at + timedelta(seconds=_SETTLE_SECONDS) <= now

api/domain/tracing/trace_run_agg.py

@@ -0,0 +1,642 @@
+"""The Agent trace run aggregate.
+
+A run is the trace of one Agent-managed execution: the destination and capture
+policy frozen when the execution started, the runtime invocations that report
+spans into it, its business terminal, and its export lifecycle. Callers pass
+the time, generated identifiers, and the facts a rule needs.
+"""
+
+import hashlib
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field, replace
+from datetime import UTC, datetime, timedelta
+from enum import StrEnum
+from typing import Any, Literal
+from uuid import UUID, uuid5
+
+from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceSpan
+from dify_trace_sdk.ids import span_id_for
+from dify_trace_sdk.model import Span, SpanKind, SpanStatus, Trace
+
+from domain.tracing.errors import TraceRunConflictError, TraceRunError, TraceRunLimitError, TraceRunNotFoundError
+from domain.tracing.normalization import (
+    legacy_outer_trace,
+    normalize_agent_span,
+    order_spans,
+    remap_outer_trace,
+    safe_attributes,
+    sanitize,
+)
+from domain.tracing.sources import SourceNodeExecution
+
+type TraceSourceKind = Literal["message", "workflow", "node"]
+
+
+class TraceRunState(StrEnum):
+    NOT_COLLECTED = "not_collected"
+    """The app did not trace when the execution started; nothing is collected or exported."""
+    COLLECTING = "collecting"
+    READY = "ready"
+    """Released by a manual retry; exported once its collection is complete."""
+    EXPORTING = "exporting"
+    RETRYING = "retrying"
+    ACCEPTED = "accepted"
+    FAILED = "failed"
+    UNCERTAIN = "uncertain"
+    """The destination may have accepted the trace; only an acknowledged retry sends it again."""
+
+    @property
+    def accepts_terminal(self) -> bool:
+        """A run that collects nothing or is closed keeps no business terminal."""
+        return self in {TraceRunState.COLLECTING, TraceRunState.READY, TraceRunState.RETRYING}
+
+
+# A closed run accepts no runtime batches and no other business terminal.
+_CLOSED_STATES = frozenset(
+    {TraceRunState.ACCEPTED, TraceRunState.EXPORTING, TraceRunState.UNCERTAIN, TraceRunState.FAILED}
+)
+_COLLECTING_STATES = frozenset({TraceRunState.COLLECTING, TraceRunState.READY})
+_SETTLED_STATES = frozenset(
+    {TraceRunState.NOT_COLLECTED, TraceRunState.ACCEPTED, TraceRunState.FAILED, TraceRunState.UNCERTAIN}
+)
+_UNFINISHED_NODE_STATUSES = frozenset({"running", "pending", "paused"})
+_SESSION_ID_CHARS = 255
+
+
+@dataclass(frozen=True, slots=True)
+class TraceInvocation:
+    """One Agent runtime invocation that reports spans into a run."""
+
+    id: str
+    run_id: str
+    agent_id: str
+    parent_span_id: str
+    span_id: str
+    node_execution_id: str | None
+    """``None`` for the invocation of an Agent app."""
+    attempt: int
+    finalized: bool
+    finalized_at: datetime | None
+    """``None`` until the runtime reports the invocation's final batch."""
+
+
+@dataclass(frozen=True, slots=True)
+class TraceBatchReceipt:
+    """What a run remembers of a stored batch to recognize its replays."""
+
+    id: str
+    invocation_id: str
+    payload_hash: str
+    """The SHA-256 of the batch's original payload."""
+
+
+@dataclass(frozen=True, slots=True)
+class TraceBatch:
+    """A runtime batch as a run stores it; its spans already follow the run's capture policy."""
+
+    id: str
+    invocation_id: str
+    payload_hash: str
+    spans: Sequence[Mapping[str, Any]]
+    final: bool
+
+
+@dataclass(kw_only=True, eq=False, slots=True)
+class TraceRunAgg:
+    """An Agent trace run, its invocations, and the receipts of its stored batches.
+
+    State changes only through the methods below. ``invocations`` keeps them by
+    attempt, then creation. ``new_batches`` holds the batches ingested since the
+    run was loaded, for the repository to store.
+    """
+
+    id: str
+    tenant_id: str
+    app_id: str
+    source_kind: TraceSourceKind
+    source_id: str
+    provider: str
+    """``""`` when the run froze no destination."""
+    config_revision: str
+    """``""`` when the run froze no destination."""
+    provider_config: Mapping[str, Any]
+    """The frozen stored configuration, secrets still encrypted."""
+    trace_id: str
+    root_span_id: str
+    session_id: str
+    capture_content: bool
+    capture_agent: bool
+    state: TraceRunState
+    outer_trace: Mapping[str, Any] | None
+    """The captured business terminal; ``None`` until it is captured."""
+    span_count: int
+    payload_bytes: int
+    attempts: int
+    error_code: str | None
+    """``None`` unless the run failed, waits for a retry, or its export outcome is uncertain."""
+    trace_url: str | None
+    provider_trace_id: str | None
+    span_urls: Mapping[str, str]
+    export_token: str | None
+    """The token fencing the current export attempt; ``None`` when no attempt holds the run."""
+    export_started_at: datetime | None
+    export_heartbeat_at: datetime | None
+    """``None`` for attempts started before heartbeats were recorded."""
+    next_attempt_at: datetime | None
+    """``None`` unless a retryable export failure waits for its backoff."""
+    accepted_at: datetime | None
+    terminal_at: datetime | None
+    """When the business terminal was captured; ``None`` until then."""
+    updated_at: datetime
+    invocations: dict[str, TraceInvocation]
+    receipts: dict[str, TraceBatchReceipt]
+    new_batches: list[TraceBatch] = field(default_factory=list, init=False)
+
+    @classmethod
+    def freeze(
+        cls,
+        *,
+        run_id: str,
+        trace_id: str,
+        tenant_id: str,
+        app_id: str,
+        source_kind: TraceSourceKind,
+        source_id: str,
+        session_id: str,
+        policy: Mapping[str, Any],
+        stored_config: Mapping[str, Any] | None,
+        now: datetime,
+    ) -> "TraceRunAgg":
+        """Freeze the app's destination and capture policy for a source.
+
+        A run collects only when the policy enables the provider it names and
+        that provider has a stored configuration; otherwise it is frozen as not
+        collected. The revision identifies the frozen policy and configuration.
+        """
+        named = policy.get("tracing_provider")
+        destination = (
+            (named, dict(stored_config))
+            if isinstance(named, str) and policy.get("enabled") is True and stored_config is not None
+            else None
+        )
+        provider, frozen = destination if destination is not None else ("", {})
+        collecting = destination is not None
+        revision = hashlib.sha256(
+            json.dumps({"config": frozen, "policy": dict(policy)}, sort_keys=True, default=str).encode()
+        ).hexdigest()
+        return cls._new(
+            run_id=run_id,
+            trace_id=trace_id,
+            tenant_id=tenant_id,
+            app_id=app_id,
+            source_kind=source_kind,
+            source_id=source_id,
+            provider=provider,
+            config_revision=revision,
+            provider_config=frozen,
+            session_id=session_id[:_SESSION_ID_CHARS] or source_id,
+            capture_content=collecting and policy.get("capture_content", False) is True,
+            capture_agent=collecting and policy.get("capture_agent", True) is True,
+            state=TraceRunState.COLLECTING if collecting else TraceRunState.NOT_COLLECTED,
+            error_code=None,
+            terminal_at=None,
+            now=now,
+        )
+
+    @classmethod
+    def initialization_failed(
+        cls,
+        *,
+        run_id: str,
+        trace_id: str,
+        tenant_id: str,
+        app_id: str,
+        source_kind: TraceSourceKind,
+        source_id: str,
+        now: datetime,
+    ) -> "TraceRunAgg":
+        """Close a source whose start snapshot is missing.
+
+        It must never fall back to an exporter without its capture policy.
+        """
+        return cls._new(
+            run_id=run_id,
+            trace_id=trace_id,
+            tenant_id=tenant_id,
+            app_id=app_id,
+            source_kind=source_kind,
+            source_id=source_id,
+            provider="",
+            config_revision="",
+            provider_config={},
+            session_id=source_id,
+            capture_content=False,
+            capture_agent=False,
+            state=TraceRunState.FAILED,
+            error_code="tracing_initialization_failed",
+            terminal_at=now,
+            now=now,
+        )
+
+    @classmethod
+    def _new(
+        cls,
+        *,
+        run_id: str,
+        trace_id: str,
+        tenant_id: str,
+        app_id: str,
+        source_kind: TraceSourceKind,
+        source_id: str,
+        provider: str,
+        config_revision: str,
+        provider_config: Mapping[str, Any],
+        session_id: str,
+        capture_content: bool,
+        capture_agent: bool,
+        state: TraceRunState,
+        error_code: str | None,
+        terminal_at: datetime | None,
+        now: datetime,
+    ) -> "TraceRunAgg":
+        return cls(
+            id=run_id,
+            tenant_id=tenant_id,
+            app_id=app_id,
+            source_kind=source_kind,
+            source_id=source_id,
+            provider=provider,
+            config_revision=config_revision,
+            provider_config=provider_config,
+            trace_id=trace_id,
+            root_span_id=span_id_for(source_id),
+            session_id=session_id,
+            capture_content=capture_content,
+            capture_agent=capture_agent,
+            state=state,
+            outer_trace=None,
+            span_count=0,
+            payload_bytes=0,
+            attempts=0,
+            error_code=error_code,
+            trace_url=None,
+            provider_trace_id=None,
+            span_urls={},
+            export_token=None,
+            export_started_at=None,
+            export_heartbeat_at=None,
+            next_attempt_at=None,
+            accepted_at=None,
+            terminal_at=terminal_at,
+            updated_at=now,
+            invocations={},
+            receipts={},
+        )
+
+    @property
+    def awaits_terminal(self) -> bool:
+        """The run collects and has not captured its business terminal yet."""
+        return self.outer_trace is None and self.state in _COLLECTING_STATES
+
+    def open_invocation(
+        self, *, agent_id: str, node_execution_id: str | None, attempt: int, call_key: str
+    ) -> TraceInvocation:
+        """Create the invocation of a call key once; a repeated call key must name the same Agent binding.
+
+        Raises:
+            TraceRunConflictError: The run stopped collecting, or the call key names another invocation.
+        """
+        if self.state not in _COLLECTING_STATES:
+            raise TraceRunConflictError("tracing_run_not_collecting")
+        identifier = str(uuid5(UUID(self.id), call_key))
+        current = self.invocations.get(identifier)
+        if current is not None:
+            if (current.agent_id, current.node_execution_id, current.attempt) != (agent_id, node_execution_id, attempt):
+                raise TraceRunConflictError("tracing_invocation_conflict")
+            return current
+        invocation = TraceInvocation(
+            id=identifier,
+            run_id=self.id,
+            agent_id=agent_id,
+            parent_span_id=span_id_for(node_execution_id) if node_execution_id else self.root_span_id,
+            span_id=span_id_for(f"agent-attempt:{identifier}"),
+            node_execution_id=node_execution_id,
+            attempt=attempt,
+            finalized=False,
+            finalized_at=None,
+        )
+        self.invocations[identifier] = invocation
+        return invocation
+
+    def ingest(self, batch: AgentTraceBatch, *, max_spans: int, max_bytes: int, now: datetime) -> int:
+        """Append a runtime batch and return its span count; a final batch finalizes its invocation.
+
+        A batch ID replays only with the same validated payload, even after the run closed.
+
+        Raises:
+            TraceRunNotFoundError: The batch's invocation is not one of this run's.
+            TraceRunConflictError: The batch conflicts with a stored one, or the run or invocation stopped collecting.
+            TraceRunLimitError: The batch exceeds the run's limits.
+        """
+        # Hash the original validated payload. Sanitization cannot turn two
+        # conflicting producer payloads into a falsely successful replay.
+        canonical = json.dumps(batch.model_dump(mode="json"), sort_keys=True, separators=(",", ":")).encode()
+        digest = hashlib.sha256(canonical).hexdigest()
+        invocation = self.invocations.get(str(batch.invocation_id))
+        if invocation is None:
+            raise TraceRunNotFoundError("tracing_invocation_not_found")
+        batch_id = str(batch.batch_id)
+        spans = batch.spans
+        previous = self.receipts.get(batch_id)
+        if previous is not None:
+            if previous.invocation_id != invocation.id or previous.payload_hash != digest:
+                raise TraceRunConflictError("tracing_batch_conflict")
+            return len(spans)
+        if self.state in _CLOSED_STATES or invocation.finalized:
+            raise TraceRunConflictError("tracing_collection_closed")
+        if any(span.trace_id != self.trace_id for span in spans):
+            raise TraceRunConflictError("tracing_trace_id_mismatch")
+        if self.span_count + len(spans) > max_spans or self.payload_bytes + len(canonical) > max_bytes:
+            raise TraceRunLimitError("tracing_payload_limit")
+        final = batch.final
+        self.new_batches.append(
+            TraceBatch(
+                id=batch_id,
+                invocation_id=invocation.id,
+                payload_hash=digest,
+                spans=[self._stored_span(span) for span in spans],
+                final=final,
+            )
+        )
+        self.receipts[batch_id] = TraceBatchReceipt(id=batch_id, invocation_id=invocation.id, payload_hash=digest)
+        self.span_count += len(spans)
+        self.payload_bytes += len(canonical)
+        self.updated_at = now
+        if final:
+            self.invocations[invocation.id] = replace(invocation, finalized=True, finalized_at=now)
+        return len(spans)
+
+    def _stored_span(self, span: AgentTraceSpan) -> dict[str, Any]:
+        stored = span.model_dump(mode="json")
+        stored["attributes"] = safe_attributes(span.attributes, capture_content=self.capture_content)
+        if not self.capture_content:
+            stored["events"] = []
+            stored["status_message"] = "Agent operation failed" if span.status == "error" else None
+        return stored
+
+    def capture_terminal(self, trace: Trace, *, now: datetime) -> None:
+        """Keep the execution's business terminal under the run's trace ID, session, and capture policy.
+
+        The Agent runtime reports a message's LLM work itself, so the
+        terminal's own LLM span is dropped.
+        """
+        synthetic = frozenset({span_id_for(f"{self.source_id}:llm")}) if self.source_kind == "message" else frozenset()
+        adopted = remap_outer_trace(
+            trace, trace_id=self.trace_id, capture_content=self.capture_content, synthetic_span_ids=synthetic
+        )
+        self._keep_terminal(adopted.model_copy(update={"session_id": self.session_id}), now=now)
+
+    def settle_preview(self, node: SourceNodeExecution, *, now: datetime) -> None:
+        """Make a finished single-node preview its own terminal; a preview reports no business terminal."""
+        if self.source_kind != "node" or self.outer_trace is not None or node.status in _UNFINISHED_NODE_STATUSES:
+            return
+        error = node.error
+        started_at = node.created_at.replace(tzinfo=UTC)
+        span = Span(
+            span_id=self.root_span_id,
+            parent_span_id=None,
+            name=node.title or "Agent preview",
+            kind=SpanKind.AGENT,
+            start_time=started_at,
+            end_time=started_at + timedelta(seconds=node.elapsed_time or 0),
+            status=SpanStatus.ERROR if error else SpanStatus.OK,
+            error=error if self.capture_content else "Node execution failed" if error else None,
+            inputs=sanitize(node.inputs) if self.capture_content else None,
+            outputs=sanitize(node.outputs) if self.capture_content else None,
+            attributes={"node_execution_id": node.id, "node_id": node.node_id, "source": "single_step"},
+        )
+        trace = Trace(
+            trace_id=self.trace_id,
+            external_trace_id=None,
+            session_id=self.session_id,
+            user_id=None,
+            parent=None,
+            spans=(span,),
+        )
+        self._keep_terminal(trace, now=now)
+
+    def _keep_terminal(self, trace: Trace, *, now: datetime) -> None:
+        if not self.state.accepts_terminal:
+            return
+        stored = trace.model_dump(mode="json")
+        if self.outer_trace is not None and self.outer_trace != stored:
+            # A duplicate terminal task must not rewrite a completed immutable
+            # span. The first persisted terminal envelope owns the export.
+            return
+        self.outer_trace = stored
+        self.root_span_id = trace.root.span_id
+        self.terminal_at = self.terminal_at or now
+        self.updated_at = now
+
+    def recovery_checked(self, error_code: str | None, *, now: datetime) -> None:
+        """Rotate incomplete work behind other pending runs, or fail it with ``error_code``.
+
+        A run that captured its terminal or left collection is not changed.
+        """
+        if not self.awaits_terminal:
+            return
+        self.updated_at = now
+        if error_code is not None:
+            self.state, self.error_code = TraceRunState.FAILED, error_code
+
+    def claim_export(
+        self, *, token: str, owner_exists: bool, collection_timeout: int, lease_seconds: int, now: datetime
+    ) -> bool:
+        """Lease a complete run for one export attempt that ``token`` fences; ``False`` when it is not exportable now.
+
+        A run is complete once its terminal is captured and every invocation is
+        finalized; ``collection_timeout`` seconds after the terminal an
+        incomplete run fails. An export whose last heartbeat is older than
+        ``lease_seconds`` becomes uncertain, and a run whose app is gone fails.
+        """
+        if self.state in _SETTLED_STATES:
+            return False
+        if self.state is TraceRunState.EXPORTING:
+            last_heartbeat = self.export_heartbeat_at or self.export_started_at
+            if last_heartbeat and last_heartbeat + timedelta(seconds=lease_seconds) < now:
+                self.state, self.error_code = TraceRunState.UNCERTAIN, "export_lease_expired"
+                self.updated_at = now
+            return False
+        if self.outer_trace is None:
+            return False
+        if any(not invocation.finalized for invocation in self.invocations.values()):
+            if self.terminal_at and self.terminal_at + timedelta(seconds=collection_timeout) < now:
+                self.state, self.error_code = TraceRunState.FAILED, "agent_trace_collection_incomplete"
+                self.updated_at = now
+            return False
+        if self.next_attempt_at and self.next_attempt_at > now:
+            return False
+        if not owner_exists:
+            self.state, self.error_code = TraceRunState.FAILED, "tracing_owner_not_found"
+            return False
+        self.state, self.export_token = TraceRunState.EXPORTING, token
+        self.export_started_at = self.export_heartbeat_at = self.updated_at = now
+        self.attempts += 1
+        return True
+
+    def renew_export_lease(self, token: str, *, now: datetime) -> bool:
+        """Renew only the current export attempt; ``False`` once the attempt no longer holds the run."""
+        if self.state is not TraceRunState.EXPORTING or self.export_token != token:
+            return False
+        self.export_heartbeat_at = self.updated_at = now
+        return True
+
+    def export_accepted(
+        self,
+        token: str,
+        *,
+        provider_trace_id: str,
+        trace_url: str | None,
+        span_urls: Mapping[str, str],
+        now: datetime,
+    ) -> None:
+        """Record the destination's acceptance.
+
+        An acceptance that arrives after its lease expired still records the
+        destination's result, unless a manual retry replaced its token.
+        """
+        late = self.state is TraceRunState.UNCERTAIN and self.error_code == "export_lease_expired"
+        if self.export_token != token or (self.state is not TraceRunState.EXPORTING and not late):
+            return
+        self._finish_export(TraceRunState.ACCEPTED, None, now=now)
+        self.provider_trace_id, self.trace_url, self.span_urls = provider_trace_id, trace_url, dict(span_urls)
+        self.accepted_at = now
+
+    def export_deferred(self, token: str, *, max_attempts: int, now: datetime) -> None:
+        """Wait with exponential backoff after a retryable failure until the attempts run out."""
+        if self.attempts >= max_attempts:
+            self.export_failed(token, "provider_retry_exhausted", now=now)
+            return
+        if not self._holds_export(token):
+            return
+        self._finish_export(TraceRunState.RETRYING, "provider_temporarily_unavailable", now=now)
+        self.next_attempt_at = now + timedelta(seconds=min(300, 5 * 2 ** min(self.attempts - 1, 6)))
+
+    def export_uncertain(self, token: str, *, now: datetime) -> None:
+        """The destination may have accepted the trace; only an acknowledged retry sends it again."""
+        if self._holds_export(token):
+            self._finish_export(TraceRunState.UNCERTAIN, "provider_acceptance_unknown", now=now)
+
+    def export_failed(self, token: str, error_code: str, *, now: datetime) -> None:
+        if self._holds_export(token):
+            self._finish_export(TraceRunState.FAILED, error_code, now=now)
+
+    def _holds_export(self, token: str) -> bool:
+        return self.state is TraceRunState.EXPORTING and self.export_token == token
+
+    def _finish_export(self, state: TraceRunState, error_code: str | None, *, now: datetime) -> None:
+        self.state, self.error_code = state, error_code
+        self.export_token = None
+        self.next_attempt_at = None
+        self.updated_at = now
+
+    def release_for_retry(self, *, acknowledge_possible_duplicates: bool, now: datetime) -> None:
+        """Release a failed or uncertain run for another export.
+
+        Raises:
+            TraceRunNotFoundError: Nothing was collected.
+            TraceRunConflictError: The run is accepted, not retryable, or needs the duplicate acknowledgement.
+        """
+        if self.state is TraceRunState.NOT_COLLECTED:
+            raise TraceRunNotFoundError("tracing_run_not_found")
+        if self.state is TraceRunState.ACCEPTED:
+            raise TraceRunConflictError("tracing_already_accepted")
+        if self.state not in {TraceRunState.FAILED, TraceRunState.UNCERTAIN} or self.outer_trace is None:
+            raise TraceRunConflictError("tracing_retry_not_allowed")
+        if any(not invocation.finalized for invocation in self.invocations.values()) or (
+            self.error_code or ""
+        ).startswith("tracing_"):
+            raise TraceRunConflictError("tracing_retry_not_allowed")
+        if self.state is TraceRunState.UNCERTAIN and not acknowledge_possible_duplicates:
+            raise TraceRunConflictError("tracing_duplicate_acknowledgement_required")
+        self.state, self.error_code, self.export_token = TraceRunState.READY, None, None
+        self.next_attempt_at = None
+        self.updated_at = now
+
+    def exported_trace(self, batches: Sequence[TraceBatch]) -> Trace:
+        """The trace this run exports: its business terminal with each invocation's spans under an Agent span.
+
+        ``batches`` are the run's stored batches by arrival.
+
+        Raises:
+            TraceRunError: The stored tree is not valid, so no export can repair it.
+            ValueError: A stored span is malformed.
+        """
+        if self.outer_trace is None:
+            raise TraceRunError("tracing_terminal_missing")
+        # Runs captured before the upgrade stored the previous trace shape, named by its root span.
+        stored = self.outer_trace
+        outer = legacy_outer_trace(stored) if "root_span_id" in stored else Trace.model_validate(stored)
+        spans = list(outer.spans)
+        internal_by_invocation: dict[str, dict[str, Span]] = {invocation_id: {} for invocation_id in self.invocations}
+        for batch in batches:
+            if batch.invocation_id not in self.invocations:
+                raise TraceRunError("tracing_invocation_not_found")
+            for data in batch.spans:
+                span = normalize_agent_span(data, capture_content=self.capture_content)
+                current = internal_by_invocation[batch.invocation_id].get(span.span_id)
+                if current is not None and current != span:
+                    raise TraceRunError("tracing_span_conflict")
+                internal_by_invocation[batch.invocation_id][span.span_id] = span
+        for invocation in self.invocations.values():
+            children = list(internal_by_invocation[invocation.id].values())
+            if not children:
+                raise TraceRunError("tracing_invocation_empty")
+            valid_ids = {child.span_id for child in children}
+            if any(
+                child.parent_span_id not in valid_ids and child.parent_span_id != invocation.span_id
+                for child in children
+            ):
+                raise TraceRunError("tracing_invocation_parent_invalid")
+            root_children = [child for child in children if child.parent_span_id == invocation.span_id]
+            if not root_children:
+                raise TraceRunError("tracing_invocation_root_missing")
+            errors = [child.error for child in root_children if child.status is SpanStatus.ERROR]
+            spans.append(
+                Span(
+                    span_id=invocation.span_id,
+                    parent_span_id=invocation.parent_span_id,
+                    name=f"Agent attempt {invocation.attempt + 1}",
+                    kind=SpanKind.AGENT,
+                    start_time=min(child.start_time for child in children),
+                    end_time=max(child.end_time for child in children),
+                    status=SpanStatus.ERROR if errors else SpanStatus.OK,
+                    error=errors[0] if errors else None,
+                    attributes={
+                        "agent_id": invocation.agent_id,
+                        "attempt": invocation.attempt,
+                        "node_execution_id": invocation.node_execution_id,
+                        "invocation_id": invocation.id,
+                    },
+                )
+            )
+            spans.extend(children)
+        trusted: dict[str, Any] = {
+            "tenant_id": self.tenant_id,
+            "app_id": self.app_id,
+            "session_id": self.session_id,
+            "dify.tenant_id": self.tenant_id,
+            "dify.app_id": self.app_id,
+            "dify.tracing.run_id": self.id,
+            "dify.source.id": self.source_id,
+        }
+        spans = [span.model_copy(update={"attributes": {**span.attributes, **trusted}}) for span in spans]
+        return Trace(
+            trace_id=self.trace_id,
+            external_trace_id=None,
+            session_id=self.session_id,
+            user_id=outer.user_id,
+            parent=None,
+            spans=order_spans(outer.root.span_id, spans),
+        )

api/domain/workflow/__init__.py

@@ -0,0 +1 @@
+"""Pure workflow contracts shared by application services and execution adapters."""

api/domain/workflow/agent_node.py

@@ -1,3 +1,5 @@
+"""Identify Dify Agent nodes without importing their execution runtime."""
+
 from collections.abc import Mapping
 from typing import Any
 

api/domain/workflow/retry_history.py

@@ -1,4 +1,4 @@
-"""Validated internal payloads for persisted workflow node retry history."""
+"""Validated retry history shared by workflow persistence and tracing."""
 
 from collections.abc import Mapping
 from typing import Any

api/enterprise/telemetry/enterprise_trace.py

@@ -35,7 +35,15 @@ from typing import Any, cast
 
 from opentelemetry.util.types import AttributeValue
 
-from core.ops.entities.trace_entity import (
+from enterprise.telemetry.entities import (
+    EnterpriseTelemetryCounter,
+    EnterpriseTelemetryEvent,
+    EnterpriseTelemetryHistogram,
+    EnterpriseTelemetrySpan,
+    TokenMetricLabels,
+)
+from enterprise.telemetry.telemetry_log import emit_metric_only_event, emit_telemetry_log
+from services.tracing.records import (
     BaseTraceInfo,
     DatasetRetrievalTraceInfo,
     DraftNodeExecutionTrace,
@@ -49,14 +57,6 @@ from core.ops.entities.trace_entity import (
     WorkflowNodeTraceInfo,
     WorkflowTraceInfo,
 )
-from enterprise.telemetry.entities import (
-    EnterpriseTelemetryCounter,
-    EnterpriseTelemetryEvent,
-    EnterpriseTelemetryHistogram,
-    EnterpriseTelemetrySpan,
-    TokenMetricLabels,
-)
-from enterprise.telemetry.telemetry_log import emit_metric_only_event, emit_telemetry_log
 
 logger = logging.getLogger(__name__)
 
@@ -261,15 +261,9 @@ class EnterpriseOtelTrace:
                 invoke_from=invoke_from,
             ),
         )
-        # Prefer wall-clock timestamps over the elapsed_time field: elapsed_time defaults
-        # to 0 in the DB and can be stale if the Celery write races with the trace task.
-        # start_time = workflow_run.created_at, end_time = workflow_run.finished_at.
-        if info.start_time and info.end_time:
-            workflow_duration = (info.end_time - info.start_time).total_seconds()
-        elif info.workflow_run_elapsed_time:
-            workflow_duration = float(info.workflow_run_elapsed_time)
-        else:
-            workflow_duration = 0.0
+        # Wall-clock timestamps, not the elapsed_time field: elapsed_time defaults to 0
+        # in the DB and can be stale if the Celery write races with the trace task.
+        workflow_duration = (info.end_time - info.start_time).total_seconds()
         self._exporter.record_histogram(
             EnterpriseTelemetryHistogram.WORKFLOW_DURATION,
             workflow_duration,

api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py

@@ -6,7 +6,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from core.db.session_factory import session_factory
-from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data
+from domain.workflow.agent_node import is_dify_agent_node_data
 from events.app_event import app_published_workflow_was_updated
 from extensions.ext_database import db
 from models.knowledge_fs import AppKnowledgeFSSpaceJoin, KnowledgeFSAppSpaceJoinType

api/extensions/application_services/tracing.py

@@ -0,0 +1,109 @@
+"""Composition of the trace pipeline, Agent trace collection, and tracing provider configurations."""
+
+from dataclasses import dataclass
+
+from sqlalchemy.orm import Session, sessionmaker
+
+from configs import dify_config
+from enterprise.telemetry.exporter import is_enterprise_telemetry_enabled
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from repositories.application_trace_repository import ApplicationTraceRepository
+from repositories.factory import DifyAPIRepositoryFactory
+from repositories.trace_record_repository import TraceRecordRepository
+from services.tracing.adapters import (
+    CeleryExportRetryQueue,
+    CeleryTraceEmitter,
+    WorkspaceTracingSecretCipher,
+    schedule_agent_trace_export,
+    send_to_enterprise_telemetry,
+)
+from services.tracing.canonical import CanonicalTraceBuilder
+from services.tracing.collection import AgentTraceSettings
+from services.tracing.delivery import TraceDeliveryService
+from services.tracing.egress import SSRFEgress
+from services.tracing.provider_configs import ProviderConfigGateway, TraceProviderCapabilitiesQuery
+from services.tracing.providers import ProviderConfigCodec
+from services.tracing.registry import EntryPointTraceProviderCatalog
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
+from services.tracing.scopes import TraceScopes
+from services.tracing.service import ApplicationTracingService
+
+
+@dataclass(frozen=True, slots=True)
+class TracingServices:
+    scopes: TraceScopes
+    delivery: TraceDeliveryService
+    agent_traces: ApplicationTracingService
+    hooks: RuntimeApplicationTracingHooks | None
+    """``None`` when the deployment does not collect Agent traces."""
+    provider_configs: ProviderConfigGateway
+    provider_capabilities: TraceProviderCapabilitiesQuery
+
+
+def build_tracing_services(*, database_client: sessionmaker[Session]) -> TracingServices:
+    egress = SSRFEgress()
+    providers = EntryPointTraceProviderCatalog()
+    codec = ProviderConfigCodec(WorkspaceTracingSecretCipher())
+    builder = CanonicalTraceBuilder()
+    configs = SQLAlchemyAppTracingConfigRepository(session_factory=database_client)
+    enterprise_telemetry = is_enterprise_telemetry_enabled()
+    workflow_runs = DifyAPIRepositoryFactory.create_api_workflow_run_repository(database_client)
+    node_executions = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(database_client)
+    agent_settings = AgentTraceSettings(
+        enabled=dify_config.OPS_AGENT_TRACE_ENABLED,
+        max_spans=dify_config.OPS_AGENT_TRACE_MAX_SPANS,
+        max_bytes=dify_config.OPS_AGENT_TRACE_MAX_BYTES,
+        collection_timeout_seconds=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT,
+        export_lease_seconds=dify_config.OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS,
+        max_export_attempts=dify_config.OPS_AGENT_TRACE_MAX_ATTEMPTS,
+        retention_days=dify_config.OPS_AGENT_TRACE_RETENTION_DAYS,
+    )
+    agent_trace_runs = ApplicationTraceRepository(database_client)
+    agent_traces = ApplicationTracingService(
+        runs=agent_trace_runs,
+        run_query=agent_trace_runs,
+        sources=agent_trace_runs,
+        # App, workspace, and credential names are enterprise telemetry dimensions only.
+        records=TraceRecordRepository(
+            database_client,
+            workflow_runs=workflow_runs,
+            node_executions=node_executions,
+            files_url=dify_config.FILES_URL,
+            enterprise_details=False,
+        ),
+        builder=builder,
+        providers=providers,
+        codec=codec,
+        egress=egress,
+        schedule_export=schedule_agent_trace_export,
+        settings=agent_settings,
+    )
+    return TracingServices(
+        scopes=TraceScopes(
+            emitter=CeleryTraceEmitter(), policies=configs, enterprise_enabled=is_enterprise_telemetry_enabled
+        ),
+        delivery=TraceDeliveryService(
+            records=TraceRecordRepository(
+                database_client,
+                workflow_runs=workflow_runs,
+                node_executions=node_executions,
+                files_url=dify_config.FILES_URL,
+                enterprise_details=enterprise_telemetry,
+            ),
+            destinations=configs,
+            providers=providers,
+            codec=codec,
+            builder=builder,
+            egress=egress,
+            retries=CeleryExportRetryQueue(),
+            enterprise_sink=send_to_enterprise_telemetry if enterprise_telemetry else None,
+            collector=agent_traces if agent_settings.enabled else None,
+            max_export_attempts=dify_config.OPS_TRACE_EXPORT_MAX_ATTEMPTS,
+        ),
+        agent_traces=agent_traces,
+        hooks=RuntimeApplicationTracingHooks(agent_traces) if agent_settings.enabled else None,
+        provider_configs=ProviderConfigGateway(providers=providers, codec=codec, egress=egress),
+        provider_capabilities=TraceProviderCapabilitiesQuery(
+            providers=providers, api_collection_enabled=agent_settings.enabled
+        ),
+    )

api/extensions/ext_application_services.py

@@ -17,7 +17,6 @@ from sqlalchemy.orm import Session, sessionmaker
 from configs import dify_config
 from constants.dsl_version import CURRENT_APP_DSL_VERSION
 from core.db.session_factory import get_session_maker
-from core.ops.base_trace_instance import bind_application_tracing_hooks
 from core.schemas.schema_manager import SchemaManager
 from core.tools.tool_file_manager import ToolFileManager
 from core.tools.tool_manager import ToolManager
@@ -39,6 +38,7 @@ from extensions.application_services.knowledge import (
 )
 from extensions.application_services.rbac import RBACServices, build_rbac_services
 from extensions.application_services.resource_access_token import build_resource_access_token_service
+from extensions.application_services.tracing import TracingServices, build_tracing_services
 from extensions.application_services.trial_app import TrialAppServices, build_trial_app_services
 from extensions.application_services.workspace import (
     WorkspaceServices,
@@ -47,6 +47,7 @@ from extensions.application_services.workspace import (
 )
 from extensions.ext_redis import RedisClientWrapper, redis_client
 from extensions.ext_storage import storage
+from extensions.tracing import bind_tracing
 from libs.helper import RateLimiter
 from libs.passport import PassportService
 from models.model import EndUser
@@ -132,13 +133,12 @@ from services.app_scoped_end_user_service import AppScopedEndUserService
 from services.app_site_service import AppSiteService
 from services.app_statistic_query import AppStatisticQuery
 from services.app_task_service import AppTaskControlService
-from services.app_tracing_config_gateway import OpsTraceManagerGateway
 from services.app_tracing_config_service import AppTracingConfigService
 from services.billing_portal_service import BillingPortalService
 from services.billing_service import BillingService
 from services.compliance_download_service import ComplianceDownloadService
 from services.conversation_command_service import ConversationCommandService
-from services.conversation_name_generator import generate_conversation_name
+from services.conversation_name_generator import DefaultModelConversationNameGenerator
 from services.conversation_name_service import ConversationNameService
 from services.conversation_query_service import ConversationQueryService
 from services.conversation_service import ConversationService
@@ -217,7 +217,7 @@ from services.step_by_step_tour_service import StepByStepTourService
 from services.system_feature_service import SystemFeatureService
 from services.tag_application_service import TagApplicationService
 from services.tool_file_download_service import ToolFileDownloadService
-from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
+from services.tracing.adapters import StoredPolicyTraceScopes
 from services.upload_file_delivery_service import UploadFileDeliveryService
 from services.web_app_runtime_query_service import WebAppRuntimeQueryService
 from services.web_authentication_adapters import (
@@ -366,6 +366,7 @@ class ApplicationServices:
     web_passport: WebPassportService
     tags: TagApplicationService
     workflow_statistics: WorkflowStatisticQueryService
+    tracing: TracingServices
 
 
 def _build_oauth_server_service(
@@ -516,9 +517,10 @@ def build_application_services(
         apps=console_apps,
         datasets=dataset_dependencies.datasets,
     )
+    tracing = build_tracing_services(database_client=database_client)
     tracing_configs = AppTracingConfigService(
         configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client),
-        provider=OpsTraceManagerGateway(),
+        provider=tracing.provider_configs,
     )
     apps = build_app_services(
         database_client=database_client,
@@ -584,7 +586,10 @@ def build_application_services(
         invitation_tokens=invitation_tokens,
     )
     conversations = ConversationRepository(session_factory=database_client)
-    conversation_names = ConversationNameService(conversations=conversations, generate_name=generate_conversation_name)
+    conversation_names = ConversationNameService(
+        conversations=conversations,
+        generate_name=DefaultModelConversationNameGenerator(trace_scopes=tracing.scopes),
+    )
     conversation_commands = ConversationCommandService(
         conversations=conversations, enqueue_delete_cleanup=ConversationService.enqueue_delete_cleanup
     )
@@ -769,7 +774,7 @@ def build_application_services(
         ),
         message_suggested_questions=MessageSuggestedQuestionsService(
             queries=suggested_questions,
-            generator=SuggestedQuestionsGenerator(),
+            generator=SuggestedQuestionsGenerator(trace_scopes=tracing.scopes),
         ),
         plugin_file_uploads=PluginFileUploadService(
             owners=SQLAlchemyPluginFileUploadOwnerRepository(session_factory=database_client),
@@ -862,6 +867,7 @@ def build_application_services(
                 session_maker=database_client,
             ),
         ),
+        tracing=tracing,
     )
 
 
@@ -879,7 +885,7 @@ def init_app(app: Flask) -> None:
         redis=redis_client,
     )
     app.extensions[_EXTENSION_KEY] = services
-    bind_application_tracing_hooks(app, RuntimeApplicationTracingHooks())
+    bind_tracing(app, scopes=StoredPolicyTraceScopes(services.tracing.scopes), hooks=services.tracing.hooks)
     bind_account_loader(app, services.accounts.identity.load_user)
 
 

api/extensions/ext_celery.py

@@ -159,6 +159,11 @@ def init_app(app: DifyApp) -> Celery:
         timezone=pytz.timezone(dify_config.LOG_TZ or "UTC"),
         task_ignore_result=True,
         task_annotations=dify_config.CELERY_TASK_ANNOTATIONS,
+        task_routes={
+            "tasks.trace_delivery_task.*": {"queue": dify_config.OPS_TRACE_QUEUE},
+            "tasks.application_trace_task.*": {"queue": dify_config.OPS_TRACE_QUEUE},
+            "schedule.collect_agent_sandbox_usage.collect_agent_sandbox_usage": {"queue": dify_config.OPS_TRACE_QUEUE},
+        },
     )
 
     if dify_config.CELERY_BACKEND == "redis":
@@ -209,6 +214,8 @@ def init_app(app: DifyApp) -> Celery:
         "tasks.knowledge_fs_upgrade_tasks",  # legacy Dataset upgrades use a dedicated queue and worker
         "tasks.knowledge_fs_background_tasks",  # local-engine tasks on dedicated prefork queues
         "tasks.workflow_run_archive_deletion_tasks",  # permanent monthly archive deletion
+        "tasks.trace_delivery_task",  # trace delivery and provider export retries
+        "tasks.ops_trace_task",  # retired endpoint; old workers must drain pre-upgrade tasks
     ]
     day = dify_config.CELERY_BEAT_SCHEDULER_TIME
 
@@ -254,6 +261,11 @@ def init_app(app: DifyApp) -> Celery:
             "task": "tasks.knowledge_fs_upgrade_tasks.cleanup_deferred_knowledge_fs_upgrade_files",
             "schedule": timedelta(seconds=dify_config.KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS),
         }
+    if dify_config.ENABLE_TRACE_PAYLOAD_SWEEP_TASK:
+        beat_schedule["sweep_trace_payloads"] = {
+            "task": "tasks.trace_delivery_task.sweep_trace_payloads",
+            "schedule": crontab(minute="30", hour="3"),
+        }
     if dify_config.OPS_AGENT_TRACE_ENABLED:
         imports.append("tasks.application_trace_task")
     if dify_config.OPS_AGENT_TRACE_ENABLED and dify_config.ENABLE_APPLICATION_TRACE_RECONCILE_TASK:

api/extensions/ext_forward_refs.py

@@ -1,49 +0,0 @@
-import logging
-
-from dify_app import DifyApp
-
-
-def is_enabled() -> bool:
-    return True
-
-
-def init_app(app: DifyApp):
-    """Resolve Pydantic forward refs that would otherwise cause circular imports.
-
-    Rebuilds models in core.app.entities.app_invoke_entities with the real TraceQueueManager type.
-    Safe to run multiple times.
-    """
-    logger = logging.getLogger(__name__)
-    try:
-        from core.app.entities.app_invoke_entities import (
-            AdvancedChatAppGenerateEntity,
-            AgentChatAppGenerateEntity,
-            AppGenerateEntity,
-            ChatAppGenerateEntity,
-            CompletionAppGenerateEntity,
-            ConversationAppGenerateEntity,
-            EasyUIBasedAppGenerateEntity,
-            RagPipelineGenerateEntity,
-            WorkflowAppGenerateEntity,
-        )
-        from core.ops.ops_trace_manager import TraceQueueManager  # heavy import, do it at startup only
-
-        ns = {"TraceQueueManager": TraceQueueManager}
-        for model in (
-            AppGenerateEntity,
-            EasyUIBasedAppGenerateEntity,
-            ConversationAppGenerateEntity,
-            ChatAppGenerateEntity,
-            CompletionAppGenerateEntity,
-            AgentChatAppGenerateEntity,
-            AdvancedChatAppGenerateEntity,
-            WorkflowAppGenerateEntity,
-            RagPipelineGenerateEntity,
-        ):
-            try:
-                model.model_rebuild(_types_namespace=ns)
-            except Exception as e:
-                logger.debug("model_rebuild skipped for %s: %s", model.__name__, e)
-    except Exception as e:
-        # Don't block app startup; just log at debug level.
-        logger.debug("ext_forward_refs init skipped: %s", e)

api/extensions/ext_sentry.py

@@ -13,24 +13,6 @@ def init_app(app: DifyApp):
 
         from graphon.model_runtime.errors.invoke import InvokeRateLimitError
 
-        try:
-            from langfuse._utils import parse_error
-
-            _langfuse_error_response = parse_error.defaultErrorResponse
-        except (ImportError, AttributeError):
-            _langfuse_error_response = (
-                "Unexpected error occurred. Please check your request"
-                " and contact support: https://langfuse.com/support."
-            )
-
-        def before_send(event, hint):
-            if "exc_info" in hint:
-                _, exc_value, _ = hint["exc_info"]
-                if _langfuse_error_response in str(exc_value):
-                    return None
-
-            return event
-
         sentry_sdk.init(
             dsn=dify_config.SENTRY_DSN,
             integrations=[FlaskIntegration(), CeleryIntegration()],
@@ -39,13 +21,11 @@ def init_app(app: DifyApp):
                 ValueError,
                 FileNotFoundError,
                 InvokeRateLimitError,
-                _langfuse_error_response,
             ],
             traces_sample_rate=dify_config.SENTRY_TRACES_SAMPLE_RATE,
             profiles_sample_rate=dify_config.SENTRY_PROFILES_SAMPLE_RATE,
             environment=dify_config.DEPLOY_ENV,
             release=f"dify-{dify_config.project.version}-{dify_config.COMMIT_SHA}",
-            before_send=before_send,
         )
 
         if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:

api/extensions/logstore/repositories/logstore_api_workflow_node_execution_repository.py

@@ -22,6 +22,7 @@ from models.enums import CreatorUserRole
 from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
 from repositories.api_workflow_node_execution_repository import (
     DifyAPIWorkflowNodeExecutionRepository,
+    WorkflowNodeExecutionPayloads,
     WorkflowNodeExecutionSnapshot,
 )
 from repositories.sqlalchemy_api_workflow_node_execution_repository import (
@@ -161,11 +162,32 @@ class LogstoreAPIWorkflowNodeExecutionRepository(DifyAPIWorkflowNodeExecutionRep
             tenant_id=tenant_id, app_id=app_id, batch_size=batch_size
         )
 
+    @override
+    def load_full_inputs(self, execution: WorkflowNodeExecutionModel) -> Mapping[str, Any] | None:
+        return execution.inputs_dict
+
+    @override
+    def load_full_outputs(self, execution: WorkflowNodeExecutionModel) -> Mapping[str, Any] | None:
+        return execution.outputs_dict
+
     @override
     def load_full_process_data(self, execution: WorkflowNodeExecutionModel) -> Mapping[str, Any] | None:
         """Return complete Process Data already stored inline by LogStore."""
         return execution.process_data_dict
 
+    @override
+    def load_full_payloads(
+        self, executions: Sequence[WorkflowNodeExecutionModel]
+    ) -> Mapping[str, WorkflowNodeExecutionPayloads]:
+        return {
+            execution.id: WorkflowNodeExecutionPayloads(
+                inputs=execution.inputs_dict,
+                process_data=execution.process_data_dict,
+                outputs=execution.outputs_dict,
+            )
+            for execution in executions
+        }
+
     @override
     def get_node_last_execution(
         self,

api/extensions/logstore/repositories/logstore_workflow_node_execution_repository.py

@@ -12,11 +12,11 @@ from collections.abc import Sequence
 from datetime import datetime
 from typing import Any, override
 
+from pydantic import TypeAdapter
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
 
 from configs import dify_config
-from core.ops.utils import JSON_DICT_ADAPTER
 from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository
 from core.repositories.factory import OrderConfig, WorkflowNodeExecutionRepository
 from core.workflow.node_execution import WorkflowNodeExecution as DifyWorkflowNodeExecution
@@ -37,6 +37,8 @@ from repositories.workflow_run_control import apply_workflow_stop_to_node, get_s
 
 logger = logging.getLogger(__name__)
 
+_JSON_DICT_ADAPTER: TypeAdapter[dict[str, Any]] = TypeAdapter(dict[str, Any])
+
 
 def _dict_to_workflow_node_execution(data: dict[str, Any]) -> WorkflowNodeExecution:
     """
@@ -50,10 +52,10 @@ def _dict_to_workflow_node_execution(data: dict[str, Any]) -> WorkflowNodeExecut
     """
     logger.debug("_dict_to_workflow_node_execution: data keys=%s", list(data.keys())[:5])
     # Parse JSON fields
-    inputs = JSON_DICT_ADAPTER.validate_json(data.get("inputs") or "{}")
-    process_data = JSON_DICT_ADAPTER.validate_json(data.get("process_data") or "{}")
-    outputs = JSON_DICT_ADAPTER.validate_json(data.get("outputs") or "{}")
-    metadata = JSON_DICT_ADAPTER.validate_json(data.get("execution_metadata") or "{}")
+    inputs = _JSON_DICT_ADAPTER.validate_json(data.get("inputs") or "{}")
+    process_data = _JSON_DICT_ADAPTER.validate_json(data.get("process_data") or "{}")
+    outputs = _JSON_DICT_ADAPTER.validate_json(data.get("outputs") or "{}")
+    metadata = _JSON_DICT_ADAPTER.validate_json(data.get("execution_metadata") or "{}")
 
     # Convert metadata to domain enum keys
     domain_metadata = {}

api/extensions/tracing.py

@@ -0,0 +1,155 @@
+"""Execution-side access to the trace pipeline, bound to one Flask application.
+
+Legacy producers in the execution kernel import only these accessors; the
+composition root binds the services they reach. Every accessor fails open:
+tracing never breaks the operation that is being traced.
+"""
+
+import logging
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable
+
+from dify_trace_sdk.emitter import TraceScope
+from dify_trace_sdk.events import TraceCorrelation
+from flask import Flask, current_app, has_app_context
+
+from core.helper.trace_id_helper import ParentTraceContext
+
+if TYPE_CHECKING:
+    from dify_agent.layers.execution_context import ApplicationTracingContext
+
+    from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
+    from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
+    from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
+
+logger = logging.getLogger(__name__)
+
+_SCOPES_KEY = "trace_scopes"
+_HOOKS_KEY = "application_tracing_hooks"
+
+
+@runtime_checkable
+class ExecutionTraceScopes(Protocol):
+    """Trace scopes for legacy producers, which hold the app's stored tracing policy as its raw column value."""
+
+    def for_execution(
+        self,
+        *,
+        tenant_id: str,
+        app_id: str | None,
+        app_tracing: str | None,
+        user_id: str | None,
+        correlation: TraceCorrelation,
+    ) -> TraceScope | None: ...
+
+    def for_app(
+        self, *, tenant_id: str, app_id: str | None, user_id: str | None, correlation: TraceCorrelation
+    ) -> TraceScope | None: ...
+
+
+@runtime_checkable
+class ApplicationTracingHooks(Protocol):
+    """Agent trace collection lifecycle; it never replaces the trace pipeline."""
+
+    def prepare_agent_app(self, context: "AgentAppRuntimeBuildContext") -> "ApplicationTracingContext | None": ...
+
+    def prepare_workflow_agent(
+        self, context: "WorkflowAgentRuntimeBuildContext", conversation_id: str | None
+    ) -> "ApplicationTracingContext | None": ...
+
+    def freeze_workflow_run(
+        self,
+        entity: "AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity",
+        *,
+        workflow_id: str,
+        workflow_run_id: str,
+        conversation_id: object,
+        graph_data: object,
+    ) -> None: ...
+
+    def capture_agent_app_terminal(
+        self,
+        *,
+        tenant_id: str,
+        app_id: str,
+        message_id: str,
+        outcome: Literal["succeeded", "failed", "cancelled"],
+        output: str | None,
+    ) -> None: ...
+
+
+def bind_tracing(app: Flask, *, scopes: ExecutionTraceScopes, hooks: ApplicationTracingHooks | None) -> None:
+    """``hooks`` is ``None`` when this deployment does not collect Agent traces."""
+    app.extensions[_SCOPES_KEY] = scopes
+    app.extensions[_HOOKS_KEY] = hooks
+
+
+def _scopes() -> ExecutionTraceScopes | None:
+    if not has_app_context():
+        return None
+    scopes = current_app.extensions.get(_SCOPES_KEY)
+    return scopes if isinstance(scopes, ExecutionTraceScopes) else None
+
+
+def correlation_from_extras(extras: Mapping[str, Any]) -> TraceCorrelation:
+    """The caller-supplied correlation an execution carries in its ``extras``."""
+    parent = extras.get("parent_trace_context")
+    if isinstance(parent, Mapping):
+        try:
+            parent = ParentTraceContext.model_validate(parent)
+        except ValueError:
+            parent = None
+    external_trace_id, session_id = extras.get("external_trace_id"), extras.get("trace_session_id")
+    return TraceCorrelation(
+        external_trace_id=external_trace_id if isinstance(external_trace_id, str) and external_trace_id else None,
+        session_id=session_id if isinstance(session_id, str) and session_id else None,
+        parent_workflow_run_id=parent.parent_workflow_run_id if isinstance(parent, ParentTraceContext) else None,
+        parent_node_execution_id=parent.parent_node_execution_id if isinstance(parent, ParentTraceContext) else None,
+    )
+
+
+def create_trace_scope(
+    *, tenant_id: str, app_id: str | None, app_tracing: str | None, user_id: str | None, extras: Mapping[str, Any]
+) -> TraceScope | None:
+    """The scope for an execution whose app row is loaded; ``None`` when nothing consumes its events."""
+    scopes = _scopes()
+    if scopes is None:
+        return None
+    try:
+        return scopes.for_execution(
+            tenant_id=tenant_id,
+            app_id=app_id,
+            app_tracing=app_tracing,
+            user_id=user_id,
+            correlation=correlation_from_extras(extras),
+        )
+    except Exception:
+        logger.warning("Tracing scope unavailable: app_id=%s", app_id, exc_info=True)
+        return None
+
+
+def create_trace_scope_for_app(
+    *, tenant_id: str, app_id: str | None, user_id: str | None, extras: Mapping[str, Any]
+) -> TraceScope | None:
+    """The scope for a caller that does not hold the app row."""
+    scopes = _scopes()
+    if scopes is None:
+        return None
+    try:
+        return scopes.for_app(
+            tenant_id=tenant_id, app_id=app_id, user_id=user_id, correlation=correlation_from_extras(extras)
+        )
+    except Exception:
+        logger.warning("Tracing scope unavailable: app_id=%s", app_id, exc_info=True)
+        return None
+
+
+def application_tracing_hooks() -> ApplicationTracingHooks | None:
+    """The Agent collection hooks; ``None`` when Agent tracing is off or no application is bound here.
+
+    Agent runs started without hooks are reported as not collected instead of failing.
+    """
+    if not has_app_context():
+        return None
+    hooks = current_app.extensions.get(_HOOKS_KEY)
+    return hooks if isinstance(hooks, ApplicationTracingHooks) else None

api/libs/datetime_utils.py

@@ -1,5 +1,8 @@
 import abc
 import datetime
+from collections.abc import Generator
+from contextlib import contextmanager
+from dataclasses import dataclass
 from typing import Protocol
 
 import pytz  # type: ignore[import-untyped]
@@ -22,6 +25,23 @@ def utc_now() -> datetime.datetime:
     return _now_func(datetime.UTC)
 
 
+@dataclass
+class Timing:
+    started_at: datetime.datetime
+    ended_at: datetime.datetime
+
+
+@contextmanager
+def measure_time() -> Generator[Timing]:
+    """Time a block in aware UTC; ``ended_at`` is final once the block exits."""
+    started_at = utc_now()
+    timing = Timing(started_at=started_at, ended_at=started_at)
+    try:
+        yield timing
+    finally:
+        timing.ended_at = utc_now()
+
+
 def naive_utc_now() -> datetime.datetime:
     """Return a naive datetime object (without timezone information)
     representing current UTC time.

api/migrations/versions/2026_10_23_1200-53c10126d551_unique_trace_app_config_provider.py

@@ -0,0 +1,50 @@
+"""keep one tracing configuration per app and provider
+
+Revision ID: 53c10126d551
+Revises: 0cc3158b50b0
+Create Date: 2026-10-23 12:00:00.000000
+
+Configuration writes lock the app row, but rows written before that lock existed can
+duplicate an (app, provider) pair, and reads then pick one arbitrarily. The most
+recently updated row wins; the unique index keeps the pair unique from now on.
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "53c10126d551"
+down_revision = "0cc3158b50b0"
+branch_labels = None
+depends_on = None
+
+_TABLE = "trace_app_config"
+_INDEX = "trace_app_config_app_provider_uq"
+
+# MySQL cannot read the table a DELETE targets except through a derived table.
+_DELETE_SUPERSEDED = sa.text(
+    """
+    DELETE FROM trace_app_config
+    WHERE id IN (
+        SELECT id FROM (
+            SELECT older.id
+            FROM trace_app_config older
+            JOIN trace_app_config newer
+              ON newer.app_id = older.app_id
+             AND newer.tracing_provider = older.tracing_provider
+             AND (
+                newer.updated_at > older.updated_at
+                OR (newer.updated_at = older.updated_at AND newer.id > older.id)
+             )
+        ) superseded
+    )
+    """
+)
+
+
+def upgrade() -> None:
+    op.execute(_DELETE_SUPERSEDED)
+    op.create_index(_INDEX, _TABLE, ["app_id", "tracing_provider"], unique=True)
+
+
+def downgrade() -> None:
+    op.drop_index(_INDEX, table_name=_TABLE)

api/models/model.py

@@ -2640,6 +2640,7 @@ class TraceAppConfig(TypeBase):
     __table_args__ = (
         sa.PrimaryKeyConstraint("id", name="tracing_app_config_pkey"),
         sa.Index("trace_app_config_app_id_idx", "app_id"),
+        sa.Index("trace_app_config_app_provider_uq", "app_id", "tracing_provider", unique=True),
     )
 
     id: Mapped[str] = mapped_column(

api/openapi/markdown/console-openapi.md

@@ -1457,7 +1457,7 @@ Get available TTS voices for an Agent and language
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
 | 200 | Provider support declarations | **application/json**: [TraceProviderCapabilitiesResponse](#traceprovidercapabilitiesresponse)<br> |
-| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| 400 | Bad Request | **application/json**: [TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
 | default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
 
 ### [DELETE] /agent/{agent_id}/trace-config
@@ -3829,7 +3829,7 @@ Update app tracing configuration
 | Code | Description | Schema |
 | ---- | ----------- | ------ |
 | 200 | Provider support declarations | **application/json**: [TraceProviderCapabilitiesResponse](#traceprovidercapabilitiesresponse)<br> |
-| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| 400 | Bad Request | **application/json**: [TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
 | default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
 
 ### [DELETE] /apps/{app_id}/trace-config
@@ -35958,13 +35958,14 @@ Enum class for tool provider
 | Name | Type | Description | Required |
 | ---- | ---- | ----------- | -------- |
 | app_id | string |  | No |
+| configured_secrets | [ string ] | Secret fields that hold a value; submit them empty to keep that value | No |
 | created_at | string |  | No |
 | error | string |  | No |
 | has_not_configured | boolean |  | No |
 | id | string |  | No |
 | is_active | boolean |  | No |
 | result | string |  | No |
-| tracing_config | object |  | No |
+| tracing_config | object | Non-secret fields; secrets are write-only and never returned | No |
 | tracing_provider | string |  | No |
 | updated_at | string |  | No |
 

api/providers/trace/README.md

@@ -1,77 +1,132 @@
 # Trace providers
 
-This directory holds **optional workspace packages** that send Dify **ops tracing** data (workflows, messages, tools, moderation, etc.) to an external observability backend (Langfuse, LangSmith, OpenTelemetry-style exporters, and others).
+Each directory here is an optional uv workspace package that sends Dify traces (messages, workflow runs, tool calls, retrieval, moderation, and so on) to one external observability backend. `trace-sdk/` is the contract every provider implements; it is not a provider itself.
 
-Unlike VDB providers, trace plugins are **not** discovered via entry points. The API core imports your package **explicitly** from `core/ops/ops_trace_manager.py` after you register the provider id and mapping.
+## The contract (`trace-sdk`, imported as `dify_trace_sdk`)
 
-## Architecture
+A provider is a subclass of `dify_trace_sdk.provider.TraceProvider[ConfigT]`:
 
-| Layer | Location | Role |
-|--------|----------|------|
-| Contracts | `api/core/ops/base_trace_instance.py`, `api/core/ops/entities/trace_entity.py`, `api/core/ops/entities/config_entity.py` | `BaseTraceInstance`, `BaseTracingConfig`, and typed `*TraceInfo` payloads |
-| Registry | `api/core/ops/ops_trace_manager.py` | `TracingProviderEnum`, `OpsTraceProviderConfigMap` — maps provider **string** → config class, encrypted keys, and trace class |
-| Your package | `api/providers/trace/trace-<name>/` | Pydantic config + subclass of `BaseTraceInstance` |
+| Member | Purpose |
+|--------|---------|
+| `id` | The stable provider ID stored in app tracing settings, such as `"langfuse"`. |
+| `config_model` | A `ProviderConfig` subclass. Mark credentials with `secret()` and values that verification resolves with `derived()`. |
+| `capabilities` | A `ProviderCapabilities` value: whether the backend prices usage natively and links to traces and spans. |
+| `verify(config)` | Checks a new or edited configuration against the backend and returns it with derived fields filled. Raises `VerificationError(code)` with a stable code that contains no secret. |
+| `project_url(config)` | The console link for a configuration. It must not perform I/O. |
+| `send(config, trace)` | Delivers one trace unit and classifies every failure (see below). |
 
-At runtime, `OpsTraceManager` decrypts stored credentials, builds your config model, caches a trace instance, and calls `trace(trace_info)` with a concrete `BaseTraceInfo` subtype.
+What `send` receives (`dify_trace_sdk.model`):
 
-## What you implement
+- A `Trace` is a parent-first tree of completed `Span`s that uses OpenTelemetry ID formats.
+  - Each span carries timing, inputs and outputs, status, and typed LLM fields (`model_name`, `model_provider`, `usage`, `cost`, `time_to_first_token`).
+  - All values are strict JSON.
+- A message trace has a `message` root and an `llm` child.
+- A workflow run has a `workflow` span with node spans and loop or iteration wrapper spans. For chatflows these hang under the `message` root.
+- Auxiliary operations join their message's trace as separate units. `trace.parent` names the message root span, and the first span of the unit points at it.
+- A workflow nested in another one is its own trace. Its root carries `parent_workflow_run_id` and `parent_node_execution_id`.
+- A unit can be sent again after a retryable failure. Derive vendor IDs deterministically from the trace and span IDs (`dify_trace_sdk.ids`), so a resend updates the same records instead of duplicating them.
 
-### 1. Config model (`BaseTracingConfig`)
+Failure classes (`dify_trace_sdk.errors`):
 
-Subclass `BaseTracingConfig` from `core.ops.entities.config_entity`. Use Pydantic validators; reuse helpers from `core.ops.utils` (for example `validate_url`, `validate_url_with_path`, `validate_project_name`) where appropriate.
+| Raise | When | Pipeline behaviour |
+|-------|------|--------------------|
+| `ExportRetryableError` | Nothing was accepted and the cause is transient (unreachable, 408/425/429/5xx). | Sends the same unit again later, up to `OPS_TRACE_EXPORT_MAX_ATTEMPTS` attempts in total. |
+| `ExportRejectedError` | The backend refused the unit, or the configuration cannot work. | Drops the unit and logs a warning. |
+| `ExportOutcomeUnknownError` | Part or all of the unit may have been accepted. | Drops the unit; sending again could duplicate it. |
 
-Fields fall into two groups used by the manager:
+Error messages are logged and must never contain credentials, request bodies, or vendor response bodies. Raise them `from None`.
 
-- **`secret_keys`** — names of fields that are **encrypted at rest** (API keys, tokens, passwords).
-- **`other_keys`** — non-secret connection settings (hosts, project names, endpoints).
+## Network access
 
-List these key names in your `OpsTraceProviderConfigMap` entry so encrypt/decrypt and merge logic stay correct.
+A provider never opens sockets, starts threads, or reads environment variables or files. All traffic goes through `self.egress`:
 
-### 2. Trace instance (`BaseTraceInstance`)
+- `egress.request(...)` sends one HTTP request through the SSRF proxy, with no retries or redirects, a bounded response, and no ambient cookies.
+- `egress.grpc_channel(endpoint)` opens a channel pinned to a vetted address.
+- The helpers in `dify_trace_sdk.exporting` and `dify_trace_sdk.otlp` turn egress failures and HTTP or gRPC statuses into the failure classes above.
 
-Subclass `BaseTraceInstance` and implement:
+If you must use a vendor SDK, route its HTTP through `EgressHttpxTransport` or `EgressRequestsSession` (`dify_trace_sdk.transports`) and disable its batching, background threads, and retries. Importing the SDK must not change process-wide state; some SDKs install a Sentry client or patch other libraries on import.
 
-```python
-def trace(self, trace_info: BaseTraceInfo) -> None: ...
-```
+## Configuration and secrets
 
-Dispatch on the concrete type with `isinstance` (see `trace_langfuse` or `trace_langsmith` for full patterns). Payload types are defined in `core/ops/entities/trace_entity.py`, including:
+Stored configurations keep the provider's fields at the top level, with `secret()` fields encrypted per workspace. Secrets are write-only:
 
-- `WorkflowTraceInfo`, `WorkflowNodeTraceInfo`, `DraftNodeExecutionTrace`
-- `MessageTraceInfo`, `ToolTraceInfo`, `ModerationTraceInfo`, `SuggestedQuestionTraceInfo`
-- `DatasetRetrievalTraceInfo`, `GenerateNameTraceInfo`, `PromptGenerationTraceInfo`
+- The console never receives them back; it gets the names of the secrets that are set.
+- When an edit leaves a secret empty or omits it, the stored value is kept.
+- Clients cannot set `derived()` fields.
 
-You may ignore categories your backend does not support; existing providers often no-op unhandled types.
+`ProviderConfig` hides inputs from validation errors; keep validator messages fixed texts too.
 
-Optional: use `get_service_account_with_tenant(app_id)` from the base class when you need tenant-scoped account context.
+## Adding a provider
 
-### 3. Register in the API core
+1. Create `api/providers/trace/trace-<name>/` with:
+   - a `pyproject.toml` for the project `dify-trace-<name>`, depending on `dify-trace-sdk` (plus any vendor SDK that is safe to import);
+   - `src/dify_trace_<name>/{config.py,provider.py,py.typed}`;
+   - tests that mirror the sources, e.g. `src/dify_trace_<name>/provider.py` maps to `tests/unit_tests/src/dify_trace_<name>/test_provider.py`.
 
-Upstream changes are required so Dify knows your provider exists:
+   List `py.typed` under `[tool.setuptools.package-data]`.
+2. Register the provider class as an entry point:
 
-1. **`TracingProviderEnum`** (`api/core/ops/entities/config_entity.py`) — add a new member whose **value** is the stable string stored in app tracing config (e.g. `"mybackend"`).
-2. **`OpsTraceProviderConfigMap.__getitem__`** (`api/core/ops/ops_trace_manager.py`) — add a `match` case for that enum member returning:
-   - `config_class`: your Pydantic config type
-   - `secret_keys` / `other_keys`: lists of field names as above
-   - `trace_instance`: your `BaseTraceInstance` subclass  
-   Lazy-import your package inside the case so missing optional installs raise a clear `ImportError`.
+   ```toml
+   [project.entry-points."dify.trace_providers"]
+   <id> = "dify_trace_<name>.provider:<Name>Provider"
+   ```
 
-If the `match` case is missing, the provider string will not resolve and tracing will be disabled for that app.
+3. Add the ID to `SUPPORTED_PROVIDERS` in `api/services/tracing/providers.py` and the provider's form fields to the web console's tracing field registry.
+4. In `api/pyproject.toml`, add `dify-trace-<name> = { workspace = true }` under `[tool.uv.sources]` and add the package to the `trace-<name>` and `trace-all` dependency groups. Then run `uv lock` and `uv sync` from `api/`.
+5. Test with `dify_trace_sdk.testing.RecordingEgress` and the sample traces there. Tests must not reach the network.
 
-## Package layout
+`trace-langfuse/` is the smallest reference implementation; `trace-arize-phoenix/` and `trace-tencent/` show OTLP over HTTP and gRPC.
 
-Each provider is a normal uv workspace member, for example:
+## How traces reach a provider
 
-- `api/providers/trace/trace-<name>/pyproject.toml` — project name `dify-trace-<name>`, dependencies on vendor SDKs
-- `api/providers/trace/trace-<name>/src/dify_trace_<name>/` — `config.py`, `<name>_trace.py`, optional `entities/`, and an empty **`py.typed`** file (PEP 561) so the API type checker treats the package as typed; list `py.typed` under `[tool.setuptools.package-data]` for that import name in `pyproject.toml`.
+1. Producers emit typed events (`dify_trace_sdk.events`) through a `TraceScope` after their records are committed:
+   - generators, task pipelines and the workflow persistence layer;
+   - moderation, tools, retrieval, conversation naming and suggested questions.
 
-Reference implementations: `trace-langfuse/`, `trace-langsmith/`, `trace-opik/`.
+   A scope exists only when the app has tracing enabled or enterprise telemetry is on.
+   The producer snapshots the event and hands it to a background publisher, so storage and
+   broker I/O do not delay the execution. Each process buffers at most 256 pending events;
+   a full buffer drops new events with a warning. Graceful shutdown drains accepted events,
+   but events not yet published can be lost if the process crashes.
+2. `tasks.trace_delivery_task` hands each event on `OPS_TRACE_QUEUE` (default `ops_trace_v2`) to `services.tracing.delivery.TraceDeliveryService`:
+   - `repositories.trace_record_repository` reloads the records through the event's workspace.
+   - Enterprise telemetry receives every record.
+   - The Agent trace collector (`services.tracing.service`) takes the executions it manages.
+   - `services.tracing.canonical` builds the trace for the app's current provider configuration, and the provider's `send` runs.
+3. A retryable failure stores only the built trace and schedules that send again. Payloads that an interrupted delivery leaves in storage are removed daily by `sweep_trace_payloads`.
 
-## Wiring into the `api` workspace
+The tracing services depend only on the ports they declare: records, destinations, the provider catalog, secret encryption, the retry queue and the Agent trace ledger. `extensions/application_services/tracing.py` is the one place that chooses their adapters (`services/tracing/adapters.py`, `services/tracing/egress.py`, `services/tracing/registry.py` and the tracing repositories) and reads tracing configuration.
 
-In `api/pyproject.toml`:
+## Rolling upgrade from the previous pipeline
 
-1. **`[tool.uv.sources]`** — `dify-trace-<name> = { workspace = true }`
-2. **`[dependency-groups]`** — add `trace-<name> = ["dify-trace-<name>"]` and include `dify-trace-<name>` in `trace-all` if it should ship with the default bundle
+The new task names and payloads are incompatible with old workers. Use separate queues while
+both versions run. `OPS_TRACE_QUEUE` defaults to `ops_trace_v2` for event delivery, export retries,
+Agent trace exports/reconciliation, payload sweeping, and the sandbox metering task that previously
+shared `ops_trace`. Celery routing is configured in `extensions/ext_celery.py`.
+This isolates tracing tasks; other queues and database migrations still need their normal
+compatibility checks when running both versions.
 
-After changing metadata, run **`uv sync`** from `api/`.
+1. Apply the database migrations and start new workers with `OPS_TRACE_QUEUE=ops_trace_v2`.
+   The Docker and development worker launchers include this queue by default and exclude
+   `ops_trace`. Explicit `CELERY_QUEUES`, `CELERY_WORKER_QUEUES`, or `--queues` lists must select
+   `ops_trace_v2` for the new pool. Keep the old tracing worker pool subscribed only to `ops_trace`.
+2. Roll out the new API and all other producers with the same setting. Replace the old Beat
+   scheduler with the new one; do not run duplicate schedules. Allow old in-flight requests,
+   workflow jobs, and their buffered trace producers to finish before stopping them.
+3. Let the old workers finish exporting their tasks. Check broker messages **and** worker
+   active, reserved/prefetched, scheduled/ETA, and retry deliveries. An empty broker queue alone
+   does not prove it has drained. Then stop the old workers gracefully and confirm no old
+   producers or workers remain. The new sweeper never deletes legacy `ops_trace/` payloads.
+4. Keeping `ops_trace_v2` is supported. To restore the old queue name, first have **new-version**
+   workers consume both `ops_trace_v2,ops_trace`. Set `OPS_TRACE_QUEUE=ops_trace` on every new API,
+   worker, and Beat process and roll them out. Keep consumers for both queues until the new
+   queue, active tasks, reservations, ETA tasks, and retries drain; a redelivery can retain its
+   original queue. Then remove `ops_trace_v2` from worker subscriptions.
+
+Do not use the new version's retired `process_trace_tasks` handler to drain old messages: it can
+only discard them, not export the legacy format. Do not purge queues or legacy payloads during
+the transition. Returning to old binaries after publishing new-format tasks also requires
+keeping new-version consumers available until those tasks finish.
+
+Saved provider configuration and credentials need no changes. Queue migration is deployment
+configuration, shared by the API, workers, and Beat; it does not change app tracing settings.

api/providers/trace/trace-aliyun/pyproject.toml

@@ -2,13 +2,12 @@
 name = "dify-trace-aliyun"
 version = "0.0.1"
 dependencies = [
-    # versions inherited from parent
-    "opentelemetry-api",
-    "opentelemetry-exporter-otlp-proto-grpc",
-    "opentelemetry-sdk",
-    "opentelemetry-semantic-conventions",
+    "dify-trace-sdk",
 ]
 description = "Dify ops tracing provider (Aliyun)."
 
+[project.entry-points."dify.trace_providers"]
+aliyun = "dify_trace_aliyun.provider:AliyunProvider"
+
 [tool.setuptools.packages.find]
 where = ["src"]

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/agent_trace.py

@@ -1,27 +0,0 @@
-"""Aliyun LLM tracing export with synchronous OTLP/HTTP acknowledgment."""
-
-from collections.abc import Mapping
-from typing import Any
-
-from core.ops.unified_trace.entities import CanonicalTrace
-from dify_trace_aliyun.configuration import AliyunConfigurationError, ingestion_url
-from services.tracing.genai_attributes import attributes
-from services.tracing.otlp_export import build_request, export_http
-from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult
-
-
-def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
-    try:
-        url = ingestion_url(config)
-    except AliyunConfigurationError:
-        raise TraceExportRejectedError("Aliyun tracing configuration is invalid") from None
-    payload = build_request(
-        trace,
-        {
-            "service.name": str(config.get("app_name") or "dify_app"),
-            "acs.arms.service.feature": "genai_app",
-        },
-        lambda span: attributes(span, trace, provider),
-    )
-    export_http(provider, url, {}, payload)
-    return TraceExportResult(trace_id=trace.trace_id)

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py

@@ -1,850 +0,0 @@
-import logging
-from collections.abc import Mapping, Sequence
-from typing import Any, override
-
-from opentelemetry.trace import SpanKind
-from sqlalchemy.orm import sessionmaker
-
-from core.ops.base_trace_instance import BaseTraceInstance
-from core.ops.entities.trace_entity import (
-    BaseTraceInfo,
-    DatasetRetrievalTraceInfo,
-    GenerateNameTraceInfo,
-    MessageTraceInfo,
-    ModerationTraceInfo,
-    SuggestedQuestionTraceInfo,
-    ToolTraceInfo,
-    WorkflowTraceInfo,
-)
-from core.repositories import DifyCoreRepositoryFactory
-from dify_trace_aliyun.config import AliyunConfig
-from dify_trace_aliyun.configuration import console_url, ingestion_url, verify_configuration
-from dify_trace_aliyun.data_exporter.traceclient import (
-    TraceClient,
-    convert_datetime_to_nanoseconds,
-    convert_to_span_id,
-    convert_to_trace_id,
-    generate_span_id,
-)
-from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData, TraceMetadata
-from dify_trace_aliyun.entities.semconv import (
-    DIFY_APP_ID,
-    DIFY_NODE_TYPE,
-    GEN_AI_AGENT_NAME,
-    GEN_AI_COMPLETION,
-    GEN_AI_INPUT_MESSAGE,
-    GEN_AI_OPERATION_NAME,
-    GEN_AI_OUTPUT_MESSAGE,
-    GEN_AI_PROMPT,
-    GEN_AI_PROVIDER_NAME,
-    GEN_AI_REACT_FINISH_REASON,
-    GEN_AI_REACT_ROUND,
-    GEN_AI_REQUEST_MODEL,
-    GEN_AI_RESPONSE_FINISH_REASON,
-    GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN,
-    GEN_AI_USAGE_INPUT_TOKENS,
-    GEN_AI_USAGE_OUTPUT_TOKENS,
-    GEN_AI_USAGE_TOTAL_TOKENS,
-    OPERATION_NAME_CHAT,
-    OPERATION_NAME_INVOKE_AGENT,
-    OPERATION_NAME_REACT,
-    RETRIEVAL_DOCUMENT,
-    RETRIEVAL_QUERY,
-    GenAISpanKind,
-)
-from dify_trace_aliyun.utils import (
-    AgentLogEntry,
-    convert_seconds_to_nanoseconds,
-    create_common_span_attributes,
-    create_gen_ai_tool_attributes,
-    create_links_from_trace_id,
-    create_status_from_agent_log_entry,
-    create_status_from_error,
-    extract_model_name_from_thought_label,
-    extract_react_round_number,
-    extract_retrieval_documents,
-    extract_tool_description,
-    extract_tool_name_from_call_label,
-    format_input_messages,
-    format_output_messages,
-    format_retrieval_documents,
-    get_user_id_from_message_data,
-    get_workflow_node_status,
-    is_llm_thought_entry,
-    is_tool_call_entry,
-    map_gen_ai_tool_type,
-    parse_agent_log_entries,
-    serialize_json_data,
-)
-from extensions.ext_database import db
-from graphon.entities import WorkflowNodeExecution
-from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKey
-from models import WorkflowNodeExecutionTriggeredFrom
-
-logger = logging.getLogger(__name__)
-
-
-class AliyunDataTrace(BaseTraceInstance):
-    def __init__(
-        self,
-        aliyun_config: AliyunConfig,
-    ):
-        super().__init__(aliyun_config)
-        self._config = aliyun_config.model_copy(deep=True)
-        self.trace_client = TraceClient(
-            service_name=self._config.app_name, endpoint=ingestion_url(self._config.model_dump())
-        )
-
-    @override
-    def trace(self, trace_info: BaseTraceInfo):
-        with self.trace_client.batch():
-            match trace_info:
-                case WorkflowTraceInfo():
-                    self.workflow_trace(trace_info)
-                case MessageTraceInfo():
-                    self.message_trace(trace_info)
-                case ModerationTraceInfo():
-                    pass
-                case SuggestedQuestionTraceInfo():
-                    self.suggested_question_trace(trace_info)
-                case DatasetRetrievalTraceInfo():
-                    self.dataset_retrieval_trace(trace_info)
-                case ToolTraceInfo():
-                    self.tool_trace(trace_info)
-                case GenerateNameTraceInfo():
-                    pass
-                case _:
-                    pass
-
-    def api_check(self) -> bool:
-        verify_configuration(self._config)
-        return True
-
-    def get_project_url(self):
-        return console_url()
-
-    def _extract_app_id(self, trace_info: BaseTraceInfo) -> str:
-        """Extract app_id from trace_info, trying metadata first then message_data."""
-        app_id = trace_info.metadata.get("app_id")
-        if app_id:
-            return str(app_id)
-        message_data = getattr(trace_info, "message_data", None)
-        if message_data is not None:
-            return str(getattr(message_data, "app_id", ""))
-        return ""
-
-    def workflow_trace(self, trace_info: WorkflowTraceInfo):
-        trace_metadata = TraceMetadata(
-            trace_id=convert_to_trace_id(trace_info.workflow_run_id),
-            workflow_span_id=convert_to_span_id(trace_info.workflow_run_id, "workflow"),
-            session_id=trace_info.metadata.get("conversation_id") or "",
-            user_id=str(trace_info.metadata.get("user_id") or ""),
-            links=create_links_from_trace_id(trace_info.trace_id),
-        )
-
-        self.add_workflow_span(trace_info, trace_metadata)
-
-        workflow_node_executions = self.get_workflow_node_executions(trace_info)
-        for node_execution in workflow_node_executions:
-            node_span = self.build_workflow_node_span(node_execution, trace_info, trace_metadata)
-            self.trace_client.add_span(node_span)
-            if node_span is not None and node_execution.node_type == BuiltinNodeTypes.AGENT:
-                for react_span in self.build_agent_react_spans(node_execution, trace_metadata):
-                    self.trace_client.add_span(react_span)
-
-    def message_trace(self, trace_info: MessageTraceInfo):
-        message_data = trace_info.message_data
-        if message_data is None:
-            return
-
-        message_id = trace_info.message_id
-        user_id = get_user_id_from_message_data(message_data)
-        status = create_status_from_error(trace_info.error)
-
-        trace_metadata = TraceMetadata(
-            trace_id=convert_to_trace_id(message_id),
-            workflow_span_id=0,
-            session_id=trace_info.metadata.get("conversation_id") or "",
-            user_id=user_id,
-            links=create_links_from_trace_id(trace_info.trace_id),
-        )
-
-        inputs_json = serialize_json_data(trace_info.inputs)
-        outputs_str = str(trace_info.outputs)
-
-        message_span_id = convert_to_span_id(message_id, "message")
-        message_span = SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=None,
-            span_id=message_span_id,
-            name="message",
-            start_time=convert_datetime_to_nanoseconds(trace_info.start_time),
-            end_time=convert_datetime_to_nanoseconds(trace_info.end_time),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.CHAIN,
-                    inputs=inputs_json,
-                    outputs=outputs_str,
-                ),
-                DIFY_APP_ID: self._extract_app_id(trace_info),
-            },
-            status=status,
-            links=trace_metadata.links,
-            span_kind=SpanKind.SERVER,
-        )
-        self.trace_client.add_span(message_span)
-
-        llm_attributes: dict[str, Any] = {
-            **create_common_span_attributes(
-                session_id=trace_metadata.session_id,
-                user_id=trace_metadata.user_id,
-                span_kind=GenAISpanKind.LLM,
-                inputs=inputs_json,
-                outputs=outputs_str,
-            ),
-            GEN_AI_OPERATION_NAME: OPERATION_NAME_CHAT,
-            GEN_AI_REQUEST_MODEL: trace_info.metadata.get("ls_model_name") or "",
-            GEN_AI_PROVIDER_NAME: trace_info.metadata.get("ls_provider") or "",
-            GEN_AI_USAGE_INPUT_TOKENS: str(trace_info.message_tokens),
-            GEN_AI_USAGE_OUTPUT_TOKENS: str(trace_info.answer_tokens),
-            GEN_AI_USAGE_TOTAL_TOKENS: str(trace_info.total_tokens),
-            GEN_AI_PROMPT: inputs_json,
-            GEN_AI_COMPLETION: outputs_str,
-        }
-        if trace_info.gen_ai_server_time_to_first_token is not None:
-            llm_attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] = convert_seconds_to_nanoseconds(
-                trace_info.gen_ai_server_time_to_first_token
-            )
-
-        llm_span = SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=message_span_id,
-            span_id=convert_to_span_id(message_id, "llm"),
-            name="llm",
-            start_time=convert_datetime_to_nanoseconds(trace_info.start_time),
-            end_time=convert_datetime_to_nanoseconds(trace_info.end_time),
-            attributes=llm_attributes,
-            status=status,
-            links=trace_metadata.links,
-        )
-        self.trace_client.add_span(llm_span)
-
-    def dataset_retrieval_trace(self, trace_info: DatasetRetrievalTraceInfo):
-        if trace_info.message_data is None:
-            return
-
-        message_id = trace_info.message_id
-
-        trace_metadata = TraceMetadata(
-            trace_id=convert_to_trace_id(message_id),
-            workflow_span_id=0,
-            session_id=trace_info.metadata.get("conversation_id") or "",
-            user_id=str(trace_info.metadata.get("user_id") or ""),
-            links=create_links_from_trace_id(trace_info.trace_id),
-        )
-
-        documents_data = extract_retrieval_documents(trace_info.documents)
-        documents_json = serialize_json_data(documents_data)
-        inputs_str = str(trace_info.inputs)
-
-        dataset_retrieval_span = SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=convert_to_span_id(message_id, "message"),
-            span_id=generate_span_id(),
-            name="dataset_retrieval",
-            start_time=convert_datetime_to_nanoseconds(trace_info.start_time),
-            end_time=convert_datetime_to_nanoseconds(trace_info.end_time),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.RETRIEVER,
-                    inputs=inputs_str,
-                    outputs=documents_json,
-                ),
-                RETRIEVAL_QUERY: inputs_str,
-                RETRIEVAL_DOCUMENT: documents_json,
-            },
-            links=trace_metadata.links,
-        )
-        self.trace_client.add_span(dataset_retrieval_span)
-
-    def tool_trace(self, trace_info: ToolTraceInfo):
-        if trace_info.message_data is None:
-            return
-
-        message_id = trace_info.message_id
-        status = create_status_from_error(trace_info.error)
-
-        trace_metadata = TraceMetadata(
-            trace_id=convert_to_trace_id(message_id),
-            workflow_span_id=0,
-            session_id=trace_info.metadata.get("conversation_id") or "",
-            user_id=str(trace_info.metadata.get("user_id") or ""),
-            links=create_links_from_trace_id(trace_info.trace_id),
-        )
-
-        tool_config = trace_info.tool_config if isinstance(trace_info.tool_config, Mapping) else {}
-        tool_inputs_json = serialize_json_data(trace_info.tool_inputs)
-        tool_result = str(trace_info.tool_outputs)
-        inputs_json = serialize_json_data(trace_info.inputs)
-        provider_type = tool_config.get("tool_provider_type") or tool_config.get("provider_type")
-
-        tool_span = SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=convert_to_span_id(message_id, "message"),
-            span_id=generate_span_id(),
-            name=trace_info.tool_name,
-            start_time=convert_datetime_to_nanoseconds(trace_info.start_time),
-            end_time=convert_datetime_to_nanoseconds(trace_info.end_time),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.TOOL,
-                    inputs=inputs_json,
-                    outputs=tool_result,
-                ),
-                **create_gen_ai_tool_attributes(
-                    tool_name=trace_info.tool_name,
-                    tool_type=map_gen_ai_tool_type(str(provider_type) if provider_type else None),
-                    tool_description=extract_tool_description(tool_config),
-                    tool_call_id=str(trace_info.metadata.get("node_execution_id") or ""),
-                    tool_call_arguments=tool_inputs_json,
-                    tool_call_result=tool_result,
-                ),
-            },
-            status=status,
-            links=trace_metadata.links,
-        )
-        self.trace_client.add_span(tool_span)
-
-    def get_workflow_node_executions(self, trace_info: WorkflowTraceInfo) -> Sequence[WorkflowNodeExecution]:
-        app_id = trace_info.metadata.get("app_id")
-        if not app_id:
-            raise ValueError("No app_id found in trace_info metadata")
-
-        service_account = self.get_service_account_with_tenant(app_id)
-
-        session_factory = sessionmaker(bind=db.engine)
-        workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
-            session_factory=session_factory,
-            tenant_id=trace_info.tenant_id,
-            user=service_account,
-            app_id=app_id,
-            triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
-        )
-
-        return workflow_node_execution_repository.get_by_workflow_execution(
-            workflow_execution_id=trace_info.workflow_run_id
-        )
-
-    def build_workflow_node_span(
-        self, node_execution: WorkflowNodeExecution, trace_info: WorkflowTraceInfo, trace_metadata: TraceMetadata
-    ):
-        try:
-            if node_execution.node_type == BuiltinNodeTypes.LLM:
-                node_span = self.build_workflow_llm_span(trace_info, node_execution, trace_metadata)
-            elif node_execution.node_type == BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL:
-                node_span = self.build_workflow_retrieval_span(trace_info, node_execution, trace_metadata)
-            elif node_execution.node_type == BuiltinNodeTypes.TOOL:
-                node_span = self.build_workflow_tool_span(trace_info, node_execution, trace_metadata)
-            elif node_execution.node_type == BuiltinNodeTypes.AGENT:
-                node_span = self.build_workflow_agent_span(trace_info, node_execution, trace_metadata)
-            else:
-                node_span = self.build_workflow_task_span(trace_info, node_execution, trace_metadata)
-            return node_span
-        except Exception as e:
-            logger.warning("Error occurred in build_workflow_node_span: %s", e, exc_info=True)
-            return None
-
-    def build_workflow_task_span(
-        self, trace_info: WorkflowTraceInfo, node_execution: WorkflowNodeExecution, trace_metadata: TraceMetadata
-    ) -> SpanData:
-        inputs = node_execution.inputs
-        if node_execution.node_type == BuiltinNodeTypes.HTTP_REQUEST and isinstance(
-            node_execution.process_data, Mapping
-        ):
-            request = node_execution.process_data.get("request")
-            if isinstance(request, str) and request.strip():
-                # Reuse the node's masked request log; do not reconstruct the outbound request.
-                inputs = {"request": request}
-        inputs_json = serialize_json_data(inputs)
-        outputs_json = serialize_json_data(node_execution.outputs)
-        return SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=trace_metadata.workflow_span_id,
-            span_id=convert_to_span_id(node_execution.id, "node"),
-            name=node_execution.title,
-            start_time=convert_datetime_to_nanoseconds(node_execution.created_at),
-            end_time=convert_datetime_to_nanoseconds(node_execution.finished_at),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.TASK,
-                    inputs=inputs_json,
-                    outputs=outputs_json,
-                ),
-                DIFY_NODE_TYPE: node_execution.node_type,
-            },
-            status=get_workflow_node_status(node_execution),
-            links=trace_metadata.links,
-        )
-
-    def build_workflow_tool_span(
-        self, trace_info: WorkflowTraceInfo, node_execution: WorkflowNodeExecution, trace_metadata: TraceMetadata
-    ) -> SpanData:
-        tool_info: Mapping[str, Any] = {}
-        if isinstance(node_execution.metadata, Mapping):
-            raw_tool_info = node_execution.metadata.get(WorkflowNodeExecutionMetadataKey.TOOL_INFO, {})
-            if isinstance(raw_tool_info, Mapping):
-                tool_info = raw_tool_info
-
-        inputs_json = serialize_json_data(node_execution.inputs or {})
-        outputs_json = serialize_json_data(node_execution.outputs)
-        provider_type = tool_info.get("provider_type")
-
-        return SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=trace_metadata.workflow_span_id,
-            span_id=convert_to_span_id(node_execution.id, "node"),
-            name=node_execution.title,
-            start_time=convert_datetime_to_nanoseconds(node_execution.created_at),
-            end_time=convert_datetime_to_nanoseconds(node_execution.finished_at),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.TOOL,
-                    inputs=inputs_json,
-                    outputs=outputs_json,
-                ),
-                **create_gen_ai_tool_attributes(
-                    tool_name=node_execution.title,
-                    tool_type=map_gen_ai_tool_type(str(provider_type) if provider_type else None),
-                    tool_description=extract_tool_description(tool_info),
-                    tool_call_id=str(node_execution.id or ""),
-                    tool_call_arguments=inputs_json,
-                    tool_call_result=outputs_json,
-                ),
-            },
-            status=get_workflow_node_status(node_execution),
-            links=trace_metadata.links,
-        )
-
-    def build_workflow_retrieval_span(
-        self, trace_info: WorkflowTraceInfo, node_execution: WorkflowNodeExecution, trace_metadata: TraceMetadata
-    ) -> SpanData:
-        input_value = str(node_execution.inputs.get("query", "")) if node_execution.inputs else ""
-        output_value = serialize_json_data(node_execution.outputs.get("result", [])) if node_execution.outputs else ""
-
-        retrieval_documents = node_execution.outputs.get("result", []) if node_execution.outputs else []
-        semantic_retrieval_documents = format_retrieval_documents(retrieval_documents)
-        semantic_retrieval_documents_json = serialize_json_data(semantic_retrieval_documents)
-
-        return SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=trace_metadata.workflow_span_id,
-            span_id=convert_to_span_id(node_execution.id, "node"),
-            name=node_execution.title,
-            start_time=convert_datetime_to_nanoseconds(node_execution.created_at),
-            end_time=convert_datetime_to_nanoseconds(node_execution.finished_at),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.RETRIEVER,
-                    inputs=input_value,
-                    outputs=output_value,
-                ),
-                RETRIEVAL_QUERY: input_value,
-                RETRIEVAL_DOCUMENT: semantic_retrieval_documents_json,
-            },
-            status=get_workflow_node_status(node_execution),
-            links=trace_metadata.links,
-        )
-
-    def build_workflow_llm_span(
-        self, trace_info: WorkflowTraceInfo, node_execution: WorkflowNodeExecution, trace_metadata: TraceMetadata
-    ) -> SpanData:
-        process_data = node_execution.process_data if isinstance(node_execution.process_data, Mapping) else {}
-        inputs = node_execution.inputs if isinstance(node_execution.inputs, Mapping) else {}
-        outputs = node_execution.outputs if isinstance(node_execution.outputs, Mapping) else {}
-        usage_data = process_data.get("usage", {}) if "usage" in process_data else outputs.get("usage", {})
-        if not isinstance(usage_data, Mapping):
-            usage_data = {}
-
-        # On invoke failure graphon leaves process_data empty, but prep already wrote
-        # model identity (and template variables / context) into node inputs.
-        prompts = process_data.get("prompts") or []
-        if prompts:
-            prompts_json = serialize_json_data(prompts)
-        elif inputs:
-            prompts_json = serialize_json_data(inputs)
-        else:
-            prompts_json = serialize_json_data([])
-
-        text_output = str(outputs.get("text") or "")
-        if not text_output:
-            text_output = str(outputs.get("error_message") or node_execution.error or "")
-
-        finish_reason = outputs.get("finish_reason") or outputs.get("error_type") or ""
-        model_name = process_data.get("model_name") or inputs.get("model_name") or ""
-        model_provider = process_data.get("model_provider") or inputs.get("model_provider") or ""
-
-        gen_ai_input_message = format_input_messages(process_data)
-        gen_ai_output_message = format_output_messages(outputs)
-
-        attributes: dict[str, Any] = {
-            **create_common_span_attributes(
-                session_id=trace_metadata.session_id,
-                user_id=trace_metadata.user_id,
-                span_kind=GenAISpanKind.LLM,
-                inputs=prompts_json,
-                outputs=text_output,
-            ),
-            GEN_AI_OPERATION_NAME: OPERATION_NAME_CHAT,
-            GEN_AI_REQUEST_MODEL: str(model_name),
-            GEN_AI_PROVIDER_NAME: str(model_provider),
-            GEN_AI_USAGE_INPUT_TOKENS: str(usage_data.get("prompt_tokens", 0)),
-            GEN_AI_USAGE_OUTPUT_TOKENS: str(usage_data.get("completion_tokens", 0)),
-            GEN_AI_USAGE_TOTAL_TOKENS: str(usage_data.get("total_tokens", 0)),
-            GEN_AI_PROMPT: prompts_json,
-            GEN_AI_COMPLETION: text_output,
-            GEN_AI_RESPONSE_FINISH_REASON: str(finish_reason),
-            GEN_AI_INPUT_MESSAGE: gen_ai_input_message,
-            GEN_AI_OUTPUT_MESSAGE: gen_ai_output_message,
-        }
-        time_to_first_token = usage_data.get("time_to_first_token")
-        if isinstance(time_to_first_token, (int, float)):
-            attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] = convert_seconds_to_nanoseconds(float(time_to_first_token))
-
-        return SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=trace_metadata.workflow_span_id,
-            span_id=convert_to_span_id(node_execution.id, "node"),
-            name=node_execution.title,
-            start_time=convert_datetime_to_nanoseconds(node_execution.created_at),
-            end_time=convert_datetime_to_nanoseconds(node_execution.finished_at),
-            attributes=attributes,
-            status=get_workflow_node_status(node_execution),
-            links=trace_metadata.links,
-        )
-
-    def build_workflow_agent_span(
-        self, trace_info: WorkflowTraceInfo, node_execution: WorkflowNodeExecution, trace_metadata: TraceMetadata
-    ) -> SpanData:
-        """Build an AGENT-kind span for an agent-strategy node (instead of a generic TASK span)."""
-        inputs_json = serialize_json_data(node_execution.inputs)
-        outputs = node_execution.outputs if isinstance(node_execution.outputs, Mapping) else {}
-        usage_data = outputs.get("usage", {})
-        if not isinstance(usage_data, Mapping):
-            usage_data = {}
-        text_output = str(outputs.get("text", ""))
-
-        attributes: dict[str, Any] = {
-            **create_common_span_attributes(
-                session_id=trace_metadata.session_id,
-                user_id=trace_metadata.user_id,
-                span_kind=GenAISpanKind.AGENT,
-                inputs=inputs_json,
-                outputs=text_output,
-            ),
-            GEN_AI_OPERATION_NAME: OPERATION_NAME_INVOKE_AGENT,
-            GEN_AI_AGENT_NAME: node_execution.title,
-            GEN_AI_USAGE_INPUT_TOKENS: str(usage_data.get("prompt_tokens", 0)),
-            GEN_AI_USAGE_OUTPUT_TOKENS: str(usage_data.get("completion_tokens", 0)),
-            GEN_AI_USAGE_TOTAL_TOKENS: str(usage_data.get("total_tokens", 0)),
-        }
-        time_to_first_token = usage_data.get("time_to_first_token")
-        if isinstance(time_to_first_token, (int, float)):
-            attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] = convert_seconds_to_nanoseconds(float(time_to_first_token))
-
-        return SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=trace_metadata.workflow_span_id,
-            span_id=convert_to_span_id(node_execution.id, "node"),
-            name=node_execution.title,
-            start_time=convert_datetime_to_nanoseconds(node_execution.created_at),
-            end_time=convert_datetime_to_nanoseconds(node_execution.finished_at),
-            attributes=attributes,
-            status=get_workflow_node_status(node_execution),
-            links=trace_metadata.links,
-        )
-
-    def build_agent_react_spans(
-        self, node_execution: WorkflowNodeExecution, trace_metadata: TraceMetadata
-    ) -> list[SpanData]:
-        """Build ReAct STEP spans and child LLM / TOOL spans from the agent execution log.
-
-        The agent log lives in ``outputs["json"]``; ``started_at``/``finished_at`` there are
-        monotonic-clock seconds, so they are mapped onto wall-clock time by anchoring the
-        earliest ``started_at`` to the node's start time. Entries without timing fall back
-        to the node's start/end times. Returns an empty list when no log is available.
-        """
-        try:
-            outputs = node_execution.outputs or {}
-            round_entries = parse_agent_log_entries(outputs)
-            if not round_entries:
-                return []
-
-            agent_span_id = convert_to_span_id(node_execution.id, "node")
-            node_start_ns = convert_datetime_to_nanoseconds(node_execution.created_at)
-            node_end_ns = convert_datetime_to_nanoseconds(node_execution.finished_at)
-
-            monotonic_starts = [
-                entry.metadata["started_at"]
-                for round_entry in round_entries
-                for entry in [round_entry, *round_entry.children]
-                if isinstance(entry.metadata.get("started_at"), (int, float))
-            ]
-            base_monotonic = min(monotonic_starts) if monotonic_starts else None
-
-            def to_wall_clock_ns(monotonic_seconds: Any, fallback: int | None) -> int | None:
-                if (
-                    isinstance(monotonic_seconds, (int, float))
-                    and base_monotonic is not None
-                    and node_start_ns is not None
-                ):
-                    return node_start_ns + convert_seconds_to_nanoseconds(float(monotonic_seconds) - base_monotonic)
-                return fallback
-
-            spans: list[SpanData] = []
-            for index, round_entry in enumerate(round_entries, start=1):
-                round_number = extract_react_round_number(round_entry.label, index)
-                step_span_id = generate_span_id()
-                step_attributes: dict[str, Any] = {
-                    **create_common_span_attributes(
-                        session_id=trace_metadata.session_id,
-                        user_id=trace_metadata.user_id,
-                        span_kind=GenAISpanKind.STEP,
-                        inputs="",
-                        outputs=serialize_json_data(round_entry.data),
-                    ),
-                    GEN_AI_OPERATION_NAME: OPERATION_NAME_REACT,
-                    GEN_AI_REACT_ROUND: round_number,
-                }
-                if round_entry.error:
-                    step_attributes[GEN_AI_REACT_FINISH_REASON] = "error"
-                spans.append(
-                    SpanData(
-                        trace_id=trace_metadata.trace_id,
-                        parent_span_id=agent_span_id,
-                        span_id=step_span_id,
-                        name=round_entry.label or f"react step {round_number}",
-                        start_time=to_wall_clock_ns(round_entry.metadata.get("started_at"), node_start_ns),
-                        end_time=to_wall_clock_ns(round_entry.metadata.get("finished_at"), node_end_ns),
-                        attributes=step_attributes,
-                        status=create_status_from_agent_log_entry(round_entry),
-                        links=trace_metadata.links,
-                    )
-                )
-
-                for child in round_entry.children:
-                    child_start = to_wall_clock_ns(child.metadata.get("started_at"), node_start_ns)
-                    child_end = to_wall_clock_ns(child.metadata.get("finished_at"), node_end_ns)
-                    if is_tool_call_entry(child):
-                        spans.append(
-                            self._build_agent_tool_call_span(
-                                entry=child,
-                                step_span_id=step_span_id,
-                                trace_metadata=trace_metadata,
-                                start_time=child_start,
-                                end_time=child_end,
-                            )
-                        )
-                    elif is_llm_thought_entry(child):
-                        spans.append(
-                            self._build_agent_llm_call_span(
-                                entry=child,
-                                step_span_id=step_span_id,
-                                trace_metadata=trace_metadata,
-                                start_time=child_start,
-                                end_time=child_end,
-                            )
-                        )
-            return spans
-        except Exception as e:
-            logger.warning("Error occurred in build_agent_react_spans: %s", e, exc_info=True)
-            return []
-
-    def _build_agent_llm_call_span(
-        self,
-        entry: AgentLogEntry,
-        step_span_id: int,
-        trace_metadata: TraceMetadata,
-        start_time: int | None,
-        end_time: int | None,
-    ) -> SpanData:
-        completion = str(entry.data.get("thought") or entry.data.get("action") or "")
-        return SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=step_span_id,
-            span_id=generate_span_id(),
-            name=entry.label or "llm",
-            start_time=start_time,
-            end_time=end_time,
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.LLM,
-                    inputs="",
-                    outputs=serialize_json_data(entry.data),
-                ),
-                GEN_AI_OPERATION_NAME: OPERATION_NAME_CHAT,
-                GEN_AI_REQUEST_MODEL: extract_model_name_from_thought_label(entry.label),
-                GEN_AI_PROVIDER_NAME: str(entry.metadata.get("provider") or ""),
-                GEN_AI_USAGE_TOTAL_TOKENS: str(entry.metadata.get("total_tokens", 0)),
-                GEN_AI_COMPLETION: completion,
-            },
-            status=create_status_from_agent_log_entry(entry),
-            links=trace_metadata.links,
-        )
-
-    def _build_agent_tool_call_span(
-        self,
-        entry: AgentLogEntry,
-        step_span_id: int,
-        trace_metadata: TraceMetadata,
-        start_time: int | None,
-        end_time: int | None,
-    ) -> SpanData:
-        tool_name = str(entry.data.get("tool_name") or extract_tool_name_from_call_label(entry.label) or "tool")
-        tool_parameters = entry.data.get("tool_call_args")
-        if tool_parameters is None:
-            tool_parameters = entry.data.get("tool_call_input")
-        if tool_parameters is None:
-            tool_parameters = entry.data
-        tool_arguments_json = serialize_json_data(tool_parameters)
-        tool_result = entry.data.get("output", entry.data)
-        tool_result_json = tool_result if isinstance(tool_result, str) else serialize_json_data(tool_result)
-        provider_type = entry.metadata.get("provider_type") or entry.data.get("provider_type")
-        return SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=step_span_id,
-            span_id=generate_span_id(),
-            name=entry.label or f"CALL {tool_name}",
-            start_time=start_time,
-            end_time=end_time,
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.TOOL,
-                    inputs=tool_arguments_json,
-                    outputs=tool_result_json,
-                ),
-                **create_gen_ai_tool_attributes(
-                    tool_name=tool_name,
-                    tool_type=map_gen_ai_tool_type(str(provider_type) if provider_type else None),
-                    tool_description=extract_tool_description(entry.data) or extract_tool_description(entry.metadata),
-                    tool_call_id=entry.id,
-                    tool_call_arguments=tool_arguments_json,
-                    tool_call_result=tool_result_json,
-                ),
-            },
-            status=create_status_from_agent_log_entry(entry),
-            links=trace_metadata.links,
-        )
-
-    def add_workflow_span(self, trace_info: WorkflowTraceInfo, trace_metadata: TraceMetadata):
-        message_span_id = None
-        if trace_info.message_id:
-            message_span_id = convert_to_span_id(trace_info.message_id, "message")
-        status = create_status_from_error(trace_info.error)
-
-        inputs_json = serialize_json_data(trace_info.workflow_run_inputs)
-        outputs_json = serialize_json_data(trace_info.workflow_run_outputs)
-
-        app_id = self._extract_app_id(trace_info)
-
-        if message_span_id:
-            message_span = SpanData(
-                trace_id=trace_metadata.trace_id,
-                parent_span_id=None,
-                span_id=message_span_id,
-                name="message",
-                start_time=convert_datetime_to_nanoseconds(trace_info.start_time),
-                end_time=convert_datetime_to_nanoseconds(trace_info.end_time),
-                attributes={
-                    **create_common_span_attributes(
-                        session_id=trace_metadata.session_id,
-                        user_id=trace_metadata.user_id,
-                        span_kind=GenAISpanKind.CHAIN,
-                        inputs=trace_info.workflow_run_inputs.get("sys.query") or "",
-                        outputs=outputs_json,
-                    ),
-                    DIFY_APP_ID: app_id,
-                },
-                status=status,
-                links=trace_metadata.links,
-                span_kind=SpanKind.SERVER,
-            )
-            self.trace_client.add_span(message_span)
-
-        workflow_span = SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=message_span_id,
-            span_id=trace_metadata.workflow_span_id,
-            name="workflow",
-            start_time=convert_datetime_to_nanoseconds(trace_info.start_time),
-            end_time=convert_datetime_to_nanoseconds(trace_info.end_time),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.CHAIN,
-                    inputs=inputs_json,
-                    outputs=outputs_json,
-                ),
-                **({DIFY_APP_ID: app_id} if message_span_id is None else {}),
-            },
-            status=status,
-            links=trace_metadata.links,
-            span_kind=SpanKind.SERVER if message_span_id is None else SpanKind.INTERNAL,
-        )
-        self.trace_client.add_span(workflow_span)
-
-    def suggested_question_trace(self, trace_info: SuggestedQuestionTraceInfo):
-        message_id = trace_info.message_id
-        status = create_status_from_error(trace_info.error)
-
-        trace_metadata = TraceMetadata(
-            trace_id=convert_to_trace_id(message_id),
-            workflow_span_id=0,
-            session_id=trace_info.metadata.get("conversation_id") or "",
-            user_id=str(trace_info.metadata.get("user_id") or ""),
-            links=create_links_from_trace_id(trace_info.trace_id),
-        )
-
-        inputs_json = serialize_json_data(trace_info.inputs)
-        suggested_question_json = serialize_json_data(trace_info.suggested_question)
-
-        suggested_question_span = SpanData(
-            trace_id=trace_metadata.trace_id,
-            parent_span_id=convert_to_span_id(message_id, "message"),
-            span_id=convert_to_span_id(message_id, "suggested_question"),
-            name="suggested_question",
-            start_time=convert_datetime_to_nanoseconds(trace_info.start_time),
-            end_time=convert_datetime_to_nanoseconds(trace_info.end_time),
-            attributes={
-                **create_common_span_attributes(
-                    session_id=trace_metadata.session_id,
-                    user_id=trace_metadata.user_id,
-                    span_kind=GenAISpanKind.LLM,
-                    inputs=inputs_json,
-                    outputs=suggested_question_json,
-                ),
-                GEN_AI_REQUEST_MODEL: trace_info.metadata.get("ls_model_name") or "",
-                GEN_AI_PROVIDER_NAME: trace_info.metadata.get("ls_provider") or "",
-                GEN_AI_PROMPT: inputs_json,
-                GEN_AI_COMPLETION: suggested_question_json,
-            },
-            status=status,
-            links=trace_metadata.links,
-        )
-        self.trace_client.add_span(suggested_question_span)

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py

@@ -1,36 +1,94 @@
-from pydantic import ConfigDict, ValidationInfo, field_validator
+"""Aliyun configuration.
 
-from core.ops.entities.config_entity import BaseTracingConfig
-from core.ops.utils import validate_url_with_path
+Users paste collector URLs that already carry the license key in their path
+(``/adapt_<key>/api/otlp/traces``). Such endpoints are rejected rather than
+parsed, and validation messages are fixed texts that never include input.
+"""
+
+import re
+from urllib.parse import unquote, urlsplit, urlunsplit
+
+from dify_trace_sdk.config import ProviderConfig, destination, name_or_default, secret
+from pydantic import field_validator
+
+DEFAULT_APP_NAME = "dify_app"
+DEFAULT_ENDPOINT = "https://tracing-analysis-dc-hz.aliyuncs.com"
+
+_INVALID_APP_NAME = "Aliyun requires an application name without control characters."
+_INVALID_LICENSE_KEY = "Aliyun requires a nonempty license key without whitespace or control characters."
+_INVALID_ENDPOINT = "Aliyun requires an HTTP(S) base URL without credentials or a collector suffix."
 
 
-class AliyunConfig(BaseTracingConfig):
-    """
-    Model class for Aliyun tracing config.
-    """
+def _is_control(char: str) -> bool:
+    return ord(char) < 32 or ord(char) == 127
 
-    # Endpoints may be pasted with an embedded LicenseKey. Even rejected input
-    # must not appear in formatted validation exceptions or chained API errors.
-    model_config = ConfigDict(hide_input_in_errors=True)
 
-    app_name: str = "dify_app"
-    license_key: str
-    endpoint: str
+def _normalized_endpoint(value: str) -> str | None:
+    """The base URL with a lowercase ASCII host and no trailing slash; ``None`` when Aliyun cannot use it."""
+    value = value.strip(" ") or DEFAULT_ENDPOINT
+    try:
+        parsed = urlsplit(value)
+        host, port = parsed.hostname, parsed.port
+        path = parsed.path.rstrip("/")
+        # Decode only the public path to detect hidden collector routes. License
+        # keys are opaque values and must never be unquoted or otherwise altered.
+        decoded_path = unquote(path, errors="strict")
+        valid = (
+            parsed.scheme in {"http", "https"}
+            and host
+            and parsed.username is None
+            and parsed.password is None
+            and "?" not in value
+            and "#" not in value
+            and "\\" not in value
+            and not any(char == " " or _is_control(char) for char in value)
+            and not parsed.netloc.endswith(":")
+            and (port is None or 1 <= port <= 65535)
+            and "%" not in host
+            and not re.search(r"%(?![0-9a-fA-F]{2})", path)
+            and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)
+            and not any(char in decoded_path for char in "\\?#%")
+            and not any(_is_control(char) for char in decoded_path)
+            and "//" not in parsed.path
+            and not any(part in {".", ".."} or part.lower().startswith("adapt_") for part in decoded_path.split("/"))
+            and not decoded_path.lower().endswith(("/v1/traces", "/api/otlp/traces"))
+        )
+        if not valid or host is None:
+            return None
+        host = host.encode("idna").decode("ascii").lower()
+    except (UnicodeError, ValueError):
+        # Return instead of raising here so no exception carrying the input is chained.
+        return None
+    authority = f"[{host}]" if ":" in host else host
+    if port is not None:
+        authority = f"{authority}:{port}"
+    return urlunsplit((parsed.scheme, authority, path, "", ""))
+
+
+class AliyunConfig(ProviderConfig):
+    app_name: str = DEFAULT_APP_NAME
+    license_key: str = secret()
+    endpoint: str = destination()
 
     @field_validator("app_name")
     @classmethod
-    def app_name_validator(cls, v, info: ValidationInfo):
-        return cls.validate_project_field(v, "dify_app")
+    def _app_name(cls, value: str) -> str:
+        if any(_is_control(char) for char in value):
+            raise ValueError(_INVALID_APP_NAME)
+        return name_or_default(value, DEFAULT_APP_NAME)
 
     @field_validator("license_key")
     @classmethod
-    def license_key_validator(cls, v, info: ValidationInfo):
-        if not v or v.strip() == "":
-            raise ValueError("License key cannot be empty")
-        return v
+    def _license_key(cls, value: str) -> str:
+        # The key becomes a URL path segment: lone surrogates cannot be percent-encoded.
+        if not value or any(char.isspace() or _is_control(char) or "\ud800" <= char <= "\udfff" for char in value):
+            raise ValueError(_INVALID_LICENSE_KEY)
+        return value
 
     @field_validator("endpoint")
     @classmethod
-    def endpoint_validator(cls, v, info: ValidationInfo):
-        # aliyun uses two URL formats, which may include a URL path
-        return validate_url_with_path(v, "https://tracing-analysis-dc-hz.aliyuncs.com")
+    def _endpoint(cls, value: str) -> str:
+        endpoint = _normalized_endpoint(value)
+        if endpoint is None:
+            raise ValueError(_INVALID_ENDPOINT)
+        return endpoint

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/configuration.py

@@ -1,158 +0,0 @@
-"""Aliyun destinations and a resource-only, zero-span collector access check.
-
-GET and URL helpers are pure. Verification makes one explicit collector request;
-it never constructs a tracer, creates span IDs, or exports business telemetry.
-"""
-
-import re
-from collections.abc import Mapping
-from typing import Literal
-from urllib.parse import quote, unquote, urlsplit, urlunsplit
-
-from dify_trace_aliyun.config import AliyunConfig
-
-type ConfigurationCode = Literal["invalid_endpoint", "invalid_app_name", "invalid_license_key", "verification_failed"]
-_MESSAGES: dict[ConfigurationCode, str] = {
-    "invalid_endpoint": "Aliyun requires an HTTP(S) base URL without credentials or a collector suffix.",
-    "invalid_app_name": "Aliyun requires an application name without control characters.",
-    "invalid_license_key": "Aliyun requires a nonempty license key without whitespace or control characters.",
-    "verification_failed": "Aliyun tracing configuration could not be verified.",
-}
-_DEFAULT_ENDPOINT = "https://tracing-analysis-dc-hz.aliyuncs.com"
-
-
-class AliyunConfigurationError(ValueError):
-    """Diagnostics never include submitted URLs, credentials, or response bodies."""
-
-    def __init__(self, code: ConfigurationCode) -> None:
-        self.code = code
-        super().__init__(_MESSAGES[code])
-
-
-def _endpoint(value: object) -> str:
-    if not isinstance(value, str):
-        raise AliyunConfigurationError("invalid_endpoint")
-    value = value.strip(" ") or _DEFAULT_ENDPOINT
-    try:
-        parsed = urlsplit(value)
-        host, port = parsed.hostname, parsed.port
-        path = parsed.path.rstrip("/")
-        # Decode only the public path to detect hidden collector routes. License
-        # keys are opaque values and must never be unquoted or otherwise altered.
-        decoded_path = unquote(path, errors="strict")
-        valid = (
-            parsed.scheme in {"http", "https"}
-            and host
-            and parsed.username is None
-            and parsed.password is None
-            and "?" not in value
-            and "#" not in value
-            and "\\" not in value
-            and not any(ord(char) <= 32 or ord(char) == 127 for char in value)
-            and not parsed.netloc.endswith(":")
-            and (port is None or 1 <= port <= 65535)
-            and "%" not in host
-            and not re.search(r"%(?![0-9a-fA-F]{2})", path)
-            and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)
-            and not any(char in decoded_path for char in "\\?#%")
-            and not any(ord(char) < 32 or ord(char) == 127 for char in decoded_path)
-            and "//" not in parsed.path
-            and not any(part in {".", ".."} or part.lower().startswith("adapt_") for part in decoded_path.split("/"))
-            and not decoded_path.lower().endswith(("/v1/traces", "/api/otlp/traces"))
-        )
-        if not valid or host is None:
-            raise AliyunConfigurationError("invalid_endpoint")
-        host = host.encode("idna").decode("ascii").lower()
-        authority = f"[{host}]" if ":" in host else host
-        if port is not None:
-            authority = f"{authority}:{port}"
-        return urlunsplit((parsed.scheme, authority, path, "", ""))
-    except (UnicodeError, ValueError):
-        raise AliyunConfigurationError("invalid_endpoint") from None
-
-
-def _license_key(value: object) -> str:
-    if (
-        not isinstance(value, str)
-        or not value
-        or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
-    ):
-        raise AliyunConfigurationError("invalid_license_key")
-    try:
-        value.encode("utf-8")
-    except UnicodeError:
-        raise AliyunConfigurationError("invalid_license_key") from None
-    return value
-
-
-def normalize_configuration(values: Mapping[str, object]) -> AliyunConfig:
-    """Validate new writes without tightening the model used for historical GET.
-
-    Masked keys survive this phase so the gateway can restore the original
-    ciphertext on PATCH. Exporters reject masks after that restoration.
-    """
-    name = values.get("app_name", "dify_app")
-    if not isinstance(name, str) or any(ord(char) < 32 or ord(char) == 127 for char in name):
-        raise AliyunConfigurationError("invalid_app_name")
-    return AliyunConfig(
-        app_name=name.strip() or "dify_app",
-        license_key=_license_key(values.get("license_key")),
-        endpoint=_endpoint(values.get("endpoint")),
-    )
-
-
-def ingestion_url(config: Mapping[str, object]) -> str:
-    """Build one destination, encoding the raw license exactly once as a path segment."""
-    normalized = normalize_configuration(config)
-    if "*" in normalized.license_key:
-        raise AliyunConfigurationError("invalid_license_key")
-    hostname = (urlsplit(normalized.endpoint).hostname or "").removesuffix(".")
-    is_sls = hostname == "log.aliyuncs.com" or hostname.endswith(".log.aliyuncs.com")
-    suffix = "api/v1/traces" if is_sls else "api/otlp/traces"
-    # Aliyun's documented path-form token contains literal '@'. RFC 3986 path
-    # segments permit it, and this collector does not treat '%40' equivalently.
-    # Preserve only that required reserved character; never unquote raw keys.
-    return f"{normalized.endpoint}/adapt_{quote(normalized.license_key, safe='@')}/{suffix}"
-
-
-def console_url() -> str:
-    """Return the platform console, not a destination-specific project or trace link."""
-    return "https://arms.console.aliyun.com/#/llm"
-
-
-def verify_configuration(config: AliyunConfig) -> None:
-    """Check access with one resource envelope and strictly zero scopes/spans.
-
-    This collector distinguishes valid from invalid licenses for this request;
-    HEAD 405 and a completely empty request are not verification. A successful
-    acknowledgment does not establish business-span retention or queryability.
-    """
-    from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
-
-    from services.tracing.otlp_export import is_aliyun_http_success_ack
-    from services.tracing.provider_common import request
-
-    normalized = normalize_configuration(config.model_dump())
-    url = ingestion_url(normalized.model_dump())
-    payload = ExportTraceServiceRequest()
-    resource = payload.resource_spans.add().resource
-    resource.attributes.add(key="service.name").value.string_value = normalized.app_name
-    try:
-        response = request(
-            "POST",
-            url,
-            "aliyun",
-            headers={"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"},
-            content=payload.SerializeToString(),
-        )
-        try:
-            # Only the account-tested resource-only ACK establishes this check.
-            # A generic empty OTLP ACK remains valid for ordinary export, but
-            # cannot be substituted as evidence that this license was checked.
-            if not is_aliyun_http_success_ack(response.status_code, response.content):
-                raise AliyunConfigurationError("verification_failed")
-        finally:
-            response.close()
-    except Exception:
-        # No URL, LicenseKey, vendor body, or nested transport exception escapes.
-        raise AliyunConfigurationError("verification_failed") from None

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py

@@ -1,187 +0,0 @@
-import hashlib
-import random
-import socket
-import uuid
-from collections.abc import Generator, Sequence
-from contextlib import contextmanager
-from contextvars import ContextVar
-from datetime import datetime
-from typing import Final
-
-from opentelemetry import trace as trace_api
-from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
-from opentelemetry.sdk.resources import Resource
-from opentelemetry.sdk.trace import ReadableSpan
-from opentelemetry.sdk.util.instrumentation import InstrumentationScope
-from opentelemetry.semconv._incubating.attributes.deployment_attributes import (  # type: ignore[import-untyped]
-    DEPLOYMENT_ENVIRONMENT,
-)
-from opentelemetry.semconv._incubating.attributes.host_attributes import (  # type: ignore[import-untyped]
-    HOST_NAME,
-)
-from opentelemetry.semconv.attributes import service_attributes
-from opentelemetry.trace import Link, SpanContext, TraceFlags
-
-from configs import dify_config
-from dify_trace_aliyun.configuration import ingestion_url
-from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData
-from dify_trace_aliyun.entities.semconv import ACS_ARMS_SERVICE_FEATURE
-from services.tracing.otlp_export import export_http
-
-INVALID_SPAN_ID: Final[int] = 0x0000000000000000
-INVALID_TRACE_ID: Final[int] = 0x00000000000000000000000000000000
-
-
-class TraceClient:
-    """Collect one dispatch locally and synchronously acknowledge its exact batch.
-
-    Instances are cached, so pending spans belong to the current context rather
-    than the client. A mapper failure discards only that dispatch; an uncertain
-    response propagates to the durable task instead of silently retrying a batch.
-    No SDK exporter, process-global configuration or background queue is created.
-    """
-
-    def __init__(self, service_name: str, endpoint: str) -> None:
-        self.endpoint = endpoint
-        self.resource = Resource(
-            attributes={
-                service_attributes.SERVICE_NAME: service_name,
-                service_attributes.SERVICE_VERSION: f"dify-{dify_config.project.version}-{dify_config.COMMIT_SHA}",
-                DEPLOYMENT_ENVIRONMENT: f"{dify_config.DEPLOY_ENV}-{dify_config.DEPLOYMENT_EDITION.value}",
-                HOST_NAME: socket.gethostname(),
-                ACS_ARMS_SERVICE_FEATURE: "genai_app",
-            }
-        )
-        self.span_builder = SpanBuilder(self.resource)
-        self._dispatch_spans: ContextVar[list[ReadableSpan] | None] = ContextVar("aliyun_dispatch_spans", default=None)
-
-    @contextmanager
-    def batch(self) -> Generator[None]:
-        """Export only after the whole mapper succeeds, and always restore context."""
-        spans: list[ReadableSpan] = []
-        token = self._dispatch_spans.set(spans)
-        try:
-            yield
-            if spans:
-                self.export(spans)
-        finally:
-            self._dispatch_spans.reset(token)
-
-    def export(self, spans: Sequence[ReadableSpan]) -> None:
-        if spans:
-            export_http("aliyun", self.endpoint, {}, encode_spans(spans))
-
-    def add_span(self, span_data: SpanData | None) -> None:
-        if span_data is None:
-            return
-        spans = self._dispatch_spans.get()
-        if spans is None:
-            raise RuntimeError("Aliyun spans require an active trace dispatch")
-        spans.append(self.span_builder.build_span(span_data))
-
-
-class SpanBuilder:
-    def __init__(self, resource: Resource) -> None:
-        self.resource = resource
-        self.instrumentation_scope = InstrumentationScope(
-            __name__,
-            "",
-            None,
-            None,
-        )
-
-    def build_span(self, span_data: SpanData) -> ReadableSpan:
-        span_context = trace_api.SpanContext(
-            trace_id=span_data.trace_id,
-            span_id=span_data.span_id,
-            is_remote=False,
-            trace_flags=trace_api.TraceFlags(trace_api.TraceFlags.SAMPLED),
-            trace_state=None,
-        )
-
-        parent_span_context = None
-        if span_data.parent_span_id is not None:
-            parent_span_context = trace_api.SpanContext(
-                trace_id=span_data.trace_id,
-                span_id=span_data.parent_span_id,
-                is_remote=False,
-                trace_flags=trace_api.TraceFlags(trace_api.TraceFlags.SAMPLED),
-                trace_state=None,
-            )
-
-        span = ReadableSpan(
-            name=span_data.name,
-            context=span_context,
-            parent=parent_span_context,
-            resource=self.resource,
-            attributes=span_data.attributes,
-            events=span_data.events,
-            links=span_data.links,
-            kind=span_data.span_kind,
-            status=span_data.status,
-            start_time=span_data.start_time,
-            end_time=span_data.end_time,
-            instrumentation_scope=self.instrumentation_scope,
-        )
-        return span
-
-
-def create_link(trace_id_str: str) -> Link:
-    placeholder_span_id = INVALID_SPAN_ID
-    try:
-        trace_id = int(trace_id_str, 16)
-    except ValueError as e:
-        raise ValueError(f"Invalid trace ID format: {trace_id_str}") from e
-
-    span_context = SpanContext(
-        trace_id=trace_id, span_id=placeholder_span_id, is_remote=False, trace_flags=TraceFlags(TraceFlags.SAMPLED)
-    )
-
-    return Link(span_context)
-
-
-def generate_span_id() -> int:
-    span_id = random.getrandbits(64)
-    while span_id == INVALID_SPAN_ID:
-        span_id = random.getrandbits(64)
-    return span_id
-
-
-def convert_to_trace_id(uuid_v4: str | None) -> int:
-    if uuid_v4 is None:
-        raise ValueError("UUID cannot be None")
-    try:
-        uuid_obj = uuid.UUID(uuid_v4)
-        return uuid_obj.int
-    except ValueError as e:
-        raise ValueError(f"Invalid UUID input: {uuid_v4}") from e
-
-
-def convert_string_to_id(string: str | None) -> int:
-    if not string:
-        return generate_span_id()
-    hash_bytes = hashlib.sha256(string.encode("utf-8")).digest()
-    return int.from_bytes(hash_bytes[:8], byteorder="big", signed=False)
-
-
-def convert_to_span_id(uuid_v4: str | None, span_type: str) -> int:
-    if uuid_v4 is None:
-        raise ValueError("UUID cannot be None")
-    try:
-        uuid_obj = uuid.UUID(uuid_v4)
-    except ValueError as e:
-        raise ValueError(f"Invalid UUID input: {uuid_v4}") from e
-    combined_key = f"{uuid_obj.hex}-{span_type}"
-    return convert_string_to_id(combined_key)
-
-
-def convert_datetime_to_nanoseconds(start_time_a: datetime | None) -> int | None:
-    if start_time_a is None:
-        return None
-    timestamp_in_seconds = start_time_a.timestamp()
-    return int(timestamp_in_seconds * 1e9)
-
-
-def build_endpoint(base_url: str, license_key: str) -> str:
-    """Compatibility wrapper around the shared, credential-safe URL builder."""
-    return ingestion_url({"endpoint": base_url, "license_key": license_key})

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/entities/aliyun_trace_entity.py

@@ -1,37 +0,0 @@
-from collections.abc import Sequence
-from dataclasses import dataclass
-from typing import Any
-
-from opentelemetry import trace as trace_api
-from opentelemetry.sdk.trace import Event
-from opentelemetry.trace import SpanKind, Status, StatusCode
-from pydantic import BaseModel, Field
-
-
-@dataclass
-class TraceMetadata:
-    """Metadata for trace operations, containing common attributes for all spans in a trace."""
-
-    trace_id: int
-    workflow_span_id: int
-    session_id: str
-    user_id: str
-    links: list[trace_api.Link]
-
-
-class SpanData(BaseModel):
-    """Data model for span information in Aliyun trace system."""
-
-    model_config = {"arbitrary_types_allowed": True}
-
-    trace_id: int = Field(..., description="The unique identifier for the trace.")
-    parent_span_id: int | None = Field(None, description="The ID of the parent span, if any.")
-    span_id: int = Field(..., description="The unique identifier for this span.")
-    name: str = Field(..., description="The name of the span.")
-    attributes: dict[str, Any] = Field(default_factory=dict, description="Attributes associated with the span.")
-    events: Sequence[Event] = Field(default_factory=list, description="Events recorded in the span.")
-    links: Sequence[trace_api.Link] = Field(default_factory=list, description="Links to other spans.")
-    status: Status = Field(default=Status(StatusCode.UNSET), description="The status of the span.")
-    start_time: int | None = Field(..., description="The start time of the span in nanoseconds.")
-    end_time: int | None = Field(..., description="The end time of the span in nanoseconds.")
-    span_kind: SpanKind = Field(default=SpanKind.INTERNAL, description="The OpenTelemetry SpanKind for this span.")

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/entities/semconv.py

@@ -1,84 +0,0 @@
-from enum import StrEnum
-from typing import Final
-
-ACS_ARMS_SERVICE_FEATURE: Final[str] = "acs.arms.service.feature"
-
-# Dify-specific attributes
-DIFY_APP_ID: Final[str] = "dify.app_id"
-DIFY_NODE_TYPE: Final[str] = "dify.node.type"
-
-# Public attributes
-GEN_AI_SESSION_ID: Final[str] = "gen_ai.session.id"
-GEN_AI_USER_ID: Final[str] = "gen_ai.user.id"
-GEN_AI_USER_NAME: Final[str] = "gen_ai.user.name"
-GEN_AI_SPAN_KIND: Final[str] = "gen_ai.span.kind"
-GEN_AI_OPERATION_NAME: Final[str] = "gen_ai.operation.name"
-GEN_AI_FRAMEWORK: Final[str] = "gen_ai.framework"
-
-# Chain attributes
-INPUT_VALUE: Final[str] = "input.value"
-OUTPUT_VALUE: Final[str] = "output.value"
-
-# Retriever attributes
-RETRIEVAL_QUERY: Final[str] = "retrieval.query"
-RETRIEVAL_DOCUMENT: Final[str] = "retrieval.document"
-
-# LLM attributes
-GEN_AI_REQUEST_MODEL: Final[str] = "gen_ai.request.model"
-GEN_AI_PROVIDER_NAME: Final[str] = "gen_ai.provider.name"
-GEN_AI_USAGE_INPUT_TOKENS: Final[str] = "gen_ai.usage.input_tokens"
-GEN_AI_USAGE_OUTPUT_TOKENS: Final[str] = "gen_ai.usage.output_tokens"
-GEN_AI_USAGE_TOTAL_TOKENS: Final[str] = "gen_ai.usage.total_tokens"
-GEN_AI_PROMPT: Final[str] = "gen_ai.prompt"
-GEN_AI_COMPLETION: Final[str] = "gen_ai.completion"
-GEN_AI_RESPONSE_FINISH_REASON: Final[str] = "gen_ai.response.finish_reason"
-# Time to first token of the model response in a streaming scenario, in nanoseconds.
-GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN: Final[str] = "gen_ai.response.time_to_first_token"
-
-GEN_AI_INPUT_MESSAGE: Final[str] = "gen_ai.input.messages"
-GEN_AI_OUTPUT_MESSAGE: Final[str] = "gen_ai.output.messages"
-
-# Tool attributes (GenAI semantic conventions)
-GEN_AI_TOOL_CALL_ID: Final[str] = "gen_ai.tool.call.id"
-GEN_AI_TOOL_DESCRIPTION: Final[str] = "gen_ai.tool.description"
-GEN_AI_TOOL_NAME: Final[str] = "gen_ai.tool.name"
-GEN_AI_TOOL_TYPE: Final[str] = "gen_ai.tool.type"
-GEN_AI_TOOL_CALL_ARGUMENTS: Final[str] = "gen_ai.tool.call.arguments"
-GEN_AI_TOOL_CALL_RESULT: Final[str] = "gen_ai.tool.call.result"
-
-# Skill attributes (conditionally required when loading a Skill)
-GEN_AI_SKILL_ID: Final[str] = "gen_ai.skill.id"
-GEN_AI_SKILL_NAME: Final[str] = "gen_ai.skill.name"
-GEN_AI_SKILL_DESCRIPTION: Final[str] = "gen_ai.skill.description"
-GEN_AI_SKILL_VERSION: Final[str] = "gen_ai.skill.version"
-
-# Agent attributes
-GEN_AI_AGENT_NAME: Final[str] = "gen_ai.agent.name"
-
-# ReAct step attributes
-GEN_AI_REACT_ROUND: Final[str] = "gen_ai.react.round"
-GEN_AI_REACT_FINISH_REASON: Final[str] = "gen_ai.react.finish_reason"
-
-# gen_ai.operation.name values (see Aliyun LLM Trace field definitions)
-OPERATION_NAME_CHAT: Final[str] = "chat"
-OPERATION_NAME_EXECUTE_TOOL: Final[str] = "execute_tool"
-OPERATION_NAME_INVOKE_AGENT: Final[str] = "invoke_agent"
-OPERATION_NAME_REACT: Final[str] = "react"
-
-# gen_ai.tool.type values
-TOOL_TYPE_FUNCTION: Final[str] = "function"
-TOOL_TYPE_EXTENSION: Final[str] = "extension"
-TOOL_TYPE_DATASTORE: Final[str] = "datastore"
-
-
-class GenAISpanKind(StrEnum):
-    CHAIN = "CHAIN"
-    RETRIEVER = "RETRIEVER"
-    RERANKER = "RERANKER"
-    LLM = "LLM"
-    EMBEDDING = "EMBEDDING"
-    TOOL = "TOOL"
-    AGENT = "AGENT"
-    TASK = "TASK"
-    # Marks one Reasoning-Acting iteration of an agent (ReAct step).
-    STEP = "STEP"

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/provider.py

@@ -0,0 +1,104 @@
+"""Aliyun LLM application monitoring over OTLP/HTTP.
+
+The license key travels in the collector URL path, so every request marks its
+URL as sensitive. The collector acknowledges an export with the literal body
+``"success"`` rather than an OTLP response; standard acknowledgments are
+accepted too.
+"""
+
+from typing import override
+from urllib.parse import quote, urlsplit
+
+from dify_trace_sdk.egress import EgressResponse
+from dify_trace_sdk.errors import EgressError, ExportRejectedError, VerificationError
+from dify_trace_sdk.genai import GenAIDialect, genai_attributes
+from dify_trace_sdk.model import SpanKind, Trace
+from dify_trace_sdk.otlp import encode_trace, post_otlp, read_otlp_ack
+from dify_trace_sdk.provider import ProviderCapabilities, SendResult, TraceProvider
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+
+from dify_trace_aliyun.config import AliyunConfig
+
+VERIFY_TIMEOUT_SECONDS = 10.0
+_DESTINATION = "Aliyun"
+_CONSOLE_URL = "https://arms.console.aliyun.com/#/llm"
+_DIALECT = GenAIDialect(
+    kind_labels={kind: kind.value.upper() for kind in SpanKind},
+    input_key="input.value",
+    output_key="output.value",
+    mark_entry_span=False,
+)
+
+
+def _ingestion_url(config: AliyunConfig) -> str:
+    """The collector route, with the raw license key encoded exactly once as a path segment."""
+    hostname = (urlsplit(config.endpoint).hostname or "").removesuffix(".")
+    is_sls = hostname == "log.aliyuncs.com" or hostname.endswith(".log.aliyuncs.com")
+    suffix = "api/v1/traces" if is_sls else "api/otlp/traces"
+    # Aliyun's documented path-form key contains a literal '@'. RFC 3986 path
+    # segments permit it, and the collector does not treat '%40' equivalently.
+    return f"{config.endpoint}/adapt_{quote(config.license_key, safe='@')}/{suffix}"
+
+
+def _is_masked(config: AliyunConfig) -> bool:
+    """A displayed mask of a key passes validation but is not a credential to send."""
+    return "*" in config.license_key
+
+
+def _is_success_ack(response: EgressResponse) -> bool:
+    """The observed acknowledgment of Aliyun's HTTP collector; never normalize other bodies into it."""
+    return response.status_code == 200 and response.content == b'"success"'
+
+
+class AliyunProvider(TraceProvider[AliyunConfig]):
+    id = "aliyun"
+    config_model = AliyunConfig
+    capabilities = ProviderCapabilities(native_cost=False, trace_url=False, span_url=False)
+
+    @override
+    def verify(self, config: AliyunConfig) -> AliyunConfig:
+        """Post one resource with zero spans and require the collector's literal acknowledgment.
+
+        The collector distinguishes valid from invalid license keys for this
+        request, whereas an empty OTLP acknowledgment proves nothing. Success
+        does not establish that business spans are retained.
+        """
+        if _is_masked(config):
+            raise VerificationError("invalid_license_key")
+        probe = ExportTraceServiceRequest()
+        resource = probe.resource_spans.add().resource
+        resource.attributes.add(key="service.name").value.string_value = config.app_name
+        try:
+            response = self.egress.request(
+                "POST",
+                _ingestion_url(config),
+                headers={"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"},
+                content=probe.SerializeToString(),
+                timeout=VERIFY_TIMEOUT_SECONDS,
+                sensitive_url=True,
+            )
+        except EgressError:
+            raise VerificationError("verification_failed") from None
+        if not _is_success_ack(response):
+            raise VerificationError("verification_failed")
+        return config
+
+    @override
+    def project_url(self, config: AliyunConfig) -> str | None:
+        return _CONSOLE_URL
+
+    @override
+    def send(self, config: AliyunConfig, trace: Trace) -> SendResult:
+        if _is_masked(config):
+            raise ExportRejectedError("Aliyun tracing configuration is invalid")
+        request = encode_trace(
+            trace,
+            resource={"service.name": config.app_name, "acs.arms.service.feature": "genai_app"},
+            attributes=lambda span: genai_attributes(span, trace, _DIALECT),
+        )
+        response = post_otlp(
+            self.egress, _ingestion_url(config), request, destination=_DESTINATION, headers={}, sensitive_url=True
+        )
+        if not _is_success_ack(response):
+            read_otlp_ack(response, _DESTINATION)
+        return SendResult(trace_id=trace.trace_id, trace_url=None, span_urls={})

api/providers/trace/trace-aliyun/src/dify_trace_aliyun/utils.py

@@ -1,369 +0,0 @@
-import json
-import re
-from collections.abc import Mapping
-from dataclasses import dataclass, field
-from typing import Any, TypedDict
-
-from opentelemetry.trace import Link, Status, StatusCode
-
-from core.rag.models.document import Document
-from dify_trace_aliyun.entities.semconv import (
-    GEN_AI_FRAMEWORK,
-    GEN_AI_OPERATION_NAME,
-    GEN_AI_SESSION_ID,
-    GEN_AI_SPAN_KIND,
-    GEN_AI_TOOL_CALL_ARGUMENTS,
-    GEN_AI_TOOL_CALL_ID,
-    GEN_AI_TOOL_CALL_RESULT,
-    GEN_AI_TOOL_DESCRIPTION,
-    GEN_AI_TOOL_NAME,
-    GEN_AI_TOOL_TYPE,
-    GEN_AI_USER_ID,
-    INPUT_VALUE,
-    OPERATION_NAME_EXECUTE_TOOL,
-    OUTPUT_VALUE,
-    TOOL_TYPE_DATASTORE,
-    TOOL_TYPE_EXTENSION,
-    TOOL_TYPE_FUNCTION,
-    GenAISpanKind,
-)
-from extensions.ext_database import db
-from graphon.entities import WorkflowNodeExecution
-from graphon.enums import WorkflowNodeExecutionStatus
-from models import EndUser
-
-# Constants
-DEFAULT_JSON_ENSURE_ASCII = False
-DEFAULT_FRAMEWORK_NAME = "dify"
-
-
-def get_user_id_from_message_data(message_data) -> str:
-    user_id = message_data.from_account_id
-    if message_data.from_end_user_id:
-        end_user_data: EndUser | None = db.session.get(EndUser, message_data.from_end_user_id)
-        if end_user_data is not None:
-            user_id = end_user_data.session_id
-    return user_id
-
-
-def create_status_from_error(error: str | None) -> Status:
-    if error:
-        return Status(StatusCode.ERROR, error)
-    return Status(StatusCode.OK)
-
-
-def get_workflow_node_status(node_execution: WorkflowNodeExecution) -> Status:
-    if node_execution.status == WorkflowNodeExecutionStatus.SUCCEEDED:
-        return Status(StatusCode.OK)
-    if node_execution.status in [WorkflowNodeExecutionStatus.FAILED, WorkflowNodeExecutionStatus.EXCEPTION]:
-        return Status(StatusCode.ERROR, str(node_execution.error))
-    return Status(StatusCode.UNSET)
-
-
-def create_links_from_trace_id(trace_id: str | None) -> list[Link]:
-    from dify_trace_aliyun.data_exporter.traceclient import create_link
-
-    links = []
-    if trace_id:
-        links.append(create_link(trace_id_str=trace_id))
-    return links
-
-
-class RetrievalDocumentMetadataDict(TypedDict):
-    dataset_id: Any
-    doc_id: Any
-    document_id: Any
-
-
-class RetrievalDocumentDict(TypedDict):
-    content: str
-    metadata: RetrievalDocumentMetadataDict
-    score: Any
-
-
-def extract_retrieval_documents(documents: list[Document]) -> list[RetrievalDocumentDict]:
-    documents_data: list[RetrievalDocumentDict] = []
-    for document in documents:
-        document_data: RetrievalDocumentDict = {
-            "content": document.page_content,
-            "metadata": {
-                "dataset_id": document.metadata.get("dataset_id"),
-                "doc_id": document.metadata.get("doc_id"),
-                "document_id": document.metadata.get("document_id"),
-            },
-            "score": document.metadata.get("score"),
-        }
-        documents_data.append(document_data)
-    return documents_data
-
-
-def serialize_json_data(data: Any, ensure_ascii: bool = DEFAULT_JSON_ENSURE_ASCII) -> str:
-    return json.dumps(data, ensure_ascii=ensure_ascii)
-
-
-def create_common_span_attributes(
-    session_id: str = "",
-    user_id: str = "",
-    span_kind: str = GenAISpanKind.CHAIN,
-    framework: str = DEFAULT_FRAMEWORK_NAME,
-    inputs: str = "",
-    outputs: str = "",
-) -> dict[str, str]:
-    return {
-        GEN_AI_SESSION_ID: session_id,
-        GEN_AI_USER_ID: user_id,
-        GEN_AI_SPAN_KIND: span_kind,
-        GEN_AI_FRAMEWORK: framework,
-        INPUT_VALUE: inputs,
-        OUTPUT_VALUE: outputs,
-    }
-
-
-def map_gen_ai_tool_type(provider_type: str | None) -> str:
-    """Map Dify tool provider type to GenAI ``gen_ai.tool.type`` values."""
-    normalized = (provider_type or "").strip().lower()
-    if normalized in {"dataset-retrieval", "datastore"}:
-        return TOOL_TYPE_DATASTORE
-    if normalized == "extension":
-        return TOOL_TYPE_EXTENSION
-    return TOOL_TYPE_FUNCTION
-
-
-def extract_tool_description(tool_meta: Mapping[str, Any] | None) -> str:
-    if not isinstance(tool_meta, Mapping):
-        return ""
-    for key in ("description", "tool_description"):
-        value = tool_meta.get(key)
-        if value:
-            return str(value)
-    return ""
-
-
-def create_gen_ai_tool_attributes(
-    *,
-    tool_name: str,
-    tool_type: str = TOOL_TYPE_FUNCTION,
-    tool_description: str = "",
-    tool_call_id: str = "",
-    tool_call_arguments: str = "",
-    tool_call_result: str = "",
-) -> dict[str, str]:
-    """Build GenAI-compliant attributes for a TOOL span."""
-    return {
-        GEN_AI_OPERATION_NAME: OPERATION_NAME_EXECUTE_TOOL,
-        GEN_AI_TOOL_NAME: tool_name,
-        GEN_AI_TOOL_TYPE: tool_type or TOOL_TYPE_FUNCTION,
-        GEN_AI_TOOL_DESCRIPTION: tool_description,
-        GEN_AI_TOOL_CALL_ID: tool_call_id,
-        GEN_AI_TOOL_CALL_ARGUMENTS: tool_call_arguments,
-        GEN_AI_TOOL_CALL_RESULT: tool_call_result,
-    }
-
-
-def format_retrieval_documents(retrieval_documents: list) -> list:
-    try:
-        if not isinstance(retrieval_documents, list):
-            return []
-
-        semantic_documents = []
-        for doc in retrieval_documents:
-            if not isinstance(doc, dict):
-                continue
-
-            metadata = doc.get("metadata", {})
-            content = doc.get("content", "")
-            title = doc.get("title", "")
-            score = metadata.get("score", 0.0)
-            document_id = metadata.get("document_id", "")
-
-            semantic_metadata = {}
-            if title:
-                semantic_metadata["title"] = title
-            if metadata.get("source"):
-                semantic_metadata["source"] = metadata["source"]
-            elif metadata.get("_source"):
-                semantic_metadata["source"] = metadata["_source"]
-            if metadata.get("doc_metadata"):
-                doc_metadata = metadata["doc_metadata"]
-                if isinstance(doc_metadata, dict):
-                    semantic_metadata.update(doc_metadata)
-
-            semantic_doc = {
-                "document": {"content": content, "metadata": semantic_metadata, "score": score, "id": document_id}
-            }
-            semantic_documents.append(semantic_doc)
-
-        return semantic_documents
-    except Exception:
-        return []
-
-
-def format_input_messages(process_data: Mapping[str, Any]) -> str:
-    try:
-        if not isinstance(process_data, dict):
-            return serialize_json_data([])
-
-        prompts = process_data.get("prompts", [])
-        if not prompts:
-            return serialize_json_data([])
-
-        valid_roles = {"system", "user", "assistant", "tool"}
-        input_messages = []
-        for prompt in prompts:
-            if not isinstance(prompt, dict):
-                continue
-
-            role = prompt.get("role", "")
-            text = prompt.get("text", "")
-
-            if not role or role not in valid_roles:
-                continue
-
-            if text:
-                message = {"role": role, "parts": [{"type": "text", "content": text}]}
-                input_messages.append(message)
-
-        return serialize_json_data(input_messages)
-    except Exception:
-        return serialize_json_data([])
-
-
-def convert_seconds_to_nanoseconds(seconds: float) -> int:
-    return int(seconds * 1e9)
-
-
-_REACT_ROUND_LABEL_PATTERN = re.compile(r"ROUND\s+(\d+)", re.IGNORECASE)
-_LLM_THOUGHT_LABEL_SUFFIX = " Thought"
-_TOOL_CALL_LABEL_PREFIX = "CALL "
-
-
-@dataclass
-class AgentLogEntry:
-    """One entry of the agent-strategy execution log (``outputs["json"]`` of an agent node).
-
-    Entries form a tree via ``parent_id``: top-level entries are ReAct rounds
-    (label like ``ROUND 1``) and their children are LLM thoughts / tool calls.
-    ``started_at``/``finished_at`` in ``metadata`` are monotonic-clock seconds
-    (``time.perf_counter``), not epoch timestamps.
-    """
-
-    id: str
-    parent_id: str | None
-    label: str
-    status: str
-    error: str | None
-    data: dict[str, Any]
-    metadata: dict[str, Any]
-    children: list["AgentLogEntry"] = field(default_factory=list)
-
-
-def parse_agent_log_entries(outputs: Mapping[str, Any]) -> list[AgentLogEntry]:
-    """Parse agent node outputs into a tree of log entries, returning top-level rounds in order.
-
-    Entries without an ``id`` (e.g. the trailing ``{"data": []}`` element) are skipped.
-    Children whose parent is missing are dropped.
-    """
-    raw_entries = outputs.get("json")
-    if not isinstance(raw_entries, list):
-        return []
-
-    entries: list[AgentLogEntry] = []
-    entries_by_id: dict[str, AgentLogEntry] = {}
-    for raw_entry in raw_entries:
-        if not isinstance(raw_entry, dict):
-            continue
-        entry_id = raw_entry.get("id")
-        if not entry_id:
-            continue
-        data = raw_entry.get("data")
-        metadata = raw_entry.get("metadata")
-        entry = AgentLogEntry(
-            id=str(entry_id),
-            parent_id=raw_entry.get("parent_id"),
-            label=str(raw_entry.get("label") or ""),
-            status=str(raw_entry.get("status") or ""),
-            error=raw_entry.get("error"),
-            data=data if isinstance(data, dict) else {},
-            metadata=metadata if isinstance(metadata, dict) else {},
-        )
-        entries.append(entry)
-        entries_by_id[entry.id] = entry
-
-    roots: list[AgentLogEntry] = []
-    for entry in entries:
-        if entry.parent_id is None:
-            roots.append(entry)
-        else:
-            parent = entries_by_id.get(entry.parent_id)
-            if parent is not None:
-                parent.children.append(entry)
-    return roots
-
-
-def extract_react_round_number(label: str, fallback: int) -> int:
-    match = _REACT_ROUND_LABEL_PATTERN.search(label)
-    if match:
-        return int(match.group(1))
-    return fallback
-
-
-def is_tool_call_entry(entry: AgentLogEntry) -> bool:
-    """Tool invocations use labels like ``CALL {tool_name}`` (also carry a provider in metadata)."""
-    return entry.label.startswith(_TOOL_CALL_LABEL_PREFIX)
-
-
-def is_llm_thought_entry(entry: AgentLogEntry) -> bool:
-    """LLM thought entries use labels like ``{model} Thought``.
-
-    Do not key off ``metadata.provider`` alone: tool CALL entries also set provider
-    (to the tool provider), which previously misclassified them as LLM spans.
-    """
-    if is_tool_call_entry(entry):
-        return False
-    return entry.label.endswith(_LLM_THOUGHT_LABEL_SUFFIX) or bool(entry.metadata.get("provider"))
-
-
-def extract_model_name_from_thought_label(label: str) -> str:
-    if label.endswith(_LLM_THOUGHT_LABEL_SUFFIX):
-        return label.removesuffix(_LLM_THOUGHT_LABEL_SUFFIX)
-    return ""
-
-
-def extract_tool_name_from_call_label(label: str) -> str:
-    if label.startswith(_TOOL_CALL_LABEL_PREFIX):
-        return label.removeprefix(_TOOL_CALL_LABEL_PREFIX).strip()
-    return ""
-
-
-def create_status_from_agent_log_entry(entry: AgentLogEntry) -> Status:
-    if entry.error:
-        return Status(StatusCode.ERROR, str(entry.error))
-    if entry.status == "success":
-        return Status(StatusCode.OK)
-    return Status(StatusCode.UNSET)
-
-
-def format_output_messages(outputs: Mapping[str, Any]) -> str:
-    try:
-        if not isinstance(outputs, dict):
-            return serialize_json_data([])
-
-        text = outputs.get("text", "")
-        finish_reason = outputs.get("finish_reason", "")
-
-        if not text:
-            return serialize_json_data([])
-
-        valid_finish_reasons = {"stop", "length", "content_filter", "tool_call", "error"}
-        if finish_reason not in valid_finish_reasons:
-            finish_reason = "stop"
-
-        output_message = {
-            "role": "assistant",
-            "parts": [{"type": "text", "content": text}],
-            "finish_reason": finish_reason,
-        }
-
-        return serialize_json_data([output_message])
-    except Exception:
-        return serialize_json_data([])

api/providers/trace/trace-aliyun/tests/unit_tests/src/dify_trace_aliyun/data_exporter/test_traceclient.py

@@ -1,160 +0,0 @@
-import uuid
-from datetime import datetime
-from unittest.mock import patch
-
-import pytest
-from dify_trace_aliyun.data_exporter.traceclient import (
-    INVALID_SPAN_ID,
-    SpanBuilder,
-    TraceClient,
-    build_endpoint,
-    convert_datetime_to_nanoseconds,
-    convert_string_to_id,
-    convert_to_span_id,
-    convert_to_trace_id,
-    create_link,
-    generate_span_id,
-)
-from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData
-from opentelemetry.sdk.resources import Resource
-from opentelemetry.sdk.trace import ReadableSpan
-from opentelemetry.trace import SpanKind, Status, StatusCode
-
-
-class TestTraceClient:
-    def test_constructor_has_no_background_exporter(self) -> None:
-        client = TraceClient(service_name="test-service", endpoint="https://collector.test")
-        assert client.endpoint == "https://collector.test"
-        assert not hasattr(client, "exporter")
-        assert not hasattr(client, "worker_thread")
-        assert not hasattr(client, "queue")
-
-    def test_add_requires_dispatch(self) -> None:
-        client = TraceClient(service_name="test-service", endpoint="https://collector.test")
-        client.add_span(None)
-        with pytest.raises(RuntimeError, match="active trace dispatch"):
-            client.add_span(SpanData(name="span", trace_id=123, span_id=456, start_time=1000, end_time=2000))
-
-
-class TestSpanBuilder:
-    def test_build_span(self):
-        resource = Resource({"service.name": "test"})
-        builder = SpanBuilder(resource)
-
-        span_data = SpanData(
-            name="test-span",
-            trace_id=123,
-            span_id=456,
-            parent_span_id=789,
-            start_time=1000,
-            end_time=2000,
-            status=Status(StatusCode.OK),
-            span_kind=SpanKind.INTERNAL,
-            attributes={"attr1": "val1"},
-            events=[],
-            links=[],
-        )
-
-        span = builder.build_span(span_data)
-        assert isinstance(span, ReadableSpan)
-        assert span.name == "test-span"
-        assert span.context is not None
-        assert span.context.trace_id == 123
-        assert span.context.span_id == 456
-        assert span.parent is not None
-        assert span.parent.span_id == 789
-        assert span.resource == resource
-        assert span.attributes == {"attr1": "val1"}
-
-    def test_build_span_no_parent(self):
-        resource = Resource({"service.name": "test"})
-        builder = SpanBuilder(resource)
-
-        span_data = SpanData(
-            name="test-span",
-            trace_id=123,
-            span_id=456,
-            parent_span_id=None,
-            start_time=1000,
-            end_time=2000,
-            status=Status(StatusCode.OK),
-            span_kind=SpanKind.INTERNAL,
-        )
-
-        span = builder.build_span(span_data)
-        assert span.parent is None
-
-
-def test_create_link():
-    trace_id_str = "0123456789abcdef0123456789abcdef"
-    link = create_link(trace_id_str)
-    assert link.context.trace_id == int(trace_id_str, 16)
-    assert link.context.span_id == INVALID_SPAN_ID
-
-    with pytest.raises(ValueError, match="Invalid trace ID format"):
-        create_link("invalid-hex")
-
-
-def test_generate_span_id():
-    # Test normal generation
-    span_id = generate_span_id()
-    assert isinstance(span_id, int)
-    assert span_id != INVALID_SPAN_ID
-
-    # Test retry loop
-    with patch("dify_trace_aliyun.data_exporter.traceclient.random.getrandbits") as mock_rand:
-        mock_rand.side_effect = [INVALID_SPAN_ID, 999]
-        span_id = generate_span_id()
-        assert span_id == 999
-        assert mock_rand.call_count == 2
-
-
-def test_convert_to_trace_id():
-    uid = str(uuid.uuid4())
-    trace_id = convert_to_trace_id(uid)
-    assert trace_id == uuid.UUID(uid).int
-
-    with pytest.raises(ValueError, match="UUID cannot be None"):
-        convert_to_trace_id(None)
-
-    with pytest.raises(ValueError, match="Invalid UUID input"):
-        convert_to_trace_id("not-a-uuid")
-
-
-def test_convert_string_to_id():
-    assert convert_string_to_id("test") > 0
-    # Test with None string
-    with patch("dify_trace_aliyun.data_exporter.traceclient.generate_span_id") as mock_gen:
-        mock_gen.return_value = 12345
-        assert convert_string_to_id(None) == 12345
-
-
-def test_convert_to_span_id():
-    uid = str(uuid.uuid4())
-    span_id = convert_to_span_id(uid, "test-type")
-    assert isinstance(span_id, int)
-
-    with pytest.raises(ValueError, match="UUID cannot be None"):
-        convert_to_span_id(None, "test")
-
-    with pytest.raises(ValueError, match="Invalid UUID input"):
-        convert_to_span_id("not-a-uuid", "test")
-
-
-def test_convert_datetime_to_nanoseconds():
-    dt = datetime(2023, 1, 1, 12, 0, 0)
-    ns = convert_datetime_to_nanoseconds(dt)
-    assert ns == int(dt.timestamp() * 1e9)
-    assert convert_datetime_to_nanoseconds(None) is None
-
-
-def test_build_endpoint():
-    license_key = "abc"
-
-    # CMS 2.0 endpoint
-    url1 = "https://log.aliyuncs.com"
-    assert build_endpoint(url1, license_key) == "https://log.aliyuncs.com/adapt_abc/api/v1/traces"
-
-    # XTrace endpoint
-    url2 = "https://example.com"
-    assert build_endpoint(url2, license_key) == "https://example.com/adapt_abc/api/otlp/traces"

api/providers/trace/trace-aliyun/tests/unit_tests/src/dify_trace_aliyun/entities/test_aliyun_trace_entity.py

@@ -1,88 +0,0 @@
-import pytest
-from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData, TraceMetadata
-from opentelemetry import trace as trace_api
-from opentelemetry.sdk.trace import Event
-from opentelemetry.trace import SpanKind, Status, StatusCode
-from pydantic import ValidationError
-
-
-class TestTraceMetadata:
-    def test_trace_metadata_init(self):
-        links = [trace_api.Link(context=trace_api.SpanContext(0, 0, False))]
-        metadata = TraceMetadata(
-            trace_id=123, workflow_span_id=456, session_id="session_1", user_id="user_1", links=links
-        )
-        assert metadata.trace_id == 123
-        assert metadata.workflow_span_id == 456
-        assert metadata.session_id == "session_1"
-        assert metadata.user_id == "user_1"
-        assert metadata.links == links
-
-
-class TestSpanData:
-    def test_span_data_init_required_fields(self):
-        span_data = SpanData(trace_id=123, span_id=456, name="test_span", start_time=1000, end_time=2000)
-        assert span_data.trace_id == 123
-        assert span_data.span_id == 456
-        assert span_data.name == "test_span"
-        assert span_data.start_time == 1000
-        assert span_data.end_time == 2000
-
-        # Check defaults
-        assert span_data.parent_span_id is None
-        assert span_data.attributes == {}
-        assert span_data.events == []
-        assert span_data.links == []
-        assert span_data.status.status_code == StatusCode.UNSET
-        assert span_data.span_kind == SpanKind.INTERNAL
-
-    def test_span_data_with_optional_fields(self):
-        event = Event(name="event_1", timestamp=1500)
-        link = trace_api.Link(context=trace_api.SpanContext(0, 0, False))
-        status = Status(StatusCode.OK)
-
-        span_data = SpanData(
-            trace_id=123,
-            parent_span_id=111,
-            span_id=456,
-            name="test_span",
-            attributes={"key": "value"},
-            events=[event],
-            links=[link],
-            status=status,
-            start_time=1000,
-            end_time=2000,
-            span_kind=SpanKind.SERVER,
-        )
-
-        assert span_data.parent_span_id == 111
-        assert span_data.attributes == {"key": "value"}
-        assert span_data.events == [event]
-        assert span_data.links == [link]
-        assert span_data.status.status_code == status.status_code
-        assert span_data.span_kind == SpanKind.SERVER
-
-    def test_span_data_missing_required_fields(self):
-        with pytest.raises(ValidationError):
-            SpanData.model_validate(
-                {
-                    "trace_id": 123,
-                    "name": "test_span",
-                    "start_time": 1000,
-                    "end_time": 2000,
-                }
-            )
-
-    def test_span_data_arbitrary_types_allowed(self):
-        # opentelemetry.trace.Status and Event are "arbitrary types" for Pydantic
-        # This test ensures they are accepted thanks to model_config
-        status = Status(StatusCode.ERROR, description="error occurred")
-        event = Event(name="exception", timestamp=1234, attributes={"exception.type": "ValueError"})
-
-        span_data = SpanData(
-            trace_id=123, span_id=456, name="test_span", status=status, events=[event], start_time=1000, end_time=2000
-        )
-
-        assert span_data.status.status_code == status.status_code
-        assert span_data.status.description == status.description
-        assert span_data.events == [event]

api/providers/trace/trace-aliyun/tests/unit_tests/src/dify_trace_aliyun/entities/test_semconv.py

@@ -1,101 +0,0 @@
-from dify_trace_aliyun.entities.semconv import (
-    ACS_ARMS_SERVICE_FEATURE,
-    GEN_AI_AGENT_NAME,
-    GEN_AI_COMPLETION,
-    GEN_AI_FRAMEWORK,
-    GEN_AI_INPUT_MESSAGE,
-    GEN_AI_OPERATION_NAME,
-    GEN_AI_OUTPUT_MESSAGE,
-    GEN_AI_PROMPT,
-    GEN_AI_PROVIDER_NAME,
-    GEN_AI_REACT_FINISH_REASON,
-    GEN_AI_REACT_ROUND,
-    GEN_AI_REQUEST_MODEL,
-    GEN_AI_RESPONSE_FINISH_REASON,
-    GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN,
-    GEN_AI_SESSION_ID,
-    GEN_AI_SKILL_DESCRIPTION,
-    GEN_AI_SKILL_ID,
-    GEN_AI_SKILL_NAME,
-    GEN_AI_SKILL_VERSION,
-    GEN_AI_SPAN_KIND,
-    GEN_AI_TOOL_CALL_ARGUMENTS,
-    GEN_AI_TOOL_CALL_ID,
-    GEN_AI_TOOL_CALL_RESULT,
-    GEN_AI_TOOL_DESCRIPTION,
-    GEN_AI_TOOL_NAME,
-    GEN_AI_TOOL_TYPE,
-    GEN_AI_USAGE_INPUT_TOKENS,
-    GEN_AI_USAGE_OUTPUT_TOKENS,
-    GEN_AI_USAGE_TOTAL_TOKENS,
-    GEN_AI_USER_ID,
-    GEN_AI_USER_NAME,
-    INPUT_VALUE,
-    OPERATION_NAME_EXECUTE_TOOL,
-    OUTPUT_VALUE,
-    RETRIEVAL_DOCUMENT,
-    RETRIEVAL_QUERY,
-    TOOL_TYPE_DATASTORE,
-    TOOL_TYPE_EXTENSION,
-    TOOL_TYPE_FUNCTION,
-    GenAISpanKind,
-)
-
-
-def test_constants():
-    assert ACS_ARMS_SERVICE_FEATURE == "acs.arms.service.feature"
-    assert GEN_AI_SESSION_ID == "gen_ai.session.id"
-    assert GEN_AI_USER_ID == "gen_ai.user.id"
-    assert GEN_AI_USER_NAME == "gen_ai.user.name"
-    assert GEN_AI_SPAN_KIND == "gen_ai.span.kind"
-    assert GEN_AI_FRAMEWORK == "gen_ai.framework"
-    assert INPUT_VALUE == "input.value"
-    assert OUTPUT_VALUE == "output.value"
-    assert RETRIEVAL_QUERY == "retrieval.query"
-    assert RETRIEVAL_DOCUMENT == "retrieval.document"
-    assert GEN_AI_REQUEST_MODEL == "gen_ai.request.model"
-    assert GEN_AI_PROVIDER_NAME == "gen_ai.provider.name"
-    assert GEN_AI_USAGE_INPUT_TOKENS == "gen_ai.usage.input_tokens"
-    assert GEN_AI_USAGE_OUTPUT_TOKENS == "gen_ai.usage.output_tokens"
-    assert GEN_AI_USAGE_TOTAL_TOKENS == "gen_ai.usage.total_tokens"
-    assert GEN_AI_PROMPT == "gen_ai.prompt"
-    assert GEN_AI_COMPLETION == "gen_ai.completion"
-    assert GEN_AI_RESPONSE_FINISH_REASON == "gen_ai.response.finish_reason"
-    assert GEN_AI_INPUT_MESSAGE == "gen_ai.input.messages"
-    assert GEN_AI_OUTPUT_MESSAGE == "gen_ai.output.messages"
-    assert GEN_AI_TOOL_CALL_ID == "gen_ai.tool.call.id"
-    assert GEN_AI_TOOL_DESCRIPTION == "gen_ai.tool.description"
-    assert GEN_AI_TOOL_NAME == "gen_ai.tool.name"
-    assert GEN_AI_TOOL_TYPE == "gen_ai.tool.type"
-    assert GEN_AI_TOOL_CALL_ARGUMENTS == "gen_ai.tool.call.arguments"
-    assert GEN_AI_TOOL_CALL_RESULT == "gen_ai.tool.call.result"
-    assert GEN_AI_SKILL_ID == "gen_ai.skill.id"
-    assert GEN_AI_SKILL_NAME == "gen_ai.skill.name"
-    assert GEN_AI_SKILL_DESCRIPTION == "gen_ai.skill.description"
-    assert GEN_AI_SKILL_VERSION == "gen_ai.skill.version"
-    assert GEN_AI_OPERATION_NAME == "gen_ai.operation.name"
-    assert OPERATION_NAME_EXECUTE_TOOL == "execute_tool"
-    assert TOOL_TYPE_FUNCTION == "function"
-    assert TOOL_TYPE_EXTENSION == "extension"
-    assert TOOL_TYPE_DATASTORE == "datastore"
-    assert GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN == "gen_ai.response.time_to_first_token"
-    assert GEN_AI_AGENT_NAME == "gen_ai.agent.name"
-    assert GEN_AI_REACT_ROUND == "gen_ai.react.round"
-    assert GEN_AI_REACT_FINISH_REASON == "gen_ai.react.finish_reason"
-
-
-def test_gen_ai_span_kind_enum():
-    assert GenAISpanKind.CHAIN == "CHAIN"
-    assert GenAISpanKind.RETRIEVER == "RETRIEVER"
-    assert GenAISpanKind.RERANKER == "RERANKER"
-    assert GenAISpanKind.LLM == "LLM"
-    assert GenAISpanKind.EMBEDDING == "EMBEDDING"
-    assert GenAISpanKind.TOOL == "TOOL"
-    assert GenAISpanKind.AGENT == "AGENT"
-    assert GenAISpanKind.TASK == "TASK"
-    assert GenAISpanKind.STEP == "STEP"
-
-    # Verify iteration works (covers the class definition)
-    kinds = list(GenAISpanKind)
-    assert len(kinds) == 9
-    assert "LLM" in kinds

api/providers/trace/trace-aliyun/tests/unit_tests/src/dify_trace_aliyun/test_config.py

@@ -1,85 +1,165 @@
+import traceback
+
 import pytest
 from dify_trace_aliyun.config import AliyunConfig
+from dify_trace_sdk.config import FormField
 from pydantic import ValidationError
 
+ENDPOINT = "http://tracing-analysis-dc-qd.aliyuncs.com"
+EMBEDDED_KEY = "DO-NOT-PRINT"
 
-class TestAliyunConfig:
-    """Test cases for AliyunConfig"""
 
-    def test_valid_config(self):
-        """Test valid Aliyun configuration"""
-        config = AliyunConfig(
-            app_name="test_app",
-            license_key="test_license_key",
-            endpoint="https://custom.tracing-analysis-dc-hz.aliyuncs.com",
-        )
-        assert config.app_name == "test_app"
-        assert config.license_key == "test_license_key"
-        assert config.endpoint == "https://custom.tracing-analysis-dc-hz.aliyuncs.com"
+def _config(**overrides: object) -> AliyunConfig:
+    return AliyunConfig.model_validate({"endpoint": ENDPOINT, "license_key": "synthetic-key", **overrides})
 
-    def test_default_values(self):
-        """Test default values are set correctly"""
-        config = AliyunConfig(license_key="test_license", endpoint="https://tracing-analysis-dc-hz.aliyuncs.com")
-        assert config.app_name == "dify_app"
 
-    def test_missing_required_fields(self):
-        """Test that required fields are enforced"""
-        with pytest.raises(ValidationError):
-            AliyunConfig.model_validate({})
+def _rendered(error: ValidationError) -> str:
+    return str(error) + repr(error) + "".join(traceback.format_exception(error))
 
-        with pytest.raises(ValidationError):
-            AliyunConfig.model_validate({"license_key": "test_license"})
 
-        with pytest.raises(ValidationError):
-            AliyunConfig.model_validate({"endpoint": "https://tracing-analysis-dc-hz.aliyuncs.com"})
+def test_form_declares_the_license_key_as_the_only_secret() -> None:
+    assert AliyunConfig.secret_fields() == {"license_key"}
+    assert AliyunConfig.derived_fields() == set()
+    assert AliyunConfig.form_fields() == (
+        FormField(name="app_name", secret=False, required=False),
+        FormField(name="license_key", secret=True, required=True),
+        FormField(name="endpoint", secret=False, required=True),
+    )
 
-    def test_app_name_validation_empty(self):
-        """Test app_name validation with empty value"""
-        config = AliyunConfig(
-            license_key="test_license", endpoint="https://tracing-analysis-dc-hz.aliyuncs.com", app_name=""
-        )
-        assert config.app_name == "dify_app"
 
-    def test_endpoint_validation_empty(self):
-        """Test endpoint validation with empty value"""
-        config = AliyunConfig(license_key="test_license", endpoint="")
-        assert config.endpoint == "https://tracing-analysis-dc-hz.aliyuncs.com"
+def test_stored_configuration_validates_unchanged() -> None:
+    stored = {
+        "app_name": "test_app",
+        "license_key": "test_license_key",
+        "endpoint": "https://custom.tracing-analysis-dc-hz.aliyuncs.com",
+    }
+    config = AliyunConfig.model_validate({**stored, "retired_field": "ignored"})
+    assert config.model_dump() == stored
+    assert AliyunConfig.model_validate(config.model_dump()) == config
 
-    def test_endpoint_validation_with_path(self):
-        """Test endpoint validation preserves path for Aliyun endpoints"""
-        config = AliyunConfig(
-            license_key="test_license", endpoint="https://tracing-analysis-dc-hz.aliyuncs.com/api/v1/traces"
-        )
-        assert config.endpoint == "https://tracing-analysis-dc-hz.aliyuncs.com/api/v1/traces"
 
-    def test_endpoint_validation_invalid_scheme(self):
-        """Test endpoint validation rejects invalid schemes"""
-        with pytest.raises(ValidationError, match="URL must start with https:// or http://"):
-            AliyunConfig(license_key="test_license", endpoint="ftp://invalid.tracing-analysis-dc-hz.aliyuncs.com")
+def test_normalization_keeps_the_license_key_and_application_name_opaque() -> None:
+    config = _config(
+        endpoint=" HTTPS://Project.CN-QINGDAO.LOG.ALIYUNCS.COM:443/apm/trace/opentelemetry/ ",
+        app_name=" 中文 agent_Name 01 ",
+        license_key="literal%2F@a+b/c",
+    )
+    assert config.endpoint == "https://project.cn-qingdao.log.aliyuncs.com:443/apm/trace/opentelemetry"
+    assert config.app_name == "中文 agent_Name 01"
+    assert config.license_key == "literal%2F@a+b/c"
 
-    def test_endpoint_validation_no_scheme(self):
-        """Test endpoint validation rejects URLs without scheme"""
-        with pytest.raises(ValidationError, match="URL must start with https:// or http://"):
-            AliyunConfig(license_key="test_license", endpoint="invalid.tracing-analysis-dc-hz.aliyuncs.com")
 
-    def test_license_key_required(self):
-        """Test that license_key is required and cannot be empty"""
-        with pytest.raises(ValidationError):
-            AliyunConfig(license_key="", endpoint="https://tracing-analysis-dc-hz.aliyuncs.com")
+@pytest.mark.parametrize("name", ["", "  ", "dify_app", "应用 & Space"])
+def test_empty_application_names_fall_back_to_the_default(name: str) -> None:
+    assert _config(app_name=name).app_name == (name.strip() or "dify_app")
 
-    def test_valid_endpoint_format_examples(self):
-        """Test valid endpoint format examples from comments"""
-        valid_endpoints = [
-            # cms2.0 public endpoint
-            "https://proj-xtrace-123456-cn-heyuan.cn-heyuan.log.aliyuncs.com/apm/trace/opentelemetry",
-            # cms2.0 intranet endpoint
-            "https://proj-xtrace-123456-cn-heyuan.cn-heyuan-intranet.log.aliyuncs.com/apm/trace/opentelemetry",
-            # xtrace public endpoint
-            "http://tracing-cn-heyuan.arms.aliyuncs.com",
-            # xtrace intranet endpoint
-            "http://tracing-cn-heyuan-internal.arms.aliyuncs.com",
-        ]
 
-        for endpoint in valid_endpoints:
-            config = AliyunConfig(license_key="test_license", endpoint=endpoint)
-            assert config.endpoint == endpoint
+def test_omitted_application_name_and_empty_endpoint_use_the_defaults() -> None:
+    config = _config(endpoint="")
+    assert config.app_name == "dify_app"
+    assert config.endpoint == "https://tracing-analysis-dc-hz.aliyuncs.com"
+
+
+@pytest.mark.parametrize(
+    "endpoint",
+    [
+        "https://proj-xtrace-123456-cn-heyuan.cn-heyuan.log.aliyuncs.com/apm/trace/opentelemetry",
+        "https://proj-xtrace-123456-cn-heyuan.cn-heyuan-intranet.log.aliyuncs.com/apm/trace/opentelemetry",
+        "http://tracing-cn-heyuan.arms.aliyuncs.com",
+        "http://tracing-cn-heyuan-internal.arms.aliyuncs.com",
+        "http://[::1]:4318/reverse",
+    ],
+)
+def test_documented_endpoint_formats_are_kept(endpoint: str) -> None:
+    assert _config(endpoint=endpoint).endpoint == endpoint
+
+
+def test_internationalized_hosts_are_ascii_encoded() -> None:
+    assert _config(endpoint="https://BÜCHER.example/collector/").endpoint == "https://xn--bcher-kva.example/collector"
+
+
+@pytest.mark.parametrize(
+    "endpoint",
+    [
+        "ftp://host.test",
+        "host.test",
+        "https:///missing",
+        "https://user:secret@host.test",
+        "https://host.test?",
+        "https://host.test#",
+        "https://host.test/path?key=DO-NOT-PRINT",
+        "https://host.test:bad",
+        "https://host.test:0",
+        "https://host.test:65536",
+        "https://host.test:",
+        "https://[invalid",
+        "\nhttps://host.test",
+        "https://ho\nst.test",
+        "https://host.test\\@elsewhere",
+        "https://host.test/a/../b",
+        "https://host.test/a/%2e%2e/b",
+        "https://host.test/a%2fb",
+        "https://host.test/%252e%252e",
+        "https://host.test/%zz",
+        "https://host.test/%ff",
+        "https://host.test/double//path",
+        "https://host.test/api/otlp/traces",
+        "https://host.test/v1/traces",
+    ],
+)
+def test_unsafe_or_completed_routes_are_rejected_without_echo(endpoint: str) -> None:
+    with pytest.raises(ValidationError) as caught:
+        _config(endpoint=endpoint)
+    rendered = _rendered(caught.value)
+    assert "Aliyun requires an HTTP(S) base URL" in rendered
+    assert "host.test" not in rendered
+    assert "DO-NOT-PRINT" not in rendered
+
+
+@pytest.mark.parametrize(
+    "endpoint",
+    [
+        f"https://tracing-analysis-dc-hz.aliyuncs.com/adapt_{EMBEDDED_KEY}@abc_{EMBEDDED_KEY}@def/api/otlp/traces",
+        f"https://host.test/reverse/adapt_{EMBEDDED_KEY}/api/v1/traces/",
+        f"https://host.test/%61dapt_{EMBEDDED_KEY}",
+    ],
+)
+def test_pasted_collector_urls_embedding_the_license_key_are_rejected_without_echo(endpoint: str) -> None:
+    with pytest.raises(ValidationError) as caught:
+        _config(license_key=EMBEDDED_KEY, endpoint=endpoint)
+    assert EMBEDDED_KEY not in _rendered(caught.value)
+
+
+@pytest.mark.parametrize("endpoint", [None, 1])
+def test_non_string_endpoints_are_rejected(endpoint: object) -> None:
+    with pytest.raises(ValidationError):
+        _config(endpoint=endpoint)
+
+
+@pytest.mark.parametrize("field", ["endpoint", "license_key"])
+def test_required_fields_are_not_invented(field: str) -> None:
+    stored = {"endpoint": ENDPOINT, "license_key": "synthetic-key"}
+    del stored[field]
+    with pytest.raises(ValidationError):
+        AliyunConfig.model_validate(stored)
+
+
+@pytest.mark.parametrize(
+    "key", ["", " ", " DO-NOT-PRINT", "DO-NOT-PRINT ", "DO-NOT\nPRINT", "DO-NOT\x00PRINT", "DO-NOT\ud800PRINT"]
+)
+def test_unusable_license_keys_are_rejected_without_echo(key: str) -> None:
+    with pytest.raises(ValidationError) as caught:
+        _config(license_key=key)
+    rendered = _rendered(caught.value)
+    assert "Aliyun requires a nonempty license key" in rendered
+    assert "DO-NOT" not in rendered
+
+
+def test_masked_license_keys_pass_validation_for_verify_and_send_to_refuse() -> None:
+    assert _config(license_key="***masked").license_key == "***masked"
+
+
+@pytest.mark.parametrize("name", ["app\nname", "app\x00name", "app\x7fname"])
+def test_application_names_with_control_characters_are_rejected(name: str) -> None:
+    with pytest.raises(ValidationError, match="application name without control characters"):
+        _config(app_name=name)

api/providers/trace/trace-aliyun/tests/unit_tests/src/dify_trace_aliyun/test_provider.py

@@ -0,0 +1,288 @@
+import traceback
+from collections.abc import Iterable
+from urllib.parse import unquote, urlsplit
+
+import pytest
+from dify_trace_aliyun.config import AliyunConfig
+from dify_trace_aliyun.provider import AliyunProvider
+from dify_trace_sdk.egress import EgressResponse
+from dify_trace_sdk.errors import (
+    EgressBlockedError,
+    EgressError,
+    ExportOutcomeUnknownError,
+    ExportRejectedError,
+    ExportRetryableError,
+    VerificationError,
+)
+from dify_trace_sdk.provider import SendResult
+from dify_trace_sdk.testing import (
+    MESSAGE_ID,
+    RecordedRequest,
+    RecordingEgress,
+    message_trace,
+    response,
+    tool_fragment,
+    workflow_trace,
+)
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+    ExportTraceServiceRequest,
+    ExportTraceServiceResponse,
+)
+from opentelemetry.proto.common.v1.common_pb2 import KeyValue
+from opentelemetry.proto.resource.v1.resource_pb2 import Resource
+from opentelemetry.proto.trace.v1.trace_pb2 import Span as OtlpSpan
+from opentelemetry.proto.trace.v1.trace_pb2 import Status
+
+LICENSE_KEY = "SYNTHETIC-LICENSE@key_1"
+CONFIG = AliyunConfig(
+    app_name="test_app", license_key=LICENSE_KEY, endpoint="https://tracing-analysis-dc-qd.aliyuncs.com/prefix"
+)
+INGESTION_URL = f"https://tracing-analysis-dc-qd.aliyuncs.com/prefix/adapt_{LICENSE_KEY}/api/otlp/traces"
+SUCCESS_ACK = response(200, b'"success"')
+
+
+def _provider(*responses: EgressResponse | Exception) -> tuple[AliyunProvider, RecordingEgress]:
+    egress = RecordingEgress(responses, grpc_target=None)
+    return AliyunProvider(egress), egress
+
+
+def _partial_success() -> bytes:
+    partial = ExportTraceServiceResponse()
+    partial.partial_success.rejected_spans = 1
+    return partial.SerializeToString()
+
+
+def _attributes(values: Iterable[KeyValue]) -> dict[str, object]:
+    decoded: dict[str, object] = {}
+    for item in values:
+        match item.value.WhichOneof("value"):
+            case "string_value":
+                decoded[item.key] = item.value.string_value
+            case "int_value":
+                decoded[item.key] = item.value.int_value
+            case "bool_value":
+                decoded[item.key] = item.value.bool_value
+            case "double_value":
+                decoded[item.key] = item.value.double_value
+            case other:
+                raise AssertionError(f"unexpected attribute type: {other}")
+    return decoded
+
+
+def _payload(sent: RecordedRequest) -> ExportTraceServiceRequest:
+    assert sent.method == "POST"
+    assert sent.sensitive_url is True
+    assert sent.headers["Content-Type"] == "application/x-protobuf"
+    assert sent.content is not None
+    return ExportTraceServiceRequest.FromString(sent.content)
+
+
+def _exported(egress: RecordingEgress) -> tuple[Resource, list[OtlpSpan]]:
+    (sent,) = egress.requests
+    assert sent.url == INGESTION_URL
+    (resource_spans,) = _payload(sent).resource_spans
+    (scope_spans,) = resource_spans.scope_spans
+    return resource_spans.resource, list(scope_spans.spans)
+
+
+def _rendered(error: Exception) -> str:
+    return str(error) + repr(error) + "".join(traceback.format_exception(error))
+
+
+@pytest.mark.parametrize("ack", [SUCCESS_ACK, response(200, "success")])
+def test_verify_posts_one_resource_without_spans_and_accepts_the_literal_ack(ack: EgressResponse) -> None:
+    provider, egress = _provider(ack)
+    assert provider.verify(CONFIG) == CONFIG
+
+    (sent,) = egress.requests
+    assert sent.url == INGESTION_URL
+    assert sent.headers == {"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"}
+    (resource_spans,) = _payload(sent).resource_spans
+    assert list(resource_spans.scope_spans) == []
+    assert _attributes(resource_spans.resource.attributes) == {"service.name": "test_app"}
+
+
+@pytest.mark.parametrize(
+    "outcome",
+    [
+        response(200, b""),
+        response(200, b"{}"),
+        response(200, b"success"),
+        response(200, b'"success"\n'),
+        response(200, b'"failure"'),
+        response(201, b'"success"'),
+        response(204, b""),
+        response(401, b"private-response"),
+        response(403, b"private-response"),
+        response(405, b"private-response"),
+        response(503, b"private-response"),
+        EgressError("refused", request_sent=False),
+        EgressError("read timeout", request_sent=True),
+        EgressBlockedError("private address"),
+    ],
+)
+def test_verify_failures_have_one_code_and_expose_neither_key_nor_body(outcome: EgressResponse | Exception) -> None:
+    provider, egress = _provider(outcome)
+    with pytest.raises(VerificationError) as caught:
+        provider.verify(CONFIG)
+    assert caught.value.code == "verification_failed"
+    rendered = _rendered(caught.value)
+    assert LICENSE_KEY not in rendered
+    assert "private-response" not in rendered
+    assert len(egress.requests) == 1
+
+
+def test_masked_license_keys_are_never_sent() -> None:
+    masked = AliyunConfig.model_validate({**CONFIG.model_dump(), "license_key": "SYNTHETIC***key"})
+    provider, egress = _provider()
+    with pytest.raises(VerificationError) as caught:
+        provider.verify(masked)
+    assert caught.value.code == "invalid_license_key"
+    with pytest.raises(ExportRejectedError):
+        provider.send(masked, message_trace())
+    assert egress.requests == []
+
+
+def test_project_url_is_the_llm_console() -> None:
+    provider, _ = _provider()
+    assert provider.project_url(CONFIG) == "https://arms.console.aliyun.com/#/llm"
+
+
+@pytest.mark.parametrize(
+    ("endpoint", "expected"),
+    [
+        ("https://log.aliyuncs.com", "https://log.aliyuncs.com/adapt_k/api/v1/traces"),
+        (
+            "https://CN-QINGDAO.LOG.ALIYUNCS.COM.:443/prefix/",
+            "https://cn-qingdao.log.aliyuncs.com.:443/prefix/adapt_k/api/v1/traces",
+        ),
+        ("https://log.aliyuncs.com.evil.test", "https://log.aliyuncs.com.evil.test/adapt_k/api/otlp/traces"),
+        ("https://evil-log.aliyuncs.com", "https://evil-log.aliyuncs.com/adapt_k/api/otlp/traces"),
+        ("https://collector.test/log.aliyuncs.com", "https://collector.test/log.aliyuncs.com/adapt_k/api/otlp/traces"),
+        ("http://[::1]:4318/reverse/", "http://[::1]:4318/reverse/adapt_k/api/otlp/traces"),
+    ],
+)
+def test_collector_route_depends_only_on_the_hostname(endpoint: str, expected: str) -> None:
+    provider, egress = _provider(SUCCESS_ACK)
+    provider.send(AliyunConfig(license_key="k", endpoint=endpoint), message_trace())
+    (sent,) = egress.requests
+    assert sent.url == expected
+
+
+@pytest.mark.parametrize("key", ["raw%2F", "raw%252F", "a/b?c#d", "left@right_one+two", "密钥"])
+def test_license_key_is_encoded_exactly_once_as_one_path_segment(key: str) -> None:
+    provider, egress = _provider(SUCCESS_ACK)
+    provider.verify(AliyunConfig(license_key=key, endpoint="https://collector.test"))
+    (sent,) = egress.requests
+    path = urlsplit(sent.url).path
+    assert path.startswith("/adapt_")
+    assert path.endswith("/api/otlp/traces")
+    encoded = path.removeprefix("/adapt_").removesuffix("/api/otlp/traces")
+    assert "/" not in encoded
+    assert unquote(encoded) == key
+
+
+def test_message_trace_is_exported_with_aliyun_genai_attributes() -> None:
+    provider, egress = _provider(SUCCESS_ACK)
+    trace = message_trace()
+    assert provider.send(CONFIG, trace) == SendResult(trace_id=trace.trace_id, trace_url=None, span_urls={})
+
+    resource, (root, llm) = _exported(egress)
+    assert _attributes(resource.attributes) == {"service.name": "test_app", "acs.arms.service.feature": "genai_app"}
+    assert root.trace_id.hex() == llm.trace_id.hex() == trace.trace_id == MESSAGE_ID.replace("-", "")
+    assert (root.span_id.hex(), root.parent_span_id) == (trace.spans[0].span_id, b"")
+    assert (llm.span_id.hex(), llm.parent_span_id.hex()) == (trace.spans[1].span_id, trace.spans[0].span_id)
+    assert (root.name, llm.name) == ("message", "llm")
+    assert root.status.code == llm.status.code == Status.STATUS_CODE_OK
+    root_attributes = _attributes(root.attributes)
+    assert (
+        root_attributes.items()
+        >= {
+            "gen_ai.span.kind": "CHAIN",
+            "gen_ai.framework": "dify",
+            "gen_ai.session.id": "conversation-1",
+            "gen_ai.user.id": "end-user-1",
+            "input.value": '{"query": "What is Dify?"}',
+            "output.value": '"An LLM application platform."',
+            "dify.app_id": "app-1",
+            "dify.tenant_id": "tenant-1",
+            "dify.message_id": MESSAGE_ID,
+        }.items()
+    )
+    assert "gen_ai.is_entry" not in root_attributes
+    assert (
+        _attributes(llm.attributes).items()
+        >= {
+            "gen_ai.span.kind": "LLM",
+            "gen_ai.operation.name": "chat",
+            "gen_ai.request.model": "gpt-4o",
+            "gen_ai.provider.name": "openai",
+            "gen_ai.usage.input_tokens": 12,
+            "gen_ai.usage.output_tokens": 8,
+            "gen_ai.usage.total_tokens": 20,
+            "gen_ai.response.time_to_first_token": 250_000_000,
+            "dify.app_id": "app-1",
+        }.items()
+    )
+
+
+def test_workflow_trace_keeps_hierarchy_kinds_and_errors() -> None:
+    provider, egress = _provider(SUCCESS_ACK)
+    trace = workflow_trace()
+    provider.send(CONFIG, trace)
+
+    _, (run, llm, tool) = _exported(egress)
+    assert [span.span_id.hex() for span in (run, llm, tool)] == [span.span_id for span in trace.spans]
+    assert (llm.parent_span_id.hex(), tool.parent_span_id.hex()) == (run.span_id.hex(), llm.span_id.hex())
+    assert [_attributes(span.attributes)["gen_ai.span.kind"] for span in (run, llm, tool)] == ["CHAIN", "LLM", "TOOL"]
+    assert (run.status.code, run.status.message) == (Status.STATUS_CODE_ERROR, "Tool failed")
+    assert (tool.status.code, tool.status.message) == (Status.STATUS_CODE_ERROR, "Tool failed")
+    assert llm.status.code == Status.STATUS_CODE_OK
+    assert _attributes(tool.attributes)["gen_ai.tool.name"] == "tool_Search"
+
+
+def test_fragment_root_points_at_the_separately_exported_message() -> None:
+    provider, egress = _provider(SUCCESS_ACK)
+    fragment = tool_fragment()
+    provider.send(CONFIG, fragment)
+
+    _, (tool,) = _exported(egress)
+    assert fragment.parent is not None
+    assert tool.trace_id.hex() == message_trace().trace_id
+    assert tool.parent_span_id.hex() == fragment.parent.span_id == message_trace().root.span_id
+    assert _attributes(tool.attributes)["gen_ai.span.kind"] == "TOOL"
+
+
+@pytest.mark.parametrize(
+    "ack",
+    [response(200, b""), response(200, ExportTraceServiceResponse().SerializeToString()), response(200, {})],
+)
+def test_standard_otlp_acknowledgments_are_accepted(ack: EgressResponse) -> None:
+    provider, _ = _provider(ack)
+    provider.send(CONFIG, message_trace())
+
+
+@pytest.mark.parametrize(
+    ("outcome", "expected"),
+    [
+        (response(200, _partial_success()), ExportOutcomeUnknownError),
+        (response(200, b"not-protobuf-\xff"), ExportOutcomeUnknownError),
+        (response(200, "failure"), ExportOutcomeUnknownError),
+        (response(201, b'"success"'), ExportOutcomeUnknownError),
+        (response(503, b"private-response"), ExportRetryableError),
+        (response(429, b"private-response"), ExportRetryableError),
+        (response(403, b"private-response"), ExportRejectedError),
+        (EgressError("refused", request_sent=False), ExportRetryableError),
+        (EgressError("read timeout", request_sent=True), ExportOutcomeUnknownError),
+        (EgressBlockedError("private address"), ExportRejectedError),
+    ],
+)
+def test_send_failures_are_classified_without_exposing_the_key(
+    outcome: EgressResponse | Exception, expected: type[Exception]
+) -> None:
+    provider, _ = _provider(outcome)
+    with pytest.raises(expected) as caught:
+        provider.send(CONFIG, message_trace())
+    rendered = _rendered(caught.value)
+    assert LICENSE_KEY not in rendered
+    assert "private-response" not in rendered

api/providers/trace/trace-aliyun/tests/unit_tests/src/dify_trace_aliyun/test_utils.py

@@ -1,316 +0,0 @@
-"""Unit tests for Aliyun trace utility transformations and database lookups."""
-
-import json
-from collections.abc import Mapping
-from typing import Any, cast
-from unittest.mock import MagicMock
-
-import pytest
-from dify_trace_aliyun.entities.semconv import (
-    GEN_AI_FRAMEWORK,
-    GEN_AI_OPERATION_NAME,
-    GEN_AI_SESSION_ID,
-    GEN_AI_SPAN_KIND,
-    GEN_AI_TOOL_CALL_ARGUMENTS,
-    GEN_AI_TOOL_NAME,
-    GEN_AI_TOOL_TYPE,
-    GEN_AI_USER_ID,
-    INPUT_VALUE,
-    OPERATION_NAME_EXECUTE_TOOL,
-    OUTPUT_VALUE,
-    TOOL_TYPE_DATASTORE,
-    TOOL_TYPE_EXTENSION,
-    TOOL_TYPE_FUNCTION,
-)
-from dify_trace_aliyun.utils import (
-    create_common_span_attributes,
-    create_gen_ai_tool_attributes,
-    create_links_from_trace_id,
-    create_status_from_error,
-    extract_retrieval_documents,
-    extract_tool_description,
-    format_input_messages,
-    format_output_messages,
-    format_retrieval_documents,
-    get_user_id_from_message_data,
-    get_workflow_node_status,
-    map_gen_ai_tool_type,
-    serialize_json_data,
-)
-from opentelemetry.trace import Link, SpanContext, StatusCode
-from sqlalchemy.orm import Session
-
-from core.rag.models.document import Document
-from graphon.enums import WorkflowNodeExecutionStatus
-from models import EndUser
-from models.enums import EndUserType
-from tests.helpers.unit.core.ops.trace_fixtures import workflow_node_execution
-
-
-def test_get_user_id_from_message_data_no_end_user(monkeypatch: pytest.MonkeyPatch):
-    message_data = MagicMock()
-    message_data.from_account_id = "account_id"
-    message_data.from_end_user_id = None
-
-    assert get_user_id_from_message_data(message_data) == "account_id"
-
-
-@pytest.mark.parametrize("sqlite3_session", [(EndUser,)], indirect=True)
-def test_get_user_id_from_message_data_with_end_user(monkeypatch: pytest.MonkeyPatch, sqlite3_session: Session) -> None:
-    message_data = MagicMock()
-    message_data.from_account_id = "account_id"
-    message_data.from_end_user_id = "end_user_id"
-
-    end_user_data = EndUser(
-        id="end_user_id",
-        tenant_id="tenant_id",
-        app_id="app_id",
-        type=EndUserType.BROWSER,
-        session_id="session_id",
-    )
-    sqlite3_session.add(end_user_data)
-    sqlite3_session.commit()
-
-    from dify_trace_aliyun.utils import db
-
-    monkeypatch.setattr(db, "session", sqlite3_session)
-
-    assert get_user_id_from_message_data(message_data) == "session_id"
-
-
-@pytest.mark.parametrize("sqlite3_session", [(EndUser,)], indirect=True)
-def test_get_user_id_from_message_data_end_user_not_found(
-    monkeypatch: pytest.MonkeyPatch, sqlite3_session: Session
-) -> None:
-    message_data = MagicMock()
-    message_data.from_account_id = "account_id"
-    message_data.from_end_user_id = "end_user_id"
-
-    from dify_trace_aliyun.utils import db
-
-    monkeypatch.setattr(db, "session", sqlite3_session)
-
-    assert get_user_id_from_message_data(message_data) == "account_id"
-
-
-def test_create_status_from_error():
-    # Case OK
-    status_ok = create_status_from_error(None)
-    assert status_ok.status_code == StatusCode.OK
-
-    # Case Error
-    status_err = create_status_from_error("some error")
-    assert status_err.status_code == StatusCode.ERROR
-    assert status_err.description == "some error"
-
-
-def test_get_workflow_node_status():
-    node_execution = workflow_node_execution()
-
-    # SUCCEEDED
-    node_execution.status = WorkflowNodeExecutionStatus.SUCCEEDED
-    status = get_workflow_node_status(node_execution)
-    assert status.status_code == StatusCode.OK
-
-    # FAILED
-    node_execution.status = WorkflowNodeExecutionStatus.FAILED
-    node_execution.error = "node fail"
-    status = get_workflow_node_status(node_execution)
-    assert status.status_code == StatusCode.ERROR
-    assert status.description == "node fail"
-
-    # EXCEPTION
-    node_execution.status = WorkflowNodeExecutionStatus.EXCEPTION
-    node_execution.error = "node exception"
-    status = get_workflow_node_status(node_execution)
-    assert status.status_code == StatusCode.ERROR
-    assert status.description == "node exception"
-
-    # UNSET/OTHER
-    node_execution.status = WorkflowNodeExecutionStatus.RUNNING
-    status = get_workflow_node_status(node_execution)
-    assert status.status_code == StatusCode.UNSET
-
-
-def test_create_links_from_trace_id(monkeypatch: pytest.MonkeyPatch):
-    # Mock create_link
-    link = Link(SpanContext(trace_id=1, span_id=2, is_remote=False))
-    import dify_trace_aliyun.data_exporter.traceclient
-
-    monkeypatch.setattr(dify_trace_aliyun.data_exporter.traceclient, "create_link", lambda trace_id_str: link)
-
-    # Trace ID None
-    assert create_links_from_trace_id(None) == []
-
-    # Trace ID Present
-    links = create_links_from_trace_id("trace_id")
-    assert len(links) == 1
-    assert links[0] == link
-
-
-def test_extract_retrieval_documents():
-    doc1 = Document(
-        page_content="content1",
-        metadata={"dataset_id": "ds1", "doc_id": "di1", "document_id": "dd1", "score": 0.9},
-    )
-    doc2 = Document(page_content="content2", metadata={"dataset_id": "ds2"})  # Missing some keys
-
-    documents = [doc1, doc2]
-    extracted = extract_retrieval_documents(documents)
-
-    assert len(extracted) == 2
-    assert extracted[0]["content"] == "content1"
-    assert extracted[0]["metadata"]["dataset_id"] == "ds1"
-    assert extracted[0]["score"] == 0.9
-
-    assert extracted[1]["content"] == "content2"
-    assert extracted[1]["metadata"]["dataset_id"] == "ds2"
-    assert extracted[1]["metadata"]["doc_id"] is None
-    assert extracted[1]["score"] is None
-
-
-def test_serialize_json_data():
-    data = {"a": 1}
-    # Test ensure_ascii default (False)
-    assert serialize_json_data(data) == json.dumps(data, ensure_ascii=False)
-    # Test ensure_ascii True
-    assert serialize_json_data(data, ensure_ascii=True) == json.dumps(data, ensure_ascii=True)
-
-
-def test_create_common_span_attributes():
-    attrs = create_common_span_attributes(
-        session_id="s1", user_id="u1", span_kind="kind1", framework="fw1", inputs="in1", outputs="out1"
-    )
-    assert attrs[GEN_AI_SESSION_ID] == "s1"
-    assert attrs[GEN_AI_USER_ID] == "u1"
-    assert attrs[GEN_AI_SPAN_KIND] == "kind1"
-    assert attrs[GEN_AI_FRAMEWORK] == "fw1"
-    assert attrs[INPUT_VALUE] == "in1"
-    assert attrs[OUTPUT_VALUE] == "out1"
-
-
-def test_format_retrieval_documents():
-    # Not a list
-    assert format_retrieval_documents(cast(list[object], "not a list")) == []
-
-    # Valid list
-    docs = [
-        {"metadata": {"score": 0.8, "document_id": "doc1", "source": "src1"}, "content": "c1", "title": "t1"},
-        {
-            "metadata": {"_source": "src2", "doc_metadata": {"extra": "val"}},
-            "content": "c2",
-            # Missing title
-        },
-        "not a dict",  # Should be skipped
-    ]
-    formatted = format_retrieval_documents(docs)
-
-    assert len(formatted) == 2
-    assert formatted[0]["document"]["content"] == "c1"
-    assert formatted[0]["document"]["metadata"]["title"] == "t1"
-    assert formatted[0]["document"]["metadata"]["source"] == "src1"
-    assert formatted[0]["document"]["score"] == 0.8
-    assert formatted[0]["document"]["id"] == "doc1"
-
-    assert formatted[1]["document"]["content"] == "c2"
-    assert formatted[1]["document"]["metadata"]["source"] == "src2"
-    assert formatted[1]["document"]["metadata"]["extra"] == "val"
-    assert "title" not in formatted[1]["document"]["metadata"]
-    assert formatted[1]["document"]["score"] == 0.0  # Default
-
-    # Exception handling
-    # We can trigger an exception by passing something that causes an error in the loop logic,
-    # but the try/except covers the whole function.
-    # Passing a list that contains something that throws when calling .get() - though dicts won't.
-    # Let's mock a dict that raises on get.
-    class BadDict:
-        def get(self, *args, **kwargs):
-            raise Exception("boom")
-
-    assert format_retrieval_documents([BadDict()]) == []
-
-
-def test_format_input_messages():
-    # Not a dict
-    assert format_input_messages(cast(Mapping[str, Any], None)) == serialize_json_data([])
-
-    # No prompts
-    assert format_input_messages({}) == serialize_json_data([])
-
-    # Valid prompts
-    process_data = {
-        "prompts": [
-            {"role": "user", "text": "hello"},
-            {"role": "assistant", "text": "hi"},
-            {"role": "system", "text": "be helpful"},
-            {"role": "tool", "text": "result"},
-            {"role": "invalid", "text": "skip me"},
-            "not a dict",
-            {"role": "user", "text": ""},  # Empty text, should be skipped? Code says `if text: message = ...`
-        ]
-    }
-    result = format_input_messages(process_data)
-    result_list = json.loads(result)
-
-    assert len(result_list) == 4
-    assert result_list[0]["role"] == "user"
-    assert result_list[0]["parts"][0]["content"] == "hello"
-    assert result_list[1]["role"] == "assistant"
-    assert result_list[2]["role"] == "system"
-    assert result_list[3]["role"] == "tool"
-
-    # Exception path
-    assert format_input_messages({"prompts": [None]}) == serialize_json_data([])
-
-
-def test_format_output_messages():
-    # Not a dict
-    assert format_output_messages(cast(Mapping[str, Any], None)) == serialize_json_data([])
-
-    # No text
-    assert format_output_messages({"finish_reason": "stop"}) == serialize_json_data([])
-
-    # Valid
-    outputs = {"text": "done", "finish_reason": "length"}
-    result = format_output_messages(outputs)
-    result_list = json.loads(result)
-    assert len(result_list) == 1
-    assert result_list[0]["role"] == "assistant"
-    assert result_list[0]["parts"][0]["content"] == "done"
-    assert result_list[0]["finish_reason"] == "length"
-
-    # Invalid finish reason
-    outputs2 = {"text": "done", "finish_reason": "unknown"}
-    result2 = format_output_messages(outputs2)
-    result_list2 = json.loads(result2)
-    assert result_list2[0]["finish_reason"] == "stop"
-
-    # Exception path
-    # Trigger exception in serialize_json_data by passing non-serializable
-    assert format_output_messages({"text": MagicMock()}) == serialize_json_data([])
-
-
-def test_map_gen_ai_tool_type():
-    assert map_gen_ai_tool_type("dataset-retrieval") == TOOL_TYPE_DATASTORE
-    assert map_gen_ai_tool_type("extension") == TOOL_TYPE_EXTENSION
-    assert map_gen_ai_tool_type("builtin") == TOOL_TYPE_FUNCTION
-    assert map_gen_ai_tool_type(None) == TOOL_TYPE_FUNCTION
-
-
-def test_extract_tool_description():
-    assert extract_tool_description({"description": "d"}) == "d"
-    assert extract_tool_description({"tool_description": "td"}) == "td"
-    assert extract_tool_description({"other": 1}) == ""
-    assert extract_tool_description(None) == ""
-
-
-def test_create_gen_ai_tool_attributes():
-    attrs = create_gen_ai_tool_attributes(
-        tool_name="search",
-        tool_type=TOOL_TYPE_DATASTORE,
-        tool_call_arguments='{"q": 1}',
-    )
-    assert attrs[GEN_AI_OPERATION_NAME] == OPERATION_NAME_EXECUTE_TOOL
-    assert attrs[GEN_AI_TOOL_NAME] == "search"
-    assert attrs[GEN_AI_TOOL_TYPE] == TOOL_TYPE_DATASTORE
-    assert attrs[GEN_AI_TOOL_CALL_ARGUMENTS] == '{"q": 1}'

api/providers/trace/trace-arize-phoenix/pyproject.toml

@@ -2,9 +2,13 @@
 name = "dify-trace-arize-phoenix"
 version = "0.0.1"
 dependencies = [
-    "arize-phoenix-otel==0.15.0",
+    "dify-trace-sdk",
 ]
 description = "Dify ops tracing provider (Arize / Phoenix)."
 
+[project.entry-points."dify.trace_providers"]
+arize = "dify_trace_arize_phoenix.provider:ArizeProvider"
+phoenix = "dify_trace_arize_phoenix.provider:PhoenixProvider"
+
 [tool.setuptools.packages.find]
 where = ["src"]

api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/_arize_project.py

@@ -0,0 +1,147 @@
+"""The link to an Arize Cloud project: looked up by ``verify``, then built without I/O.
+
+Arize Cloud addresses a project by organization, space, and project IDs, while
+a configuration names it. The lookup runs only for the default Cloud collector,
+so credentials for a custom or regional collector never reach an assumed API
+host. It is optional: a key may ingest without read access, so every failure
+means no link rather than a failed verification.
+"""
+
+import re
+import time
+from collections.abc import Mapping
+from typing import TypeIs
+from urllib.parse import quote, urlencode, urlsplit
+
+from dify_trace_sdk.egress import Egress
+from dify_trace_sdk.errors import EgressError
+
+from dify_trace_arize_phoenix.config import ArizeConfig
+
+API_BASE = "https://api.arize.com/v2"
+UI_BASE = "https://app.arize.com"
+CLOUD_COLLECTOR_HOST = "otlp.arize.com"
+LOOKUP_BUDGET_SECONDS = 5.0
+"""Shared by every lookup request, because a user waits for verification."""
+MAX_RESPONSE_BYTES = 1024 * 1024
+PAGE_LIMIT = 100
+MAX_PAGES = 3
+MAX_CURSOR_LENGTH = 4096
+_OPAQUE_ID = re.compile(r"[A-Za-z0-9+/_=-]{1,1024}")
+
+
+def _opaque_id(value: object) -> TypeIs[str]:
+    # Never decode IDs (e.g. to derive an organization from a space) or accept URL metacharacters.
+    return isinstance(value, str) and _OPAQUE_ID.fullmatch(value) is not None
+
+
+def _cloud_collector(endpoint: str) -> bool:
+    parsed = urlsplit(endpoint)
+    try:
+        port = parsed.port
+    except ValueError:
+        return False
+    return parsed.scheme == "https" and parsed.hostname == CLOUD_COLLECTOR_HOST and port in {None, 443}
+
+
+def _identity(config: ArizeConfig) -> dict[str, str] | None:
+    """What a destination is bound to, or ``None`` when the collector is not the default Cloud one."""
+    if not _cloud_collector(config.endpoint) or not _opaque_id(config.space_id):
+        return None
+    return {"endpoint": config.endpoint, "space_id": config.space_id, "project": config.project}
+
+
+def project_url(config: ArizeConfig) -> str | None:
+    """Built from the stored destination alone: presentation reads the space ID still encrypted.
+
+    Verification resolves the destination again on every save, so a destination
+    that names another endpoint or project only remains from a manual edit.
+    """
+    destination = config.arize_project_destination
+    if destination is None or not _cloud_collector(config.endpoint):
+        return None
+    if destination.get("endpoint") != config.endpoint or destination.get("project") != config.project:
+        return None
+    organization_id, space_id, project_id = (
+        destination.get("organization_id"),
+        destination.get("space_id"),
+        destination.get("project_id"),
+    )
+    if not _opaque_id(organization_id) or not _opaque_id(space_id) or not _opaque_id(project_id):
+        return None
+    return (
+        f"{UI_BASE}/organizations/{quote(organization_id, safe='')}"
+        f"/spaces/{quote(space_id, safe='')}/projects/{quote(project_id, safe='')}"
+    )
+
+
+def resolve_destination(egress: Egress, config: ArizeConfig) -> dict[str, str] | None:
+    """Find the one project of the space named exactly ``config.project``.
+
+    The API's name filter matches substrings, so up to ``MAX_PAGES`` pages are
+    read for an exact match. Denied reads, malformed or ambiguous answers, and
+    an exhausted budget all mean no destination.
+    """
+    identity = _identity(config)
+    if identity is None:
+        return None
+    deadline = time.monotonic() + LOOKUP_BUDGET_SECONDS
+    headers = {"Authorization": f"Bearer {config.api_key}", "Accept": "application/json"}
+
+    def read(path: str) -> Mapping[str, object] | None:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            return None
+        try:
+            # Every lookup URL embeds the space ID, which is a secret field.
+            response = egress.request(
+                "GET", API_BASE + path, headers=headers, content=None, timeout=remaining, sensitive_url=True
+            )
+        except EgressError:
+            return None
+        if response.status_code != 200 or len(response.content) > MAX_RESPONSE_BYTES or time.monotonic() > deadline:
+            return None
+        try:
+            body = response.json()
+        except (ValueError, RecursionError):
+            return None
+        return body if isinstance(body, dict) else None
+
+    space = read(f"/spaces/{quote(config.space_id, safe='')}")
+    organization_id = space.get("organization_id") if space is not None else None
+    if space is None or space.get("id") != config.space_id or not _opaque_id(organization_id):
+        return None
+    project_ids: set[str] = set()
+    seen_cursors: set[str] = set()
+    query: dict[str, str | int] = {"space_id": config.space_id, "name": config.project, "limit": PAGE_LIMIT}
+    for _ in range(MAX_PAGES):
+        page = read("/projects?" + urlencode(query, quote_via=quote))
+        projects = page.get("projects") if page is not None else None
+        pagination = page.get("pagination") if page is not None else None
+        if not isinstance(projects, list) or len(projects) > PAGE_LIMIT or not isinstance(pagination, dict):
+            return None
+        for project in projects:
+            if not isinstance(project, dict) or project.get("space_id") != config.space_id:
+                return None
+            project_id, name = project.get("id"), project.get("name")
+            if not _opaque_id(project_id) or not isinstance(name, str):
+                return None
+            if name == config.project:
+                project_ids.add(project_id)
+        if len(project_ids) > 1:
+            return None
+        has_more, cursor = pagination.get("has_more"), pagination.get("next_cursor")
+        if has_more is False:
+            if len(project_ids) != 1:
+                return None
+            return {**identity, "organization_id": organization_id, "project_id": project_ids.pop()}
+        if (
+            has_more is not True
+            or not isinstance(cursor, str)
+            or not 0 < len(cursor) <= MAX_CURSOR_LENGTH
+            or cursor in seen_cursors
+        ):
+            return None
+        seen_cursors.add(cursor)
+        query["cursor"] = cursor
+    return None

api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/agent_trace.py

@@ -1,54 +0,0 @@
-"""Stable-ID OpenInference mapping for Phoenix HTTP and Arize gRPC ingestion."""
-
-from collections.abc import Mapping
-from typing import Any
-
-from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalTrace
-from services.tracing.otlp_export import build_request, export_grpc, export_http
-from services.tracing.provider_common import endpoint, json_value, token_usage
-from services.tracing.provider_dispatch import TraceExportResult
-
-
-def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
-    url = endpoint(
-        config.get("endpoint", "https://otlp.arize.com" if provider == "arize" else "https://app.phoenix.arize.com")
-    )
-    api_key = str(config.get("api_key") or "")
-    project = str(config.get("project") or "default")
-
-    def attributes(span: CanonicalSpan) -> dict[str, object]:
-        result: dict[str, object] = {
-            "openinference.span.kind": span.kind.value.upper(),
-            "input.value": json_value(span.inputs),
-            "input.mime_type": "application/json",
-            "output.value": json_value(span.outputs),
-            "output.mime_type": "application/json",
-            "metadata": json_value(span.metadata),
-            "session.id": trace.session_id,
-            "dify.span.id": span.id,
-        }
-        if span.kind is CanonicalSpanKind.LLM:
-            model = span.metadata.get("model") or span.metadata.get("model_name")
-            if model:
-                result["llm.model_name"] = str(model)
-            if span.metadata.get("model_provider"):
-                result["llm.provider"] = str(span.metadata["model_provider"])
-            names = {"input": "prompt", "output": "completion", "total": "total"}
-            result.update({f"llm.token_count.{names[key]}": value for key, value in token_usage(span).items()})
-        elif span.kind is CanonicalSpanKind.TOOL:
-            result["tool.name"] = span.name
-            result["tool.parameters"] = json_value(span.inputs)
-        return result
-
-    payload = build_request(
-        trace,
-        {"service.name": "dify-application", "openinference.project.name": project, "model_id": project},
-        attributes,
-    )
-    headers = {"api_key": api_key, "authorization": f"Bearer {api_key}"} if api_key or provider == "arize" else {}
-    if provider == "arize":
-        headers["space_id"] = str(config.get("space_id") or "")
-        export_grpc(provider, url, headers, payload)
-    else:
-        export_http(provider, f"{url}/v1/traces", headers, payload)
-    return TraceExportResult(trace_id=trace.trace_id)

api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_configuration.py

@@ -1,304 +0,0 @@
-"""Optional Arize Cloud project metadata, resolved only on explicit config writes.
-
-Ingestion verification is separate from metadata reads. A key without read access
-may still ingest successfully, so an unavailable project link is never a reason
-to reject that configuration. Readers use only the frozen destination cache.
-"""
-
-import hashlib
-import json
-import re
-import time
-from collections.abc import Mapping
-from typing import Literal
-from urllib.parse import quote, urlsplit
-
-from pydantic import JsonValue, TypeAdapter
-
-from core.helper import ssrf_proxy
-from dify_trace_arize_phoenix.config import ArizeConfig
-
-DESTINATION_KEY = "arize_project_destination"
-_API_BASE = "https://api.arize.com/v2"
-_UI_BASE = "https://app.arize.com"
-_BUDGET_SECONDS = 5.0
-_MAX_RESPONSE_BYTES = 1024 * 1024
-_PAGE_LIMIT = 100
-_MAX_PAGES = 3
-_JSON_OBJECT = TypeAdapter(dict[str, JsonValue])
-_VERIFY_TIMEOUT_SECONDS = 5.0
-
-type VerificationCode = Literal[
-    "invalid_configuration", "authentication_failed", "permission_denied", "connection_failed", "unexpected_response"
-]
-
-
-class ArizeConfigurationError(ValueError):
-    """Fixed diagnostics never contain credentials, vendor details or endpoints."""
-
-    def __init__(self, code: VerificationCode) -> None:
-        self.code = code
-        super().__init__(f"Arize ingestion verification failed: {code}")
-
-
-def verify_configuration(config: ArizeConfig) -> None:
-    """Require one synchronous OTLP ACK, without creating any span or project.
-
-    The default Arize collector was verified to authenticate this resource-only
-    request for both valid and invalid credentials. It establishes this ingest
-    protocol's access, not REST read permission or a project's eventual creation.
-    No SDK exporter, span processor or global/environment credentials are used.
-    """
-    import grpc  # type: ignore[import-untyped]
-    from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value
-    from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
-    from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import TraceServiceStub
-
-    try:
-        parsed = urlsplit(config.endpoint)
-        host, port = parsed.hostname, parsed.port
-        if (
-            parsed.scheme not in {"http", "https"}
-            or not host
-            or parsed.username is not None
-            or parsed.password is not None
-            or (port is not None and not 1 <= port <= 65535)
-        ):
-            raise ArizeConfigurationError("invalid_configuration")
-        for value in (config.api_key, config.space_id):
-            if (
-                not isinstance(value, str)
-                or not value
-                or "*" in value
-                or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
-            ):
-                raise ArizeConfigurationError("invalid_configuration")
-    except ValueError:
-        raise ArizeConfigurationError("invalid_configuration") from None
-
-    payload = ExportTraceServiceRequest()
-    resource = payload.resource_spans.add().resource
-    for name in ("model_id", "openinference.project.name"):
-        resource.attributes.add(key=name).value.string_value = config.project or "default"
-    headers = (
-        ("api_key", config.api_key),
-        ("space_id", config.space_id),
-        ("authorization", f"Bearer {config.api_key}"),
-    )
-    authority = f"[{host}]" if ":" in host else host
-    target = f"{authority}:{port or 443}"
-    options = (("grpc.enable_retries", 0), ("grpc.max_receive_message_length", _MAX_RESPONSE_BYTES))
-    channel: grpc.Channel | None = None
-    context = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
-    try:
-        try:
-            channel = (
-                grpc.insecure_channel(target, options=options)
-                if parsed.scheme == "http"
-                else grpc.secure_channel(target, grpc.ssl_channel_credentials(), options=options)
-            )
-            result = TraceServiceStub(channel).Export(
-                payload, metadata=headers, timeout=_VERIFY_TIMEOUT_SECONDS, wait_for_ready=False
-            )
-            if result.partial_success.rejected_spans or result.partial_success.error_message:
-                raise ArizeConfigurationError("unexpected_response")
-        finally:
-            if channel is not None:
-                channel.close()
-    except ArizeConfigurationError:
-        raise
-    except grpc.RpcError as error:
-        code = error.code()
-        if code is grpc.StatusCode.UNAUTHENTICATED:
-            raise ArizeConfigurationError("authentication_failed") from None
-        if code is grpc.StatusCode.PERMISSION_DENIED:
-            raise ArizeConfigurationError("permission_denied") from None
-        raise ArizeConfigurationError("connection_failed") from None
-    except Exception:
-        raise ArizeConfigurationError("connection_failed") from None
-    finally:
-        detach(context)
-
-
-def _opaque_id(value: object) -> bool:
-    # IDs remain opaque. In particular, never derive organization IDs by
-    # decoding the project or space IDs, or accept path/query metacharacters.
-    return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9+/_=-]{1,1024}", value) is not None
-
-
-def _identity(values: Mapping[str, object]) -> dict[str, str] | None:
-    try:
-        config = ArizeConfig.model_validate(
-            {key: values[key] for key in ("endpoint", "space_id", "project") if key in values}
-        )
-        endpoint = urlsplit(config.endpoint)
-        # These fixed API/UI hosts belong to the documented default collector.
-        # Custom or other regional collectors keep ingestion support, but must
-        # not send their credentials to an assumed Cloud metadata destination.
-        if (
-            endpoint.scheme != "https"
-            or endpoint.hostname != "otlp.arize.com"
-            or endpoint.port not in (None, 443)
-            or endpoint.username is not None
-            or endpoint.password is not None
-            or not _opaque_id(config.space_id)
-        ):
-            return None
-    except ValueError:
-        return None
-    assert config.space_id is not None
-    return {"endpoint": config.endpoint, "space_id": config.space_id, "project": config.project or "default"}
-
-
-def project_url(values: Mapping[str, object]) -> str | None:
-    """Compute only a verified cached destination; never perform vendor I/O."""
-    destination = values.get(DESTINATION_KEY)
-    if not isinstance(destination, Mapping):
-        return None
-    identity = _identity(destination)
-    # Legacy Arize encrypts space_id as well as api_key. Bind the public cache
-    # to this exact persisted record without decrypting it on status reads.
-    digest = _source_digest(values)
-    if identity is None or digest is None or destination.get("source_digest") != digest:
-        return None
-    organization_id, project_id = destination.get("organization_id"), destination.get("project_id")
-    if not _opaque_id(organization_id) or not _opaque_id(project_id):
-        return None
-    assert isinstance(organization_id, str)
-    assert isinstance(project_id, str)
-    return (
-        f"{_UI_BASE}/organizations/{quote(organization_id, safe='')}"
-        f"/spaces/{quote(identity['space_id'], safe='')}/projects/{quote(project_id, safe='')}"
-    )
-
-
-def _source_digest(values: Mapping[str, object]) -> str | None:
-    parts = [values.get(key) for key in ("endpoint", "space_id", "project")]
-    if not all(isinstance(part, str) for part in parts):
-        return None
-    return hashlib.sha256(json.dumps(parts, ensure_ascii=True, separators=(",", ":")).encode()).hexdigest()
-
-
-def bind_destination(values: Mapping[str, object], destination: dict[str, str]) -> dict[str, str]:
-    """Bind server-resolved metadata after legacy secret encryption has finished."""
-    digest = _source_digest(values)
-    if digest is None:
-        raise ValueError("Arize persisted destination is incomplete")
-    return {**destination, "source_digest": digest}
-
-
-def _get_json(
-    path: str, headers: dict[str, str], deadline: float, *, params: dict[str, str | int] | None = None
-) -> dict[str, JsonValue] | None:
-    remaining = deadline - time.monotonic()
-    if remaining <= 0:
-        return None
-    try:
-        response = ssrf_proxy.get(
-            _API_BASE + path,
-            headers=headers,
-            params=params,
-            auth=None,
-            timeout=remaining,
-            max_retries=0,
-            follow_redirects=False,
-            stream_response=True,
-        )
-        try:
-            if response.status_code != 200 or time.monotonic() >= deadline:
-                return None
-            encoding = response.headers.get("content-encoding", "identity").strip().lower()
-            if encoding not in {"", "identity"}:
-                return None
-            content = bytearray()
-            for chunk in response.iter_bytes():
-                if time.monotonic() >= deadline or len(content) + len(chunk) > _MAX_RESPONSE_BYTES:
-                    return None
-                content.extend(chunk)
-            if time.monotonic() >= deadline:
-                return None
-            return _JSON_OBJECT.validate_json(content, strict=True)
-        finally:
-            response.close()
-    except Exception:
-        # Metadata is optional. Do not expose request headers, response bodies,
-        # credential-bearing exceptions, or turn read permission into an ingest
-        # permission requirement.
-        return None
-
-
-def resolve_project_destination(values: Mapping[str, object]) -> dict[str, str] | None:
-    """Resolve exact project/space ownership within one shared five-second budget.
-
-    The documented name filter is a substring match. Read at most three scoped
-    pages and require one exact name/space match. Missing organization IDs on
-    older servers, denied reads, malformed responses and budget exhaustion all
-    mean no optional link, without modifying ingestion configuration or history.
-    """
-    identity = _identity(values)
-    key = values.get("api_key")
-    if (
-        identity is None
-        or not isinstance(key, str)
-        or not key
-        or "*" in key
-        or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in key)
-    ):
-        return None
-    deadline = time.monotonic() + _BUDGET_SECONDS
-    headers = {
-        "Authorization": f"Bearer {key}",
-        "Accept": "application/json",
-        "Accept-Encoding": "identity",
-        "Cookie": "",
-    }
-    space = _get_json("/spaces/" + quote(identity["space_id"], safe=""), headers, deadline)
-    if space is None or space.get("id") != identity["space_id"] or not _opaque_id(space.get("organization_id")):
-        return None
-    organization_id = space["organization_id"]
-    assert isinstance(organization_id, str)
-    project_ids: set[str] = set()
-    seen_cursors: set[str] = set()
-    params: dict[str, str | int] = {
-        "space_id": identity["space_id"],
-        "name": identity["project"],
-        "limit": _PAGE_LIMIT,
-    }
-    for _ in range(_MAX_PAGES):
-        page = _get_json("/projects", headers, deadline, params=params)
-        if page is None:
-            return None
-        projects, pagination = page.get("projects"), page.get("pagination")
-        if not isinstance(projects, list) or len(projects) > _PAGE_LIMIT or not isinstance(pagination, dict):
-            return None
-        for project in projects:
-            if (
-                not isinstance(project, dict)
-                or not _opaque_id(project.get("id"))
-                or not isinstance(project.get("name"), str)
-                or project.get("space_id") != identity["space_id"]
-            ):
-                return None
-            if project["name"] == identity["project"]:
-                project_id = project["id"]
-                assert isinstance(project_id, str)
-                project_ids.add(project_id)
-                if len(project_ids) > 1:
-                    return None
-        has_more = pagination.get("has_more")
-        if has_more is False:
-            if len(project_ids) != 1:
-                return None
-            return {**identity, "organization_id": organization_id, "project_id": next(iter(project_ids))}
-        cursor = pagination.get("next_cursor")
-        if (
-            has_more is not True
-            or not isinstance(cursor, str)
-            or not cursor
-            or len(cursor) > 4096
-            or cursor in seen_cursors
-        ):
-            return None
-        seen_cursors.add(cursor)
-        params = {**params, "cursor": cursor}
-    return None