From 9fbd9dcd33767612436cb41e88a400a086cf6b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Tue, 10 Nov 2026 05:08:50 +0000 Subject: [PATCH] refactor(api): isolate native webhook runtime (#45912) Co-authored-by: Stream --- api/.importlinter | 1 + api/controllers/trigger/webhook.py | 116 +- api/controllers/trigger/webhook_request.py | 673 ++++++++ api/core/trigger/debug/event_selectors.py | 4 +- .../sync_webhook_when_app_created.py | 39 +- .../application_services/workspace.py | 23 + api/repositories/trigger/app_repository.py | 125 +- api/services/trigger/webhook_gateway.py | 22 + api/services/trigger/webhook_service.py | 1159 ++----------- .../trigger/workflow_dispatch_gateway.py | 6 + .../trigger/workflow_dispatch_service.py | 2 + .../services/test_webhook_service.py | 230 +-- .../test_webhook_service_relationships.py | 589 ------- .../containers/trigger/test_trigger_e2e.py | 65 +- .../common/test_access_response.py | 26 +- .../inner_api/workspace/test_workspace.py | 164 +- .../controllers/trigger/test_webhook.py | 791 +++++---- .../trigger/test_webhook_request.py | 548 ++++++ .../trigger/debug/test_event_selectors.py | 15 +- .../test_sync_webhook_when_app_created.py | 118 +- .../trigger/test_webhook_runtime.py | 240 +++ .../services/trigger/test_webhook_service.py | 1475 ----------------- 22 files changed, 2500 insertions(+), 3931 deletions(-) create mode 100644 api/controllers/trigger/webhook_request.py create mode 100644 api/services/trigger/webhook_gateway.py delete mode 100644 api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py create mode 100644 api/tests/unit_tests/controllers/trigger/test_webhook_request.py create mode 100644 api/tests/unit_tests/repositories/trigger/test_webhook_runtime.py delete mode 100644 api/tests/unit_tests/services/trigger/test_webhook_service.py diff --git a/api/.importlinter b/api/.importlinter index a3b4693c404..c97281ff205 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -1443,6 +1443,7 @@ source_modules = services.trigger.subscription_binding_service services.async_workflow_service services.trigger.workflow_dispatch_service + services.trigger.webhook_service forbidden_modules = controllers extensions diff --git a/api/controllers/trigger/webhook.py b/api/controllers/trigger/webhook.py index 6c83fb1e9fc..152d3ad8850 100644 --- a/api/controllers/trigger/webhook.py +++ b/api/controllers/trigger/webhook.py @@ -1,17 +1,21 @@ import logging import time +from http import HTTPStatus from flask import jsonify, request from werkzeug.exceptions import RequestEntityTooLarge +from werkzeug.wrappers import Response from controllers.common.access_response import webhook_not_found_response from controllers.trigger import bp +from controllers.trigger.webhook_request import WebhookRequestParser, WebhookRequestValidationError from core.trigger.debug.event_bus import TriggerDebugEventBus from core.trigger.debug.events import WebhookDebugEvent, build_webhook_pool_key +from core.trigger.utils.endpoint import generate_webhook_trigger_endpoint from enums import QuotaType from extensions.ext_application_services import application_services from services.errors.app import QuotaExceededError -from services.trigger.webhook_service import RawWebhookDataDict, WebhookService +from services.trigger.webhook_service import WebhookContext, WebhookNotFoundError, build_webhook_workflow_inputs logger = logging.getLogger(__name__) @@ -30,40 +34,19 @@ def _get_quota_exceeded_message(feature: str) -> str: return _QUOTA_EXCEEDED_MESSAGES.get(quota_type, _DEFAULT_QUOTA_EXCEEDED_MESSAGE) -def _prepare_webhook_execution(webhook_id: str, is_debug: bool = False): - """Fetch trigger context, extract request data, and validate payload using unified processing. - - Args: - webhook_id: The webhook ID to process - is_debug: If True, skip status validation for debug mode - """ - webhook_trigger, workflow, node_config = WebhookService.get_webhook_trigger_and_workflow( - webhook_id, is_debug=is_debug +def _prepare_webhook_execution(webhook_id: str, *, debug: bool) -> tuple[WebhookContext, WebhookRequestParser]: + services = application_services().workspaces + context = services.webhooks.prepare(webhook_id=webhook_id, debug=debug) + parser = WebhookRequestParser( + request=request, + files=services.webhook_files, + resolve_actor=lambda webhook_trigger: services.webhooks.resolve_actor(context=webhook_trigger), ) - - webhook_data: RawWebhookDataDict - try: - # Use new unified extraction and validation - webhook_data = WebhookService.extract_and_validate_webhook_data( - webhook_trigger, - node_config, - end_users=application_services().app_scoped_end_users.commands, - ) - return webhook_trigger, workflow, node_config, webhook_data, None - except ValueError as e: - # Provide minimal context for error reporting without risking another parse failure - webhook_data = { - "method": request.method, - "headers": dict(request.headers), - "query_params": dict(request.args), - "body": {}, - "files": {}, - } - return webhook_trigger, workflow, node_config, webhook_data, str(e) + return context, parser @bp.route("/webhook/", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]) -def handle_webhook(webhook_id: str): +def handle_webhook(webhook_id: str) -> Response | tuple[Response, int]: """ Handle webhook trigger calls. @@ -71,22 +54,17 @@ def handle_webhook(webhook_id: str): configured webhook trigger settings. """ try: - webhook_trigger, workflow, node_config, webhook_data, error = _prepare_webhook_execution(webhook_id) - if error: - return jsonify({"error": "Bad Request", "message": error}), 400 + context, parser = _prepare_webhook_execution(webhook_id, debug=False) + try: + webhook_data = parser.extract_and_validate_webhook_data(context, context.node_config) + except WebhookRequestValidationError as error: + return jsonify({"error": "Bad Request", "message": error.public_message}), HTTPStatus.BAD_REQUEST # Process webhook call (send to Celery) - WebhookService.trigger_workflow_execution( - webhook_trigger, - webhook_data, - workflow, - end_users=application_services().app_scoped_end_users.commands, - trigger_status=application_services().apps.trigger_status, - workflow_dispatch=application_services().workspaces.workflow_dispatch, - ) + application_services().workspaces.webhooks.execute(context=context, data=webhook_data, actor=parser.actor) # Return configured response - response_data, status_code = WebhookService.generate_webhook_response(node_config) + response_data, status_code = parser.generate_webhook_response(context.node_config) return jsonify(response_data), status_code except QuotaExceededError as error: @@ -95,18 +73,20 @@ def handle_webhook(webhook_id: str): "error": "Too Many Requests", "message": _get_quota_exceeded_message(error.feature), } - ), 429 - except ValueError: + ), HTTPStatus.TOO_MANY_REQUESTS + except WebhookNotFoundError: return webhook_not_found_response() except RequestEntityTooLarge: raise except Exception: logger.exception("Webhook processing failed for %s", webhook_id) - return jsonify({"error": "Internal server error", "message": "An internal error has occurred."}), 500 + return jsonify( + {"error": "Internal server error", "message": "An internal error has occurred."} + ), HTTPStatus.INTERNAL_SERVER_ERROR @bp.route("/webhook-debug/", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]) -def handle_webhook_debug(webhook_id: str): +def handle_webhook_debug(webhook_id: str) -> Response | tuple[Response, int]: """Handle webhook debug calls without triggering production workflow execution. The debug webhook endpoint is only for draft inspection flows. It never enqueues @@ -116,22 +96,24 @@ def handle_webhook_debug(webhook_id: str): effectively dropped. """ try: - webhook_trigger, _, node_config, webhook_data, error = _prepare_webhook_execution(webhook_id, is_debug=True) - if error: - return jsonify({"error": "Bad Request", "message": error}), 400 + context, parser = _prepare_webhook_execution(webhook_id, debug=True) + try: + webhook_data = parser.extract_and_validate_webhook_data(context, context.node_config) + except WebhookRequestValidationError as error: + return jsonify({"error": "Bad Request", "message": error.public_message}), HTTPStatus.BAD_REQUEST - workflow_inputs = WebhookService.build_workflow_inputs(webhook_data) + workflow_inputs = build_webhook_workflow_inputs(webhook_data) # Generate pool key and dispatch debug event pool_key: str = build_webhook_pool_key( - tenant_id=webhook_trigger.tenant_id, - app_id=webhook_trigger.app_id, - node_id=webhook_trigger.node_id, + tenant_id=context.tenant_id, + app_id=context.app_id, + node_id=context.node_id, ) event = WebhookDebugEvent( - request_id=f"webhook_debug_{webhook_trigger.webhook_id}_{int(time.time() * 1000)}", + request_id=f"webhook_debug_{context.webhook_id}_{int(time.time() * 1000)}", timestamp=int(time.time()), - node_id=webhook_trigger.node_id, + node_id=context.node_id, payload={ "inputs": workflow_inputs, "webhook_data": webhook_data, @@ -139,17 +121,17 @@ def handle_webhook_debug(webhook_id: str): }, ) dispatch_count = TriggerDebugEventBus.dispatch( - tenant_id=webhook_trigger.tenant_id, + tenant_id=context.tenant_id, event=event, pool_key=pool_key, ) if dispatch_count == 0: logger.warning( "Webhook debug request dropped without an active listener for webhook %s (tenant=%s, app=%s, node=%s)", - webhook_trigger.webhook_id, - webhook_trigger.tenant_id, - webhook_trigger.app_id, - webhook_trigger.node_id, + context.webhook_id, + context.tenant_id, + context.app_id, + context.node_id, ) return ( jsonify( @@ -159,18 +141,20 @@ def handle_webhook_debug(webhook_id: str): "The webhook debug URL only works while the Variable Inspector is listening. " "Use the published webhook URL to execute the workflow in Celery." ), - "execution_url": webhook_trigger.webhook_url, + "execution_url": generate_webhook_trigger_endpoint(context.webhook_id), } ), - 409, + HTTPStatus.CONFLICT, ) - response_data, status_code = WebhookService.generate_webhook_response(node_config) + response_data, status_code = parser.generate_webhook_response(context.node_config) return jsonify(response_data), status_code - except ValueError: + except WebhookNotFoundError: return webhook_not_found_response() except RequestEntityTooLarge: raise except Exception: logger.exception("Webhook debug processing failed for %s", webhook_id) - return jsonify({"error": "Internal server error", "message": "An internal error has occurred."}), 500 + return jsonify( + {"error": "Internal server error", "message": "An internal error has occurred."} + ), HTTPStatus.INTERNAL_SERVER_ERROR diff --git a/api/controllers/trigger/webhook_request.py b/api/controllers/trigger/webhook_request.py new file mode 100644 index 00000000000..f92983ad097 --- /dev/null +++ b/api/controllers/trigger/webhook_request.py @@ -0,0 +1,673 @@ +"""HTTP parsing and response formatting for native webhook triggers.""" + +import json +import logging +import math +import mimetypes +from collections.abc import Callable, Mapping, Sequence +from typing import Any, NotRequired, TypedDict + +import orjson +from flask import Request +from werkzeug.datastructures import FileStorage +from werkzeug.exceptions import RequestEntityTooLarge + +from configs import dify_config +from core.workflow.nodes.trigger_webhook.entities import ( + ContentType, + WebhookBodyParameter, + WebhookData, + WebhookParameter, +) +from graphon.entities.graph_config import NodeConfigDict +from graphon.file import File +from graphon.variables.types import ArrayValidation, SegmentType +from services.trigger.webhook_service import RawWebhookDataDict, WebhookContext, WebhookFiles +from services.workflow.entities import TriggerActor + +try: + import magic +except ImportError: + magic = None # type: ignore[assignment] + +logger = logging.getLogger(__name__) + + +class ValidationResultDict(TypedDict): + valid: bool + error: NotRequired[str] + + +class WebhookRequestValidationError(ValueError): + """Expose only messages intentionally generated for invalid webhook requests.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.public_message = message + + +class WebhookRequestParser: + def __init__( + self, + *, + request: Request, + files: WebhookFiles, + resolve_actor: Callable[[WebhookContext], TriggerActor] | None = None, + ) -> None: + self._request = request + self._files = files + self._resolve_actor = resolve_actor + self.actor: TriggerActor | None = None + + @staticmethod + def _sanitize_key(key: str) -> str: + if not isinstance(key, str): + return key + return key.replace("-", "_") + + def extract_and_validate_webhook_data( + self, webhook_trigger: WebhookContext, node_config: NodeConfigDict + ) -> RawWebhookDataDict: + """Extract and validate webhook data in a single unified process. + + Args: + webhook_trigger: The webhook trigger object containing metadata + node_config: The node configuration containing validation rules + + Returns: + dict[str, Any]: Processed and validated webhook data with correct types + + Raises: + ValueError: If validation fails (HTTP method mismatch, missing required fields, type errors) + """ + # Extract raw data first + raw_data = self.extract_webhook_data(webhook_trigger) + + # Validate HTTP metadata (method, content-type) + node_data = WebhookData.model_validate(node_config["data"], from_attributes=True) + validation_result = self._validate_http_metadata(raw_data, node_data) + if not validation_result["valid"]: + raise WebhookRequestValidationError(validation_result.get("error", "Validation failed")) + + # Process and validate data according to configuration + processed_data = self._process_and_validate_data(raw_data, node_data) + + return processed_data + + def extract_webhook_data(self, webhook_trigger: WebhookContext) -> RawWebhookDataDict: + """Extract raw data from incoming webhook request without type conversion. + + Args: + webhook_trigger: The webhook trigger object for file processing context + + Returns: + dict[str, Any]: Raw webhook data containing: + - method: HTTP method + - headers: Request headers + - query_params: Query parameters as strings + - body: Request body (varies by content type; JSON parsing errors raise ValueError) + - files: Uploaded files (if any) + """ + self._validate_content_length() + + data: RawWebhookDataDict = { + "method": self._request.method, + "headers": dict(self._request.headers), + "query_params": dict(self._request.args), + "body": {}, + "files": {}, + } + + # Extract and normalize content type + content_type = self._extract_content_type(dict(self._request.headers)) + + # Route to appropriate extractor based on content type + extractors: dict[str, Callable[[], tuple[dict[str, Any], dict[str, Any]]]] = { + "application/json": self._extract_json_body, + "application/x-www-form-urlencoded": self._extract_form_body, + "multipart/form-data": lambda: self._extract_multipart_body(webhook_trigger), + "application/octet-stream": lambda: self._extract_octet_stream_body(webhook_trigger), + "text/plain": self._extract_text_body, + } + + extractor = extractors.get(content_type) + if not extractor: + # Default to text/plain for unknown content types + logger.warning("Unknown Content-Type: %s, treating as text/plain", content_type) + extractor = self._extract_text_body + + # Extract body and files + body_data, files_data = extractor() + data["body"] = body_data + data["files"] = files_data + + return data + + def _process_and_validate_data(self, raw_data: RawWebhookDataDict, node_data: WebhookData) -> RawWebhookDataDict: + """Process and validate webhook data according to node configuration. + + Args: + raw_data: Raw webhook data from extraction + node_data: Node configuration containing validation and type rules + + Returns: + dict[str, Any]: Processed data with validated types + + Raises: + ValueError: If validation fails or required fields are missing + """ + result = raw_data.copy() + + # Validate and process headers + self._validate_required_headers(raw_data["headers"], node_data.headers) + + # Process query parameters with type conversion and validation + result["query_params"] = self._process_parameters(raw_data["query_params"], node_data.params, is_form_data=True) + + # Process body parameters based on content type + result["body"] = self._process_body_parameters(raw_data["body"], node_data.body, node_data.content_type) + + return result + + def _validate_content_length(self) -> None: + """Validate request content length against maximum allowed size.""" + content_length = self._request.content_length + if content_length and content_length > dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE: + raise RequestEntityTooLarge( + f"Webhook request too large: {content_length} bytes exceeds maximum allowed size " + f"of {dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE} bytes" + ) + + def _extract_json_body(self) -> tuple[dict[str, Any], dict[str, Any]]: + """Extract JSON body from self._request. + + Returns: + tuple: (body_data, files_data) where: + - body_data: Parsed JSON content + - files_data: Empty dict (JSON requests don't contain files) + + Raises: + ValueError: If JSON parsing fails + """ + raw_body = self._request.get_data(cache=True) + if not raw_body or raw_body.strip() == b"": + return {}, {} + + try: + body = orjson.loads(raw_body) + except orjson.JSONDecodeError as exc: + logger.warning("Failed to parse JSON body: %s", exc) + message = f"{exc.msg}: line {exc.lineno} column {exc.colno} (char {exc.pos})" + raise WebhookRequestValidationError(f"Invalid JSON body: {message}") from exc + return body, {} + + def _extract_form_body(self) -> tuple[dict[str, Any], dict[str, Any]]: + """Extract form-urlencoded body from self._request. + + Returns: + tuple: (body_data, files_data) where: + - body_data: Form data as key-value pairs + - files_data: Empty dict (form-urlencoded requests don't contain files) + """ + return dict(self._request.form), {} + + def _extract_multipart_body(self, webhook_trigger: WebhookContext) -> tuple[dict[str, Any], dict[str, Any]]: + """Extract multipart/form-data body and files from self._request. + + Args: + webhook_trigger: Webhook trigger for file processing context + + Returns: + tuple: (body_data, files_data) where: + - body_data: Form data as key-value pairs + - files_data: Processed file objects indexed by field name + """ + body = dict(self._request.form) + files = self._process_file_uploads(self._request.files, webhook_trigger) if self._request.files else {} + return body, files + + def _extract_octet_stream_body(self, webhook_trigger: WebhookContext) -> tuple[dict[str, Any], dict[str, Any]]: + """Extract binary data as file from self._request. + + Args: + webhook_trigger: Webhook trigger for file processing context + + Returns: + tuple: (body_data, files_data) where: + - body_data: Dict with 'raw' key containing file object or None + - files_data: Empty dict + """ + try: + file_content = self._request.get_data() + if file_content: + mimetype = self._detect_binary_mimetype(file_content) + file_obj = self._create_file_from_binary(file_content, mimetype, webhook_trigger) + return {"raw": file_obj.to_dict()}, {} + else: + return {"raw": None}, {} + except Exception: + logger.exception("Failed to process octet-stream data") + return {"raw": None}, {} + + def _extract_text_body(self) -> tuple[dict[str, Any], dict[str, Any]]: + """Extract text/plain body from self._request. + + Returns: + tuple: (body_data, files_data) where: + - body_data: Dict with 'raw' key containing text content + - files_data: Empty dict (text requests don't contain files) + """ + try: + body = {"raw": self._request.get_data(as_text=True)} + except Exception: + logger.warning("Failed to extract text body") + body = {"raw": ""} + return body, {} + + @staticmethod + def _detect_binary_mimetype(file_content: bytes) -> str: + """Guess MIME type for binary payloads using python-magic when available.""" + if magic is not None: + try: + detected = magic.from_buffer(file_content[:1024], mime=True) + if detected: + return detected + except Exception: + logger.debug("python-magic detection failed for octet-stream payload") + return "application/octet-stream" + + def _process_file_uploads( + self, files: Mapping[str, FileStorage], webhook_trigger: WebhookContext + ) -> dict[str, Any]: + """Process file uploads using ToolFileManager. + + Args: + files: Flask request files object containing uploaded files + webhook_trigger: Webhook trigger for tenant and user context + + Returns: + dict[str, Any]: Processed file objects indexed by field name + """ + processed_files = {} + + for name, file in files.items(): + if file and file.filename: + try: + file_content = file.stream.read() + mimetype = file.content_type or mimetypes.guess_type(file.filename)[0] or "application/octet-stream" + file_obj = self._create_file_from_binary(file_content, mimetype, webhook_trigger) + processed_files[name] = file_obj.to_dict() + except Exception: + logger.exception("Failed to process file upload '%s'", name) + # Continue processing other files + + return processed_files + + def _create_file_from_binary(self, file_content: bytes, mimetype: str, webhook_trigger: WebhookContext) -> File: + if self.actor is None and self._resolve_actor is not None: + self.actor = self._resolve_actor(webhook_trigger) + user_id = self.actor.id if self.actor is not None else webhook_trigger.created_by + return self._files.create( + tenant_id=webhook_trigger.tenant_id, + user_id=user_id, + content=file_content, + mimetype=mimetype, + ) + + def _process_parameters( + self, + raw_params: dict[str, str], + param_configs: Sequence[WebhookParameter], + *, + is_form_data: bool = False, + ) -> dict[str, Any]: + """Process parameters with unified validation and type conversion. + + Args: + raw_params: Raw parameter values as strings + param_configs: List of parameter configuration dictionaries + is_form_data: Whether the parameters are from form data (requiring string conversion) + + Returns: + dict[str, Any]: Processed parameters with validated types + + Raises: + ValueError: If required parameters are missing or validation fails + """ + processed = {} + configured_params = {config.name: config for config in param_configs} + + # Process configured parameters + for param_config in param_configs: + name = param_config.name + param_type = param_config.type + required = param_config.required + + # Check required parameters + if required and name not in raw_params: + raise WebhookRequestValidationError(f"Required parameter missing: {name}") + + if name in raw_params: + raw_value = raw_params[name] + processed[name] = self._validate_and_convert_value(name, raw_value, param_type, is_form_data) + + # Include unconfigured parameters as strings + for name, value in raw_params.items(): + if name not in configured_params: + processed[name] = value + + return processed + + def _process_body_parameters( + self, + raw_body: dict[str, Any], + body_configs: Sequence[WebhookBodyParameter], + content_type: ContentType, + ) -> dict[str, Any]: + """Process body parameters based on content type and configuration. + + Args: + raw_body: Raw body data from request + body_configs: List of body parameter configuration dictionaries + content_type: The request content type + + Returns: + dict[str, Any]: Processed body parameters with validated types + + Raises: + ValueError: If required body parameters are missing or validation fails + """ + match content_type: + case ContentType.TEXT | ContentType.BINARY: + # For text/plain and octet-stream, validate required content exists + if body_configs and any(config.required for config in body_configs): + raw_content = raw_body.get("raw") + if not raw_content: + raise WebhookRequestValidationError(f"Required body content missing for {content_type} request") + return raw_body + case _: + pass + + # For structured data (JSON, form-data, etc.) + processed = {} + configured_params: dict[str, WebhookBodyParameter] = {config.name: config for config in body_configs} + + for body_config in body_configs: + name = body_config.name + param_type = body_config.type + required = body_config.required + + # Multipart file validation is handled separately in the extraction phase. + if not (param_type == SegmentType.FILE and content_type == ContentType.FORM_DATA): + if required and name not in raw_body: + raise WebhookRequestValidationError(f"Required body parameter missing: {name}") + + if name in raw_body: + raw_value = raw_body[name] + is_form_data = content_type in [ContentType.FORM_URLENCODED, ContentType.FORM_DATA] + processed[name] = self._validate_and_convert_value(name, raw_value, param_type, is_form_data) + + # Include unconfigured parameters + for name, value in raw_body.items(): + if name not in configured_params: + processed[name] = value + + return processed + + def _validate_and_convert_value( + self, param_name: str, value: Any, param_type: SegmentType | str, is_form_data: bool + ) -> Any: + """Unified validation and type conversion for parameter values. + + Args: + param_name: Name of the parameter for error reporting + value: The value to validate and convert + param_type: The expected parameter type (SegmentType) + is_form_data: Whether the value is from form data (requiring string conversion) + + Returns: + Any: The validated and converted value + + Raises: + ValueError: If validation or conversion fails. The original validation + error is preserved as ``__cause__`` for debugging. + """ + try: + if is_form_data: + # Form data comes as strings and needs conversion + return self._convert_form_value(param_name, value, param_type) + else: + # JSON data should already be in correct types, just validate + return self._validate_json_value(param_name, value, param_type) + except WebhookRequestValidationError as error: + raise WebhookRequestValidationError( + f"Parameter '{param_name}' validation failed: {error.public_message}" + ) from error + + def _convert_form_value(self, param_name: str, value: str, param_type: SegmentType | str) -> Any: + """Convert form data string values to specified types. + + Args: + param_name: Name of the parameter for error reporting + value: The string value to convert + param_type: The target type to convert to (SegmentType) + + Returns: + Any: The converted value in the appropriate type + + Raises: + ValueError: If the value cannot be converted to the specified type + """ + match param_type: + case SegmentType.STRING: + return value + case SegmentType.NUMBER: + if not self._can_convert_to_number(value): + raise WebhookRequestValidationError(f"Cannot convert '{value}' to number") + try: + # Parse integers exactly before float conversion can round them. + return int(value) + except ValueError: + pass + numeric_value = float(value) + return int(numeric_value) if numeric_value.is_integer() else numeric_value + case SegmentType.BOOLEAN: + lower_value = value.lower() + bool_map = {"true": True, "false": False, "1": True, "0": False, "yes": True, "no": False} + if lower_value not in bool_map: + raise WebhookRequestValidationError(f"Cannot convert '{value}' to boolean") + return bool_map[lower_value] + case ( + SegmentType.OBJECT + | SegmentType.FILE + | SegmentType.ARRAY_ANY + | SegmentType.ARRAY_STRING + | SegmentType.ARRAY_NUMBER + | SegmentType.ARRAY_OBJECT + | SegmentType.ARRAY_FILE + | SegmentType.ARRAY_BOOLEAN + | SegmentType.SECRET + | SegmentType.INTEGER + | SegmentType.FLOAT + | SegmentType.NONE + | SegmentType.GROUP + ): + raise WebhookRequestValidationError( + f"Unsupported type '{param_type}' for form data parameter '{param_name}'" + ) + case _: + raise WebhookRequestValidationError( + f"Unsupported type '{param_type}' for form data parameter '{param_name}'" + ) + + def _validate_json_value(self, param_name: str, value: Any, param_type: SegmentType | str) -> Any: + """Validate JSON values against expected types. + + Args: + param_name: Name of the parameter for error reporting + value: The value to validate + param_type: The expected parameter type (SegmentType) + + Returns: + Any: The validated value (unchanged if valid) + + Raises: + ValueError: If the value type doesn't match the expected type + """ + param_type_enum = self._coerce_segment_type(param_type, param_name=param_name) + if param_type_enum is None: + return value + + if not param_type_enum.is_valid(value, array_validation=ArrayValidation.ALL) or self._is_boolean_for_number( + value, param_type_enum + ): + actual_type = type(value).__name__ + expected_type = self._expected_type_label(param_type_enum) + raise WebhookRequestValidationError(f"Expected {expected_type}, got {actual_type}") + + return value + + @staticmethod + def _is_boolean_for_number(value: Any, param_type: SegmentType) -> bool: + match param_type: + case SegmentType.NUMBER | SegmentType.INTEGER | SegmentType.FLOAT: + return isinstance(value, bool) + case SegmentType.ARRAY_NUMBER: + return isinstance(value, list) and any(isinstance(item, bool) for item in value) + case _: + return False + + def _coerce_segment_type(self, param_type: SegmentType | str, *, param_name: str) -> SegmentType | None: + if isinstance(param_type, SegmentType): + return param_type + try: + return SegmentType(param_type) + except Exception: + logger.warning("Unknown parameter type: %s for parameter %s", param_type, param_name) + return None + + @staticmethod + def _expected_type_label(param_type: SegmentType) -> str: + match param_type: + case SegmentType.ARRAY_STRING: + return "array of strings" + case SegmentType.ARRAY_NUMBER: + return "array of numbers" + case SegmentType.ARRAY_BOOLEAN: + return "array of booleans" + case SegmentType.ARRAY_OBJECT: + return "array of objects" + case _: + return param_type.value + + def _validate_required_headers(self, headers: dict[str, Any], header_configs: Sequence[WebhookParameter]) -> None: + """Validate required headers are present. + + Args: + headers: Request headers dictionary + header_configs: List of header configuration dictionaries + + Raises: + ValueError: If required headers are missing + """ + headers_lower = {k.lower(): v for k, v in headers.items()} + headers_sanitized = {self._sanitize_key(k).lower(): v for k, v in headers.items()} + for header_config in header_configs: + if header_config.required: + header_name = header_config.name + sanitized_name = self._sanitize_key(header_name).lower() + if header_name.lower() not in headers_lower and sanitized_name not in headers_sanitized: + raise WebhookRequestValidationError(f"Required header missing: {header_name}") + + def _validate_http_metadata(self, webhook_data: RawWebhookDataDict, node_data: WebhookData) -> ValidationResultDict: + """Validate HTTP method and content-type. + + Args: + webhook_data: Extracted webhook data containing method and headers + node_data: Node configuration containing expected method and content-type + + Returns: + dict[str, Any]: Validation result with 'valid' key and optional 'error' key + """ + # Validate HTTP method + configured_method = node_data.method.value.upper() + request_method = webhook_data["method"].upper() + if configured_method != request_method: + return self._validation_error(f"HTTP method mismatch. Expected {configured_method}, got {request_method}") + + # Validate Content-type + configured_content_type = node_data.content_type.value.lower() + request_content_type = self._extract_content_type(webhook_data["headers"]) + + if configured_content_type != request_content_type: + return self._validation_error( + f"Content-type mismatch. Expected {configured_content_type}, got {request_content_type}" + ) + + return {"valid": True} + + def _extract_content_type(self, headers: dict[str, Any]) -> str: + """Extract and normalize content-type from headers. + + Args: + headers: Request headers dictionary + + Returns: + str: Normalized content-type (main type without parameters) + """ + content_type = headers.get("Content-Type", "").lower() + if not content_type: + content_type = headers.get("content-type", "application/json").lower() + # Extract the main content type (ignore parameters like boundary) + return content_type.split(";")[0].strip() + + def _validation_error(self, error_message: str) -> ValidationResultDict: + """Create a standard validation error response. + + Args: + error_message: The error message to include + + Returns: + dict[str, Any]: Validation error response with 'valid' and 'error' keys + """ + return {"valid": False, "error": error_message} + + def _can_convert_to_number(self, value: str) -> bool: + """Check if a string can be converted to a finite number.""" + try: + return math.isfinite(float(value)) + except ValueError: + return False + + def generate_webhook_response(self, node_config: NodeConfigDict) -> tuple[Any, int]: + """Generate HTTP response based on node configuration. + + Args: + node_config: Node configuration containing response settings + + Returns: + tuple[dict[str, Any], int]: Response data and HTTP status code + """ + node_data = WebhookData.model_validate(node_config["data"], from_attributes=True) + + # Get configured status code and response body + status_code = node_data.status_code + response_body = node_data.response_body + + # Parse response body as JSON if it's valid JSON, otherwise return as text + try: + if response_body: + try: + response_data = ( + json.loads(response_body) + if response_body.strip().startswith(("{", "[")) + else {"message": response_body} + ) + except json.JSONDecodeError: + response_data = {"message": response_body} + else: + response_data = {"status": "success", "message": "Webhook processed successfully"} + except Exception: + response_data = {"message": response_body or "Webhook processed successfully"} + + return response_data, status_code diff --git a/api/core/trigger/debug/event_selectors.py b/api/core/trigger/debug/event_selectors.py index 8e304fe1df5..b9fe186134d 100644 --- a/api/core/trigger/debug/event_selectors.py +++ b/api/core/trigger/debug/event_selectors.py @@ -139,13 +139,13 @@ class WebhookTriggerDebugEventPoller(TriggerDebugEventPoller): if not webhook_event: return None - from services.trigger.webhook_service import WebhookService + from services.trigger.webhook_service import build_webhook_workflow_inputs payload = webhook_event.payload or {} workflow_inputs = payload.get("inputs") if workflow_inputs is None: webhook_data = payload.get("webhook_data", {}) - workflow_inputs = WebhookService.build_workflow_inputs(webhook_data) + workflow_inputs = build_webhook_workflow_inputs(webhook_data) workflow_args: Mapping[str, Any] = { "inputs": workflow_inputs or {}, diff --git a/api/events/event_handlers/sync_webhook_when_app_created.py b/api/events/event_handlers/sync_webhook_when_app_created.py index 7fd23a8dfa4..c29c9c24307 100644 --- a/api/events/event_handlers/sync_webhook_when_app_created.py +++ b/api/events/event_handlers/sync_webhook_when_app_created.py @@ -1,32 +1,31 @@ -import logging +from sqlalchemy.orm import Session +from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated +from extensions.ext_application_services import application_services from models.model import App, AppMode from models.workflow import Workflow -from services.trigger.webhook_service import WebhookService - -logger = logging.getLogger(__name__) @app_draft_workflow_was_synced.connect -def handle(sender, synced_draft_workflow: Workflow, **kwargs): - """ - While creating a workflow or updating a workflow, we may need to sync - its webhook relationships in DB. - """ - app: App = sender - if app.mode != AppMode.WORKFLOW.value: - # only handle workflow app, chatflow is not supported yet +def handle(sender: App, synced_draft_workflow: Workflow, **kwargs: object) -> None: + if sender.mode != AppMode.WORKFLOW.value: return - - WebhookService.sync_webhook_relationships(app, synced_draft_workflow, remove_stale=False) + application_services().workspaces.webhook_draft_bindings.sync( + tenant_id=sender.tenant_id, + app_id=sender.id, + created_by=sender.created_by, + node_ids=[node_id for node_id, _ in synced_draft_workflow.walk_nodes(TRIGGER_WEBHOOK_NODE_TYPE)], + ) @app_published_workflow_was_updated.connect -def handle_published(sender, published_workflow: Workflow, **kwargs): - """Remove stale webhook relationships after their node deletion is published.""" - app: App = sender - if app.mode != AppMode.WORKFLOW.value: +def handle_published(sender: App, published_workflow: Workflow, *, session: Session, **kwargs: object) -> None: + if sender.mode != AppMode.WORKFLOW.value: return - - WebhookService.sync_webhook_relationships(app, published_workflow, remove_stale=True) + application_services().workspaces.create_webhook_publication_service(session).sync( + tenant_id=sender.tenant_id, + app_id=sender.id, + created_by=sender.created_by, + node_ids=[node_id for node_id, _ in published_workflow.walk_nodes(TRIGGER_WEBHOOK_NODE_TYPE)], + ) diff --git a/api/extensions/application_services/workspace.py b/api/extensions/application_services/workspace.py index 04c0359b067..e529ef87eab 100644 --- a/api/extensions/application_services/workspace.py +++ b/api/extensions/application_services/workspace.py @@ -7,7 +7,9 @@ from functools import partial from sqlalchemy.orm import Session, sessionmaker from configs import dify_config +from core.app.file_access import DatabaseFileAccessController from core.plugin.impl.oauth import OAuthHandler +from core.tools.tool_file_manager import ToolFileManager from core.trigger.trigger_manager import TriggerManager from core.trigger.utils.encryption import delete_cache_for_subscription from extensions.ext_redis import RedisClientWrapper @@ -16,6 +18,7 @@ from repositories.app.console_repository import ConsoleAppRepository from repositories.credentials.query_repository import CredentialQueryRepository from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository from repositories.sqlalchemy_workflow_trigger_log_repository import SQLAlchemyWorkflowTriggerDispatchRepository +from repositories.trigger.app_repository import SQLAlchemyAppTriggerRepository from repositories.trigger.oauth_client_repository import TriggerOAuthClientRepository from repositories.trigger.subscription_builder_repository import RedisTriggerSubscriptionBuilderRepository from repositories.trigger.subscription_repository import ( @@ -60,6 +63,8 @@ from services.trigger.subscription_renewal_service import TriggerSubscriptionRen from services.trigger.subscription_service import TriggerSubscriptionService from services.trigger.trigger_request_service import TriggerHttpRequestCachingService from services.trigger.trigger_subscription_builder_service import TriggerSubscriptionBuilderService +from services.trigger.webhook_gateway import ToolWebhookFiles +from services.trigger.webhook_service import WebhookBindingService, WebhookFiles, WebhookService from services.trigger.workflow_dispatch_service import PluginTriggerWorkflowDispatchService, TriggerEndUserProvisioner from services.workflow.queue_dispatcher import CeleryWorkflowQueuePublisher from services.workspace.gateways import ( @@ -109,6 +114,10 @@ class WorkspaceServices: trigger_endpoints: TriggerEndpointService trigger_events: TriggerEventService workflow_dispatch: AsyncWorkflowService + webhooks: WebhookService + webhook_files: WebhookFiles + webhook_draft_bindings: WebhookBindingService + create_webhook_publication_service: Callable[[Session], WebhookBindingService] trigger_workflow_dispatch: PluginTriggerWorkflowDispatchService trigger_oauth_clients: TriggerOAuthClientService trigger_oauth: TriggerOAuthService @@ -201,6 +210,7 @@ def build_workspace_services( workflow_dispatch = AsyncWorkflowService( workflows=workflow_queries, logs=execution_logs, queues=queues, reserve_quota=QuotaService.reserve ) + app_triggers = SQLAlchemyAppTriggerRepository(session_factory=database_client) return WorkspaceServices( queries=WorkspaceQueryService(workspaces=workspaces, plans=DeploymentWorkspacePlanGateway()), management=WorkspaceService( @@ -248,6 +258,19 @@ def build_workspace_services( ), trigger_events=trigger_events, workflow_dispatch=workflow_dispatch, + webhooks=WebhookService( + webhooks=app_triggers, + end_users=trigger_end_users, + dispatch=workflow_dispatch, + reserve_quota=QuotaService.reserve, + mark_rate_limited=mark_rate_limited, + ), + webhook_files=ToolWebhookFiles(manager=ToolFileManager(), access=DatabaseFileAccessController()), + webhook_draft_bindings=WebhookBindingService(sync_bindings=app_triggers.sync_draft_webhooks), + # Publish binding changes atomically with the workflow that owns them. + create_webhook_publication_service=lambda session: WebhookBindingService( + sync_bindings=partial(app_triggers.sync_webhooks, session=session, remove_stale=True) + ), trigger_workflow_dispatch=PluginTriggerWorkflowDispatchService( workflows=workflow_queries, logs=execution_logs, diff --git a/api/repositories/trigger/app_repository.py b/api/repositories/trigger/app_repository.py index 1895bb0fd80..9a95e51667e 100644 --- a/api/repositories/trigger/app_repository.py +++ b/api/repositories/trigger/app_repository.py @@ -1,15 +1,18 @@ """App trigger management with workspace-scoped, bounded database transactions.""" +import secrets from collections.abc import Sequence +from copy import deepcopy from typing import override from sqlalchemy import select, update from sqlalchemy.orm import Session, sessionmaker from machinery.context import RequestContext -from models.enums import AppTriggerStatus -from models.model import AppMode +from models.enums import AppTriggerStatus, AppTriggerType +from models.model import App, AppMode from models.trigger import AppTrigger, WorkflowWebhookTrigger +from models.workflow import NodeNotFoundError, Workflow from repositories.app.console_repository import require_console_app from services.app.console_service import ConsoleAppNotFoundError from services.trigger.app_trigger_store import ( @@ -20,6 +23,7 @@ from services.trigger.app_trigger_store import ( WebhookTriggerNotFoundError, WebhookTriggerRecord, ) +from services.trigger.webhook_service import UnavailableWebhook, WebhookContext, WebhookNotFoundError def _require_workflow_app(session: Session, context: RequestContext, app_id: str) -> None: @@ -45,6 +49,123 @@ class SQLAlchemyAppTriggerRepository(AppTriggerStore): def __init__(self, *, session_factory: sessionmaker[Session]) -> None: self._session_factory = session_factory + def get_webhook_context(self, *, webhook_id: str, debug: bool) -> WebhookContext | UnavailableWebhook: + with self._session_factory() as session: + webhook = session.scalar( + select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.webhook_id == webhook_id).limit(1) + ) + if webhook is None: + raise WebhookNotFoundError(f"Webhook not found: {webhook_id}") + status = None + if debug: + workflow = session.scalar( + select(Workflow) + .where( + Workflow.tenant_id == webhook.tenant_id, + Workflow.app_id == webhook.app_id, + Workflow.version == Workflow.VERSION_DRAFT, + ) + .order_by(Workflow.created_at.desc()) + .limit(1) + ) + else: + trigger = session.scalar( + select(AppTrigger) + .where( + AppTrigger.tenant_id == webhook.tenant_id, + AppTrigger.app_id == webhook.app_id, + AppTrigger.node_id == webhook.node_id, + AppTrigger.trigger_type == AppTriggerType.TRIGGER_WEBHOOK, + ) + .limit(1) + ) + if trigger is None: + raise WebhookNotFoundError(f"App trigger not found for webhook {webhook_id}") + status = AppTriggerStatus(trigger.status) + # Preserve admission precedence even when an inactive app's stored graph is broken. + if status != AppTriggerStatus.ENABLED: + return UnavailableWebhook(tenant_id=webhook.tenant_id, status=status) + workflow = session.scalar( + select(Workflow) + .join(App, App.workflow_id == Workflow.id) + .where( + App.id == webhook.app_id, + App.tenant_id == webhook.tenant_id, + Workflow.app_id == webhook.app_id, + Workflow.tenant_id == webhook.tenant_id, + Workflow.version != Workflow.VERSION_DRAFT, + ) + ) + if workflow is None: + raise WebhookNotFoundError(f"Workflow not found for app {webhook.app_id}") + try: + node_config = deepcopy(workflow.get_node_config_by_id(webhook.node_id)) + except NodeNotFoundError as error: + raise WebhookNotFoundError(str(error)) from error + return WebhookContext( + tenant_id=webhook.tenant_id, + app_id=webhook.app_id, + webhook_id=webhook.webhook_id, + node_id=webhook.node_id, + created_by=webhook.created_by, + workflow_id=workflow.id, + node_config=node_config, + status=status, + ) + + def sync_draft_webhooks( + self, *, tenant_id: str, app_id: str, created_by: str | None, node_ids: Sequence[str] + ) -> None: + # Draft signals are emitted after the draft transaction commits. + with self._session_factory.begin() as session: + self.sync_webhooks( + session=session, + tenant_id=tenant_id, + app_id=app_id, + created_by=created_by, + node_ids=node_ids, + remove_stale=False, + ) + + def sync_webhooks( + self, + *, + session: Session, + tenant_id: str, + app_id: str, + created_by: str | None, + node_ids: Sequence[str], + remove_stale: bool, + ) -> None: + """Retain URLs across draft undo; published deletion joins the publisher transaction.""" + app = session.scalar(select(App).where(App.tenant_id == tenant_id, App.id == app_id).with_for_update()) + if app is None: + raise WebhookNotFoundError(f"App not found: {app_id}") + rows = session.scalars( + select(WorkflowWebhookTrigger).where( + WorkflowWebhookTrigger.tenant_id == tenant_id, + WorkflowWebhookTrigger.app_id == app_id, + ) + ).all() + existing = {row.node_id: row for row in rows} + desired = set(node_ids) + for node_id in desired - existing.keys(): + if created_by is None: + raise RuntimeError("App is missing created_by; cannot create webhook trigger") + session.add( + WorkflowWebhookTrigger( + tenant_id=tenant_id, + app_id=app_id, + created_by=created_by, + node_id=node_id, + webhook_id=secrets.token_urlsafe(18)[:24], + ) + ) + if remove_stale: + for row in rows: + if row.node_id not in desired: + session.delete(row) + @override def get_webhook(self, context: RequestContext, app_id: str, node_id: str) -> WebhookTriggerRecord: with self._session_factory() as session: diff --git a/api/services/trigger/webhook_gateway.py b/api/services/trigger/webhook_gateway.py new file mode 100644 index 00000000000..ffacfce1e41 --- /dev/null +++ b/api/services/trigger/webhook_gateway.py @@ -0,0 +1,22 @@ +"""Native webhook file storage through the shared tool-file owner.""" + +from core.app.file_access import DatabaseFileAccessController +from core.tools.tool_file_manager import ToolFileManager +from factories import file_factory +from graphon.file import File, FileTransferMethod + + +class ToolWebhookFiles: + def __init__(self, *, manager: ToolFileManager, access: DatabaseFileAccessController) -> None: + self._manager = manager + self._access = access + + def create(self, *, tenant_id: str, user_id: str, content: bytes, mimetype: str) -> File: + record = self._manager.create_file_by_raw( + user_id=user_id, tenant_id=tenant_id, conversation_id=None, file_binary=content, mimetype=mimetype + ) + return file_factory.build_from_mapping( + mapping={"tool_file_id": record.id, "transfer_method": FileTransferMethod.TOOL_FILE.value}, + tenant_id=tenant_id, + access_controller=self._access, + ) diff --git a/api/services/trigger/webhook_service.py b/api/services/trigger/webhook_service.py index 1ad52af19af..e55541b98ce 100644 --- a/api/services/trigger/webhook_service.py +++ b/api/services/trigger/webhook_service.py @@ -1,1085 +1,148 @@ -import json -import logging -import math -import mimetypes -import secrets -from collections.abc import Callable, Mapping, Sequence -from typing import Any, NotRequired, Protocol, TypedDict +"""Native webhook admission and execution, independent of HTTP and SQL sessions.""" -import orjson -from flask import g, has_request_context, request -from pydantic import BaseModel -from sqlalchemy import select -from sqlalchemy.orm import Session, sessionmaker -from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import RequestEntityTooLarge +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any, Protocol, TypedDict -from configs import dify_config -from core.app.file_access import DatabaseFileAccessController -from core.tools.tool_file_manager import ToolFileManager -from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE -from core.workflow.nodes.trigger_webhook.entities import ( - ContentType, - WebhookBodyParameter, - WebhookData, - WebhookParameter, -) from enums import QuotaType -from extensions.ext_database import db -from extensions.ext_redis import redis_client -from factories import file_factory from graphon.entities.graph_config import NodeConfigDict -from graphon.file import FileTransferMethod -from graphon.variables.types import ArrayValidation, SegmentType -from models.enums import AppTriggerStatus, AppTriggerType, CreatorUserRole, EndUserType -from models.model import App, EndUser -from models.trigger import AppTrigger, WorkflowWebhookTrigger -from models.workflow import Workflow +from graphon.file import File +from models.enums import AppTriggerStatus from services.async_workflow_service import TriggerWorkflowDispatcher from services.errors.app import QuotaExceededError -from services.quota_service import QuotaService -from services.trigger.app_trigger_service import AppTriggerService +from services.quota_service import QuotaCharge +from services.trigger.workflow_dispatch_service import TriggerEndUserProvisioner from services.workflow.entities import TriggerActor, WebhookTriggerData -from services.workflow_service import WorkflowService -try: - import magic -except ImportError: - magic = None # type: ignore[assignment] -logger = logging.getLogger(__name__) -_file_access_controller = DatabaseFileAccessController() - -# Request-local cache of the trigger end user resolved for each (tenant_id, app_id), so the files a -# webhook request uploads and the run it queues always share the same owner. -_TRIGGER_END_USER_CACHE_KEY = "_webhook_trigger_end_users" +class WebhookNotFoundError(Exception): + """The webhook or its admitted workflow is unavailable.""" class RawWebhookDataDict(TypedDict): method: str headers: dict[str, str] - query_params: dict[str, str] + query_params: dict[str, Any] body: dict[str, Any] files: dict[str, Any] -class ValidationResultDict(TypedDict): - valid: bool - error: NotRequired[str] - - class WorkflowInputsDict(TypedDict): webhook_data: RawWebhookDataDict webhook_headers: dict[str, str] - webhook_query_params: dict[str, str] + webhook_query_params: dict[str, Any] webhook_body: dict[str, Any] -class WebhookEndUserProvisioner(Protocol): - def get_or_create_end_user_by_type( - self, - type: EndUserType, - tenant_id: str, - app_id: str, - user_id: str | None = None, - ) -> EndUser: ... +@dataclass(frozen=True, slots=True) +class WebhookContext: + tenant_id: str + app_id: str + webhook_id: str + node_id: str + created_by: str + workflow_id: str + node_config: NodeConfigDict + # Draft lookup does not require a published AppTrigger record. + status: AppTriggerStatus | None + + +@dataclass(frozen=True, slots=True) +class UnavailableWebhook: + """Inactive published triggers are admitted before loading workflow configuration.""" + + tenant_id: str + status: AppTriggerStatus + + +class WebhookQuery(Protocol): + def get_webhook_context(self, *, webhook_id: str, debug: bool) -> WebhookContext | UnavailableWebhook: ... + + +class WebhookFiles(Protocol): + def create(self, *, tenant_id: str, user_id: str, content: bytes, mimetype: str) -> File: ... + + +class WebhookBindingWriter(Protocol): + def __call__(self, *, tenant_id: str, app_id: str, created_by: str | None, node_ids: Sequence[str]) -> None: ... + + +def build_webhook_workflow_inputs(webhook_data: RawWebhookDataDict) -> WorkflowInputsDict: + return { + "webhook_data": webhook_data, + "webhook_headers": webhook_data.get("headers", {}), + "webhook_query_params": webhook_data.get("query_params", {}), + "webhook_body": webhook_data.get("body", {}), + } + + +class WebhookBindingService: + MAX_WEBHOOK_NODES_PER_WORKFLOW = 5 + + def __init__(self, *, sync_bindings: WebhookBindingWriter) -> None: + self._sync_bindings = sync_bindings + + def sync(self, *, tenant_id: str, app_id: str, created_by: str | None, node_ids: Sequence[str]) -> None: + if len(node_ids) > self.MAX_WEBHOOK_NODES_PER_WORKFLOW: + raise ValueError( + "Workflow exceeds maximum webhook node limit. " + f"Found {len(node_ids)} webhook nodes, maximum allowed is {self.MAX_WEBHOOK_NODES_PER_WORKFLOW}" + ) + self._sync_bindings(tenant_id=tenant_id, app_id=app_id, created_by=created_by, node_ids=node_ids) class WebhookService: - """Service for handling webhook operations.""" - - __WEBHOOK_NODE_CACHE_KEY__ = "webhook_nodes" - MAX_WEBHOOK_NODES_PER_WORKFLOW = 5 # Maximum allowed webhook nodes per workflow - - @staticmethod - def _sanitize_key(key: str) -> str: - """Normalize external keys (headers/params) to workflow-safe variables.""" - if not isinstance(key, str): - return key - return key.replace("-", "_") - - @classmethod - def get_webhook_trigger_and_workflow( - cls, webhook_id: str, is_debug: bool = False - ) -> tuple[WorkflowWebhookTrigger, Workflow, NodeConfigDict]: - """Get webhook trigger, workflow, and node configuration. - - Args: - webhook_id: The webhook ID to look up - is_debug: If True, use the draft workflow graph and skip the trigger enabled status check - - Returns: - A tuple containing: - - WorkflowWebhookTrigger: The webhook trigger object - - Workflow: The associated workflow object - - Mapping[str, Any]: The node configuration data - - Raises: - QuotaExceededError: If the app trigger is rate limited - ValueError: If webhook not found, app trigger not found, trigger disabled, or workflow not found - """ - with Session(db.engine) as session: - # Get webhook trigger - webhook_trigger = session.scalar( - select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.webhook_id == webhook_id).limit(1) - ) - if not webhook_trigger: - raise ValueError(f"Webhook not found: {webhook_id}") - - if is_debug: - workflow = session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == webhook_trigger.tenant_id, - Workflow.app_id == webhook_trigger.app_id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - .order_by(Workflow.created_at.desc()) - .limit(1) - ) - else: - # Check if the corresponding AppTrigger exists - app_trigger = session.scalar( - select(AppTrigger) - .where( - AppTrigger.tenant_id == webhook_trigger.tenant_id, - AppTrigger.app_id == webhook_trigger.app_id, - AppTrigger.node_id == webhook_trigger.node_id, - AppTrigger.trigger_type == AppTriggerType.TRIGGER_WEBHOOK, - ) - .limit(1) - ) - - if not app_trigger: - raise ValueError(f"App trigger not found for webhook {webhook_id}") - - # Only check enabled status if not in debug mode - - if app_trigger.status == AppTriggerStatus.RATE_LIMITED: - raise QuotaExceededError( - feature=QuotaType.TRIGGER.value, - tenant_id=webhook_trigger.tenant_id, - required=1, - ) - - if app_trigger.status != AppTriggerStatus.ENABLED: - raise ValueError(f"Webhook trigger is disabled for webhook {webhook_id}") - - app = session.scalar( - select(App) - .where( - App.tenant_id == webhook_trigger.tenant_id, - App.id == webhook_trigger.app_id, - ) - .limit(1) - ) - if not app: - raise ValueError(f"App not found for webhook {webhook_id}") - - workflow = WorkflowService().get_published_workflow(app, session=session) - if not workflow: - raise ValueError(f"Workflow not found for app {webhook_trigger.app_id}") - - node_config = workflow.get_node_config_by_id(webhook_trigger.node_id) - - return webhook_trigger, workflow, node_config - - @classmethod - def extract_and_validate_webhook_data( - cls, - webhook_trigger: WorkflowWebhookTrigger, - node_config: NodeConfigDict, + def __init__( + self, *, - end_users: WebhookEndUserProvisioner, - ) -> RawWebhookDataDict: - """Extract and validate webhook data in a single unified process. - - Args: - webhook_trigger: The webhook trigger object containing metadata - node_config: The node configuration containing validation rules - end_users: Provisioner for the trigger end user that owns uploaded files - - Returns: - dict[str, Any]: Processed and validated webhook data with correct types - - Raises: - ValueError: If validation fails (HTTP method mismatch, missing required fields, type errors) - """ - # Extract raw data first - raw_data = cls.extract_webhook_data(webhook_trigger, end_users=end_users) - - # Validate HTTP metadata (method, content-type) - node_data = WebhookData.model_validate(node_config["data"], from_attributes=True) - validation_result = cls._validate_http_metadata(raw_data, node_data) - if not validation_result["valid"]: - raise ValueError(validation_result.get("error", "Validation failed")) - - # Process and validate data according to configuration - processed_data = cls._process_and_validate_data(raw_data, node_data) - - return processed_data - - @classmethod - def extract_webhook_data( - cls, webhook_trigger: WorkflowWebhookTrigger, *, end_users: WebhookEndUserProvisioner - ) -> RawWebhookDataDict: - """Extract raw data from incoming webhook request without type conversion. - - Args: - webhook_trigger: The webhook trigger object for file processing context - end_users: Provisioner for the trigger end user that owns uploaded files - - Returns: - dict[str, Any]: Raw webhook data containing: - - method: HTTP method - - headers: Request headers - - query_params: Query parameters as strings - - body: Request body (varies by content type; JSON parsing errors raise ValueError) - - files: Uploaded files (if any) - """ - cls._validate_content_length() - - data: RawWebhookDataDict = { - "method": request.method, - "headers": dict(request.headers), - "query_params": dict(request.args), - "body": {}, - "files": {}, - } - - # Extract and normalize content type - content_type = cls._extract_content_type(dict(request.headers)) - - # Route to appropriate extractor based on content type - extractors: dict[str, Callable[[], tuple[dict[str, Any], dict[str, Any]]]] = { - "application/json": cls._extract_json_body, - "application/x-www-form-urlencoded": cls._extract_form_body, - "multipart/form-data": lambda: cls._extract_multipart_body(webhook_trigger, end_users), - "application/octet-stream": lambda: cls._extract_octet_stream_body(webhook_trigger, end_users), - "text/plain": cls._extract_text_body, - } - - extractor = extractors.get(content_type) - if not extractor: - # Default to text/plain for unknown content types - logger.warning("Unknown Content-Type: %s, treating as text/plain", content_type) - extractor = cls._extract_text_body - - # Extract body and files - body_data, files_data = extractor() - data["body"] = body_data - data["files"] = files_data - - return data - - @classmethod - def _process_and_validate_data(cls, raw_data: RawWebhookDataDict, node_data: WebhookData) -> RawWebhookDataDict: - """Process and validate webhook data according to node configuration. - - Args: - raw_data: Raw webhook data from extraction - node_data: Node configuration containing validation and type rules - - Returns: - dict[str, Any]: Processed data with validated types - - Raises: - ValueError: If validation fails or required fields are missing - """ - result = raw_data.copy() - - # Validate and process headers - cls._validate_required_headers(raw_data["headers"], node_data.headers) - - # Process query parameters with type conversion and validation - result["query_params"] = cls._process_parameters(raw_data["query_params"], node_data.params, is_form_data=True) - - # Process body parameters based on content type - result["body"] = cls._process_body_parameters(raw_data["body"], node_data.body, node_data.content_type) - - return result - - @classmethod - def _validate_content_length(cls) -> None: - """Validate request content length against maximum allowed size.""" - content_length = request.content_length - if content_length and content_length > dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE: - raise RequestEntityTooLarge( - f"Webhook request too large: {content_length} bytes exceeds maximum allowed size " - f"of {dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE} bytes" - ) - - @classmethod - def _extract_json_body(cls) -> tuple[dict[str, Any], dict[str, Any]]: - """Extract JSON body from request. - - Returns: - tuple: (body_data, files_data) where: - - body_data: Parsed JSON content - - files_data: Empty dict (JSON requests don't contain files) - - Raises: - ValueError: If JSON parsing fails - """ - raw_body = request.get_data(cache=True) - if not raw_body or raw_body.strip() == b"": - return {}, {} - - try: - body = orjson.loads(raw_body) - except orjson.JSONDecodeError as exc: - logger.warning("Failed to parse JSON body: %s", exc) - raise ValueError(f"Invalid JSON body: {exc}") from exc - return body, {} - - @classmethod - def _extract_form_body(cls) -> tuple[dict[str, Any], dict[str, Any]]: - """Extract form-urlencoded body from request. - - Returns: - tuple: (body_data, files_data) where: - - body_data: Form data as key-value pairs - - files_data: Empty dict (form-urlencoded requests don't contain files) - """ - return dict(request.form), {} - - @classmethod - def _extract_multipart_body( - cls, webhook_trigger: WorkflowWebhookTrigger, end_users: WebhookEndUserProvisioner - ) -> tuple[dict[str, Any], dict[str, Any]]: - """Extract multipart/form-data body and files from request. - - Args: - webhook_trigger: Webhook trigger for file processing context - end_users: Provisioner for the trigger end user that owns uploaded files - - Returns: - tuple: (body_data, files_data) where: - - body_data: Form data as key-value pairs - - files_data: Processed file objects indexed by field name - """ - body = dict(request.form) - files = cls._process_file_uploads(request.files, webhook_trigger, end_users) if request.files else {} - return body, files - - @classmethod - def _extract_octet_stream_body( - cls, webhook_trigger: WorkflowWebhookTrigger, end_users: WebhookEndUserProvisioner - ) -> tuple[dict[str, Any], dict[str, Any]]: - """Extract binary data as file from request. - - Args: - webhook_trigger: Webhook trigger for file processing context - end_users: Provisioner for the trigger end user that owns the uploaded file - - Returns: - tuple: (body_data, files_data) where: - - body_data: Dict with 'raw' key containing file object or None - - files_data: Empty dict - """ - try: - file_content = request.get_data() - if file_content: - mimetype = cls._detect_binary_mimetype(file_content) - end_user = cls._resolve_trigger_end_user(webhook_trigger, end_users) - file_obj = cls._create_file_from_binary(file_content, mimetype, webhook_trigger, end_user) - return {"raw": file_obj.to_dict()}, {} - else: - return {"raw": None}, {} - except Exception: - logger.exception("Failed to process octet-stream data") - return {"raw": None}, {} - - @classmethod - def _extract_text_body(cls) -> tuple[dict[str, Any], dict[str, Any]]: - """Extract text/plain body from request. - - Returns: - tuple: (body_data, files_data) where: - - body_data: Dict with 'raw' key containing text content - - files_data: Empty dict (text requests don't contain files) - """ - try: - body = {"raw": request.get_data(as_text=True)} - except Exception: - logger.warning("Failed to extract text body") - body = {"raw": ""} - return body, {} - - @staticmethod - def _detect_binary_mimetype(file_content: bytes) -> str: - """Guess MIME type for binary payloads using python-magic when available.""" - if magic is not None: - try: - detected = magic.from_buffer(file_content[:1024], mime=True) - if detected: - return detected - except Exception: - logger.debug("python-magic detection failed for octet-stream payload") - return "application/octet-stream" - - @staticmethod - def _resolve_trigger_end_user( - webhook_trigger: WorkflowWebhookTrigger, end_users: WebhookEndUserProvisioner - ) -> EndUser: - """Return the app's trigger end user, the identity that webhook workflow runs execute as. - - Within a request, the end user is resolved once and reused from request-local context. A - second lookup could select another trigger end user that a concurrent request created in - the meantime, and the run would then not own the files this request uploaded. - """ - cache: dict[tuple[str, str], EndUser] | None = ( - g.setdefault(_TRIGGER_END_USER_CACHE_KEY, {}) if has_request_context() else None - ) - key = (webhook_trigger.tenant_id, webhook_trigger.app_id) - if cache is not None and key in cache: - return cache[key] - - end_user = end_users.get_or_create_end_user_by_type( - type=EndUserType.TRIGGER, - tenant_id=webhook_trigger.tenant_id, - app_id=webhook_trigger.app_id, - user_id=None, - ) - if cache is not None: - cache[key] = end_user - return end_user - - @classmethod - def _process_file_uploads( - cls, - files: Mapping[str, FileStorage], - webhook_trigger: WorkflowWebhookTrigger, - end_users: WebhookEndUserProvisioner, - ) -> dict[str, Any]: - """Process file uploads using ToolFileManager. - - Args: - files: Flask request files object containing uploaded files - webhook_trigger: Webhook trigger for tenant and app context - end_users: Provisioner for the trigger end user that owns uploaded files - - Returns: - dict[str, Any]: Processed file objects indexed by field name - """ - processed_files = {} - end_user = cls._resolve_trigger_end_user(webhook_trigger, end_users) - - for name, file in files.items(): - if file and file.filename: - try: - file_content = file.stream.read() - mimetype = file.content_type or mimetypes.guess_type(file.filename)[0] or "application/octet-stream" - file_obj = cls._create_file_from_binary(file_content, mimetype, webhook_trigger, end_user) - processed_files[name] = file_obj.to_dict() - except Exception: - logger.exception("Failed to process file upload '%s'", name) - # Continue processing other files - - return processed_files - - @classmethod - def _create_file_from_binary( - cls, file_content: bytes, mimetype: str, webhook_trigger: WorkflowWebhookTrigger, end_user: EndUser - ) -> Any: - """Create a file object from binary content using ToolFileManager. - - Args: - file_content: The binary content of the file - mimetype: The MIME type of the file - webhook_trigger: Webhook trigger for tenant context - end_user: Trigger end user that owns the file. The async workflow run executes as this - end user, and its file access scope only loads tool files the end user owns. - - Returns: - Any: A file object built from the binary content - """ - tool_file_manager = ToolFileManager() - - # Create file using ToolFileManager - tool_file = tool_file_manager.create_file_by_raw( - user_id=end_user.id, - tenant_id=webhook_trigger.tenant_id, - conversation_id=None, - file_binary=file_content, - mimetype=mimetype, - ) - - # Build File object - mapping = { - "tool_file_id": tool_file.id, - "transfer_method": FileTransferMethod.TOOL_FILE.value, - } - return file_factory.build_from_mapping( - mapping=mapping, - tenant_id=webhook_trigger.tenant_id, - access_controller=_file_access_controller, - ) - - @classmethod - def _process_parameters( - cls, - raw_params: dict[str, str], - param_configs: Sequence[WebhookParameter], - *, - is_form_data: bool = False, - ) -> dict[str, Any]: - """Process parameters with unified validation and type conversion. - - Args: - raw_params: Raw parameter values as strings - param_configs: List of parameter configuration dictionaries - is_form_data: Whether the parameters are from form data (requiring string conversion) - - Returns: - dict[str, Any]: Processed parameters with validated types - - Raises: - ValueError: If required parameters are missing or validation fails - """ - processed = {} - configured_params = {config.name: config for config in param_configs} - - # Process configured parameters - for param_config in param_configs: - name = param_config.name - param_type = param_config.type - required = param_config.required - - # Check required parameters - if required and name not in raw_params: - raise ValueError(f"Required parameter missing: {name}") - - if name in raw_params: - raw_value = raw_params[name] - processed[name] = cls._validate_and_convert_value(name, raw_value, param_type, is_form_data) - - # Include unconfigured parameters as strings - for name, value in raw_params.items(): - if name not in configured_params: - processed[name] = value - - return processed - - @classmethod - def _process_body_parameters( - cls, - raw_body: dict[str, Any], - body_configs: Sequence[WebhookBodyParameter], - content_type: ContentType, - ) -> dict[str, Any]: - """Process body parameters based on content type and configuration. - - Args: - raw_body: Raw body data from request - body_configs: List of body parameter configuration dictionaries - content_type: The request content type - - Returns: - dict[str, Any]: Processed body parameters with validated types - - Raises: - ValueError: If required body parameters are missing or validation fails - """ - match content_type: - case ContentType.TEXT | ContentType.BINARY: - # For text/plain and octet-stream, validate required content exists - if body_configs and any(config.required for config in body_configs): - raw_content = raw_body.get("raw") - if not raw_content: - raise ValueError(f"Required body content missing for {content_type} request") - return raw_body - case _: - pass - - # For structured data (JSON, form-data, etc.) - processed = {} - configured_params: dict[str, WebhookBodyParameter] = {config.name: config for config in body_configs} - - for body_config in body_configs: - name = body_config.name - param_type = body_config.type - required = body_config.required - - # Handle file parameters for multipart data - if param_type == SegmentType.FILE and content_type == ContentType.FORM_DATA: - # File validation is handled separately in extract phase - continue - - # Check required parameters - if required and name not in raw_body: - raise ValueError(f"Required body parameter missing: {name}") - - if name in raw_body: - raw_value = raw_body[name] - is_form_data = content_type in [ContentType.FORM_URLENCODED, ContentType.FORM_DATA] - processed[name] = cls._validate_and_convert_value(name, raw_value, param_type, is_form_data) - - # Include unconfigured parameters - for name, value in raw_body.items(): - if name not in configured_params: - processed[name] = value - - return processed - - @classmethod - def _validate_and_convert_value( - cls, param_name: str, value: Any, param_type: SegmentType | str, is_form_data: bool - ) -> Any: - """Unified validation and type conversion for parameter values. - - Args: - param_name: Name of the parameter for error reporting - value: The value to validate and convert - param_type: The expected parameter type (SegmentType) - is_form_data: Whether the value is from form data (requiring string conversion) - - Returns: - Any: The validated and converted value - - Raises: - ValueError: If validation or conversion fails. The original validation - error is preserved as ``__cause__`` for debugging. - """ - try: - if is_form_data: - # Form data comes as strings and needs conversion - return cls._convert_form_value(param_name, value, param_type) - else: - # JSON data should already be in correct types, just validate - return cls._validate_json_value(param_name, value, param_type) - except Exception as e: - raise ValueError(f"Parameter '{param_name}' validation failed: {str(e)}") from e - - @classmethod - def _convert_form_value(cls, param_name: str, value: str, param_type: SegmentType | str) -> Any: - """Convert form data string values to specified types. - - Args: - param_name: Name of the parameter for error reporting - value: The string value to convert - param_type: The target type to convert to (SegmentType) - - Returns: - Any: The converted value in the appropriate type - - Raises: - ValueError: If the value cannot be converted to the specified type - """ - match param_type: - case SegmentType.STRING: - return value - case SegmentType.NUMBER: - if not cls._can_convert_to_number(value): - raise ValueError(f"Cannot convert '{value}' to number") - try: - # float() rounds integers above 2**53, so parse integers exactly first. - return int(value) - except ValueError: - pass - numeric_value = float(value) - return int(numeric_value) if numeric_value.is_integer() else numeric_value - case SegmentType.BOOLEAN: - lower_value = value.lower() - bool_map = {"true": True, "false": False, "1": True, "0": False, "yes": True, "no": False} - if lower_value not in bool_map: - raise ValueError(f"Cannot convert '{value}' to boolean") - return bool_map[lower_value] - case ( - SegmentType.OBJECT - | SegmentType.FILE - | SegmentType.ARRAY_ANY - | SegmentType.ARRAY_STRING - | SegmentType.ARRAY_NUMBER - | SegmentType.ARRAY_OBJECT - | SegmentType.ARRAY_FILE - | SegmentType.ARRAY_BOOLEAN - | SegmentType.SECRET - | SegmentType.INTEGER - | SegmentType.FLOAT - | SegmentType.NONE - | SegmentType.GROUP - ): - raise ValueError(f"Unsupported type '{param_type}' for form data parameter '{param_name}'") - case _: - raise ValueError(f"Unsupported type '{param_type}' for form data parameter '{param_name}'") - - @classmethod - def _validate_json_value(cls, param_name: str, value: Any, param_type: SegmentType | str) -> Any: - """Validate JSON values against expected types. - - Args: - param_name: Name of the parameter for error reporting - value: The value to validate - param_type: The expected parameter type (SegmentType) - - Returns: - Any: The validated value (unchanged if valid) - - Raises: - ValueError: If the value type doesn't match the expected type - """ - param_type_enum = cls._coerce_segment_type(param_type, param_name=param_name) - if param_type_enum is None: - return value - - matches_type = param_type_enum.is_valid(value, array_validation=ArrayValidation.ALL) - if not matches_type or cls._is_boolean_for_number(value, param_type_enum): - actual_type = type(value).__name__ - expected_type = cls._expected_type_label(param_type_enum) - raise ValueError(f"Expected {expected_type}, got {actual_type}") - - return value - - @staticmethod - def _is_boolean_for_number(value: Any, param_type: SegmentType) -> bool: - """Return True when a JSON boolean is given for a number parameter. - - JSON ``true``/``false`` decode to ``bool``, a subclass of ``int``, so - ``SegmentType.is_valid`` accepts them as numbers. - """ - match param_type: - case SegmentType.NUMBER | SegmentType.INTEGER | SegmentType.FLOAT: - return isinstance(value, bool) - case SegmentType.ARRAY_NUMBER: - return isinstance(value, list) and any(isinstance(item, bool) for item in value) - case _: - return False - - @classmethod - def _coerce_segment_type(cls, param_type: SegmentType | str, *, param_name: str) -> SegmentType | None: - if isinstance(param_type, SegmentType): - return param_type - try: - return SegmentType(param_type) - except Exception: - logger.warning("Unknown parameter type: %s for parameter %s", param_type, param_name) - return None - - @staticmethod - def _expected_type_label(param_type: SegmentType) -> str: - match param_type: - case SegmentType.ARRAY_STRING: - return "array of strings" - case SegmentType.ARRAY_NUMBER: - return "array of numbers" - case SegmentType.ARRAY_BOOLEAN: - return "array of booleans" - case SegmentType.ARRAY_OBJECT: - return "array of objects" - case _: - return param_type.value - - @classmethod - def _validate_required_headers(cls, headers: dict[str, Any], header_configs: Sequence[WebhookParameter]) -> None: - """Validate required headers are present. - - Args: - headers: Request headers dictionary - header_configs: List of header configuration dictionaries - - Raises: - ValueError: If required headers are missing - """ - headers_lower = {k.lower(): v for k, v in headers.items()} - headers_sanitized = {cls._sanitize_key(k).lower(): v for k, v in headers.items()} - for header_config in header_configs: - if header_config.required: - header_name = header_config.name - sanitized_name = cls._sanitize_key(header_name).lower() - if header_name.lower() not in headers_lower and sanitized_name not in headers_sanitized: - raise ValueError(f"Required header missing: {header_name}") - - @classmethod - def _validate_http_metadata(cls, webhook_data: RawWebhookDataDict, node_data: WebhookData) -> ValidationResultDict: - """Validate HTTP method and content-type. - - Args: - webhook_data: Extracted webhook data containing method and headers - node_data: Node configuration containing expected method and content-type - - Returns: - dict[str, Any]: Validation result with 'valid' key and optional 'error' key - """ - # Validate HTTP method - configured_method = node_data.method.value.upper() - request_method = webhook_data["method"].upper() - if configured_method != request_method: - return cls._validation_error(f"HTTP method mismatch. Expected {configured_method}, got {request_method}") - - # Validate Content-type - configured_content_type = node_data.content_type.value.lower() - request_content_type = cls._extract_content_type(webhook_data["headers"]) - - if configured_content_type != request_content_type: - return cls._validation_error( - f"Content-type mismatch. Expected {configured_content_type}, got {request_content_type}" - ) - - return {"valid": True} - - @classmethod - def _extract_content_type(cls, headers: dict[str, Any]) -> str: - """Extract and normalize content-type from headers. - - Args: - headers: Request headers dictionary - - Returns: - str: Normalized content-type (main type without parameters) - """ - content_type = headers.get("Content-Type", "").lower() - if not content_type: - content_type = headers.get("content-type", "application/json").lower() - # Extract the main content type (ignore parameters like boundary) - return content_type.split(";")[0].strip() - - @classmethod - def _validation_error(cls, error_message: str) -> ValidationResultDict: - """Create a standard validation error response. - - Args: - error_message: The error message to include - - Returns: - dict[str, Any]: Validation error response with 'valid' and 'error' keys - """ - return {"valid": False, "error": error_message} - - @classmethod - def _can_convert_to_number(cls, value: str) -> bool: - """Check if a string can be converted to a finite number.""" - try: - return math.isfinite(float(value)) - except ValueError: - return False - - @classmethod - def build_workflow_inputs(cls, webhook_data: RawWebhookDataDict) -> WorkflowInputsDict: - """Construct workflow inputs payload from webhook data. - - Args: - webhook_data: Processed webhook data containing headers, query params, and body - - Returns: - dict[str, Any]: Workflow inputs formatted for execution - """ - return { - "webhook_data": webhook_data, - "webhook_headers": webhook_data.get("headers", {}), - "webhook_query_params": webhook_data.get("query_params", {}), - "webhook_body": webhook_data.get("body", {}), - } - - @classmethod - def trigger_workflow_execution( - cls, - webhook_trigger: WorkflowWebhookTrigger, - webhook_data: RawWebhookDataDict, - workflow: Workflow, - *, - end_users: WebhookEndUserProvisioner, - trigger_status: AppTriggerService, - workflow_dispatch: TriggerWorkflowDispatcher, + webhooks: WebhookQuery, + end_users: TriggerEndUserProvisioner, + dispatch: TriggerWorkflowDispatcher, + reserve_quota: Callable[[QuotaType, str], QuotaCharge], + mark_rate_limited: Callable[[str], None], ) -> None: - """Trigger workflow execution via AsyncWorkflowService. + self._webhooks = webhooks + self._end_users = end_users + self._dispatch = dispatch + self._reserve_quota = reserve_quota + self._mark_rate_limited = mark_rate_limited - Args: - webhook_trigger: The webhook trigger object - webhook_data: Processed webhook data for workflow inputs - workflow: The workflow to execute + def prepare(self, *, webhook_id: str, debug: bool) -> WebhookContext: + context = self._webhooks.get_webhook_context(webhook_id=webhook_id, debug=debug) + if isinstance(context, UnavailableWebhook): + if context.status == AppTriggerStatus.RATE_LIMITED: + raise QuotaExceededError(feature=QuotaType.TRIGGER.value, tenant_id=context.tenant_id, required=1) + raise WebhookNotFoundError(f"Webhook trigger is disabled for webhook {webhook_id}") + return context - Raises: - QuotaExceededError: If the tenant has exhausted its trigger or workflow execution quota - ValueError: If tenant owner is not found - Exception: If workflow execution fails - """ + def resolve_actor(self, *, context: WebhookContext) -> TriggerActor: + return self._end_users.get_or_create_trigger_end_user(tenant_id=context.tenant_id, app_id=context.app_id) + + def execute( + self, + *, + context: WebhookContext, + data: RawWebhookDataDict, + actor: TriggerActor | None = None, + ) -> None: + resolved_actor = actor or self.resolve_actor(context=context) try: - workflow_inputs = cls.build_workflow_inputs(webhook_data) - - trigger_data = WebhookTriggerData( - app_id=webhook_trigger.app_id, - workflow_id=workflow.id, - root_node_id=webhook_trigger.node_id, - inputs=workflow_inputs, - tenant_id=webhook_trigger.tenant_id, - ) - - end_user = cls._resolve_trigger_end_user(webhook_trigger, end_users) - - try: - quota_charge = QuotaService.reserve(QuotaType.TRIGGER, webhook_trigger.tenant_id) - except QuotaExceededError: - trigger_status.mark_tenant_triggers_rate_limited(webhook_trigger.tenant_id) - raise - - try: - workflow_dispatch.trigger_workflow_async( - actor=TriggerActor(id=end_user.id, role=CreatorUserRole.END_USER), trigger_data=trigger_data - ) - quota_charge.commit() - except Exception: - quota_charge.refund() - raise - - except QuotaExceededError as e: - logger.info( - "Tenant %s quota exceeded for feature %s, skipping webhook trigger %s", - webhook_trigger.tenant_id, - e.feature, - webhook_trigger.webhook_id, - ) + charge = self._reserve_quota(QuotaType.TRIGGER, context.tenant_id) + except QuotaExceededError: + self._mark_rate_limited(context.tenant_id) raise - except Exception: - logger.exception("Failed to trigger workflow for webhook %s", webhook_trigger.webhook_id) - raise - - @classmethod - def generate_webhook_response(cls, node_config: NodeConfigDict) -> tuple[dict[str, Any], int]: - """Generate HTTP response based on node configuration. - - Args: - node_config: Node configuration containing response settings - - Returns: - tuple[dict[str, Any], int]: Response data and HTTP status code - """ - node_data = WebhookData.model_validate(node_config["data"], from_attributes=True) - - # Get configured status code and response body - status_code = node_data.status_code - response_body = node_data.response_body - - # Parse response body as JSON if it's valid JSON, otherwise return as text try: - if response_body: - try: - response_data = ( - json.loads(response_body) - if response_body.strip().startswith(("{", "[")) - else {"message": response_body} - ) - except json.JSONDecodeError: - response_data = {"message": response_body} - else: - response_data = {"status": "success", "message": "Webhook processed successfully"} - except Exception: - response_data = {"message": response_body or "Webhook processed successfully"} - - return response_data, status_code - - @classmethod - def sync_webhook_relationships(cls, app: App, workflow: Workflow, *, remove_stale: bool = True): - """ - Sync webhook relationships in DB. - - 1. Check if the workflow has any webhook trigger nodes - 2. Fetch the nodes from DB, see if there were any webhook records already - 3. Diff the nodes and the webhook records, creating missing records and optionally deleting stale records - - Draft workflow synchronization preserves stale records so undo can restore a - webhook node without changing its URL. Published workflow synchronization - removes stale records after the deletion becomes effective. - - Approach: - Frequent DB operations may cause performance issues, using Redis to cache it instead. - If any record exists, cache it. - - Limits: - - Maximum 5 webhook nodes per workflow - """ - - class Cache(BaseModel): - """ - Cache model for webhook nodes - """ - - record_id: str - node_id: str - webhook_id: str - - nodes_id_in_graph = [node_id for node_id, _ in workflow.walk_nodes(TRIGGER_WEBHOOK_NODE_TYPE)] - - # Check webhook node limit - if len(nodes_id_in_graph) > cls.MAX_WEBHOOK_NODES_PER_WORKFLOW: - raise ValueError( - f"Workflow exceeds maximum webhook node limit. " - f"Found {len(nodes_id_in_graph)} webhook nodes, maximum allowed is {cls.MAX_WEBHOOK_NODES_PER_WORKFLOW}" + self._dispatch.trigger_workflow_async( + actor=resolved_actor, + trigger_data=WebhookTriggerData( + tenant_id=context.tenant_id, + app_id=context.app_id, + workflow_id=context.workflow_id, + root_node_id=context.node_id, + inputs=build_webhook_workflow_inputs(data), + ), ) - - not_found_in_cache: list[str] = [] - for node_id in nodes_id_in_graph: - # firstly check if the node exists in cache - if not redis_client.get(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}"): - not_found_in_cache.append(node_id) - continue - - lock_key = f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:apps:{app.id}:lock" - lock = redis_client.lock(lock_key, timeout=10) - lock_acquired = False - - try: - # acquire the lock with blocking and timeout - lock_acquired = lock.acquire(blocking=True, blocking_timeout=10) - if not lock_acquired: - logger.warning("Failed to acquire lock for webhook sync, app %s", app.id) - raise RuntimeError("Failed to acquire lock for webhook trigger synchronization") - - with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session: - # fetch the non-cached nodes from DB - all_records = session.scalars( - select(WorkflowWebhookTrigger).where( - WorkflowWebhookTrigger.app_id == app.id, - WorkflowWebhookTrigger.tenant_id == app.tenant_id, - ) - ).all() - - nodes_id_in_db = {node.node_id: node for node in all_records} - - # get the nodes not found both in cache and DB - nodes_not_found = [node_id for node_id in not_found_in_cache if node_id not in nodes_id_in_db] - - # create new webhook records - for node_id in nodes_not_found: - if app.created_by is None: - raise RuntimeError("App is missing created_by; cannot create webhook trigger") - webhook_record = WorkflowWebhookTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id=node_id, - webhook_id=cls.generate_webhook_id(), - created_by=app.created_by, - ) - session.add(webhook_record) - session.flush() - cache = Cache(record_id=webhook_record.id, node_id=node_id, webhook_id=webhook_record.webhook_id) - redis_client.set( - f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}", cache.model_dump_json(), ex=60 * 60 - ) - - if remove_stale: - # Delete relationships only when reconciling an effective published workflow. - for node_id in nodes_id_in_db: - if node_id not in nodes_id_in_graph: - session.delete(nodes_id_in_db[node_id]) - redis_client.delete(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}") + charge.commit() except Exception: - logger.exception("Failed to sync webhook relationships for app %s", app.id) + charge.refund() raise - finally: - # release the lock only if it was acquired - if lock_acquired: - try: - lock.release() - except Exception: - logger.exception("Failed to release lock for webhook sync, app %s", app.id) - - @classmethod - def generate_webhook_id(cls) -> str: - """ - Generate unique 24-character webhook ID - - Deduplication is not needed, DB already has unique constraint on webhook_id. - """ - # Generate 24-character random string - return secrets.token_urlsafe(18)[:24] # token_urlsafe gives base64url, take first 24 chars diff --git a/api/services/trigger/workflow_dispatch_gateway.py b/api/services/trigger/workflow_dispatch_gateway.py index 15b84a167b9..8335330eb94 100644 --- a/api/services/trigger/workflow_dispatch_gateway.py +++ b/api/services/trigger/workflow_dispatch_gateway.py @@ -12,6 +12,12 @@ class AppScopedTriggerEndUsers: def __init__(self, *, end_users: AppScopedEndUserService[EndUser]) -> None: self._end_users = end_users + def get_or_create_trigger_end_user(self, *, tenant_id: str, app_id: str) -> TriggerActor: + record = self._end_users.get_or_create_end_user_by_type( + type=EndUserType.TRIGGER, tenant_id=tenant_id, app_id=app_id, user_id=None + ) + return TriggerActor(id=record.id, role=CreatorUserRole.END_USER) + def create_trigger_end_users( self, *, tenant_id: str, app_ids: list[str], user_id: str ) -> Mapping[str, TriggerActor]: diff --git a/api/services/trigger/workflow_dispatch_service.py b/api/services/trigger/workflow_dispatch_service.py index c3fa7ae9708..a9a1baf4f00 100644 --- a/api/services/trigger/workflow_dispatch_service.py +++ b/api/services/trigger/workflow_dispatch_service.py @@ -28,6 +28,8 @@ logger = logging.getLogger(__name__) class TriggerEndUserProvisioner(Protocol): + def get_or_create_trigger_end_user(self, *, tenant_id: str, app_id: str) -> TriggerActor: ... + def create_trigger_end_users( self, *, diff --git a/api/tests/e2e_tests/containers/services/test_webhook_service.py b/api/tests/e2e_tests/containers/services/test_webhook_service.py index 6dc943d9ef1..6e826bfc772 100644 --- a/api/tests/e2e_tests/containers/services/test_webhook_service.py +++ b/api/tests/e2e_tests/containers/services/test_webhook_service.py @@ -1,166 +1,82 @@ -"""Database integration tests for webhook trigger and workflow lookup.""" +"""Native debug HTTP uses the real Redis waiting pool and never queues workflow execution.""" -import json -from typing import TypedDict +from contextvars import Context +from http import HTTPStatus +from uuid import uuid4 import pytest -from faker import Faker -from flask import Flask -from sqlalchemy.orm import Session +from sqlalchemy import select -from models.account import Account, Tenant -from models.enums import AppTriggerStatus, AppTriggerType -from models.model import App -from models.trigger import AppTrigger, WorkflowWebhookTrigger -from models.workflow import Workflow -from models.workflow import WorkflowType as ModelWorkflowType -from services.trigger.webhook_service import WebhookService -from tests.e2e_tests.containers.helpers import accounts as account_fixtures -from tests.e2e_tests.containers.helpers import generate_valid_password +from core.trigger.debug.event_bus import TriggerDebugEventBus +from core.trigger.debug.events import WebhookDebugEvent, build_webhook_pool_key +from core.trigger.utils.endpoint import generate_webhook_trigger_endpoint +from models.enums import AppTriggerStatus +from models.model import EndUser +from models.trigger import AppTrigger, WorkflowTriggerLog +from tests.e2e_tests.containers.services import test_trigger_subscription_builder +from tests.unit_tests.controllers.console.workspace import test_trigger_oauth_client +from tests.unit_tests.controllers.console.workspace.test_trigger_oauth_client import _Harness +from tests.unit_tests.controllers.trigger import test_webhook +from tests.unit_tests.controllers.trigger.test_webhook import _configure, _Webhook + +application_redis = test_trigger_subscription_builder.application_redis +sqlite_engine = test_trigger_subscription_builder.sqlite_engine +sqlite_session_factory = test_trigger_subscription_builder.sqlite_session_factory +harness = test_trigger_oauth_client.harness +webhook = test_webhook.webhook -class WebhookIntegrationData(TypedDict): - tenant: Tenant - account: Account - app: App - workflow: Workflow - webhook_trigger: WorkflowWebhookTrigger - webhook_id: str - app_trigger: AppTrigger - - -@pytest.fixture -def test_data( - db_session_with_containers: Session, - monkeypatch: pytest.MonkeyPatch, -) -> WebhookIntegrationData: - """Persist the webhook graph with account and workspace creation enabled.""" - - fake = Faker() - monkeypatch.setattr( - "services.account.login_adapters.SystemFeatureService.is_registration_allowed", - lambda: True, - ) - monkeypatch.setattr( - "services.account.login_adapters.SystemFeatureService.is_workspace_creation_allowed", - lambda: True, - ) - account = account_fixtures.create_account( - email=fake.email(), - name=fake.name(), - interface_language="en-US", - password=generate_valid_password(fake), - session=db_session_with_containers, - ) - account_fixtures.create_owner_workspace(account, name=fake.company(), session=db_session_with_containers) - tenant = account.current_tenant - assert tenant is not None - app = App( - tenant_id=tenant.id, - name=fake.company(), - description=fake.text(), - mode="workflow", - icon="", - icon_background="", - enable_site=True, - enable_api=True, - ) - db_session_with_containers.add(app) - db_session_with_containers.flush() - workflow_data = { - "nodes": [ - { - "id": "webhook_node", - "type": "webhook", - "data": { - "type": "trigger-webhook", - "title": "Test Webhook", - "method": "post", - "content_type": "application/json", - "headers": [ - {"name": "Authorization", "required": True}, - {"name": "Content-Type", "required": False}, - ], - "params": [ - {"name": "version", "required": True}, - {"name": "format", "required": False}, - ], - "body": [ - {"name": "message", "type": "string", "required": True}, - {"name": "count", "type": "number", "required": False}, - {"name": "upload", "type": "file", "required": False}, - ], - "status_code": 200, - "response_body": '{"status": "success"}', - "timeout": 30, - }, - } - ], - "edges": [], - } - workflow = Workflow( - tenant_id=tenant.id, - app_id=app.id, - type=ModelWorkflowType("workflow"), - graph=json.dumps(workflow_data), - _features=json.dumps({}), - created_by=account.id, - _environment_variables=[], - _conversation_variables=[], - version="1.0", - ) - db_session_with_containers.add(workflow) - db_session_with_containers.flush() - app.workflow_id = workflow.id - webhook_id = fake.uuid4()[:16] - webhook_trigger = WorkflowWebhookTrigger( - app_id=app.id, - node_id="webhook_node", - tenant_id=tenant.id, - webhook_id=webhook_id, - created_by=account.id, - ) - db_session_with_containers.add(webhook_trigger) - db_session_with_containers.flush() - app_trigger = AppTrigger( - tenant_id=tenant.id, - app_id=app.id, - node_id="webhook_node", - trigger_type=AppTriggerType.TRIGGER_WEBHOOK, - provider_name="webhook", - title="Test Webhook", - status=AppTriggerStatus.ENABLED, - ) - db_session_with_containers.add(app_trigger) - db_session_with_containers.commit() - return { - "tenant": tenant, - "account": account, - "app": app, - "workflow": workflow, - "webhook_trigger": webhook_trigger, - "webhook_id": webhook_id, - "app_trigger": app_trigger, - } - - -class TestWebhookServiceDatabaseLookup: - def test_get_webhook_trigger_and_workflow_success( - self, - test_data: WebhookIntegrationData, - flask_app_with_containers: Flask, - ) -> None: - with flask_app_with_containers.app_context(): - webhook_trigger, workflow, node_config = WebhookService.get_webhook_trigger_and_workflow( - test_data["webhook_id"] +@pytest.mark.parametrize("listener", ["absent", "other-app", "matching"]) +def test_debug_http_requires_matching_listener_and_preserves_draft_response( + harness: _Harness, webhook: _Webhook, listener: str +) -> None: + _configure(harness, webhook, status_code=202, response_body='["accepted"]') + with harness.session_factory.begin() as session: + trigger = session.get(AppTrigger, webhook.trigger_id) + assert trigger is not None + trigger.status = AppTriggerStatus.DISABLED + listener_app = str(uuid4()) if listener == "other-app" else webhook.app_id + pool_key = build_webhook_pool_key(tenant_id=harness.tenant_id, app_id=listener_app, node_id=webhook.node_id) + if listener != "absent": + assert ( + TriggerDebugEventBus.poll( + event_type=WebhookDebugEvent, + pool_key=pool_key, + tenant_id=harness.tenant_id, + user_id=harness.account_id, + app_id=listener_app, + node_id=webhook.node_id, ) + is None + ) - assert webhook_trigger.webhook_id == test_data["webhook_id"] - assert workflow.app_id == test_data["app"].id - assert node_config["id"] == "webhook_node" - assert node_config["data"].title == "Test Webhook" + response = Context().run(harness.client.post, webhook.debug_url, json={"ref": "main"}) - def test_get_webhook_trigger_and_workflow_not_found(self, flask_app_with_containers: Flask) -> None: - with flask_app_with_containers.app_context(): - with pytest.raises(ValueError, match="Webhook not found"): - WebhookService.get_webhook_trigger_and_workflow("nonexistent_webhook") + if listener == "matching": + assert response.status_code == HTTPStatus.ACCEPTED + assert response.get_json() == ["accepted"] + event = TriggerDebugEventBus.poll( + event_type=WebhookDebugEvent, + pool_key=pool_key, + tenant_id=harness.tenant_id, + user_id=harness.account_id, + app_id=webhook.app_id, + node_id=webhook.node_id, + ) + assert event is not None + assert event.node_id == webhook.node_id + assert event.payload["webhook_data"]["body"] == {"ref": "main"} + assert event.payload["inputs"]["webhook_body"] == {"ref": "main"} + else: + assert response.status_code == HTTPStatus.CONFLICT + assert response.get_json() == { + "error": "No active debug listener", + "message": ( + "The webhook debug URL only works while the Variable Inspector is listening. " + "Use the published webhook URL to execute the workflow in Celery." + ), + "execution_url": generate_webhook_trigger_endpoint(webhook.webhook_id), + } + with harness.session_factory() as session: + assert session.scalar(select(EndUser)) is None + assert session.scalar(select(WorkflowTriggerLog)) is None diff --git a/api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py b/api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py deleted file mode 100644 index 13612433125..00000000000 --- a/api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py +++ /dev/null @@ -1,589 +0,0 @@ -from __future__ import annotations - -import json -import logging -from datetime import datetime -from unittest.mock import MagicMock, patch -from uuid import uuid4 - -import pytest -from flask import Flask -from sqlalchemy import select -from sqlalchemy.orm import Session - -from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE -from enums import QuotaType -from extensions.ext_application_services import application_services -from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole -from models.enums import AppTriggerStatus, AppTriggerType, EndUserType -from models.model import App, EndUser -from models.trigger import AppTrigger, WorkflowWebhookTrigger -from models.workflow import Workflow -from models.workflow import WorkflowType as ModelWorkflowType -from services.errors.app import QuotaExceededError -from services.trigger.webhook_service import WebhookService - - -class _EndUserServiceStub: - def __init__(self, result: EndUser | Exception) -> None: - self._result = result - self.calls: list[tuple[EndUserType, str, str, str | None]] = [] - - def get_or_create_end_user_by_type( - self, - type: EndUserType, - tenant_id: str, - app_id: str, - user_id: str | None = None, - ) -> EndUser: - self.calls.append((type, tenant_id, app_id, user_id)) - if isinstance(self._result, Exception): - raise self._result - return self._result - - -def _end_user(*, tenant_id: str, app_id: str) -> EndUser: - return EndUser( - id=str(uuid4()), - tenant_id=tenant_id, - app_id=app_id, - type=EndUserType.TRIGGER, - session_id="trigger-session", - ) - - -class WebhookServiceRelationshipFactory: - @staticmethod - def create_account_and_tenant(db_session_with_containers: Session) -> tuple[Account, Tenant]: - account = Account( - name=f"Account {uuid4()}", - email=f"webhook-{uuid4()}@example.com", - password="hashed-password", - password_salt="salt", - interface_language="en-US", - timezone="UTC", - ) - db_session_with_containers.add(account) - db_session_with_containers.commit() - - tenant = Tenant(name=f"Tenant {uuid4()}", plan="basic", status="normal") - db_session_with_containers.add(tenant) - db_session_with_containers.commit() - - join = TenantAccountJoin( - tenant_id=tenant.id, - account_id=account.id, - role=TenantAccountRole.OWNER, - current=True, - ) - db_session_with_containers.add(join) - db_session_with_containers.commit() - - account.set_current_tenant_with_session(tenant, session=db_session_with_containers) - return account, tenant - - @staticmethod - def create_app(db_session_with_containers: Session, tenant: Tenant, account: Account) -> App: - app = App( - tenant_id=tenant.id, - name=f"Webhook App {uuid4()}", - description="", - mode="workflow", - icon_type="emoji", - icon="bot", - icon_background="#FFFFFF", - enable_site=False, - enable_api=True, - api_rpm=100, - api_rph=100, - is_demo=False, - is_public=False, - is_universal=False, - created_by=account.id, - updated_by=account.id, - ) - db_session_with_containers.add(app) - db_session_with_containers.commit() - return app - - @staticmethod - def create_workflow( - db_session_with_containers: Session, - *, - app: App, - account: Account, - node_ids: list[str], - version: str, - ) -> Workflow: - graph = { - "nodes": [ - { - "id": node_id, - "data": { - "type": TRIGGER_WEBHOOK_NODE_TYPE, - "title": f"Webhook {node_id}", - "method": "post", - "content_type": "application/json", - "headers": [], - "params": [], - "body": [], - "status_code": 200, - "response_body": '{"status": "ok"}', - "timeout": 30, - }, - } - for node_id in node_ids - ], - "edges": [], - } - - workflow = Workflow( - tenant_id=app.tenant_id, - app_id=app.id, - type=ModelWorkflowType("workflow"), - graph=json.dumps(graph), - _features=json.dumps({}), - created_by=account.id, - updated_by=account.id, - _environment_variables=[], - _conversation_variables=[], - version=version, - ) - db_session_with_containers.add(workflow) - db_session_with_containers.commit() - return workflow - - @staticmethod - def create_webhook_trigger( - db_session_with_containers: Session, - *, - app: App, - account: Account, - node_id: str, - webhook_id: str | None = None, - ) -> WorkflowWebhookTrigger: - webhook_trigger = WorkflowWebhookTrigger( - app_id=app.id, - node_id=node_id, - tenant_id=app.tenant_id, - webhook_id=webhook_id or uuid4().hex[:24], - created_by=account.id, - ) - db_session_with_containers.add(webhook_trigger) - db_session_with_containers.commit() - return webhook_trigger - - @staticmethod - def create_app_trigger( - db_session_with_containers: Session, - *, - app: App, - node_id: str, - status: AppTriggerStatus, - ) -> AppTrigger: - app_trigger = AppTrigger( - tenant_id=app.tenant_id, - app_id=app.id, - node_id=node_id, - trigger_type=AppTriggerType.TRIGGER_WEBHOOK, - provider_name="webhook", - title=f"Webhook {node_id}", - status=status, - ) - db_session_with_containers.add(app_trigger) - db_session_with_containers.commit() - return app_trigger - - -class TestWebhookServiceLookupWithContainers: - def test_get_webhook_trigger_and_workflow_raises_when_app_trigger_missing( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-14.001" - ) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - - with pytest.raises(ValueError, match="App trigger not found"): - WebhookService.get_webhook_trigger_and_workflow(webhook_trigger.webhook_id) - - def test_get_webhook_trigger_and_workflow_raises_when_app_trigger_rate_limited( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-14.001" - ) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - factory.create_app_trigger( - db_session_with_containers, app=app, node_id="node-1", status=AppTriggerStatus.RATE_LIMITED - ) - - with pytest.raises(QuotaExceededError) as exc_info: - WebhookService.get_webhook_trigger_and_workflow(webhook_trigger.webhook_id) - - assert exc_info.value.feature == QuotaType.TRIGGER.value - assert exc_info.value.tenant_id == tenant.id - assert exc_info.value.required == 1 - - def test_get_webhook_trigger_and_workflow_raises_when_app_trigger_disabled( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-14.001" - ) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - factory.create_app_trigger( - db_session_with_containers, app=app, node_id="node-1", status=AppTriggerStatus.DISABLED - ) - - with pytest.raises(ValueError, match="disabled"): - WebhookService.get_webhook_trigger_and_workflow(webhook_trigger.webhook_id) - - def test_get_webhook_trigger_and_workflow_raises_when_workflow_missing( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - factory.create_app_trigger( - db_session_with_containers, app=app, node_id="node-1", status=AppTriggerStatus.ENABLED - ) - - with pytest.raises(ValueError, match="Workflow not found"): - WebhookService.get_webhook_trigger_and_workflow(webhook_trigger.webhook_id) - - def test_get_webhook_trigger_and_workflow_uses_app_workflow_id( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - current_workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-14.001" - ) - newer_workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-15.001" - ) - current_workflow.created_at = datetime(2026, 4, 14) - newer_workflow.created_at = datetime(2026, 4, 15) - app.workflow_id = current_workflow.id - db_session_with_containers.commit() - - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - factory.create_app_trigger( - db_session_with_containers, app=app, node_id="node-1", status=AppTriggerStatus.ENABLED - ) - - got_trigger, got_workflow, got_node_config = WebhookService.get_webhook_trigger_and_workflow( - webhook_trigger.webhook_id - ) - - assert got_trigger.id == webhook_trigger.id - assert got_workflow.id == current_workflow.id - assert got_workflow.id != newer_workflow.id - assert got_node_config["id"] == "node-1" - - def test_get_webhook_trigger_and_workflow_returns_debug_draft_workflow( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - factory.create_workflow( - db_session_with_containers, - app=app, - account=account, - node_ids=["published-node"], - version="2026-04-14.001", - ) - draft_workflow = factory.create_workflow( - db_session_with_containers, - app=app, - account=account, - node_ids=["debug-node"], - version=Workflow.VERSION_DRAFT, - ) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="debug-node" - ) - - got_trigger, got_workflow, got_node_config = WebhookService.get_webhook_trigger_and_workflow( - webhook_trigger.webhook_id, - is_debug=True, - ) - - assert got_trigger.id == webhook_trigger.id - assert got_workflow.id == draft_workflow.id - assert got_node_config["id"] == "debug-node" - - -class TestWebhookServiceTriggerExecutionWithContainers: - def test_trigger_workflow_execution_triggers_async_workflow_successfully( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-14.001" - ) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - - end_user = _end_user(tenant_id=tenant.id, app_id=app.id) - webhook_data = {"body": {"value": 1}, "headers": {}, "query_params": {}, "files": {}, "method": "POST"} - - quota_charge = MagicMock() - - with ( - patch( - "services.trigger.webhook_service.QuotaService.reserve", - return_value=quota_charge, - ) as mock_reserve, - patch("services.async_workflow_service.AsyncWorkflowService.trigger_workflow_async") as mock_trigger, - ): - WebhookService.trigger_workflow_execution( - webhook_trigger, - webhook_data, - workflow, - end_users=_EndUserServiceStub(end_user), - trigger_status=application_services().apps.trigger_status, - workflow_dispatch=application_services().workspaces.workflow_dispatch, - ) - - mock_reserve.assert_called_once() - reserve_args = mock_reserve.call_args.args - assert reserve_args[0] == QuotaType.TRIGGER - assert reserve_args[1] == webhook_trigger.tenant_id - quota_charge.commit.assert_called_once() - mock_trigger.assert_called_once() - trigger_args = mock_trigger.call_args.kwargs - assert trigger_args["actor"].id == end_user.id - assert trigger_args["trigger_data"].workflow_id == workflow.id - assert trigger_args["trigger_data"].root_node_id == webhook_trigger.node_id - - def test_trigger_workflow_execution_marks_tenant_rate_limited_when_quota_exceeded( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-14.001" - ) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - - app_trigger = factory.create_app_trigger( - db_session_with_containers, app=app, node_id="node-1", status=AppTriggerStatus.ENABLED - ) - with patch( - "services.trigger.webhook_service.QuotaService.reserve", - side_effect=QuotaExceededError(feature="trigger", tenant_id=tenant.id, required=1), - ): - with pytest.raises(QuotaExceededError): - WebhookService.trigger_workflow_execution( - webhook_trigger, - {"body": {}, "headers": {}, "query_params": {}, "files": {}, "method": "POST"}, - workflow, - end_users=_EndUserServiceStub(_end_user(tenant_id=tenant.id, app_id=app.id)), - trigger_status=application_services().apps.trigger_status, - workflow_dispatch=application_services().workspaces.workflow_dispatch, - ) - - db_session_with_containers.refresh(app_trigger) - assert app_trigger.status == AppTriggerStatus.RATE_LIMITED - - def test_trigger_workflow_execution_logs_and_reraises_unexpected_errors( - self, - db_session_with_containers: Session, - flask_app_with_containers: Flask, - caplog: pytest.LogCaptureFixture, - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version="2026-04-14.001" - ) - webhook_trigger = factory.create_webhook_trigger( - db_session_with_containers, app=app, account=account, node_id="node-1" - ) - caplog.set_level(logging.ERROR, logger="services.trigger.webhook_service") - - with pytest.raises(RuntimeError, match="boom"): - WebhookService.trigger_workflow_execution( - webhook_trigger, - {"body": {}, "headers": {}, "query_params": {}, "files": {}, "method": "POST"}, - workflow, - end_users=_EndUserServiceStub(RuntimeError("boom")), - trigger_status=application_services().apps.trigger_status, - workflow_dispatch=application_services().workspaces.workflow_dispatch, - ) - - assert caplog.messages.count(f"Failed to trigger workflow for webhook {webhook_trigger.webhook_id}") == 1 - - -class TestWebhookServiceRelationshipSyncWithContainers: - def test_sync_webhook_relationships_raises_when_workflow_exceeds_node_limit( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - node_ids = [f"node-{index}" for index in range(WebhookService.MAX_WEBHOOK_NODES_PER_WORKFLOW + 1)] - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=node_ids, version=Workflow.VERSION_DRAFT - ) - - with pytest.raises(ValueError, match="maximum webhook node limit"): - WebhookService.sync_webhook_relationships(app, workflow) - - def test_sync_webhook_relationships_raises_when_lock_not_acquired( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version=Workflow.VERSION_DRAFT - ) - lock = MagicMock() - lock.acquire.return_value = False - - with patch("services.trigger.webhook_service.redis_client.lock", return_value=lock): - with pytest.raises(RuntimeError, match="Failed to acquire lock"): - WebhookService.sync_webhook_relationships(app, workflow) - - def test_sync_webhook_relationships_creates_missing_records_and_deletes_stale_records( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - stale_trigger = factory.create_webhook_trigger( - db_session_with_containers, - app=app, - account=account, - node_id="node-stale", - webhook_id="stale-webhook-id-000001", - ) - stale_trigger_id = stale_trigger.id - workflow = factory.create_workflow( - db_session_with_containers, - app=app, - account=account, - node_ids=["node-new"], - version=Workflow.VERSION_DRAFT, - ) - - with patch( - "services.trigger.webhook_service.WebhookService.generate_webhook_id", return_value="new-webhook-id-000001" - ): - WebhookService.sync_webhook_relationships(app, workflow) - - db_session_with_containers.expire_all() - records = db_session_with_containers.scalars( - select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.app_id == app.id) - ).all() - - assert [record.node_id for record in records] == ["node-new"] - assert records[0].webhook_id == "new-webhook-id-000001" - assert db_session_with_containers.get(WorkflowWebhookTrigger, stale_trigger_id) is None - - def test_sync_webhook_relationships_sets_redis_cache_for_new_record( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, - app=app, - account=account, - node_ids=["node-cache"], - version=Workflow.VERSION_DRAFT, - ) - cache_key = f"{WebhookService.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:node-cache" - - with patch( - "services.trigger.webhook_service.WebhookService.generate_webhook_id", return_value="cache-webhook-id-00001" - ): - WebhookService.sync_webhook_relationships(app, workflow) - - cached_payload = WebhookServiceRelationshipFactory._read_cache(cache_key) - assert cached_payload is not None - assert cached_payload["node_id"] == "node-cache" - assert cached_payload["webhook_id"] == "cache-webhook-id-00001" - - def test_sync_webhook_relationships_logs_when_lock_release_fails( - self, - db_session_with_containers: Session, - flask_app_with_containers: Flask, - caplog: pytest.LogCaptureFixture, - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=[], version=Workflow.VERSION_DRAFT - ) - lock = MagicMock() - lock.acquire.return_value = True - lock.release.side_effect = RuntimeError("release failed") - caplog.set_level(logging.ERROR, logger="services.trigger.webhook_service") - - with patch("services.trigger.webhook_service.redis_client.lock", return_value=lock): - WebhookService.sync_webhook_relationships(app, workflow) - - assert caplog.messages.count(f"Failed to release lock for webhook sync, app {app.id}") == 1 - - -def _read_cache(cache_key: str) -> dict[str, str] | None: - from extensions.ext_redis import redis_client - - cached = redis_client.get(cache_key) - if not cached: - return None - if isinstance(cached, bytes): - cached = cached.decode("utf-8") - return json.loads(cached) - - -WebhookServiceRelationshipFactory._read_cache = staticmethod(_read_cache) diff --git a/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py b/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py index aa3cddbbca2..621cac10a8c 100644 --- a/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py +++ b/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py @@ -11,6 +11,7 @@ from uuid import uuid4 import pytest from flask import Flask from flask.testing import FlaskClient +from kombu.simple import SimpleQueue from sqlalchemy import select from sqlalchemy.orm import Session @@ -47,6 +48,8 @@ from services.trigger.subscription_query_service import TriggerSubscriptionRunti from services.workflow.entities import TriggerActor, TriggerData from services.workflow_service import WorkflowService from tasks import trigger_processing_tasks +from tests.helpers.unit.config_override import config_overrides_context +from tests.unit_tests.controllers.trigger import test_webhook as native_webhook_tests from .conftest import RecordingScheduleTask @@ -55,6 +58,8 @@ WEBHOOK_ID_PRODUCTION = "wh1234567890123456789012" WEBHOOK_ID_DEBUG = "whdebug1234567890123456" TEST_TRIGGER_URL = "https://trigger.example.com/base" +broker = native_webhook_tests.broker + def _build_workflow_graph(root_node_id: str, trigger_type: str) -> str: """Build a minimal workflow graph JSON for testing.""" @@ -156,7 +161,7 @@ def test_webhook_trigger_creates_trigger_log( db_session_with_containers: Session, tenant_and_account: tuple[Tenant, Account], app_model: App, - monkeypatch: pytest.MonkeyPatch, + broker: SimpleQueue, ) -> None: """Production webhook trigger should create a trigger log in the database.""" tenant, account = tenant_and_account @@ -198,39 +203,10 @@ def test_webhook_trigger_creates_trigger_log( db_session_with_containers.add_all([webhook_trigger, app_trigger]) db_session_with_containers.commit() - def _fake_trigger_workflow_async( - self: AsyncWorkflowService, *, actor: TriggerActor, trigger_data: TriggerData - ) -> SimpleNamespace: - session = db_session_with_containers - log = WorkflowTriggerLog( - tenant_id=trigger_data.tenant_id, - app_id=trigger_data.app_id, - workflow_id=trigger_data.workflow_id, - root_node_id=trigger_data.root_node_id, - trigger_metadata=trigger_data.trigger_metadata.model_dump_json() if trigger_data.trigger_metadata else "{}", - trigger_type=trigger_data.trigger_type, - workflow_run_id=None, - outputs=None, - trigger_data=trigger_data.model_dump_json(), - inputs=json.dumps(dict(trigger_data.inputs)), - status=WorkflowTriggerStatus.SUCCEEDED, - error="", - queue_name="triggered_workflow_dispatcher", - celery_task_id="celery-test", - created_by_role=CreatorUserRole.ACCOUNT, - created_by=account.id, + with config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY): + response = test_client_with_containers.post( + f"/triggers/webhook/{webhook_trigger.webhook_id}", json={"foo": "bar"} ) - session.add(log) - session.commit() - return SimpleNamespace(workflow_trigger_log_id=log.id, task_id=None, status="queued", queue="test") - - monkeypatch.setattr( - AsyncWorkflowService, - "trigger_workflow_async", - _fake_trigger_workflow_async, - ) - - response = test_client_with_containers.post(f"/triggers/webhook/{webhook_trigger.webhook_id}", json={"foo": "bar"}) assert response.status_code == 200 @@ -238,7 +214,14 @@ def test_webhook_trigger_creates_trigger_log( logs = db_session_with_containers.scalars( select(WorkflowTriggerLog).where(WorkflowTriggerLog.app_id == app_model.id) ).all() - assert logs, "Webhook trigger should create trigger log" + assert len(logs) == 1 + assert logs[0].status == WorkflowTriggerStatus.QUEUED + assert logs[0].root_node_id == webhook_node_id + assert json.loads(logs[0].inputs)["webhook_body"] == {"foo": "bar"} + message = broker.get(block=False) + assert message.payload[0] == [{"workflow_trigger_log_id": logs[0].id}] + assert message.headers["id"] == logs[0].celery_task_id + message.ack() @pytest.mark.parametrize("schedule_type", ["visual", "cron"]) @@ -390,7 +373,6 @@ def test_webhook_debug_dispatches_event( db_session_with_containers: Session, tenant_and_account: tuple[Tenant, Account], app_model: App, - monkeypatch: pytest.MonkeyPatch, ) -> None: """Webhook single-step debug should dispatch debug event and be pollable.""" tenant, account = tenant_and_account @@ -421,13 +403,6 @@ def test_webhook_debug_dispatches_event( db_session_with_containers.add(webhook_trigger) db_session_with_containers.commit() - debug_events: list[dict[str, Any]] = [] - original_dispatch = TriggerDebugEventBus.dispatch - monkeypatch.setattr( - "controllers.trigger.webhook.TriggerDebugEventBus.dispatch", - lambda **kwargs: (debug_events.append(kwargs), original_dispatch(**kwargs))[1], - ) - # Listener polls first to enter waiting pool poller = WebhookTriggerDebugEventPoller( tenant_id=tenant.id, @@ -445,12 +420,14 @@ def test_webhook_debug_dispatches_event( ) assert response.status_code == 200 - assert debug_events, "Debug event should be sent to event bus" # Second poll should get the event event = poller.poll() assert event is not None assert event.workflow_args["inputs"]["webhook_body"]["foo"] == "bar" - assert debug_events[0]["pool_key"].endswith(f":{app_model.id}:{webhook_node_id}") + assert ( + db_session_with_containers.scalar(select(WorkflowTriggerLog).where(WorkflowTriggerLog.app_id == app_model.id)) + is None + ) def test_schedule_trigger_creates_trigger_log( diff --git a/api/tests/unit_tests/controllers/common/test_access_response.py b/api/tests/unit_tests/controllers/common/test_access_response.py index 573e7d7e2bb..4bb1ff71d3b 100644 --- a/api/tests/unit_tests/controllers/common/test_access_response.py +++ b/api/tests/unit_tests/controllers/common/test_access_response.py @@ -16,7 +16,7 @@ from pytest_mock import MockerFixture from sqlalchemy import event from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, object_session, sessionmaker -from werkzeug.exceptions import NotFound, Unauthorized +from werkzeug.exceptions import Unauthorized import controllers.service_api.wraps as token_wraps import services.api_token_service as token_service @@ -27,7 +27,6 @@ from controllers.common.access_response import ( register_auth_error_response, ) from controllers.mcp import mcp -from controllers.trigger import webhook from extensions import ext_request_logging from extensions.ext_redis import RedisClientWrapper from libs.external_api import ExternalApi @@ -553,29 +552,6 @@ def test_mcp_reads_body_outside_sessions_and_rechecks_identity_before_execution( assert writes == [] -@pytest.mark.parametrize("route", ["webhook", "webhook-debug"]) -def test_webhook_missing_identity_uses_native_html_without_disclosing_id( - http_app: Flask, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, route: Literal["webhook", "webhook-debug"] -) -> None: - monkeypatch.setattr(webhook, "request", request) - monkeypatch.setattr(webhook, "jsonify", jsonify) - mocker.patch.object( - webhook.WebhookService, - "get_webhook_trigger_and_workflow", - side_effect=ValueError("Webhook not found: private-fixture"), - ) - execute = mocker.patch.object(webhook.WebhookService, "trigger_workflow_execution") - handler = webhook.handle_webhook if route == "webhook" else webhook.handle_webhook_debug - http_app.add_url_rule(f"/triggers/{route}/", view_func=handler, methods=["POST"]) - response = http_app.test_client().post(f"/triggers/{route}/private-fixture", json={}) - assert response.status_code == 404 - assert response.data == NotFound().get_body().encode() - assert response.headers["Content-Type"] == "text/html; charset=utf-8" - assert response.headers["Cache-Control"] == "no-store" - assert b"private-fixture" not in response.data - execute.assert_not_called() - - @pytest.mark.parametrize("status", [401, 403, 429]) def test_openapi_local_hook_preserves_native_details_and_oauth_challenge(http_app: Flask, status: int) -> None: payload = { diff --git a/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py b/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py index 6e3b0431e19..e41f21e6495 100644 --- a/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py +++ b/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py @@ -1,7 +1,6 @@ from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime -from functools import partial from inspect import unwrap from types import SimpleNamespace from unittest.mock import Mock, create_autospec @@ -17,65 +16,18 @@ from werkzeug.exceptions import UnprocessableEntity from controllers.inner_api.workspace import workspace as controller from controllers.inner_api.wraps import InnerApiUnauthorizedError -from core.plugin.impl.oauth import OAuthHandler -from core.trigger.trigger_manager import TriggerManager -from core.trigger.utils.encryption import delete_cache_for_subscription +from enums import DeploymentEdition from extensions.application_services.workspace import WorkspaceServices +from extensions.ext_application_services import build_application_services from extensions.ext_redis import RedisClientWrapper from libs.external_api import register_external_error_handlers -from repositories.app_definition_query_repository import AppDefinitionQueryRepository -from repositories.app_scoped_end_user_repository import AppScopedEndUserRepo -from repositories.credentials.query_repository import CredentialQueryRepository -from repositories.sqlalchemy_workflow_trigger_log_repository import SQLAlchemyWorkflowTriggerDispatchRepository -from repositories.trigger.app_repository import SQLAlchemyAppTriggerRepository -from repositories.trigger.oauth_client_repository import TriggerOAuthClientRepository -from repositories.trigger.subscription_builder_repository import RedisTriggerSubscriptionBuilderRepository -from repositories.trigger.subscription_repository import ( - RedisTriggerSubscriptionLocks, - RedisTriggerSubscriptionRefreshReservations, - TriggerSubscriptionRepository, -) from services.account_errors import AccountNotFoundError -from services.app_scoped_end_user_service import AppScopedEndUserService -from services.async_workflow_service import AsyncWorkflowService from services.errors.workspace import ( InvalidWorkspaceMemberRoleError, WorkspaceNotFoundError, WorkspaceOwnerNotFoundError, WorkspacesLimitExceededError, ) -from services.plugin.plugin_parameter_gateway import DaemonPluginParameterGateway -from services.plugin.plugin_parameter_service import PluginParameterService -from services.quota_service import QuotaService -from services.trigger.app_trigger_service import AppTriggerService -from services.trigger.endpoint_gateway import PluginTriggerEndpointGateway, enqueue_triggered_workflows -from services.trigger.endpoint_service import TriggerEndpointService -from services.trigger.event_gateway import PluginTriggerEventGateway -from services.trigger.event_service import TriggerEventService -from services.trigger.oauth_client_service import TriggerOAuthClientService -from services.trigger.oauth_gateway import ( - OAuthProxyContextGateway, - PluginTriggerOAuthGateway, - TriggerOAuthBuilderGateway, -) -from services.trigger.oauth_service import TriggerOAuthService -from services.trigger.provider_gateway import PluginTriggerProviderQuery -from services.trigger.subscription_binding_service import TriggerSubscriptionBindingService -from services.trigger.subscription_codec import ProviderTriggerSubscriptionCodec -from services.trigger.subscription_gateway import ( - PluginTriggerSubscriptionGateway, - enqueue_subscription_refresh, - invalidate_subscription_properties, -) -from services.trigger.subscription_query_service import TriggerSubscriptionQueryService -from services.trigger.subscription_refresh_service import TriggerSubscriptionRefreshService -from services.trigger.subscription_renewal_service import TriggerSubscriptionRenewalService -from services.trigger.subscription_service import TriggerSubscriptionService -from services.trigger.trigger_request_service import TriggerHttpRequestCachingService -from services.trigger.trigger_subscription_builder_service import TriggerSubscriptionBuilderService -from services.trigger.workflow_dispatch_gateway import AppScopedTriggerEndUsers -from services.trigger.workflow_dispatch_service import PluginTriggerWorkflowDispatchService -from services.workflow.queue_dispatcher import CeleryWorkflowQueuePublisher from services.workspace.contracts import CreatedWorkspace, WorkspaceMembership from services.workspace.provisioning_service import WorkspaceProvisioningService from services.workspace.service import WorkspaceQueryService, WorkspaceService @@ -146,55 +98,15 @@ class TestWorkspaceCases: @pytest.fixture def provisioning(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> Mock: provisioning = create_autospec(WorkspaceProvisioningService, instance=True) - oauth_provider = PluginTriggerOAuthGateway(oauth=OAuthHandler()) - oauth_clients = TriggerOAuthClientService( - clients=TriggerOAuthClientRepository(session_factory=sqlite_session_factory), providers=oauth_provider - ) redis = RedisClientWrapper() redis.initialize(Redis()) - subscription_management = TriggerSubscriptionService( - subscriptions=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - providers=PluginTriggerSubscriptionGateway(), - locks=RedisTriggerSubscriptionLocks(redis=redis), - codec=ProviderTriggerSubscriptionCodec(), - invalidate_cache=delete_cache_for_subscription, - ) - builders = TriggerSubscriptionBuilderService( - builders=RedisTriggerSubscriptionBuilderRepository(redis=redis), - get_provider=TriggerManager.get_trigger_provider, - subscribe=TriggerManager.subscribe_trigger, - create_subscription=subscription_management.create, - ) - subscription_queries = TriggerSubscriptionQueryService( - credentials=CredentialQueryRepository(session_factory=sqlite_session_factory), - schemas=PluginTriggerProviderQuery(), - subscriptions=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - ) - renewal = TriggerSubscriptionRenewalService( - subscriptions=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - providers=PluginTriggerSubscriptionGateway(), - oauth=oauth_provider, - clients=oauth_clients, - locks=RedisTriggerSubscriptionLocks(redis=redis), - codec=ProviderTriggerSubscriptionCodec(), - credential_threshold=10, - subscription_threshold=10, - console_api_url="https://console.example.test", - invalidate_cache=delete_cache_for_subscription, - invalidate_properties=invalidate_subscription_properties, - ) - workflow_queries = AppDefinitionQueryRepository(session_factory=sqlite_session_factory) - logs = SQLAlchemyWorkflowTriggerDispatchRepository(session_factory=sqlite_session_factory) - queues = CeleryWorkflowQueuePublisher() - events = TriggerEventService( - subscriptions=subscription_queries, - plugins=PluginTriggerEventGateway(), - requests=TriggerHttpRequestCachingService, - ) - workflow_dispatch = AsyncWorkflowService( - workflows=workflow_queries, logs=logs, queues=queues, reserve_quota=QuotaService.reserve - ) - services = WorkspaceServices( + services = replace( + build_application_services( + database_client=sqlite_session_factory, + deployment_edition=DeploymentEdition.COMMUNITY, + initialization_password="", + redis=redis, + ).workspaces, queries=create_autospec(WorkspaceQueryService, instance=True), management=create_autospec(WorkspaceService, instance=True), provisioning=provisioning, @@ -204,62 +116,6 @@ class TestWorkspaceCases: invitations=Mock(), identity=Mock(), resource_cleanup=Mock(), - plugin_parameters=PluginParameterService( - credentials=CredentialQueryRepository(session_factory=sqlite_session_factory), - subscriptions=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - builders=builders, - plugins=DaemonPluginParameterGateway(), - ), - trigger_providers=PluginTriggerProviderQuery(), - trigger_subscriptions=subscription_queries, - trigger_binding_queries=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - create_trigger_binding_service=lambda session: TriggerSubscriptionBindingService( - sync_bindings=partial( - TriggerSubscriptionRepository(session_factory=sqlite_session_factory).sync_workflow_bindings, - session=session, - ) - ), - trigger_events=events, - workflow_dispatch=workflow_dispatch, - trigger_workflow_dispatch=PluginTriggerWorkflowDispatchService( - workflows=workflow_queries, - logs=logs, - queues=queues, - dispatch=workflow_dispatch, - events=events, - bindings=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - end_users=AppScopedTriggerEndUsers( - end_users=AppScopedEndUserService( - end_users=AppScopedEndUserRepo(session_factory=sqlite_session_factory) - ) - ), - reserve_quota=QuotaService.reserve, - mark_rate_limited=AppTriggerService( - triggers=SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory) - ).mark_tenant_triggers_rate_limited, - ), - trigger_builders=builders, - trigger_endpoints=TriggerEndpointService( - subscriptions=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - provider=PluginTriggerEndpointGateway(), - requests=TriggerHttpRequestCachingService, - publish=enqueue_triggered_workflows, - ), - trigger_subscription_management=subscription_management, - trigger_subscription_renewal=renewal, - trigger_subscription_refresh=TriggerSubscriptionRefreshService( - subscriptions=TriggerSubscriptionRepository(session_factory=sqlite_session_factory), - reservations=RedisTriggerSubscriptionRefreshReservations(redis=redis), - publish=enqueue_subscription_refresh, - renewal=renewal, - ), - trigger_oauth_clients=oauth_clients, - trigger_oauth=TriggerOAuthService( - clients=oauth_clients, - providers=oauth_provider, - contexts=OAuthProxyContextGateway(), - builders=TriggerOAuthBuilderGateway(builders=builders), - ), ) monkeypatch.setattr(controller, "application_services", lambda: Services(services)) return provisioning diff --git a/api/tests/unit_tests/controllers/trigger/test_webhook.py b/api/tests/unit_tests/controllers/trigger/test_webhook.py index abe88f7ad3a..a6ad66f0169 100644 --- a/api/tests/unit_tests/controllers/trigger/test_webhook.py +++ b/api/tests/unit_tests/controllers/trigger/test_webhook.py @@ -1,361 +1,516 @@ -from dataclasses import dataclass +"""Native webhook HTTP requests use real admission, SQL, quota HTTP and task publishing.""" + +import json +from collections.abc import Iterator +from contextvars import Context +from dataclasses import dataclass, field +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, HTTPServer from io import BytesIO -from unittest.mock import MagicMock, patch +from threading import Thread +from typing import override +from uuid import uuid4 import pytest -from flask import Flask -from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import NotFound, RequestEntityTooLarge +from celery import Celery +from kombu.simple import SimpleQueue +from sqlalchemy import Engine, delete, select +from sqlalchemy.pool import QueuePool +from werkzeug.test import TestResponse -import controllers.trigger.webhook as module -from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom -from core.app.file_access import DatabaseFileAccessController, FileAccessScope, bind_file_access_scope -from factories import file_factory -from models.enums import DEFAULT_END_USER_SESSION_ID, EndUserType -from models.model import EndUser -from models.trigger import WorkflowWebhookTrigger +from controllers.trigger import bp +from core.app.workflow.file_runtime import create_dify_workflow_file_runtime +from enums import DeploymentEdition +from extensions.ext_storage import storage +from graphon.file.runtime import use_workflow_file_runtime +from models.enums import AppTriggerStatus, AppTriggerType, EndUserType, WorkflowTriggerStatus +from models.model import AppMode, EndUser +from models.tools import ToolFile +from models.trigger import AppTrigger, WorkflowTriggerLog, WorkflowWebhookTrigger from models.workflow import Workflow -from repositories.trigger.app_repository import SQLAlchemyAppTriggerRepository -from services.errors.app import QuotaExceededError -from services.trigger.app_trigger_service import AppTriggerService -from tests.helpers.unit.model_factories import make_end_user, make_workflow +from tasks.async_workflow_tasks import execute_workflow_sandbox +from tasks.workflow_cfs_scheduler.entities import AsyncWorkflowQueue +from tests.helpers.unit.config_override import config_overrides_context +from tests.helpers.unit.model_factories import make_app, make_workflow +from tests.unit_tests.controllers.console.workspace import test_trigger_oauth_client +from tests.unit_tests.controllers.console.workspace.test_trigger_oauth_client import _Harness - -class _RequestStub: - method = "POST" - headers = {"x-test": "1"} - args = {"a": "b"} - - -@dataclass(eq=False, repr=False) -class _AppScopedEndUserServicesStub: - commands: object +application_redis = test_trigger_oauth_client.application_redis +harness = test_trigger_oauth_client.harness @dataclass(frozen=True) -class _AppServicesStub: - trigger_status: AppTriggerService +class _Webhook: + app_id: str + webhook_id: str + workflow_id: str + draft_id: str + trigger_id: str + node_id: str = "webhook-node" + + @property + def url(self) -> str: + return f"/triggers/webhook/{self.webhook_id}" + + @property + def debug_url(self) -> str: + return f"/triggers/webhook-debug/{self.webhook_id}" -@dataclass(eq=False, repr=False) -class _WorkspaceServicesStub: - workflow_dispatch: object - - -class _ApplicationServicesStub: - def __init__(self, commands: object, trigger_status: AppTriggerService) -> None: - self.app_scoped_end_users = _AppScopedEndUserServicesStub(commands) - self.apps = _AppServicesStub(trigger_status) - self.workspaces = _WorkspaceServicesStub(object()) - - -class _EndUserCommandsStub: - def __init__(self, end_user: EndUser) -> None: - self._end_user = end_user - self.calls: list[tuple[EndUserType, str, str, str | None]] = [] - - def get_or_create_end_user_by_type( - self, - type: EndUserType, - tenant_id: str, - app_id: str, - user_id: str | None = None, - ) -> EndUser: - self.calls.append((type, tenant_id, app_id, user_id)) - return self._end_user - - -@pytest.fixture(autouse=True) -def mock_request(): - module.request = _RequestStub() - - -@pytest.fixture(autouse=True) -def mock_jsonify(): - module.jsonify = lambda payload: payload - - -@pytest.fixture(autouse=True) -def end_user_commands(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> object: - commands = object() - triggers = AppTriggerService(triggers=SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory)) - monkeypatch.setattr(module, "application_services", lambda: _ApplicationServicesStub(commands, triggers)) - return commands - - -def _webhook_trigger() -> WorkflowWebhookTrigger: - return WorkflowWebhookTrigger( - webhook_id="wh-1", - tenant_id="tenant-1", - app_id="app-1", - node_id="node-1", - created_by="account-1", - ) - - -def _workflow() -> Workflow: - return make_workflow(workflow_id="workflow-1") - - -class TestPrepareWebhookExecution: - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data") - def test_prepare_success(self, mock_extract, mock_get): - webhook_trigger = _webhook_trigger() - workflow = _workflow() - mock_get.return_value = (webhook_trigger, workflow, "node_config") - mock_extract.return_value = {"data": "ok"} - - result = module._prepare_webhook_execution("wh-1") - - assert result == (webhook_trigger, workflow, "node_config", {"data": "ok"}, None) - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data", side_effect=ValueError("bad")) - def test_prepare_validation_error(self, mock_extract, mock_get): - webhook_trigger = _webhook_trigger() - workflow = _workflow() - mock_get.return_value = (webhook_trigger, workflow, "node_config") - - trigger, workflow, node_config, webhook_data, error = module._prepare_webhook_execution("wh-1") - - assert error == "bad" - assert webhook_data["method"] == "POST" - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - def test_prepare_stores_uploads_for_the_trigger_end_user_run( - self, - mock_get: MagicMock, - app: Flask, - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - ) -> None: - node_config = { - "data": { - "method": "post", - "content_type": "multipart/form-data", - "body": [{"name": "chat_history_file", "type": "file", "required": True}], +@pytest.fixture +def webhook(harness: _Harness) -> _Webhook: + harness.app.register_blueprint(bp) + app_id, webhook_id = str(uuid4()), uuid4().hex[:24] + graph: dict[str, object] = { + "nodes": [ + { + "id": "webhook-node", + "data": { + "type": "trigger-webhook", + "title": "Incoming webhook", + "method": "post", + "content_type": "application/json", + "headers": [], + "params": [], + "body": [], + }, } - } - mock_get.return_value = (_webhook_trigger(), _workflow(), node_config) - trigger_end_user = make_end_user( - end_user_id="trigger-end-user-1", - tenant_id="tenant-1", - app_id="app-1", - end_user_type=EndUserType.TRIGGER, - session_id=DEFAULT_END_USER_SESSION_ID, + ], + "edges": [], + } + published, draft = [ + make_workflow( + tenant_id=harness.tenant_id, + app_id=app_id, + version=version, + graph=graph, + created_by=harness.account_id, ) - end_users = _EndUserCommandsStub(trigger_end_user) - triggers = AppTriggerService(triggers=SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory)) - monkeypatch.setattr(module, "application_services", lambda: _ApplicationServicesStub(end_users, triggers)) - monkeypatch.setattr("core.tools.tool_file_manager.storage.save", lambda *_args: None) - - with app.test_request_context( - "/triggers/webhook/wh-1", - method="POST", - data={"chat_history_file": (BytesIO(b"# chat"), "chat.md", "text/markdown")}, - content_type="multipart/form-data", - ): - _, _, _, webhook_data, error = module._prepare_webhook_execution("wh-1") - - assert error is None - tool_file_id = webhook_data["files"]["chat_history_file"]["related_id"] - # The async run executes as the trigger end user, whose scope only loads tool files it owns. - run_scope = FileAccessScope( - tenant_id="tenant-1", - user_id=trigger_end_user.id, - user_from=UserFrom.END_USER, - invoke_from=InvokeFrom.SERVICE_API, + for version in ("published", Workflow.VERSION_DRAFT) + ] + with harness.session_factory.begin() as session: + trigger = AppTrigger( + tenant_id=harness.tenant_id, + app_id=app_id, + node_id="webhook-node", + title="Incoming webhook", + trigger_type=AppTriggerType.TRIGGER_WEBHOOK, + status=AppTriggerStatus.ENABLED, ) - with bind_file_access_scope(run_scope): - file = file_factory.build_from_mapping( - mapping={"transfer_method": "tool_file", "tool_file_id": tool_file_id}, - tenant_id="tenant-1", - access_controller=DatabaseFileAccessController(), + session.add_all( + [ + published, + draft, + make_app(app_id=app_id, tenant_id=harness.tenant_id, workflow_id=published.id, mode=AppMode.WORKFLOW), + trigger, + WorkflowWebhookTrigger( + tenant_id=harness.tenant_id, + app_id=app_id, + webhook_id=webhook_id, + node_id="webhook-node", + created_by=harness.account_id, + ), + ] + ) + session.flush() + return _Webhook(app_id, webhook_id, published.id, draft.id, trigger.id) + + +def _configure(harness: _Harness, webhook: _Webhook, **values: object) -> None: + with harness.session_factory.begin() as session: + for workflow_id in (webhook.workflow_id, webhook.draft_id): + workflow = session.get(Workflow, workflow_id) + assert workflow is not None + graph = json.loads(workflow.graph) + graph["nodes"][0]["data"].update(values) + workflow.graph = json.dumps(graph) + + +@pytest.fixture +def broker() -> Iterator[SimpleQueue]: + previous_app = execute_workflow_sandbox.app + app = Celery("native-webhook-http-test", broker="memory://", set_as_current=False) + app.conf.update(task_always_eager=False, task_serializer="json") + app.set_current() + try: + with app.connection_for_write() as connection: + with SimpleQueue(connection.channel(), AsyncWorkflowQueue.SANDBOX_QUEUE) as queue: + queue.clear() + try: + yield queue + finally: + queue.clear() + finally: + previous_app.set_current() + app.close() + + +@dataclass +class _Billing: + requests: list[tuple[str, dict[str, object]]] = field(default_factory=list) + open_connections: list[int] = field(default_factory=list) + exhausted_feature: str | None = None + + +@pytest.fixture +def billing(sqlite_engine: Engine) -> Iterator[_Billing]: + billing = _Billing() + pool = sqlite_engine.pool + assert isinstance(pool, QueuePool) + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.respond( + { + "enabled": True, + "subscription": { + "plan": "sandbox", + "interval": "month", + "education": False, + "status": "active", + "cancel_at_period_end": False, + "paid_period_end": 0, + }, + "members": {"size": 1, "limit": 10}, + "apps": {"size": 1, "limit": 10}, + "vector_space": None, + "knowledge_rate_limit": {"size": 0, "limit": 10}, + "documents_upload_quota": {"size": 0, "limit": 10}, + "annotation_quota_limit": {"size": 0, "limit": 10}, + "docs_processing": "standard", + "can_replace_logo": False, + "model_load_balancing_enabled": False, + "knowledge_pipeline_publish_enabled": False, + "next_credit_reset_date": 0, + } ) - assert file.related_id == tool_file_id - assert end_users.calls == [(EndUserType.TRIGGER, "tenant-1", "app-1", None)] + + def do_POST(self) -> None: + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + billing.requests.append((self.path, payload)) + billing.open_connections.append(pool.checkedout()) + if self.path == "/quota/reserve": + exhausted = payload["feature_key"] == billing.exhausted_feature + self.respond( + { + "reservation_id": "" if exhausted else str(uuid4()), + "available": 0 if exhausted else 10, + "reserved": 0 if exhausted else 1, + } + ) + elif self.path == "/quota/commit": + self.respond({"available": 9, "reserved": 0, "refunded": 0}) + else: + assert self.path == "/quota/release" + self.respond({"available": 10, "reserved": 0, "released": 1}) + + def respond(self, data: dict[str, object]) -> None: + body = json.dumps(data).encode() + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + @override + def log_message(self, format: str, *args: object) -> None: + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}" + with config_overrides_context(BILLING_API_URL=url, BILLING_QUOTA_API_URL=url): + yield billing + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) -class TestHandleWebhook: - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data") - @patch.object(module.WebhookService, "trigger_workflow_execution") - @patch.object(module.WebhookService, "generate_webhook_response") - def test_success( - self, - mock_generate, - mock_trigger, - mock_extract, - mock_get, - end_user_commands, - ): - mock_get.return_value = (_webhook_trigger(), _workflow(), "node_config") - mock_extract.return_value = {"input": "x"} - mock_generate.return_value = ({"ok": True}, 200) +def test_success_publishes_actual_workflow_task_and_preserves_actor_and_inputs( + harness: _Harness, webhook: _Webhook, broker: SimpleQueue, billing: _Billing +) -> None: + with config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD): + response = Context().run(harness.client.post, webhook.url + "?attempt=2", json={"ref": "main"}) - response, status = module.handle_webhook("wh-1") + assert response.status_code == HTTPStatus.OK + assert response.get_json() == {"status": "success", "message": "Webhook processed successfully"} + assert response.headers["Content-Type"] == "application/json" + message = broker.get(block=False) + with harness.session_factory() as session: + log = session.scalar(select(WorkflowTriggerLog).where(WorkflowTriggerLog.app_id == webhook.app_id)) + actor = session.scalar(select(EndUser).where(EndUser.app_id == webhook.app_id)) + assert log is not None + assert actor is not None + assert actor.tenant_id == harness.tenant_id + assert actor.type == EndUserType.TRIGGER + assert log.created_by == actor.id + assert log.tenant_id == harness.tenant_id + assert log.workflow_id == webhook.workflow_id + assert log.root_node_id == webhook.node_id + assert log.status == WorkflowTriggerStatus.QUEUED + inputs = json.loads(log.inputs) + assert inputs["webhook_body"] == {"ref": "main"} + assert inputs["webhook_query_params"] == {"attempt": "2"} + assert inputs["webhook_data"]["method"] == "POST" + assert message.payload[0] == [{"workflow_trigger_log_id": log.id}] + assert message.headers["id"] == log.celery_task_id + assert message.headers["task"] == execute_workflow_sandbox.name + message.ack() + assert broker.qsize() == 0 + assert billing.open_connections + assert not any(billing.open_connections) - assert status == 200 - assert response["ok"] is True - mock_trigger.assert_called_once() - assert mock_trigger.call_args.kwargs["end_users"] is end_user_commands - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data", side_effect=ValueError("bad")) - def test_bad_request(self, mock_extract, mock_get): - mock_get.return_value = (_webhook_trigger(), _workflow(), "node_config") +@pytest.mark.parametrize( + ("body", "status", "expected"), + [ + ('{"accepted":true}', 202, {"accepted": True}), + ('["one",2]', 201, ["one", 2]), + ("accepted", 200, {"message": "accepted"}), + ], +) +def test_configured_response_preserves_status_and_json_shape( + harness: _Harness, webhook: _Webhook, broker: SimpleQueue, body: str, status: int, expected: object +) -> None: + _configure(harness, webhook, response_body=body, status_code=status) + response = Context().run(harness.client.post, webhook.url, json={}) + assert response.status_code == status + assert response.get_json() == expected + assert broker.qsize() == 1 - response, status = module.handle_webhook("wh-1") - assert status == 400 - assert response["error"] == "Bad Request" +@pytest.mark.parametrize("debug", [False, True]) +@pytest.mark.parametrize("problem", ["method", "json", "required"]) +def test_request_errors_are_400_before_execution( + harness: _Harness, webhook: _Webhook, debug: bool, problem: str +) -> None: + url = webhook.debug_url if debug else webhook.url + if problem == "method": + response = Context().run(harness.client.get, url) + elif problem == "json": + response = Context().run(harness.client.post, url, data="{bad", content_type="application/json") + else: + _configure(harness, webhook, body=[{"name": "required", "type": "string", "required": True}]) + response = Context().run(harness.client.post, url, json={}) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.get_json()["error"] == "Bad Request" + assert response.get_json()["message"] + with harness.session_factory() as session: + assert session.scalar(select(WorkflowTriggerLog)) is None + assert session.scalar(select(EndUser)) is None - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data") - @patch.object( - module.WebhookService, - "trigger_workflow_execution", - side_effect=QuotaExceededError(feature="trigger", tenant_id="tenant-1", required=1), + +@pytest.mark.parametrize("debug", [False, True]) +def test_invalid_stored_webhook_configuration_does_not_expose_details( + harness: _Harness, webhook: _Webhook, debug: bool +) -> None: + _configure(harness, webhook, body=[{"name": "amount", "type": "private-config-value", "required": True}]) + + response = Context().run(harness.client.post, webhook.debug_url if debug else webhook.url, json={"amount": 1}) + + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + assert response.get_json() == {"error": "Internal server error", "message": "An internal error has occurred."} + assert b"private-config-value" not in response.data + with harness.session_factory() as session: + assert session.scalar(select(WorkflowTriggerLog)) is None + assert session.scalar(select(EndUser)) is None + + +@pytest.mark.parametrize("debug", [False, True]) +def test_unknown_webhook_is_opaque_noncacheable_404(harness: _Harness, webhook: _Webhook, debug: bool) -> None: + url = webhook.debug_url if debug else webhook.url + with harness.session_factory.begin() as session: + session.execute(delete(WorkflowWebhookTrigger)) + response = Context().run(harness.client.post, url, json={}) + assert response.status_code == HTTPStatus.NOT_FOUND + assert response.content_type == "text/html; charset=utf-8" + assert response.headers["Cache-Control"] == "no-store" + assert webhook.webhook_id not in response.get_data(as_text=True) + + +@pytest.mark.parametrize("debug", [False, True]) +@pytest.mark.parametrize("broken_graph", ["missing-node", "empty-nodes", "malformed"]) +def test_webhook_distinguishes_missing_node_from_corrupt_workflow( + harness: _Harness, webhook: _Webhook, debug: bool, broken_graph: str +) -> None: + workflow_id = webhook.draft_id if debug else webhook.workflow_id + with harness.session_factory.begin() as session: + workflow = session.get(Workflow, workflow_id) + assert workflow is not None + if broken_graph == "missing-node": + graph = json.loads(workflow.graph) + graph["nodes"][0]["id"] = "different-node" + workflow.graph = json.dumps(graph) + elif broken_graph == "empty-nodes": + workflow.graph = '{"nodes": [], "edges": []}' + else: + workflow.graph = "{malformed" + + response = Context().run(harness.client.post, webhook.debug_url if debug else webhook.url, json={}) + + if broken_graph == "missing-node": + assert response.status_code == HTTPStatus.NOT_FOUND + assert response.headers["Cache-Control"] == "no-store" + else: + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + assert response.get_json() == {"error": "Internal server error", "message": "An internal error has occurred."} + with harness.session_factory() as session: + assert session.scalar(select(WorkflowTriggerLog)) is None + assert session.scalar(select(EndUser)) is None + + +@pytest.mark.parametrize("debug", [False, True]) +def test_body_limit_remains_413(harness: _Harness, webhook: _Webhook, debug: bool) -> None: + with config_overrides_context(WEBHOOK_REQUEST_BODY_MAX_SIZE=4): + response = Context().run(harness.client.post, webhook.debug_url if debug else webhook.url, json={"large": True}) + assert response.status_code == HTTPStatus.REQUEST_ENTITY_TOO_LARGE + + +@pytest.mark.parametrize("status", [AppTriggerStatus.DISABLED, AppTriggerStatus.RATE_LIMITED]) +def test_published_status_blocks_execution_but_draft_still_reaches_request_validation( + harness: _Harness, webhook: _Webhook, status: AppTriggerStatus +) -> None: + with harness.session_factory.begin() as session: + trigger = session.get(AppTrigger, webhook.trigger_id) + assert trigger is not None + trigger.status = status + response = Context().run(harness.client.post, webhook.url, json={}) + assert response.status_code == ( + HTTPStatus.NOT_FOUND if status == AppTriggerStatus.DISABLED else HTTPStatus.TOO_MANY_REQUESTS ) - def test_quota_exceeded(self, mock_trigger, mock_extract, mock_get): - mock_get.return_value = (_webhook_trigger(), _workflow(), "node_config") - mock_extract.return_value = {"input": "x"} - - response, status = module.handle_webhook("wh-1") - - assert status == 429 - assert response == { + if status == AppTriggerStatus.RATE_LIMITED: + assert response.get_json() == { "error": "Too Many Requests", "message": "Trigger event quota exceeded. Please upgrade your plan.", } + debug = Context().run(harness.client.post, webhook.debug_url, data="{bad", content_type="application/json") + assert debug.status_code == HTTPStatus.BAD_REQUEST - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data") - @patch.object( - module.WebhookService, - "trigger_workflow_execution", - side_effect=QuotaExceededError(feature="workflow", tenant_id="tenant-1", required=1), - ) - def test_workflow_quota_exceeded(self, mock_trigger, mock_extract, mock_get): - mock_get.return_value = (_webhook_trigger(), _workflow(), "node_config") - mock_extract.return_value = {"input": "x"} - response, status = module.handle_webhook("wh-1") +@pytest.mark.parametrize("status", [AppTriggerStatus.DISABLED, AppTriggerStatus.RATE_LIMITED]) +@pytest.mark.parametrize("broken_workflow", ["missing", "empty-nodes", "malformed-graph"]) +def test_unavailable_trigger_status_precedes_published_workflow_lookup_and_parsing( + harness: _Harness, webhook: _Webhook, status: AppTriggerStatus, broken_workflow: str +) -> None: + with harness.session_factory.begin() as session: + trigger = session.get(AppTrigger, webhook.trigger_id) + workflow = session.get(Workflow, webhook.workflow_id) + assert trigger is not None + assert workflow is not None + trigger.status = status + if broken_workflow == "missing": + session.delete(workflow) + elif broken_workflow == "empty-nodes": + workflow.graph = '{"nodes": [], "edges": []}' + else: + workflow.graph = "{malformed" - assert status == 429 - assert response == { - "error": "Too Many Requests", - "message": "Workflow execution quota exceeded. Please upgrade your plan.", - } + response = Context().run(harness.client.post, webhook.url, json={}) - @patch.object( - module.WebhookService, - "get_webhook_trigger_and_workflow", - side_effect=QuotaExceededError(feature="trigger", tenant_id="tenant-1", required=1), - ) - def test_rate_limited(self, mock_get): - response, status = module.handle_webhook("wh-1") - - assert status == 429 - assert response == { + if status == AppTriggerStatus.RATE_LIMITED: + assert response.status_code == HTTPStatus.TOO_MANY_REQUESTS + assert response.get_json() == { "error": "Too Many Requests", "message": "Trigger event quota exceeded. Please upgrade your plan.", } - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=ValueError("missing")) - def test_value_error_not_found(self, mock_get): - response = module.handle_webhook("wh-1") - assert response.status_code == 404 - assert response.data == NotFound().get_body().encode() - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=RequestEntityTooLarge()) - def test_request_entity_too_large(self, mock_get): - with pytest.raises(RequestEntityTooLarge): - module.handle_webhook("wh-1") - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=Exception("boom")) - def test_internal_error(self, mock_get): - response, status = module.handle_webhook("wh-1") - - assert status == 500 - assert response["error"] == "Internal server error" + else: + assert response.status_code == HTTPStatus.NOT_FOUND + assert response.content_type == "text/html; charset=utf-8" + assert response.headers["Cache-Control"] == "no-store" + # The intact draft continues to use its own config despite the published status and corruption. + debug = Context().run(harness.client.post, webhook.debug_url, data="{bad", content_type="application/json") + assert debug.status_code == HTTPStatus.BAD_REQUEST + with harness.session_factory() as session: + assert session.scalar(select(WorkflowTriggerLog)) is None + assert session.scalar(select(EndUser)) is None -class TestHandleWebhookDebug: - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data") - @patch.object(module.WebhookService, "build_workflow_inputs", return_value={"x": 1}) - @patch.object(module.TriggerDebugEventBus, "dispatch", return_value=0) - def test_debug_requires_active_listener( - self, - mock_dispatch, - mock_build_inputs, - mock_extract, - mock_get, - ): - webhook_trigger = _webhook_trigger() - mock_get.return_value = (webhook_trigger, None, "node_config") - mock_extract.return_value = {"method": "POST"} - - response, status = module.handle_webhook_debug("wh-1") - - assert status == 409 - assert response["error"] == "No active debug listener" - assert response["message"] == ( - "The webhook debug URL only works while the Variable Inspector is listening. " - "Use the published webhook URL to execute the workflow in Celery." +@pytest.mark.parametrize( + ("feature", "message"), + [ + ("trigger_event", "Trigger event quota exceeded. Please upgrade your plan."), + ("api_rate_limit", "Workflow execution quota exceeded. Please upgrade your plan."), + ], +) +def test_real_quota_denial_returns_specific_429_without_publishing( + harness: _Harness, webhook: _Webhook, broker: SimpleQueue, billing: _Billing, feature: str, message: str +) -> None: + billing.exhausted_feature = feature + with config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD): + response = Context().run(harness.client.post, webhook.url, json={}) + assert response.status_code == HTTPStatus.TOO_MANY_REQUESTS + assert response.get_json() == {"error": "Too Many Requests", "message": message} + assert broker.qsize() == 0 + with harness.session_factory() as session: + log = session.scalar(select(WorkflowTriggerLog)) + if feature == "trigger_event": + assert log is None + else: + assert log is not None + assert log.status == WorkflowTriggerStatus.RATE_LIMITED + assert log.celery_task_id is None + trigger = session.get(AppTrigger, webhook.trigger_id) + assert trigger is not None + assert trigger.status == ( + AppTriggerStatus.RATE_LIMITED if feature == "trigger_event" else AppTriggerStatus.ENABLED ) - assert response["execution_url"] == webhook_trigger.webhook_url - mock_dispatch.assert_called_once() + assert not any(billing.open_connections) - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data") - @patch.object(module.WebhookService, "build_workflow_inputs", return_value={"x": 1}) - @patch.object(module.TriggerDebugEventBus, "dispatch", return_value=1) - @patch.object(module.WebhookService, "generate_webhook_response") - def test_debug_success( - self, - mock_generate, - mock_dispatch, - mock_build_inputs, - mock_extract, - mock_get, - ): - mock_get.return_value = (_webhook_trigger(), None, "node_config") - mock_extract.return_value = {"method": "POST"} - mock_generate.return_value = ({"ok": True}, 200) - response, status = module.handle_webhook_debug("wh-1") +def test_corrupt_stored_workflow_returns_500_instead_of_client_validation_error( + harness: _Harness, webhook: _Webhook +) -> None: + with harness.session_factory.begin() as session: + workflow = session.get(Workflow, webhook.workflow_id) + assert workflow is not None + workflow.graph = '{"nodes": [], "edges": []}' + response = Context().run(harness.client.post, webhook.url, json={}) + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + assert response.get_json() == {"error": "Internal server error", "message": "An internal error has occurred."} - assert status == 200 - assert response["ok"] is True - mock_dispatch.assert_called_once() - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") - @patch.object(module.WebhookService, "extract_and_validate_webhook_data", side_effect=ValueError("bad")) - def test_debug_bad_request(self, mock_extract, mock_get): - mock_get.return_value = (_webhook_trigger(), None, "node_config") +@pytest.mark.parametrize("content_type", ["multipart/form-data", "application/octet-stream"]) +def test_uploaded_files_use_real_tool_file_storage_and_detached_workflow_inputs( + harness: _Harness, webhook: _Webhook, broker: SimpleQueue, content_type: str +) -> None: + _configure(harness, webhook, content_type=content_type) + content = b"webhook upload bytes" + data = ( + {"upload": (BytesIO(content), "sample.txt"), "caption": "incoming"} + if content_type == "multipart/form-data" + else content + ) - response, status = module.handle_webhook_debug("wh-1") + def send() -> TestResponse: + # The minimal Flask harness binds the same concrete file runtime as production DifyAppContext. + with use_workflow_file_runtime(create_dify_workflow_file_runtime()): + return harness.client.post(webhook.url, data=data, content_type=content_type) - assert status == 400 - assert response["error"] == "Bad Request" - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=ValueError("missing")) - def test_debug_not_found(self, mock_get): - response = module.handle_webhook_debug("wh-1") - assert response.status_code == 404 - assert response.data == NotFound().get_body().encode() - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=RequestEntityTooLarge()) - def test_debug_request_entity_too_large(self, mock_get): - with pytest.raises(RequestEntityTooLarge): - module.handle_webhook_debug("wh-1") - - @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=Exception("boom")) - def test_debug_internal_error(self, mock_get): - response, status = module.handle_webhook_debug("wh-1") - - assert status == 500 - assert response["error"] == "Internal server error" + response = Context().run(send) + assert response.status_code == HTTPStatus.OK + assert broker.qsize() == 1 + with harness.session_factory() as session: + tool_file = session.scalar(select(ToolFile)) + actor = session.scalar(select(EndUser).where(EndUser.app_id == webhook.app_id)) + assert tool_file is not None + assert actor is not None + assert tool_file.tenant_id == harness.tenant_id + assert tool_file.user_id == actor.id + assert tool_file.size == len(content) + file_key = tool_file.file_key + log = session.scalar(select(WorkflowTriggerLog).where(WorkflowTriggerLog.app_id == webhook.app_id)) + assert log is not None + assert log.created_by == actor.id + inputs = json.loads(log.inputs) + saved_file = ( + inputs["webhook_data"]["files"]["upload"] + if content_type == "multipart/form-data" + else inputs["webhook_body"]["raw"] + ) + assert saved_file["transfer_method"] == "tool_file" + assert saved_file["size"] == len(content) + assert saved_file["mime_type"] == tool_file.mimetype + assert storage.load_once(file_key) == content diff --git a/api/tests/unit_tests/controllers/trigger/test_webhook_request.py b/api/tests/unit_tests/controllers/trigger/test_webhook_request.py new file mode 100644 index 00000000000..d0f33d1c0c0 --- /dev/null +++ b/api/tests/unit_tests/controllers/trigger/test_webhook_request.py @@ -0,0 +1,548 @@ +"""Real HTTP requests and value conversion retain the native webhook contract.""" + +from collections.abc import Mapping + +import pytest +from flask import Flask, request +from werkzeug.exceptions import RequestEntityTooLarge + +from controllers.trigger.webhook_request import WebhookRequestParser +from core.app.file_access import DatabaseFileAccessController +from core.tools.tool_file_manager import ToolFileManager +from core.workflow.nodes.trigger_webhook.entities import ( + ContentType, + WebhookBodyParameter, + WebhookData, + WebhookParameter, +) +from graphon.entities.graph_config import NodeConfigDict +from graphon.variables.types import SegmentType +from services.trigger.webhook_gateway import ToolWebhookFiles +from services.trigger.webhook_service import RawWebhookDataDict, WebhookContext, build_webhook_workflow_inputs +from tests.helpers.unit.config_override import config_overrides_context + + +@pytest.fixture +def flask_app() -> Flask: + return Flask(__name__) + + +@pytest.fixture +def parser() -> WebhookRequestParser: + return WebhookRequestParser( + request=request, files=ToolWebhookFiles(manager=ToolFileManager(), access=DatabaseFileAccessController()) + ) + + +def _webhook_trigger() -> WebhookContext: + return WebhookContext( + tenant_id="tenant-123", + app_id="app-123", + node_id="node-123", + created_by="account-123", + webhook_id="webhook-123", + workflow_id="workflow-123", + node_config=_node({}), + status=None, + ) + + +def _workflow_trigger() -> WebhookContext: + return _webhook_trigger() + + +class TestWebhookRequestParserUnit: + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("1234567890123456789", 1234567890123456789), + ("9007199254740993", 9007199254740993), + ("-9007199254740993", -9007199254740993), + ("1e3", 1000), + ], + ) + def test_form_number_preserves_exact_integers(self, parser: WebhookRequestParser, raw: str, expected: int) -> None: + assert parser._convert_form_value("amount", raw, SegmentType.NUMBER) == expected + + @pytest.mark.parametrize("raw", ["nan", "NaN", "inf", "-Infinity", "1e400"]) + def test_form_number_rejects_non_finite_values(self, parser: WebhookRequestParser, raw: str) -> None: + with pytest.raises(ValueError, match="to number"): + parser._convert_form_value("amount", raw, SegmentType.NUMBER) + + @pytest.mark.parametrize(("value", "kind"), [(True, "number"), ([1, True], "array[number]")]) + def test_json_number_rejects_booleans(self, parser: WebhookRequestParser, value: object, kind: str) -> None: + with pytest.raises(ValueError, match="Expected"): + parser._validate_json_value("amount", value, kind) + + def test_rejects_oversized_request(self, parser: WebhookRequestParser, flask_app: Flask) -> None: + with config_overrides_context(WEBHOOK_REQUEST_BODY_MAX_SIZE=1): + with flask_app.test_request_context("/webhook", method="POST", data="ab"): + with pytest.raises(RequestEntityTooLarge): + parser.extract_webhook_data(_webhook_trigger()) + + def test_extract_webhook_data_json(self, parser: WebhookRequestParser) -> None: + """Test webhook data extraction from JSON request.""" + app = Flask(__name__) + with app.test_request_context( + "/webhook", + method="POST", + headers={"Content-Type": "application/json", "Authorization": "Bearer token"}, + query_string="version=1&format=json", + json={"message": "hello", "count": 42}, + ): + webhook_trigger = _webhook_trigger() + webhook_data = parser.extract_webhook_data(webhook_trigger) + assert webhook_data["method"] == "POST" + assert webhook_data["headers"]["Authorization"] == "Bearer token" + assert webhook_data["query_params"]["version"] == "1" + assert webhook_data["query_params"]["format"] == "json" + assert webhook_data["body"]["message"] == "hello" + assert webhook_data["body"]["count"] == 42 + assert webhook_data["files"] == {} + + def test_extract_webhook_data_query_params_remain_strings(self, parser: WebhookRequestParser) -> None: + """Query parameters should be extracted as raw strings without automatic conversion.""" + app = Flask(__name__) + with app.test_request_context( + "/webhook", + method="GET", + headers={"Content-Type": "application/json"}, + query_string="count=42&threshold=3.14&enabled=true¬e=text", + ): + webhook_trigger = _webhook_trigger() + webhook_data = parser.extract_webhook_data(webhook_trigger) + assert webhook_data["query_params"]["count"] == "42" + assert webhook_data["query_params"]["threshold"] == "3.14" + assert webhook_data["query_params"]["enabled"] == "true" + assert webhook_data["query_params"]["note"] == "text" + + def test_extract_webhook_data_form_urlencoded(self, parser: WebhookRequestParser) -> None: + """Test webhook data extraction from form URL encoded request.""" + app = Flask(__name__) + with app.test_request_context( + "/webhook", + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={"username": "test", "password": "secret"}, + ): + webhook_trigger = _webhook_trigger() + webhook_data = parser.extract_webhook_data(webhook_trigger) + assert webhook_data["method"] == "POST" + assert webhook_data["body"]["username"] == "test" + assert webhook_data["body"]["password"] == "secret" + + def test_extract_webhook_data_raw_text(self, parser: WebhookRequestParser) -> None: + """Test webhook data extraction from raw text request.""" + app = Flask(__name__) + with app.test_request_context( + "/webhook", method="POST", headers={"Content-Type": "text/plain"}, data="raw text content" + ): + webhook_trigger = _webhook_trigger() + webhook_data = parser.extract_webhook_data(webhook_trigger) + assert webhook_data["method"] == "POST" + assert webhook_data["body"]["raw"] == "raw text content" + + def test_extract_webhook_data_invalid_json(self, parser: WebhookRequestParser) -> None: + """Test webhook data extraction with invalid JSON.""" + app = Flask(__name__) + with app.test_request_context( + "/webhook", method="POST", headers={"Content-Type": "application/json"}, data="invalid json" + ): + webhook_trigger = _webhook_trigger() + with pytest.raises(ValueError, match="Invalid JSON body"): + parser.extract_webhook_data(webhook_trigger) + + def test_generate_webhook_response_default(self, parser: WebhookRequestParser) -> None: + """Test webhook response generation with default values.""" + node_config = _node({}) + response_data, status_code = parser.generate_webhook_response(node_config) + assert status_code == 200 + assert response_data["status"] == "success" + assert "Webhook processed successfully" in response_data["message"] + + def test_generate_webhook_response_custom_json(self, parser: WebhookRequestParser) -> None: + """Test webhook response generation with custom JSON response.""" + node_config = _node({"status_code": 201, "response_body": '{"result": "created", "id": 123}'}) + response_data, status_code = parser.generate_webhook_response(node_config) + assert status_code == 201 + assert response_data["result"] == "created" + assert response_data["id"] == 123 + + def test_generate_webhook_response_custom_text(self, parser: WebhookRequestParser) -> None: + """Test webhook response generation with custom text response.""" + node_config = _node({"status_code": 202, "response_body": "Request accepted for processing"}) + response_data, status_code = parser.generate_webhook_response(node_config) + assert status_code == 202 + assert response_data["message"] == "Request accepted for processing" + + def test_generate_webhook_response_invalid_json(self, parser: WebhookRequestParser) -> None: + """Test webhook response generation with invalid JSON response.""" + node_config = _node({"status_code": 400, "response_body": '{"invalid": json}'}) + response_data, status_code = parser.generate_webhook_response(node_config) + assert status_code == 400 + assert response_data["message"] == '{"invalid": json}' + + def test_generate_webhook_response_empty_response_body(self, parser: WebhookRequestParser) -> None: + """Test webhook response generation with empty response body.""" + node_config = _node({"status_code": 204, "response_body": ""}) + response_data, status_code = parser.generate_webhook_response(node_config) + assert status_code == 204 + assert response_data["status"] == "success" + assert "Webhook processed successfully" in response_data["message"] + + def test_generate_webhook_response_array_json(self, parser: WebhookRequestParser) -> None: + """Test webhook response generation with JSON array response.""" + node_config = _node({"status_code": 200, "response_body": '[{"id": 1}, {"id": 2}]'}) + response_data, status_code = parser.generate_webhook_response(node_config) + assert status_code == 200 + assert isinstance(response_data, list) + assert len(response_data) == 2 + assert response_data[0]["id"] == 1 + assert response_data[1]["id"] == 2 + + def test_validate_json_value_string(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for string type.""" + result = parser._validate_json_value("name", "hello", "string") + assert result == "hello" + with pytest.raises(ValueError, match="Expected string, got int"): + parser._validate_json_value("name", 123, "string") + + def test_validate_json_value_number(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for number type.""" + result = parser._validate_json_value("count", 42, "number") + assert result == 42 + result = parser._validate_json_value("price", 19.99, "number") + assert result == 19.99 + with pytest.raises(ValueError, match="Expected number, got str"): + parser._validate_json_value("count", "42", "number") + + def test_validate_json_value_bool(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for boolean type.""" + result = parser._validate_json_value("enabled", True, "boolean") + assert result is True + result = parser._validate_json_value("enabled", False, "boolean") + assert result is False + with pytest.raises(ValueError, match="Expected boolean, got str"): + parser._validate_json_value("enabled", "true", "boolean") + + def test_validate_json_value_object(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for object type.""" + result = parser._validate_json_value("user", {"name": "John", "age": 30}, "object") + assert result == {"name": "John", "age": 30} + with pytest.raises(ValueError, match="Expected object, got str"): + parser._validate_json_value("user", "not_an_object", "object") + + def test_validate_json_value_array_string(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for array[string] type.""" + result = parser._validate_json_value("tags", ["tag1", "tag2", "tag3"], "array[string]") + assert result == ["tag1", "tag2", "tag3"] + with pytest.raises(ValueError, match="Expected array of strings, got str"): + parser._validate_json_value("tags", "not_an_array", "array[string]") + with pytest.raises(ValueError, match="Expected array of strings, got list"): + parser._validate_json_value("tags", ["tag1", 123, "tag3"], "array[string]") + + def test_validate_json_value_array_number(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for array[number] type.""" + result = parser._validate_json_value("scores", [1, 2.5, 3, 4.7], "array[number]") + assert result == [1, 2.5, 3, 4.7] + with pytest.raises(ValueError, match="Expected array of numbers, got list"): + parser._validate_json_value("scores", [1, "2", 3], "array[number]") + + def test_validate_json_value_array_bool(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for array[boolean] type.""" + result = parser._validate_json_value("flags", [True, False, True], "array[boolean]") + assert result == [True, False, True] + with pytest.raises(ValueError, match="Expected array of booleans, got list"): + parser._validate_json_value("flags", [True, "false", True], "array[boolean]") + + def test_validate_json_value_array_object(self, parser: WebhookRequestParser) -> None: + """Test JSON value validation for array[object] type.""" + result = parser._validate_json_value("users", [{"name": "John"}, {"name": "Jane"}], "array[object]") + assert result == [{"name": "John"}, {"name": "Jane"}] + with pytest.raises(ValueError, match="Expected array of objects, got list"): + parser._validate_json_value("users", [{"name": "John"}, "not_object"], "array[object]") + + def test_convert_form_value_string(self, parser: WebhookRequestParser) -> None: + """Test form value conversion for string type.""" + result = parser._convert_form_value("test", "hello", "string") + assert result == "hello" + + def test_convert_form_value_number(self, parser: WebhookRequestParser) -> None: + """Test form value conversion for number type.""" + result = parser._convert_form_value("count", "42", "number") + assert result == 42 + result = parser._convert_form_value("price", "19.99", "number") + assert result == 19.99 + with pytest.raises(ValueError, match="Cannot convert 'not_a_number' to number"): + parser._convert_form_value("count", "not_a_number", "number") + + def test_convert_form_value_boolean(self, parser: WebhookRequestParser) -> None: + """Test form value conversion for boolean type.""" + assert parser._convert_form_value("flag", "true", "boolean") is True + assert parser._convert_form_value("flag", "1", "boolean") is True + assert parser._convert_form_value("flag", "yes", "boolean") is True + assert parser._convert_form_value("flag", "false", "boolean") is False + assert parser._convert_form_value("flag", "0", "boolean") is False + assert parser._convert_form_value("flag", "no", "boolean") is False + with pytest.raises(ValueError, match="Cannot convert 'maybe' to boolean"): + parser._convert_form_value("flag", "maybe", "boolean") + + def test_extract_and_validate_webhook_data_success(self, parser: WebhookRequestParser) -> None: + """Test successful unified data extraction and validation.""" + app = Flask(__name__) + with app.test_request_context( + "/webhook", + method="POST", + headers={"Content-Type": "application/json"}, + query_string="count=42&enabled=true", + json={"message": "hello", "age": 25}, + ): + webhook_trigger = _webhook_trigger() + node_config = _node( + { + "method": "post", + "content_type": "application/json", + "params": [ + {"name": "count", "type": "number", "required": True}, + {"name": "enabled", "type": "boolean", "required": True}, + ], + "body": [ + {"name": "message", "type": "string", "required": True}, + {"name": "age", "type": "number", "required": True}, + ], + } + ) + result = parser.extract_and_validate_webhook_data(webhook_trigger, node_config) + assert result["query_params"]["count"] == 42 + assert result["query_params"]["enabled"] is True + assert result["body"]["message"] == "hello" + assert result["body"]["age"] == 25 + + def test_extract_and_validate_webhook_data_invalid_json_error(self, parser: WebhookRequestParser) -> None: + """Invalid JSON should bubble up as a ValueError with details.""" + app = Flask(__name__) + with app.test_request_context( + "/webhook", method="POST", headers={"Content-Type": "application/json"}, data='{"invalid": }' + ): + webhook_trigger = _webhook_trigger() + node_config = _node({"method": "post", "content_type": "application/json"}) + with pytest.raises(ValueError, match="Invalid JSON body"): + parser.extract_and_validate_webhook_data(webhook_trigger, node_config) + + def test_extract_and_validate_webhook_data_validation_error(self, parser: WebhookRequestParser) -> None: + """Test unified data extraction with validation error.""" + app = Flask(__name__) + with app.test_request_context("/webhook", method="GET", headers={"Content-Type": "application/json"}): + webhook_trigger = _webhook_trigger() + node_config = _node({"method": "post", "content_type": "application/json"}) + with pytest.raises(ValueError, match="HTTP method mismatch"): + parser.extract_and_validate_webhook_data(webhook_trigger, node_config) + + def test_extract_and_validate_webhook_request_missing_required_header(self, parser: WebhookRequestParser) -> None: + app = Flask(__name__) + with app.test_request_context("/webhook", method="POST", headers={"Content-Type": "application/json"}): + node_config = _node( + { + "method": "post", + "content_type": "application/json", + "headers": [{"name": "Authorization", "required": True}], + } + ) + with pytest.raises(ValueError, match="Required header missing: Authorization"): + parser.extract_and_validate_webhook_data(_webhook_trigger(), node_config) + + def test_extract_and_validate_webhook_request_case_insensitive_headers(self, parser: WebhookRequestParser) -> None: + app = Flask(__name__) + with app.test_request_context( + "/webhook", + method="POST", + headers={"Content-Type": "application/json", "authorization": "Bearer token"}, + json={"message": "hello"}, + ): + node_config = _node( + { + "method": "post", + "content_type": "application/json", + "headers": [{"name": "Authorization", "required": True}], + "body": [{"name": "message", "type": "string", "required": True}], + } + ) + result = parser.extract_and_validate_webhook_data(_webhook_trigger(), node_config) + assert result["headers"].get("Authorization") == "Bearer token" + + def test_extract_and_validate_webhook_request_missing_required_param(self, parser: WebhookRequestParser) -> None: + app = Flask(__name__) + with app.test_request_context( + "/webhook", method="POST", headers={"Content-Type": "application/json"}, json={"message": "hello"} + ): + node_config = _node( + { + "method": "post", + "content_type": "application/json", + "params": [{"name": "version", "required": True}], + "body": [{"name": "message", "type": "string", "required": True}], + } + ) + with pytest.raises(ValueError, match="Required parameter missing: version"): + parser.extract_and_validate_webhook_data(_webhook_trigger(), node_config) + + def test_extract_and_validate_webhook_request_missing_required_body_param( + self, parser: WebhookRequestParser + ) -> None: + app = Flask(__name__) + with app.test_request_context("/webhook", method="POST", headers={"Content-Type": "application/json"}, json={}): + node_config = _node( + { + "method": "post", + "content_type": "application/json", + "body": [{"name": "message", "type": "string", "required": True}], + } + ) + with pytest.raises(ValueError, match="Required body parameter missing: message"): + parser.extract_and_validate_webhook_data(_webhook_trigger(), node_config) + + def test_extract_and_validate_webhook_request_missing_required_file(self, parser: WebhookRequestParser) -> None: + app = Flask(__name__) + with app.test_request_context( + "/webhook", method="POST", data={"note": "test"}, content_type="multipart/form-data" + ): + node_config = _node( + { + "method": "post", + "content_type": "multipart/form-data", + "body": [{"name": "file", "type": "file", "required": True}], + } + ) + result = parser.extract_and_validate_webhook_data(_webhook_trigger(), node_config) + assert result["files"] == {} + + +class TestWebhookRequestParserExtractionFallbacks: + def test_extract_octet_stream_body_should_return_none_when_empty_payload( + self, parser: WebhookRequestParser, flask_app: Flask + ) -> None: + webhook_trigger = _workflow_trigger() + with flask_app.test_request_context("/webhook", method="POST", data=b""): + body, files = parser._extract_octet_stream_body(webhook_trigger) + assert body == {"raw": None} + assert files == {} + + +class TestWebhookRequestParserValidationAndConversion: + @pytest.mark.parametrize( + ("raw_value", "param_type", "expected"), + [ + ("42", SegmentType.NUMBER, 42), + ("3.14", SegmentType.NUMBER, float("3.14")), + ("yes", SegmentType.BOOLEAN, True), + ("no", SegmentType.BOOLEAN, False), + ], + ) + def test_convert_form_value_should_convert_supported_types( + self, parser: WebhookRequestParser, raw_value: str, param_type: str, expected: object + ) -> None: + result = parser._convert_form_value("param", raw_value, param_type) + assert result == expected + + def test_convert_form_value_should_raise_for_unsupported_type(self, parser: WebhookRequestParser) -> None: + with pytest.raises(ValueError, match="Unsupported type"): + parser._convert_form_value("p", "x", SegmentType.FILE) + + def test_validate_and_convert_value_should_wrap_conversion_errors(self, parser: WebhookRequestParser) -> None: + with pytest.raises(ValueError, match="validation failed"): + parser._validate_and_convert_value("param", "bad", SegmentType.NUMBER, is_form_data=True) + + def test_process_parameters_should_raise_when_required_parameter_missing( + self, parser: WebhookRequestParser + ) -> None: + raw_params = {"optional": "x"} + config = [WebhookParameter(name="required_param", type=SegmentType.STRING, required=True)] + with pytest.raises(ValueError, match="Required parameter missing"): + parser._process_parameters(raw_params, config, is_form_data=True) + + def test_process_parameters_should_include_unconfigured_parameters(self, parser: WebhookRequestParser) -> None: + raw_params = {"known": "1", "unknown": "x"} + config = [WebhookParameter(name="known", type=SegmentType.NUMBER, required=False)] + result = parser._process_parameters(raw_params, config, is_form_data=True) + assert result == {"known": 1, "unknown": "x"} + + def test_process_body_parameters_should_raise_when_required_text_raw_is_missing( + self, parser: WebhookRequestParser + ) -> None: + with pytest.raises(ValueError, match="Required body content missing"): + parser._process_body_parameters( + raw_body={"raw": ""}, + body_configs=[WebhookBodyParameter(name="raw", required=True)], + content_type=ContentType.TEXT, + ) + + def test_process_body_parameters_should_skip_file_config_for_multipart_form_data( + self, parser: WebhookRequestParser + ) -> None: + raw_body = {"message": "hello", "extra": "x"} + body_configs = [ + WebhookBodyParameter(name="upload", type=SegmentType.FILE, required=True), + WebhookBodyParameter(name="message", type=SegmentType.STRING, required=True), + ] + result = parser._process_body_parameters(raw_body, body_configs, ContentType.FORM_DATA) + assert result == {"message": "hello", "extra": "x"} + + def test_validate_required_headers_should_accept_sanitized_header_names(self, parser: WebhookRequestParser) -> None: + headers = {"x_api_key": "123"} + configs = [WebhookParameter(name="x-api-key", required=True)] + parser._validate_required_headers(headers, configs) + + def test_validate_required_headers_should_raise_when_required_header_missing( + self, parser: WebhookRequestParser + ) -> None: + headers = {"x-other": "123"} + configs = [WebhookParameter(name="x-api-key", required=True)] + with pytest.raises(ValueError, match="Required header missing"): + parser._validate_required_headers(headers, configs) + + def test_validate_http_metadata_should_return_content_type_mismatch_error( + self, parser: WebhookRequestParser + ) -> None: + webhook_data: RawWebhookDataDict = { + "method": "POST", + "headers": {"Content-Type": "application/json"}, + "query_params": {}, + "body": {}, + "files": {}, + } + node_data = WebhookData(method="post", content_type=ContentType.TEXT) + result = parser._validate_http_metadata(webhook_data, node_data) + assert result["valid"] is False + assert "Content-type mismatch" in result["error"] + + def test_extract_content_type_should_fallback_to_lowercase_header_key(self, parser: WebhookRequestParser) -> None: + headers = {"content-type": "application/json; charset=utf-8"} + assert parser._extract_content_type(headers) == "application/json" + + def test_build_workflow_inputs_should_include_expected_keys(self) -> None: + webhook_data: RawWebhookDataDict = { + "headers": {"h": "v"}, + "query_params": {"q": 1}, + "body": {"b": 2}, + "method": "POST", + "files": {}, + } + result = build_webhook_workflow_inputs(webhook_data) + assert result["webhook_data"] == webhook_data + assert result["webhook_headers"] == {"h": "v"} + assert result["webhook_query_params"] == {"q": 1} + assert result["webhook_body"] == {"b": 2} + + +class TestWebhookRequestParserUtilities: + def test_generate_webhook_response_should_fallback_when_response_body_is_not_json( + self, parser: WebhookRequestParser + ) -> None: + node_config = _node({"status_code": 200, "response_body": "{bad-json"}) + body, status = parser.generate_webhook_response(node_config) + assert status == 200 + assert "message" in body + + +def _node(data: Mapping[str, object]) -> NodeConfigDict: + return {"id": "hook", "data": WebhookData.model_validate(data)} diff --git a/api/tests/unit_tests/core/trigger/debug/test_event_selectors.py b/api/tests/unit_tests/core/trigger/debug/test_event_selectors.py index cbb7868b354..e35b152a6d7 100644 --- a/api/tests/unit_tests/core/trigger/debug/test_event_selectors.py +++ b/api/tests/unit_tests/core/trigger/debug/test_event_selectors.py @@ -232,15 +232,16 @@ class TestDebugEventSelectorsCases: ) mock_bus.poll.return_value = event - with patch("services.trigger.webhook_service.WebhookService") as mock_webhook_svc: - mock_webhook_svc.build_workflow_inputs.return_value = {"parsed": "data"} - - poller = WebhookTriggerDebugEventPoller(**_make_poller_args()) - result = poller.poll() + poller = WebhookTriggerDebugEventPoller(**_make_poller_args()) + result = poller.poll() assert result is not None - assert result.workflow_args["inputs"] == {"parsed": "data"} - mock_webhook_svc.build_workflow_inputs.assert_called_once_with({"body": "raw"}) + assert result.workflow_args["inputs"] == { + "webhook_data": {"body": "raw"}, + "webhook_headers": {}, + "webhook_query_params": {}, + "webhook_body": "raw", + } @patch("core.trigger.debug.event_selectors.TriggerDebugEventBus") def test_returns_none_when_no_event(self, mock_bus: MagicMock) -> None: diff --git a/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py b/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py index 9f450474ec0..39087790d7d 100644 --- a/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py +++ b/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py @@ -1,32 +1,102 @@ -from types import SimpleNamespace -from typing import cast -from unittest.mock import patch +"""Webhook listeners use real application wiring and caller-owned publication SQL.""" -from events.event_handlers import sync_webhook_when_app_created -from events.event_handlers.sync_webhook_when_app_created import handle -from models.model import App +import json +from uuid import uuid4 + +import pytest +from sqlalchemy import select +from sqlalchemy.orm import Session, sessionmaker + +from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated +from events.event_handlers.sync_webhook_when_app_created import handle, handle_published +from models.model import App, AppMode +from models.trigger import WorkflowWebhookTrigger from models.workflow import Workflow +from tests.unit_tests.events.event_handlers.test_sync_plugin_trigger_when_app_created import binding_app # noqa: F401 +from tests.unit_tests.repositories.trigger.test_webhook_runtime import WebhookSeed +from tests.unit_tests.repositories.trigger.test_webhook_runtime import webhook_seed as _webhook_seed + +webhook_seed = _webhook_seed -def test_draft_sync_preserves_stale_webhook_relationships() -> None: - app = cast(App, SimpleNamespace(mode="workflow")) - workflow = cast(Workflow, SimpleNamespace()) +def _workflow(app: App, node_ids: list[str]) -> Workflow: + return Workflow.new( + tenant_id=app.tenant_id, + app_id=app.id, + type="workflow", + version="published", + graph=json.dumps( + {"nodes": [{"id": node_id, "data": {"type": "trigger-webhook"}} for node_id in node_ids], "edges": []} + ), + features="{}", + created_by=str(uuid4()), + environment_variables=[], + conversation_variables=[], + rag_pipeline_variables=[], + ) - with patch( - "events.event_handlers.sync_webhook_when_app_created.WebhookService.sync_webhook_relationships" - ) as sync_webhook_relationships: + +def _bindings(session: Session, app_id: str) -> dict[str, str]: + return { + binding.node_id: binding.webhook_id + for binding in session.scalars(select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.app_id == app_id)) + } + + +def test_draft_listener_preserves_url_for_undo( + sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed +) -> None: + with sqlite_session_factory() as session: + app = session.get(App, webhook_seed.app_id) + assert app is not None + session.expunge(app) + handle(app, synced_draft_workflow=_workflow(app, ["new"])) + with sqlite_session_factory() as session: + bindings = _bindings(session, app.id) + assert bindings["hook"] == webhook_seed.webhook_id + assert len(bindings["new"]) == 24 + handle(app, synced_draft_workflow=_workflow(app, ["hook"])) + with sqlite_session_factory() as session: + assert _bindings(session, app.id) == bindings + + +@pytest.mark.parametrize("commit", [True, False], ids=["commit", "rollback"]) +def test_published_listener_removes_stale_bindings_in_publishers_transaction( + sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed, commit: bool +) -> None: + with sqlite_session_factory() as session: + app = session.get(App, webhook_seed.app_id) + assert app is not None + published = _workflow(app, ["new"]) + session.add(published) + app.workflow_id = published.id + handle_published(app, published_workflow=published, session=session) + assert set(_bindings(session, app.id)) == {"new"} + if commit: + session.commit() + else: + session.rollback() + with sqlite_session_factory() as session: + app = session.get(App, webhook_seed.app_id) + assert app is not None + assert app.workflow_id == (published.id if commit else webhook_seed.workflow_id) + bindings = _bindings(session, app.id) + assert set(bindings) == ({"new"} if commit else {"hook"}) + if not commit: + assert bindings["hook"] == webhook_seed.webhook_id + + +def test_non_workflow_app_is_ignored(sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed) -> None: + with sqlite_session_factory() as session: + app = session.get(App, webhook_seed.app_id) + assert app is not None + app.mode = AppMode.CHAT + workflow = _workflow(app, ["new"]) handle(app, synced_draft_workflow=workflow) - - sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=False) + handle_published(app, published_workflow=workflow, session=session) + assert _bindings(session, app.id) == {"hook": webhook_seed.webhook_id} -def test_published_sync_removes_stale_webhook_relationships() -> None: - app = cast(App, SimpleNamespace(mode="workflow")) - workflow = cast(Workflow, SimpleNamespace()) - - with patch( - "events.event_handlers.sync_webhook_when_app_created.WebhookService.sync_webhook_relationships" - ) as sync_webhook_relationships: - sync_webhook_when_app_created.handle_published(app, published_workflow=workflow) - - sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=True) +def test_webhook_listeners_are_registered() -> None: + assert handle in app_draft_workflow_was_synced.receivers_for(App()) + assert handle_published in app_published_workflow_was_updated.receivers_for(App()) diff --git a/api/tests/unit_tests/repositories/trigger/test_webhook_runtime.py b/api/tests/unit_tests/repositories/trigger/test_webhook_runtime.py new file mode 100644 index 00000000000..12b5562602e --- /dev/null +++ b/api/tests/unit_tests/repositories/trigger/test_webhook_runtime.py @@ -0,0 +1,240 @@ +"""Real SQL ownership and publication transactions for native webhooks.""" + +import json +from dataclasses import dataclass +from uuid import uuid4 + +import pytest +from sqlalchemy import Engine, select +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import QueuePool + +from models.enums import AppTriggerStatus, AppTriggerType +from models.model import App, AppMode +from models.trigger import AppTrigger, WorkflowWebhookTrigger +from models.workflow import Workflow, WorkflowDataError +from repositories.trigger.app_repository import SQLAlchemyAppTriggerRepository +from services.trigger.webhook_service import WebhookBindingService, WebhookContext, WebhookNotFoundError + + +@dataclass(frozen=True) +class WebhookSeed: + tenant_id: str + app_id: str + account_id: str + webhook_id: str + workflow_id: str + draft_id: str + trigger_id: str + + +@pytest.fixture +def webhook_seed(sqlite_session_factory: sessionmaker[Session]) -> WebhookSeed: + tenant_id, account_id = str(uuid4()), str(uuid4()) + with sqlite_session_factory.begin() as session: + app = App( + tenant_id=tenant_id, + name="Webhook app", + mode=AppMode.WORKFLOW, + created_by=account_id, + enable_site=True, + enable_api=True, + max_active_requests=0, + ) + session.add(app) + session.flush() + graph = json.dumps( + { + "nodes": [ + { + "id": "hook", + "data": { + "type": "trigger-webhook", + "title": "Webhook", + "method": "post", + "content_type": "application/json", + "headers": [], + "params": [], + "body": [], + }, + } + ], + "edges": [], + } + ) + workflows = [ + Workflow.new( + tenant_id=tenant_id, + app_id=app.id, + type="workflow", + version=version, + graph=graph, + features="{}", + created_by=account_id, + environment_variables=[], + conversation_variables=[], + rag_pipeline_variables=[], + ) + for version in ("published", Workflow.VERSION_DRAFT) + ] + app.workflow_id = workflows[0].id + webhook = WorkflowWebhookTrigger( + tenant_id=tenant_id, + app_id=app.id, + node_id="hook", + created_by=account_id, + webhook_id="abcdefghijklmnopqrstuvwx", + ) + trigger = AppTrigger( + tenant_id=tenant_id, + app_id=app.id, + node_id="hook", + title="Webhook", + trigger_type=AppTriggerType.TRIGGER_WEBHOOK, + status=AppTriggerStatus.ENABLED, + ) + session.add_all([*workflows, webhook, trigger]) + session.flush() + return WebhookSeed( + tenant_id, app.id, account_id, webhook.webhook_id, workflows[0].id, workflows[1].id, trigger.id + ) + + +def test_webhook_context_is_detached_and_uses_the_selected_version( + sqlite_session_factory: sessionmaker[Session], sqlite_engine: Engine, webhook_seed: WebhookSeed +) -> None: + repository = SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory) + live = repository.get_webhook_context(webhook_id=webhook_seed.webhook_id, debug=False) + draft = repository.get_webhook_context(webhook_id=webhook_seed.webhook_id, debug=True) + assert isinstance(live, WebhookContext) + assert isinstance(draft, WebhookContext) + assert live.workflow_id == webhook_seed.workflow_id + assert live.status == AppTriggerStatus.ENABLED + assert draft.workflow_id == webhook_seed.draft_id + assert draft.status is None + pool = sqlite_engine.pool + assert isinstance(pool, QueuePool) + assert pool.checkedout() == 0 + assert live.node_config["data"]["type"] == "trigger-webhook" + live.node_config["id"] = "local mutation" + refreshed = repository.get_webhook_context(webhook_id=webhook_seed.webhook_id, debug=False) + assert isinstance(refreshed, WebhookContext) + assert refreshed.node_config["data"]["title"] == "Webhook" + + +@pytest.mark.parametrize("mismatch", ["tenant", "app", "node", "type", "missing"]) +def test_published_webhook_requires_matching_app_trigger( + sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed, mismatch: str +) -> None: + with sqlite_session_factory.begin() as session: + trigger = session.get(AppTrigger, webhook_seed.trigger_id) + assert trigger is not None + if mismatch == "tenant": + trigger.tenant_id = str(uuid4()) + elif mismatch == "app": + trigger.app_id = str(uuid4()) + elif mismatch == "node": + trigger.node_id = "decoy" + elif mismatch == "type": + trigger.trigger_type = AppTriggerType.TRIGGER_PLUGIN + else: + session.delete(trigger) + repository = SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory) + with pytest.raises(WebhookNotFoundError): + repository.get_webhook_context(webhook_id=webhook_seed.webhook_id, debug=False) + draft = repository.get_webhook_context(webhook_id=webhook_seed.webhook_id, debug=True) + assert isinstance(draft, WebhookContext) + assert draft.workflow_id == webhook_seed.draft_id + + +@pytest.mark.parametrize("mismatch", ["tenant", "app", "draft", "missing_node"]) +def test_published_webhook_rejects_invalid_workflow_owner_chain( + sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed, mismatch: str +) -> None: + with sqlite_session_factory.begin() as session: + workflow = session.get(Workflow, webhook_seed.workflow_id) + assert workflow is not None + if mismatch == "tenant": + workflow.tenant_id = str(uuid4()) + elif mismatch == "app": + workflow.app_id = str(uuid4()) + elif mismatch == "draft": + workflow.version = Workflow.VERSION_DRAFT + else: + workflow.graph = '{"nodes": [], "edges": []}' + repository = SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory) + error = WorkflowDataError if mismatch == "missing_node" else WebhookNotFoundError + with pytest.raises(error): + repository.get_webhook_context(webhook_id=webhook_seed.webhook_id, debug=False) + + +def test_draft_undo_keeps_url_and_published_delete_joins_caller_transaction( + sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed +) -> None: + repository = SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory) + service = WebhookBindingService(sync_bindings=repository.sync_draft_webhooks) + service.sync( + tenant_id=webhook_seed.tenant_id, app_id=webhook_seed.app_id, created_by=webhook_seed.account_id, node_ids=[] + ) + service.sync( + tenant_id=webhook_seed.tenant_id, + app_id=webhook_seed.app_id, + created_by=webhook_seed.account_id, + node_ids=["hook", "new"], + ) + with sqlite_session_factory() as session: + rows = session.scalars(select(WorkflowWebhookTrigger)).all() + assert len(rows) == 2 + assert {r.webhook_id for r in rows if r.node_id == "hook"} == {webhook_seed.webhook_id} + repository.sync_webhooks( + session=session, + tenant_id=webhook_seed.tenant_id, + app_id=webhook_seed.app_id, + created_by=webhook_seed.account_id, + node_ids=[], + remove_stale=True, + ) + session.flush() + assert session.scalars(select(WorkflowWebhookTrigger)).all() == [] + session.rollback() + with sqlite_session_factory.begin() as session: + assert len(session.scalars(select(WorkflowWebhookTrigger)).all()) == 2 + repository.sync_webhooks( + session=session, + tenant_id=webhook_seed.tenant_id, + app_id=webhook_seed.app_id, + created_by=webhook_seed.account_id, + node_ids=["new"], + remove_stale=True, + ) + with sqlite_session_factory() as session: + assert [r.node_id for r in session.scalars(select(WorkflowWebhookTrigger))] == ["new"] + + +def test_missing_app_creator_is_rejected_only_when_creating_webhooks( + sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed +) -> None: + repository = SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory) + repository.sync_draft_webhooks( + tenant_id=webhook_seed.tenant_id, app_id=webhook_seed.app_id, created_by=None, node_ids=["hook"] + ) + with pytest.raises(RuntimeError, match="App is missing created_by"): + repository.sync_draft_webhooks( + tenant_id=webhook_seed.tenant_id, app_id=webhook_seed.app_id, created_by=None, node_ids=["hook", "new"] + ) + + +def test_binding_limit_validated_before_writing( + sqlite_session_factory: sessionmaker[Session], webhook_seed: WebhookSeed +) -> None: + repository = SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory) + service = WebhookBindingService(sync_bindings=repository.sync_draft_webhooks) + with pytest.raises(ValueError, match="maximum webhook node limit"): + service.sync( + tenant_id=webhook_seed.tenant_id, + app_id=webhook_seed.app_id, + created_by=webhook_seed.account_id, + node_ids=[str(i) for i in range(6)], + ) + with sqlite_session_factory() as session: + assert len(session.scalars(select(WorkflowWebhookTrigger)).all()) == 1 diff --git a/api/tests/unit_tests/services/trigger/test_webhook_service.py b/api/tests/unit_tests/services/trigger/test_webhook_service.py deleted file mode 100644 index a9b66bdb658..00000000000 --- a/api/tests/unit_tests/services/trigger/test_webhook_service.py +++ /dev/null @@ -1,1475 +0,0 @@ -import logging -from io import BytesIO -from types import SimpleNamespace -from typing import Any, cast -from unittest.mock import MagicMock, patch - -import pytest -from flask import Flask -from sqlalchemy import Engine as _webhook_service_relationships_Engine -from sqlalchemy import select -from sqlalchemy.engine import Engine as _webhook_service_Engine -from sqlalchemy.orm import Session, sessionmaker -from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import RequestEntityTooLarge - -from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE -from core.workflow.nodes.trigger_webhook.entities import ( - ContentType, - WebhookBodyParameter, - WebhookData, - WebhookParameter, -) -from graphon.variables.types import SegmentType -from models.enums import AppTriggerStatus, AppTriggerType, CreatorUserRole, EndUserType -from models.model import App, AppMode, EndUser -from models.tools import ToolFile -from models.trigger import AppTrigger, WorkflowWebhookTrigger -from models.workflow import Workflow -from repositories.trigger.app_repository import SQLAlchemyAppTriggerRepository -from services.errors.app import QuotaExceededError -from services.trigger import webhook_service -from services.trigger import webhook_service as service_module -from services.trigger import webhook_service as webhook_service_module -from services.trigger.app_trigger_service import AppTriggerService -from services.trigger.webhook_service import WebhookService -from services.workflow.entities import AsyncTriggerResponse, TriggerActor, TriggerData -from tests.helpers.unit.model_factories import make_app, make_end_user, make_workflow - - -@pytest.fixture -def trigger_status(sqlite_session_factory: sessionmaker[Session]) -> AppTriggerService: - return AppTriggerService(triggers=SQLAlchemyAppTriggerRepository(session_factory=sqlite_session_factory)) - - -class _EndUserServiceStub: - def __init__(self, result: EndUser | Exception) -> None: - self._result = result - self.calls: list[tuple[EndUserType, str, str, str | None]] = [] - - def get_or_create_end_user_by_type( - self, - type: EndUserType, - tenant_id: str, - app_id: str, - user_id: str | None = None, - ) -> EndUser: - self.calls.append((type, tenant_id, app_id, user_id)) - if isinstance(self._result, Exception): - raise self._result - return self._result - - -class _EndUserSequenceStub: - """Returns the given end users one per lookup, then keeps returning the last one.""" - - def __init__(self, *results: EndUser) -> None: - self._results = results - self.calls: list[tuple[EndUserType, str, str, str | None]] = [] - - def get_or_create_end_user_by_type( - self, - type: EndUserType, - tenant_id: str, - app_id: str, - user_id: str | None = None, - ) -> EndUser: - self.calls.append((type, tenant_id, app_id, user_id)) - return self._results[min(len(self.calls), len(self._results)) - 1] - - -def _webhook_trigger( - *, - webhook_id: str = "webhook-123", - tenant_id: str = "tenant-123", - app_id: str = "app-123", - node_id: str = "node-123", - created_by: str = "account-123", -) -> WorkflowWebhookTrigger: - return WorkflowWebhookTrigger( - webhook_id=webhook_id, - tenant_id=tenant_id, - app_id=app_id, - node_id=node_id, - created_by=created_by, - ) - - -def _tool_file() -> ToolFile: - tool_file = ToolFile( - user_id="user-123", - tenant_id="test_tenant", - conversation_id=None, - file_key="webhook/test.txt", - mimetype="text/plain", - name="test.txt", - size=7, - ) - tool_file.id = "test_file_id" - return tool_file - - -def _workflow( - *, - workflow_id: str = "workflow-123", - tenant_id: str = "tenant-123", - app_id: str = "app-123", - version: str = Workflow.VERSION_DRAFT, -) -> Workflow: - return make_workflow( - workflow_id=workflow_id, - tenant_id=tenant_id, - app_id=app_id, - version=version, - created_by="account-123", - ) - - -def _app(*, tenant_id: str = "tenant-123", app_id: str = "app-123") -> App: - return make_app( - app_id=app_id, - tenant_id=tenant_id, - name="Webhook App", - mode=AppMode.WORKFLOW, - icon_type=None, - max_active_requests=0, - ) - - -def _app_trigger(*, status: AppTriggerStatus = AppTriggerStatus.ENABLED) -> AppTrigger: - return AppTrigger( - tenant_id="tenant-123", - app_id="app-123", - node_id="node-123", - trigger_type=AppTriggerType.TRIGGER_WEBHOOK, - title="Webhook", - status=status, - ) - - -def _end_user(end_user_id: str = "end-user-123") -> EndUser: - return make_end_user( - end_user_id=end_user_id, - tenant_id="tenant-123", - app_id="app-123", - end_user_type=EndUserType.TRIGGER, - session_id="webhook-session", - ) - - -def _end_users() -> _EndUserServiceStub: - return _EndUserServiceStub(_end_user()) - - -class _WorkflowDispatcher: - def __init__(self, error: Exception | None = None) -> None: - self.error = error - self.actor: TriggerActor | None = None - self.data: TriggerData | None = None - - def trigger_workflow_async(self, *, actor: TriggerActor, trigger_data: TriggerData) -> AsyncTriggerResponse: - self.actor, self.data = actor, trigger_data - if self.error is not None: - raise self.error - return AsyncTriggerResponse(workflow_trigger_log_id="log", task_id="task", status="queued", queue="workflow") - - -class TestWebhookServiceCases: - class TestWebhookServiceLookup: - def test_debug_lookup_scopes_draft_workflow_to_trigger_owner( - self, - sqlite_session: Session, - sqlite_engine: _webhook_service_Engine, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - target_workflow = _workflow() - tenant_decoy = _workflow( - workflow_id="workflow-decoy", - tenant_id="tenant-other", - app_id="app-123", - ) - sqlite_session.add_all([_webhook_trigger(), target_workflow, tenant_decoy]) - sqlite_session.commit() - monkeypatch.setattr(webhook_service_module, "db", SimpleNamespace(engine=sqlite_engine)) - node_config = {"id": "node-123", "data": {}} - - with patch.object(Workflow, "get_node_config_by_id", autospec=True, return_value=node_config) as get_node: - webhook_trigger, workflow, result_node_config = WebhookService.get_webhook_trigger_and_workflow( - "webhook-123", is_debug=True - ) - - assert webhook_trigger.tenant_id == "tenant-123" - assert workflow.id == target_workflow.id - assert result_node_config is node_config - get_node.assert_called_once_with(workflow, "node-123") - - def test_published_lookup_uses_persisted_owner_chain( - self, - sqlite_session: Session, - sqlite_engine: _webhook_service_Engine, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - sqlite_session.add_all([_webhook_trigger(), _app_trigger(), _app()]) - sqlite_session.commit() - monkeypatch.setattr(webhook_service_module, "db", SimpleNamespace(engine=sqlite_engine)) - published_workflow = _workflow(version="published") - node_config = {"id": "node-123", "data": {}} - - with ( - patch.object(webhook_service_module, "WorkflowService") as workflow_service_class, - patch.object(Workflow, "get_node_config_by_id", autospec=True, return_value=node_config), - ): - get_published_workflow = workflow_service_class.return_value.get_published_workflow - get_published_workflow.return_value = published_workflow - webhook_trigger, workflow, result_node_config = WebhookService.get_webhook_trigger_and_workflow( - "webhook-123" - ) - - assert webhook_trigger.app_id == "app-123" - assert workflow is published_workflow - assert result_node_config is node_config - persisted_app = get_published_workflow.call_args.args[0] - assert isinstance(persisted_app, App) - assert persisted_app.id == "app-123" - assert persisted_app.tenant_id == "tenant-123" - - def test_published_lookup_rejects_cross_tenant_app_decoy( - self, - sqlite_session: Session, - sqlite_engine: _webhook_service_Engine, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - sqlite_session.add_all( - [ - _webhook_trigger(), - _app_trigger(), - _app(tenant_id="tenant-other"), - ] - ) - sqlite_session.commit() - monkeypatch.setattr(webhook_service_module, "db", SimpleNamespace(engine=sqlite_engine)) - - with pytest.raises(ValueError, match="App not found"): - WebhookService.get_webhook_trigger_and_workflow("webhook-123") - - @pytest.mark.parametrize( - "status", - [AppTriggerStatus.DISABLED, AppTriggerStatus.UNAUTHORIZED], - ) - def test_published_lookup_rejects_inactive_trigger( - self, - status: AppTriggerStatus, - sqlite_session: Session, - sqlite_engine: _webhook_service_Engine, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - sqlite_session.add_all([_webhook_trigger(), _app_trigger(status=status)]) - sqlite_session.commit() - monkeypatch.setattr(webhook_service_module, "db", SimpleNamespace(engine=sqlite_engine)) - - with pytest.raises(ValueError, match="disabled"): - WebhookService.get_webhook_trigger_and_workflow("webhook-123") - - def test_published_lookup_reports_rate_limited_trigger( - self, - sqlite_session: Session, - sqlite_engine: _webhook_service_Engine, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - sqlite_session.add_all([_webhook_trigger(), _app_trigger(status=AppTriggerStatus.RATE_LIMITED)]) - sqlite_session.commit() - monkeypatch.setattr(webhook_service_module, "db", SimpleNamespace(engine=sqlite_engine)) - - with pytest.raises(QuotaExceededError): - WebhookService.get_webhook_trigger_and_workflow("webhook-123") - - class TestWebhookServiceUnit: - """Webhook business-logic tests with isolated sessions where the service owns their lifecycle.""" - - def test_trigger_workflow_execution_propagates_quota_error_without_error_log( - self, - caplog: pytest.LogCaptureFixture, - trigger_status: AppTriggerService, - ) -> None: - quota_charge = MagicMock() - quota_error = QuotaExceededError(feature="workflow", tenant_id="tenant-123", required=1) - dispatch = _WorkflowDispatcher(error=quota_error) - caplog.set_level(logging.INFO) - with patch("services.trigger.webhook_service.QuotaService.reserve", return_value=quota_charge): - with pytest.raises(QuotaExceededError) as exc_info: - WebhookService.trigger_workflow_execution( - _webhook_trigger(), - {"body": {}, "headers": {}, "query_params": {}, "files": {}, "method": "POST"}, - _workflow(), - end_users=_EndUserServiceStub(_end_user()), - workflow_dispatch=dispatch, - trigger_status=trigger_status, - ) - assert exc_info.value is quota_error - quota_charge.refund.assert_called_once_with() - assert len(caplog.records) == 1 - assert caplog.records[0].levelno == logging.INFO - - def test_trigger_workflow_execution_success(self, trigger_status: AppTriggerService) -> None: - quota_charge = MagicMock() - dispatch = _WorkflowDispatcher() - end_user = _end_user() - with patch.object(webhook_service_module.QuotaService, "reserve", return_value=quota_charge): - WebhookService.trigger_workflow_execution( - _webhook_trigger(), - {"method": "POST", "headers": {}, "query_params": {}, "body": {}, "files": {}}, - _workflow(), - end_users=_EndUserServiceStub(end_user), - workflow_dispatch=dispatch, - trigger_status=trigger_status, - ) - assert dispatch.actor is not None - assert dispatch.actor.id == end_user.id - assert dispatch.actor.role == CreatorUserRole.END_USER - assert dispatch.data is not None - assert dispatch.data.workflow_id == _workflow().id - quota_charge.commit.assert_called_once_with() - quota_charge.refund.assert_not_called() - - def test_trigger_workflow_execution_end_user_service_failure(self, trigger_status: AppTriggerService) -> None: - dispatch = _WorkflowDispatcher() - with pytest.raises(ValueError, match="Failed to create end user"): - WebhookService.trigger_workflow_execution( - _webhook_trigger(), - {"method": "POST", "headers": {}, "query_params": {}, "body": {}, "files": {}}, - _workflow(), - end_users=_EndUserServiceStub(ValueError("Failed to create end user")), - workflow_dispatch=dispatch, - trigger_status=trigger_status, - ) - assert dispatch.actor is None - - def test_trigger_workflow_execution_reuses_the_end_user_that_owns_the_uploads( - self, trigger_status: AppTriggerService - ) -> None: - # The request creates trigger end user B and uploads under it while a concurrent request - # commits trigger end user A, so a second lookup would select A. The run must stay with B, - # because only the file's owner can load it. - app = Flask(__name__) - webhook_trigger = _webhook_trigger() - upload_owner = _end_user("end-user-b") - concurrent_end_user = _end_user("end-user-a") - end_users = _EndUserSequenceStub(upload_owner, concurrent_end_user) - dispatch = _WorkflowDispatcher() - - with ( - app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/octet-stream"}, - data=b"payload", - ), - patch.object(WebhookService, "_detect_binary_mimetype", return_value="text/plain", autospec=True), - patch.object(WebhookService, "_create_file_from_binary", autospec=True) as mock_create, - patch.object(webhook_service_module, "db"), - patch.object(webhook_service_module.QuotaService, "reserve"), - ): - mock_create.return_value.to_dict.return_value = {"file": "data"} - webhook_data = WebhookService.extract_webhook_data(webhook_trigger, end_users=end_users) - WebhookService.trigger_workflow_execution( - webhook_trigger, - webhook_data, - _workflow(), - end_users=end_users, - trigger_status=trigger_status, - workflow_dispatch=dispatch, - ) - - assert mock_create.call_args[0][3] is upload_owner - assert dispatch.actor is not None - assert dispatch.actor.id == upload_owner.id - assert end_users.calls == [(EndUserType.TRIGGER, "tenant-123", "app-123", None)] - - def test_trigger_end_user_is_looked_up_again_in_the_next_request(self) -> None: - app = Flask(__name__) - webhook_trigger = _webhook_trigger() - first_end_user = _end_user("end-user-1") - second_end_user = _end_user("end-user-2") - end_users = _EndUserSequenceStub(first_end_user, second_end_user) - - with app.test_request_context("/webhook", method="POST"): - assert WebhookService._resolve_trigger_end_user(webhook_trigger, end_users) is first_end_user - assert WebhookService._resolve_trigger_end_user(webhook_trigger, end_users) is first_end_user - with app.test_request_context("/webhook", method="POST"): - assert WebhookService._resolve_trigger_end_user(webhook_trigger, end_users) is second_end_user - - assert len(end_users.calls) == 2 - - def test_extract_webhook_data_json(self): - """Test webhook data extraction from JSON request.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json", "Authorization": "Bearer token"}, - query_string="version=1&format=json", - json={"message": "hello", "count": 42}, - ): - webhook_trigger = _webhook_trigger() - webhook_data = WebhookService.extract_webhook_data(webhook_trigger, end_users=_end_users()) - - assert webhook_data["method"] == "POST" - assert webhook_data["headers"]["Authorization"] == "Bearer token" - # Query params are now extracted as raw strings - assert webhook_data["query_params"]["version"] == "1" - assert webhook_data["query_params"]["format"] == "json" - assert webhook_data["body"]["message"] == "hello" - assert webhook_data["body"]["count"] == 42 - assert webhook_data["files"] == {} - - def test_extract_webhook_data_query_params_remain_strings(self): - """Query parameters should be extracted as raw strings without automatic conversion.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", - method="GET", - headers={"Content-Type": "application/json"}, - query_string="count=42&threshold=3.14&enabled=true¬e=text", - ): - webhook_trigger = _webhook_trigger() - webhook_data = WebhookService.extract_webhook_data(webhook_trigger, end_users=_end_users()) - - # After refactoring, raw extraction keeps query params as strings - assert webhook_data["query_params"]["count"] == "42" - assert webhook_data["query_params"]["threshold"] == "3.14" - assert webhook_data["query_params"]["enabled"] == "true" - assert webhook_data["query_params"]["note"] == "text" - - def test_extract_webhook_data_form_urlencoded(self): - """Test webhook data extraction from form URL encoded request.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - data={"username": "test", "password": "secret"}, - ): - webhook_trigger = _webhook_trigger() - webhook_data = WebhookService.extract_webhook_data(webhook_trigger, end_users=_end_users()) - - assert webhook_data["method"] == "POST" - assert webhook_data["body"]["username"] == "test" - assert webhook_data["body"]["password"] == "secret" - - def test_extract_webhook_data_multipart_with_files(self): - """Test webhook data extraction from multipart form with files.""" - app = Flask(__name__) - - # Create a mock file - file_content = b"test file content" - file_storage = FileStorage(stream=BytesIO(file_content), filename="test.txt", content_type="text/plain") - - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "multipart/form-data"}, - data={"message": "test", "file": file_storage}, - ): - webhook_trigger = _webhook_trigger() - webhook_trigger.tenant_id = "test_tenant" - - with patch.object(WebhookService, "_process_file_uploads", autospec=True) as mock_process_files: - mock_process_files.return_value = {"file": "mocked_file_obj"} - - webhook_data = WebhookService.extract_webhook_data(webhook_trigger, end_users=_end_users()) - - assert webhook_data["method"] == "POST" - assert webhook_data["body"]["message"] == "test" - assert webhook_data["files"]["file"] == "mocked_file_obj" - mock_process_files.assert_called_once() - - def test_extract_webhook_data_raw_text(self): - """Test webhook data extraction from raw text request.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", method="POST", headers={"Content-Type": "text/plain"}, data="raw text content" - ): - webhook_trigger = _webhook_trigger() - webhook_data = WebhookService.extract_webhook_data(webhook_trigger, end_users=_end_users()) - - assert webhook_data["method"] == "POST" - assert webhook_data["body"]["raw"] == "raw text content" - - def test_extract_octet_stream_body_uses_detected_mime(self): - """Octet-stream uploads should rely on detected MIME type.""" - app = Flask(__name__) - binary_content = b"plain text data" - - with app.test_request_context( - "/webhook", method="POST", headers={"Content-Type": "application/octet-stream"}, data=binary_content - ): - webhook_trigger = _webhook_trigger() - end_user = _end_user() - end_users = _EndUserServiceStub(end_user) - mock_file = MagicMock() - mock_file.to_dict.return_value = {"file": "data"} - - with ( - patch.object( - WebhookService, "_detect_binary_mimetype", return_value="text/plain", autospec=True - ) as mock_detect, - patch.object(WebhookService, "_create_file_from_binary", autospec=True) as mock_create, - ): - mock_create.return_value = mock_file - body, files = WebhookService._extract_octet_stream_body(webhook_trigger, end_users) - - assert body["raw"] == {"file": "data"} - assert files == {} - mock_detect.assert_called_once_with(binary_content) - mock_create.assert_called_once() - args = mock_create.call_args[0] - assert args[0] == binary_content - assert args[1] == "text/plain" - assert args[2] is webhook_trigger - assert args[3] is end_user - assert end_users.calls == [(EndUserType.TRIGGER, "tenant-123", "app-123", None)] - - def test_detect_binary_mimetype_uses_magic(self, monkeypatch: pytest.MonkeyPatch): - """python-magic output should be used when available.""" - fake_magic = MagicMock() - fake_magic.from_buffer.return_value = "image/png" - monkeypatch.setattr("services.trigger.webhook_service.magic", fake_magic) - - result = WebhookService._detect_binary_mimetype(b"binary data") - - assert result == "image/png" - fake_magic.from_buffer.assert_called_once() - - def test_detect_binary_mimetype_fallback_without_magic(self, monkeypatch: pytest.MonkeyPatch): - """Fallback MIME type should be used when python-magic is unavailable.""" - monkeypatch.setattr("services.trigger.webhook_service.magic", None) - - result = WebhookService._detect_binary_mimetype(b"binary data") - - assert result == "application/octet-stream" - - def test_detect_binary_mimetype_handles_magic_exception( - self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture - ): - """Fallback MIME type should be used when python-magic raises an exception.""" - try: - import magic as real_magic - except ImportError: - pytest.skip("python-magic is not installed") - - fake_magic = MagicMock() - fake_magic.from_buffer.side_effect = real_magic.MagicException("magic error") - monkeypatch.setattr("services.trigger.webhook_service.magic", fake_magic) - caplog.set_level(logging.DEBUG, logger="services.trigger.webhook_service") - - result = WebhookService._detect_binary_mimetype(b"binary data") - - assert result == "application/octet-stream" - assert "python-magic detection failed for octet-stream payload" in caplog.messages - - def test_extract_webhook_data_invalid_json(self): - """Test webhook data extraction with invalid JSON.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", method="POST", headers={"Content-Type": "application/json"}, data="invalid json" - ): - webhook_trigger = _webhook_trigger() - with pytest.raises(ValueError, match="Invalid JSON body"): - WebhookService.extract_webhook_data(webhook_trigger, end_users=_end_users()) - - def test_generate_webhook_response_default(self): - """Test webhook response generation with default values.""" - node_config = {"data": {}} - - response_data, status_code = WebhookService.generate_webhook_response(node_config) - - assert status_code == 200 - assert response_data["status"] == "success" - assert "Webhook processed successfully" in response_data["message"] - - def test_generate_webhook_response_custom_json(self): - """Test webhook response generation with custom JSON response.""" - node_config = {"data": {"status_code": 201, "response_body": '{"result": "created", "id": 123}'}} - - response_data, status_code = WebhookService.generate_webhook_response(node_config) - - assert status_code == 201 - assert response_data["result"] == "created" - assert response_data["id"] == 123 - - def test_generate_webhook_response_custom_text(self): - """Test webhook response generation with custom text response.""" - node_config = {"data": {"status_code": 202, "response_body": "Request accepted for processing"}} - - response_data, status_code = WebhookService.generate_webhook_response(node_config) - - assert status_code == 202 - assert response_data["message"] == "Request accepted for processing" - - def test_generate_webhook_response_invalid_json(self): - """Test webhook response generation with invalid JSON response.""" - node_config = {"data": {"status_code": 400, "response_body": '{"invalid": json}'}} - - response_data, status_code = WebhookService.generate_webhook_response(node_config) - - assert status_code == 400 - assert response_data["message"] == '{"invalid": json}' - - def test_generate_webhook_response_empty_response_body(self): - """Test webhook response generation with empty response body.""" - node_config = {"data": {"status_code": 204, "response_body": ""}} - - response_data, status_code = WebhookService.generate_webhook_response(node_config) - - assert status_code == 204 - assert response_data["status"] == "success" - assert "Webhook processed successfully" in response_data["message"] - - def test_generate_webhook_response_array_json(self): - """Test webhook response generation with JSON array response.""" - node_config = {"data": {"status_code": 200, "response_body": '[{"id": 1}, {"id": 2}]'}} - - response_data, status_code = WebhookService.generate_webhook_response(node_config) - - assert status_code == 200 - assert isinstance(response_data, list) - assert len(response_data) == 2 - assert response_data[0]["id"] == 1 - assert response_data[1]["id"] == 2 - - @patch("services.trigger.webhook_service.ToolFileManager", autospec=True) - @patch("services.trigger.webhook_service.file_factory", autospec=True) - def test_process_file_uploads_success(self, mock_file_factory, mock_tool_file_manager): - """Test successful file upload processing.""" - # Mock ToolFileManager - mock_tool_file_instance = mock_tool_file_manager.return_value # Mock file creation - mock_tool_file_instance.create_file_by_raw.return_value = _tool_file() - - # Mock file factory - mock_file_obj = MagicMock() - mock_file_factory.build_from_mapping.return_value = mock_file_obj - - # Create mock files - files = { - "file1": MagicMock(filename="test1.txt", content_type="text/plain"), - "file2": MagicMock(filename="test2.jpg", content_type="image/jpeg"), - } - - # Mock file reads - files["file1"].stream.read.return_value = b"content1" - files["file2"].stream.read.return_value = b"content2" - - webhook_trigger = _webhook_trigger() - webhook_trigger.tenant_id = "test_tenant" - end_users = _end_users() - - result = WebhookService._process_file_uploads(files, webhook_trigger, end_users) - - assert len(result) == 2 - assert "file1" in result - assert "file2" in result - - # Verify file processing was called for each file - assert mock_tool_file_manager.call_count == 2 - assert mock_file_factory.build_from_mapping.call_count == 2 - # Both files belong to the trigger end user, resolved once per request - assert end_users.calls == [(EndUserType.TRIGGER, "test_tenant", "app-123", None)] - create_calls = mock_tool_file_instance.create_file_by_raw.call_args_list - assert [call.kwargs["user_id"] for call in create_calls] == ["end-user-123", "end-user-123"] - - @patch("services.trigger.webhook_service.ToolFileManager", autospec=True) - @patch("services.trigger.webhook_service.file_factory", autospec=True) - def test_process_file_uploads_with_errors(self, mock_file_factory, mock_tool_file_manager): - """Test file upload processing with errors.""" - # Mock ToolFileManager - mock_tool_file_instance = mock_tool_file_manager.return_value # Mock file creation - mock_tool_file_instance.create_file_by_raw.return_value = _tool_file() - - # Mock file factory - mock_file_obj = MagicMock() - mock_file_factory.build_from_mapping.return_value = mock_file_obj - - # Create mock files, one will fail - files = { - "good_file": MagicMock(filename="test.txt", content_type="text/plain"), - "bad_file": MagicMock(filename="test.bad", content_type="text/plain"), - } - - files["good_file"].stream.read.return_value = b"content" - files["bad_file"].stream.read.side_effect = Exception("Read error") - - webhook_trigger = _webhook_trigger() - webhook_trigger.tenant_id = "test_tenant" - - result = WebhookService._process_file_uploads(files, webhook_trigger, _end_users()) - - # Should process the good file and skip the bad one - assert len(result) == 1 - assert "good_file" in result - assert "bad_file" not in result - - def test_process_file_uploads_empty_filename(self): - """Test file upload processing with empty filename.""" - files = { - "no_filename": MagicMock(filename="", content_type="text/plain"), - "none_filename": MagicMock(filename=None, content_type="text/plain"), - } - - webhook_trigger = _webhook_trigger() - webhook_trigger.tenant_id = "test_tenant" - - result = WebhookService._process_file_uploads(files, webhook_trigger, _end_users()) - - # Should skip files without filenames - assert len(result) == 0 - - def test_validate_json_value_string(self): - """Test JSON value validation for string type.""" - # Valid string - result = WebhookService._validate_json_value("name", "hello", "string") - assert result == "hello" - - # Invalid string (number) - should raise ValueError - with pytest.raises(ValueError, match="Expected string, got int"): - WebhookService._validate_json_value("name", 123, "string") - - def test_validate_json_value_number(self): - """Test JSON value validation for number type.""" - # Valid integer - result = WebhookService._validate_json_value("count", 42, "number") - assert result == 42 - - # Valid float - result = WebhookService._validate_json_value("price", 19.99, "number") - assert result == 19.99 - - # Invalid number (string) - should raise ValueError - with pytest.raises(ValueError, match="Expected number, got str"): - WebhookService._validate_json_value("count", "42", "number") - - # JSON true/false are bools, which Python treats as ints: still not a number - with pytest.raises(ValueError, match="Expected number, got bool"): - WebhookService._validate_json_value("count", True, "number") - with pytest.raises(ValueError, match="Expected number, got bool"): - WebhookService._validate_json_value("count", False, "number") - - def test_validate_json_value_bool(self): - """Test JSON value validation for boolean type.""" - # Valid boolean - result = WebhookService._validate_json_value("enabled", True, "boolean") - assert result is True - - result = WebhookService._validate_json_value("enabled", False, "boolean") - assert result is False - - # Invalid boolean (string) - should raise ValueError - with pytest.raises(ValueError, match="Expected boolean, got str"): - WebhookService._validate_json_value("enabled", "true", "boolean") - - def test_validate_json_value_object(self): - """Test JSON value validation for object type.""" - # Valid object - result = WebhookService._validate_json_value("user", {"name": "John", "age": 30}, "object") - assert result == {"name": "John", "age": 30} - - # Invalid object (string) - should raise ValueError - with pytest.raises(ValueError, match="Expected object, got str"): - WebhookService._validate_json_value("user", "not_an_object", "object") - - def test_validate_json_value_array_string(self): - """Test JSON value validation for array[string] type.""" - # Valid array of strings - result = WebhookService._validate_json_value("tags", ["tag1", "tag2", "tag3"], "array[string]") - assert result == ["tag1", "tag2", "tag3"] - - # Invalid - not an array - with pytest.raises(ValueError, match="Expected array of strings, got str"): - WebhookService._validate_json_value("tags", "not_an_array", "array[string]") - - # Invalid - array with non-strings - with pytest.raises(ValueError, match="Expected array of strings, got list"): - WebhookService._validate_json_value("tags", ["tag1", 123, "tag3"], "array[string]") - - def test_validate_json_value_array_number(self): - """Test JSON value validation for array[number] type.""" - # Valid array of numbers - result = WebhookService._validate_json_value("scores", [1, 2.5, 3, 4.7], "array[number]") - assert result == [1, 2.5, 3, 4.7] - - # Invalid - array with non-numbers - with pytest.raises(ValueError, match="Expected array of numbers, got list"): - WebhookService._validate_json_value("scores", [1, "2", 3], "array[number]") - - # Invalid - array with a boolean - with pytest.raises(ValueError, match="Expected array of numbers, got list"): - WebhookService._validate_json_value("scores", [1, True, 3], "array[number]") - - def test_validate_json_value_array_bool(self): - """Test JSON value validation for array[boolean] type.""" - # Valid array of booleans - result = WebhookService._validate_json_value("flags", [True, False, True], "array[boolean]") - assert result == [True, False, True] - - # Invalid - array with non-booleans - with pytest.raises(ValueError, match="Expected array of booleans, got list"): - WebhookService._validate_json_value("flags", [True, "false", True], "array[boolean]") - - def test_validate_json_value_array_object(self): - """Test JSON value validation for array[object] type.""" - # Valid array of objects - result = WebhookService._validate_json_value("users", [{"name": "John"}, {"name": "Jane"}], "array[object]") - assert result == [{"name": "John"}, {"name": "Jane"}] - - # Invalid - array with non-objects - with pytest.raises(ValueError, match="Expected array of objects, got list"): - WebhookService._validate_json_value("users", [{"name": "John"}, "not_object"], "array[object]") - - def test_convert_form_value_string(self): - """Test form value conversion for string type.""" - result = WebhookService._convert_form_value("test", "hello", "string") - assert result == "hello" - - def test_convert_form_value_number(self): - """Test form value conversion for number type.""" - # Integer - result = WebhookService._convert_form_value("count", "42", "number") - assert result == 42 - - # Float - result = WebhookService._convert_form_value("price", "19.99", "number") - assert result == 19.99 - - # Invalid number - with pytest.raises(ValueError, match="Cannot convert 'not_a_number' to number"): - WebhookService._convert_form_value("count", "not_a_number", "number") - - def test_convert_form_value_boolean(self): - """Test form value conversion for boolean type.""" - # True values - assert WebhookService._convert_form_value("flag", "true", "boolean") is True - assert WebhookService._convert_form_value("flag", "1", "boolean") is True - assert WebhookService._convert_form_value("flag", "yes", "boolean") is True - - # False values - assert WebhookService._convert_form_value("flag", "false", "boolean") is False - assert WebhookService._convert_form_value("flag", "0", "boolean") is False - assert WebhookService._convert_form_value("flag", "no", "boolean") is False - - # Invalid boolean - with pytest.raises(ValueError, match="Cannot convert 'maybe' to boolean"): - WebhookService._convert_form_value("flag", "maybe", "boolean") - - def test_extract_and_validate_webhook_data_success(self): - """Test successful unified data extraction and validation.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json"}, - query_string="count=42&enabled=true", - json={"message": "hello", "age": 25}, - ): - webhook_trigger = _webhook_trigger() - node_config = { - "data": { - "method": "post", - "content_type": "application/json", - "params": [ - {"name": "count", "type": "number", "required": True}, - {"name": "enabled", "type": "boolean", "required": True}, - ], - "body": [ - {"name": "message", "type": "string", "required": True}, - {"name": "age", "type": "number", "required": True}, - ], - } - } - - result = WebhookService.extract_and_validate_webhook_data( - webhook_trigger, node_config, end_users=_end_users() - ) - - # Check that types are correctly converted - assert result["query_params"]["count"] == 42 # Converted to int - assert result["query_params"]["enabled"] is True # Converted to bool - assert result["body"]["message"] == "hello" # Already string - assert result["body"]["age"] == 25 # Already number - - def test_extract_and_validate_webhook_data_invalid_json_error(self): - """Invalid JSON should bubble up as a ValueError with details.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json"}, - data='{"invalid": }', - ): - webhook_trigger = _webhook_trigger() - node_config = { - "data": { - "method": "post", - "content_type": "application/json", - } - } - - with pytest.raises(ValueError, match="Invalid JSON body"): - WebhookService.extract_and_validate_webhook_data( - webhook_trigger, node_config, end_users=_end_users() - ) - - def test_extract_and_validate_webhook_data_validation_error(self): - """Test unified data extraction with validation error.""" - app = Flask(__name__) - - with app.test_request_context( - "/webhook", - method="GET", # Wrong method - headers={"Content-Type": "application/json"}, - ): - webhook_trigger = _webhook_trigger() - node_config = { - "data": { - "method": "post", # Expects POST - "content_type": "application/json", - } - } - - with pytest.raises(ValueError, match="HTTP method mismatch"): - WebhookService.extract_and_validate_webhook_data( - webhook_trigger, node_config, end_users=_end_users() - ) - - def test_extract_and_validate_webhook_request_missing_required_header(self) -> None: - app = Flask(__name__) - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json"}, - ): - node_config = { - "data": { - "method": "post", - "content_type": "application/json", - "headers": [{"name": "Authorization", "required": True}], - } - } - - with pytest.raises(ValueError, match="Required header missing: Authorization"): - WebhookService.extract_and_validate_webhook_data( - _webhook_trigger(), node_config, end_users=_end_users() - ) - - def test_extract_and_validate_webhook_request_case_insensitive_headers(self) -> None: - app = Flask(__name__) - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json", "authorization": "Bearer token"}, - json={"message": "hello"}, - ): - node_config = { - "data": { - "method": "post", - "content_type": "application/json", - "headers": [{"name": "Authorization", "required": True}], - "body": [{"name": "message", "type": "string", "required": True}], - } - } - - result = WebhookService.extract_and_validate_webhook_data( - _webhook_trigger(), node_config, end_users=_end_users() - ) - - assert result["headers"].get("Authorization") == "Bearer token" - - def test_extract_and_validate_webhook_request_missing_required_param(self) -> None: - app = Flask(__name__) - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json"}, - json={"message": "hello"}, - ): - node_config = { - "data": { - "method": "post", - "content_type": "application/json", - "params": [{"name": "version", "required": True}], - "body": [{"name": "message", "type": "string", "required": True}], - } - } - - with pytest.raises(ValueError, match="Required parameter missing: version"): - WebhookService.extract_and_validate_webhook_data( - _webhook_trigger(), node_config, end_users=_end_users() - ) - - def test_extract_and_validate_webhook_request_missing_required_body_param(self) -> None: - app = Flask(__name__) - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json"}, - json={}, - ): - node_config = { - "data": { - "method": "post", - "content_type": "application/json", - "body": [{"name": "message", "type": "string", "required": True}], - } - } - - with pytest.raises(ValueError, match="Required body parameter missing: message"): - WebhookService.extract_and_validate_webhook_data( - _webhook_trigger(), node_config, end_users=_end_users() - ) - - def test_extract_and_validate_webhook_request_rejects_boolean_for_number_body_param(self) -> None: - app = Flask(__name__) - with app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/json"}, - json={"amount": True}, - ): - node_config = { - "data": { - "method": "post", - "content_type": "application/json", - "body": [{"name": "amount", "type": "number", "required": True}], - } - } - - with pytest.raises(ValueError, match="Parameter 'amount' validation failed: Expected number, got bool"): - WebhookService.extract_and_validate_webhook_data( - _webhook_trigger(), node_config, end_users=_end_users() - ) - - def test_extract_and_validate_webhook_request_missing_required_file(self) -> None: - app = Flask(__name__) - with app.test_request_context( - "/webhook", - method="POST", - data={"note": "test"}, - content_type="multipart/form-data", - ): - node_config = { - "data": { - "method": "post", - "content_type": "multipart/form-data", - "body": [{"name": "file", "type": "file", "required": True}], - } - } - - result = WebhookService.extract_and_validate_webhook_data( - _webhook_trigger(), node_config, end_users=_end_users() - ) - - assert result["files"] == {} - - def test_debug_mode_parameter_handling(self): - """Test that the debug mode parameter is properly handled in _prepare_webhook_execution.""" - from controllers.trigger.webhook import _prepare_webhook_execution - - # Mock the WebhookService methods - with ( - patch.object(WebhookService, "get_webhook_trigger_and_workflow", autospec=True) as mock_get_trigger, - patch.object(WebhookService, "extract_and_validate_webhook_data", autospec=True) as mock_extract, - patch("controllers.trigger.webhook.application_services", autospec=True), - ): - webhook_trigger = _webhook_trigger() - workflow = _workflow() - mock_config = {"data": {"test": "config"}} - mock_data = {"test": "data"} - - mock_get_trigger.return_value = (webhook_trigger, workflow, mock_config) - mock_extract.return_value = mock_data - - result = _prepare_webhook_execution("test_webhook", is_debug=False) - assert result == (webhook_trigger, workflow, mock_config, mock_data, None) - - # Reset mock - mock_get_trigger.reset_mock() - - result = _prepare_webhook_execution("test_webhook", is_debug=True) - assert result == (webhook_trigger, workflow, mock_config, mock_data, None) - - -def _workflow_trigger(**kwargs: Any) -> WorkflowWebhookTrigger: - values = { - "webhook_id": "webhook-123", - "tenant_id": "tenant-1", - "app_id": "app-1", - "node_id": "node-1", - "created_by": "user-1", - } - values.update(kwargs) - return WorkflowWebhookTrigger(**values) - - -class TestWebhookServiceAdditionalCases: - @staticmethod - @pytest.fixture - def flask_app() -> Flask: - return Flask(__name__) - - class TestWebhookServiceExtractionFallbacks: - def test_extract_webhook_data_should_use_text_fallback_for_unknown_content_type( - self, - flask_app: Flask, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, - ) -> None: - webhook_trigger = _workflow_trigger() - - with caplog.at_level(logging.WARNING, logger="services.trigger.webhook_service"): - with flask_app.test_request_context( - "/webhook", - method="POST", - headers={"Content-Type": "application/vnd.custom"}, - data="plain content", - ): - result = WebhookService.extract_webhook_data(webhook_trigger, end_users=_end_users()) - - assert result["body"] == {"raw": "plain content"} - assert any(r.levelno >= logging.WARNING for r in caplog.records) - - def test_extract_webhook_data_should_raise_for_request_too_large( - self, - flask_app: Flask, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - from tests.helpers.unit.config_override import apply_config_overrides - - apply_config_overrides(monkeypatch, WEBHOOK_REQUEST_BODY_MAX_SIZE=1) - - with flask_app.test_request_context("/webhook", method="POST", data="ab"): - with pytest.raises(RequestEntityTooLarge): - WebhookService.extract_webhook_data(_workflow_trigger(), end_users=_end_users()) - - def test_extract_octet_stream_body_should_return_none_when_empty_payload(self, flask_app: Flask) -> None: - webhook_trigger = _workflow_trigger() - - with flask_app.test_request_context("/webhook", method="POST", data=b""): - body, files = WebhookService._extract_octet_stream_body(webhook_trigger, _end_users()) - - assert body == {"raw": None} - assert files == {} - - def test_extract_octet_stream_body_should_return_none_when_processing_raises( - self, - flask_app: Flask, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - webhook_trigger = _workflow_trigger() - monkeypatch.setattr( - WebhookService, "_detect_binary_mimetype", MagicMock(return_value="application/octet-stream") - ) - monkeypatch.setattr(WebhookService, "_create_file_from_binary", MagicMock(side_effect=RuntimeError("boom"))) - - with flask_app.test_request_context("/webhook", method="POST", data=b"abc"): - body, files = WebhookService._extract_octet_stream_body(webhook_trigger, _end_users()) - - assert body == {"raw": None} - assert files == {} - - def test_extract_text_body_should_return_empty_string_when_request_read_fails( - self, - flask_app: Flask, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - monkeypatch.setattr("flask.wrappers.Request.get_data", MagicMock(side_effect=RuntimeError("read error"))) - - with flask_app.test_request_context("/webhook", method="POST", data="abc"): - body, files = WebhookService._extract_text_body() - - assert body == {"raw": ""} - assert files == {} - - def test_detect_binary_mimetype_should_fallback_when_magic_raises( - self, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - fake_magic = MagicMock() - fake_magic.from_buffer.side_effect = RuntimeError("magic failed") - monkeypatch.setattr(service_module, "magic", fake_magic) - - result = WebhookService._detect_binary_mimetype(b"binary") - - assert result == "application/octet-stream" - - def test_process_file_uploads_should_use_octet_stream_fallback_when_mimetype_unknown( - self, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - webhook_trigger = _workflow_trigger(created_by="user-1", tenant_id="tenant-1") - file_obj = MagicMock() - file_obj.to_dict.return_value = {"id": "f-1"} - monkeypatch.setattr(WebhookService, "_create_file_from_binary", MagicMock(return_value=file_obj)) - monkeypatch.setattr(service_module.mimetypes, "guess_type", MagicMock(return_value=(None, None))) - - uploaded = MagicMock() - uploaded.filename = "file.unknown" - uploaded.content_type = None - uploaded.read.return_value = b"content" - - result = WebhookService._process_file_uploads({"f": uploaded}, webhook_trigger, _end_users()) - - assert result == {"f": {"id": "f-1"}} - - def test_create_file_from_binary_should_call_tool_file_manager_and_file_factory( - self, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - webhook_trigger = _workflow_trigger(created_by="user-1", tenant_id="tenant-1") - manager = MagicMock() - manager.create_file_by_raw.return_value = SimpleNamespace(id="tool-file-1") - monkeypatch.setattr(service_module, "ToolFileManager", MagicMock(return_value=manager)) - expected_file = MagicMock() - monkeypatch.setattr( - service_module.file_factory, "build_from_mapping", MagicMock(return_value=expected_file) - ) - - result = WebhookService._create_file_from_binary(b"abc", "text/plain", webhook_trigger, _end_user()) - - assert result is expected_file - manager.create_file_by_raw.assert_called_once() - assert manager.create_file_by_raw.call_args.kwargs["user_id"] == "end-user-123" - - class TestWebhookServiceValidationAndConversion: - @pytest.mark.parametrize( - ("raw_value", "param_type", "expected"), - [ - ("42", SegmentType.NUMBER, 42), - ("3.14", SegmentType.NUMBER, float("3.14")), - ("yes", SegmentType.BOOLEAN, True), - ("no", SegmentType.BOOLEAN, False), - ], - ) - def test_convert_form_value_should_convert_supported_types( - self, - raw_value: str, - param_type: str, - expected: Any, - ) -> None: - result = WebhookService._convert_form_value("param", raw_value, param_type) - assert result == expected - - @pytest.mark.parametrize( - ("raw_value", "expected"), - [ - ("1234567890123456789", 1234567890123456789), - ("9007199254740993", 9007199254740993), - ("-9007199254740993", -9007199254740993), - ("1e3", 1000), - ], - ) - def test_convert_form_value_should_keep_integers_exact(self, raw_value: str, expected: int) -> None: - result = WebhookService._convert_form_value("param", raw_value, SegmentType.NUMBER) - - assert result == expected - assert isinstance(result, int) - - @pytest.mark.parametrize("raw_value", ["nan", "NaN", "inf", "-Infinity", "1e400"]) - def test_convert_form_value_should_reject_non_finite_numbers(self, raw_value: str) -> None: - with pytest.raises(ValueError, match="to number"): - WebhookService._convert_form_value("param", raw_value, SegmentType.NUMBER) - - def test_convert_form_value_should_raise_for_unsupported_type(self) -> None: - with pytest.raises(ValueError, match="Unsupported type"): - WebhookService._convert_form_value("p", "x", SegmentType.FILE) - - def test_validate_json_value_should_return_original_for_unmapped_supported_segment_type( - self, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, - ) -> None: - with caplog.at_level(logging.WARNING, logger="services.trigger.webhook_service"): - result = WebhookService._validate_json_value("param", {"x": 1}, "unsupported-type") - - assert result == {"x": 1} - assert any(r.levelno >= logging.WARNING for r in caplog.records) - - def test_validate_and_convert_value_should_wrap_conversion_errors(self) -> None: - with pytest.raises(ValueError, match="validation failed"): - WebhookService._validate_and_convert_value("param", "bad", SegmentType.NUMBER, is_form_data=True) - - def test_process_parameters_should_raise_when_required_parameter_missing(self) -> None: - raw_params = {"optional": "x"} - config = [WebhookParameter(name="required_param", type=SegmentType.STRING, required=True)] - - with pytest.raises(ValueError, match="Required parameter missing"): - WebhookService._process_parameters(raw_params, config, is_form_data=True) - - def test_process_parameters_should_include_unconfigured_parameters(self) -> None: - raw_params = {"known": "1", "unknown": "x"} - config = [WebhookParameter(name="known", type=SegmentType.NUMBER, required=False)] - - result = WebhookService._process_parameters(raw_params, config, is_form_data=True) - - assert result == {"known": 1, "unknown": "x"} - - def test_process_body_parameters_should_raise_when_required_text_raw_is_missing(self) -> None: - with pytest.raises(ValueError, match="Required body content missing"): - WebhookService._process_body_parameters( - raw_body={"raw": ""}, - body_configs=[WebhookBodyParameter(name="raw", required=True)], - content_type=ContentType.TEXT, - ) - - def test_process_body_parameters_should_skip_file_config_for_multipart_form_data(self) -> None: - raw_body = {"message": "hello", "extra": "x"} - body_configs = [ - WebhookBodyParameter(name="upload", type=SegmentType.FILE, required=True), - WebhookBodyParameter(name="message", type=SegmentType.STRING, required=True), - ] - - result = WebhookService._process_body_parameters(raw_body, body_configs, ContentType.FORM_DATA) - - assert result == {"message": "hello", "extra": "x"} - - def test_validate_required_headers_should_accept_sanitized_header_names(self) -> None: - headers = {"x_api_key": "123"} - configs = [WebhookParameter(name="x-api-key", required=True)] - - WebhookService._validate_required_headers(headers, configs) - - def test_validate_required_headers_should_raise_when_required_header_missing(self) -> None: - headers = {"x-other": "123"} - configs = [WebhookParameter(name="x-api-key", required=True)] - - with pytest.raises(ValueError, match="Required header missing"): - WebhookService._validate_required_headers(headers, configs) - - def test_validate_http_metadata_should_return_content_type_mismatch_error(self) -> None: - webhook_data = {"method": "POST", "headers": {"Content-Type": "application/json"}} - node_data = WebhookData(method="post", content_type=ContentType.TEXT) - - result = WebhookService._validate_http_metadata(webhook_data, node_data) - - assert result["valid"] is False - assert "Content-type mismatch" in result["error"] - - def test_extract_content_type_should_fallback_to_lowercase_header_key(self) -> None: - headers = {"content-type": "application/json; charset=utf-8"} - assert WebhookService._extract_content_type(headers) == "application/json" - - def test_build_workflow_inputs_should_include_expected_keys(self) -> None: - webhook_data = {"headers": {"h": "v"}, "query_params": {"q": 1}, "body": {"b": 2}} - - result = WebhookService.build_workflow_inputs(webhook_data) - - assert result["webhook_data"] == webhook_data - assert result["webhook_headers"] == {"h": "v"} - assert result["webhook_query_params"] == {"q": 1} - assert result["webhook_body"] == {"b": 2} - - class TestWebhookServiceUtilities: - def test_generate_webhook_response_should_fallback_when_response_body_is_not_json(self) -> None: - node_config = {"data": {"status_code": 200, "response_body": "{bad-json"}} - - body, status = WebhookService.generate_webhook_response(node_config) - - assert status == 200 - assert "message" in body - - def test_generate_webhook_id_should_return_24_character_identifier(self) -> None: - webhook_id = WebhookService.generate_webhook_id() - - assert isinstance(webhook_id, str) - assert len(webhook_id) == 24 - - def test_sanitize_key_should_return_original_value_for_non_string_input(self) -> None: - result = WebhookService._sanitize_key(123) # type: ignore[arg-type] - assert result == 123 - - -class TestWebhookServiceRelationshipsCases: - @staticmethod - def test_draft_sync_preserves_webhook_id_when_deleted_node_is_restored( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: _webhook_service_relationships_Engine - ) -> None: - session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - app = cast( - App, - SimpleNamespace(id="00000000-0000-0000-0000-000000000001", tenant_id="tenant-1", created_by="user-1"), - ) - assert app.created_by is not None - webhook_trigger = WorkflowWebhookTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id="webhook-node", - webhook_id="stable-webhook-id", - created_by=app.created_by, - ) - with session_maker.begin() as session: - session.add(webhook_trigger) - original_record_id = webhook_trigger.id - - redis = MagicMock() - redis.get.return_value = None - redis.lock.return_value.acquire.return_value = True - monkeypatch.setattr(webhook_service, "db", SimpleNamespace(engine=sqlite_engine)) - monkeypatch.setattr(webhook_service, "redis_client", redis) - - deleted_draft = cast(Workflow, SimpleNamespace(walk_nodes=lambda _node_type: iter(()))) - WebhookService.sync_webhook_relationships(app, deleted_draft, remove_stale=False) - - restored_draft = cast( - Workflow, - SimpleNamespace( - walk_nodes=lambda _node_type: iter([("webhook-node", {"type": TRIGGER_WEBHOOK_NODE_TYPE})]) - ), - ) - WebhookService.sync_webhook_relationships(app, restored_draft, remove_stale=False) - - with Session(sqlite_engine) as session: - records = session.scalars(select(WorkflowWebhookTrigger)).all() - - assert len(records) == 1 - assert records[0].id == original_record_id - assert records[0].webhook_id == "stable-webhook-id" - - @staticmethod - def test_published_sync_removes_stale_webhook_relationships( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: _webhook_service_relationships_Engine - ) -> None: - session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - app = cast( - App, - SimpleNamespace(id="00000000-0000-0000-0000-000000000001", tenant_id="tenant-1", created_by="user-1"), - ) - assert app.created_by is not None - webhook_trigger = WorkflowWebhookTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id="deleted-webhook-node", - webhook_id="deleted-webhook-id", - created_by=app.created_by, - ) - with session_maker.begin() as session: - session.add(webhook_trigger) - - redis = MagicMock() - redis.get.return_value = None - redis.lock.return_value.acquire.return_value = True - monkeypatch.setattr(webhook_service, "db", SimpleNamespace(engine=sqlite_engine)) - monkeypatch.setattr(webhook_service, "redis_client", redis) - - published_workflow = cast(Workflow, SimpleNamespace(walk_nodes=lambda _node_type: iter(()))) - WebhookService.sync_webhook_relationships(app, published_workflow, remove_stale=True) - - with session_maker() as session: - records = session.scalars(select(WorkflowWebhookTrigger)).all() - assert records == [] - redis.delete.assert_called_once_with( - f"{WebhookService.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:deleted-webhook-node" - )