fix: limit API request body size (#21474)

docs/faq.mdx

@@ -326,13 +326,11 @@ If your model is not currently supported but has APIs compatible with those of O
 
 ### How do I change the file size limit?
 
-The Go file-manager upload endpoint allows up to 1 GiB per file by default. The dataset document upload path has a separate 128 MiB per-file limit. Check which upload endpoint rejected the file before changing deployment settings.
+The Go API limits non-multipart request bodies to 1 MiB by default. Set `MAX_JSON_CONTENT_LENGTH` to change that limit in bytes. Multipart upload requests share a 1 GiB default total-body limit; set `MAX_MULTIPART_CONTENT_LENGTH` to change it in bytes. Upload endpoints can enforce smaller per-file or per-request limits.
 
-To change the file-manager upload limit in a Go Compose deployment, set `MAX_CONTENT_LENGTH` in **docker/.env** to the desired number of bytes. The default 1 GiB value is `1073741824`.
+The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file, and document-info uploads allow 100 MiB per file; increasing the shared multipart limit does not raise those per-file limits.
 
-If you raise the limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service. Lowering `MAX_CONTENT_LENGTH` below 1 GiB does not require changing the default Nginx limit.
-
-These settings do not raise the dataset document upload path's fixed 128 MiB limit.
+If you raise the multipart limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service.
 
 ---
 

internal/common/environments.go

@@ -88,6 +88,8 @@ const (
 	EnvDocEngine                         = "DOC_ENGINE"
 	EnvMaxFileNumPerUser                 = "MAX_FILE_NUM_PER_USER"
 	EnvMaxContentLength                  = "MAX_CONTENT_LENGTH"
+	EnvMaxJSONContentLength              = "MAX_JSON_CONTENT_LENGTH"
+	EnvMaxMultipartContentLength         = "MAX_MULTIPART_CONTENT_LENGTH"
 	EnvRAGFlowDictPath                   = "RAGFLOW_DICT_PATH"
 	EnvDefaultSuperuserEmail             = "DEFAULT_SUPERUSER_EMAIL"
 	EnvDefaultSuperuserNickname          = "DEFAULT_SUPERUSER_NICKNAME"

internal/common/http.go

@@ -38,8 +38,44 @@ import (
 const (
 	redactedLogValue      = "[REDACTED]"
 	maxLoggedVectorFloats = 3
+
+	// DefaultMaxJSONRequestBodyBytes is the default limit for non-multipart requests.
+	DefaultMaxJSONRequestBodyBytes int64 = 1 << 20
+	// DefaultMaxMultipartRequestBodyBytes is the default total limit for multipart uploads.
+	DefaultMaxMultipartRequestBodyBytes int64 = 1 << 30
+	// MaxWebhookRequestBodyBytes is the default and maximum request body size for agent webhooks.
+	MaxWebhookRequestBodyBytes int64 = 10 * 1_000_000
 )
 
+// GetMaxJSONRequestBodyBytes returns the configured non-multipart request-body
+// limit in bytes, defaulting to 1 MiB.
+func GetMaxJSONRequestBodyBytes() int64 {
+	return getMaxRequestBodyBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes)
+}
+
+// GetMaxMultipartRequestBodyBytes returns the configured multipart upload
+// request-body limit in bytes, defaulting to 1 GiB. MAX_CONTENT_LENGTH remains
+// a fallback for existing deployments when the new variable is unset.
+func GetMaxMultipartRequestBodyBytes() int64 {
+	return getMaxRequestBodyBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes)
+}
+
+func getMaxRequestBodyBytes(envName, fallbackEnvName string, defaultBytes int64) int64 {
+	raw := strings.TrimSpace(GetEnv(envName))
+	if raw == "" && fallbackEnvName != "" {
+		raw = strings.TrimSpace(GetEnv(fallbackEnvName))
+	}
+	if raw == "" {
+		return defaultBytes
+	}
+
+	limit, err := strconv.ParseInt(raw, 10, 64)
+	if err != nil || limit <= 0 {
+		return defaultBytes
+	}
+	return limit
+}
+
 var ssrfHttpClient *http.Client
 var schemeSafeHttpClient *http.Client
 

internal/handler/agent_webhook_security.go

@@ -60,13 +60,8 @@ const (
 	bytesPerKB int64 = 1000
 	bytesPerMB int64 = bytesPerKB * 1000
 
-	// webhookBodyMaxBytes is the hard ceiling above which a configured
-	// max_body_size is rejected at config-validation time. Stated
-	// in decimal MB to match the kb/mb unit base above; 10 MB
-	// decimal = 10 000 000 bytes. The python reference
-	// (agent_api.py:1652) uses 10 * 1024 * 1024 = 10 485 760 bytes,
-	// so this is ~486 KB stricter than python.
-	webhookBodyMaxBytes int64 = 10 * bytesPerMB
+	// webhookBodyMaxBytes is the default and hard maximum request body size.
+	webhookBodyMaxBytes int64 = common.MaxWebhookRequestBodyBytes
 
 	// webhookRateLimitTimeout is the redis call timeout used for the
 	// strict token-bucket lookup. Short on purpose — a security gate
@@ -149,12 +144,11 @@ func validateMaxBodySize(c *gin.Context, cfg map[string]any) error {
 	return nil
 }
 
-// parseMaxBodySize returns the byte limit configured by
-// `max_body_size` (with the 10 MB cap enforced) or 0 when no limit
-// is configured. The handler uses this to wrap c.Request.Body in
-// http.MaxBytesReader so the actual stream read is bounded — the
-// Content-Length header check alone is insufficient because a client
-// can advertise a small Content-Length and stream more.
+// parseMaxBodySize returns the configured body limit, defaulting to 10 MB
+// when `max_body_size` is absent. Configured values cannot exceed that cap.
+// The handler wraps c.Request.Body in http.MaxBytesReader so the actual stream
+// read is bounded — the Content-Length header check alone is insufficient
+// because a client can advertise a small Content-Length and stream more.
 //
 // Unit base: 1 kb = 1 000 B, 1 mb = 1 000 000 B (SI decimal). Per
 // user request; note this DIVERGES from the python reference
@@ -166,11 +160,18 @@ func validateMaxBodySize(c *gin.Context, cfg map[string]any) error {
 // the 10 MB cap. We check the parsed number against the cap before
 // multiplying.
 func parseMaxBodySize(cfg map[string]any) (int64, error) {
-	raw, ok := cfg["max_body_size"].(string)
-	if !ok || raw == "" {
-		return 0, nil
+	rawValue, exists := cfg["max_body_size"]
+	if !exists || rawValue == nil {
+		return webhookBodyMaxBytes, nil
+	}
+	raw, ok := rawValue.(string)
+	if !ok {
+		return 0, fmt.Errorf("invalid max_body_size format")
 	}
 	sizeStr := strings.ToLower(strings.TrimSpace(raw))
+	if sizeStr == "" {
+		return webhookBodyMaxBytes, nil
+	}
 	var limit int64
 	switch {
 	case strings.HasSuffix(sizeStr, "kb"):

internal/handler/agent_webhook_security_test.go

@@ -357,6 +357,22 @@ func TestValidateMaxBodySize_OverflowGuard(t *testing.T) {
 	}
 }
 
+func TestParseMaxBodySizeDefaultsTo10MB(t *testing.T) {
+	for _, cfg := range []map[string]any{
+		nil,
+		{},
+		{"max_body_size": ""},
+	} {
+		got, err := parseMaxBodySize(cfg)
+		if err != nil {
+			t.Fatalf("parseMaxBodySize(%v): %v", cfg, err)
+		}
+		if got != common.MaxWebhookRequestBodyBytes {
+			t.Errorf("parseMaxBodySize(%v) = %d, want %d", cfg, got, common.MaxWebhookRequestBodyBytes)
+		}
+	}
+}
+
 // TestParseMaxBodySize_DecimalUnits documents the per-user-request
 // SI-decimal unit base: 1 kb = 1 000 B, 1 mb = 1 000 000 B.
 // (The python reference uses 1 024 / 1 048 576.)

internal/router/request_body_limit.go

@@ -0,0 +1,64 @@
+// Package router owns the API request-size policy.
+package router
+
+import (
+	"mime"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+
+	"ragflow/internal/common"
+)
+
+const agentWebhookRoute = "/api/v1/agents/:canvas_id/webhook"
+
+// requestBodyLimitMiddleware applies configurable limits to API request bodies.
+// Multipart upload routes use a shared limit; handlers may enforce lower
+// endpoint-specific limits. Public Agent webhooks retain their 10 MB ceiling.
+func requestBodyLimitMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if c.Request.Body == nil || c.Request.Body == http.NoBody {
+			c.Next()
+			return
+		}
+
+		limit := common.GetMaxJSONRequestBodyBytes()
+		fullPath := c.FullPath()
+		switch {
+		case fullPath == agentWebhookRoute:
+			limit = common.MaxWebhookRequestBodyBytes
+		case isMultipartUploadRequest(c):
+			limit = common.GetMaxMultipartRequestBodyBytes()
+		}
+
+		if c.Request.ContentLength > limit {
+			common.ResponseWithHttpCodeData(c, http.StatusRequestEntityTooLarge, common.CodeBadRequest, nil, "request body too large")
+			c.Abort()
+			return
+		}
+		c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, limit)
+		c.Next()
+	}
+}
+
+func isMultipartUploadRequest(c *gin.Context) bool {
+	if c.Request.Method != http.MethodPost || !isMultipartFormData(c.Request.Header.Get("Content-Type")) {
+		return false
+	}
+
+	switch c.FullPath() {
+	case "/api/v1/files",
+		"/api/v1/agents/:canvas_id/upload",
+		"/api/v1/chat/audio/transcription",
+		"/api/v1/documents/upload",
+		"/api/v1/datasets/:dataset_id/documents":
+		return true
+	default:
+		return false
+	}
+}
+
+func isMultipartFormData(contentType string) bool {
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	return err == nil && mediaType == "multipart/form-data"
+}

internal/router/request_body_limit_test.go

@@ -0,0 +1,142 @@
+package router
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+
+	"ragflow/internal/common"
+)
+
+func TestRequestBodyLimitMiddleware(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	t.Setenv("MAX_JSON_CONTENT_LENGTH", "")
+	t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "")
+	t.Setenv("MAX_CONTENT_LENGTH", "")
+
+	engine := gin.New()
+	engine.Use(requestBodyLimitMiddleware())
+	readBody := func(c *gin.Context) {
+		_, err := io.Copy(io.Discard, c.Request.Body)
+		if err != nil {
+			var maxErr *http.MaxBytesError
+			if errors.As(err, &maxErr) {
+				c.Status(http.StatusRequestEntityTooLarge)
+				return
+			}
+			c.Status(http.StatusBadRequest)
+			return
+		}
+		c.Status(http.StatusNoContent)
+	}
+	engine.POST("/api/v1/test", readBody)
+	engine.POST("/api/v1/files", readBody)
+	engine.DELETE("/api/v1/files", readBody)
+	engine.POST("/api/v1/agents/:canvas_id/upload", readBody)
+	engine.POST("/api/v1/chat/audio/transcription", readBody)
+	engine.POST("/api/v1/documents/upload", readBody)
+	engine.POST("/api/v1/datasets/:dataset_id/documents", readBody)
+	engine.POST(agentWebhookRoute, readBody)
+
+	t.Run("rejects oversized default request", func(t *testing.T) {
+		body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1))
+		req := httptest.NewRequest(http.MethodPost, "/api/v1/test", bytes.NewReader(body))
+		req.Header.Set("Content-Type", "application/json")
+		resp := httptest.NewRecorder()
+		engine.ServeHTTP(resp, req)
+		if resp.Code != http.StatusRequestEntityTooLarge {
+			t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge)
+		}
+	})
+
+	t.Run("bounds chunked request body", func(t *testing.T) {
+		body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1))
+		req := httptest.NewRequest(http.MethodPost, "/api/v1/test", bytes.NewReader(body))
+		req.ContentLength = -1
+		req.TransferEncoding = []string{"chunked"}
+		req.Header.Set("Content-Type", "application/json")
+		resp := httptest.NewRecorder()
+		engine.ServeHTTP(resp, req)
+		if resp.Code != http.StatusRequestEntityTooLarge {
+			t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge)
+		}
+	})
+
+	t.Run("allows webhook body up to 10 MB", func(t *testing.T) {
+		body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1))
+		req := httptest.NewRequest(http.MethodPost, "/api/v1/agents/a1/webhook", bytes.NewReader(body))
+		req.Header.Set("Content-Type", "application/json")
+		resp := httptest.NewRecorder()
+		engine.ServeHTTP(resp, req)
+		if resp.Code != http.StatusNoContent {
+			t.Fatalf("status=%d, want %d", resp.Code, http.StatusNoContent)
+		}
+	})
+
+	t.Run("rejects webhook body above 10 MB", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/api/v1/agents/a1/webhook", bytes.NewReader(nil))
+		req.ContentLength = common.MaxWebhookRequestBodyBytes + 1
+		resp := httptest.NewRecorder()
+		engine.ServeHTTP(resp, req)
+		if resp.Code != http.StatusRequestEntityTooLarge {
+			t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge)
+		}
+	})
+
+	t.Run("does not exempt non-upload methods on file routes", func(t *testing.T) {
+		body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1))
+		req := httptest.NewRequest(http.MethodDelete, "/api/v1/files", bytes.NewReader(body))
+		req.Header.Set("Content-Type", "multipart/form-data; boundary=test")
+		resp := httptest.NewRecorder()
+		engine.ServeHTTP(resp, req)
+		if resp.Code != http.StatusRequestEntityTooLarge {
+			t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge)
+		}
+	})
+
+	t.Run("uses configured JSON request-body limit", func(t *testing.T) {
+		t.Setenv("MAX_JSON_CONTENT_LENGTH", "16")
+		body := bytes.Repeat([]byte("x"), 17)
+		request := httptest.NewRequest(http.MethodPost, "/api/v1/test", bytes.NewReader(body))
+		request.Header.Set("Content-Type", "application/json")
+		response := httptest.NewRecorder()
+		engine.ServeHTTP(response, request)
+		if response.Code != http.StatusRequestEntityTooLarge {
+			t.Fatalf("status=%d, want %d", response.Code, http.StatusRequestEntityTooLarge)
+		}
+	})
+
+	t.Run("uses configured multipart limit for every upload route", func(t *testing.T) {
+		t.Setenv("MAX_JSON_CONTENT_LENGTH", "8")
+		t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "16")
+		paths := []string{
+			"/api/v1/files",
+			"/api/v1/agents/a1/upload",
+			"/api/v1/chat/audio/transcription",
+			"/api/v1/documents/upload",
+			"/api/v1/datasets/d1/documents",
+		}
+		for _, path := range paths {
+			request := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(bytes.Repeat([]byte("x"), 12)))
+			request.Header.Set("Content-Type", "multipart/form-data; boundary=test")
+			response := httptest.NewRecorder()
+			engine.ServeHTTP(response, request)
+			if response.Code != http.StatusNoContent {
+				t.Errorf("POST %s status=%d, want %d for body above JSON limit", path, response.Code, http.StatusNoContent)
+			}
+
+			request = httptest.NewRequest(http.MethodPost, path, bytes.NewReader(bytes.Repeat([]byte("x"), 17)))
+			request.Header.Set("Content-Type", "multipart/form-data; boundary=test")
+			response = httptest.NewRecorder()
+			engine.ServeHTTP(response, request)
+			if response.Code != http.StatusRequestEntityTooLarge {
+				t.Errorf("POST %s status=%d, want %d for body above multipart limit", path, response.Code, http.StatusRequestEntityTooLarge)
+			}
+		}
+	})
+}

internal/router/router.go

@@ -136,6 +136,7 @@ func NewRouter(
 
 // Setup setup routes
 func (r *Router) Setup(engine *gin.Engine) {
+	engine.Use(requestBodyLimitMiddleware())
 	SetupEERouter(engine)
 
 	// Mark all responses from Go with a header for debugging.

internal/service/file/file_test.go

@@ -42,20 +42,27 @@ func testFileService() *FileService {
 }
 
 func TestDeploymentUploadMaxBytes(t *testing.T) {
+	t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "")
 	t.Setenv("MAX_CONTENT_LENGTH", "")
 	if got := DeploymentUploadMaxBytes(); got != defaultDeploymentUploadMaxBytes {
 		t.Fatalf("default limit = %d, want %d", got, defaultDeploymentUploadMaxBytes)
 	}
 
-	t.Setenv("MAX_CONTENT_LENGTH", "12345")
+	t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "12345")
 	if got := DeploymentUploadMaxBytes(); got != 12345 {
 		t.Fatalf("configured limit = %d, want 12345", got)
 	}
 
-	t.Setenv("MAX_CONTENT_LENGTH", "invalid")
+	t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "invalid")
 	if got := DeploymentUploadMaxBytes(); got != defaultDeploymentUploadMaxBytes {
 		t.Fatalf("invalid limit = %d, want default %d", got, defaultDeploymentUploadMaxBytes)
 	}
+
+	t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "")
+	t.Setenv("MAX_CONTENT_LENGTH", "54321")
+	if got := DeploymentUploadMaxBytes(); got != 54321 {
+		t.Fatalf("legacy configured limit = %d, want 54321", got)
+	}
 }
 
 func TestReadDeploymentUploadDataLimit(t *testing.T) {

internal/service/file/file_upload.go

@@ -13,23 +13,14 @@ import (
 	"ragflow/internal/permission"
 	"ragflow/internal/storage"
 	"ragflow/internal/utility"
-	"strconv"
 	"strings"
 	"time"
 )
 
-const defaultDeploymentUploadMaxBytes int64 = 1 << 30
+const defaultDeploymentUploadMaxBytes int64 = common.DefaultMaxMultipartRequestBodyBytes
 
 func DeploymentUploadMaxBytes() int64 {
-	raw := common.GetEnv(common.EnvMaxContentLength)
-	if raw == "" {
-		return defaultDeploymentUploadMaxBytes
-	}
-	n, err := strconv.ParseInt(raw, 10, 64)
-	if err != nil || n <= 0 {
-		return defaultDeploymentUploadMaxBytes
-	}
-	return n
+	return common.GetMaxMultipartRequestBodyBytes()
 }
 
 // UploadFile uploads files to a folder