From 2f995489118416b1d4a341c1989cfcc03214c730 Mon Sep 17 00:00:00 2001 From: buua436 Date: Mon, 26 Oct 2026 19:27:58 +0800 Subject: [PATCH] fix: limit API request body size (#21474) --- docs/faq.mdx | 8 +- internal/common/environments.go | 2 + internal/common/http.go | 36 +++++ internal/handler/agent_webhook_security.go | 33 ++-- .../handler/agent_webhook_security_test.go | 16 ++ internal/router/request_body_limit.go | 64 ++++++++ internal/router/request_body_limit_test.go | 142 ++++++++++++++++++ internal/router/router.go | 1 + internal/service/file/file_test.go | 11 +- internal/service/file/file_upload.go | 13 +- 10 files changed, 292 insertions(+), 34 deletions(-) create mode 100644 internal/router/request_body_limit.go create mode 100644 internal/router/request_body_limit_test.go diff --git a/docs/faq.mdx b/docs/faq.mdx index 7c263f08f8..21d053efe8 100644 --- a/docs/faq.mdx +++ b/docs/faq.mdx @@ -326,13 +326,11 @@ If your model is not currently supported but has APIs compatible with those of O ### How do I change the file size limit? -The Go file-manager upload endpoint allows up to 1 GiB per file by default. The dataset document upload path has a separate 128 MiB per-file limit. Check which upload endpoint rejected the file before changing deployment settings. +The Go API limits non-multipart request bodies to 1 MiB by default. Set `MAX_JSON_CONTENT_LENGTH` to change that limit in bytes. Multipart upload requests share a 1 GiB default total-body limit; set `MAX_MULTIPART_CONTENT_LENGTH` to change it in bytes. Upload endpoints can enforce smaller per-file or per-request limits. -To change the file-manager upload limit in a Go Compose deployment, set `MAX_CONTENT_LENGTH` in **docker/.env** to the desired number of bytes. The default 1 GiB value is `1073741824`. +The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file, and document-info uploads allow 100 MiB per file; increasing the shared multipart limit does not raise those per-file limits. -If you raise the limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service. Lowering `MAX_CONTENT_LENGTH` below 1 GiB does not require changing the default Nginx limit. - -These settings do not raise the dataset document upload path's fixed 128 MiB limit. +If you raise the multipart limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service. --- diff --git a/internal/common/environments.go b/internal/common/environments.go index 068e32b7b6..220b0d5ac9 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -88,6 +88,8 @@ const ( EnvDocEngine = "DOC_ENGINE" EnvMaxFileNumPerUser = "MAX_FILE_NUM_PER_USER" EnvMaxContentLength = "MAX_CONTENT_LENGTH" + EnvMaxJSONContentLength = "MAX_JSON_CONTENT_LENGTH" + EnvMaxMultipartContentLength = "MAX_MULTIPART_CONTENT_LENGTH" EnvRAGFlowDictPath = "RAGFLOW_DICT_PATH" EnvDefaultSuperuserEmail = "DEFAULT_SUPERUSER_EMAIL" EnvDefaultSuperuserNickname = "DEFAULT_SUPERUSER_NICKNAME" diff --git a/internal/common/http.go b/internal/common/http.go index 30e55aa2db..0ae932a920 100644 --- a/internal/common/http.go +++ b/internal/common/http.go @@ -38,8 +38,44 @@ import ( const ( redactedLogValue = "[REDACTED]" maxLoggedVectorFloats = 3 + + // DefaultMaxJSONRequestBodyBytes is the default limit for non-multipart requests. + DefaultMaxJSONRequestBodyBytes int64 = 1 << 20 + // DefaultMaxMultipartRequestBodyBytes is the default total limit for multipart uploads. + DefaultMaxMultipartRequestBodyBytes int64 = 1 << 30 + // MaxWebhookRequestBodyBytes is the default and maximum request body size for agent webhooks. + MaxWebhookRequestBodyBytes int64 = 10 * 1_000_000 ) +// GetMaxJSONRequestBodyBytes returns the configured non-multipart request-body +// limit in bytes, defaulting to 1 MiB. +func GetMaxJSONRequestBodyBytes() int64 { + return getMaxRequestBodyBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes) +} + +// GetMaxMultipartRequestBodyBytes returns the configured multipart upload +// request-body limit in bytes, defaulting to 1 GiB. MAX_CONTENT_LENGTH remains +// a fallback for existing deployments when the new variable is unset. +func GetMaxMultipartRequestBodyBytes() int64 { + return getMaxRequestBodyBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes) +} + +func getMaxRequestBodyBytes(envName, fallbackEnvName string, defaultBytes int64) int64 { + raw := strings.TrimSpace(GetEnv(envName)) + if raw == "" && fallbackEnvName != "" { + raw = strings.TrimSpace(GetEnv(fallbackEnvName)) + } + if raw == "" { + return defaultBytes + } + + limit, err := strconv.ParseInt(raw, 10, 64) + if err != nil || limit <= 0 { + return defaultBytes + } + return limit +} + var ssrfHttpClient *http.Client var schemeSafeHttpClient *http.Client diff --git a/internal/handler/agent_webhook_security.go b/internal/handler/agent_webhook_security.go index 227fb370a7..7b2f27a737 100644 --- a/internal/handler/agent_webhook_security.go +++ b/internal/handler/agent_webhook_security.go @@ -60,13 +60,8 @@ const ( bytesPerKB int64 = 1000 bytesPerMB int64 = bytesPerKB * 1000 - // webhookBodyMaxBytes is the hard ceiling above which a configured - // max_body_size is rejected at config-validation time. Stated - // in decimal MB to match the kb/mb unit base above; 10 MB - // decimal = 10 000 000 bytes. The python reference - // (agent_api.py:1652) uses 10 * 1024 * 1024 = 10 485 760 bytes, - // so this is ~486 KB stricter than python. - webhookBodyMaxBytes int64 = 10 * bytesPerMB + // webhookBodyMaxBytes is the default and hard maximum request body size. + webhookBodyMaxBytes int64 = common.MaxWebhookRequestBodyBytes // webhookRateLimitTimeout is the redis call timeout used for the // strict token-bucket lookup. Short on purpose — a security gate @@ -149,12 +144,11 @@ func validateMaxBodySize(c *gin.Context, cfg map[string]any) error { return nil } -// parseMaxBodySize returns the byte limit configured by -// `max_body_size` (with the 10 MB cap enforced) or 0 when no limit -// is configured. The handler uses this to wrap c.Request.Body in -// http.MaxBytesReader so the actual stream read is bounded — the -// Content-Length header check alone is insufficient because a client -// can advertise a small Content-Length and stream more. +// parseMaxBodySize returns the configured body limit, defaulting to 10 MB +// when `max_body_size` is absent. Configured values cannot exceed that cap. +// The handler wraps c.Request.Body in http.MaxBytesReader so the actual stream +// read is bounded — the Content-Length header check alone is insufficient +// because a client can advertise a small Content-Length and stream more. // // Unit base: 1 kb = 1 000 B, 1 mb = 1 000 000 B (SI decimal). Per // user request; note this DIVERGES from the python reference @@ -166,11 +160,18 @@ func validateMaxBodySize(c *gin.Context, cfg map[string]any) error { // the 10 MB cap. We check the parsed number against the cap before // multiplying. func parseMaxBodySize(cfg map[string]any) (int64, error) { - raw, ok := cfg["max_body_size"].(string) - if !ok || raw == "" { - return 0, nil + rawValue, exists := cfg["max_body_size"] + if !exists || rawValue == nil { + return webhookBodyMaxBytes, nil + } + raw, ok := rawValue.(string) + if !ok { + return 0, fmt.Errorf("invalid max_body_size format") } sizeStr := strings.ToLower(strings.TrimSpace(raw)) + if sizeStr == "" { + return webhookBodyMaxBytes, nil + } var limit int64 switch { case strings.HasSuffix(sizeStr, "kb"): diff --git a/internal/handler/agent_webhook_security_test.go b/internal/handler/agent_webhook_security_test.go index 66900a6a2b..b43d06ccbc 100644 --- a/internal/handler/agent_webhook_security_test.go +++ b/internal/handler/agent_webhook_security_test.go @@ -357,6 +357,22 @@ func TestValidateMaxBodySize_OverflowGuard(t *testing.T) { } } +func TestParseMaxBodySizeDefaultsTo10MB(t *testing.T) { + for _, cfg := range []map[string]any{ + nil, + {}, + {"max_body_size": ""}, + } { + got, err := parseMaxBodySize(cfg) + if err != nil { + t.Fatalf("parseMaxBodySize(%v): %v", cfg, err) + } + if got != common.MaxWebhookRequestBodyBytes { + t.Errorf("parseMaxBodySize(%v) = %d, want %d", cfg, got, common.MaxWebhookRequestBodyBytes) + } + } +} + // TestParseMaxBodySize_DecimalUnits documents the per-user-request // SI-decimal unit base: 1 kb = 1 000 B, 1 mb = 1 000 000 B. // (The python reference uses 1 024 / 1 048 576.) diff --git a/internal/router/request_body_limit.go b/internal/router/request_body_limit.go new file mode 100644 index 0000000000..c9b41452be --- /dev/null +++ b/internal/router/request_body_limit.go @@ -0,0 +1,64 @@ +// Package router owns the API request-size policy. +package router + +import ( + "mime" + "net/http" + + "github.com/gin-gonic/gin" + + "ragflow/internal/common" +) + +const agentWebhookRoute = "/api/v1/agents/:canvas_id/webhook" + +// requestBodyLimitMiddleware applies configurable limits to API request bodies. +// Multipart upload routes use a shared limit; handlers may enforce lower +// endpoint-specific limits. Public Agent webhooks retain their 10 MB ceiling. +func requestBodyLimitMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + if c.Request.Body == nil || c.Request.Body == http.NoBody { + c.Next() + return + } + + limit := common.GetMaxJSONRequestBodyBytes() + fullPath := c.FullPath() + switch { + case fullPath == agentWebhookRoute: + limit = common.MaxWebhookRequestBodyBytes + case isMultipartUploadRequest(c): + limit = common.GetMaxMultipartRequestBodyBytes() + } + + if c.Request.ContentLength > limit { + common.ResponseWithHttpCodeData(c, http.StatusRequestEntityTooLarge, common.CodeBadRequest, nil, "request body too large") + c.Abort() + return + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, limit) + c.Next() + } +} + +func isMultipartUploadRequest(c *gin.Context) bool { + if c.Request.Method != http.MethodPost || !isMultipartFormData(c.Request.Header.Get("Content-Type")) { + return false + } + + switch c.FullPath() { + case "/api/v1/files", + "/api/v1/agents/:canvas_id/upload", + "/api/v1/chat/audio/transcription", + "/api/v1/documents/upload", + "/api/v1/datasets/:dataset_id/documents": + return true + default: + return false + } +} + +func isMultipartFormData(contentType string) bool { + mediaType, _, err := mime.ParseMediaType(contentType) + return err == nil && mediaType == "multipart/form-data" +} diff --git a/internal/router/request_body_limit_test.go b/internal/router/request_body_limit_test.go new file mode 100644 index 0000000000..e85103cd0d --- /dev/null +++ b/internal/router/request_body_limit_test.go @@ -0,0 +1,142 @@ +package router + +import ( + "bytes" + "errors" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + "ragflow/internal/common" +) + +func TestRequestBodyLimitMiddleware(t *testing.T) { + gin.SetMode(gin.TestMode) + t.Setenv("MAX_JSON_CONTENT_LENGTH", "") + t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "") + t.Setenv("MAX_CONTENT_LENGTH", "") + + engine := gin.New() + engine.Use(requestBodyLimitMiddleware()) + readBody := func(c *gin.Context) { + _, err := io.Copy(io.Discard, c.Request.Body) + if err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + c.Status(http.StatusRequestEntityTooLarge) + return + } + c.Status(http.StatusBadRequest) + return + } + c.Status(http.StatusNoContent) + } + engine.POST("/api/v1/test", readBody) + engine.POST("/api/v1/files", readBody) + engine.DELETE("/api/v1/files", readBody) + engine.POST("/api/v1/agents/:canvas_id/upload", readBody) + engine.POST("/api/v1/chat/audio/transcription", readBody) + engine.POST("/api/v1/documents/upload", readBody) + engine.POST("/api/v1/datasets/:dataset_id/documents", readBody) + engine.POST(agentWebhookRoute, readBody) + + t.Run("rejects oversized default request", func(t *testing.T) { + body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/test", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + engine.ServeHTTP(resp, req) + if resp.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge) + } + }) + + t.Run("bounds chunked request body", func(t *testing.T) { + body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/test", bytes.NewReader(body)) + req.ContentLength = -1 + req.TransferEncoding = []string{"chunked"} + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + engine.ServeHTTP(resp, req) + if resp.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge) + } + }) + + t.Run("allows webhook body up to 10 MB", func(t *testing.T) { + body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/agents/a1/webhook", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + engine.ServeHTTP(resp, req) + if resp.Code != http.StatusNoContent { + t.Fatalf("status=%d, want %d", resp.Code, http.StatusNoContent) + } + }) + + t.Run("rejects webhook body above 10 MB", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/api/v1/agents/a1/webhook", bytes.NewReader(nil)) + req.ContentLength = common.MaxWebhookRequestBodyBytes + 1 + resp := httptest.NewRecorder() + engine.ServeHTTP(resp, req) + if resp.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge) + } + }) + + t.Run("does not exempt non-upload methods on file routes", func(t *testing.T) { + body := bytes.Repeat([]byte("x"), int(common.DefaultMaxJSONRequestBodyBytes+1)) + req := httptest.NewRequest(http.MethodDelete, "/api/v1/files", bytes.NewReader(body)) + req.Header.Set("Content-Type", "multipart/form-data; boundary=test") + resp := httptest.NewRecorder() + engine.ServeHTTP(resp, req) + if resp.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d, want %d", resp.Code, http.StatusRequestEntityTooLarge) + } + }) + + t.Run("uses configured JSON request-body limit", func(t *testing.T) { + t.Setenv("MAX_JSON_CONTENT_LENGTH", "16") + body := bytes.Repeat([]byte("x"), 17) + request := httptest.NewRequest(http.MethodPost, "/api/v1/test", bytes.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + engine.ServeHTTP(response, request) + if response.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d, want %d", response.Code, http.StatusRequestEntityTooLarge) + } + }) + + t.Run("uses configured multipart limit for every upload route", func(t *testing.T) { + t.Setenv("MAX_JSON_CONTENT_LENGTH", "8") + t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "16") + paths := []string{ + "/api/v1/files", + "/api/v1/agents/a1/upload", + "/api/v1/chat/audio/transcription", + "/api/v1/documents/upload", + "/api/v1/datasets/d1/documents", + } + for _, path := range paths { + request := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(bytes.Repeat([]byte("x"), 12))) + request.Header.Set("Content-Type", "multipart/form-data; boundary=test") + response := httptest.NewRecorder() + engine.ServeHTTP(response, request) + if response.Code != http.StatusNoContent { + t.Errorf("POST %s status=%d, want %d for body above JSON limit", path, response.Code, http.StatusNoContent) + } + + request = httptest.NewRequest(http.MethodPost, path, bytes.NewReader(bytes.Repeat([]byte("x"), 17))) + request.Header.Set("Content-Type", "multipart/form-data; boundary=test") + response = httptest.NewRecorder() + engine.ServeHTTP(response, request) + if response.Code != http.StatusRequestEntityTooLarge { + t.Errorf("POST %s status=%d, want %d for body above multipart limit", path, response.Code, http.StatusRequestEntityTooLarge) + } + } + }) +} diff --git a/internal/router/router.go b/internal/router/router.go index de9a1f1af2..070389eed0 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -136,6 +136,7 @@ func NewRouter( // Setup setup routes func (r *Router) Setup(engine *gin.Engine) { + engine.Use(requestBodyLimitMiddleware()) SetupEERouter(engine) // Mark all responses from Go with a header for debugging. diff --git a/internal/service/file/file_test.go b/internal/service/file/file_test.go index e59ebdadcc..9666b6d1ad 100644 --- a/internal/service/file/file_test.go +++ b/internal/service/file/file_test.go @@ -42,20 +42,27 @@ func testFileService() *FileService { } func TestDeploymentUploadMaxBytes(t *testing.T) { + t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "") t.Setenv("MAX_CONTENT_LENGTH", "") if got := DeploymentUploadMaxBytes(); got != defaultDeploymentUploadMaxBytes { t.Fatalf("default limit = %d, want %d", got, defaultDeploymentUploadMaxBytes) } - t.Setenv("MAX_CONTENT_LENGTH", "12345") + t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "12345") if got := DeploymentUploadMaxBytes(); got != 12345 { t.Fatalf("configured limit = %d, want 12345", got) } - t.Setenv("MAX_CONTENT_LENGTH", "invalid") + t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "invalid") if got := DeploymentUploadMaxBytes(); got != defaultDeploymentUploadMaxBytes { t.Fatalf("invalid limit = %d, want default %d", got, defaultDeploymentUploadMaxBytes) } + + t.Setenv("MAX_MULTIPART_CONTENT_LENGTH", "") + t.Setenv("MAX_CONTENT_LENGTH", "54321") + if got := DeploymentUploadMaxBytes(); got != 54321 { + t.Fatalf("legacy configured limit = %d, want 54321", got) + } } func TestReadDeploymentUploadDataLimit(t *testing.T) { diff --git a/internal/service/file/file_upload.go b/internal/service/file/file_upload.go index 3f7a037717..f352418d55 100644 --- a/internal/service/file/file_upload.go +++ b/internal/service/file/file_upload.go @@ -13,23 +13,14 @@ import ( "ragflow/internal/permission" "ragflow/internal/storage" "ragflow/internal/utility" - "strconv" "strings" "time" ) -const defaultDeploymentUploadMaxBytes int64 = 1 << 30 +const defaultDeploymentUploadMaxBytes int64 = common.DefaultMaxMultipartRequestBodyBytes func DeploymentUploadMaxBytes() int64 { - raw := common.GetEnv(common.EnvMaxContentLength) - if raw == "" { - return defaultDeploymentUploadMaxBytes - } - n, err := strconv.ParseInt(raw, 10, 64) - if err != nil || n <= 0 { - return defaultDeploymentUploadMaxBytes - } - return n + return common.GetMaxMultipartRequestBodyBytes() } // UploadFile uploads files to a folder