🐛 Limit forward proxy response body size #18991

Author: 88250

docs/API.ja.md

@@ -1624,6 +1624,8 @@
         * `hex`
 
       `text`は既存の動作を維持し、該当する場合は文字コードをUTF-8に変換します。バイナリエンコーディングは文字コード変換前のレスポンス本文データに適用され、gzip展開など既存のHTTPコンテンツデコード動作は変わりません。
+
+      HTTPコンテンツデコード後のレスポンス本文は32 MiBに制限されます。上限を超えた場合、部分的な本文を返さずにエラーコード`10`を返します。大きなファイルやストリーミングレスポンスには`/api/network/proxy`を使用してください。
 * 戻り値
 
   ```json

docs/API.md

@@ -1630,6 +1630,8 @@ Note: To ensure data security, access to this interface is prohibited in Publish
         * `hex`
 
       `text` preserves the existing behavior and converts the character set to UTF-8 when applicable. The binary encodings encode the response body before character-set conversion; existing HTTP content decoding behavior, such as gzip decompression, is unchanged.
+
+      The response body is limited to 32 MiB after HTTP content decoding. If the limit is exceeded, the API returns error code `10` without a partial body. Use `/api/network/proxy` for large files or streaming responses.
 * Return value
 
   ```json

docs/API.zh-CN.md

@@ -1624,6 +1624,8 @@
         * `hex`
 
       `text` 保持现有行为，在适用时将字符集转换为 UTF-8。二进制编码作用于字符集转换前的响应正文数据；gzip 解压等现有 HTTP 内容解码行为不变。
+
+      HTTP 内容解码后的响应正文上限为 32 MiB，超限时返回错误码 `10` 且不返回部分正文。大文件或流式响应请使用 `/api/network/proxy`。
 * 返回值
 
   ```json

kernel/api/network.go

@@ -21,6 +21,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -37,6 +38,8 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
+const maxForwardProxyResponseSize int64 = 32 * 1024 * 1024
+
 type File struct {
 	Filename string
 	Header   textproto.MIMEHeader
@@ -186,7 +189,7 @@ func forwardProxy(c *gin.Context) {
 	}
 
 	client := getSafeClient(time.Duration(timeout) * time.Millisecond)
-	responseEncoding := configureForwardProxyResponseEncoding(client, arg["responseEncoding"])
+	responseEncoding := configureForwardProxyClient(client, maxForwardProxyResponseSize, arg["responseEncoding"])
 	if redirectArg, ok := arg["redirect"].(bool); ok && !redirectArg {
 		client.SetRedirectPolicy(req.NoRedirectPolicy())
 	}
@@ -263,17 +266,15 @@ func forwardProxy(c *gin.Context) {
 	}
 
 	started := time.Now()
-	resp, err := request.Send(method, destURL)
-	if err != nil {
-		ret.Code = 8
-		ret.Msg = "forward request failed: " + err.Error()
+	resp, bodyData, err := sendForwardProxyRequest(request, method, destURL)
+	if errors.Is(err, req.ErrResponseBodyTooLarge) {
+		ret.Code = 10
+		ret.Msg = fmt.Sprintf("response body too large: limit is %d bytes", maxForwardProxyResponseSize)
 		return
 	}
-
-	bodyData, err := io.ReadAll(resp.Body)
 	if err != nil {
-		ret.Code = 9
-		ret.Msg = "read response body failed: " + err.Error()
+		ret.Code = 8
+		ret.Msg = "forward request failed: " + err.Error()
 		return
 	}
 
@@ -324,7 +325,9 @@ func forwardProxy(c *gin.Context) {
 	//	elapsed.Seconds(), len(bodyData), data["url"], headers, contentType, arg["payload"], data["status"], shortBody)
 }
 
-func configureForwardProxyResponseEncoding(client *req.Client, value any) string {
+func configureForwardProxyClient(client *req.Client, maxResponseSize int64, value any) string {
+	client.SetMaxResponseSize(maxResponseSize)
+
 	responseEncoding, ok := value.(string)
 	if !ok {
 		return "text"
@@ -339,6 +342,15 @@ func configureForwardProxyResponseEncoding(client *req.Client, value any) string
 	}
 }
 
+func sendForwardProxyRequest(request *req.Request, method, destURL string) (response *req.Response, body []byte, err error) {
+	response, err = request.Send(method, destURL)
+	if err != nil {
+		return
+	}
+	body = response.Bytes()
+	return
+}
+
 // 创建安全的 HTTP Client，防止 SSRF 和 DNS 重绑定
 func getSafeClient(timeout time.Duration) *req.Client {
 	dialer := util.SSRFSafeDialer(timeout)

kernel/api/network_test.go

@@ -2,9 +2,12 @@ package api
 
 import (
 	"bytes"
+	"compress/gzip"
+	"errors"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 
 	"github.com/imroc/req/v3"
@@ -43,7 +46,7 @@ func TestConfigureForwardProxyResponseEncoding(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			client := req.C()
-			if encoding := configureForwardProxyResponseEncoding(client, test.value); encoding != test.wantEncoding {
+			if encoding := configureForwardProxyClient(client, maxForwardProxyResponseSize, test.value); encoding != test.wantEncoding {
 				t.Fatalf("encoding = %q, want %q", encoding, test.wantEncoding)
 			}
 
@@ -61,3 +64,72 @@ func TestConfigureForwardProxyResponseEncoding(t *testing.T) {
 		})
 	}
 }
+
+func TestForwardProxyResponseSizeLimit(t *testing.T) {
+	const limit int64 = 64
+	body := bytes.Repeat([]byte("a"), int(limit)+1)
+	var compressed bytes.Buffer
+	gzipWriter := gzip.NewWriter(&compressed)
+	if _, err := gzipWriter.Write(body); err != nil {
+		t.Fatalf("compress response body failed: %s", err)
+	}
+	if err := gzipWriter.Close(); err != nil {
+		t.Fatalf("close gzip writer failed: %s", err)
+	}
+	if int64(compressed.Len()) >= limit {
+		t.Fatalf("compressed response body size = %d, want less than %d", compressed.Len(), limit)
+	}
+
+	tests := []struct {
+		name             string
+		responseEncoding any
+		handler          http.HandlerFunc
+	}{
+		{
+			name:             "content length",
+			responseEncoding: "text",
+			handler: func(writer http.ResponseWriter, _ *http.Request) {
+				writer.Header().Set("Content-Length", strconv.Itoa(len(body)))
+				_, _ = writer.Write(body)
+			},
+		},
+		{
+			name:             "chunked",
+			responseEncoding: "base64",
+			handler: func(writer http.ResponseWriter, _ *http.Request) {
+				writer.WriteHeader(http.StatusOK)
+				writer.(http.Flusher).Flush()
+				_, _ = writer.Write(body)
+			},
+		},
+		{
+			name:             "compressed",
+			responseEncoding: "hex",
+			handler: func(writer http.ResponseWriter, _ *http.Request) {
+				writer.Header().Set("Content-Encoding", "gzip")
+				writer.Header().Set("Content-Length", strconv.Itoa(compressed.Len()))
+				_, _ = writer.Write(compressed.Bytes())
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(test.handler)
+			defer server.Close()
+
+			client := req.C()
+			configureForwardProxyClient(client, limit, test.responseEncoding)
+			response, responseBody, err := sendForwardProxyRequest(client.R(), http.MethodGet, server.URL)
+			if !errors.Is(err, req.ErrResponseBodyTooLarge) {
+				t.Fatalf("error = %v, want ErrResponseBodyTooLarge", err)
+			}
+			if response == nil {
+				t.Fatal("response is nil")
+			}
+			if responseBody != nil {
+				t.Fatalf("response body = %x, want nil", responseBody)
+			}
+		})
+	}
+}