From fcd414a50dcd35197c3fc2ea8af2b09d7965255f Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 29 Jul 2026 00:59:55 +0800 Subject: [PATCH 1/3] [fix](be) Clarify JSON stream load size limit units ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: JSON Stream Load body-size errors displayed both byte counts without units next to a configuration named streaming_load_json_max_mb, making the configured limit misleading. Keep the existing threshold and byte comparison unchanged, label both exact byte values, and show the snapshotted configured MB value. Add a BE unit test using the reported request size. ### Release note Clarify units in the JSON Stream Load body-size limit error message. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=StreamLoadTest.* -j48 (2 tests passed) - Behavior changed: Yes. The JSON Stream Load size-limit error now identifies byte values and the configured MB value. - Does this need documentation: No --- be/src/service/http/action/stream_load.cpp | 10 ++++---- be/test/service/http/stream_load_test.cpp | 28 +++++++++++++++++++++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/be/src/service/http/action/stream_load.cpp b/be/src/service/http/action/stream_load.cpp index 92cdb5844fd3a0..94ddeabff0de59 100644 --- a/be/src/service/http/action/stream_load.cpp +++ b/be/src/service/http/action/stream_load.cpp @@ -312,7 +312,8 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptrbody_bytes = 0; size_t csv_max_body_bytes = config::streaming_load_max_mb * 1024 * 1024; - size_t json_max_body_bytes = config::streaming_load_json_max_mb * 1024 * 1024; + const auto json_max_body_mb = config::streaming_load_json_max_mb; + size_t json_max_body_bytes = json_max_body_mb * 1024 * 1024; bool read_json_by_line = false; if (!http_req->header(HTTP_READ_JSON_BY_LINE).empty()) { if (iequal(http_req->header(HTTP_READ_JSON_BY_LINE), "true")) { @@ -330,9 +331,10 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptrformat == TFileFormatType::FORMAT_JSON) && (ctx->body_bytes > json_max_body_bytes) && !read_json_by_line) { return Status::Error( - "json body size {} exceed BE's conf `streaming_load_json_max_mb` {}. increase " - "it if you are sure this load is reasonable", - ctx->body_bytes, json_max_body_bytes); + "json body size {} bytes exceeds the limit of {} bytes set by BE config " + "`streaming_load_json_max_mb` (currently {} MB). Increase it if you are sure " + "this load is reasonable", + ctx->body_bytes, json_max_body_bytes, json_max_body_mb); } // csv max body size else if (ctx->body_bytes > csv_max_body_bytes) { diff --git a/be/test/service/http/stream_load_test.cpp b/be/test/service/http/stream_load_test.cpp index 1ed2a1307bbc8f..c7d1ef9160a157 100644 --- a/be/test/service/http/stream_load_test.cpp +++ b/be/test/service/http/stream_load_test.cpp @@ -28,6 +28,7 @@ #include "event2/http_struct.h" #include "evhttp.h" #include "load/group_commit/wal/wal_manager.h" +#include "load/stream_load/stream_load_context.h" #include "runtime/exec_env.h" #include "service/http/ev_http_server.h" #include "service/http/http_channel.h" @@ -37,6 +38,7 @@ #include "service/http/http_headers.h" #include "service/http/http_request.h" #include "service/http/utils.h" +#include "util/defer_op.h" namespace doris { @@ -123,4 +125,28 @@ TEST_F(StreamLoadTest, TestHeader) { evhttp_request_free(evhttp_req); } } -} // namespace doris \ No newline at end of file + +TEST_F(StreamLoadTest, JsonBodySizeLimitErrorIncludesUnits) { + const auto original_json_max_mb = config::streaming_load_json_max_mb; + Defer restore_json_max_mb { + [original_json_max_mb] { config::streaming_load_json_max_mb = original_json_max_mb; }}; + config::streaming_load_json_max_mb = 100; + + auto* evhttp_req = evhttp_request_new(nullptr, nullptr); + HttpRequest req(evhttp_req); + req.set_header(HttpHeaders::AUTHORIZATION, "Basic cm9vdDo="); + req.set_header(HTTP_FORMAT_KEY, "json"); + req.set_header(HttpHeaders::CONTENT_LENGTH, "113403999"); + + StreamLoadAction action(nullptr); + auto ctx = std::make_shared(nullptr); + auto status = action._on_header(&req, ctx); + + EXPECT_TRUE(status.is()); + EXPECT_EQ(status.to_string_no_stack(), + "[E-217]json body size 113403999 bytes exceeds the limit of 104857600 bytes set by " + "BE config `streaming_load_json_max_mb` (currently 100 MB). Increase it if you are " + "sure this load is reasonable"); + evhttp_request_free(evhttp_req); +} +} // namespace doris From 09fa8751ae439d0d4d2a2f5ffc73ca52ec6c1a0d Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 29 Jul 2026 15:38:10 +0800 Subject: [PATCH 2/3] [fix](be) Report stream load size limits in MiB ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Stream Load size-limit errors mixed byte counts with configuration names expressed in MB, which made both JSON and CSV failures misleading. Report the actual request size and configured limit consistently in MiB for JSON Stream Load, CSV Stream Load, and HTTP Stream without changing any thresholds or rejection behavior. Add BE unit coverage for all three paths. ### Release note Report Stream Load body-size limit errors consistently in MiB. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=StreamLoadTest.* -j48 (4 tests passed) - Behavior changed: Yes. JSON and CSV Stream Load size-limit errors now display both values in MiB. - Does this need documentation: No --- be/src/service/http/action/http_stream.cpp | 10 +++-- be/src/service/http/action/stream_load.cpp | 17 +++++--- be/test/service/http/stream_load_test.cpp | 51 ++++++++++++++++++++-- 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/be/src/service/http/action/http_stream.cpp b/be/src/service/http/action/http_stream.cpp index e8b8d974211524..41b46f429cd206 100644 --- a/be/src/service/http/action/http_stream.cpp +++ b/be/src/service/http/action/http_stream.cpp @@ -67,6 +67,8 @@ using namespace ErrorCode; namespace { +constexpr size_t MEBIBYTE = 1024 * 1024; + bool is_compressed_file_scan(const TPipelineFragmentParams& params) { if (!params.__isset.file_scan_params) { return false; @@ -231,7 +233,8 @@ Status HttpStreamAction::_on_header(HttpRequest* http_req, std::shared_ptrbody_bytes = 0; - size_t csv_max_body_bytes = config::streaming_load_max_mb * 1024 * 1024; + const auto csv_max_body_mb = config::streaming_load_max_mb; + size_t csv_max_body_bytes = csv_max_body_mb * MEBIBYTE; if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) { try { ctx->body_bytes = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH)); @@ -243,9 +246,10 @@ Status HttpStreamAction::_on_header(HttpRequest* http_req, std::shared_ptrbody_bytes > csv_max_body_bytes) { LOG(WARNING) << "body exceed max size." << ctx->brief(); return Status::Error( - "body size {} exceed BE's conf `streaming_load_max_mb` {}. increase it if you " + "body size {:.2f} MiB exceeds the limit of {} MiB set by BE config " + "`streaming_load_max_mb`. Increase it if you " "are sure this load is reasonable", - ctx->body_bytes, csv_max_body_bytes); + static_cast(ctx->body_bytes) / MEBIBYTE, csv_max_body_mb); } } diff --git a/be/src/service/http/action/stream_load.cpp b/be/src/service/http/action/stream_load.cpp index 94ddeabff0de59..0a82339b9d342a 100644 --- a/be/src/service/http/action/stream_load.cpp +++ b/be/src/service/http/action/stream_load.cpp @@ -83,6 +83,7 @@ bvar::LatencyRecorder g_stream_load_commit_and_publish_latency_ms("stream_load", "commit_and_publish_ms"); static constexpr size_t MIN_CHUNK_SIZE = 64 * 1024; +static constexpr size_t MEBIBYTE = 1024 * 1024; static const std::string CHUNK = "chunked"; static const std::string OFF_MODE = "off_mode"; static const std::string SYNC_MODE = "sync_mode"; @@ -311,9 +312,10 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptrbody_bytes = 0; - size_t csv_max_body_bytes = config::streaming_load_max_mb * 1024 * 1024; + const auto csv_max_body_mb = config::streaming_load_max_mb; + size_t csv_max_body_bytes = csv_max_body_mb * MEBIBYTE; const auto json_max_body_mb = config::streaming_load_json_max_mb; - size_t json_max_body_bytes = json_max_body_mb * 1024 * 1024; + size_t json_max_body_bytes = json_max_body_mb * MEBIBYTE; bool read_json_by_line = false; if (!http_req->header(HTTP_READ_JSON_BY_LINE).empty()) { if (iequal(http_req->header(HTTP_READ_JSON_BY_LINE), "true")) { @@ -331,18 +333,19 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptrformat == TFileFormatType::FORMAT_JSON) && (ctx->body_bytes > json_max_body_bytes) && !read_json_by_line) { return Status::Error( - "json body size {} bytes exceeds the limit of {} bytes set by BE config " - "`streaming_load_json_max_mb` (currently {} MB). Increase it if you are sure " + "json body size {:.2f} MiB exceeds the limit of {} MiB set by BE config " + "`streaming_load_json_max_mb`. Increase it if you are sure " "this load is reasonable", - ctx->body_bytes, json_max_body_bytes, json_max_body_mb); + static_cast(ctx->body_bytes) / MEBIBYTE, json_max_body_mb); } // csv max body size else if (ctx->body_bytes > csv_max_body_bytes) { LOG(WARNING) << "body exceed max size." << ctx->brief(); return Status::Error( - "body size {} exceed BE's conf `streaming_load_max_mb` {}. increase it if you " + "body size {:.2f} MiB exceeds the limit of {} MiB set by BE config " + "`streaming_load_max_mb`. Increase it if you " "are sure this load is reasonable", - ctx->body_bytes, csv_max_body_bytes); + static_cast(ctx->body_bytes) / MEBIBYTE, csv_max_body_mb); } } else { #ifndef BE_TEST diff --git a/be/test/service/http/stream_load_test.cpp b/be/test/service/http/stream_load_test.cpp index c7d1ef9160a157..4318894ad6e2f0 100644 --- a/be/test/service/http/stream_load_test.cpp +++ b/be/test/service/http/stream_load_test.cpp @@ -30,6 +30,7 @@ #include "load/group_commit/wal/wal_manager.h" #include "load/stream_load/stream_load_context.h" #include "runtime/exec_env.h" +#include "service/http/action/http_stream.h" #include "service/http/ev_http_server.h" #include "service/http/http_channel.h" #include "service/http/http_common.h" @@ -126,7 +127,7 @@ TEST_F(StreamLoadTest, TestHeader) { } } -TEST_F(StreamLoadTest, JsonBodySizeLimitErrorIncludesUnits) { +TEST_F(StreamLoadTest, JsonBodySizeLimitErrorUsesMiB) { const auto original_json_max_mb = config::streaming_load_json_max_mb; Defer restore_json_max_mb { [original_json_max_mb] { config::streaming_load_json_max_mb = original_json_max_mb; }}; @@ -144,9 +145,51 @@ TEST_F(StreamLoadTest, JsonBodySizeLimitErrorIncludesUnits) { EXPECT_TRUE(status.is()); EXPECT_EQ(status.to_string_no_stack(), - "[E-217]json body size 113403999 bytes exceeds the limit of 104857600 bytes set by " - "BE config `streaming_load_json_max_mb` (currently 100 MB). Increase it if you are " - "sure this load is reasonable"); + "[E-217]json body size 108.15 MiB exceeds the limit of 100 MiB set by BE config " + "`streaming_load_json_max_mb`. Increase it if you are sure this load is reasonable"); + evhttp_request_free(evhttp_req); +} + +TEST_F(StreamLoadTest, CsvBodySizeLimitErrorUsesMiB) { + const auto original_max_mb = config::streaming_load_max_mb; + Defer restore_max_mb {[original_max_mb] { config::streaming_load_max_mb = original_max_mb; }}; + config::streaming_load_max_mb = 100; + + auto* evhttp_req = evhttp_request_new(nullptr, nullptr); + HttpRequest req(evhttp_req); + req.set_header(HttpHeaders::AUTHORIZATION, "Basic cm9vdDo="); + req.set_header(HTTP_FORMAT_KEY, "csv"); + req.set_header(HttpHeaders::CONTENT_LENGTH, "113403999"); + + StreamLoadAction action(nullptr); + auto ctx = std::make_shared(nullptr); + auto status = action._on_header(&req, ctx); + + EXPECT_TRUE(status.is()); + EXPECT_EQ(status.to_string_no_stack(), + "[E-217]body size 108.15 MiB exceeds the limit of 100 MiB set by BE config " + "`streaming_load_max_mb`. Increase it if you are sure this load is reasonable"); + evhttp_request_free(evhttp_req); +} + +TEST_F(StreamLoadTest, HttpStreamBodySizeLimitErrorUsesMiB) { + const auto original_max_mb = config::streaming_load_max_mb; + Defer restore_max_mb {[original_max_mb] { config::streaming_load_max_mb = original_max_mb; }}; + config::streaming_load_max_mb = 100; + + auto* evhttp_req = evhttp_request_new(nullptr, nullptr); + HttpRequest req(evhttp_req); + req.set_header(HttpHeaders::AUTHORIZATION, "Basic cm9vdDo="); + req.set_header(HttpHeaders::CONTENT_LENGTH, "113403999"); + + HttpStreamAction action(nullptr); + auto ctx = std::make_shared(nullptr); + auto status = action._on_header(&req, ctx); + + EXPECT_TRUE(status.is()); + EXPECT_EQ(status.to_string_no_stack(), + "[E-217]body size 108.15 MiB exceeds the limit of 100 MiB set by BE config " + "`streaming_load_max_mb`. Increase it if you are sure this load is reasonable"); evhttp_request_free(evhttp_req); } } // namespace doris From 4cf9d16f41d865ba8b68512b648279d5dc997dee Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 30 Jul 2026 10:56:45 +0800 Subject: [PATCH 3/3] [fix](be) Refine stream load size limit messages ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: The JSON and CSV Stream Load size-limit errors did not use the requested wording for their BE configuration references. Update both messages to name BE's conf directly while preserving the existing limits and MiB conversion, and adjust the exact unit-test expectations. ### Release note Improve JSON and CSV Stream Load size-limit error wording. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter='StreamLoadTest.*' -j48 - Behavior changed: Yes (error message wording only) - Does this need documentation: No --- be/src/service/http/action/stream_load.cpp | 8 ++++---- be/test/service/http/stream_load_test.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/be/src/service/http/action/stream_load.cpp b/be/src/service/http/action/stream_load.cpp index 0a82339b9d342a..7e426cf8287717 100644 --- a/be/src/service/http/action/stream_load.cpp +++ b/be/src/service/http/action/stream_load.cpp @@ -333,8 +333,8 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptrformat == TFileFormatType::FORMAT_JSON) && (ctx->body_bytes > json_max_body_bytes) && !read_json_by_line) { return Status::Error( - "json body size {:.2f} MiB exceeds the limit of {} MiB set by BE config " - "`streaming_load_json_max_mb`. Increase it if you are sure " + "json body size {:.2f} MiB exceeds the limit of {} MiB set by BE's conf " + "streaming_load_json_max_mb. Increase it if you are sure " "this load is reasonable", static_cast(ctx->body_bytes) / MEBIBYTE, json_max_body_mb); } @@ -342,8 +342,8 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptrbody_bytes > csv_max_body_bytes) { LOG(WARNING) << "body exceed max size." << ctx->brief(); return Status::Error( - "body size {:.2f} MiB exceeds the limit of {} MiB set by BE config " - "`streaming_load_max_mb`. Increase it if you " + "body size {:.2f} MiB exceeds the limit of {} MiB set by BE's conf " + "streaming_load_max_mb. Increase it if you " "are sure this load is reasonable", static_cast(ctx->body_bytes) / MEBIBYTE, csv_max_body_mb); } diff --git a/be/test/service/http/stream_load_test.cpp b/be/test/service/http/stream_load_test.cpp index 4318894ad6e2f0..2fb85f3893bccf 100644 --- a/be/test/service/http/stream_load_test.cpp +++ b/be/test/service/http/stream_load_test.cpp @@ -145,8 +145,8 @@ TEST_F(StreamLoadTest, JsonBodySizeLimitErrorUsesMiB) { EXPECT_TRUE(status.is()); EXPECT_EQ(status.to_string_no_stack(), - "[E-217]json body size 108.15 MiB exceeds the limit of 100 MiB set by BE config " - "`streaming_load_json_max_mb`. Increase it if you are sure this load is reasonable"); + "[E-217]json body size 108.15 MiB exceeds the limit of 100 MiB set by BE's conf " + "streaming_load_json_max_mb. Increase it if you are sure this load is reasonable"); evhttp_request_free(evhttp_req); } @@ -167,8 +167,8 @@ TEST_F(StreamLoadTest, CsvBodySizeLimitErrorUsesMiB) { EXPECT_TRUE(status.is()); EXPECT_EQ(status.to_string_no_stack(), - "[E-217]body size 108.15 MiB exceeds the limit of 100 MiB set by BE config " - "`streaming_load_max_mb`. Increase it if you are sure this load is reasonable"); + "[E-217]body size 108.15 MiB exceeds the limit of 100 MiB set by BE's conf " + "streaming_load_max_mb. Increase it if you are sure this load is reasonable"); evhttp_request_free(evhttp_req); }