support for consuming JSON batches

e2e/file_file_split/config.yml

@@ -0,0 +1,14 @@
+pipelines:
+  file_file_split:
+    settings:
+      decoder: json
+      split_json_array: true
+      split_json_array_field: data
+    input:
+      type: file
+      persistence_mode: async
+      watching_dir: SOME_DIR
+      offsets_file: SOME_FILE
+      offsets_op: reset
+    output:
+      type: file

e2e/file_file_split/file_file_split.go

@@ -0,0 +1,96 @@
+package file_file_split
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	uuid "github.com/satori/go.uuid"
+
+	"github.com/ozontech/file.d/cfg"
+	"github.com/ozontech/file.d/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Config for file-file-split plugin e2e test
+type Config struct {
+	FilesDir  string
+	Count     int
+	Lines     int
+	BatchSize int
+	RetTime   string
+}
+
+// Configure sets additional fields for input and output plugins
+func (c *Config) Configure(t *testing.T, conf *cfg.Config, pipelineName string) {
+	c.FilesDir = t.TempDir()
+	offsetsDir := t.TempDir()
+
+	input := conf.Pipelines[pipelineName].Raw.Get("input")
+	input.Set("watching_dir", c.FilesDir)
+	input.Set("filename_pattern", "split-input-*.log")
+	input.Set("offsets_file", filepath.Join(offsetsDir, "offsets.yaml"))
+
+	output := conf.Pipelines[pipelineName].Raw.Get("output")
+	output.Set("target_file", path.Join(c.FilesDir, "file-d.log"))
+	output.Set("retention_interval", c.RetTime)
+}
+
+func (c *Config) Send(t *testing.T) {
+	wg := &sync.WaitGroup{}
+	wg.Add(c.Count)
+	for i := 0; i < c.Count; i++ {
+		go func() {
+			defer wg.Done()
+			u := strings.ReplaceAll(uuid.NewV4().String(), "-", "")
+			name := path.Join(c.FilesDir, fmt.Sprintf("split-input-%s.log", u))
+			file, err := os.Create(name)
+			if err != nil {
+				log.Fatalf("failed to create file: %s", err.Error())
+			}
+
+			var sb strings.Builder
+			for j := 0; j < c.Lines; j++ {
+				sb.WriteString(buildBatch(j, c.BatchSize))
+				sb.WriteByte('\n')
+			}
+
+			if _, err = file.WriteString(sb.String()); err != nil {
+				log.Fatalf("failed to write to file: %s", err.Error())
+			}
+			if err = file.Close(); err != nil {
+				log.Fatalf("failed to close file: %s", err.Error())
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func (c *Config) Validate(t *testing.T) {
+	logFilePattern := path.Join(c.FilesDir, "file-d*.log")
+	expected := c.Count * c.Lines * c.BatchSize
+	test.WaitProcessEvents(t, expected, 3*time.Second, 30*time.Second, logFilePattern)
+	matches := test.GetMatches(t, logFilePattern)
+	assert.True(t, len(matches) > 0, "no files with processed events")
+	require.Equal(t, expected, test.CountLines(t, logFilePattern), "wrong number of processed events after split")
+}
+
+func buildBatch(lineIdx, size int) string {
+	var sb strings.Builder
+	sb.WriteString(`{"data":[`)
+	for i := range size {
+		if i > 0 {
+			sb.WriteByte(',')
+		}
+		fmt.Fprintf(&sb, `{"m":"line-%d-%d"}`, lineIdx, i)
+	}
+	sb.WriteString(`]}`)
+	return sb.String()
+}

e2e/start_work_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ozontech/file.d/e2e/file_elasticsearch"
 	"github.com/ozontech/file.d/e2e/file_es_split"
 	"github.com/ozontech/file.d/e2e/file_file"
+	"github.com/ozontech/file.d/e2e/file_file_split"
 	"github.com/ozontech/file.d/e2e/file_loki"
 	"github.com/ozontech/file.d/e2e/file_socket"
 	"github.com/ozontech/file.d/e2e/http_file"
@@ -122,6 +123,16 @@ func TestE2EStabilityWorkCase(t *testing.T) {
 			},
 			cfgPath: "./file_file/config.yml",
 		},
+		{
+			name: "file_file_split",
+			e2eTest: &file_file_split.Config{
+				Count:     5,
+				Lines:     50,
+				BatchSize: 10,
+				RetTime:   "1s",
+			},
+			cfgPath: "./file_file_split/config.yml",
+		},
 		{
 			name: "http_file",
 			e2eTest: &http_file.Config{

fd/util.go

@@ -35,6 +35,8 @@ func extractPipelineParams(settings *simplejson.Json) *pipeline.Settings {
 	var antispamExceptions antispam.Exceptions
 	var antispamRules antispam.Rules
 	var antispamSampler *antispam.Sampler
+	var splitJSONArrayField []string
+	splitJSONArray := pipeline.DefaultSplitJSONArray
 
 	metricHoldDuration := pipeline.DefaultMetricHoldDuration
 	metricMaxLabelValueLength := pipeline.DefaultMetricMaxLabelValueLength
@@ -152,6 +154,11 @@ func extractPipelineParams(settings *simplejson.Json) *pipeline.Settings {
 			metricHoldDuration = i
 		}
 
+		splitJSONArray = settings.Get("split_json_array").MustBool()
+		if str := settings.Get("split_json_array_field").MustString(); str != "" {
+			splitJSONArrayField = cfg.ParseFieldSelector(str)
+		}
+
 		metricMaxLabelValueLength = metrics.Get("max_label_value_length").MustInt()
 		if metricMaxLabelValueLength < 0 {
 			logger.Warn("negative max_label_value_length value, metric label truncation is disabled")
@@ -180,6 +187,8 @@ func extractPipelineParams(settings *simplejson.Json) *pipeline.Settings {
 		EventTimeout:        eventTimeout,
 		StreamField:         streamField,
 		IsStrict:            isStrict,
+		SplitJSONArray:      splitJSONArray,
+		SplitJSONArrayField: splitJSONArrayField,
 		Pool:                pipeline.PoolType(pool),
 		Metric: &pipeline.MetricSettings{
 			HoldDuration:        metricHoldDuration,

pipeline/README.idoc.md

@@ -108,6 +108,18 @@ Whether to fatal on decoding error.
 
 <br>
 
+**`split_json_array`** *`bool`* *`default=false`*
+
+Splitting of incoming JSON arrays into separate events before event pool allocation. Each array element becomes its own event. Applies only when the pipeline decoder is `json`.
+
+<br>
+
+**`split_json_array_field`** *`string`*
+
+Path to the JSON array of objects. Alternative to the `split` action plugin, applied before event pool allocation. Only used together with `split_json_array: true`.
+
+<br>
+
 **`metric_hold_duration`** *`string`* *`default=30m`* 
 
 The amount of time the metric can be idle until it is deleted. Used for deleting rarely updated metrics to save metrics storage resources. The value must be passed in format of duration (`<number>(ms|s|m|h)`).

pipeline/README.md

@@ -108,6 +108,18 @@ Whether to fatal on decoding error.
 
 <br>
 
+**`split_json_array`** *`bool`* *`default=false`*
+
+Splitting of incoming JSON arrays into separate events before event pool allocation. Each array element becomes its own event. Applies only when the pipeline decoder is `json`.
+
+<br>
+
+**`split_json_array_field`** *`string`*
+
+Path to the JSON array of objects. Alternative to the `split` action plugin, applied before event pool allocation. Only used together with `split_json_array: true`.
+
+<br>
+
 **`metric_hold_duration`** *`string`* *`default=30m`* 
 
 The amount of time the metric can be idle until it is deleted. Used for deleting rarely updated metrics to save metrics storage resources. The value must be passed in format of duration (`<number>(ms|s|m|h)`).

pipeline/event.go

@@ -16,6 +16,10 @@ import (
 type Event struct {
 	kind Kind
 
+	// set to true for non-last chunks of a split batch, so pipeline.finalize()
+	// skips input.Commit() and Event.Offset is committed only once
+	skipInputCommit bool
+
 	children []*Event
 
 	Root *insaneJSON.Root
@@ -116,6 +120,7 @@ func (e *Event) reset() {
 	e.stream = nil
 	e.children = e.children[:0]
 	e.kind = EventKindRegular
+	e.skipInputCommit = false
 }
 
 func (e *Event) StreamNameBytes() []byte {