diff --git a/benchmarks/benchmarks_test.go b/benchmarks/benchmarks_test.go index 4dcb2819..75c5a0ab 100644 --- a/benchmarks/benchmarks_test.go +++ b/benchmarks/benchmarks_test.go @@ -24,14 +24,19 @@ import ( "github.com/splunk/stef/go/pkg" ) -var speedEncodings = []encodings.MetricEncoding{ - &otlp.OTLPEncoding{}, +var metricsSpeedEncodings = []encodings.MetricEncoding{ + &otlp.MetricsEncoding{}, &stef.STEFEncoding{Opts: pkg.WriterOptions{Compression: pkg.CompressionNone}}, &stef.STEFUEncoding{Opts: pkg.WriterOptions{Compression: pkg.CompressionNone}}, &parquetenc.Encoding{}, &otelarrow.OtelArrowEncoding{}, } +var logsSpeedEncodings = []encodings.LogEncoding{ + &otlp.LogsEncoding{}, + &stef.LogsSTEFEncoding{Opts: pkg.WriterOptions{Compression: pkg.CompressionNone}}, +} + var benchmarkDataVariations = []struct { generator generators.Generator }{ @@ -49,6 +54,11 @@ var benchmarkDataVariations = []struct { var chart = BarOutput{} +type encodingNamer interface { + Name() string + LongName() string +} + func TestMain(m *testing.M) { // call flag.Parse() here if TestMain uses flags chart.Begin() @@ -58,9 +68,9 @@ func TestMain(m *testing.M) { func addZstdCompressTime( b *testing.B, - encoding encodings.MetricEncoding, + encoding encodingNamer, bodyBytes []byte, - dataPointCount int, + unitCount int, ) { if !chartsEnabled() { return @@ -76,11 +86,12 @@ func addZstdCompressTime( log.Fatal("compression failed") } } + value := float64(b.Elapsed().Nanoseconds()) / float64(b.N*unitCount) chart.RecordStacked( b, encoding.LongName(), "Zstd Compress", - float64(b.Elapsed().Nanoseconds())/float64(b.N*dataPointCount), + value, ) }, ) @@ -88,9 +99,9 @@ func addZstdCompressTime( func addZstdDecompressTime( b *testing.B, - encoding encodings.MetricEncoding, + encoding encodingNamer, bodyBytes []byte, - dataPointCount int, + unitCount int, ) { if !chartsEnabled() { return @@ -110,28 +121,28 @@ func addZstdDecompressTime( log.Fatal(err) } } + value := float64(b.Elapsed().Nanoseconds()) / float64(b.N*unitCount) chart.RecordStacked( b, encoding.LongName(), "Zstd Decompress", - float64(b.Elapsed().Nanoseconds())/float64(b.N*dataPointCount), + value, ) }, ) } -func BenchmarkSerializeNative(b *testing.B) { +func BenchmarkMetricsSerializeNative(b *testing.B) { chart.BeginSection("Speed Benchmarks") - chart.BeginChart("Serialization Speed", b) + chart.BeginChart("Metrics Serialize", "ns/point", b) defer chart.EndChart( - "ns/point", charts.WithColorsOpts(opts.Colors{"#92C5F9", "#12C5F9"}), ) for _, dataVariation := range benchmarkDataVariations { - for _, encoding := range speedEncodings { + for _, encoding := range metricsSpeedEncodings { if _, ok := encoding.(*otelarrow.OtelArrowEncoding); ok { // Skip Arrow, it does not have native serialization continue @@ -167,15 +178,55 @@ func BenchmarkSerializeNative(b *testing.B) { b.ReportAllocs() } -func BenchmarkDeserializeNative(b *testing.B) { - chart.BeginChart("Deserialization Speed", b) +func BenchmarkLogsSerializeNative(b *testing.B) { + chart.BeginChart("Logs Serialize", "ns/log", b) + defer chart.EndChart( + charts.WithColorsOpts(opts.Colors{"#92C5F9", "#12C5F9"}), + ) + + logsData, err := testutils.ReadOTLPLogsFile("testdata/astronomy-otellogs.zst") + require.NoError(b, err) + logCount := logsData.LogRecordCount() + require.Positive(b, logCount) + + for _, encoding := range logsSpeedEncodings { + inmem, err := encoding.FromOTLP(logsData) + require.NoError(b, err) + b.Run( + fmt.Sprintf("%s/serialize", encoding.Name()), + func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + bodyBytes, err := encoding.Encode(inmem) + if err != nil || bodyBytes == nil { + b.Fatal(err) + } + } + chart.RecordStacked( + b, + encoding.LongName(), + "Serialize", + float64(b.Elapsed().Nanoseconds())/float64(b.N*logCount), + ) + }, + ) + + bodyBytes, err := encoding.Encode(inmem) + require.NoError(b, err) + require.NotNil(b, bodyBytes) + addZstdCompressTime(b, encoding, bodyBytes, logCount) + } + b.ReportAllocs() +} + +func BenchmarkMetricsDeserializeNative(b *testing.B) { + chart.BeginChart("Metrics Deserialize", "ns/point", b) defer chart.EndChart( - "ns/point", charts.WithColorsOpts(opts.Colors{"#92C5F9", "#12C5F9"}), ) for _, dataVariation := range benchmarkDataVariations { - for _, encoding := range speedEncodings { + for _, encoding := range metricsSpeedEncodings { if _, ok := encoding.(*otelarrow.OtelArrowEncoding); ok { // Skip Arrow, it does not have native serialization continue @@ -224,15 +275,14 @@ func nsPerPoint(b *testing.B, dataPointCount int) float64 { return math.Round(float64(b.Elapsed().Nanoseconds()) / float64(totalDpCount)) } -func BenchmarkSerializeFromPdata(b *testing.B) { - chart.BeginChart("Serialization From pdata Speed", b) +func BenchmarkMetricsSerializeFromPdata(b *testing.B) { + chart.BeginChart("Metrics Serialize From pdata", "ns/point", b) defer chart.EndChart( - "ns/point", charts.WithColorsOpts(opts.Colors{"#92C5F9", "#12C5F9"}), ) for _, dataVariation := range benchmarkDataVariations { - for _, encoding := range speedEncodings { + for _, encoding := range metricsSpeedEncodings { if dataVariation.generator.GetName() == "hostandcollector-otelmetrics.zst" && encoding.Name() == "ARROW" { // Skip due to bug in Arrow encoding @@ -269,15 +319,56 @@ func BenchmarkSerializeFromPdata(b *testing.B) { b.ReportAllocs() } -func BenchmarkDeserializeToPdata(b *testing.B) { - chart.BeginChart("Deserialization To pdata Speed", b) +func BenchmarkLogsSerializeFromPdata(b *testing.B) { + chart.BeginChart("Logs Serialize From pdata", "ns/log", b) + defer chart.EndChart( + charts.WithColorsOpts(opts.Colors{"#92C5F9", "#12C5F9"}), + ) + + logsData, err := testutils.ReadOTLPLogsFile("testdata/astronomy-otellogs.zst") + require.NoError(b, err) + logCount := logsData.LogRecordCount() + require.Positive(b, logCount) + + for _, encoding := range logsSpeedEncodings { + b.Run( + fmt.Sprintf("%s/serialize", encoding.Name()), + func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + inmem, err := encoding.FromOTLP(logsData) + require.NoError(b, err) + bodyBytes, err := encoding.Encode(inmem) + require.NoError(b, err) + require.NotNil(b, bodyBytes) + } + chart.Record( + b, + encoding.LongName(), + "CPU time to serialize one log", + nsPerPoint(b, logCount), + ) + }, + ) + + inmem, err := encoding.FromOTLP(logsData) + require.NoError(b, err) + bodyBytes, err := encoding.Encode(inmem) + require.NoError(b, err) + require.NotNil(b, bodyBytes) + addZstdCompressTime(b, encoding, bodyBytes, logCount) + } + b.ReportAllocs() +} + +func BenchmarkMetricsDeserializeToPdata(b *testing.B) { + chart.BeginChart("Metrics Deserialize To pdata", "ns/point", b) defer chart.EndChart( - "ns/point", charts.WithColorsOpts(opts.Colors{"#92C5F9", "#12C5F9"}), ) for _, dataVariation := range benchmarkDataVariations { - for _, encoding := range speedEncodings { + for _, encoding := range metricsSpeedEncodings { if dataVariation.generator.GetName() == "hostandcollector-otelmetrics.zst" && encoding.Name() == "ARROW" { // Skip due to bug in Arrow encoding @@ -316,7 +407,48 @@ func BenchmarkDeserializeToPdata(b *testing.B) { b.ReportAllocs() } -func BenchmarkReaderReadMany(b *testing.B) { +func BenchmarkLogsDeserializeToPdata(b *testing.B) { + chart.BeginChart("Logs Deserialize To pdata", "ns/log", b) + defer chart.EndChart( + charts.WithColorsOpts(opts.Colors{"#92C5F9", "#12C5F9"}), + ) + + logsData, err := testutils.ReadOTLPLogsFile("testdata/astronomy-otellogs.zst") + require.NoError(b, err) + logCount := logsData.LogRecordCount() + require.Positive(b, logCount) + + for _, encoding := range logsSpeedEncodings { + inmem, err := encoding.FromOTLP(logsData) + require.NoError(b, err) + bodyBytes, err := encoding.Encode(inmem) + require.NoError(b, err) + require.NotNil(b, bodyBytes) + + b.Run( + fmt.Sprintf("%s/deserialize", encoding.Name()), + func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := encoding.ToOTLP(bodyBytes) + if err != nil { + b.Fatal(err) + } + } + chart.Record( + b, + encoding.LongName(), + "CPU time to deserialize one log", + nsPerPoint(b, logCount), + ) + }, + ) + addZstdDecompressTime(b, encoding, bodyBytes, logCount) + } + b.ReportAllocs() +} + +func BenchmarkMetricsReaderReadMany(b *testing.B) { generator := &generators.File{ FilePath: "testdata/hipstershop-otelmetrics.zst", } @@ -364,7 +496,7 @@ func BenchmarkReaderReadMany(b *testing.B) { ) } -func BenchmarkSTEFReaderRead(b *testing.B) { +func BenchmarkMetricsSTEFReaderRead(b *testing.B) { generator := &generators.File{ FilePath: "testdata/hipstershop-otelmetrics.zst", } @@ -405,9 +537,9 @@ func BenchmarkSTEFReaderRead(b *testing.B) { var multipartFiles = []string{"astronomy-otelmetrics"} -func BenchmarkSTEFSerializeMultipart(b *testing.B) { +func BenchmarkMetricsSTEFSerializeMultipart(b *testing.B) { for _, file := range multipartFiles { - parts, err := testutils.ReadMultipartOTLPFile("testdata/" + file + ".zst") + parts, err := testutils.ReadMultipartMetricsOTLPFile("testdata/" + file + ".zst") require.NoError(b, err) b.Run( file, @@ -442,9 +574,9 @@ func BenchmarkSTEFSerializeMultipart(b *testing.B) { } } -func BenchmarkSTEFDeserializeMultipart(b *testing.B) { +func BenchmarkMetricsSTEFDeserializeMultipart(b *testing.B) { for _, file := range multipartFiles { - parts, err := testutils.ReadMultipartOTLPFile("testdata/" + file + ".zst") + parts, err := testutils.ReadMultipartMetricsOTLPFile("testdata/" + file + ".zst") require.NoError(b, err) b.Run( file, diff --git a/benchmarks/charts.go b/benchmarks/charts.go index e06dde43..aa2a547f 100644 --- a/benchmarks/charts.go +++ b/benchmarks/charts.go @@ -16,6 +16,7 @@ import ( type BarOutput struct { t testing.TB title string + unit string file *os.File results map[string]map[string]float64 enabled bool @@ -25,13 +26,14 @@ func chartsEnabled() bool { return os.Getenv("UPDATE_BENCH_HTML") != "" } -func (c *BarOutput) BeginChart(title string, t testing.TB) { +func (c *BarOutput) BeginChart(title, unit string, t testing.TB) { c.t = t c.title = title + c.unit = unit c.results = map[string]map[string]float64{} } -func (c *BarOutput) EndChart(unit string, globalopts ...charts.GlobalOpts) { +func (c *BarOutput) EndChart(globalopts ...charts.GlobalOpts) { if !c.enabled { return } @@ -64,7 +66,7 @@ func (c *BarOutput) EndChart(unit string, globalopts ...charts.GlobalOpts) { charts.WithAnimation(false), charts.WithYAxisOpts( opts.YAxis{ - Name: unit, + Name: c.unit, }, ), ) @@ -117,14 +119,14 @@ func roundFloat(val float64, decimals int) float64 { func (c *BarOutput) Record(b *testing.B, encoding string, series string, val float64) { if b != nil { - b.ReportMetric(val, "ns/point") + b.ReportMetric(val, c.unit) } c.results[encoding] = map[string]float64{series: val} } func (c *BarOutput) RecordStacked(b *testing.B, encoding string, series string, val float64) { if b != nil { - b.ReportMetric(val, "ns/point") + b.ReportMetric(val, c.unit) } if c.results[encoding] == nil { c.results[encoding] = map[string]float64{} diff --git a/benchmarks/cmd/otlp2stef/main.go b/benchmarks/cmd/otlp2stef/main.go index f74a2e0a..235da7e8 100644 --- a/benchmarks/cmd/otlp2stef/main.go +++ b/benchmarks/cmd/otlp2stef/main.go @@ -35,7 +35,7 @@ func main() { } func convertOTLPtoSTEF(inputFilePath, outputFilePathNoExt, compressionMethod string) { - all, err := testutils.ReadOTLPFile(inputFilePath) + all, err := testutils.ReadOTLPMetricsFile(inputFilePath) if err != nil { log.Fatal(err) } diff --git a/benchmarks/correctness_test.go b/benchmarks/correctness_test.go index 04b86ebe..5abbde6a 100644 --- a/benchmarks/correctness_test.go +++ b/benchmarks/correctness_test.go @@ -8,17 +8,20 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" "github.com/splunk/stef/benchmarks/encodings/stef" "github.com/splunk/stef/benchmarks/testutils" "github.com/splunk/stef/go/otel/otelstef" + "github.com/splunk/stef/go/pdata/logs" + logtesttools "github.com/splunk/stef/go/pdata/logs/testtools" "github.com/splunk/stef/go/pdata/metrics" "github.com/splunk/stef/go/pdata/metrics/testtools" "github.com/splunk/stef/go/pkg" ) -func TestConvertSTEFFromToOTLP(t *testing.T) { +func TestMetricsConvertSTEFFromToOTLP(t *testing.T) { tests := []struct { file string }{ @@ -33,7 +36,7 @@ func TestConvertSTEFFromToOTLP(t *testing.T) { for _, test := range tests { t.Run( test.file, func(t *testing.T) { - otlpDataSrc, err := testutils.ReadOTLPFile(test.file) + otlpDataSrc, err := testutils.ReadOTLPMetricsFile(test.file) require.NoError(t, err) testtools.NormalizeMetrics(otlpDataSrc) @@ -69,6 +72,33 @@ func TestConvertSTEFFromToOTLP(t *testing.T) { } } +func TestLogsConvertSTEFFromToOTLP(t *testing.T) { + otlpDataSrc, err := testutils.ReadOTLPLogsFile("testdata/astronomy-otellogs.zst") + require.NoError(t, err) + + logtesttools.NormalizeLogs(otlpDataSrc) + srcCount := otlpDataSrc.LogRecordCount() + + buf := &pkg.MemChunkWriter{} + writer, err := otelstef.NewLogsWriter(buf, pkg.WriterOptions{}) + require.NoError(t, err) + + toStef := logs.OtlpToStefUnsorted{Sorted: true} + require.NoError(t, toStef.Convert(otlpDataSrc, writer)) + require.NoError(t, writer.Flush()) + + reader, err := otelstef.NewLogsReader(bytes.NewBuffer(buf.Bytes())) + require.NoError(t, err) + + toOtlp := logs.StefToOtlpUnsorted{} + otlpDataCopy, err := toOtlp.Convert(reader, true) + require.NoError(t, err) + + logtesttools.NormalizeLogs(otlpDataCopy) + assert.EqualValues(t, srcCount, otlpDataCopy.LogRecordCount()) + assert.True(t, bytes.Equal(logsToBytes(t, otlpDataSrc), logsToBytes(t, otlpDataCopy))) +} + func toBytes(t *testing.T, data pmetric.Metrics) []byte { marshaler := pmetric.ProtoMarshaler{} bytes, err := marshaler.MarshalMetrics(data) @@ -76,6 +106,13 @@ func toBytes(t *testing.T, data pmetric.Metrics) []byte { return bytes } +func logsToBytes(t *testing.T, data plog.Logs) []byte { + marshaler := plog.ProtoMarshaler{} + dataBytes, err := marshaler.MarshalLogs(data) + require.NoError(t, err) + return dataBytes +} + func TestTEFMultiPart(t *testing.T) { var testInputOtlpFiles = []string{ "testdata/astronomy-otelmetrics.zst", @@ -88,7 +125,7 @@ func TestTEFMultiPart(t *testing.T) { t.Run( inputFile, func(t *testing.T) { - parts, err := testutils.ReadMultipartOTLPFile(inputFile) + parts, err := testutils.ReadMultipartMetricsOTLPFile(inputFile) require.NoError(t, err) tefStream, err := tefEncoding.StartMultipart("") diff --git a/benchmarks/encodings/log_encoding.go b/benchmarks/encodings/log_encoding.go new file mode 100644 index 00000000..5c22c397 --- /dev/null +++ b/benchmarks/encodings/log_encoding.go @@ -0,0 +1,12 @@ +package encodings + +import "go.opentelemetry.io/collector/pdata/plog" + +type LogEncoding interface { + Name() string + LongName() string + FromOTLP(batch plog.Logs) (InMemoryData, error) + Encode(data InMemoryData) ([]byte, error) + Decode([]byte) (any, error) + ToOTLP(data []byte) (plog.Logs, error) +} diff --git a/benchmarks/encodings/otlp/logs.go b/benchmarks/encodings/otlp/logs.go new file mode 100644 index 00000000..92605b13 --- /dev/null +++ b/benchmarks/encodings/otlp/logs.go @@ -0,0 +1,37 @@ +package otlp + +import ( + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/benchmarks/encodings" +) + +type LogsEncoding struct{} + +var _ encodings.LogEncoding = (*LogsEncoding)(nil) + +func (*LogsEncoding) FromOTLP(data plog.Logs) (encodings.InMemoryData, error) { + return data, nil +} + +func (*LogsEncoding) Encode(data encodings.InMemoryData) ([]byte, error) { + marshaler := plog.ProtoMarshaler{} + return marshaler.MarshalLogs(data.(plog.Logs)) +} + +func (d *LogsEncoding) Decode(data []byte) (any, error) { + return d.ToOTLP(data) +} + +func (*LogsEncoding) ToOTLP(data []byte) (plog.Logs, error) { + unmarshaler := plog.ProtoUnmarshaler{} + return unmarshaler.UnmarshalLogs(data) +} + +func (*LogsEncoding) Name() string { + return "OTLP" +} + +func (*LogsEncoding) LongName() string { + return "Protobuf OTLP" +} diff --git a/benchmarks/encodings/otlp/otlp.go b/benchmarks/encodings/otlp/metrics.go similarity index 50% rename from benchmarks/encodings/otlp/otlp.go rename to benchmarks/encodings/otlp/metrics.go index 4ab2200d..9288906a 100644 --- a/benchmarks/encodings/otlp/otlp.go +++ b/benchmarks/encodings/otlp/metrics.go @@ -7,40 +7,40 @@ import ( "github.com/splunk/stef/benchmarks/testutils" ) -type OTLPEncoding struct { +type MetricsEncoding struct { } -func (d *OTLPEncoding) FromOTLP(data pmetric.Metrics) (encodings.InMemoryData, error) { +func (d *MetricsEncoding) FromOTLP(data pmetric.Metrics) (encodings.InMemoryData, error) { return data, nil } -func (d *OTLPEncoding) Encode(data encodings.InMemoryData) ([]byte, error) { +func (d *MetricsEncoding) Encode(data encodings.InMemoryData) ([]byte, error) { marshaler := pmetric.ProtoMarshaler{} return marshaler.MarshalMetrics(data.(pmetric.Metrics)) } -func (d *OTLPEncoding) Decode(b []byte) (any, error) { +func (d *MetricsEncoding) Decode(b []byte) (any, error) { return d.ToOTLP(b) } -func (*OTLPEncoding) ToOTLP(data []byte) (pmetric.Metrics, error) { +func (*MetricsEncoding) ToOTLP(data []byte) (pmetric.Metrics, error) { marshaler := pmetric.ProtoUnmarshaler{} return marshaler.UnmarshalMetrics(data) } -func (*OTLPEncoding) Name() string { +func (*MetricsEncoding) Name() string { return "OTLP" } -func (*OTLPEncoding) LongName() string { +func (*MetricsEncoding) LongName() string { return "Protobuf OTLP" } -type otlpMultipart struct { +type metricsMultipart struct { compression string bytes []byte } -func (o *otlpMultipart) AppendPart(part pmetric.Metrics) error { +func (o *metricsMultipart) AppendPart(part pmetric.Metrics) error { marshaler := pmetric.ProtoMarshaler{} b, err := marshaler.MarshalMetrics(part) if err != nil { @@ -55,10 +55,10 @@ func (o *otlpMultipart) AppendPart(part pmetric.Metrics) error { return nil } -func (o *otlpMultipart) FinishStream() ([]byte, error) { +func (o *metricsMultipart) FinishStream() ([]byte, error) { return o.bytes, nil } -func (d *OTLPEncoding) StartMultipart(compression string) (encodings.MetricMultipartStream, error) { - return &otlpMultipart{compression: compression}, nil +func (d *MetricsEncoding) StartMultipart(compression string) (encodings.MetricMultipartStream, error) { + return &metricsMultipart{compression: compression}, nil } diff --git a/benchmarks/encodings/stef/logs.go b/benchmarks/encodings/stef/logs.go new file mode 100644 index 00000000..fd3701e5 --- /dev/null +++ b/benchmarks/encodings/stef/logs.go @@ -0,0 +1,77 @@ +package stef + +import ( + "bytes" + "io" + + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/benchmarks/encodings" + "github.com/splunk/stef/go/otel/otelstef" + logconvert "github.com/splunk/stef/go/pdata/logs" + "github.com/splunk/stef/go/pdata/logs/sortedbyresource" + "github.com/splunk/stef/go/pkg" +) + +// LogsSTEFEncoding is resource-sorted STEF format for logs. +type LogsSTEFEncoding struct { + Opts pkg.WriterOptions +} + +var _ encodings.LogEncoding = (*LogsSTEFEncoding)(nil) + +func (*LogsSTEFEncoding) FromOTLP(data plog.Logs) (encodings.InMemoryData, error) { + return sortedbyresource.OtlpToSortedTree(data), nil +} + +func (d *LogsSTEFEncoding) Encode(data encodings.InMemoryData) ([]byte, error) { + tree := data.(*sortedbyresource.SortedTree) + output := &pkg.MemChunkWriter{} + writer, err := otelstef.NewLogsWriter(output, d.Opts) + if err != nil { + return nil, err + } + if err := tree.ToStef(writer); err != nil { + return nil, err + } + if err := writer.Flush(); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func (*LogsSTEFEncoding) Decode(data []byte) (any, error) { + reader, err := otelstef.NewLogsReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + for { + if err := reader.Read(pkg.ReadOptions{}); err != nil { + if err == io.EOF { + return nil, nil + } + return nil, err + } + } +} + +func (*LogsSTEFEncoding) ToOTLP(data []byte) (plog.Logs, error) { + reader, err := otelstef.NewLogsReader(bytes.NewReader(data)) + if err != nil { + return plog.NewLogs(), err + } + converter := logconvert.StefToOtlpUnsorted{} + return converter.Convert(reader, true) +} + +func (e *LogsSTEFEncoding) Name() string { + name := "STEF" + if e.Opts.Compression != pkg.CompressionNone { + name += "Z" + } + return name +} + +func (*LogsSTEFEncoding) LongName() string { + return "STEF" +} diff --git a/benchmarks/generators/file.go b/benchmarks/generators/file.go index 00869d57..3456c66b 100644 --- a/benchmarks/generators/file.go +++ b/benchmarks/generators/file.go @@ -23,7 +23,7 @@ func (f *File) GetName() string { } func (f *File) Generate() pmetric.Metrics { - all, err := testutils.ReadOTLPFile(f.FilePath) + all, err := testutils.ReadOTLPMetricsFile(f.FilePath) if err != nil { log.Fatalf("Error reading file %s: %v", f.FilePath, err) } diff --git a/benchmarks/otlp2sortedtree_test.go b/benchmarks/otlp2sortedtree_test.go index 4a52ae7b..00c711a4 100644 --- a/benchmarks/otlp2sortedtree_test.go +++ b/benchmarks/otlp2sortedtree_test.go @@ -36,7 +36,7 @@ func mapContains[K any](searchFor, inTree *b.Tree[K, bool]) bool { } func TestConvertFromOTLP(t *testing.T) { - otlpData, err := testutils.ReadOTLPFile("testdata/hipstershop-otelmetrics.zst") + otlpData, err := testutils.ReadOTLPMetricsFile("testdata/hipstershop-otelmetrics.zst") require.NoError(t, err) sorted, err := sortedbymetric.OtlpToSortedTree(otlpData) diff --git a/benchmarks/readwrite_test.go b/benchmarks/readwrite_test.go index b3bc507e..fe0c9629 100644 --- a/benchmarks/readwrite_test.go +++ b/benchmarks/readwrite_test.go @@ -76,7 +76,7 @@ func TestCopy(t *testing.T) { } } -func BenchmarkReadSTEF(b *testing.B) { +func BenchmarkMetricsReadSTEF(b *testing.B) { tefBytes, err := os.ReadFile("testdata/generated/hipstershop-otelmetrics.stefz") require.NoError(b, err) @@ -130,7 +130,7 @@ func BenchmarkReadSTEF(b *testing.B) { b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*recCount), "ns/point") } -func BenchmarkReadSTEFZ(b *testing.B) { +func BenchmarkMetricsReadSTEFZ(b *testing.B) { tefBytes, err := os.ReadFile("testdata/generated/hipstershop-otelmetrics.stefz") require.NoError(b, err) @@ -157,7 +157,7 @@ func BenchmarkReadSTEFZ(b *testing.B) { b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*recCount), "ns/point") } -func BenchmarkReadSTEFZWriteSTEF(b *testing.B) { +func BenchmarkMetricsReadSTEFZWriteSTEF(b *testing.B) { tefBytes, err := os.ReadFile("testdata/generated/hipstershop-otelmetrics.stefz") require.NoError(b, err) diff --git a/benchmarks/size_test.go b/benchmarks/size_test.go index c878527b..e2562a49 100644 --- a/benchmarks/size_test.go +++ b/benchmarks/size_test.go @@ -12,6 +12,7 @@ import ( "github.com/go-echarts/go-echarts/v2/charts" "github.com/go-echarts/go-echarts/v2/opts" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/ptrace" "github.com/splunk/stef/benchmarks/encodings" @@ -22,6 +23,7 @@ import ( "github.com/splunk/stef/benchmarks/generators" "github.com/splunk/stef/benchmarks/testutils" "github.com/splunk/stef/go/otel/otelstef" + pdatalogs "github.com/splunk/stef/go/pdata/logs" "github.com/splunk/stef/go/pdata/traces" "github.com/splunk/stef/go/pkg" ) @@ -69,7 +71,7 @@ var metricsDataVariations = []struct { } var sizeEncodings = []encodings.MetricEncoding{ - &otlp.OTLPEncoding{}, + &otlp.MetricsEncoding{}, &stef.STEFEncoding{Opts: pkg.WriterOptions{Compression: pkg.CompressionNone}}, &stef.STEFUEncoding{Opts: pkg.WriterOptions{Compression: pkg.CompressionNone}}, &parquetenc.Encoding{}, @@ -98,7 +100,7 @@ func TestMetricsSize(t *testing.T) { wantChart := strings.HasSuffix(dataVariation.generator.GetName(), ".zst") if wantChart { - chart.BeginChart("Dataset: "+dataVariation.generator.GetName(), t) + chart.BeginChart("Dataset: "+dataVariation.generator.GetName(), "Bytes/point", t) } for _, encoding := range sizeEncodings { @@ -179,7 +181,6 @@ func TestMetricsSize(t *testing.T) { if wantChart { chart.EndChart( - "Bytes/point", charts.WithColorsOpts(opts.Colors{"#92C5F9"}), ) } @@ -201,7 +202,7 @@ func TestMetricsMultipart(t *testing.T) { } testMultipartEncodings := []encodings.MetricMultipartEncoding{ - &otlp.OTLPEncoding{}, + &otlp.MetricsEncoding{}, &stef.STEFEncoding{}, &stef.STEFUEncoding{}, &otelarrow.OtelArrowEncoding{}, @@ -213,7 +214,7 @@ func TestMetricsMultipart(t *testing.T) { for _, compression := range compressions { for _, dataset := range datasets { - chart.BeginChart("Dataset: "+dataset.name, t) + chart.BeginChart("Dataset: "+dataset.name, "Bytes/point", t) fmt.Printf("%-30s %4v %9v %4v\n", dataset.name, "Comp", "Bytes", "Ratio") @@ -225,7 +226,7 @@ func TestMetricsMultipart(t *testing.T) { // Encode each part one after another and write to the same STEF stream. // This models more closely the operation of STEF exporter in Collector. - parts, err := testutils.ReadMultipartOTLPFile("testdata/" + dataset.name + ".zst") + parts, err := testutils.ReadMultipartMetricsOTLPFile("testdata/" + dataset.name + ".zst") require.NoError(t, err) pointCount := 0 @@ -258,7 +259,6 @@ func TestMetricsMultipart(t *testing.T) { } chart.EndChart( - "Bytes/point", charts.WithColorsOpts(opts.Colors{"#87BB62"}), ) } @@ -266,7 +266,7 @@ func TestMetricsMultipart(t *testing.T) { } func TestSTEFVeryShortFrames(t *testing.T) { - input, err := testutils.ReadOTLPFile("testdata/hipstershop-otelmetrics.zst") + input, err := testutils.ReadOTLPMetricsFile("testdata/hipstershop-otelmetrics.zst") require.NoError(t, err) compressions := []pkg.Compression{pkg.CompressionNone, pkg.CompressionZstd} @@ -348,7 +348,7 @@ func TestTracesMultipart(t *testing.T) { } fmt.Println(compressionStr) - chart.BeginChart("Dataset: "+fileName, t) + chart.BeginChart("Dataset: "+fileName, "Bytes/span", t) otlpSize := 0 spanCount := 0 @@ -382,8 +382,11 @@ func TestTracesMultipart(t *testing.T) { converter := traces.OtlpToStefUnsorted{Sorted: sorted} - for i := 0; i < len(traceData); i++ { - err = converter.Convert(traceData[i].(ptrace.Traces), writer) + for i := 0; i < len(otlpParts); i++ { + tracesPart, err := u.UnmarshalTraces(otlpParts[i]) + require.NoError(t, err) + + err = converter.Convert(tracesPart, writer) require.NoError(t, err) err = writer.Flush() @@ -412,7 +415,110 @@ func TestTracesMultipart(t *testing.T) { ) } chart.EndChart( - "Bytes/span", + charts.WithColorsOpts(opts.Colors{"#87BB62"}), + ) + } + } +} + +func TestLogsMultipart(t *testing.T) { + u := plog.ProtoUnmarshaler{} + + fileNames := []string{"astronomy-otellogs"} + + chart.BeginSection("Size - Many Batches, Multipart Logs") + + for _, fileName := range fileNames { + fmt.Println("======= " + fileName) + + var otlpParts [][]byte + logsData, err := testutils.ReadMultipartOTLPFileGeneric( + "testdata/"+fileName+".zst", func(data []byte) (any, error) { + otlpParts = append(otlpParts, data) + return u.UnmarshalLogs(data) + }, + ) + require.NoError(t, err) + + compressions := []pkg.Compression{pkg.CompressionNone, pkg.CompressionZstd} + sorteds := []bool{false, true} + + for _, compression := range compressions { + var compressionStr string + if compression == pkg.CompressionZstd { + compressionStr = "zstd" + } else { + compressionStr = "none" + } + fmt.Println(compressionStr) + + chart.BeginChart("Dataset: "+fileName, "Bytes/log", t) + + otlpSize := 0 + logCount := 0 + for i := 0; i < len(logsData); i++ { + if compression == pkg.CompressionNone { + otlpSize += len(otlpParts[i]) + } else { + otlpZstd := testutils.CompressZstd(otlpParts[i]) + otlpSize += len(otlpZstd) + } + logCount += logsData[i].(plog.Logs).LogRecordCount() + } + + chart.Record( + nil, "OTLP", "Bytes/log, compression="+compressionStr, + unitSize(otlpSize, logCount), + ) + + for _, sorted := range sorteds { + var sortedStr string + if sorted { + sortedStr = "Sorted" + } else { + sortedStr = "Unsorted" + } + fmt.Println(sortedStr) + + outputBuf := &pkg.MemChunkWriter{} + writer, err := otelstef.NewLogsWriter(outputBuf, pkg.WriterOptions{Compression: compression}) + require.NoError(t, err) + + converter := pdatalogs.OtlpToStefUnsorted{Sorted: sorted} + + for i := 0; i < len(otlpParts); i++ { + logsPart, err := u.UnmarshalLogs(otlpParts[i]) + require.NoError(t, err) + + err = converter.Convert(logsPart, writer) + require.NoError(t, err) + + err = writer.Flush() + require.NoError(t, err) + } + + stefSize := len(outputBuf.Bytes()) + + fmt.Printf( + "Logs OTLP: %8d (%5.1f bytes/log)\n", + otlpSize, + float64(otlpSize)/float64(logCount), + ) + fmt.Printf( + "Logs STEF: %8d (%5.1f bytes/log)\n", + stefSize, + float64(stefSize)/float64(logCount), + ) + fmt.Printf( + "Ratio: %8.2f\n", float64(otlpSize)/float64(stefSize), + ) + + chart.Record( + nil, "STEF "+sortedStr, "Bytes/log, compression="+compressionStr, + unitSize(stefSize, logCount), + ) + } + chart.EndChart( charts.WithColorsOpts(opts.Colors{"#87BB62"}), ) } diff --git a/benchmarks/testdata/astronomy-otellogs.zst b/benchmarks/testdata/astronomy-otellogs.zst new file mode 100644 index 00000000..198730cc Binary files /dev/null and b/benchmarks/testdata/astronomy-otellogs.zst differ diff --git a/benchmarks/testutils/testutils.go b/benchmarks/testutils/testutils.go index cec5a226..c37f9f6c 100644 --- a/benchmarks/testutils/testutils.go +++ b/benchmarks/testutils/testutils.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/klauspost/compress/zstd" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" ) @@ -43,7 +44,7 @@ func openOTLPFile(filePath string) (io.Reader, func(), error) { return reader, closer, nil } -func ReadMultipartOTLPFile(filePath string) ([]pmetric.Metrics, error) { +func ReadMultipartMetricsOTLPFile(filePath string) ([]pmetric.Metrics, error) { var result []pmetric.Metrics reader, closer, err := openOTLPFile(filePath) @@ -135,9 +136,9 @@ func ReadMultipartOTLPFileGeneric(filePath string, unmarshaler func([]byte) (any return result, nil } -func ReadOTLPFile(filePath string) (pmetric.Metrics, error) { +func ReadOTLPMetricsFile(filePath string) (pmetric.Metrics, error) { combined := pmetric.NewMetrics() - parts, err := ReadMultipartOTLPFile(filePath) + parts, err := ReadMultipartMetricsOTLPFile(filePath) if err != nil { return combined, err } @@ -146,3 +147,20 @@ func ReadOTLPFile(filePath string) (pmetric.Metrics, error) { } return combined, nil } + +func ReadOTLPLogsFile(filePath string) (plog.Logs, error) { + combined := plog.NewLogs() + unmarshaler := plog.ProtoUnmarshaler{} + parts, err := ReadMultipartOTLPFileGeneric( + filePath, func(data []byte) (any, error) { + return unmarshaler.UnmarshalLogs(data) + }, + ) + if err != nil { + return combined, err + } + for _, part := range parts { + part.(plog.Logs).ResourceLogs().MoveAndAppendTo(combined.ResourceLogs()) + } + return combined, nil +} diff --git a/docs/benchmarks.html b/docs/benchmarks.html index 55c373ff..692a1f95 100644 --- a/docs/benchmarks.html +++ b/docs/benchmarks.html @@ -9,186 +9,247 @@

Size Benchmarks - One Large Batch

-
+
- +
-
+
- +
-
+
- +

Size - Many Batches, Multipart Metrics

-
+
- +
-
+
- +
-
+
- +
-
+
- +

Size - Many Batches, Multipart Traces

-
+
+
+ + + +
+
+
+ + + +
+
- +
-
+
- + +

Size - Many Batches, Multipart Logs

-
+
- +
-
+
- +

Speed Benchmarks

-
+
- +
-
+
- + + +
+
+
+ +
-
+
- +
-
+
- + + +
+
+
+ + + +
+
+
+ + - \ No newline at end of file + diff --git a/examples/ints/internal/ints/modifiedfields.go b/examples/ints/internal/ints/modifiedfields.go index 2f02cad8..6d5b84ba 100644 --- a/examples/ints/internal/ints/modifiedfields.go +++ b/examples/ints/internal/ints/modifiedfields.go @@ -115,6 +115,13 @@ func (m *modifiedFieldsMultimap) fixParent(parentModifiedFields *modifiedFields) m.vals.parent = parentModifiedFields } +func (m *modifiedFieldsMultimap) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.keys.parent = parentModifiedFields + m.vals.parent = parentModifiedFields + m.keys.parentBit = parentModifiedBit + m.vals.parentBit = parentModifiedBit +} + // Returns a bitmask to be set in keys or vals to mark modification of // multimap element at index. func (m *modifiedFieldsMultimap) maskForIndex(index int) uint64 { diff --git a/examples/ints/internal/ints/record.go b/examples/ints/internal/ints/record.go index cf7ef42e..1f071f36 100644 --- a/examples/ints/internal/ints/record.go +++ b/examples/ints/internal/ints/record.go @@ -77,6 +77,12 @@ func (s *Record) fixParent(parentModifiedFields *modifiedFields) { s.modifiedFields.parent = parentModifiedFields } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Record) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit +} + func (s *Record) freeze() { s.modifiedFields.freeze() } @@ -118,6 +124,12 @@ func (s *Record) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Record) clearModifiedRecursively() { + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Record) computeDiff(val *Record) (ret bool) { @@ -134,17 +146,14 @@ func (s *Record) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Record) cloneShared(allocators *Allocators) Record { - return s.Clone(allocators) -} - -func (s *Record) Clone(allocators *Allocators) Record { - c := Record{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Record) CloneTo(dst *Record, allocators *Allocators) { + *dst = Record{ uint64: s.uint64, } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/examples/jsonl/internal/jsonstef/jsonobject.go b/examples/jsonl/internal/jsonstef/jsonobject.go index 06cde78a..3b5c6891 100644 --- a/examples/jsonl/internal/jsonstef/jsonobject.go +++ b/examples/jsonl/internal/jsonstef/jsonobject.go @@ -60,6 +60,15 @@ func (m *JsonObject) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointers and bits in a newly copied multimap. +func (m *JsonObject) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.modifiedElems.attachParent(parentModifiedFields, parentModifiedBit) + elems := m.elems[:m.initedCount] + for i := 0; i < len(elems); i++ { + elems[i].value.attachParent(&m.modifiedElems.vals, m.modifiedElems.maskForIndex(i)) + } +} + func (m *JsonObject) canBeShared() bool { // Multimap can never be shared. return false @@ -122,6 +131,14 @@ func (m *JsonObject) setUnmodifiedRecursively() { m.modifiedElems.setUnmodifiedAll() } +// clearModifiedRecursively clears modification state in all mutable elements. +func (m *JsonObject) clearModifiedRecursively() { + for i := 0; i < len(m.elems); i++ { + m.elems[i].value.clearModifiedRecursively() + } + m.modifiedElems.setUnmodifiedAll() +} + // computeDiff compares m to val and returns true if they differ. // All fields that are different in m will be marked as modified. func (m *JsonObject) computeDiff(val *JsonObject) (ret bool) { diff --git a/examples/jsonl/internal/jsonstef/jsonvalue.go b/examples/jsonl/internal/jsonstef/jsonvalue.go index c7d39ae9..5b510fd9 100644 --- a/examples/jsonl/internal/jsonstef/jsonvalue.go +++ b/examples/jsonl/internal/jsonstef/jsonvalue.go @@ -82,6 +82,19 @@ func (s *JsonValue) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointer and bit in a newly copied oneof. +func (s *JsonValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.parentModifiedFields = parentModifiedFields + s.parentModifiedBit = parentModifiedBit + + switch s.Type() { + case JsonValueTypeObject: + s.objectPtr().attachParent(parentModifiedFields, parentModifiedBit) + case JsonValueTypeArray: + s.arrayPtr().attachParent(parentModifiedFields, parentModifiedBit) + } +} + type JsonValueType byte const ( @@ -244,32 +257,41 @@ func (s *JsonValue) canBeShared() bool { return false } +func (s *JsonValue) clone(allocators *Allocators) *JsonValue { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(JsonValue{}))) + c := allocators.JsonValue.Alloc() + s.CloneTo(c, allocators) + return c +} + func (s *JsonValue) cloneShared(allocators *Allocators) *JsonValue { - // Oneof is not shareable, so cloneShared is just a Clone. - return s.Clone(allocators) + // Oneof is not shareable, so cloneShared is just a clone. + return s.clone(allocators) } -func (s *JsonValue) Clone(allocators *Allocators) *JsonValue { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(JsonValue{}))) - c := allocators.JsonValue.Alloc() - c.clearValSetType(s.Type()) +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *JsonValue) CloneTo(dst *JsonValue, allocators *Allocators) { + *dst = JsonValue{} + dst.clearValSetType(s.Type()) switch s.Type() { case JsonValueTypeObject: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(JsonObject{}))) - c.allocObjectAlloc(allocators) - copyToNewJsonObject(c.objectPtr(), s.objectPtr(), allocators) + dst.allocObjectAlloc(allocators) + copyToNewJsonObject(dst.objectPtr(), s.objectPtr(), allocators) case JsonValueTypeArray: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(JsonValueArray{}))) - c.allocArrayAlloc(allocators) - copyToNewJsonValueArray(c.arrayPtr(), s.arrayPtr(), allocators) + dst.allocArrayAlloc(allocators) + copyToNewJsonValueArray(dst.arrayPtr(), s.arrayPtr(), allocators) case JsonValueTypeString: - c.setString(s.String()) + dst.setString(s.String()) case JsonValueTypeNumber: - *c.numberPtr() = *s.numberPtr() + *dst.numberPtr() = *s.numberPtr() case JsonValueTypeBool: - *c.boolPtr() = *s.boolPtr() + *dst.boolPtr() = *s.boolPtr() } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -371,6 +393,16 @@ func (s *JsonValue) setUnmodifiedRecursively() { } } +// clearModifiedRecursively clears modification state in all mutable descendants. +func (s *JsonValue) clearModifiedRecursively() { + switch s.Type() { + case JsonValueTypeObject: + s.objectPtr().clearModifiedRecursively() + case JsonValueTypeArray: + s.arrayPtr().clearModifiedRecursively() + } +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *JsonValue) computeDiff(val *JsonValue) (ret bool) { diff --git a/examples/jsonl/internal/jsonstef/jsonvaluearray.go b/examples/jsonl/internal/jsonstef/jsonvaluearray.go index 05e017f2..0607eaf0 100644 --- a/examples/jsonl/internal/jsonstef/jsonvaluearray.go +++ b/examples/jsonl/internal/jsonstef/jsonvaluearray.go @@ -52,6 +52,15 @@ func (e *JsonValueArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *JsonValueArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *JsonValueArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *JsonValueArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *JsonValueArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *JsonValueArray) computeDiff(val *JsonValueArray) (ret bool) { diff --git a/examples/jsonl/internal/jsonstef/modifiedfields.go b/examples/jsonl/internal/jsonstef/modifiedfields.go index d7d07bd7..b39bec1d 100644 --- a/examples/jsonl/internal/jsonstef/modifiedfields.go +++ b/examples/jsonl/internal/jsonstef/modifiedfields.go @@ -115,6 +115,13 @@ func (m *modifiedFieldsMultimap) fixParent(parentModifiedFields *modifiedFields) m.vals.parent = parentModifiedFields } +func (m *modifiedFieldsMultimap) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.keys.parent = parentModifiedFields + m.vals.parent = parentModifiedFields + m.keys.parentBit = parentModifiedBit + m.vals.parentBit = parentModifiedBit +} + // Returns a bitmask to be set in keys or vals to mark modification of // multimap element at index. func (m *modifiedFieldsMultimap) maskForIndex(index int) uint64 { diff --git a/examples/jsonl/internal/jsonstef/record.go b/examples/jsonl/internal/jsonstef/record.go index b233bb83..bc4d81e8 100644 --- a/examples/jsonl/internal/jsonstef/record.go +++ b/examples/jsonl/internal/jsonstef/record.go @@ -85,6 +85,13 @@ func (s *Record) fixParent(parentModifiedFields *modifiedFields) { s.value.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Record) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.value.attachParent(&s.modifiedFields, fieldModifiedRecordValue) +} + func (s *Record) freeze() { if s.isFrozen() { return @@ -126,6 +133,13 @@ func (s *Record) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Record) clearModifiedRecursively() { + s.value.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Record) computeDiff(val *Record) (ret bool) { @@ -142,17 +156,13 @@ func (s *Record) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Record) cloneShared(allocators *Allocators) Record { - return s.Clone(allocators) -} - -func (s *Record) Clone(allocators *Allocators) Record { - c := Record{ - value: s.value.cloneShared(allocators), - } - return c +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Record) CloneTo(dst *Record, allocators *Allocators) { + *dst = Record{} + dst.value = s.value.cloneShared(allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/examples/profile/internal/profile/function.go b/examples/profile/internal/profile/function.go index fc001788..d7d22ada 100644 --- a/examples/profile/internal/profile/function.go +++ b/examples/profile/internal/profile/function.go @@ -98,6 +98,12 @@ func (s *Function) fixParent(parentModifiedFields *modifiedFields) { s.modifiedFields.parent = parentModifiedFields } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Function) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit +} + // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, // which can improve encoding performance. @@ -221,6 +227,12 @@ func (s *Function) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Function) clearModifiedRecursively() { + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Function) computeDiff(val *Function) (ret bool) { @@ -252,6 +264,14 @@ func (s *Function) canBeShared() bool { return s.isFrozen() } +// clone returns a clone of s allocated using allocators. +func (s *Function) clone(allocators *Allocators) *Function { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Function{}))) + c := allocators.Function.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). func (s *Function) cloneShared(allocators *Allocators) *Function { @@ -259,19 +279,20 @@ func (s *Function) cloneShared(allocators *Allocators) *Function { // If s is frozen it means it is safe to share without cloning. return s } - return s.Clone(allocators) + return s.clone(allocators) } -func (s *Function) Clone(allocators *Allocators) *Function { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Function{}))) - c := allocators.Function.Alloc() - *c = Function{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Function) CloneTo(dst *Function, allocators *Allocators) { + *dst = Function{ name: s.name, systemName: s.systemName, filename: s.filename, startLine: s.startLine, } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -461,7 +482,7 @@ func (d *FunctionEncoderDict) Add(val *Function) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -795,7 +816,7 @@ func (d *FunctionDecoder) Decode(dstPtr **Function) error { // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } diff --git a/examples/profile/internal/profile/labels.go b/examples/profile/internal/profile/labels.go index b7dddb25..5570a06c 100644 --- a/examples/profile/internal/profile/labels.go +++ b/examples/profile/internal/profile/labels.go @@ -60,6 +60,15 @@ func (m *Labels) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointers and bits in a newly copied multimap. +func (m *Labels) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.modifiedElems.attachParent(parentModifiedFields, parentModifiedBit) + elems := m.elems[:m.initedCount] + for i := 0; i < len(elems); i++ { + elems[i].value.attachParent(&m.modifiedElems.vals, m.modifiedElems.maskForIndex(i)) + } +} + func (m *Labels) canBeShared() bool { // Multimap can never be shared. return false @@ -122,6 +131,14 @@ func (m *Labels) setUnmodifiedRecursively() { m.modifiedElems.setUnmodifiedAll() } +// clearModifiedRecursively clears modification state in all mutable elements. +func (m *Labels) clearModifiedRecursively() { + for i := 0; i < len(m.elems); i++ { + m.elems[i].value.clearModifiedRecursively() + } + m.modifiedElems.setUnmodifiedAll() +} + // computeDiff compares m to val and returns true if they differ. // All fields that are different in m will be marked as modified. func (m *Labels) computeDiff(val *Labels) (ret bool) { diff --git a/examples/profile/internal/profile/labelvalue.go b/examples/profile/internal/profile/labelvalue.go index 4ad0c6e9..e9ae19dc 100644 --- a/examples/profile/internal/profile/labelvalue.go +++ b/examples/profile/internal/profile/labelvalue.go @@ -76,6 +76,17 @@ func (s *LabelValue) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointer and bit in a newly copied oneof. +func (s *LabelValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.parentModifiedFields = parentModifiedFields + s.parentModifiedBit = parentModifiedBit + + switch s.Type() { + case LabelValueTypeNum: + s.numPtr().attachParent(parentModifiedFields, parentModifiedBit) + } +} + type LabelValueType byte const ( @@ -169,23 +180,21 @@ func (s *LabelValue) canBeShared() bool { return false } -func (s *LabelValue) cloneShared(allocators *Allocators) LabelValue { - // Oneof is not shareable, so cloneShared is just a Clone. - return s.Clone(allocators) -} - -func (s *LabelValue) Clone(allocators *Allocators) LabelValue { - c := LabelValue{} - c.clearValSetType(s.Type()) +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *LabelValue) CloneTo(dst *LabelValue, allocators *Allocators) { + *dst = LabelValue{} + dst.clearValSetType(s.Type()) switch s.Type() { case LabelValueTypeStr: - c.setStr(s.Str()) + dst.setStr(s.Str()) case LabelValueTypeNum: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(NumValue{}))) - c.allocNumAlloc(allocators) - copyToNewNumValue(c.numPtr(), s.numPtr(), allocators) + dst.allocNumAlloc(allocators) + copyToNewNumValue(dst.numPtr(), s.numPtr(), allocators) } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -260,6 +269,14 @@ func (s *LabelValue) setUnmodifiedRecursively() { } } +// clearModifiedRecursively clears modification state in all mutable descendants. +func (s *LabelValue) clearModifiedRecursively() { + switch s.Type() { + case LabelValueTypeNum: + s.numPtr().clearModifiedRecursively() + } +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *LabelValue) computeDiff(val *LabelValue) (ret bool) { diff --git a/examples/profile/internal/profile/line.go b/examples/profile/internal/profile/line.go index 5dac1f49..3236014e 100644 --- a/examples/profile/internal/profile/line.go +++ b/examples/profile/internal/profile/line.go @@ -89,6 +89,15 @@ func (s *Line) fixParent(parentModifiedFields *modifiedFields) { s.function.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Line) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + if !s.function.canBeShared() { + s.function.attachParent(&s.modifiedFields, fieldModifiedLineFunction) + } +} + func (s *Line) freeze() { if s.isFrozen() { return @@ -117,7 +126,7 @@ func (s *Line) SetFunction(v *Function) { } } else { if s.function.canBeShared() { - s.function = s.function.Clone(&Allocators{}) + s.function = s.function.clone(&Allocators{}) } s.function.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedLineFunction) @@ -199,6 +208,15 @@ func (s *Line) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Line) clearModifiedRecursively() { + if !s.function.canBeShared() { + s.function.clearModifiedRecursively() + } + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Line) computeDiff(val *Line) (ret bool) { @@ -225,19 +243,16 @@ func (s *Line) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Line) cloneShared(allocators *Allocators) Line { - return s.Clone(allocators) -} - -func (s *Line) Clone(allocators *Allocators) Line { - c := Line{ - function: s.function.cloneShared(allocators), - line: s.line, - column: s.column, +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Line) CloneTo(dst *Line, allocators *Allocators) { + *dst = Line{ + line: s.line, + column: s.column, } - return c + dst.function = s.function.cloneShared(allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -316,12 +331,12 @@ func (s *Line) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter *mu s.function = new(Function) s.function.init(&s.modifiedFields, fieldModifiedLineFunction) } else { - s.function = s.function.Clone(&Allocators{}) + s.function = s.function.clone(&Allocators{}) } } if s.function.canBeShared() { // function may be shared by pointer. Clone it to have exclusive ownership. - s.function = s.function.Clone(&Allocators{}) + s.function = s.function.clone(&Allocators{}) } s.function.mutateRandom(random, schem, limiter) diff --git a/examples/profile/internal/profile/linearray.go b/examples/profile/internal/profile/linearray.go index 9ff44955..26263cc7 100644 --- a/examples/profile/internal/profile/linearray.go +++ b/examples/profile/internal/profile/linearray.go @@ -52,6 +52,15 @@ func (e *LineArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *LineArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *LineArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *LineArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *LineArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *LineArray) computeDiff(val *LineArray) (ret bool) { diff --git a/examples/profile/internal/profile/location.go b/examples/profile/internal/profile/location.go index 59b650a9..2b2068ed 100644 --- a/examples/profile/internal/profile/location.go +++ b/examples/profile/internal/profile/location.go @@ -107,6 +107,16 @@ func (s *Location) fixParent(parentModifiedFields *modifiedFields) { s.lines.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Location) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + if !s.mapping.canBeShared() { + s.mapping.attachParent(&s.modifiedFields, fieldModifiedLocationMapping) + } + s.lines.attachParent(&s.modifiedFields, fieldModifiedLocationLines) +} + // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, // which can improve encoding performance. @@ -143,7 +153,7 @@ func (s *Location) SetMapping(v *Mapping) { } } else { if s.mapping.canBeShared() { - s.mapping = s.mapping.Clone(&Allocators{}) + s.mapping = s.mapping.clone(&Allocators{}) } s.mapping.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedLocationMapping) @@ -246,6 +256,16 @@ func (s *Location) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Location) clearModifiedRecursively() { + if !s.mapping.canBeShared() { + s.mapping.clearModifiedRecursively() + } + s.lines.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Location) computeDiff(val *Location) (ret bool) { @@ -277,6 +297,14 @@ func (s *Location) canBeShared() bool { return s.isFrozen() } +// clone returns a clone of s allocated using allocators. +func (s *Location) clone(allocators *Allocators) *Location { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Location{}))) + c := allocators.Location.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). func (s *Location) cloneShared(allocators *Allocators) *Location { @@ -284,19 +312,20 @@ func (s *Location) cloneShared(allocators *Allocators) *Location { // If s is frozen it means it is safe to share without cloning. return s } - return s.Clone(allocators) + return s.clone(allocators) } -func (s *Location) Clone(allocators *Allocators) *Location { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Location{}))) - c := allocators.Location.Alloc() - *c = Location{ - mapping: s.mapping.cloneShared(allocators), +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Location) CloneTo(dst *Location, allocators *Allocators) { + *dst = Location{ address: s.address, isFolded: s.isFolded, } - copyToNewLineArray(&c.lines, &s.lines, allocators) - return c + dst.mapping = s.mapping.cloneShared(allocators) + copyToNewLineArray(&dst.lines, &s.lines, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -377,12 +406,12 @@ func (s *Location) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter s.mapping = new(Mapping) s.mapping.init(&s.modifiedFields, fieldModifiedLocationMapping) } else { - s.mapping = s.mapping.Clone(&Allocators{}) + s.mapping = s.mapping.clone(&Allocators{}) } } if s.mapping.canBeShared() { // mapping may be shared by pointer. Clone it to have exclusive ownership. - s.mapping = s.mapping.Clone(&Allocators{}) + s.mapping = s.mapping.clone(&Allocators{}) } s.mapping.mutateRandom(random, schem, limiter) @@ -526,7 +555,7 @@ func (d *LocationEncoderDict) Add(val *Location) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -912,7 +941,7 @@ func (d *LocationDecoder) Decode(dstPtr **Location) error { // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } diff --git a/examples/profile/internal/profile/locationarray.go b/examples/profile/internal/profile/locationarray.go index 61b57042..cbab389c 100644 --- a/examples/profile/internal/profile/locationarray.go +++ b/examples/profile/internal/profile/locationarray.go @@ -52,6 +52,17 @@ func (e *LocationArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *LocationArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + if !e.elems[i].canBeShared() { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } + } +} + func (e *LocationArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +107,16 @@ func (e *LocationArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *LocationArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + if !e.elems[i].canBeShared() { + e.elems[i].clearModifiedRecursively() + } + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *LocationArray) computeDiff(val *LocationArray) (ret bool) { @@ -287,7 +308,7 @@ func (a *LocationArray) mutateRandom(random *rand.Rand, schem *schema.Schema, li if random.IntN(2*len(a.elems)) == 0 { if a.elems[i].canBeShared() { // Elem may be shared by pointer. Clone it to have exclusive ownership. - a.elems[i] = a.elems[i].Clone(&Allocators{}) + a.elems[i] = a.elems[i].clone(&Allocators{}) } a.elems[i].mutateRandom(random, schem, limiter) } diff --git a/examples/profile/internal/profile/mapping.go b/examples/profile/internal/profile/mapping.go index fd8f0b60..7fe407dc 100644 --- a/examples/profile/internal/profile/mapping.go +++ b/examples/profile/internal/profile/mapping.go @@ -113,6 +113,12 @@ func (s *Mapping) fixParent(parentModifiedFields *modifiedFields) { s.modifiedFields.parent = parentModifiedFields } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Mapping) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit +} + // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, // which can improve encoding performance. @@ -361,6 +367,12 @@ func (s *Mapping) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Mapping) clearModifiedRecursively() { + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Mapping) computeDiff(val *Mapping) (ret bool) { @@ -417,6 +429,14 @@ func (s *Mapping) canBeShared() bool { return s.isFrozen() } +// clone returns a clone of s allocated using allocators. +func (s *Mapping) clone(allocators *Allocators) *Mapping { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Mapping{}))) + c := allocators.Mapping.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). func (s *Mapping) cloneShared(allocators *Allocators) *Mapping { @@ -424,13 +444,13 @@ func (s *Mapping) cloneShared(allocators *Allocators) *Mapping { // If s is frozen it means it is safe to share without cloning. return s } - return s.Clone(allocators) + return s.clone(allocators) } -func (s *Mapping) Clone(allocators *Allocators) *Mapping { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Mapping{}))) - c := allocators.Mapping.Alloc() - *c = Mapping{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Mapping) CloneTo(dst *Mapping, allocators *Allocators) { + *dst = Mapping{ memoryStart: s.memoryStart, memoryLimit: s.memoryLimit, fileOffset: s.fileOffset, @@ -441,7 +461,8 @@ func (s *Mapping) Clone(allocators *Allocators) *Mapping { hasLineNumbers: s.hasLineNumbers, hasInlineFrames: s.hasInlineFrames, } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -721,7 +742,7 @@ func (d *MappingEncoderDict) Add(val *Mapping) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -1260,7 +1281,7 @@ func (d *MappingDecoder) Decode(dstPtr **Mapping) error { // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } diff --git a/examples/profile/internal/profile/modifiedfields.go b/examples/profile/internal/profile/modifiedfields.go index 95d09368..12bb565f 100644 --- a/examples/profile/internal/profile/modifiedfields.go +++ b/examples/profile/internal/profile/modifiedfields.go @@ -115,6 +115,13 @@ func (m *modifiedFieldsMultimap) fixParent(parentModifiedFields *modifiedFields) m.vals.parent = parentModifiedFields } +func (m *modifiedFieldsMultimap) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.keys.parent = parentModifiedFields + m.vals.parent = parentModifiedFields + m.keys.parentBit = parentModifiedBit + m.vals.parentBit = parentModifiedBit +} + // Returns a bitmask to be set in keys or vals to mark modification of // multimap element at index. func (m *modifiedFieldsMultimap) maskForIndex(index int) uint64 { diff --git a/examples/profile/internal/profile/numvalue.go b/examples/profile/internal/profile/numvalue.go index 7be0cf36..ec83bc64 100644 --- a/examples/profile/internal/profile/numvalue.go +++ b/examples/profile/internal/profile/numvalue.go @@ -80,6 +80,12 @@ func (s *NumValue) fixParent(parentModifiedFields *modifiedFields) { s.modifiedFields.parent = parentModifiedFields } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *NumValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit +} + func (s *NumValue) freeze() { s.modifiedFields.freeze() } @@ -146,6 +152,12 @@ func (s *NumValue) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *NumValue) clearModifiedRecursively() { + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *NumValue) computeDiff(val *NumValue) (ret bool) { @@ -167,18 +179,15 @@ func (s *NumValue) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *NumValue) cloneShared(allocators *Allocators) NumValue { - return s.Clone(allocators) -} - -func (s *NumValue) Clone(allocators *Allocators) NumValue { - c := NumValue{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *NumValue) CloneTo(dst *NumValue, allocators *Allocators) { + *dst = NumValue{ val: s.val, unit: s.unit, } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/examples/profile/internal/profile/profilemetadata.go b/examples/profile/internal/profile/profilemetadata.go index 25f8763b..eb270879 100644 --- a/examples/profile/internal/profile/profilemetadata.go +++ b/examples/profile/internal/profile/profilemetadata.go @@ -113,6 +113,19 @@ func (s *ProfileMetadata) fixParent(parentModifiedFields *modifiedFields) { s.defaultSampleType.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *ProfileMetadata) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + if !s.periodType.canBeShared() { + s.periodType.attachParent(&s.modifiedFields, fieldModifiedProfileMetadataPeriodType) + } + s.comments.attachParent(&s.modifiedFields, fieldModifiedProfileMetadataComments) + if !s.defaultSampleType.canBeShared() { + s.defaultSampleType.attachParent(&s.modifiedFields, fieldModifiedProfileMetadataDefaultSampleType) + } +} + func (s *ProfileMetadata) freeze() { if s.isFrozen() { return @@ -239,7 +252,7 @@ func (s *ProfileMetadata) SetPeriodType(v *SampleValueType) { } } else { if s.periodType.canBeShared() { - s.periodType = s.periodType.Clone(&Allocators{}) + s.periodType = s.periodType.clone(&Allocators{}) } s.periodType.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedProfileMetadataPeriodType) @@ -314,7 +327,7 @@ func (s *ProfileMetadata) SetDefaultSampleType(v *SampleValueType) { } } else { if s.defaultSampleType.canBeShared() { - s.defaultSampleType = s.defaultSampleType.Clone(&Allocators{}) + s.defaultSampleType = s.defaultSampleType.clone(&Allocators{}) } s.defaultSampleType.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedProfileMetadataDefaultSampleType) @@ -361,6 +374,19 @@ func (s *ProfileMetadata) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *ProfileMetadata) clearModifiedRecursively() { + if !s.periodType.canBeShared() { + s.periodType.clearModifiedRecursively() + } + s.comments.clearModifiedRecursively() + if !s.defaultSampleType.canBeShared() { + s.defaultSampleType.clearModifiedRecursively() + } + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *ProfileMetadata) computeDiff(val *ProfileMetadata) (ret bool) { @@ -412,24 +438,21 @@ func (s *ProfileMetadata) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *ProfileMetadata) cloneShared(allocators *Allocators) ProfileMetadata { - return s.Clone(allocators) -} - -func (s *ProfileMetadata) Clone(allocators *Allocators) ProfileMetadata { - c := ProfileMetadata{ - dropFrames: s.dropFrames, - keepFrames: s.keepFrames, - timeNanos: s.timeNanos, - durationNanos: s.durationNanos, - periodType: s.periodType.cloneShared(allocators), - period: s.period, - defaultSampleType: s.defaultSampleType.cloneShared(allocators), +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *ProfileMetadata) CloneTo(dst *ProfileMetadata, allocators *Allocators) { + *dst = ProfileMetadata{ + dropFrames: s.dropFrames, + keepFrames: s.keepFrames, + timeNanos: s.timeNanos, + durationNanos: s.durationNanos, + period: s.period, } - copyToNewStringArray(&c.comments, &s.comments, allocators) - return c + dst.periodType = s.periodType.cloneShared(allocators) + copyToNewStringArray(&dst.comments, &s.comments, allocators) + dst.defaultSampleType = s.defaultSampleType.cloneShared(allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -567,12 +590,12 @@ func (s *ProfileMetadata) mutateRandom(random *rand.Rand, schem *schema.Schema, s.periodType = new(SampleValueType) s.periodType.init(&s.modifiedFields, fieldModifiedProfileMetadataPeriodType) } else { - s.periodType = s.periodType.Clone(&Allocators{}) + s.periodType = s.periodType.clone(&Allocators{}) } } if s.periodType.canBeShared() { // periodType may be shared by pointer. Clone it to have exclusive ownership. - s.periodType = s.periodType.Clone(&Allocators{}) + s.periodType = s.periodType.clone(&Allocators{}) } s.periodType.mutateRandom(random, schem, limiter) @@ -605,12 +628,12 @@ func (s *ProfileMetadata) mutateRandom(random *rand.Rand, schem *schema.Schema, s.defaultSampleType = new(SampleValueType) s.defaultSampleType.init(&s.modifiedFields, fieldModifiedProfileMetadataDefaultSampleType) } else { - s.defaultSampleType = s.defaultSampleType.Clone(&Allocators{}) + s.defaultSampleType = s.defaultSampleType.clone(&Allocators{}) } } if s.defaultSampleType.canBeShared() { // defaultSampleType may be shared by pointer. Clone it to have exclusive ownership. - s.defaultSampleType = s.defaultSampleType.Clone(&Allocators{}) + s.defaultSampleType = s.defaultSampleType.clone(&Allocators{}) } s.defaultSampleType.mutateRandom(random, schem, limiter) diff --git a/examples/profile/internal/profile/sample.go b/examples/profile/internal/profile/sample.go index 93f155ee..16303a91 100644 --- a/examples/profile/internal/profile/sample.go +++ b/examples/profile/internal/profile/sample.go @@ -98,6 +98,16 @@ func (s *Sample) fixParent(parentModifiedFields *modifiedFields) { s.labels.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Sample) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.metadata.attachParent(&s.modifiedFields, fieldModifiedSampleMetadata) + s.locations.attachParent(&s.modifiedFields, fieldModifiedSampleLocations) + s.values.attachParent(&s.modifiedFields, fieldModifiedSampleValues) + s.labels.attachParent(&s.modifiedFields, fieldModifiedSampleLabels) +} + func (s *Sample) freeze() { if s.isFrozen() { return @@ -205,6 +215,16 @@ func (s *Sample) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Sample) clearModifiedRecursively() { + s.metadata.clearModifiedRecursively() + s.locations.clearModifiedRecursively() + s.values.clearModifiedRecursively() + s.labels.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Sample) computeDiff(val *Sample) (ret bool) { @@ -236,19 +256,16 @@ func (s *Sample) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Sample) cloneShared(allocators *Allocators) Sample { - return s.Clone(allocators) -} - -func (s *Sample) Clone(allocators *Allocators) Sample { - c := Sample{} - copyToNewProfileMetadata(&c.metadata, &s.metadata, allocators) - copyToNewLocationArray(&c.locations, &s.locations, allocators) - copyToNewSampleValueArray(&c.values, &s.values, allocators) - copyToNewLabels(&c.labels, &s.labels, allocators) - return c +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Sample) CloneTo(dst *Sample, allocators *Allocators) { + *dst = Sample{} + copyToNewProfileMetadata(&dst.metadata, &s.metadata, allocators) + copyToNewLocationArray(&dst.locations, &s.locations, allocators) + copyToNewSampleValueArray(&dst.values, &s.values, allocators) + copyToNewLabels(&dst.labels, &s.labels, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/examples/profile/internal/profile/samplevalue.go b/examples/profile/internal/profile/samplevalue.go index 2238a010..50d54ca1 100644 --- a/examples/profile/internal/profile/samplevalue.go +++ b/examples/profile/internal/profile/samplevalue.go @@ -86,6 +86,15 @@ func (s *SampleValue) fixParent(parentModifiedFields *modifiedFields) { s.type_.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *SampleValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + if !s.type_.canBeShared() { + s.type_.attachParent(&s.modifiedFields, fieldModifiedSampleValueType) + } +} + func (s *SampleValue) freeze() { if s.isFrozen() { return @@ -138,7 +147,7 @@ func (s *SampleValue) SetType(v *SampleValueType) { } } else { if s.type_.canBeShared() { - s.type_ = s.type_.Clone(&Allocators{}) + s.type_ = s.type_.clone(&Allocators{}) } s.type_.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedSampleValueType) @@ -171,6 +180,15 @@ func (s *SampleValue) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *SampleValue) clearModifiedRecursively() { + if !s.type_.canBeShared() { + s.type_.clearModifiedRecursively() + } + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *SampleValue) computeDiff(val *SampleValue) (ret bool) { @@ -192,18 +210,15 @@ func (s *SampleValue) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *SampleValue) cloneShared(allocators *Allocators) SampleValue { - return s.Clone(allocators) -} - -func (s *SampleValue) Clone(allocators *Allocators) SampleValue { - c := SampleValue{ - val: s.val, - type_: s.type_.cloneShared(allocators), +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *SampleValue) CloneTo(dst *SampleValue, allocators *Allocators) { + *dst = SampleValue{ + val: s.val, } - return c + dst.type_ = s.type_.cloneShared(allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -287,12 +302,12 @@ func (s *SampleValue) mutateRandom(random *rand.Rand, schem *schema.Schema, limi s.type_ = new(SampleValueType) s.type_.init(&s.modifiedFields, fieldModifiedSampleValueType) } else { - s.type_ = s.type_.Clone(&Allocators{}) + s.type_ = s.type_.clone(&Allocators{}) } } if s.type_.canBeShared() { // type_ may be shared by pointer. Clone it to have exclusive ownership. - s.type_ = s.type_.Clone(&Allocators{}) + s.type_ = s.type_.clone(&Allocators{}) } s.type_.mutateRandom(random, schem, limiter) diff --git a/examples/profile/internal/profile/samplevaluearray.go b/examples/profile/internal/profile/samplevaluearray.go index a441d282..d9cd2d68 100644 --- a/examples/profile/internal/profile/samplevaluearray.go +++ b/examples/profile/internal/profile/samplevaluearray.go @@ -52,6 +52,15 @@ func (e *SampleValueArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *SampleValueArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *SampleValueArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *SampleValueArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *SampleValueArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *SampleValueArray) computeDiff(val *SampleValueArray) (ret bool) { diff --git a/examples/profile/internal/profile/samplevaluetype.go b/examples/profile/internal/profile/samplevaluetype.go index 42e7387a..85e1411d 100644 --- a/examples/profile/internal/profile/samplevaluetype.go +++ b/examples/profile/internal/profile/samplevaluetype.go @@ -92,6 +92,12 @@ func (s *SampleValueType) fixParent(parentModifiedFields *modifiedFields) { s.modifiedFields.parent = parentModifiedFields } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *SampleValueType) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit +} + // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, // which can improve encoding performance. @@ -165,6 +171,12 @@ func (s *SampleValueType) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *SampleValueType) clearModifiedRecursively() { + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *SampleValueType) computeDiff(val *SampleValueType) (ret bool) { @@ -186,6 +198,14 @@ func (s *SampleValueType) canBeShared() bool { return s.isFrozen() } +// clone returns a clone of s allocated using allocators. +func (s *SampleValueType) clone(allocators *Allocators) *SampleValueType { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(SampleValueType{}))) + c := allocators.SampleValueType.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). func (s *SampleValueType) cloneShared(allocators *Allocators) *SampleValueType { @@ -193,17 +213,18 @@ func (s *SampleValueType) cloneShared(allocators *Allocators) *SampleValueType { // If s is frozen it means it is safe to share without cloning. return s } - return s.Clone(allocators) + return s.clone(allocators) } -func (s *SampleValueType) Clone(allocators *Allocators) *SampleValueType { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(SampleValueType{}))) - c := allocators.SampleValueType.Alloc() - *c = SampleValueType{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *SampleValueType) CloneTo(dst *SampleValueType, allocators *Allocators) { + *dst = SampleValueType{ type_: s.type_, unit: s.unit, } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -357,7 +378,7 @@ func (d *SampleValueTypeEncoderDict) Add(val *SampleValueType) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -609,7 +630,7 @@ func (d *SampleValueTypeDecoder) Decode(dstPtr **SampleValueType) error { // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } diff --git a/examples/profile/internal/profile/stringarray.go b/examples/profile/internal/profile/stringarray.go index 2511d8fb..8e84370f 100644 --- a/examples/profile/internal/profile/stringarray.go +++ b/examples/profile/internal/profile/stringarray.go @@ -51,6 +51,12 @@ func (e *StringArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *StringArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit +} + func (e *StringArray) canBeShared() bool { // An array can never be shared. return false @@ -97,6 +103,11 @@ func (e *StringArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *StringArray) clearModifiedRecursively() { + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *StringArray) computeDiff(val *StringArray) (ret bool) { diff --git a/go/otel/otel.stef b/go/otel/otel.stef index 644dca23..76e8b38e 100644 --- a/go/otel/otel.stef +++ b/go/otel/otel.stef @@ -1,7 +1,7 @@ // This is Otel/STEF schema: a representation of OpenTelemetry data model // as STEF records. Data model is virtually a direct mapping from OpenTelemetry -// Protobuf IDL for metrics and traces, see: https://github.com/open-telemetry/opentelemetry-proto/tree/main/opentelemetry/proto -// TODO: add Logs and Profiles. +// Protobuf IDL for metrics, logs and traces, see: https://github.com/open-telemetry/opentelemetry-proto/tree/main/opentelemetry/proto +// TODO: add Profiles. package com.example.otelstef @@ -213,3 +213,56 @@ struct SpanStatus { Message string Code uint64 } + +//////////////////////////////////////////////////////////////// +// Log signal schema. +//////////////////////////////////////////////////////////////// + +struct Logs root { + Envelope Envelope + Resource Resource + Scope Scope + Log LogRecord +} + +struct LogRecord { + TimeUnixNano uint64 + ObservedTimeUnixNano uint64 + SeverityNumber SeverityNumber + SeverityText string dict(LogSeverityText) + Body AnyValue + Attributes Attributes + DroppedAttributesCount uint64 + Flags uint64 + TraceID bytes + SpanID bytes + EventName string dict(LogEventName) +} + +enum SeverityNumber { + Unspecified = 0 + Trace = 1 + Trace2 = 2 + Trace3 = 3 + Trace4 = 4 + Debug = 5 + Debug2 = 6 + Debug3 = 7 + Debug4 = 8 + Info = 9 + Info2 = 10 + Info3 = 11 + Info4 = 12 + Warn = 13 + Warn2 = 14 + Warn3 = 15 + Warn4 = 16 + Error = 17 + Error2 = 18 + Error3 = 19 + Error4 = 20 + Fatal = 21 + Fatal2 = 22 + Fatal3 = 23 + Fatal4 = 24 +} diff --git a/go/otel/otelstef/anyvalue.go b/go/otel/otelstef/anyvalue.go index 8920c8a6..d173016e 100644 --- a/go/otel/otelstef/anyvalue.go +++ b/go/otel/otelstef/anyvalue.go @@ -82,6 +82,19 @@ func (s *AnyValue) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointer and bit in a newly copied oneof. +func (s *AnyValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.parentModifiedFields = parentModifiedFields + s.parentModifiedBit = parentModifiedBit + + switch s.Type() { + case AnyValueTypeArray: + s.arrayPtr().attachParent(parentModifiedFields, parentModifiedBit) + case AnyValueTypeKVList: + s.kVListPtr().attachParent(parentModifiedFields, parentModifiedBit) + } +} + type AnyValueType byte const ( @@ -287,36 +300,45 @@ func (s *AnyValue) canBeShared() bool { return false } +func (s *AnyValue) clone(allocators *Allocators) *AnyValue { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(AnyValue{}))) + c := allocators.AnyValue.Alloc() + s.CloneTo(c, allocators) + return c +} + func (s *AnyValue) cloneShared(allocators *Allocators) *AnyValue { - // Oneof is not shareable, so cloneShared is just a Clone. - return s.Clone(allocators) + // Oneof is not shareable, so cloneShared is just a clone. + return s.clone(allocators) } -func (s *AnyValue) Clone(allocators *Allocators) *AnyValue { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(AnyValue{}))) - c := allocators.AnyValue.Alloc() - c.clearValSetType(s.Type()) +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *AnyValue) CloneTo(dst *AnyValue, allocators *Allocators) { + *dst = AnyValue{} + dst.clearValSetType(s.Type()) switch s.Type() { case AnyValueTypeString: - c.setString(s.String()) + dst.setString(s.String()) case AnyValueTypeBool: - *c.boolPtr() = *s.boolPtr() + *dst.boolPtr() = *s.boolPtr() case AnyValueTypeInt64: - *c.int64Ptr() = *s.int64Ptr() + *dst.int64Ptr() = *s.int64Ptr() case AnyValueTypeFloat64: - *c.float64Ptr() = *s.float64Ptr() + *dst.float64Ptr() = *s.float64Ptr() case AnyValueTypeArray: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(AnyValueArray{}))) - c.allocArrayAlloc(allocators) - copyToNewAnyValueArray(c.arrayPtr(), s.arrayPtr(), allocators) + dst.allocArrayAlloc(allocators) + copyToNewAnyValueArray(dst.arrayPtr(), s.arrayPtr(), allocators) case AnyValueTypeKVList: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(KeyValueList{}))) - c.allocKVListAlloc(allocators) - copyToNewKeyValueList(c.kVListPtr(), s.kVListPtr(), allocators) + dst.allocKVListAlloc(allocators) + copyToNewKeyValueList(dst.kVListPtr(), s.kVListPtr(), allocators) case AnyValueTypeBytes: - c.setBytes(s.Bytes()) + dst.setBytes(s.Bytes()) } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -433,6 +455,16 @@ func (s *AnyValue) setUnmodifiedRecursively() { } } +// clearModifiedRecursively clears modification state in all mutable descendants. +func (s *AnyValue) clearModifiedRecursively() { + switch s.Type() { + case AnyValueTypeArray: + s.arrayPtr().clearModifiedRecursively() + case AnyValueTypeKVList: + s.kVListPtr().clearModifiedRecursively() + } +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *AnyValue) computeDiff(val *AnyValue) (ret bool) { diff --git a/go/otel/otelstef/anyvaluearray.go b/go/otel/otelstef/anyvaluearray.go index f234dd27..8451f48e 100644 --- a/go/otel/otelstef/anyvaluearray.go +++ b/go/otel/otelstef/anyvaluearray.go @@ -52,6 +52,15 @@ func (e *AnyValueArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *AnyValueArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *AnyValueArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *AnyValueArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *AnyValueArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *AnyValueArray) computeDiff(val *AnyValueArray) (ret bool) { diff --git a/go/otel/otelstef/attributes.go b/go/otel/otelstef/attributes.go index 839f3660..b8017d14 100644 --- a/go/otel/otelstef/attributes.go +++ b/go/otel/otelstef/attributes.go @@ -60,6 +60,15 @@ func (m *Attributes) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointers and bits in a newly copied multimap. +func (m *Attributes) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.modifiedElems.attachParent(parentModifiedFields, parentModifiedBit) + elems := m.elems[:m.initedCount] + for i := 0; i < len(elems); i++ { + elems[i].value.attachParent(&m.modifiedElems.vals, m.modifiedElems.maskForIndex(i)) + } +} + func (m *Attributes) canBeShared() bool { // Multimap can never be shared. return false @@ -122,6 +131,14 @@ func (m *Attributes) setUnmodifiedRecursively() { m.modifiedElems.setUnmodifiedAll() } +// clearModifiedRecursively clears modification state in all mutable elements. +func (m *Attributes) clearModifiedRecursively() { + for i := 0; i < len(m.elems); i++ { + m.elems[i].value.clearModifiedRecursively() + } + m.modifiedElems.setUnmodifiedAll() +} + // computeDiff compares m to val and returns true if they differ. // All fields that are different in m will be marked as modified. func (m *Attributes) computeDiff(val *Attributes) (ret bool) { diff --git a/go/otel/otelstef/clone_test.go b/go/otel/otelstef/clone_test.go new file mode 100644 index 00000000..e6d41383 --- /dev/null +++ b/go/otel/otelstef/clone_test.go @@ -0,0 +1,74 @@ +package otelstef + +import ( + "bytes" + "testing" + + "github.com/splunk/stef/go/pkg" +) + +func TestLogsCloneTracksExistingAttributeValueModifications(t *testing.T) { + var source Logs + source.Init() + source.Log().Attributes().EnsureLen(1) + source.Log().Attributes().SetKey(0, "key") + source.Log().Attributes().Value(0).SetInt64(1) + + var allocators Allocators + var copy Logs + source.CloneTo(©, &allocators) + + if copy.IsLogModified() || copy.Log().IsAttributesModified() { + t.Fatal("clone is unexpectedly marked modified") + } + + copy.Log().Attributes().Value(0).SetInt64(2) + + if !copy.IsLogModified() || !copy.Log().IsAttributesModified() { + t.Fatal("attribute value modification did not propagate through clone") + } +} + +func TestLogsCloneModificationSurvivesEncoding(t *testing.T) { + var buf pkg.MemChunkWriter + writer, err := NewLogsWriter(&buf, pkg.WriterOptions{}) + if err != nil { + t.Fatal(err) + } + + writer.Record.Log().Attributes().EnsureLen(1) + writer.Record.Log().Attributes().SetKey(0, "key") + writer.Record.Log().Attributes().Value(0).SetInt64(1) + if err := writer.Write(); err != nil { + t.Fatal(err) + } + + var allocators Allocators + var cloned Logs + writer.Record.CloneTo(&cloned, &allocators) + cloned.Log().Attributes().Value(0).SetInt64(2) + writer.Record.CopyFrom(&cloned) + if err := writer.Write(); err != nil { + t.Fatal(err) + } + if err := writer.Flush(); err != nil { + t.Fatal(err) + } + + reader, err := NewLogsReader(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatal(err) + } + if err := reader.Read(pkg.ReadOptions{}); err != nil { + t.Fatal(err) + } + if got := reader.Record.Log().Attributes().Value(0).Int64(); got != 1 { + t.Fatalf("first attribute value = %d, want 1", got) + } + if err := reader.Read(pkg.ReadOptions{}); err != nil { + t.Fatal(err) + } + if got := reader.Record.Log().Attributes().Value(0).Int64(); got != 2 { + t.Fatalf("cloned attribute value after encoding = %d, want 2", got) + } +} diff --git a/go/otel/otelstef/common.go b/go/otel/otelstef/common.go index b64af1a5..a62207be 100644 --- a/go/otel/otelstef/common.go +++ b/go/otel/otelstef/common.go @@ -10,7 +10,7 @@ import ( ) // The STEF IDL string. Used for testing. -const allSchemaContent = "// This is Otel/STEF schema: a representation of OpenTelemetry data model\n// as STEF records. Data model is virtually a direct mapping from OpenTelemetry\n// Protobuf IDL for metrics and traces, see: https://github.com/open-telemetry/opentelemetry-proto/tree/main/opentelemetry/proto\n// TODO: add Logs and Profiles.\n\npackage com.example.otelstef\n\n////////////////////////////////////////////////////////////////\n// Schema that is common across all signals.\n////////////////////////////////////////////////////////////////\n\nmultimap Attributes {\n key string dict(AttributeKey)\n value AnyValue\n}\n\noneof AnyValue {\n String string dict(AnyValueString)\n Bool bool\n Int64 int64\n Float64 float64\n Array []AnyValue\n KVList KeyValueList\n Bytes bytes\n}\n\nmultimap KeyValueList {\n key string\n value AnyValue\n}\n\nstruct Resource dict(Resource) {\n // All SchemaURL fields use the same (shared) dict.\n SchemaURL string dict(SchemaURL)\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct Scope dict(Scope) {\n Name string dict(ScopeName)\n Version string dict(ScopeVersion)\n SchemaURL string dict(SchemaURL)\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct Envelope {\n Attributes EnvelopeAttributes\n}\n\nmultimap EnvelopeAttributes {\n key string\n value bytes\n}\n\n////////////////////////////////////////////////////////////////\n// Metric signal schema.\n////////////////////////////////////////////////////////////////\n\nstruct Metrics root {\n Envelope Envelope\n Metric Metric\n Resource Resource\n Scope Scope\n Attributes Attributes\n Point Point\n}\n\nstruct Metric dict(Metric) {\n Name string dict(MetricName)\n Description string dict(MetricDescription)\n Unit string dict(MetricUnit)\n Type MetricType\n Metadata Attributes\n HistogramBounds []float64\n AggregationTemporality AggregationTemporality\n Monotonic bool\n}\n\nenum MetricType {\n\tGauge = 0\n\tSum = 1\n\tHistogram = 2\n\tExpHistogram = 3\n\tSummary = 4\n}\n\nenum AggregationTemporality {\n Unspecified = 0\n Delta = 1\n Cumulative = 2\n}\n\n// Point represents a metric data point.\nstruct Point {\n StartTimestamp uint64\n Timestamp uint64\n Value PointValue\n Exemplars []Exemplar\n}\n\noneof PointValue {\n Int64 int64\n Float64 float64\n Histogram HistogramValue\n ExpHistogram ExpHistogramValue\n Summary SummaryValue\n}\n\nstruct HistogramValue {\n Count int64\n Sum float64 optional\n Min float64 optional\n Max float64 optional\n BucketCounts []uint64\n}\n\nstruct ExpHistogramValue {\n Count uint64\n Sum float64 optional\n Min float64 optional\n Max float64 optional\n Scale int64\n ZeroCount uint64\n PositiveBuckets ExpHistogramBuckets\n NegativeBuckets ExpHistogramBuckets\n ZeroThreshold float64\n}\n\nstruct ExpHistogramBuckets {\n Offset int64\n BucketCounts []uint64\n}\n\nstruct SummaryValue {\n Count uint64\n Sum float64\n QuantileValues []QuantileValue\n}\n\nstruct QuantileValue {\n Quantile float64\n Value float64\n}\n\nstruct Exemplar {\n Timestamp uint64\n Value ExemplarValue\n SpanID bytes\n TraceID bytes\n FilteredAttributes Attributes\n}\n\noneof ExemplarValue {\n Int64 int64\n Float64 float64\n}\n\n////////////////////////////////////////////////////////////////\n// Trace signal schema.\n////////////////////////////////////////////////////////////////\n\nstruct Spans root {\n Envelope Envelope\n Resource Resource\n Scope Scope\n Span Span\n}\n\nstruct Span {\n TraceID bytes\n SpanID bytes\n TraceState string\n ParentSpanID bytes\n Flags uint64\n Name string dict(SpanName)\n Kind SpanKind\n StartTimeUnixNano uint64\n EndTimeUnixNano uint64\n Attributes Attributes\n DroppedAttributesCount uint64\n Events []Event\n Links []Link\n Status SpanStatus\n}\n\nenum SpanKind {\n Unspecified = 0\n Internal = 1\n Server = 2\n Client = 3\n Producer = 4\n Consumer = 5\n}\n\nstruct Link {\n TraceID bytes\n SpanID bytes\n TraceState string\n Flags uint64\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct Event {\n Name string dict(SpanEventName)\n TimeUnixNano uint64\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct SpanStatus {\n Message string\n Code uint64\n}\n" +const allSchemaContent = "// This is Otel/STEF schema: a representation of OpenTelemetry data model\n// as STEF records. Data model is virtually a direct mapping from OpenTelemetry\n// Protobuf IDL for metrics, logs and traces, see: https://github.com/open-telemetry/opentelemetry-proto/tree/main/opentelemetry/proto\n// TODO: add Profiles.\n\npackage com.example.otelstef\n\n////////////////////////////////////////////////////////////////\n// Schema that is common across all signals.\n////////////////////////////////////////////////////////////////\n\nmultimap Attributes {\n key string dict(AttributeKey)\n value AnyValue\n}\n\noneof AnyValue {\n String string dict(AnyValueString)\n Bool bool\n Int64 int64\n Float64 float64\n Array []AnyValue\n KVList KeyValueList\n Bytes bytes\n}\n\nmultimap KeyValueList {\n key string\n value AnyValue\n}\n\nstruct Resource dict(Resource) {\n // All SchemaURL fields use the same (shared) dict.\n SchemaURL string dict(SchemaURL)\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct Scope dict(Scope) {\n Name string dict(ScopeName)\n Version string dict(ScopeVersion)\n SchemaURL string dict(SchemaURL)\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct Envelope {\n Attributes EnvelopeAttributes\n}\n\nmultimap EnvelopeAttributes {\n key string\n value bytes\n}\n\n////////////////////////////////////////////////////////////////\n// Metric signal schema.\n////////////////////////////////////////////////////////////////\n\nstruct Metrics root {\n Envelope Envelope\n Metric Metric\n Resource Resource\n Scope Scope\n Attributes Attributes\n Point Point\n}\n\nstruct Metric dict(Metric) {\n Name string dict(MetricName)\n Description string dict(MetricDescription)\n Unit string dict(MetricUnit)\n Type MetricType\n Metadata Attributes\n HistogramBounds []float64\n AggregationTemporality AggregationTemporality\n Monotonic bool\n}\n\nenum MetricType {\n\tGauge = 0\n\tSum = 1\n\tHistogram = 2\n\tExpHistogram = 3\n\tSummary = 4\n}\n\nenum AggregationTemporality {\n Unspecified = 0\n Delta = 1\n Cumulative = 2\n}\n\n// Point represents a metric data point.\nstruct Point {\n StartTimestamp uint64\n Timestamp uint64\n Value PointValue\n Exemplars []Exemplar\n}\n\noneof PointValue {\n Int64 int64\n Float64 float64\n Histogram HistogramValue\n ExpHistogram ExpHistogramValue\n Summary SummaryValue\n}\n\nstruct HistogramValue {\n Count int64\n Sum float64 optional\n Min float64 optional\n Max float64 optional\n BucketCounts []uint64\n}\n\nstruct ExpHistogramValue {\n Count uint64\n Sum float64 optional\n Min float64 optional\n Max float64 optional\n Scale int64\n ZeroCount uint64\n PositiveBuckets ExpHistogramBuckets\n NegativeBuckets ExpHistogramBuckets\n ZeroThreshold float64\n}\n\nstruct ExpHistogramBuckets {\n Offset int64\n BucketCounts []uint64\n}\n\nstruct SummaryValue {\n Count uint64\n Sum float64\n QuantileValues []QuantileValue\n}\n\nstruct QuantileValue {\n Quantile float64\n Value float64\n}\n\nstruct Exemplar {\n Timestamp uint64\n Value ExemplarValue\n SpanID bytes\n TraceID bytes\n FilteredAttributes Attributes\n}\n\noneof ExemplarValue {\n Int64 int64\n Float64 float64\n}\n\n////////////////////////////////////////////////////////////////\n// Trace signal schema.\n////////////////////////////////////////////////////////////////\n\nstruct Spans root {\n Envelope Envelope\n Resource Resource\n Scope Scope\n Span Span\n}\n\nstruct Span {\n TraceID bytes\n SpanID bytes\n TraceState string\n ParentSpanID bytes\n Flags uint64\n Name string dict(SpanName)\n Kind SpanKind\n StartTimeUnixNano uint64\n EndTimeUnixNano uint64\n Attributes Attributes\n DroppedAttributesCount uint64\n Events []Event\n Links []Link\n Status SpanStatus\n}\n\nenum SpanKind {\n Unspecified = 0\n Internal = 1\n Server = 2\n Client = 3\n Producer = 4\n Consumer = 5\n}\n\nstruct Link {\n TraceID bytes\n SpanID bytes\n TraceState string\n Flags uint64\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct Event {\n Name string dict(SpanEventName)\n TimeUnixNano uint64\n Attributes Attributes\n DroppedAttributesCount uint64\n}\n\nstruct SpanStatus {\n Message string\n Code uint64\n}\n\n////////////////////////////////////////////////////////////////\n// Log signal schema.\n////////////////////////////////////////////////////////////////\n\nstruct Logs root {\n Envelope Envelope\n Resource Resource\n Scope Scope\n Log LogRecord\n}\n\nstruct LogRecord {\n TimeUnixNano uint64\n ObservedTimeUnixNano uint64\n SeverityNumber SeverityNumber\n SeverityText string dict(LogSeverityText)\n Body AnyValue\n Attributes Attributes\n DroppedAttributesCount uint64\n Flags uint64\n TraceID bytes\n SpanID bytes\n EventName string dict(LogEventName)\n}\n\nenum SeverityNumber {\n Unspecified = 0\n Trace = 1\n Trace2 = 2\n Trace3 = 3\n Trace4 = 4\n Debug = 5\n Debug2 = 6\n Debug3 = 7\n Debug4 = 8\n Info = 9\n Info2 = 10\n Info3 = 11\n Info4 = 12\n Warn = 13\n Warn2 = 14\n Warn3 = 15\n Warn4 = 16\n Error = 17\n Error2 = 18\n Error3 = 19\n Error4 = 20\n Fatal = 21\n Fatal2 = 22\n Fatal3 = 23\n Fatal4 = 24\n}\n" // StructFieldCounts is used to keep track of the number of fields in each struct // of an associated schema and to iterate over the fields. @@ -27,6 +27,8 @@ type StructFieldCounts struct { countExpHistogramValue uint // Number of fields in ExpHistogramValue struct. countHistogramValue uint // Number of fields in HistogramValue struct. countLink uint // Number of fields in Link struct. + countLogRecord uint // Number of fields in LogRecord struct. + countLogs uint // Number of fields in Logs struct. countMetric uint // Number of fields in Metric struct. countMetrics uint // Number of fields in Metrics struct. countPoint uint // Number of fields in Point struct. @@ -61,6 +63,8 @@ func (s *StructFieldCounts) Init(overrideSchema *schema.WireSchema) { s.countExpHistogramValue = math.MaxUint s.countHistogramValue = math.MaxUint s.countLink = math.MaxUint + s.countLogRecord = math.MaxUint + s.countLogs = math.MaxUint s.countMetric = math.MaxUint s.countMetrics = math.MaxUint s.countPoint = math.MaxUint @@ -134,6 +138,14 @@ func (s *StructFieldCounts) LinkFieldCount() (uint, error) { return getFieldCount(&s.countLink, "Link", s.overrideSchema, &s.overrideSchemaIter, 6) } +func (s *StructFieldCounts) LogRecordFieldCount() (uint, error) { + return getFieldCount(&s.countLogRecord, "LogRecord", s.overrideSchema, &s.overrideSchemaIter, 11) +} + +func (s *StructFieldCounts) LogsFieldCount() (uint, error) { + return getFieldCount(&s.countLogs, "Logs", s.overrideSchema, &s.overrideSchemaIter, 4) +} + func (s *StructFieldCounts) MetricFieldCount() (uint, error) { return getFieldCount(&s.countMetric, "Metric", s.overrideSchema, &s.overrideSchemaIter, 8) } @@ -192,6 +204,8 @@ type Allocators struct { ExpHistogramValue ExpHistogramValueAllocator HistogramValue HistogramValueAllocator Link LinkAllocator + LogRecord LogRecordAllocator + Logs LogsAllocator Metric MetricAllocator Metrics MetricsAllocator Point PointAllocator diff --git a/go/otel/otelstef/envelope.go b/go/otel/otelstef/envelope.go index 2862a67a..05ae8e56 100644 --- a/go/otel/otelstef/envelope.go +++ b/go/otel/otelstef/envelope.go @@ -80,6 +80,13 @@ func (s *Envelope) fixParent(parentModifiedFields *modifiedFields) { s.attributes.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Envelope) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.attributes.attachParent(&s.modifiedFields, fieldModifiedEnvelopeAttributes) +} + func (s *Envelope) freeze() { if s.isFrozen() { return @@ -121,6 +128,13 @@ func (s *Envelope) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Envelope) clearModifiedRecursively() { + s.attributes.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Envelope) computeDiff(val *Envelope) (ret bool) { @@ -137,16 +151,13 @@ func (s *Envelope) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Envelope) cloneShared(allocators *Allocators) Envelope { - return s.Clone(allocators) -} - -func (s *Envelope) Clone(allocators *Allocators) Envelope { - c := Envelope{} - copyToNewEnvelopeAttributes(&c.attributes, &s.attributes, allocators) - return c +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Envelope) CloneTo(dst *Envelope, allocators *Allocators) { + *dst = Envelope{} + copyToNewEnvelopeAttributes(&dst.attributes, &s.attributes, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/envelopeattributes.go b/go/otel/otelstef/envelopeattributes.go index df7ef2ea..266eec9a 100644 --- a/go/otel/otelstef/envelopeattributes.go +++ b/go/otel/otelstef/envelopeattributes.go @@ -58,6 +58,14 @@ func (m *EnvelopeAttributes) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointers and bits in a newly copied multimap. +func (m *EnvelopeAttributes) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.modifiedElems.attachParent(parentModifiedFields, parentModifiedBit) + elems := m.elems[:m.initedCount] + for i := 0; i < len(elems); i++ { + } +} + func (m *EnvelopeAttributes) canBeShared() bool { // Multimap can never be shared. return false @@ -113,6 +121,13 @@ func (m *EnvelopeAttributes) setUnmodifiedRecursively() { m.modifiedElems.setUnmodifiedAll() } +// clearModifiedRecursively clears modification state in all mutable elements. +func (m *EnvelopeAttributes) clearModifiedRecursively() { + for i := 0; i < len(m.elems); i++ { + } + m.modifiedElems.setUnmodifiedAll() +} + // computeDiff compares m to val and returns true if they differ. // All fields that are different in m will be marked as modified. func (m *EnvelopeAttributes) computeDiff(val *EnvelopeAttributes) (ret bool) { diff --git a/go/otel/otelstef/event.go b/go/otel/otelstef/event.go index f371c7ab..f6e75f90 100644 --- a/go/otel/otelstef/event.go +++ b/go/otel/otelstef/event.go @@ -89,6 +89,13 @@ func (s *Event) fixParent(parentModifiedFields *modifiedFields) { s.attributes.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Event) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.attributes.attachParent(&s.modifiedFields, fieldModifiedEventAttributes) +} + func (s *Event) freeze() { if s.isFrozen() { return @@ -205,6 +212,13 @@ func (s *Event) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Event) clearModifiedRecursively() { + s.attributes.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Event) computeDiff(val *Event) (ret bool) { @@ -236,20 +250,17 @@ func (s *Event) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Event) cloneShared(allocators *Allocators) Event { - return s.Clone(allocators) -} - -func (s *Event) Clone(allocators *Allocators) Event { - c := Event{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Event) CloneTo(dst *Event, allocators *Allocators) { + *dst = Event{ name: s.name, timeUnixNano: s.timeUnixNano, droppedAttributesCount: s.droppedAttributesCount, } - copyToNewAttributes(&c.attributes, &s.attributes, allocators) - return c + copyToNewAttributes(&dst.attributes, &s.attributes, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/eventarray.go b/go/otel/otelstef/eventarray.go index 3b0c59b2..e328e599 100644 --- a/go/otel/otelstef/eventarray.go +++ b/go/otel/otelstef/eventarray.go @@ -52,6 +52,15 @@ func (e *EventArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *EventArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *EventArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *EventArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *EventArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *EventArray) computeDiff(val *EventArray) (ret bool) { diff --git a/go/otel/otelstef/exemplar.go b/go/otel/otelstef/exemplar.go index bd7ea3fb..8dce6782 100644 --- a/go/otel/otelstef/exemplar.go +++ b/go/otel/otelstef/exemplar.go @@ -95,6 +95,14 @@ func (s *Exemplar) fixParent(parentModifiedFields *modifiedFields) { s.filteredAttributes.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Exemplar) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.value.attachParent(&s.modifiedFields, fieldModifiedExemplarValue) + s.filteredAttributes.attachParent(&s.modifiedFields, fieldModifiedExemplarFilteredAttributes) +} + func (s *Exemplar) freeze() { if s.isFrozen() { return @@ -233,6 +241,14 @@ func (s *Exemplar) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Exemplar) clearModifiedRecursively() { + s.value.clearModifiedRecursively() + s.filteredAttributes.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Exemplar) computeDiff(val *Exemplar) (ret bool) { @@ -269,21 +285,18 @@ func (s *Exemplar) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Exemplar) cloneShared(allocators *Allocators) Exemplar { - return s.Clone(allocators) -} - -func (s *Exemplar) Clone(allocators *Allocators) Exemplar { - c := Exemplar{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Exemplar) CloneTo(dst *Exemplar, allocators *Allocators) { + *dst = Exemplar{ timestamp: s.timestamp, spanID: s.spanID, traceID: s.traceID, } - copyToNewExemplarValue(&c.value, &s.value, allocators) - copyToNewAttributes(&c.filteredAttributes, &s.filteredAttributes, allocators) - return c + copyToNewExemplarValue(&dst.value, &s.value, allocators) + copyToNewAttributes(&dst.filteredAttributes, &s.filteredAttributes, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/exemplararray.go b/go/otel/otelstef/exemplararray.go index 7750b267..d5d6103e 100644 --- a/go/otel/otelstef/exemplararray.go +++ b/go/otel/otelstef/exemplararray.go @@ -52,6 +52,15 @@ func (e *ExemplarArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *ExemplarArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *ExemplarArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *ExemplarArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *ExemplarArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *ExemplarArray) computeDiff(val *ExemplarArray) (ret bool) { diff --git a/go/otel/otelstef/exemplarvalue.go b/go/otel/otelstef/exemplarvalue.go index ff9d6c87..11e9dab1 100644 --- a/go/otel/otelstef/exemplarvalue.go +++ b/go/otel/otelstef/exemplarvalue.go @@ -73,6 +73,15 @@ func (s *ExemplarValue) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointer and bit in a newly copied oneof. +func (s *ExemplarValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.parentModifiedFields = parentModifiedFields + s.parentModifiedBit = parentModifiedBit + + switch s.Type() { + } +} + type ExemplarValueType byte const ( @@ -159,21 +168,19 @@ func (s *ExemplarValue) canBeShared() bool { return false } -func (s *ExemplarValue) cloneShared(allocators *Allocators) ExemplarValue { - // Oneof is not shareable, so cloneShared is just a Clone. - return s.Clone(allocators) -} - -func (s *ExemplarValue) Clone(allocators *Allocators) ExemplarValue { - c := ExemplarValue{} - c.clearValSetType(s.Type()) +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *ExemplarValue) CloneTo(dst *ExemplarValue, allocators *Allocators) { + *dst = ExemplarValue{} + dst.clearValSetType(s.Type()) switch s.Type() { case ExemplarValueTypeInt64: - *c.int64Ptr() = *s.int64Ptr() + *dst.int64Ptr() = *s.int64Ptr() case ExemplarValueTypeFloat64: - *c.float64Ptr() = *s.float64Ptr() + *dst.float64Ptr() = *s.float64Ptr() } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -241,6 +248,12 @@ func (s *ExemplarValue) setUnmodifiedRecursively() { } } +// clearModifiedRecursively clears modification state in all mutable descendants. +func (s *ExemplarValue) clearModifiedRecursively() { + switch s.Type() { + } +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *ExemplarValue) computeDiff(val *ExemplarValue) (ret bool) { diff --git a/go/otel/otelstef/exphistogrambuckets.go b/go/otel/otelstef/exphistogrambuckets.go index dc06b7a6..b4eb297a 100644 --- a/go/otel/otelstef/exphistogrambuckets.go +++ b/go/otel/otelstef/exphistogrambuckets.go @@ -83,6 +83,13 @@ func (s *ExpHistogramBuckets) fixParent(parentModifiedFields *modifiedFields) { s.bucketCounts.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *ExpHistogramBuckets) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.bucketCounts.attachParent(&s.modifiedFields, fieldModifiedExpHistogramBucketsBucketCounts) +} + func (s *ExpHistogramBuckets) freeze() { if s.isFrozen() { return @@ -149,6 +156,13 @@ func (s *ExpHistogramBuckets) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *ExpHistogramBuckets) clearModifiedRecursively() { + s.bucketCounts.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *ExpHistogramBuckets) computeDiff(val *ExpHistogramBuckets) (ret bool) { @@ -170,18 +184,15 @@ func (s *ExpHistogramBuckets) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *ExpHistogramBuckets) cloneShared(allocators *Allocators) ExpHistogramBuckets { - return s.Clone(allocators) -} - -func (s *ExpHistogramBuckets) Clone(allocators *Allocators) ExpHistogramBuckets { - c := ExpHistogramBuckets{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *ExpHistogramBuckets) CloneTo(dst *ExpHistogramBuckets, allocators *Allocators) { + *dst = ExpHistogramBuckets{ offset: s.offset, } - copyToNewUint64Array(&c.bucketCounts, &s.bucketCounts, allocators) - return c + copyToNewUint64Array(&dst.bucketCounts, &s.bucketCounts, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/exphistogramvalue.go b/go/otel/otelstef/exphistogramvalue.go index 73151ea8..fabcb785 100644 --- a/go/otel/otelstef/exphistogramvalue.go +++ b/go/otel/otelstef/exphistogramvalue.go @@ -119,6 +119,14 @@ func (s *ExpHistogramValue) fixParent(parentModifiedFields *modifiedFields) { s.negativeBuckets.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *ExpHistogramValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.positiveBuckets.attachParent(&s.modifiedFields, fieldModifiedExpHistogramValuePositiveBuckets) + s.negativeBuckets.attachParent(&s.modifiedFields, fieldModifiedExpHistogramValueNegativeBuckets) +} + func (s *ExpHistogramValue) freeze() { if s.isFrozen() { return @@ -399,6 +407,14 @@ func (s *ExpHistogramValue) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *ExpHistogramValue) clearModifiedRecursively() { + s.positiveBuckets.clearModifiedRecursively() + s.negativeBuckets.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *ExpHistogramValue) computeDiff(val *ExpHistogramValue) (ret bool) { @@ -497,14 +513,10 @@ func (s *ExpHistogramValue) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *ExpHistogramValue) cloneShared(allocators *Allocators) ExpHistogramValue { - return s.Clone(allocators) -} - -func (s *ExpHistogramValue) Clone(allocators *Allocators) ExpHistogramValue { - c := ExpHistogramValue{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *ExpHistogramValue) CloneTo(dst *ExpHistogramValue, allocators *Allocators) { + *dst = ExpHistogramValue{ count: s.count, sum: s.sum, min: s.min, @@ -513,9 +525,11 @@ func (s *ExpHistogramValue) Clone(allocators *Allocators) ExpHistogramValue { zeroCount: s.zeroCount, zeroThreshold: s.zeroThreshold, } - copyToNewExpHistogramBuckets(&c.positiveBuckets, &s.positiveBuckets, allocators) - copyToNewExpHistogramBuckets(&c.negativeBuckets, &s.negativeBuckets, allocators) - return c + copyToNewExpHistogramBuckets(&dst.positiveBuckets, &s.positiveBuckets, allocators) + copyToNewExpHistogramBuckets(&dst.negativeBuckets, &s.negativeBuckets, allocators) + dst.optionalFieldsPresent = s.optionalFieldsPresent + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/float64array.go b/go/otel/otelstef/float64array.go index 5d03bd1c..8b94ac10 100644 --- a/go/otel/otelstef/float64array.go +++ b/go/otel/otelstef/float64array.go @@ -51,6 +51,12 @@ func (e *Float64Array) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *Float64Array) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit +} + func (e *Float64Array) canBeShared() bool { // An array can never be shared. return false @@ -97,6 +103,11 @@ func (e *Float64Array) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *Float64Array) clearModifiedRecursively() { + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *Float64Array) computeDiff(val *Float64Array) (ret bool) { diff --git a/go/otel/otelstef/histogramvalue.go b/go/otel/otelstef/histogramvalue.go index 18a9f9a1..f14534cc 100644 --- a/go/otel/otelstef/histogramvalue.go +++ b/go/otel/otelstef/histogramvalue.go @@ -104,6 +104,13 @@ func (s *HistogramValue) fixParent(parentModifiedFields *modifiedFields) { s.bucketCounts.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *HistogramValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.bucketCounts.attachParent(&s.modifiedFields, fieldModifiedHistogramValueBucketCounts) +} + func (s *HistogramValue) freeze() { if s.isFrozen() { return @@ -287,6 +294,13 @@ func (s *HistogramValue) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *HistogramValue) clearModifiedRecursively() { + s.bucketCounts.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *HistogramValue) computeDiff(val *HistogramValue) (ret bool) { @@ -365,21 +379,19 @@ func (s *HistogramValue) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *HistogramValue) cloneShared(allocators *Allocators) HistogramValue { - return s.Clone(allocators) -} - -func (s *HistogramValue) Clone(allocators *Allocators) HistogramValue { - c := HistogramValue{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *HistogramValue) CloneTo(dst *HistogramValue, allocators *Allocators) { + *dst = HistogramValue{ count: s.count, sum: s.sum, min: s.min, max: s.max, } - copyToNewUint64Array(&c.bucketCounts, &s.bucketCounts, allocators) - return c + copyToNewUint64Array(&dst.bucketCounts, &s.bucketCounts, allocators) + dst.optionalFieldsPresent = s.optionalFieldsPresent + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/keyvaluelist.go b/go/otel/otelstef/keyvaluelist.go index 05c8523b..114adaea 100644 --- a/go/otel/otelstef/keyvaluelist.go +++ b/go/otel/otelstef/keyvaluelist.go @@ -60,6 +60,15 @@ func (m *KeyValueList) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointers and bits in a newly copied multimap. +func (m *KeyValueList) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.modifiedElems.attachParent(parentModifiedFields, parentModifiedBit) + elems := m.elems[:m.initedCount] + for i := 0; i < len(elems); i++ { + elems[i].value.attachParent(&m.modifiedElems.vals, m.modifiedElems.maskForIndex(i)) + } +} + func (m *KeyValueList) canBeShared() bool { // Multimap can never be shared. return false @@ -122,6 +131,14 @@ func (m *KeyValueList) setUnmodifiedRecursively() { m.modifiedElems.setUnmodifiedAll() } +// clearModifiedRecursively clears modification state in all mutable elements. +func (m *KeyValueList) clearModifiedRecursively() { + for i := 0; i < len(m.elems); i++ { + m.elems[i].value.clearModifiedRecursively() + } + m.modifiedElems.setUnmodifiedAll() +} + // computeDiff compares m to val and returns true if they differ. // All fields that are different in m will be marked as modified. func (m *KeyValueList) computeDiff(val *KeyValueList) (ret bool) { diff --git a/go/otel/otelstef/link.go b/go/otel/otelstef/link.go index beffd4f8..b4fd391a 100644 --- a/go/otel/otelstef/link.go +++ b/go/otel/otelstef/link.go @@ -95,6 +95,13 @@ func (s *Link) fixParent(parentModifiedFields *modifiedFields) { s.attributes.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Link) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.attributes.attachParent(&s.modifiedFields, fieldModifiedLinkAttributes) +} + func (s *Link) freeze() { if s.isFrozen() { return @@ -261,6 +268,13 @@ func (s *Link) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Link) clearModifiedRecursively() { + s.attributes.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Link) computeDiff(val *Link) (ret bool) { @@ -302,22 +316,19 @@ func (s *Link) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Link) cloneShared(allocators *Allocators) Link { - return s.Clone(allocators) -} - -func (s *Link) Clone(allocators *Allocators) Link { - c := Link{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Link) CloneTo(dst *Link, allocators *Allocators) { + *dst = Link{ traceID: s.traceID, spanID: s.spanID, traceState: s.traceState, flags: s.flags, droppedAttributesCount: s.droppedAttributesCount, } - copyToNewAttributes(&c.attributes, &s.attributes, allocators) - return c + copyToNewAttributes(&dst.attributes, &s.attributes, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/linkarray.go b/go/otel/otelstef/linkarray.go index a12698b8..d12f1bdb 100644 --- a/go/otel/otelstef/linkarray.go +++ b/go/otel/otelstef/linkarray.go @@ -52,6 +52,15 @@ func (e *LinkArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *LinkArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *LinkArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *LinkArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *LinkArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *LinkArray) computeDiff(val *LinkArray) (ret bool) { diff --git a/go/otel/otelstef/logrecord.go b/go/otel/otelstef/logrecord.go new file mode 100644 index 00000000..65448328 --- /dev/null +++ b/go/otel/otelstef/logrecord.go @@ -0,0 +1,1541 @@ +// Code generated by stefc. DO NOT EDIT. +package otelstef + +import ( + "bytes" + "fmt" + "math/rand/v2" + "strings" + "unsafe" + + "github.com/splunk/stef/go/pkg" + "github.com/splunk/stef/go/pkg/codecs" + "github.com/splunk/stef/go/pkg/schema" +) + +var _ = strings.Compare +var _ = codecs.StringEncoder{} +var _ = schema.WireSchema{} +var _ = bytes.NewBuffer + +type LogRecord struct { + timeUnixNano uint64 + observedTimeUnixNano uint64 + severityNumber uint64 + severityText string + body *AnyValue + attributes Attributes + droppedAttributesCount uint64 + flags uint64 + traceID pkg.Bytes + spanID pkg.Bytes + eventName string + + // modifiedFields keeps track of which fields are modified. + modifiedFields modifiedFields +} + +const LogRecordStructName = "LogRecord" + +// Bitmasks for "modified" flags for each field. +const ( + fieldModifiedLogRecordTimeUnixNano = uint64(1 << iota) + fieldModifiedLogRecordObservedTimeUnixNano + fieldModifiedLogRecordSeverityNumber + fieldModifiedLogRecordSeverityText + fieldModifiedLogRecordBody + fieldModifiedLogRecordAttributes + fieldModifiedLogRecordDroppedAttributesCount + fieldModifiedLogRecordFlags + fieldModifiedLogRecordTraceID + fieldModifiedLogRecordSpanID + fieldModifiedLogRecordEventName +) + +// Init must be called once, before the LogRecord is used. +func (s *LogRecord) Init() { + s.init(nil, 0) +} + +func NewLogRecord() *LogRecord { + var s LogRecord + s.init(nil, 0) + return &s +} + +func (s *LogRecord) init(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + + s.body = &AnyValue{} + s.body.init(&s.modifiedFields, fieldModifiedLogRecordBody) + s.attributes.init(&s.modifiedFields, fieldModifiedLogRecordAttributes) +} + +func (s *LogRecord) initAlloc(parentModifiedFields *modifiedFields, parentModifiedBit uint64, allocators *Allocators) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(AnyValue{}))) + s.body = allocators.AnyValue.Alloc() + s.body.initAlloc(&s.modifiedFields, fieldModifiedLogRecordBody, allocators) + s.attributes.initAlloc(&s.modifiedFields, fieldModifiedLogRecordAttributes, allocators) +} + +// Reset the struct to its initial state. +func (s *LogRecord) Reset() { + s.reset() + s.setUnmodifiedRecursively() +} + +// reset the struct to its initial state, as if init() was just called. +// Will not reset internal fields such as parentModifiedFields. +func (s *LogRecord) reset() { + if s.modifiedFields.isFrozen() { + panic("cannot modify frozen LogRecord") + } + s.timeUnixNano = 0 + s.observedTimeUnixNano = 0 + s.severityNumber = 0 + s.severityText = "" + if s.body != nil { + s.body.reset() + } + s.attributes.reset() + s.droppedAttributesCount = 0 + s.flags = 0 + s.traceID = pkg.EmptyBytes + s.spanID = pkg.EmptyBytes + s.eventName = "" +} + +// fixParent sets the parentModifiedFields pointer to the supplied value. +// This is used when the parent is moved in memory for example because the parent +// an array element and the array was expanded. +func (s *LogRecord) fixParent(parentModifiedFields *modifiedFields) { + s.modifiedFields.parent = parentModifiedFields + s.body.fixParent(&s.modifiedFields) + s.attributes.fixParent(&s.modifiedFields) +} + +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *LogRecord) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.body.attachParent(&s.modifiedFields, fieldModifiedLogRecordBody) + s.attributes.attachParent(&s.modifiedFields, fieldModifiedLogRecordAttributes) +} + +func (s *LogRecord) freeze() { + if s.isFrozen() { + return + } + s.modifiedFields.freeze() + s.body.freeze() + s.attributes.freeze() +} + +func (s *LogRecord) isFrozen() bool { + return s.modifiedFields.isFrozen() +} + +func (s *LogRecord) TimeUnixNano() uint64 { + return s.timeUnixNano +} + +// SetTimeUnixNano sets the value of TimeUnixNano field. +func (s *LogRecord) SetTimeUnixNano(v uint64) { + if s.timeUnixNano != v { + s.timeUnixNano = v + s.modifiedFields.markModified(fieldModifiedLogRecordTimeUnixNano) + } +} + +func (s *LogRecord) markTimeUnixNanoModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordTimeUnixNano) +} + +// IsTimeUnixNanoModified returns true the value of TimeUnixNano field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsTimeUnixNanoModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordTimeUnixNano != 0 +} + +func (s *LogRecord) ObservedTimeUnixNano() uint64 { + return s.observedTimeUnixNano +} + +// SetObservedTimeUnixNano sets the value of ObservedTimeUnixNano field. +func (s *LogRecord) SetObservedTimeUnixNano(v uint64) { + if s.observedTimeUnixNano != v { + s.observedTimeUnixNano = v + s.modifiedFields.markModified(fieldModifiedLogRecordObservedTimeUnixNano) + } +} + +func (s *LogRecord) markObservedTimeUnixNanoModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordObservedTimeUnixNano) +} + +// IsObservedTimeUnixNanoModified returns true the value of ObservedTimeUnixNano field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsObservedTimeUnixNanoModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordObservedTimeUnixNano != 0 +} + +func (s *LogRecord) SeverityNumber() SeverityNumber { + return SeverityNumber(s.severityNumber) +} + +// SetSeverityNumber sets the value of SeverityNumber field. +func (s *LogRecord) SetSeverityNumber(v SeverityNumber) { + if s.severityNumber != uint64(v) { + s.severityNumber = uint64(v) + s.modifiedFields.markModified(fieldModifiedLogRecordSeverityNumber) + } +} + +func (s *LogRecord) markSeverityNumberModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordSeverityNumber) +} + +// IsSeverityNumberModified returns true the value of SeverityNumber field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsSeverityNumberModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordSeverityNumber != 0 +} + +func (s *LogRecord) SeverityText() string { + return s.severityText +} + +// SetSeverityText sets the value of SeverityText field. +func (s *LogRecord) SetSeverityText(v string) { + if s.severityText != v { + s.severityText = v + s.modifiedFields.markModified(fieldModifiedLogRecordSeverityText) + } +} + +func (s *LogRecord) markSeverityTextModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordSeverityText) +} + +// IsSeverityTextModified returns true the value of SeverityText field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsSeverityTextModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordSeverityText != 0 +} + +func (s *LogRecord) Body() *AnyValue { + return s.body +} + +func (s *LogRecord) markBodyModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordBody) +} + +// IsBodyModified returns true the value of Body field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsBodyModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordBody != 0 +} + +func (s *LogRecord) Attributes() *Attributes { + return &s.attributes +} + +func (s *LogRecord) markAttributesModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordAttributes) +} + +// IsAttributesModified returns true the value of Attributes field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsAttributesModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordAttributes != 0 +} + +func (s *LogRecord) DroppedAttributesCount() uint64 { + return s.droppedAttributesCount +} + +// SetDroppedAttributesCount sets the value of DroppedAttributesCount field. +func (s *LogRecord) SetDroppedAttributesCount(v uint64) { + if s.droppedAttributesCount != v { + s.droppedAttributesCount = v + s.modifiedFields.markModified(fieldModifiedLogRecordDroppedAttributesCount) + } +} + +func (s *LogRecord) markDroppedAttributesCountModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordDroppedAttributesCount) +} + +// IsDroppedAttributesCountModified returns true the value of DroppedAttributesCount field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsDroppedAttributesCountModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordDroppedAttributesCount != 0 +} + +func (s *LogRecord) Flags() uint64 { + return s.flags +} + +// SetFlags sets the value of Flags field. +func (s *LogRecord) SetFlags(v uint64) { + if s.flags != v { + s.flags = v + s.modifiedFields.markModified(fieldModifiedLogRecordFlags) + } +} + +func (s *LogRecord) markFlagsModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordFlags) +} + +// IsFlagsModified returns true the value of Flags field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsFlagsModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordFlags != 0 +} + +func (s *LogRecord) TraceID() pkg.Bytes { + return s.traceID +} + +// SetTraceID sets the value of TraceID field. +func (s *LogRecord) SetTraceID(v pkg.Bytes) { + if s.traceID != v { + s.traceID = v + s.modifiedFields.markModified(fieldModifiedLogRecordTraceID) + } +} + +func (s *LogRecord) markTraceIDModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordTraceID) +} + +// IsTraceIDModified returns true the value of TraceID field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsTraceIDModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordTraceID != 0 +} + +func (s *LogRecord) SpanID() pkg.Bytes { + return s.spanID +} + +// SetSpanID sets the value of SpanID field. +func (s *LogRecord) SetSpanID(v pkg.Bytes) { + if s.spanID != v { + s.spanID = v + s.modifiedFields.markModified(fieldModifiedLogRecordSpanID) + } +} + +func (s *LogRecord) markSpanIDModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordSpanID) +} + +// IsSpanIDModified returns true the value of SpanID field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsSpanIDModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordSpanID != 0 +} + +func (s *LogRecord) EventName() string { + return s.eventName +} + +// SetEventName sets the value of EventName field. +func (s *LogRecord) SetEventName(v string) { + if s.eventName != v { + s.eventName = v + s.modifiedFields.markModified(fieldModifiedLogRecordEventName) + } +} + +func (s *LogRecord) markEventNameModified() { + s.modifiedFields.markModified(fieldModifiedLogRecordEventName) +} + +// IsEventNameModified returns true the value of EventName field was modified since +// LogRecord was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *LogRecord) IsEventNameModified() bool { + return s.modifiedFields.mask&fieldModifiedLogRecordEventName != 0 +} + +func (s *LogRecord) setModifiedRecursively() { + s.body.setModifiedRecursively() + s.attributes.setModifiedRecursively() + s.modifiedFields.mask = + fieldModifiedLogRecordTimeUnixNano | + fieldModifiedLogRecordObservedTimeUnixNano | + fieldModifiedLogRecordSeverityNumber | + fieldModifiedLogRecordSeverityText | + fieldModifiedLogRecordBody | + fieldModifiedLogRecordAttributes | + fieldModifiedLogRecordDroppedAttributesCount | + fieldModifiedLogRecordFlags | + fieldModifiedLogRecordTraceID | + fieldModifiedLogRecordSpanID | + fieldModifiedLogRecordEventName | 0 +} + +func (s *LogRecord) setUnmodifiedRecursively() { + if s.IsBodyModified() { + s.body.setUnmodifiedRecursively() + } + if s.IsAttributesModified() { + s.attributes.setUnmodifiedRecursively() + } + s.modifiedFields.mask = 0 +} + +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *LogRecord) clearModifiedRecursively() { + s.body.clearModifiedRecursively() + s.attributes.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + +// computeDiff compares s and val and returns true if they differ. +// All fields that are different in s will be marked as modified. +func (s *LogRecord) computeDiff(val *LogRecord) (ret bool) { + // Compare TimeUnixNano field. + if s.timeUnixNano != val.timeUnixNano { + s.modifiedFields.setModified(fieldModifiedLogRecordTimeUnixNano) + ret = true + } + // Compare ObservedTimeUnixNano field. + if s.observedTimeUnixNano != val.observedTimeUnixNano { + s.modifiedFields.setModified(fieldModifiedLogRecordObservedTimeUnixNano) + ret = true + } + // Compare SeverityNumber field. + if s.severityNumber != val.severityNumber { + s.modifiedFields.setModified(fieldModifiedLogRecordSeverityNumber) + ret = true + } + // Compare SeverityText field. + if s.severityText != val.severityText { + s.modifiedFields.setModified(fieldModifiedLogRecordSeverityText) + ret = true + } + // Compare Body field. + if s.body.computeDiff(val.body) { + s.modifiedFields.setModified(fieldModifiedLogRecordBody) + ret = true + } + // Compare Attributes field. + if s.attributes.computeDiff(&val.attributes) { + s.modifiedFields.setModified(fieldModifiedLogRecordAttributes) + ret = true + } + // Compare DroppedAttributesCount field. + if s.droppedAttributesCount != val.droppedAttributesCount { + s.modifiedFields.setModified(fieldModifiedLogRecordDroppedAttributesCount) + ret = true + } + // Compare Flags field. + if s.flags != val.flags { + s.modifiedFields.setModified(fieldModifiedLogRecordFlags) + ret = true + } + // Compare TraceID field. + if s.traceID != val.traceID { + s.modifiedFields.setModified(fieldModifiedLogRecordTraceID) + ret = true + } + // Compare SpanID field. + if s.spanID != val.spanID { + s.modifiedFields.setModified(fieldModifiedLogRecordSpanID) + ret = true + } + // Compare EventName field. + if s.eventName != val.eventName { + s.modifiedFields.setModified(fieldModifiedLogRecordEventName) + ret = true + } + return ret +} + +// canBeShared returns true if s is safe to share by pointer without cloning (for example if s is frozen). +func (s *LogRecord) canBeShared() bool { + return false +} + +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *LogRecord) CloneTo(dst *LogRecord, allocators *Allocators) { + *dst = LogRecord{ + timeUnixNano: s.timeUnixNano, + observedTimeUnixNano: s.observedTimeUnixNano, + severityNumber: s.severityNumber, + severityText: s.severityText, + droppedAttributesCount: s.droppedAttributesCount, + flags: s.flags, + traceID: s.traceID, + spanID: s.spanID, + eventName: s.eventName, + } + dst.body = s.body.cloneShared(allocators) + copyToNewAttributes(&dst.attributes, &s.attributes, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() +} + +// ByteSize returns approximate memory usage in bytes. Used to calculate +// memory used by dictionaries. +func (s *LogRecord) byteSize() uint { + return uint(unsafe.Sizeof(*s)) + + s.body.byteSize() + s.attributes.byteSize() + 0 +} + +// Copy from src to dst, overwriting existing data in dst. +func copyLogRecord(dst *LogRecord, src *LogRecord) { + dst.SetTimeUnixNano(src.timeUnixNano) + dst.SetObservedTimeUnixNano(src.observedTimeUnixNano) + dst.SetSeverityNumber(SeverityNumber(src.severityNumber)) + dst.SetSeverityText(src.severityText) + + if src.body.canBeShared() { + if src.body.computeDiff(dst.body) { + dst.body = src.body + dst.markBodyModified() + } + } else { + if dst.body == nil || dst.body.canBeShared() { // Not allowed to modify shared data. + dst.body = new(AnyValue) + dst.body.init(&dst.modifiedFields, fieldModifiedLogRecordBody) + } + copyAnyValue(dst.body, src.body) + } + copyAttributes(&dst.attributes, &src.attributes) + dst.SetDroppedAttributesCount(src.droppedAttributesCount) + dst.SetFlags(src.flags) + dst.SetTraceID(src.traceID) + dst.SetSpanID(src.spanID) + dst.SetEventName(src.eventName) +} + +// Copy from src to dst. dst is assumed to be just inited. +func copyToNewLogRecord(dst *LogRecord, src *LogRecord, allocators *Allocators) { + dst.SetTimeUnixNano(src.timeUnixNano) + dst.SetObservedTimeUnixNano(src.observedTimeUnixNano) + dst.SetSeverityNumber(SeverityNumber(src.severityNumber)) + dst.SetSeverityText(src.severityText) + + if src.body.canBeShared() { + dst.body = src.body + } else { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(AnyValue{}))) + dst.body = allocators.AnyValue.Alloc() + dst.body.init(&dst.modifiedFields, fieldModifiedLogRecordBody) + copyToNewAnyValue(dst.body, src.body, allocators) + } + + copyToNewAttributes(&dst.attributes, &src.attributes, allocators) + dst.SetDroppedAttributesCount(src.droppedAttributesCount) + dst.SetFlags(src.flags) + dst.SetTraceID(src.traceID) + dst.SetSpanID(src.spanID) + dst.SetEventName(src.eventName) +} + +// CopyFrom() performs a deep copy from src. +func (s *LogRecord) CopyFrom(src *LogRecord) { + copyLogRecord(s, src) +} + +// mutateRandom mutates fields in a random, deterministic manner using +// random parameter as a deterministic generator. Only fields that exist +// in the schem are mutated, allowing to generate data for specified schema. +// The state of "modified" flags is undefined after this operation. +func (s *LogRecord) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter *mutateRandomLimiter) { + // Get the field count for this struct from the schema. If the schema specifies + // fewer field count than the one we have in this code then we will not mutate + // fields that are not in the schema. + fieldCount, err := schem.FieldCount("LogRecord") + if err != nil { + panic(fmt.Sprintf("cannot get field count for %s: %v", "LogRecord", err)) + } + + const randRange = max(11, 2) // At least 2 to ensure we don't recurse infinitely if there is only 1 field. + + if fieldCount <= 0 { + return // TimeUnixNano and all subsequent fields are skipped. + } + // Maybe mutate TimeUnixNano + if random.IntN(randRange) == 0 { + s.SetTimeUnixNano(pkg.Uint64Random(random)) + } + if fieldCount <= 1 { + return // ObservedTimeUnixNano and all subsequent fields are skipped. + } + // Maybe mutate ObservedTimeUnixNano + if random.IntN(randRange) == 0 { + s.SetObservedTimeUnixNano(pkg.Uint64Random(random)) + } + if fieldCount <= 2 { + return // SeverityNumber and all subsequent fields are skipped. + } + // Maybe mutate SeverityNumber + if random.IntN(randRange) == 0 { + s.SetSeverityNumber(SeverityNumber(pkg.Uint64Random(random) % 25)) + } + if fieldCount <= 3 { + return // SeverityText and all subsequent fields are skipped. + } + // Maybe mutate SeverityText + if random.IntN(randRange) == 0 { + s.SetSeverityText(pkg.StringRandom(random)) + } + if fieldCount <= 4 { + return // Body and all subsequent fields are skipped. + } + // Maybe mutate Body + if random.IntN(randRange) == 0 { + s.body.mutateRandom(random, schem, limiter) + } + if fieldCount <= 5 { + return // Attributes and all subsequent fields are skipped. + } + // Maybe mutate Attributes + if random.IntN(randRange) == 0 { + s.attributes.mutateRandom(random, schem, limiter) + } + if fieldCount <= 6 { + return // DroppedAttributesCount and all subsequent fields are skipped. + } + // Maybe mutate DroppedAttributesCount + if random.IntN(randRange) == 0 { + s.SetDroppedAttributesCount(pkg.Uint64Random(random)) + } + if fieldCount <= 7 { + return // Flags and all subsequent fields are skipped. + } + // Maybe mutate Flags + if random.IntN(randRange) == 0 { + s.SetFlags(pkg.Uint64Random(random)) + } + if fieldCount <= 8 { + return // TraceID and all subsequent fields are skipped. + } + // Maybe mutate TraceID + if random.IntN(randRange) == 0 { + s.SetTraceID(pkg.BytesRandom(random)) + } + if fieldCount <= 9 { + return // SpanID and all subsequent fields are skipped. + } + // Maybe mutate SpanID + if random.IntN(randRange) == 0 { + s.SetSpanID(pkg.BytesRandom(random)) + } + if fieldCount <= 10 { + return // EventName and all subsequent fields are skipped. + } + // Maybe mutate EventName + if random.IntN(randRange) == 0 { + s.SetEventName(pkg.StringRandom(random)) + } +} + +// IsEqual performs deep comparison and returns true if struct is equal to right. +func (s *LogRecord) IsEqual(right *LogRecord) bool { + // Compare TimeUnixNano field. + if !pkg.Uint64Equal(s.timeUnixNano, right.timeUnixNano) { + return false + } + // Compare ObservedTimeUnixNano field. + if !pkg.Uint64Equal(s.observedTimeUnixNano, right.observedTimeUnixNano) { + return false + } + // Compare SeverityNumber field. + if !pkg.Uint64Equal(s.severityNumber, right.severityNumber) { + return false + } + // Compare SeverityText field. + if !pkg.StringEqual(s.severityText, right.severityText) { + return false + } + // Compare Body field. + if !s.body.IsEqual(right.body) { + return false + } + // Compare Attributes field. + if !s.attributes.IsEqual(&right.attributes) { + return false + } + // Compare DroppedAttributesCount field. + if !pkg.Uint64Equal(s.droppedAttributesCount, right.droppedAttributesCount) { + return false + } + // Compare Flags field. + if !pkg.Uint64Equal(s.flags, right.flags) { + return false + } + // Compare TraceID field. + if !pkg.BytesEqual(s.traceID, right.traceID) { + return false + } + // Compare SpanID field. + if !pkg.BytesEqual(s.spanID, right.spanID) { + return false + } + // Compare EventName field. + if !pkg.StringEqual(s.eventName, right.eventName) { + return false + } + + return true +} + +// CmpLogRecord performs deep comparison and returns an integer that +// will be 0 if left == right, negative if left < right, positive if left > right. +func CmpLogRecord(left, right *LogRecord) int { + // Compare TimeUnixNano field. + if c := pkg.Uint64Compare(left.timeUnixNano, right.timeUnixNano); c != 0 { + return c + } + // Compare ObservedTimeUnixNano field. + if c := pkg.Uint64Compare(left.observedTimeUnixNano, right.observedTimeUnixNano); c != 0 { + return c + } + // Compare SeverityNumber field. + if c := pkg.Uint64Compare(left.severityNumber, right.severityNumber); c != 0 { + return c + } + // Compare SeverityText field. + if c := strings.Compare(left.severityText, right.severityText); c != 0 { + return c + } + // Compare Body field. + if c := CmpAnyValue(left.body, right.body); c != 0 { + return c + } + // Compare Attributes field. + if c := CmpAttributes(&left.attributes, &right.attributes); c != 0 { + return c + } + // Compare DroppedAttributesCount field. + if c := pkg.Uint64Compare(left.droppedAttributesCount, right.droppedAttributesCount); c != 0 { + return c + } + // Compare Flags field. + if c := pkg.Uint64Compare(left.flags, right.flags); c != 0 { + return c + } + // Compare TraceID field. + if c := pkg.BytesCompare(left.traceID, right.traceID); c != 0 { + return c + } + // Compare SpanID field. + if c := pkg.BytesCompare(left.spanID, right.spanID); c != 0 { + return c + } + // Compare EventName field. + if c := strings.Compare(left.eventName, right.eventName); c != 0 { + return c + } + return 0 +} + +// LogRecordEncoder implements encoding of LogRecord +type LogRecordEncoder struct { + buf pkg.BitsWriter + limiter *pkg.SizeLimiter + + // forceModifiedFields is set to a mask to force the next encoding operation + // write the fields, whether they are modified or no. This is used after frame + // restarts so that the data can be decoded from the frame start. + forceModifiedFields uint64 + + timeUnixNanoEncoder codecs.Uint64Encoder + observedTimeUnixNanoEncoder codecs.Uint64Encoder + severityNumberEncoder codecs.Uint64Encoder + severityTextEncoder codecs.StringDictEncoder + bodyEncoder *AnyValueEncoder + isBodyRecursive bool // Indicates Body field's type is recursive. + attributesEncoder *AttributesEncoder + isAttributesRecursive bool // Indicates Attributes field's type is recursive. + droppedAttributesCountEncoder codecs.Uint64Encoder + flagsEncoder codecs.Uint64Encoder + traceIDEncoder codecs.BytesEncoder + spanIDEncoder codecs.BytesEncoder + eventNameEncoder codecs.StringDictEncoder + + allocators *Allocators + + keepFieldMask uint64 + fieldCount uint +} + +func (e *LogRecordEncoder) Init(state *WriterState, columns *pkg.WriteColumnSet) error { + // Remember this encoder in the state so that we can detect recursion. + if state.LogRecordEncoder != nil { + panic("cannot initialize LogRecordEncoder: already initialized") + } + state.LogRecordEncoder = e + defer func() { state.LogRecordEncoder = nil }() + + e.limiter = &state.limiter + e.allocators = &state.Allocators + + // Number of fields in the output data schema. + var err error + e.fieldCount, err = state.StructFieldCounts.LogRecordFieldCount() + if err != nil { + return fmt.Errorf("cannot find struct %s in override schema: %w", "LogRecord", err) + } + // Set that many 1 bits in the keepFieldMask. All fields with higher number + // will be skipped when encoding. + e.keepFieldMask = ^(^uint64(0) << e.fieldCount) + + // Init encoder for TimeUnixNano field. + if e.fieldCount <= 0 { + return nil // TimeUnixNano and all subsequent fields are skipped. + } + err = e.timeUnixNanoEncoder.Init(e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for ObservedTimeUnixNano field. + if e.fieldCount <= 1 { + return nil // ObservedTimeUnixNano and all subsequent fields are skipped. + } + err = e.observedTimeUnixNanoEncoder.Init(e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for SeverityNumber field. + if e.fieldCount <= 2 { + return nil // SeverityNumber and all subsequent fields are skipped. + } + err = e.severityNumberEncoder.Init(e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for SeverityText field. + if e.fieldCount <= 3 { + return nil // SeverityText and all subsequent fields are skipped. + } + err = e.severityTextEncoder.Init(&state.LogSeverityText, e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for Body field. + if e.fieldCount <= 4 { + return nil // Body and all subsequent fields are skipped. + } + if state.AnyValueEncoder != nil { + // Recursion detected, use the existing encoder. + e.bodyEncoder = state.AnyValueEncoder + e.isBodyRecursive = true + } else { + e.bodyEncoder = new(AnyValueEncoder) + err = e.bodyEncoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + + // Init encoder for Attributes field. + if e.fieldCount <= 5 { + return nil // Attributes and all subsequent fields are skipped. + } + if state.AttributesEncoder != nil { + // Recursion detected, use the existing encoder. + e.attributesEncoder = state.AttributesEncoder + e.isAttributesRecursive = true + } else { + e.attributesEncoder = new(AttributesEncoder) + err = e.attributesEncoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + + // Init encoder for DroppedAttributesCount field. + if e.fieldCount <= 6 { + return nil // DroppedAttributesCount and all subsequent fields are skipped. + } + err = e.droppedAttributesCountEncoder.Init(e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for Flags field. + if e.fieldCount <= 7 { + return nil // Flags and all subsequent fields are skipped. + } + err = e.flagsEncoder.Init(e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for TraceID field. + if e.fieldCount <= 8 { + return nil // TraceID and all subsequent fields are skipped. + } + err = e.traceIDEncoder.Init(e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for SpanID field. + if e.fieldCount <= 9 { + return nil // SpanID and all subsequent fields are skipped. + } + err = e.spanIDEncoder.Init(e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + // Init encoder for EventName field. + if e.fieldCount <= 10 { + return nil // EventName and all subsequent fields are skipped. + } + err = e.eventNameEncoder.Init(&state.LogEventName, e.limiter, columns.AddSubColumn()) + if err != nil { + return err + } + + return nil +} + +func (e *LogRecordEncoder) Reset() { + // Since we are resetting the state of encoder make sure the next Encode() + // call forcedly writes all fields and does not attempt to skip. + e.forceModifiedFields = e.keepFieldMask + + if e.fieldCount <= 0 { + return // TimeUnixNano and all subsequent fields are skipped. + } + e.timeUnixNanoEncoder.Reset() + if e.fieldCount <= 1 { + return // ObservedTimeUnixNano and all subsequent fields are skipped. + } + e.observedTimeUnixNanoEncoder.Reset() + if e.fieldCount <= 2 { + return // SeverityNumber and all subsequent fields are skipped. + } + e.severityNumberEncoder.Reset() + if e.fieldCount <= 3 { + return // SeverityText and all subsequent fields are skipped. + } + e.severityTextEncoder.Reset() + if e.fieldCount <= 4 { + return // Body and all subsequent fields are skipped. + } + if !e.isBodyRecursive { + e.bodyEncoder.Reset() + } + if e.fieldCount <= 5 { + return // Attributes and all subsequent fields are skipped. + } + if !e.isAttributesRecursive { + e.attributesEncoder.Reset() + } + if e.fieldCount <= 6 { + return // DroppedAttributesCount and all subsequent fields are skipped. + } + e.droppedAttributesCountEncoder.Reset() + if e.fieldCount <= 7 { + return // Flags and all subsequent fields are skipped. + } + e.flagsEncoder.Reset() + if e.fieldCount <= 8 { + return // TraceID and all subsequent fields are skipped. + } + e.traceIDEncoder.Reset() + if e.fieldCount <= 9 { + return // SpanID and all subsequent fields are skipped. + } + e.spanIDEncoder.Reset() + if e.fieldCount <= 10 { + return // EventName and all subsequent fields are skipped. + } + e.eventNameEncoder.Reset() +} + +// Encode encodes val into buf +func (e *LogRecordEncoder) Encode(val *LogRecord) { + var bitCount uint + + // Mask that describes what fields are encoded. Start with all modified fields. + fieldMask := val.modifiedFields.mask + + // If forceModifiedFields we need to set to 1 all bits so that we + // force writing of all fields. + fieldMask |= e.forceModifiedFields + e.forceModifiedFields = 0 + + // Only write fields that we want to write. See Init() for keepFieldMask. + fieldMask &= e.keepFieldMask + + // Write bits to indicate which fields follow. + e.buf.WriteBits(fieldMask, e.fieldCount) + bitCount += e.fieldCount + + // Encode modified, present fields. + + if fieldMask&fieldModifiedLogRecordTimeUnixNano != 0 { + // Encode TimeUnixNano + e.timeUnixNanoEncoder.Encode(val.timeUnixNano) + } + + if fieldMask&fieldModifiedLogRecordObservedTimeUnixNano != 0 { + // Encode ObservedTimeUnixNano + e.observedTimeUnixNanoEncoder.Encode(val.observedTimeUnixNano) + } + + if fieldMask&fieldModifiedLogRecordSeverityNumber != 0 { + // Encode SeverityNumber + e.severityNumberEncoder.Encode(val.severityNumber) + } + + if fieldMask&fieldModifiedLogRecordSeverityText != 0 { + // Encode SeverityText + e.severityTextEncoder.Encode(val.severityText) + } + + if fieldMask&fieldModifiedLogRecordBody != 0 { + // Encode Body + e.bodyEncoder.Encode(val.body) + } + + if fieldMask&fieldModifiedLogRecordAttributes != 0 { + // Encode Attributes + e.attributesEncoder.Encode(&val.attributes) + } + + if fieldMask&fieldModifiedLogRecordDroppedAttributesCount != 0 { + // Encode DroppedAttributesCount + e.droppedAttributesCountEncoder.Encode(val.droppedAttributesCount) + } + + if fieldMask&fieldModifiedLogRecordFlags != 0 { + // Encode Flags + e.flagsEncoder.Encode(val.flags) + } + + if fieldMask&fieldModifiedLogRecordTraceID != 0 { + // Encode TraceID + e.traceIDEncoder.Encode(val.traceID) + } + + if fieldMask&fieldModifiedLogRecordSpanID != 0 { + // Encode SpanID + e.spanIDEncoder.Encode(val.spanID) + } + + if fieldMask&fieldModifiedLogRecordEventName != 0 { + // Encode EventName + e.eventNameEncoder.Encode(val.eventName) + } + + // Account written bits in the limiter. + e.limiter.AddFrameBits(bitCount) + + // Mark all fields non-modified so that next Encode() correctly + // encodes only fields that change after this. + val.modifiedFields.mask = 0 +} + +// CollectColumns collects all buffers from all encoders into buf. +func (e *LogRecordEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { + columnSet.SetBits(&e.buf) + colIdx := 0 + // Collect TimeUnixNano field. + if e.fieldCount <= 0 { + return // TimeUnixNano and subsequent fields are skipped. + } + e.timeUnixNanoEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect ObservedTimeUnixNano field. + if e.fieldCount <= 1 { + return // ObservedTimeUnixNano and subsequent fields are skipped. + } + e.observedTimeUnixNanoEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect SeverityNumber field. + if e.fieldCount <= 2 { + return // SeverityNumber and subsequent fields are skipped. + } + e.severityNumberEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect SeverityText field. + if e.fieldCount <= 3 { + return // SeverityText and subsequent fields are skipped. + } + e.severityTextEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect Body field. + if e.fieldCount <= 4 { + return // Body and subsequent fields are skipped. + } + if !e.isBodyRecursive { + e.bodyEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + } + // Collect Attributes field. + if e.fieldCount <= 5 { + return // Attributes and subsequent fields are skipped. + } + if !e.isAttributesRecursive { + e.attributesEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + } + // Collect DroppedAttributesCount field. + if e.fieldCount <= 6 { + return // DroppedAttributesCount and subsequent fields are skipped. + } + e.droppedAttributesCountEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect Flags field. + if e.fieldCount <= 7 { + return // Flags and subsequent fields are skipped. + } + e.flagsEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect TraceID field. + if e.fieldCount <= 8 { + return // TraceID and subsequent fields are skipped. + } + e.traceIDEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect SpanID field. + if e.fieldCount <= 9 { + return // SpanID and subsequent fields are skipped. + } + e.spanIDEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + // Collect EventName field. + if e.fieldCount <= 10 { + return // EventName and subsequent fields are skipped. + } + e.eventNameEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ +} + +// LogRecordDecoder implements decoding of LogRecord +type LogRecordDecoder struct { + buf pkg.BitsReader + column *pkg.ReadableColumn + fieldCount uint + + timeUnixNanoDecoder codecs.Uint64Decoder + + observedTimeUnixNanoDecoder codecs.Uint64Decoder + + severityNumberDecoder codecs.Uint64Decoder + + severityTextDecoder codecs.StringDictDecoder + + bodyDecoder *AnyValueDecoder + isBodyRecursive bool + attributesDecoder *AttributesDecoder + isAttributesRecursive bool + droppedAttributesCountDecoder codecs.Uint64Decoder + + flagsDecoder codecs.Uint64Decoder + + traceIDDecoder codecs.BytesDecoder + + spanIDDecoder codecs.BytesDecoder + + eventNameDecoder codecs.StringDictDecoder + + allocators *Allocators +} + +// Init is called once in the lifetime of the stream. +func (d *LogRecordDecoder) Init(state *ReaderState, columns *pkg.ReadColumnSet) error { + // Remember this decoder in the state so that we can detect recursion. + if state.LogRecordDecoder != nil { + panic("cannot initialize LogRecordDecoder: already initialized") + } + state.LogRecordDecoder = d + defer func() { state.LogRecordDecoder = nil }() + + d.allocators = &state.Allocators + + // Number of fields in the input data schema. + var err error + d.fieldCount, err = state.StructFieldCounts.LogRecordFieldCount() + if err != nil { + return fmt.Errorf("cannot find struct %s in override schema: %w", "LogRecord", err) + } + if d.fieldCount > 11 { + return pkg.ErrTooManyFieldsToDecode + } + + d.column = columns.Column() + + if d.fieldCount <= 0 { + return nil // TimeUnixNano and subsequent fields are skipped. + } + err = d.timeUnixNanoDecoder.Init(columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 1 { + return nil // ObservedTimeUnixNano and subsequent fields are skipped. + } + err = d.observedTimeUnixNanoDecoder.Init(columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 2 { + return nil // SeverityNumber and subsequent fields are skipped. + } + err = d.severityNumberDecoder.Init(columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 3 { + return nil // SeverityText and subsequent fields are skipped. + } + err = d.severityTextDecoder.Init(&state.LogSeverityText, columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 4 { + return nil // Body and subsequent fields are skipped. + } + if state.AnyValueDecoder != nil { + // Recursion detected, use the existing decoder. + d.bodyDecoder = state.AnyValueDecoder + d.isBodyRecursive = true // Mark that we are using a recursive decoder. + } else { + d.bodyDecoder = new(AnyValueDecoder) + err = d.bodyDecoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + if d.fieldCount <= 5 { + return nil // Attributes and subsequent fields are skipped. + } + if state.AttributesDecoder != nil { + // Recursion detected, use the existing decoder. + d.attributesDecoder = state.AttributesDecoder + d.isAttributesRecursive = true // Mark that we are using a recursive decoder. + } else { + d.attributesDecoder = new(AttributesDecoder) + err = d.attributesDecoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + if d.fieldCount <= 6 { + return nil // DroppedAttributesCount and subsequent fields are skipped. + } + err = d.droppedAttributesCountDecoder.Init(columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 7 { + return nil // Flags and subsequent fields are skipped. + } + err = d.flagsDecoder.Init(columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 8 { + return nil // TraceID and subsequent fields are skipped. + } + err = d.traceIDDecoder.Init(columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 9 { + return nil // SpanID and subsequent fields are skipped. + } + err = d.spanIDDecoder.Init(columns.AddSubColumn()) + if err != nil { + return err + } + if d.fieldCount <= 10 { + return nil // EventName and subsequent fields are skipped. + } + err = d.eventNameDecoder.Init(&state.LogEventName, columns.AddSubColumn()) + if err != nil { + return err + } + + return nil +} + +// Continue is called at the start of the frame to continue decoding column data. +// This should set the decoder's source buffer, so the new decoding continues from +// the supplied column data. This should NOT reset the internal state of the decoder, +// since columns can cross frame boundaries and the new column data is considered +// continuation of that same column in the previous frame. +func (d *LogRecordDecoder) Continue() { + d.buf.Reset(d.column.Data()) + + if d.fieldCount <= 0 { + return // TimeUnixNano and subsequent fields are skipped. + } + d.timeUnixNanoDecoder.Continue() + if d.fieldCount <= 1 { + return // ObservedTimeUnixNano and subsequent fields are skipped. + } + d.observedTimeUnixNanoDecoder.Continue() + if d.fieldCount <= 2 { + return // SeverityNumber and subsequent fields are skipped. + } + d.severityNumberDecoder.Continue() + if d.fieldCount <= 3 { + return // SeverityText and subsequent fields are skipped. + } + d.severityTextDecoder.Continue() + if d.fieldCount <= 4 { + return // Body and subsequent fields are skipped. + } + + if !d.isBodyRecursive { + d.bodyDecoder.Continue() + } + + if d.fieldCount <= 5 { + return // Attributes and subsequent fields are skipped. + } + + if !d.isAttributesRecursive { + d.attributesDecoder.Continue() + } + + if d.fieldCount <= 6 { + return // DroppedAttributesCount and subsequent fields are skipped. + } + d.droppedAttributesCountDecoder.Continue() + if d.fieldCount <= 7 { + return // Flags and subsequent fields are skipped. + } + d.flagsDecoder.Continue() + if d.fieldCount <= 8 { + return // TraceID and subsequent fields are skipped. + } + d.traceIDDecoder.Continue() + if d.fieldCount <= 9 { + return // SpanID and subsequent fields are skipped. + } + d.spanIDDecoder.Continue() + if d.fieldCount <= 10 { + return // EventName and subsequent fields are skipped. + } + d.eventNameDecoder.Continue() +} + +func (d *LogRecordDecoder) Reset() { + + if d.fieldCount <= 0 { + return // TimeUnixNano and all subsequent fields are skipped. + } + d.timeUnixNanoDecoder.Reset() + if d.fieldCount <= 1 { + return // ObservedTimeUnixNano and all subsequent fields are skipped. + } + d.observedTimeUnixNanoDecoder.Reset() + if d.fieldCount <= 2 { + return // SeverityNumber and all subsequent fields are skipped. + } + d.severityNumberDecoder.Reset() + if d.fieldCount <= 3 { + return // SeverityText and all subsequent fields are skipped. + } + d.severityTextDecoder.Reset() + if d.fieldCount <= 4 { + return // Body and all subsequent fields are skipped. + } + + if !d.isBodyRecursive { + d.bodyDecoder.Reset() + } + + if d.fieldCount <= 5 { + return // Attributes and all subsequent fields are skipped. + } + + if !d.isAttributesRecursive { + d.attributesDecoder.Reset() + } + + if d.fieldCount <= 6 { + return // DroppedAttributesCount and all subsequent fields are skipped. + } + d.droppedAttributesCountDecoder.Reset() + if d.fieldCount <= 7 { + return // Flags and all subsequent fields are skipped. + } + d.flagsDecoder.Reset() + if d.fieldCount <= 8 { + return // TraceID and all subsequent fields are skipped. + } + d.traceIDDecoder.Reset() + if d.fieldCount <= 9 { + return // SpanID and all subsequent fields are skipped. + } + d.spanIDDecoder.Reset() + if d.fieldCount <= 10 { + return // EventName and all subsequent fields are skipped. + } + d.eventNameDecoder.Reset() +} + +func (d *LogRecordDecoder) Decode(dstPtr *LogRecord) error { + val := dstPtr + + // Read bits that indicate which fields follow. + val.modifiedFields.mask = d.buf.PeekBits(d.fieldCount) + d.buf.Consume(d.fieldCount) + + if val.modifiedFields.mask&fieldModifiedLogRecordTimeUnixNano != 0 { // TimeUnixNano is changed. + + if err := d.timeUnixNanoDecoder.Decode(&val.timeUnixNano); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordObservedTimeUnixNano != 0 { // ObservedTimeUnixNano is changed. + + if err := d.observedTimeUnixNanoDecoder.Decode(&val.observedTimeUnixNano); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordSeverityNumber != 0 { // SeverityNumber is changed. + + if err := d.severityNumberDecoder.Decode(&val.severityNumber); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordSeverityText != 0 { // SeverityText is changed. + + if err := d.severityTextDecoder.Decode(&val.severityText); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordBody != 0 { // Body is changed. + + if val.body == nil { + if err := d.allocators.allocSizeChecker.PrepAllocSize(uint(unsafe.Sizeof(*val.body))); err != nil { + return err + } + val.body = d.allocators.AnyValue.Alloc() + val.body.init(&val.modifiedFields, fieldModifiedLogRecordBody) + } + + if err := d.bodyDecoder.Decode(val.body); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordAttributes != 0 { // Attributes is changed. + + if err := d.attributesDecoder.Decode(&val.attributes); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordDroppedAttributesCount != 0 { // DroppedAttributesCount is changed. + + if err := d.droppedAttributesCountDecoder.Decode(&val.droppedAttributesCount); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordFlags != 0 { // Flags is changed. + + if err := d.flagsDecoder.Decode(&val.flags); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordTraceID != 0 { // TraceID is changed. + + if err := d.traceIDDecoder.Decode(&val.traceID); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordSpanID != 0 { // SpanID is changed. + + if err := d.spanIDDecoder.Decode(&val.spanID); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogRecordEventName != 0 { // EventName is changed. + + if err := d.eventNameDecoder.Decode(&val.eventName); err != nil { + return err + } + } + + return d.buf.Error() +} + +// LogRecordAllocator implements a custom allocator for LogRecord. +// It maintains a pool of pre-allocated LogRecord and grows the pool +// dynamically as needed, up to a maximum size of 64 elements. +type LogRecordAllocator struct { + pool []LogRecord + ofs int +} + +// Alloc returns the next available LogRecord from the pool. +// If the pool is exhausted, it grows the pool by doubling its size +// up to a maximum of 64 elements. +func (a *LogRecordAllocator) Alloc() *LogRecord { + if a.ofs < len(a.pool) { + // Get the next available LogRecord from the pool + a.ofs++ + return &a.pool[a.ofs-1] + } + // We've exhausted the current pool, prealloc a new pool. + return a.prealloc() +} + +//go:noinline +func (a *LogRecordAllocator) prealloc() *LogRecord { + // prealloc expands the pool by doubling its size, up to a maximum of 64 elements. + // If the pool is empty, it starts with 1 element. + newLen := min(max(len(a.pool)*2, 1), 64) + a.pool = make([]LogRecord, newLen) + a.ofs = 1 + return &a.pool[0] +} diff --git a/go/otel/otelstef/logs.go b/go/otel/otelstef/logs.go new file mode 100644 index 00000000..1199900e --- /dev/null +++ b/go/otel/otelstef/logs.go @@ -0,0 +1,1027 @@ +// Code generated by stefc. DO NOT EDIT. +package otelstef + +import ( + "bytes" + "fmt" + "math/rand/v2" + "strings" + "unsafe" + + "github.com/splunk/stef/go/pkg" + "github.com/splunk/stef/go/pkg/codecs" + "github.com/splunk/stef/go/pkg/schema" +) + +var _ = strings.Compare +var _ = codecs.StringEncoder{} +var _ = schema.WireSchema{} +var _ = bytes.NewBuffer + +type Logs struct { + envelope Envelope + resource *Resource + scope *Scope + log LogRecord + + // modifiedFields keeps track of which fields are modified. + modifiedFields modifiedFields +} + +const LogsStructName = "Logs" + +// Bitmasks for "modified" flags for each field. +const ( + fieldModifiedLogsEnvelope = uint64(1 << iota) + fieldModifiedLogsResource + fieldModifiedLogsScope + fieldModifiedLogsLog +) + +// Init must be called once, before the Logs is used. +func (s *Logs) Init() { + s.init(nil, 0) +} + +func NewLogs() *Logs { + var s Logs + s.init(nil, 0) + return &s +} + +func (s *Logs) init(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + + s.envelope.init(&s.modifiedFields, fieldModifiedLogsEnvelope) + s.resource = &Resource{} + s.resource.init(&s.modifiedFields, fieldModifiedLogsResource) + s.scope = &Scope{} + s.scope.init(&s.modifiedFields, fieldModifiedLogsScope) + s.log.init(&s.modifiedFields, fieldModifiedLogsLog) +} + +func (s *Logs) initAlloc(parentModifiedFields *modifiedFields, parentModifiedBit uint64, allocators *Allocators) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + + s.envelope.initAlloc(&s.modifiedFields, fieldModifiedLogsEnvelope, allocators) + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Resource{}))) + s.resource = allocators.Resource.Alloc() + s.resource.initAlloc(&s.modifiedFields, fieldModifiedLogsResource, allocators) + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Scope{}))) + s.scope = allocators.Scope.Alloc() + s.scope.initAlloc(&s.modifiedFields, fieldModifiedLogsScope, allocators) + s.log.initAlloc(&s.modifiedFields, fieldModifiedLogsLog, allocators) +} + +// Reset the struct to its initial state. +func (s *Logs) Reset() { + s.reset() + s.setUnmodifiedRecursively() +} + +// reset the struct to its initial state, as if init() was just called. +// Will not reset internal fields such as parentModifiedFields. +func (s *Logs) reset() { + if s.modifiedFields.isFrozen() { + panic("cannot modify frozen Logs") + } + s.envelope.reset() + s.resource = &emptyResource + s.scope = &emptyScope + s.log.reset() +} + +// fixParent sets the parentModifiedFields pointer to the supplied value. +// This is used when the parent is moved in memory for example because the parent +// an array element and the array was expanded. +func (s *Logs) fixParent(parentModifiedFields *modifiedFields) { + s.modifiedFields.parent = parentModifiedFields + s.envelope.fixParent(&s.modifiedFields) + s.resource.fixParent(&s.modifiedFields) + s.scope.fixParent(&s.modifiedFields) + s.log.fixParent(&s.modifiedFields) +} + +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Logs) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.envelope.attachParent(&s.modifiedFields, fieldModifiedLogsEnvelope) + if !s.resource.canBeShared() { + s.resource.attachParent(&s.modifiedFields, fieldModifiedLogsResource) + } + if !s.scope.canBeShared() { + s.scope.attachParent(&s.modifiedFields, fieldModifiedLogsScope) + } + s.log.attachParent(&s.modifiedFields, fieldModifiedLogsLog) +} + +func (s *Logs) freeze() { + if s.isFrozen() { + return + } + s.modifiedFields.freeze() + s.envelope.freeze() + s.resource.freeze() + s.scope.freeze() + s.log.freeze() +} + +func (s *Logs) isFrozen() bool { + return s.modifiedFields.isFrozen() +} + +func (s *Logs) Envelope() *Envelope { + return &s.envelope +} + +func (s *Logs) markEnvelopeModified() { + s.modifiedFields.markModified(fieldModifiedLogsEnvelope) +} + +// IsEnvelopeModified returns true the value of Envelope field was modified since +// Logs was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *Logs) IsEnvelopeModified() bool { + return s.modifiedFields.mask&fieldModifiedLogsEnvelope != 0 +} + +// Resource returns a readonly value. Use SetResource() to modify it. +func (s *Logs) Resource() *Resource { + return s.resource +} + +// SetResource sets the value of Resource field. +func (s *Logs) SetResource(v *Resource) { + if v.canBeShared() { + // v can be shared by pointer. Compute its difference from current resource + if v.computeDiff(s.resource) { + // It is different. Update to it. + s.resource = v + s.modifiedFields.markModified(fieldModifiedLogsResource) + } + } else { + if s.resource.canBeShared() { + s.resource = s.resource.clone(&Allocators{}) + } + s.resource.CopyFrom(v) + s.modifiedFields.markModified(fieldModifiedLogsResource) + } +} + +func (s *Logs) markResourceModified() { + s.modifiedFields.markModified(fieldModifiedLogsResource) +} + +// IsResourceModified returns true the value of Resource field was modified since +// Logs was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *Logs) IsResourceModified() bool { + return s.modifiedFields.mask&fieldModifiedLogsResource != 0 +} + +// Scope returns a readonly value. Use SetScope() to modify it. +func (s *Logs) Scope() *Scope { + return s.scope +} + +// SetScope sets the value of Scope field. +func (s *Logs) SetScope(v *Scope) { + if v.canBeShared() { + // v can be shared by pointer. Compute its difference from current scope + if v.computeDiff(s.scope) { + // It is different. Update to it. + s.scope = v + s.modifiedFields.markModified(fieldModifiedLogsScope) + } + } else { + if s.scope.canBeShared() { + s.scope = s.scope.clone(&Allocators{}) + } + s.scope.CopyFrom(v) + s.modifiedFields.markModified(fieldModifiedLogsScope) + } +} + +func (s *Logs) markScopeModified() { + s.modifiedFields.markModified(fieldModifiedLogsScope) +} + +// IsScopeModified returns true the value of Scope field was modified since +// Logs was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *Logs) IsScopeModified() bool { + return s.modifiedFields.mask&fieldModifiedLogsScope != 0 +} + +func (s *Logs) Log() *LogRecord { + return &s.log +} + +func (s *Logs) markLogModified() { + s.modifiedFields.markModified(fieldModifiedLogsLog) +} + +// IsLogModified returns true the value of Log field was modified since +// Logs was created, encoded or decoded. If the field is modified +// it will be encoded by the next Write() operation. If the field is decoded by the +// next Read() operation the modified flag will be set. +func (s *Logs) IsLogModified() bool { + return s.modifiedFields.mask&fieldModifiedLogsLog != 0 +} + +func (s *Logs) setModifiedRecursively() { + s.envelope.setModifiedRecursively() + s.resource.setModifiedRecursively() + s.scope.setModifiedRecursively() + s.log.setModifiedRecursively() + s.modifiedFields.mask = + fieldModifiedLogsEnvelope | + fieldModifiedLogsResource | + fieldModifiedLogsScope | + fieldModifiedLogsLog | 0 +} + +func (s *Logs) setUnmodifiedRecursively() { + if s.IsEnvelopeModified() { + s.envelope.setUnmodifiedRecursively() + } + if s.IsResourceModified() { + s.resource.setUnmodifiedRecursively() + } + if s.IsScopeModified() { + s.scope.setUnmodifiedRecursively() + } + if s.IsLogModified() { + s.log.setUnmodifiedRecursively() + } + s.modifiedFields.mask = 0 +} + +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Logs) clearModifiedRecursively() { + s.envelope.clearModifiedRecursively() + if !s.resource.canBeShared() { + s.resource.clearModifiedRecursively() + } + if !s.scope.canBeShared() { + s.scope.clearModifiedRecursively() + } + s.log.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + +// computeDiff compares s and val and returns true if they differ. +// All fields that are different in s will be marked as modified. +func (s *Logs) computeDiff(val *Logs) (ret bool) { + // Compare Envelope field. + if s.envelope.computeDiff(&val.envelope) { + s.modifiedFields.setModified(fieldModifiedLogsEnvelope) + ret = true + } + // Compare Resource field. + if s.resource.computeDiff(val.resource) { + s.modifiedFields.setModified(fieldModifiedLogsResource) + ret = true + } + // Compare Scope field. + if s.scope.computeDiff(val.scope) { + s.modifiedFields.setModified(fieldModifiedLogsScope) + ret = true + } + // Compare Log field. + if s.log.computeDiff(&val.log) { + s.modifiedFields.setModified(fieldModifiedLogsLog) + ret = true + } + return ret +} + +// canBeShared returns true if s is safe to share by pointer without cloning (for example if s is frozen). +func (s *Logs) canBeShared() bool { + return false +} + +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Logs) CloneTo(dst *Logs, allocators *Allocators) { + *dst = Logs{} + copyToNewEnvelope(&dst.envelope, &s.envelope, allocators) + dst.resource = s.resource.cloneShared(allocators) + dst.scope = s.scope.cloneShared(allocators) + copyToNewLogRecord(&dst.log, &s.log, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() +} + +// ByteSize returns approximate memory usage in bytes. Used to calculate +// memory used by dictionaries. +func (s *Logs) byteSize() uint { + return uint(unsafe.Sizeof(*s)) + + s.envelope.byteSize() + s.resource.byteSize() + s.scope.byteSize() + s.log.byteSize() + 0 +} + +// Copy from src to dst, overwriting existing data in dst. +func copyLogs(dst *Logs, src *Logs) { + copyEnvelope(&dst.envelope, &src.envelope) + + if src.resource.canBeShared() { + if src.resource.computeDiff(dst.resource) { + dst.resource = src.resource + dst.markResourceModified() + } + } else { + if dst.resource == nil || dst.resource.canBeShared() { // Not allowed to modify shared data. + dst.resource = new(Resource) + dst.resource.init(&dst.modifiedFields, fieldModifiedLogsResource) + } + copyResource(dst.resource, src.resource) + } + + if src.scope.canBeShared() { + if src.scope.computeDiff(dst.scope) { + dst.scope = src.scope + dst.markScopeModified() + } + } else { + if dst.scope == nil || dst.scope.canBeShared() { // Not allowed to modify shared data. + dst.scope = new(Scope) + dst.scope.init(&dst.modifiedFields, fieldModifiedLogsScope) + } + copyScope(dst.scope, src.scope) + } + copyLogRecord(&dst.log, &src.log) +} + +// Copy from src to dst. dst is assumed to be just inited. +func copyToNewLogs(dst *Logs, src *Logs, allocators *Allocators) { + copyToNewEnvelope(&dst.envelope, &src.envelope, allocators) + + if src.resource.canBeShared() { + dst.resource = src.resource + } else { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Resource{}))) + dst.resource = allocators.Resource.Alloc() + dst.resource.init(&dst.modifiedFields, fieldModifiedLogsResource) + copyToNewResource(dst.resource, src.resource, allocators) + } + + if src.scope.canBeShared() { + dst.scope = src.scope + } else { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Scope{}))) + dst.scope = allocators.Scope.Alloc() + dst.scope.init(&dst.modifiedFields, fieldModifiedLogsScope) + copyToNewScope(dst.scope, src.scope, allocators) + } + + copyToNewLogRecord(&dst.log, &src.log, allocators) +} + +// CopyFrom() performs a deep copy from src. +func (s *Logs) CopyFrom(src *Logs) { + copyLogs(s, src) +} + +// mutateRandom mutates fields in a random, deterministic manner using +// random parameter as a deterministic generator. Only fields that exist +// in the schem are mutated, allowing to generate data for specified schema. +// The state of "modified" flags is undefined after this operation. +func (s *Logs) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter *mutateRandomLimiter) { + // Get the field count for this struct from the schema. If the schema specifies + // fewer field count than the one we have in this code then we will not mutate + // fields that are not in the schema. + fieldCount, err := schem.FieldCount("Logs") + if err != nil { + panic(fmt.Sprintf("cannot get field count for %s: %v", "Logs", err)) + } + + const randRange = max(4, 2) // At least 2 to ensure we don't recurse infinitely if there is only 1 field. + + if fieldCount <= 0 { + return // Envelope and all subsequent fields are skipped. + } + // Maybe mutate Envelope + if random.IntN(randRange) == 0 { + s.envelope.mutateRandom(random, schem, limiter) + } + if fieldCount <= 1 { + return // Resource and all subsequent fields are skipped. + } + // Maybe mutate Resource + if random.IntN(randRange) == 0 { + if random.IntN(10) == 0 { + // Freeze and replace with a clone to test frozen object dictionary handling. + s.resource.Freeze() + if random.IntN(10) == 0 { + // Reset to brand new object once in a while to test the code path + // where a dict-based is not mutated, but created from scratch. + s.resource = new(Resource) + s.resource.init(&s.modifiedFields, fieldModifiedLogsResource) + } else { + s.resource = s.resource.clone(&Allocators{}) + } + } + if s.resource.canBeShared() { + // resource may be shared by pointer. Clone it to have exclusive ownership. + s.resource = s.resource.clone(&Allocators{}) + } + + s.resource.mutateRandom(random, schem, limiter) + } + if fieldCount <= 2 { + return // Scope and all subsequent fields are skipped. + } + // Maybe mutate Scope + if random.IntN(randRange) == 0 { + if random.IntN(10) == 0 { + // Freeze and replace with a clone to test frozen object dictionary handling. + s.scope.Freeze() + if random.IntN(10) == 0 { + // Reset to brand new object once in a while to test the code path + // where a dict-based is not mutated, but created from scratch. + s.scope = new(Scope) + s.scope.init(&s.modifiedFields, fieldModifiedLogsScope) + } else { + s.scope = s.scope.clone(&Allocators{}) + } + } + if s.scope.canBeShared() { + // scope may be shared by pointer. Clone it to have exclusive ownership. + s.scope = s.scope.clone(&Allocators{}) + } + + s.scope.mutateRandom(random, schem, limiter) + } + if fieldCount <= 3 { + return // Log and all subsequent fields are skipped. + } + // Maybe mutate Log + if random.IntN(randRange) == 0 { + s.log.mutateRandom(random, schem, limiter) + } +} + +// IsEqual performs deep comparison and returns true if struct is equal to right. +func (s *Logs) IsEqual(right *Logs) bool { + // Compare Envelope field. + if !s.envelope.IsEqual(&right.envelope) { + return false + } + // Compare Resource field. + if !s.resource.IsEqual(right.resource) { + return false + } + // Compare Scope field. + if !s.scope.IsEqual(right.scope) { + return false + } + // Compare Log field. + if !s.log.IsEqual(&right.log) { + return false + } + + return true +} + +// CmpLogs performs deep comparison and returns an integer that +// will be 0 if left == right, negative if left < right, positive if left > right. +func CmpLogs(left, right *Logs) int { + // Compare Envelope field. + if c := CmpEnvelope(&left.envelope, &right.envelope); c != 0 { + return c + } + // Compare Resource field. + if c := CmpResource(left.resource, right.resource); c != 0 { + return c + } + // Compare Scope field. + if c := CmpScope(left.scope, right.scope); c != 0 { + return c + } + // Compare Log field. + if c := CmpLogRecord(&left.log, &right.log); c != 0 { + return c + } + return 0 +} + +// LogsEncoder implements encoding of Logs +type LogsEncoder struct { + buf pkg.BitsWriter + limiter *pkg.SizeLimiter + + // forceModifiedFields is set to a mask to force the next encoding operation + // write the fields, whether they are modified or no. This is used after frame + // restarts so that the data can be decoded from the frame start. + forceModifiedFields uint64 + + envelopeEncoder *EnvelopeEncoder + isEnvelopeRecursive bool // Indicates Envelope field's type is recursive. + resourceEncoder *ResourceEncoder + isResourceRecursive bool // Indicates Resource field's type is recursive. + scopeEncoder *ScopeEncoder + isScopeRecursive bool // Indicates Scope field's type is recursive. + logEncoder *LogRecordEncoder + isLogRecursive bool // Indicates Log field's type is recursive. + + allocators *Allocators + + keepFieldMask uint64 + fieldCount uint +} + +func (e *LogsEncoder) Init(state *WriterState, columns *pkg.WriteColumnSet) error { + // Remember this encoder in the state so that we can detect recursion. + if state.LogsEncoder != nil { + panic("cannot initialize LogsEncoder: already initialized") + } + state.LogsEncoder = e + defer func() { state.LogsEncoder = nil }() + + e.limiter = &state.limiter + e.allocators = &state.Allocators + + // Number of fields in the output data schema. + var err error + e.fieldCount, err = state.StructFieldCounts.LogsFieldCount() + if err != nil { + return fmt.Errorf("cannot find struct %s in override schema: %w", "Logs", err) + } + // Set that many 1 bits in the keepFieldMask. All fields with higher number + // will be skipped when encoding. + e.keepFieldMask = ^(^uint64(0) << e.fieldCount) + + // Init encoder for Envelope field. + if e.fieldCount <= 0 { + return nil // Envelope and all subsequent fields are skipped. + } + if state.EnvelopeEncoder != nil { + // Recursion detected, use the existing encoder. + e.envelopeEncoder = state.EnvelopeEncoder + e.isEnvelopeRecursive = true + } else { + e.envelopeEncoder = new(EnvelopeEncoder) + err = e.envelopeEncoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + + // Init encoder for Resource field. + if e.fieldCount <= 1 { + return nil // Resource and all subsequent fields are skipped. + } + if state.ResourceEncoder != nil { + // Recursion detected, use the existing encoder. + e.resourceEncoder = state.ResourceEncoder + e.isResourceRecursive = true + } else { + e.resourceEncoder = new(ResourceEncoder) + err = e.resourceEncoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + + // Init encoder for Scope field. + if e.fieldCount <= 2 { + return nil // Scope and all subsequent fields are skipped. + } + if state.ScopeEncoder != nil { + // Recursion detected, use the existing encoder. + e.scopeEncoder = state.ScopeEncoder + e.isScopeRecursive = true + } else { + e.scopeEncoder = new(ScopeEncoder) + err = e.scopeEncoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + + // Init encoder for Log field. + if e.fieldCount <= 3 { + return nil // Log and all subsequent fields are skipped. + } + if state.LogRecordEncoder != nil { + // Recursion detected, use the existing encoder. + e.logEncoder = state.LogRecordEncoder + e.isLogRecursive = true + } else { + e.logEncoder = new(LogRecordEncoder) + err = e.logEncoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + + return nil +} + +func (e *LogsEncoder) Reset() { + // Since we are resetting the state of encoder make sure the next Encode() + // call forcedly writes all fields and does not attempt to skip. + e.forceModifiedFields = e.keepFieldMask + + if e.fieldCount <= 0 { + return // Envelope and all subsequent fields are skipped. + } + if !e.isEnvelopeRecursive { + e.envelopeEncoder.Reset() + } + if e.fieldCount <= 1 { + return // Resource and all subsequent fields are skipped. + } + if !e.isResourceRecursive { + e.resourceEncoder.Reset() + } + if e.fieldCount <= 2 { + return // Scope and all subsequent fields are skipped. + } + if !e.isScopeRecursive { + e.scopeEncoder.Reset() + } + if e.fieldCount <= 3 { + return // Log and all subsequent fields are skipped. + } + if !e.isLogRecursive { + e.logEncoder.Reset() + } +} + +// Encode encodes val into buf +func (e *LogsEncoder) Encode(val *Logs) { + var bitCount uint + + // Mask that describes what fields are encoded. Start with all modified fields. + fieldMask := val.modifiedFields.mask + + // If forceModifiedFields we need to set to 1 all bits so that we + // force writing of all fields. + fieldMask |= e.forceModifiedFields + e.forceModifiedFields = 0 + + // Only write fields that we want to write. See Init() for keepFieldMask. + fieldMask &= e.keepFieldMask + + // Write bits to indicate which fields follow. + e.buf.WriteBits(fieldMask, e.fieldCount) + bitCount += e.fieldCount + + // Encode modified, present fields. + + if fieldMask&fieldModifiedLogsEnvelope != 0 { + // Encode Envelope + e.envelopeEncoder.Encode(&val.envelope) + } + + if fieldMask&fieldModifiedLogsResource != 0 { + // Encode Resource + e.resourceEncoder.Encode(val.resource) + } + + if fieldMask&fieldModifiedLogsScope != 0 { + // Encode Scope + e.scopeEncoder.Encode(val.scope) + } + + if fieldMask&fieldModifiedLogsLog != 0 { + // Encode Log + e.logEncoder.Encode(&val.log) + } + + // Account written bits in the limiter. + e.limiter.AddFrameBits(bitCount) + + // Mark all fields non-modified so that next Encode() correctly + // encodes only fields that change after this. + val.modifiedFields.mask = 0 +} + +// CollectColumns collects all buffers from all encoders into buf. +func (e *LogsEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { + columnSet.SetBits(&e.buf) + colIdx := 0 + // Collect Envelope field. + if e.fieldCount <= 0 { + return // Envelope and subsequent fields are skipped. + } + if !e.isEnvelopeRecursive { + e.envelopeEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + } + // Collect Resource field. + if e.fieldCount <= 1 { + return // Resource and subsequent fields are skipped. + } + if !e.isResourceRecursive { + e.resourceEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + } + // Collect Scope field. + if e.fieldCount <= 2 { + return // Scope and subsequent fields are skipped. + } + if !e.isScopeRecursive { + e.scopeEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + } + // Collect Log field. + if e.fieldCount <= 3 { + return // Log and subsequent fields are skipped. + } + if !e.isLogRecursive { + e.logEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ + } +} + +// LogsDecoder implements decoding of Logs +type LogsDecoder struct { + buf pkg.BitsReader + column *pkg.ReadableColumn + fieldCount uint + + envelopeDecoder *EnvelopeDecoder + isEnvelopeRecursive bool + resourceDecoder *ResourceDecoder + isResourceRecursive bool + scopeDecoder *ScopeDecoder + isScopeRecursive bool + logDecoder *LogRecordDecoder + isLogRecursive bool + allocators *Allocators +} + +// Init is called once in the lifetime of the stream. +func (d *LogsDecoder) Init(state *ReaderState, columns *pkg.ReadColumnSet) error { + // Remember this decoder in the state so that we can detect recursion. + if state.LogsDecoder != nil { + panic("cannot initialize LogsDecoder: already initialized") + } + state.LogsDecoder = d + defer func() { state.LogsDecoder = nil }() + + d.allocators = &state.Allocators + + // Number of fields in the input data schema. + var err error + d.fieldCount, err = state.StructFieldCounts.LogsFieldCount() + if err != nil { + return fmt.Errorf("cannot find struct %s in override schema: %w", "Logs", err) + } + if d.fieldCount > 4 { + return pkg.ErrTooManyFieldsToDecode + } + + if d.fieldCount == 0 { + return pkg.ErrEmptyRootStructDisallowed + } + + d.column = columns.Column() + + if d.fieldCount <= 0 { + return nil // Envelope and subsequent fields are skipped. + } + if state.EnvelopeDecoder != nil { + // Recursion detected, use the existing decoder. + d.envelopeDecoder = state.EnvelopeDecoder + d.isEnvelopeRecursive = true // Mark that we are using a recursive decoder. + } else { + d.envelopeDecoder = new(EnvelopeDecoder) + err = d.envelopeDecoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + if d.fieldCount <= 1 { + return nil // Resource and subsequent fields are skipped. + } + if state.ResourceDecoder != nil { + // Recursion detected, use the existing decoder. + d.resourceDecoder = state.ResourceDecoder + d.isResourceRecursive = true // Mark that we are using a recursive decoder. + } else { + d.resourceDecoder = new(ResourceDecoder) + err = d.resourceDecoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + if d.fieldCount <= 2 { + return nil // Scope and subsequent fields are skipped. + } + if state.ScopeDecoder != nil { + // Recursion detected, use the existing decoder. + d.scopeDecoder = state.ScopeDecoder + d.isScopeRecursive = true // Mark that we are using a recursive decoder. + } else { + d.scopeDecoder = new(ScopeDecoder) + err = d.scopeDecoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + if d.fieldCount <= 3 { + return nil // Log and subsequent fields are skipped. + } + if state.LogRecordDecoder != nil { + // Recursion detected, use the existing decoder. + d.logDecoder = state.LogRecordDecoder + d.isLogRecursive = true // Mark that we are using a recursive decoder. + } else { + d.logDecoder = new(LogRecordDecoder) + err = d.logDecoder.Init(state, columns.AddSubColumn()) + } + if err != nil { + return err + } + + return nil +} + +// Continue is called at the start of the frame to continue decoding column data. +// This should set the decoder's source buffer, so the new decoding continues from +// the supplied column data. This should NOT reset the internal state of the decoder, +// since columns can cross frame boundaries and the new column data is considered +// continuation of that same column in the previous frame. +func (d *LogsDecoder) Continue() { + d.buf.Reset(d.column.Data()) + + if d.fieldCount <= 0 { + return // Envelope and subsequent fields are skipped. + } + + if !d.isEnvelopeRecursive { + d.envelopeDecoder.Continue() + } + + if d.fieldCount <= 1 { + return // Resource and subsequent fields are skipped. + } + + if !d.isResourceRecursive { + d.resourceDecoder.Continue() + } + + if d.fieldCount <= 2 { + return // Scope and subsequent fields are skipped. + } + + if !d.isScopeRecursive { + d.scopeDecoder.Continue() + } + + if d.fieldCount <= 3 { + return // Log and subsequent fields are skipped. + } + + if !d.isLogRecursive { + d.logDecoder.Continue() + } + +} + +func (d *LogsDecoder) Reset() { + + if d.fieldCount <= 0 { + return // Envelope and all subsequent fields are skipped. + } + + if !d.isEnvelopeRecursive { + d.envelopeDecoder.Reset() + } + + if d.fieldCount <= 1 { + return // Resource and all subsequent fields are skipped. + } + + if !d.isResourceRecursive { + d.resourceDecoder.Reset() + } + + if d.fieldCount <= 2 { + return // Scope and all subsequent fields are skipped. + } + + if !d.isScopeRecursive { + d.scopeDecoder.Reset() + } + + if d.fieldCount <= 3 { + return // Log and all subsequent fields are skipped. + } + + if !d.isLogRecursive { + d.logDecoder.Reset() + } + +} + +func (d *LogsDecoder) Decode(dstPtr *Logs) error { + val := dstPtr + + // Read bits that indicate which fields follow. + val.modifiedFields.mask = d.buf.PeekBits(d.fieldCount) + d.buf.Consume(d.fieldCount) + + if val.modifiedFields.mask&fieldModifiedLogsEnvelope != 0 { // Envelope is changed. + + if err := d.envelopeDecoder.Decode(&val.envelope); err != nil { + return err + } + } + + if val.modifiedFields.mask&fieldModifiedLogsResource != 0 { // Resource is changed. + + if val.resource == nil { + if err := d.allocators.allocSizeChecker.PrepAllocSize(uint(unsafe.Sizeof(*val.resource))); err != nil { + return err + } + val.resource = d.allocators.Resource.Alloc() + val.resource.init(&val.modifiedFields, fieldModifiedLogsResource) + } + + if err := d.resourceDecoder.Decode(&val.resource); err != nil { + return err + } + if val.resource == nil { + return pkg.ErrDecodeError // Non-optional fields cannot be nil + } + } + + if val.modifiedFields.mask&fieldModifiedLogsScope != 0 { // Scope is changed. + + if val.scope == nil { + if err := d.allocators.allocSizeChecker.PrepAllocSize(uint(unsafe.Sizeof(*val.scope))); err != nil { + return err + } + val.scope = d.allocators.Scope.Alloc() + val.scope.init(&val.modifiedFields, fieldModifiedLogsScope) + } + + if err := d.scopeDecoder.Decode(&val.scope); err != nil { + return err + } + if val.scope == nil { + return pkg.ErrDecodeError // Non-optional fields cannot be nil + } + } + + if val.modifiedFields.mask&fieldModifiedLogsLog != 0 { // Log is changed. + + if err := d.logDecoder.Decode(&val.log); err != nil { + return err + } + } + + return d.buf.Error() +} + +// LogsAllocator implements a custom allocator for Logs. +// It maintains a pool of pre-allocated Logs and grows the pool +// dynamically as needed, up to a maximum size of 64 elements. +type LogsAllocator struct { + pool []Logs + ofs int +} + +// Alloc returns the next available Logs from the pool. +// If the pool is exhausted, it grows the pool by doubling its size +// up to a maximum of 64 elements. +func (a *LogsAllocator) Alloc() *Logs { + if a.ofs < len(a.pool) { + // Get the next available Logs from the pool + a.ofs++ + return &a.pool[a.ofs-1] + } + // We've exhausted the current pool, prealloc a new pool. + return a.prealloc() +} + +//go:noinline +func (a *LogsAllocator) prealloc() *Logs { + // prealloc expands the pool by doubling its size, up to a maximum of 64 elements. + // If the pool is empty, it starts with 1 element. + newLen := min(max(len(a.pool)*2, 1), 64) + a.pool = make([]Logs, newLen) + a.ofs = 1 + return &a.pool[0] +} + +var wireSchemaLogs = []byte{0x06, 0x04, 0x01, 0x03, 0x07, 0x05, 0x0B} + +func LogsWireSchema() (schema.WireSchema, error) { + var w schema.WireSchema + if err := w.Deserialize(bytes.NewReader(wireSchemaLogs)); err != nil { + return w, err + } + return w, nil +} diff --git a/go/otel/otelstef/logsreader.go b/go/otel/otelstef/logsreader.go new file mode 100644 index 00000000..d56818bc --- /dev/null +++ b/go/otel/otelstef/logsreader.go @@ -0,0 +1,140 @@ +// Code generated by stefc. DO NOT EDIT. +package otelstef + +import ( + "bufio" + "errors" + "fmt" + "io" + + "github.com/splunk/stef/go/pkg" + "github.com/splunk/stef/go/pkg/schema" +) + +type LogsReader struct { + // Record contains the record that was just read by the last Read() operation. + // Do not modify this field externally. The next Read() will overwrite the Record. + Record Logs + + base pkg.BaseReader + decoder LogsDecoder + state ReaderState + recordPtr *Logs +} + +func NewLogsReader(source io.Reader) (*LogsReader, error) { + bufferedSource := bufio.NewReaderSize(source, 64*1024) + reader := &LogsReader{} + + reader.Record.Init() + reader.recordPtr = &reader.Record + + if err := reader.base.Init(bufferedSource); err != nil { + return nil, err + } + + if err := reader.initSchema(); err != nil { + return nil, err + } + + return reader, nil +} + +func (r *LogsReader) initSchema() error { + ownSchema, err := LogsWireSchema() + if err != nil { + return err + } + if err := r.base.ReadVarHeader(ownSchema); err != nil { + return err + } + + r.state.Init(r.base.Schema) + err = r.decoder.Init(&r.state, &r.base.ReadBufs.Columns) + if err != nil { + return fmt.Errorf("schema is not compatible with Reader: %w", err) + } + + if !r.state.StructFieldCounts.AllFetched() { + return errors.New("override schema iterator is not done, decoded data schema is likely incompatible") + } + + return nil +} + +func (f *LogsReader) Header() *pkg.FixedHeader { + return &f.base.FixedHeader +} + +func (f *LogsReader) UserData() map[string]string { + return f.base.VarHeader.UserData +} + +// Read the next record. After Read() returns successfully the record +// will be accessible in LogsReader.Record field. +// +// If ReadOptions.TillEndOfFrame=true and end of the current +// frame is reached Read() will return pkg.ErrEndOfFrame. Calling Read() +// after that with ReadOptions.TillEndOfFrame=false will correctly load +// the next frame (if any) and read the next record from the frame. +// The very first Read() must use ReadOptions.TillEndOfFrame=false +// otherwise pkg.ErrEndOfFrame will be returned since no frames are +// initially loaded in the Reader. See more details in pkg.ReadOptions. +// +// If Read() encounters a decoding error one of the pkg.Err values will +// be returned. +// If underlying source io.Reader returns any error then Read() will +// either return that error or a decoding error if source io.Reader +// returned an error prematurely while more data was expected in STEF stream. +// +// For well-formed streams that don't encounter decoding errors Read() will +// return io.EOF once end of the underlying source io.Reader is reached +// (assuming io.Reader returns io.EOF itself). +func (r *LogsReader) Read(opts pkg.ReadOptions) error { + for r.base.FrameRecordCount == 0 { + if opts.TillEndOfFrame { + return pkg.ErrEndOfFrame + } + if err := r.nextFrame(); err != nil { + return err + } + } + r.base.FrameRecordCount-- + r.base.RecordCount++ + + // Track new memory allocated during decoding of this record. + r.state.Allocators.allocSizeChecker.ResetAllocSize() + + return r.decoder.Decode(r.recordPtr) +} + +func (r *LogsReader) RecordCount() uint64 { + return r.base.RecordCount +} + +func (r *LogsReader) nextFrame() error { + frameFlags, err := r.base.NextFrame() + if err != nil { + return err + } + + if frameFlags&pkg.RestartDictionaries != 0 { + // The frame that has just started indicates that the dictionaries + // must be restarted. Reset all dictionaries. + r.state.ResetDicts() + } + + if frameFlags&pkg.RestartCodecs != 0 { + // The frame that has just started indicates that the decoders + // must be restarted. + r.decoder.Reset() + } + + r.decoder.Continue() + return nil +} + +// Schema returns the schema of the STEF stream being read. +func (r *LogsReader) Schema() *schema.WireSchema { + return r.base.Schema +} diff --git a/go/otel/otelstef/logswriter.go b/go/otel/otelstef/logswriter.go new file mode 100644 index 00000000..c4da3fd1 --- /dev/null +++ b/go/otel/otelstef/logswriter.go @@ -0,0 +1,210 @@ +// Code generated by stefc. DO NOT EDIT. +package otelstef + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + + "github.com/splunk/stef/go/pkg" +) + +type LogsWriter struct { + Record Logs + + opts pkg.WriterOptions + dst pkg.ChunkWriter + frameEncoder pkg.FrameEncoder + encoder LogsEncoder + state WriterState + writeBufs pkg.WriteBufs + frameRecordCount uint64 + recordCount uint64 +} + +func NewLogsWriter(dst pkg.ChunkWriter, opts pkg.WriterOptions) (*LogsWriter, error) { + writer := &LogsWriter{ + dst: dst, + opts: opts, + } + + // Set default options. + if writer.opts.MaxUncompressedFrameByteSize == 0 { + writer.opts.MaxUncompressedFrameByteSize = pkg.DefaultMaxFrameSize + } + if writer.opts.MaxTotalDictSize == 0 { + writer.opts.MaxTotalDictSize = pkg.DefaultMaxTotalDictSize + } + + if writer.opts.Schema != nil { + // If the schema is overridden must include the descriptor so that the readers + // can decode the data correctly. + writer.opts.IncludeDescriptor = true + + ownSchema, err := LogsWireSchema() + if err != nil { + return nil, err + } + if _, err := ownSchema.Compatible(writer.opts.Schema); err != nil { + return nil, fmt.Errorf("schema is not compatible with Writer: %w", err) + } + } + + writer.Record.Init() + writer.state.Init(&writer.opts) + if err := writer.encoder.Init(&writer.state, &writer.writeBufs.Columns); err != nil { + return nil, err + } + + if !writer.state.StructFieldCounts.AllFetched() { + return nil, errors.New("override schema iterator is not done, likely supplied schema override does not match the generated code") + } + + if err := writer.frameEncoder.Init(dst, writer.opts.Compression); err != nil { + return nil, err + } + + if err := writer.writeFixedHeader(); err != nil { + return nil, err + } + + if err := writer.writeVarHeader(); err != nil { + return nil, err + } + + return writer, nil +} + +func (f *LogsWriter) writeFixedHeader() error { + var hdrTail []byte + + var versionAndType byte = pkg.HdrFormatVersion + hdrTail = append(hdrTail, versionAndType) + + flags := byte(f.opts.Compression) & pkg.HdrFlagsCompressionMethod + + hdrTail = append(hdrTail, flags) + hdrTailSize := uint64(len(hdrTail)) + + var hdrFull []byte + hdrFull = append(hdrFull, pkg.HdrSignature...) + hdrFull = binary.AppendUvarint(hdrFull, hdrTailSize) + hdrFull = append(hdrFull, hdrTail...) + + err := f.dst.WriteChunk(hdrFull, nil) + return err +} + +func (w *LogsWriter) writeVarHeader() error { + // Serialize varheader + hdr := pkg.VarHeader{} + if w.opts.IncludeDescriptor { + if w.opts.Schema != nil { + var buf bytes.Buffer + err := w.opts.Schema.Serialize(&buf) + if err != nil { + return fmt.Errorf("could not marshal schema: %w", err) + } + hdr.SchemaWireBytes = buf.Bytes() + } else { + hdr.SchemaWireBytes = []byte(wireSchemaLogs) + } + } + + if len(w.opts.UserData) > 0 { + hdr.UserData = w.opts.UserData + } + + var buf bytes.Buffer + err := hdr.Serialize(&buf) + if err != nil { + return err + } + + // Write to the frame + _, err = w.frameEncoder.Write(buf.Bytes()) + if err != nil { + return err + } + + // Close the the frame. + if err := w.frameEncoder.CloseFrame(); err != nil { + return err + } + + // Open a new frame. + w.frameEncoder.OpenFrame(0) + + return nil +} + +func (w *LogsWriter) Write() error { + w.encoder.Encode(&w.Record) + w.frameRecordCount++ + + nextFrameFlags := w.opts.FrameRestartFlags + restartFrame := false + if w.state.limiter.DictLimitReached() || (nextFrameFlags&pkg.RestartDictionaries != 0) { + w.state.ResetDicts() + nextFrameFlags = w.opts.FrameRestartFlags | pkg.RestartDictionaries + restartFrame = true + } + + if w.state.limiter.FrameLimitReached() { + restartFrame = true + } + + if restartFrame { + if err := w.restartFrame(nextFrameFlags); err != nil { + return err + } + } + + w.recordCount++ + + return nil +} + +func (w *LogsWriter) RecordCount() uint64 { + return w.recordCount +} + +func (w *LogsWriter) restartFrame(nextFrameFlags pkg.FrameFlags) error { + if nextFrameFlags&pkg.RestartCodecs != 0 { + w.encoder.Reset() + } + + // Write record count. + if _, err := w.frameEncoder.Write(binary.AppendUvarint(nil, w.frameRecordCount)); err != nil { + return err + } + w.frameRecordCount = 0 + + // Write from column buffers to the frame buffer. + w.encoder.CollectColumns(&w.writeBufs.Columns) + if err := w.writeBufs.WriteTo(&w.frameEncoder); err != nil { + return err + } + + // Close the current frame. + if err := w.frameEncoder.CloseFrame(); err != nil { + return err + } + + // Open a new frame. + w.frameEncoder.OpenFrame(nextFrameFlags) + + // Reset the frame size limiter, this is a new frame. + w.state.limiter.ResetFrameSize() + + return nil +} + +func (w *LogsWriter) Flush() error { + if w.frameRecordCount == 0 { + // Nothing to flush. + return nil + } + return w.restartFrame(w.opts.FrameRestartFlags) +} diff --git a/go/otel/otelstef/logswriter_test.go b/go/otel/otelstef/logswriter_test.go new file mode 100644 index 00000000..56290b2b --- /dev/null +++ b/go/otel/otelstef/logswriter_test.go @@ -0,0 +1,251 @@ +// Code generated by stefc. DO NOT EDIT. +package otelstef + +import ( + "bytes" + "fmt" + "io" + "math/rand/v2" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/splunk/stef/go/pkg" + "github.com/splunk/stef/go/pkg/idl" + "github.com/splunk/stef/go/pkg/schema" +) + +// genLogsRecords generates a number of records pseudo-randomly +// using the supplied Rand generator. Generated records will be always +// the same for the same input state of Rand generator. +// Generated records will only have fields set (mutated) that are defined +// in the supplied schema. This allows testing schema evolution. +func genLogsRecords(random *rand.Rand, schem *schema.Schema) (records []Logs) { + const recCount = 1000 + var record Logs + record.Init() + + records = make([]Logs, recCount) + for i := 0; i < recCount; i++ { + limiter := &mutateRandomLimiter{} + record.mutateRandom(random, schem, limiter) + records[i].Init() + records[i].CopyFrom(&record) + } + + return records +} + +func testLogsWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { + retVal = true + + random := rand.New(rand.NewPCG(seed, 0)) + + // Load the schema from the allSchemaContent variable. + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(t, err, "seed %v", seed) + + schem, err = schem.PrunedForRoot("Logs") + require.NoError(t, err, "seed %v", seed) + + if random.IntN(2) == 0 { + // Randomly shrink the schema in approximately half of the test runs. + // This is to test that the writer/reader can handle schema changes. + schema.ShrinkRandomly(random, schem) + } + wireSchema := schema.NewWireSchema(schem, "Logs") + + for _, opt := range testWriterOpts { + t.Run( + "", func(t *testing.T) { + succeeded := false + defer func() { + if !succeeded { + retVal = false + fmt.Printf("Test failed with seed %v\n", seed) + } + }() + + // Write data according to (possibly modified) schema + optCpy := opt + optCpy.Schema = &wireSchema + + buf := &pkg.MemChunkWriter{} + writer, err := NewLogsWriter(buf, optCpy) + require.NoError(t, err, "seed %v", seed) + + // Generate records pseudo-randomly + records := genLogsRecords(random, schem) + // Write the records + for i := 0; i < len(records); i++ { + writer.Record.CopyFrom(&records[i]) + err = writer.Write() + require.NoError(t, err, "record %d seed %v", i, seed) + } + err = writer.Flush() + require.NoError(t, err, "seed %v", seed) + + // Read the records and compare to written. + reader, err := NewLogsReader(bytes.NewBuffer(buf.Bytes())) + require.NoError(t, err, "seed %v", seed) + + for i := 0; i < len(records); i++ { + err := reader.Read(pkg.ReadOptions{}) + require.NoError(t, err, "record %d seed %v", i, seed) + require.True(t, reader.Record.IsEqual(&records[i]), "record %d seed %v", i, seed) + } + err = reader.Read(pkg.ReadOptions{}) + require.ErrorIs(t, err, io.EOF, seed) + + succeeded = true + }, + ) + } + return retVal +} + +func TestLogsWriteRead(t *testing.T) { + seedFileName := "../../../seeds/otelstef_Logs_seeds.txt" + seedsBytes, err := os.ReadFile(seedFileName) + var seeds []string + if err == nil { + seeds = strings.Split(strings.TrimSpace(string(seedsBytes)), "\n") + } + + for _, seedStr := range seeds { + seed, err := strconv.ParseUint(seedStr, 10, 64) + fmt.Printf("Testing with seed from file: %v\n", seed) + require.NoError(t, err, "parsing seed from file "+seedFileName) + testLogsWriteReadSeed(t, seed) + } + + // Choose a seed (non-pseudo) randomly. We will print the seed + // on failure for easy reproduction. + seed := uint64(time.Now().UnixNano()) + + succeeded := false + defer func() { + if !succeeded { + fmt.Printf("Test failed with seed %v, adding to seed file\n", seed) + err := os.WriteFile(seedFileName, + []byte(string(seedsBytes)+fmt.Sprintf("%v\n", seed)), 0644) + require.NoError(t, err) + } + }() + + succeeded = testLogsWriteReadSeed(t, seed) +} + +func TestLogsWriteReadLong(t *testing.T) { + // Tests writing and reading a larger number of records + // in a single writer/reader session. + + seed := uint64(time.Now().UnixNano()) + random := rand.New(rand.NewPCG(seed, 0)) + + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(t, err, "seed %v", seed) + + schem, err = schem.PrunedForRoot("Logs") + require.NoError(t, err, "seed %v", seed) + + mem := &pkg.MemReaderWriter{} + + writer, err := NewLogsWriter(mem, pkg.WriterOptions{Compression: pkg.CompressionZstd}) + require.NoError(t, err, "seed %v", seed) + + reader, err := NewLogsReader(mem) + require.NoError(t, err, "seed %v", seed) + + iterations := 10 + if os.Getenv("STEF_ENABLE_SLOW_TESTS") == "1" { + iterations = 100 + } + + var record Logs + record.Init() + + records := make([]Logs, 1000) + for i := 0; i < iterations; i++ { + recCount := 1 + random.IntN(1000) + records = records[:recCount] + + // Write some records. + for recIdx := range records { + limiter := &mutateRandomLimiter{} + record.mutateRandom(random, schem, limiter) + records[recIdx] = Logs{} + records[recIdx].Init() + records[recIdx].CopyFrom(&record) + writer.Record.CopyFrom(&record) + err = writer.Write() + require.NoError(t, err, "record %d:%d seed %v", i, recIdx, seed) + } + + // Flush to make sure the records are in the MemReaderWriter and can be + // read back. + err = writer.Flush() + require.NoError(t, err, "iteration %d seed %v", i, seed) + + // Read the records back and compare. + for recIdx := range records { + err = reader.Read(pkg.ReadOptions{}) + require.NoError(t, err, "record %d:%d seed %v", i, recIdx, seed) + require.True(t, reader.Record.IsEqual(&records[recIdx]), "record %d:%d seed %v", i, recIdx, seed) + } + } + + err = reader.Read(pkg.ReadOptions{}) + require.ErrorIs(t, err, io.EOF, seed) +} + +func FuzzLogsReader(f *testing.F) { + f.Add([]byte("")) + + random := rand.New(rand.NewPCG(0, 0)) + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(f, err) + + for _, opt := range testWriterOpts { + for i := 0; i <= 3; i++ { + buf := &pkg.MemChunkWriter{} + writer, err := NewLogsWriter(buf, opt) + require.NoError(f, err) + + recCount := (1 << (2 * i)) - 1 + var record Logs + record.Init() + for range recCount { + limiter := &mutateRandomLimiter{} + record.mutateRandom(random, schem, limiter) + writer.Record.CopyFrom(&record) + err = writer.Write() + require.NoError(f, err) + } + + err = writer.Flush() + require.NoError(f, err) + + f.Add(buf.Bytes()) + } + } + + f.Fuzz( + func(t *testing.T, data []byte) { + reader, err := NewLogsReader(bytes.NewBuffer(data)) + if err != nil { + return + } + for { + err = reader.Read(pkg.ReadOptions{}) + if err != nil { + break + } + } + }, + ) +} diff --git a/go/otel/otelstef/metric.go b/go/otel/otelstef/metric.go index 171db392..aaa0a98f 100644 --- a/go/otel/otelstef/metric.go +++ b/go/otel/otelstef/metric.go @@ -116,6 +116,14 @@ func (s *Metric) fixParent(parentModifiedFields *modifiedFields) { s.histogramBounds.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Metric) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.metadata.attachParent(&s.modifiedFields, fieldModifiedMetricMetadata) + s.histogramBounds.attachParent(&s.modifiedFields, fieldModifiedMetricHistogramBounds) +} + // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, // which can improve encoding performance. @@ -336,6 +344,14 @@ func (s *Metric) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Metric) clearModifiedRecursively() { + s.metadata.clearModifiedRecursively() + s.histogramBounds.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Metric) computeDiff(val *Metric) (ret bool) { @@ -387,6 +403,14 @@ func (s *Metric) canBeShared() bool { return s.isFrozen() } +// clone returns a clone of s allocated using allocators. +func (s *Metric) clone(allocators *Allocators) *Metric { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Metric{}))) + c := allocators.Metric.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). func (s *Metric) cloneShared(allocators *Allocators) *Metric { @@ -394,13 +418,13 @@ func (s *Metric) cloneShared(allocators *Allocators) *Metric { // If s is frozen it means it is safe to share without cloning. return s } - return s.Clone(allocators) + return s.clone(allocators) } -func (s *Metric) Clone(allocators *Allocators) *Metric { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Metric{}))) - c := allocators.Metric.Alloc() - *c = Metric{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Metric) CloneTo(dst *Metric, allocators *Allocators) { + *dst = Metric{ name: s.name, description: s.description, unit: s.unit, @@ -408,9 +432,10 @@ func (s *Metric) Clone(allocators *Allocators) *Metric { aggregationTemporality: s.aggregationTemporality, monotonic: s.monotonic, } - copyToNewAttributes(&c.metadata, &s.metadata, allocators) - copyToNewFloat64Array(&c.histogramBounds, &s.histogramBounds, allocators) - return c + copyToNewAttributes(&dst.metadata, &s.metadata, allocators) + copyToNewFloat64Array(&dst.histogramBounds, &s.histogramBounds, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -674,7 +699,7 @@ func (d *MetricEncoderDict) Add(val *Metric) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -1224,7 +1249,7 @@ func (d *MetricDecoder) Decode(dstPtr **Metric) error { // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } diff --git a/go/otel/otelstef/metrics.go b/go/otel/otelstef/metrics.go index b86f5934..d240161b 100644 --- a/go/otel/otelstef/metrics.go +++ b/go/otel/otelstef/metrics.go @@ -119,6 +119,24 @@ func (s *Metrics) fixParent(parentModifiedFields *modifiedFields) { s.point.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Metrics) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.envelope.attachParent(&s.modifiedFields, fieldModifiedMetricsEnvelope) + if !s.metric.canBeShared() { + s.metric.attachParent(&s.modifiedFields, fieldModifiedMetricsMetric) + } + if !s.resource.canBeShared() { + s.resource.attachParent(&s.modifiedFields, fieldModifiedMetricsResource) + } + if !s.scope.canBeShared() { + s.scope.attachParent(&s.modifiedFields, fieldModifiedMetricsScope) + } + s.attributes.attachParent(&s.modifiedFields, fieldModifiedMetricsAttributes) + s.point.attachParent(&s.modifiedFields, fieldModifiedMetricsPoint) +} + func (s *Metrics) freeze() { if s.isFrozen() { return @@ -168,7 +186,7 @@ func (s *Metrics) SetMetric(v *Metric) { } } else { if s.metric.canBeShared() { - s.metric = s.metric.Clone(&Allocators{}) + s.metric = s.metric.clone(&Allocators{}) } s.metric.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedMetricsMetric) @@ -203,7 +221,7 @@ func (s *Metrics) SetResource(v *Resource) { } } else { if s.resource.canBeShared() { - s.resource = s.resource.Clone(&Allocators{}) + s.resource = s.resource.clone(&Allocators{}) } s.resource.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedMetricsResource) @@ -238,7 +256,7 @@ func (s *Metrics) SetScope(v *Scope) { } } else { if s.scope.canBeShared() { - s.scope = s.scope.Clone(&Allocators{}) + s.scope = s.scope.clone(&Allocators{}) } s.scope.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedMetricsScope) @@ -327,6 +345,24 @@ func (s *Metrics) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Metrics) clearModifiedRecursively() { + s.envelope.clearModifiedRecursively() + if !s.metric.canBeShared() { + s.metric.clearModifiedRecursively() + } + if !s.resource.canBeShared() { + s.resource.clearModifiedRecursively() + } + if !s.scope.canBeShared() { + s.scope.clearModifiedRecursively() + } + s.attributes.clearModifiedRecursively() + s.point.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Metrics) computeDiff(val *Metrics) (ret bool) { @@ -368,22 +404,18 @@ func (s *Metrics) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Metrics) cloneShared(allocators *Allocators) Metrics { - return s.Clone(allocators) -} - -func (s *Metrics) Clone(allocators *Allocators) Metrics { - c := Metrics{ - metric: s.metric.cloneShared(allocators), - resource: s.resource.cloneShared(allocators), - scope: s.scope.cloneShared(allocators), - } - copyToNewEnvelope(&c.envelope, &s.envelope, allocators) - copyToNewAttributes(&c.attributes, &s.attributes, allocators) - copyToNewPoint(&c.point, &s.point, allocators) - return c +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Metrics) CloneTo(dst *Metrics, allocators *Allocators) { + *dst = Metrics{} + copyToNewEnvelope(&dst.envelope, &s.envelope, allocators) + dst.metric = s.metric.cloneShared(allocators) + dst.resource = s.resource.cloneShared(allocators) + dst.scope = s.scope.cloneShared(allocators) + copyToNewAttributes(&dst.attributes, &s.attributes, allocators) + copyToNewPoint(&dst.point, &s.point, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -515,12 +547,12 @@ func (s *Metrics) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter s.metric = new(Metric) s.metric.init(&s.modifiedFields, fieldModifiedMetricsMetric) } else { - s.metric = s.metric.Clone(&Allocators{}) + s.metric = s.metric.clone(&Allocators{}) } } if s.metric.canBeShared() { // metric may be shared by pointer. Clone it to have exclusive ownership. - s.metric = s.metric.Clone(&Allocators{}) + s.metric = s.metric.clone(&Allocators{}) } s.metric.mutateRandom(random, schem, limiter) @@ -539,12 +571,12 @@ func (s *Metrics) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter s.resource = new(Resource) s.resource.init(&s.modifiedFields, fieldModifiedMetricsResource) } else { - s.resource = s.resource.Clone(&Allocators{}) + s.resource = s.resource.clone(&Allocators{}) } } if s.resource.canBeShared() { // resource may be shared by pointer. Clone it to have exclusive ownership. - s.resource = s.resource.Clone(&Allocators{}) + s.resource = s.resource.clone(&Allocators{}) } s.resource.mutateRandom(random, schem, limiter) @@ -563,12 +595,12 @@ func (s *Metrics) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter s.scope = new(Scope) s.scope.init(&s.modifiedFields, fieldModifiedMetricsScope) } else { - s.scope = s.scope.Clone(&Allocators{}) + s.scope = s.scope.clone(&Allocators{}) } } if s.scope.canBeShared() { // scope may be shared by pointer. Clone it to have exclusive ownership. - s.scope = s.scope.Clone(&Allocators{}) + s.scope = s.scope.clone(&Allocators{}) } s.scope.mutateRandom(random, schem, limiter) diff --git a/go/otel/otelstef/modifiedfields.go b/go/otel/otelstef/modifiedfields.go index 15301bce..e0abea28 100644 --- a/go/otel/otelstef/modifiedfields.go +++ b/go/otel/otelstef/modifiedfields.go @@ -115,6 +115,13 @@ func (m *modifiedFieldsMultimap) fixParent(parentModifiedFields *modifiedFields) m.vals.parent = parentModifiedFields } +func (m *modifiedFieldsMultimap) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.keys.parent = parentModifiedFields + m.vals.parent = parentModifiedFields + m.keys.parentBit = parentModifiedBit + m.vals.parentBit = parentModifiedBit +} + // Returns a bitmask to be set in keys or vals to mark modification of // multimap element at index. func (m *modifiedFieldsMultimap) maskForIndex(index int) uint64 { diff --git a/go/otel/otelstef/point.go b/go/otel/otelstef/point.go index bfc50857..3c29c5df 100644 --- a/go/otel/otelstef/point.go +++ b/go/otel/otelstef/point.go @@ -92,6 +92,14 @@ func (s *Point) fixParent(parentModifiedFields *modifiedFields) { s.exemplars.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Point) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.value.attachParent(&s.modifiedFields, fieldModifiedPointValue) + s.exemplars.attachParent(&s.modifiedFields, fieldModifiedPointExemplars) +} + func (s *Point) freeze() { if s.isFrozen() { return @@ -205,6 +213,14 @@ func (s *Point) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Point) clearModifiedRecursively() { + s.value.clearModifiedRecursively() + s.exemplars.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Point) computeDiff(val *Point) (ret bool) { @@ -236,20 +252,17 @@ func (s *Point) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Point) cloneShared(allocators *Allocators) Point { - return s.Clone(allocators) -} - -func (s *Point) Clone(allocators *Allocators) Point { - c := Point{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Point) CloneTo(dst *Point, allocators *Allocators) { + *dst = Point{ startTimestamp: s.startTimestamp, timestamp: s.timestamp, } - copyToNewPointValue(&c.value, &s.value, allocators) - copyToNewExemplarArray(&c.exemplars, &s.exemplars, allocators) - return c + copyToNewPointValue(&dst.value, &s.value, allocators) + copyToNewExemplarArray(&dst.exemplars, &s.exemplars, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/pointvalue.go b/go/otel/otelstef/pointvalue.go index 2d6a71f5..a221545b 100644 --- a/go/otel/otelstef/pointvalue.go +++ b/go/otel/otelstef/pointvalue.go @@ -86,6 +86,21 @@ func (s *PointValue) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointer and bit in a newly copied oneof. +func (s *PointValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.parentModifiedFields = parentModifiedFields + s.parentModifiedBit = parentModifiedBit + + switch s.Type() { + case PointValueTypeHistogram: + s.histogramPtr().attachParent(parentModifiedFields, parentModifiedBit) + case PointValueTypeExpHistogram: + s.expHistogramPtr().attachParent(parentModifiedFields, parentModifiedBit) + case PointValueTypeSummary: + s.summaryPtr().attachParent(parentModifiedFields, parentModifiedBit) + } +} + type PointValueType byte const ( @@ -253,33 +268,31 @@ func (s *PointValue) canBeShared() bool { return false } -func (s *PointValue) cloneShared(allocators *Allocators) PointValue { - // Oneof is not shareable, so cloneShared is just a Clone. - return s.Clone(allocators) -} - -func (s *PointValue) Clone(allocators *Allocators) PointValue { - c := PointValue{} - c.clearValSetType(s.Type()) +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *PointValue) CloneTo(dst *PointValue, allocators *Allocators) { + *dst = PointValue{} + dst.clearValSetType(s.Type()) switch s.Type() { case PointValueTypeInt64: - *c.int64Ptr() = *s.int64Ptr() + *dst.int64Ptr() = *s.int64Ptr() case PointValueTypeFloat64: - *c.float64Ptr() = *s.float64Ptr() + *dst.float64Ptr() = *s.float64Ptr() case PointValueTypeHistogram: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(HistogramValue{}))) - c.allocHistogramAlloc(allocators) - copyToNewHistogramValue(c.histogramPtr(), s.histogramPtr(), allocators) + dst.allocHistogramAlloc(allocators) + copyToNewHistogramValue(dst.histogramPtr(), s.histogramPtr(), allocators) case PointValueTypeExpHistogram: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(ExpHistogramValue{}))) - c.allocExpHistogramAlloc(allocators) - copyToNewExpHistogramValue(c.expHistogramPtr(), s.expHistogramPtr(), allocators) + dst.allocExpHistogramAlloc(allocators) + copyToNewExpHistogramValue(dst.expHistogramPtr(), s.expHistogramPtr(), allocators) case PointValueTypeSummary: allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(SummaryValue{}))) - c.allocSummaryAlloc(allocators) - copyToNewSummaryValue(c.summaryPtr(), s.summaryPtr(), allocators) + dst.allocSummaryAlloc(allocators) + copyToNewSummaryValue(dst.summaryPtr(), s.summaryPtr(), allocators) } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -386,6 +399,18 @@ func (s *PointValue) setUnmodifiedRecursively() { } } +// clearModifiedRecursively clears modification state in all mutable descendants. +func (s *PointValue) clearModifiedRecursively() { + switch s.Type() { + case PointValueTypeHistogram: + s.histogramPtr().clearModifiedRecursively() + case PointValueTypeExpHistogram: + s.expHistogramPtr().clearModifiedRecursively() + case PointValueTypeSummary: + s.summaryPtr().clearModifiedRecursively() + } +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *PointValue) computeDiff(val *PointValue) (ret bool) { diff --git a/go/otel/otelstef/quantilevalue.go b/go/otel/otelstef/quantilevalue.go index b684b99e..09826fcd 100644 --- a/go/otel/otelstef/quantilevalue.go +++ b/go/otel/otelstef/quantilevalue.go @@ -80,6 +80,12 @@ func (s *QuantileValue) fixParent(parentModifiedFields *modifiedFields) { s.modifiedFields.parent = parentModifiedFields } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *QuantileValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit +} + func (s *QuantileValue) freeze() { s.modifiedFields.freeze() } @@ -146,6 +152,12 @@ func (s *QuantileValue) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *QuantileValue) clearModifiedRecursively() { + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *QuantileValue) computeDiff(val *QuantileValue) (ret bool) { @@ -167,18 +179,15 @@ func (s *QuantileValue) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *QuantileValue) cloneShared(allocators *Allocators) QuantileValue { - return s.Clone(allocators) -} - -func (s *QuantileValue) Clone(allocators *Allocators) QuantileValue { - c := QuantileValue{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *QuantileValue) CloneTo(dst *QuantileValue, allocators *Allocators) { + *dst = QuantileValue{ quantile: s.quantile, value: s.value, } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/quantilevaluearray.go b/go/otel/otelstef/quantilevaluearray.go index 0ef85e73..e12d8ca3 100644 --- a/go/otel/otelstef/quantilevaluearray.go +++ b/go/otel/otelstef/quantilevaluearray.go @@ -52,6 +52,15 @@ func (e *QuantileValueArray) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *QuantileValueArray) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + for i := range e.elems { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } +} + func (e *QuantileValueArray) canBeShared() bool { // An array can never be shared. return false @@ -96,6 +105,14 @@ func (e *QuantileValueArray) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *QuantileValueArray) clearModifiedRecursively() { + for i := 0; i < len(e.elems); i++ { + e.elems[i].clearModifiedRecursively() + } + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *QuantileValueArray) computeDiff(val *QuantileValueArray) (ret bool) { diff --git a/go/otel/otelstef/readerstate.go b/go/otel/otelstef/readerstate.go index 967697fe..9091e8c7 100644 --- a/go/otel/otelstef/readerstate.go +++ b/go/otel/otelstef/readerstate.go @@ -14,6 +14,8 @@ type ReaderState struct { // Dictionaries AnyValueString codecs.StringDictDecoderDict AttributeKey codecs.StringDictDecoderDict + LogEventName codecs.StringDictDecoderDict + LogSeverityText codecs.StringDictDecoderDict Metric MetricDecoderDict MetricDescription codecs.StringDictDecoderDict MetricName codecs.StringDictDecoderDict @@ -44,6 +46,8 @@ type ReaderState struct { KeyValueListDecoder *KeyValueListDecoder LinkDecoder *LinkDecoder LinkArrayDecoder *LinkArrayDecoder + LogRecordDecoder *LogRecordDecoder + LogsDecoder *LogsDecoder MetricDecoder *MetricDecoder MetricsDecoder *MetricsDecoder PointDecoder *PointDecoder @@ -66,6 +70,8 @@ func (d *ReaderState) Init(overrideSchema *schema.WireSchema) { d.AnyValueString.Init() d.AttributeKey.Init() + d.LogEventName.Init() + d.LogSeverityText.Init() d.Metric.Init() d.MetricDescription.Init() d.MetricName.Init() @@ -84,6 +90,8 @@ func (d *ReaderState) Init(overrideSchema *schema.WireSchema) { func (d *ReaderState) ResetDicts() { d.AnyValueString.Reset() d.AttributeKey.Reset() + d.LogEventName.Reset() + d.LogSeverityText.Reset() d.Metric.Reset() d.MetricDescription.Reset() d.MetricName.Reset() diff --git a/go/otel/otelstef/resource.go b/go/otel/otelstef/resource.go index 89b0131a..e5251da0 100644 --- a/go/otel/otelstef/resource.go +++ b/go/otel/otelstef/resource.go @@ -98,6 +98,13 @@ func (s *Resource) fixParent(parentModifiedFields *modifiedFields) { s.attributes.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Resource) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.attributes.attachParent(&s.modifiedFields, fieldModifiedResourceAttributes) +} + // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, // which can improve encoding performance. @@ -196,6 +203,13 @@ func (s *Resource) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Resource) clearModifiedRecursively() { + s.attributes.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Resource) computeDiff(val *Resource) (ret bool) { @@ -222,6 +236,14 @@ func (s *Resource) canBeShared() bool { return s.isFrozen() } +// clone returns a clone of s allocated using allocators. +func (s *Resource) clone(allocators *Allocators) *Resource { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Resource{}))) + c := allocators.Resource.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). func (s *Resource) cloneShared(allocators *Allocators) *Resource { @@ -229,18 +251,19 @@ func (s *Resource) cloneShared(allocators *Allocators) *Resource { // If s is frozen it means it is safe to share without cloning. return s } - return s.Clone(allocators) + return s.clone(allocators) } -func (s *Resource) Clone(allocators *Allocators) *Resource { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Resource{}))) - c := allocators.Resource.Alloc() - *c = Resource{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Resource) CloneTo(dst *Resource, allocators *Allocators) { + *dst = Resource{ schemaURL: s.schemaURL, droppedAttributesCount: s.droppedAttributesCount, } - copyToNewAttributes(&c.attributes, &s.attributes, allocators) - return c + copyToNewAttributes(&dst.attributes, &s.attributes, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -413,7 +436,7 @@ func (d *ResourceEncoderDict) Add(val *Resource) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -732,7 +755,7 @@ func (d *ResourceDecoder) Decode(dstPtr **Resource) error { // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } diff --git a/go/otel/otelstef/scope.go b/go/otel/otelstef/scope.go index 3b6e4315..4a595957 100644 --- a/go/otel/otelstef/scope.go +++ b/go/otel/otelstef/scope.go @@ -104,6 +104,13 @@ func (s *Scope) fixParent(parentModifiedFields *modifiedFields) { s.attributes.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Scope) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.attributes.attachParent(&s.modifiedFields, fieldModifiedScopeAttributes) +} + // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, // which can improve encoding performance. @@ -252,6 +259,13 @@ func (s *Scope) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Scope) clearModifiedRecursively() { + s.attributes.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Scope) computeDiff(val *Scope) (ret bool) { @@ -288,6 +302,14 @@ func (s *Scope) canBeShared() bool { return s.isFrozen() } +// clone returns a clone of s allocated using allocators. +func (s *Scope) clone(allocators *Allocators) *Scope { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Scope{}))) + c := allocators.Scope.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). func (s *Scope) cloneShared(allocators *Allocators) *Scope { @@ -295,20 +317,21 @@ func (s *Scope) cloneShared(allocators *Allocators) *Scope { // If s is frozen it means it is safe to share without cloning. return s } - return s.Clone(allocators) + return s.clone(allocators) } -func (s *Scope) Clone(allocators *Allocators) *Scope { - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof(Scope{}))) - c := allocators.Scope.Alloc() - *c = Scope{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Scope) CloneTo(dst *Scope, allocators *Allocators) { + *dst = Scope{ name: s.name, version: s.version, schemaURL: s.schemaURL, droppedAttributesCount: s.droppedAttributesCount, } - copyToNewAttributes(&c.attributes, &s.attributes, allocators) - return c + copyToNewAttributes(&dst.attributes, &s.attributes, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -517,7 +540,7 @@ func (d *ScopeEncoderDict) Add(val *Scope) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -918,7 +941,7 @@ func (d *ScopeDecoder) Decode(dstPtr **Scope) error { // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } diff --git a/go/otel/otelstef/severitynumber.go b/go/otel/otelstef/severitynumber.go new file mode 100644 index 00000000..49b08b2a --- /dev/null +++ b/go/otel/otelstef/severitynumber.go @@ -0,0 +1,32 @@ +// Code generated by stefc. DO NOT EDIT. +package otelstef + +type SeverityNumber uint64 + +const ( + SeverityNumberUnspecified SeverityNumber = 0 + SeverityNumberTrace SeverityNumber = 1 + SeverityNumberTrace2 SeverityNumber = 2 + SeverityNumberTrace3 SeverityNumber = 3 + SeverityNumberTrace4 SeverityNumber = 4 + SeverityNumberDebug SeverityNumber = 5 + SeverityNumberDebug2 SeverityNumber = 6 + SeverityNumberDebug3 SeverityNumber = 7 + SeverityNumberDebug4 SeverityNumber = 8 + SeverityNumberInfo SeverityNumber = 9 + SeverityNumberInfo2 SeverityNumber = 10 + SeverityNumberInfo3 SeverityNumber = 11 + SeverityNumberInfo4 SeverityNumber = 12 + SeverityNumberWarn SeverityNumber = 13 + SeverityNumberWarn2 SeverityNumber = 14 + SeverityNumberWarn3 SeverityNumber = 15 + SeverityNumberWarn4 SeverityNumber = 16 + SeverityNumberError SeverityNumber = 17 + SeverityNumberError2 SeverityNumber = 18 + SeverityNumberError3 SeverityNumber = 19 + SeverityNumberError4 SeverityNumber = 20 + SeverityNumberFatal SeverityNumber = 21 + SeverityNumberFatal2 SeverityNumber = 22 + SeverityNumberFatal3 SeverityNumber = 23 + SeverityNumberFatal4 SeverityNumber = 24 +) diff --git a/go/otel/otelstef/span.go b/go/otel/otelstef/span.go index 8049820d..6b692035 100644 --- a/go/otel/otelstef/span.go +++ b/go/otel/otelstef/span.go @@ -128,6 +128,16 @@ func (s *Span) fixParent(parentModifiedFields *modifiedFields) { s.status.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Span) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.attributes.attachParent(&s.modifiedFields, fieldModifiedSpanAttributes) + s.events.attachParent(&s.modifiedFields, fieldModifiedSpanEvents) + s.links.attachParent(&s.modifiedFields, fieldModifiedSpanLinks) + s.status.attachParent(&s.modifiedFields, fieldModifiedSpanStatus) +} + func (s *Span) freeze() { if s.isFrozen() { return @@ -485,6 +495,16 @@ func (s *Span) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Span) clearModifiedRecursively() { + s.attributes.clearModifiedRecursively() + s.events.clearModifiedRecursively() + s.links.clearModifiedRecursively() + s.status.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Span) computeDiff(val *Span) (ret bool) { @@ -566,14 +586,10 @@ func (s *Span) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Span) cloneShared(allocators *Allocators) Span { - return s.Clone(allocators) -} - -func (s *Span) Clone(allocators *Allocators) Span { - c := Span{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Span) CloneTo(dst *Span, allocators *Allocators) { + *dst = Span{ traceID: s.traceID, spanID: s.spanID, traceState: s.traceState, @@ -585,11 +601,12 @@ func (s *Span) Clone(allocators *Allocators) Span { endTimeUnixNano: s.endTimeUnixNano, droppedAttributesCount: s.droppedAttributesCount, } - copyToNewAttributes(&c.attributes, &s.attributes, allocators) - copyToNewEventArray(&c.events, &s.events, allocators) - copyToNewLinkArray(&c.links, &s.links, allocators) - copyToNewSpanStatus(&c.status, &s.status, allocators) - return c + copyToNewAttributes(&dst.attributes, &s.attributes, allocators) + copyToNewEventArray(&dst.events, &s.events, allocators) + copyToNewLinkArray(&dst.links, &s.links, allocators) + copyToNewSpanStatus(&dst.status, &s.status, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/spans.go b/go/otel/otelstef/spans.go index d64d4422..17505aa1 100644 --- a/go/otel/otelstef/spans.go +++ b/go/otel/otelstef/spans.go @@ -104,6 +104,20 @@ func (s *Spans) fixParent(parentModifiedFields *modifiedFields) { s.span.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *Spans) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.envelope.attachParent(&s.modifiedFields, fieldModifiedSpansEnvelope) + if !s.resource.canBeShared() { + s.resource.attachParent(&s.modifiedFields, fieldModifiedSpansResource) + } + if !s.scope.canBeShared() { + s.scope.attachParent(&s.modifiedFields, fieldModifiedSpansScope) + } + s.span.attachParent(&s.modifiedFields, fieldModifiedSpansSpan) +} + func (s *Spans) freeze() { if s.isFrozen() { return @@ -151,7 +165,7 @@ func (s *Spans) SetResource(v *Resource) { } } else { if s.resource.canBeShared() { - s.resource = s.resource.Clone(&Allocators{}) + s.resource = s.resource.clone(&Allocators{}) } s.resource.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedSpansResource) @@ -186,7 +200,7 @@ func (s *Spans) SetScope(v *Scope) { } } else { if s.scope.canBeShared() { - s.scope = s.scope.Clone(&Allocators{}) + s.scope = s.scope.clone(&Allocators{}) } s.scope.CopyFrom(v) s.modifiedFields.markModified(fieldModifiedSpansScope) @@ -249,6 +263,20 @@ func (s *Spans) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *Spans) clearModifiedRecursively() { + s.envelope.clearModifiedRecursively() + if !s.resource.canBeShared() { + s.resource.clearModifiedRecursively() + } + if !s.scope.canBeShared() { + s.scope.clearModifiedRecursively() + } + s.span.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *Spans) computeDiff(val *Spans) (ret bool) { @@ -280,20 +308,16 @@ func (s *Spans) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *Spans) cloneShared(allocators *Allocators) Spans { - return s.Clone(allocators) -} - -func (s *Spans) Clone(allocators *Allocators) Spans { - c := Spans{ - resource: s.resource.cloneShared(allocators), - scope: s.scope.cloneShared(allocators), - } - copyToNewEnvelope(&c.envelope, &s.envelope, allocators) - copyToNewSpan(&c.span, &s.span, allocators) - return c +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *Spans) CloneTo(dst *Spans, allocators *Allocators) { + *dst = Spans{} + copyToNewEnvelope(&dst.envelope, &s.envelope, allocators) + dst.resource = s.resource.cloneShared(allocators) + dst.scope = s.scope.cloneShared(allocators) + copyToNewSpan(&dst.span, &s.span, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -401,12 +425,12 @@ func (s *Spans) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter *m s.resource = new(Resource) s.resource.init(&s.modifiedFields, fieldModifiedSpansResource) } else { - s.resource = s.resource.Clone(&Allocators{}) + s.resource = s.resource.clone(&Allocators{}) } } if s.resource.canBeShared() { // resource may be shared by pointer. Clone it to have exclusive ownership. - s.resource = s.resource.Clone(&Allocators{}) + s.resource = s.resource.clone(&Allocators{}) } s.resource.mutateRandom(random, schem, limiter) @@ -425,12 +449,12 @@ func (s *Spans) mutateRandom(random *rand.Rand, schem *schema.Schema, limiter *m s.scope = new(Scope) s.scope.init(&s.modifiedFields, fieldModifiedSpansScope) } else { - s.scope = s.scope.Clone(&Allocators{}) + s.scope = s.scope.clone(&Allocators{}) } } if s.scope.canBeShared() { // scope may be shared by pointer. Clone it to have exclusive ownership. - s.scope = s.scope.Clone(&Allocators{}) + s.scope = s.scope.clone(&Allocators{}) } s.scope.mutateRandom(random, schem, limiter) diff --git a/go/otel/otelstef/spanstatus.go b/go/otel/otelstef/spanstatus.go index 83f84e59..1baed495 100644 --- a/go/otel/otelstef/spanstatus.go +++ b/go/otel/otelstef/spanstatus.go @@ -80,6 +80,12 @@ func (s *SpanStatus) fixParent(parentModifiedFields *modifiedFields) { s.modifiedFields.parent = parentModifiedFields } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *SpanStatus) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit +} + func (s *SpanStatus) freeze() { s.modifiedFields.freeze() } @@ -146,6 +152,12 @@ func (s *SpanStatus) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *SpanStatus) clearModifiedRecursively() { + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *SpanStatus) computeDiff(val *SpanStatus) (ret bool) { @@ -167,18 +179,15 @@ func (s *SpanStatus) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *SpanStatus) cloneShared(allocators *Allocators) SpanStatus { - return s.Clone(allocators) -} - -func (s *SpanStatus) Clone(allocators *Allocators) SpanStatus { - c := SpanStatus{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *SpanStatus) CloneTo(dst *SpanStatus, allocators *Allocators) { + *dst = SpanStatus{ message: s.message, code: s.code, } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/summaryvalue.go b/go/otel/otelstef/summaryvalue.go index ed023a62..3ad18594 100644 --- a/go/otel/otelstef/summaryvalue.go +++ b/go/otel/otelstef/summaryvalue.go @@ -86,6 +86,13 @@ func (s *SummaryValue) fixParent(parentModifiedFields *modifiedFields) { s.quantileValues.fixParent(&s.modifiedFields) } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *SummaryValue) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + s.quantileValues.attachParent(&s.modifiedFields, fieldModifiedSummaryValueQuantileValues) +} + func (s *SummaryValue) freeze() { if s.isFrozen() { return @@ -177,6 +184,13 @@ func (s *SummaryValue) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *SummaryValue) clearModifiedRecursively() { + s.quantileValues.clearModifiedRecursively() + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *SummaryValue) computeDiff(val *SummaryValue) (ret bool) { @@ -203,19 +217,16 @@ func (s *SummaryValue) canBeShared() bool { return false } -// cloneShared returns a clone of s. It may return s if it is safe to share without cloning -// (for example if s is frozen). -func (s *SummaryValue) cloneShared(allocators *Allocators) SummaryValue { - return s.Clone(allocators) -} - -func (s *SummaryValue) Clone(allocators *Allocators) SummaryValue { - c := SummaryValue{ +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *SummaryValue) CloneTo(dst *SummaryValue, allocators *Allocators) { + *dst = SummaryValue{ count: s.count, sum: s.sum, } - copyToNewQuantileValueArray(&c.quantileValues, &s.quantileValues, allocators) - return c + copyToNewQuantileValueArray(&dst.quantileValues, &s.quantileValues, allocators) + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate diff --git a/go/otel/otelstef/uint64array.go b/go/otel/otelstef/uint64array.go index a719b69c..1e05df38 100644 --- a/go/otel/otelstef/uint64array.go +++ b/go/otel/otelstef/uint64array.go @@ -51,6 +51,12 @@ func (e *Uint64Array) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *Uint64Array) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit +} + func (e *Uint64Array) canBeShared() bool { // An array can never be shared. return false @@ -97,6 +103,11 @@ func (e *Uint64Array) setUnmodifiedRecursively() { } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e *Uint64Array) clearModifiedRecursively() { + +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *Uint64Array) computeDiff(val *Uint64Array) (ret bool) { diff --git a/go/otel/otelstef/writerstate.go b/go/otel/otelstef/writerstate.go index 91397071..d3808afe 100644 --- a/go/otel/otelstef/writerstate.go +++ b/go/otel/otelstef/writerstate.go @@ -16,6 +16,8 @@ type WriterState struct { // Dictionaries AnyValueString codecs.StringDictEncoderDict AttributeKey codecs.StringDictEncoderDict + LogEventName codecs.StringDictEncoderDict + LogSeverityText codecs.StringDictEncoderDict Metric MetricEncoderDict MetricDescription codecs.StringDictEncoderDict MetricName codecs.StringDictEncoderDict @@ -46,6 +48,8 @@ type WriterState struct { KeyValueListEncoder *KeyValueListEncoder LinkEncoder *LinkEncoder LinkArrayEncoder *LinkArrayEncoder + LogRecordEncoder *LogRecordEncoder + LogsEncoder *LogsEncoder MetricEncoder *MetricEncoder MetricsEncoder *MetricsEncoder PointEncoder *PointEncoder @@ -71,6 +75,8 @@ func (d *WriterState) Init(opts *pkg.WriterOptions) { // Init dictionaries d.AnyValueString.Init(&d.limiter) d.AttributeKey.Init(&d.limiter) + d.LogEventName.Init(&d.limiter) + d.LogSeverityText.Init(&d.limiter) d.Metric.Init(&d.limiter) d.MetricDescription.Init(&d.limiter) d.MetricName.Init(&d.limiter) @@ -89,6 +95,8 @@ func (d *WriterState) ResetDicts() { d.limiter.ResetDict() d.AnyValueString.Reset() d.AttributeKey.Reset() + d.LogEventName.Reset() + d.LogSeverityText.Reset() d.Metric.Reset() d.MetricDescription.Reset() d.MetricName.Reset() diff --git a/go/pdata/internal/otlptools/compare.go b/go/pdata/internal/otlptools/compare.go index 1318fe40..a4969bfd 100644 --- a/go/pdata/internal/otlptools/compare.go +++ b/go/pdata/internal/otlptools/compare.go @@ -1,9 +1,13 @@ package otlptools import ( + "bytes" + "cmp" + "slices" "strings" "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" ) @@ -21,6 +25,9 @@ func Map2attrs(attributes pcommon.Map) []AttrAccessible { return true }, ) + slices.SortFunc(attrs, func(a, b AttrAccessible) int { + return strings.Compare(a.Key, b.Key) + }) return attrs } @@ -29,6 +36,10 @@ func CmpResourceMetrics(left, right pmetric.ResourceMetrics) int { if c != 0 { return c } + c = cmp.Compare(left.Resource().DroppedAttributesCount(), right.Resource().DroppedAttributesCount()) + if c != 0 { + return c + } return CmpAttrs(left.Resource().Attributes(), right.Resource().Attributes()) } @@ -37,6 +48,22 @@ func CmpResourceSpans(left, right ptrace.ResourceSpans) int { if c != 0 { return c } + c = cmp.Compare(left.Resource().DroppedAttributesCount(), right.Resource().DroppedAttributesCount()) + if c != 0 { + return c + } + return CmpAttrs(left.Resource().Attributes(), right.Resource().Attributes()) +} + +func CmpResourceLogs(left, right plog.ResourceLogs) int { + c := strings.Compare(left.SchemaUrl(), right.SchemaUrl()) + if c != 0 { + return c + } + c = cmp.Compare(left.Resource().DroppedAttributesCount(), right.Resource().DroppedAttributesCount()) + if c != 0 { + return c + } return CmpAttrs(left.Resource().Attributes(), right.Resource().Attributes()) } @@ -53,6 +80,10 @@ func CmpScopeMetrics(left, right pmetric.ScopeMetrics) int { if c != 0 { return c } + c = cmp.Compare(left.Scope().DroppedAttributesCount(), right.Scope().DroppedAttributesCount()) + if c != 0 { + return c + } return CmpAttrs(left.Scope().Attributes(), right.Scope().Attributes()) } @@ -70,10 +101,73 @@ func CmpScopeSpans(left, right ptrace.ScopeSpans) int { if c != 0 { return c } + c = cmp.Compare(left.Scope().DroppedAttributesCount(), right.Scope().DroppedAttributesCount()) + if c != 0 { + return c + } return CmpAttrs(left.Scope().Attributes(), right.Scope().Attributes()) } +func CmpScopeLogs(left, right plog.ScopeLogs) int { + c := strings.Compare(left.Scope().Name(), right.Scope().Name()) + if c != 0 { + return c + } + c = strings.Compare(left.Scope().Version(), right.Scope().Version()) + if c != 0 { + return c + } + c = strings.Compare(left.SchemaUrl(), right.SchemaUrl()) + if c != 0 { + return c + } + c = cmp.Compare(left.Scope().DroppedAttributesCount(), right.Scope().DroppedAttributesCount()) + if c != 0 { + return c + } + return CmpAttrs(left.Scope().Attributes(), right.Scope().Attributes()) +} + +// CmpLogRecords compares log records by every field encoded by the STEF logs +// converter. It provides a deterministic order for otherwise identical +// resource and scope groupings. +func CmpLogRecords(left, right plog.LogRecord) int { + if c := cmp.Compare(left.Timestamp(), right.Timestamp()); c != 0 { + return c + } + if c := cmp.Compare(left.ObservedTimestamp(), right.ObservedTimestamp()); c != 0 { + return c + } + leftTraceID, rightTraceID := left.TraceID(), right.TraceID() + if c := bytes.Compare(leftTraceID[:], rightTraceID[:]); c != 0 { + return c + } + leftSpanID, rightSpanID := left.SpanID(), right.SpanID() + if c := bytes.Compare(leftSpanID[:], rightSpanID[:]); c != 0 { + return c + } + if c := cmp.Compare(left.SeverityNumber(), right.SeverityNumber()); c != 0 { + return c + } + if c := strings.Compare(left.SeverityText(), right.SeverityText()); c != 0 { + return c + } + if c := CmpVal(left.Body(), right.Body()); c != 0 { + return c + } + if c := CmpAttrs(left.Attributes(), right.Attributes()); c != 0 { + return c + } + if c := cmp.Compare(left.DroppedAttributesCount(), right.DroppedAttributesCount()); c != 0 { + return c + } + if c := cmp.Compare(left.Flags(), right.Flags()); c != 0 { + return c + } + return strings.Compare(left.EventName(), right.EventName()) +} + func CmpAttrs(a, b pcommon.Map) int { left := Map2attrs(a) right := Map2attrs(b) @@ -114,6 +208,12 @@ func CmpVal(left, right pcommon.Value) int { case pcommon.ValueTypeBool: return CmpBool(left.Bool(), right.Bool()) + case pcommon.ValueTypeDouble: + return cmp.Compare(left.Double(), right.Double()) + + case pcommon.ValueTypeBytes: + return bytes.Compare(left.Bytes().AsRaw(), right.Bytes().AsRaw()) + case pcommon.ValueTypeSlice: left := left.Slice() right := right.Slice() @@ -131,8 +231,11 @@ func CmpVal(left, right pcommon.Value) int { case pcommon.ValueTypeEmpty: return 0 + case pcommon.ValueTypeMap: + return CmpAttrs(left.Map(), right.Map()) + default: - panic("comparison not implemented") + panic("unknown value type") } } diff --git a/go/pdata/internal/otlptools/otlpval2tef.go b/go/pdata/internal/otlptools/otlpval2tef.go index 07f4a34f..2e6d8e65 100644 --- a/go/pdata/internal/otlptools/otlpval2tef.go +++ b/go/pdata/internal/otlptools/otlpval2tef.go @@ -35,6 +35,11 @@ type Otlp2Stef struct { attrElems []elem } +// Value converts an OTLP AnyValue to its STEF representation. +func (o *Otlp2Stef) Value(src pcommon.Value, dst *otelstef.AnyValue) { + otlpValueToTefAnyValue(src, dst) +} + func (o *Otlp2Stef) ResourceSorted(dst *otelstef.Resource, src pcommon.Resource, schemaUrl string) { dst.SetSchemaURL(schemaUrl) o.MapSorted(src.Attributes(), dst.Attributes()) @@ -139,6 +144,7 @@ func otlpValueToTefAnyValue(val pcommon.Value, into *otelstef.AnyValue) { func(k string, v pcommon.Value) bool { kvList.SetKey(i, k) otlpValueToTefAnyValue(v, kvList.Value(i)) + i++ return true }, ) diff --git a/go/pdata/internal/otlptools/tef2otlpval.go b/go/pdata/internal/otlptools/tef2otlpval.go index 3f6d94de..24e64989 100644 --- a/go/pdata/internal/otlptools/tef2otlpval.go +++ b/go/pdata/internal/otlptools/tef2otlpval.go @@ -30,7 +30,7 @@ func TefToOtlpMap(in *otelstef.Attributes, out pcommon.Map) error { //decoder := anyvalue.Decoder{} for i := 0; i < in.Len(); i++ { val := out.PutEmpty(in.Key(i)) - err := tefAnyValueToOtlp(in.Value(i), val) + err := TefAnyValueToOtlp(in.Value(i), val) if err != nil { return err } @@ -40,7 +40,7 @@ func TefToOtlpMap(in *otelstef.Attributes, out pcommon.Map) error { var errDecode = errors.New("decode error") -func tefAnyValueToOtlp(anyVal *otelstef.AnyValue, into pcommon.Value) error { +func TefAnyValueToOtlp(anyVal *otelstef.AnyValue, into pcommon.Value) error { switch anyVal.Type() { case otelstef.AnyValueTypeString: into.SetStr(anyVal.String()) @@ -65,7 +65,7 @@ func tefAnyValueToOtlp(anyVal *otelstef.AnyValue, into pcommon.Value) error { arr := anyVal.Array() for i := 0; i < arr.Len(); i++ { val := values.AppendEmpty() - err := tefAnyValueToOtlp(arr.At(i), val) + err := TefAnyValueToOtlp(arr.At(i), val) if err != nil { return err } @@ -76,7 +76,7 @@ func tefAnyValueToOtlp(anyVal *otelstef.AnyValue, into pcommon.Value) error { kvList := anyVal.KVList() for i := 0; i < kvList.Len(); i++ { val := values.PutEmpty(kvList.Key(i)) - err := tefAnyValueToOtlp(kvList.Value(i), val) + err := TefAnyValueToOtlp(kvList.Value(i), val) if err != nil { return err } diff --git a/go/pdata/logs/otlp2stef.go b/go/pdata/logs/otlp2stef.go new file mode 100644 index 00000000..8e9946db --- /dev/null +++ b/go/pdata/logs/otlp2stef.go @@ -0,0 +1,20 @@ +package logs + +import ( + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/otel/otelstef" +) + +// OtlpToStef defines a converter from OTLP logs to STEF format. +type OtlpToStef interface { + // Convert writes OTLP logs to the provided STEF writer. + // It does not call Flush on the writer. + Convert(src plog.Logs, writer *otelstef.LogsWriter) error +} + +// NewOtlpToStef returns an OTLP logs converter. When sorted is true, the +// converter groups equal resources and scopes and sorts log records by time. +func NewOtlpToStef(sorted bool) OtlpToStef { + return &OtlpToStefUnsorted{Sorted: sorted} +} diff --git a/go/pdata/logs/otlp2stef_unsorted.go b/go/pdata/logs/otlp2stef_unsorted.go new file mode 100644 index 00000000..a40b55a8 --- /dev/null +++ b/go/pdata/logs/otlp2stef_unsorted.go @@ -0,0 +1,123 @@ +package logs + +import ( + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/otel/otelstef" + "github.com/splunk/stef/go/pdata/internal/otlptools" + "github.com/splunk/stef/go/pkg" +) + +// OtlpToStefUnsorted converts OTLP logs to STEF. If Sorted is true, it first +// groups equal resources and scopes and orders log records by timestamp. +type OtlpToStefUnsorted struct { + otlp2stef otlptools.Otlp2Stef + Sorted bool +} + +var _ OtlpToStef = (*OtlpToStefUnsorted)(nil) + +func (c *OtlpToStefUnsorted) Convert(src plog.Logs, writer *otelstef.LogsWriter) error { + otlp2stef := &c.otlp2stef + resourceLogs := src.ResourceLogs() + + if c.Sorted { + resourceLogs.Sort(func(a, b plog.ResourceLogs) bool { + return otlptools.CmpResourceLogs(a, b) < 0 + }) + for i := 0; i < resourceLogs.Len()-1; { + if otlptools.CmpResourceLogs(resourceLogs.At(i), resourceLogs.At(i+1)) != 0 { + i++ + continue + } + resourceLogs.At(i + 1).ScopeLogs().MoveAndAppendTo(resourceLogs.At(i).ScopeLogs()) + removeResourceLogsAt(resourceLogs, i+1) + } + } + + for i := 0; i < resourceLogs.Len(); i++ { + rl := resourceLogs.At(i) + if c.Sorted { + otlp2stef.ResourceSorted(writer.Record.Resource(), rl.Resource(), rl.SchemaUrl()) + } else { + otlp2stef.ResourceUnsorted(writer.Record.Resource(), rl.Resource(), rl.SchemaUrl()) + } + scopeLogs := rl.ScopeLogs() + + if c.Sorted { + scopeLogs.Sort(func(a, b plog.ScopeLogs) bool { + return otlptools.CmpScopeLogs(a, b) < 0 + }) + for j := 0; j < scopeLogs.Len()-1; { + if otlptools.CmpScopeLogs(scopeLogs.At(j), scopeLogs.At(j+1)) != 0 { + j++ + continue + } + scopeLogs.At(j + 1).LogRecords().MoveAndAppendTo(scopeLogs.At(j).LogRecords()) + removeScopeLogsAt(scopeLogs, j+1) + } + } + + for j := 0; j < scopeLogs.Len(); j++ { + sl := scopeLogs.At(j) + if c.Sorted { + otlp2stef.ScopeSorted(writer.Record.Scope(), sl.Scope(), sl.SchemaUrl()) + sortLogRecords(sl.LogRecords()) + } else { + otlp2stef.ScopeUnsorted(writer.Record.Scope(), sl.Scope(), sl.SchemaUrl()) + } + + for k := 0; k < sl.LogRecords().Len(); k++ { + logRecordToStef(sl.LogRecords().At(k), writer.Record.Log(), otlp2stef, c.Sorted) + if err := writer.Write(); err != nil { + return err + } + } + } + } + return nil +} + +func removeResourceLogsAt(logs plog.ResourceLogsSlice, index int) { + i := 0 + logs.RemoveIf(func(plog.ResourceLogs) bool { + remove := i == index + i++ + return remove + }) +} + +func removeScopeLogsAt(logs plog.ScopeLogsSlice, index int) { + i := 0 + logs.RemoveIf(func(plog.ScopeLogs) bool { + remove := i == index + i++ + return remove + }) +} + +func sortLogRecords(logs plog.LogRecordSlice) { + logs.Sort(func(a, b plog.LogRecord) bool { + return otlptools.CmpLogRecords(a, b) < 0 + }) +} + +func logRecordToStef(src plog.LogRecord, dst *otelstef.LogRecord, converter *otlptools.Otlp2Stef, sorted bool) { + dst.SetTimeUnixNano(uint64(src.Timestamp())) + dst.SetObservedTimeUnixNano(uint64(src.ObservedTimestamp())) + dst.SetSeverityNumber(otelstef.SeverityNumber(src.SeverityNumber())) + dst.SetSeverityText(src.SeverityText()) + converter.Value(src.Body(), dst.Body()) + if sorted { + converter.MapSorted(src.Attributes(), dst.Attributes()) + } else { + converter.MapUnsorted(src.Attributes(), dst.Attributes()) + } + dst.SetDroppedAttributesCount(uint64(src.DroppedAttributesCount())) + dst.SetFlags(uint64(src.Flags())) + traceID := src.TraceID() + spanID := src.SpanID() + dst.SetTraceID(pkg.Bytes(traceID[:])) + dst.SetSpanID(pkg.Bytes(spanID[:])) + dst.SetEventName(src.EventName()) +} diff --git a/go/pdata/logs/otlp2stef_unsorted_test.go b/go/pdata/logs/otlp2stef_unsorted_test.go new file mode 100644 index 00000000..017bfc3b --- /dev/null +++ b/go/pdata/logs/otlp2stef_unsorted_test.go @@ -0,0 +1,106 @@ +package logs + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/otel/otelstef" + "github.com/splunk/stef/go/pkg" +) + +func TestConvertLogRecord(t *testing.T) { + src := plog.NewLogs() + rl := src.ResourceLogs().AppendEmpty() + rl.SetSchemaUrl("resource-schema") + rl.Resource().Attributes().PutStr("service.name", "checkout") + rl.Resource().SetDroppedAttributesCount(1) + + sl := rl.ScopeLogs().AppendEmpty() + sl.SetSchemaUrl("scope-schema") + sl.Scope().SetName("logger") + sl.Scope().SetVersion("1.2.3") + sl.Scope().Attributes().PutBool("scope-attribute", true) + sl.Scope().SetDroppedAttributesCount(2) + + logRecord := sl.LogRecords().AppendEmpty() + logRecord.SetTimestamp(pcommon.Timestamp(100)) + logRecord.SetObservedTimestamp(pcommon.Timestamp(200)) + logRecord.SetSeverityNumber(plog.SeverityNumberError) + logRecord.SetSeverityText("ERROR") + body := logRecord.Body().SetEmptyMap() + body.PutStr("message", "failed") + body.PutInt("attempt", 3) + logRecord.Attributes().PutStr("log.attribute", "value") + logRecord.SetDroppedAttributesCount(3) + logRecord.SetFlags(plog.DefaultLogRecordFlags.WithIsSampled(true)) + traceID := pcommon.TraceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + spanID := pcommon.SpanID{1, 2, 3, 4, 5, 6, 7, 8} + logRecord.SetTraceID(traceID) + logRecord.SetSpanID(spanID) + logRecord.SetEventName("exception") + + buf := &pkg.MemChunkWriter{} + writer, err := otelstef.NewLogsWriter(buf, pkg.WriterOptions{}) + require.NoError(t, err) + require.NoError(t, (&OtlpToStefUnsorted{}).Convert(src, writer)) + require.NoError(t, writer.Flush()) + require.EqualValues(t, 1, writer.RecordCount()) + + reader, err := otelstef.NewLogsReader(bytes.NewReader(buf.Bytes())) + require.NoError(t, err) + require.NoError(t, reader.Read(pkg.ReadOptions{})) + + require.Equal(t, "resource-schema", reader.Record.Resource().SchemaURL()) + require.EqualValues(t, 1, reader.Record.Resource().DroppedAttributesCount()) + require.Equal(t, "logger", reader.Record.Scope().Name()) + require.Equal(t, "1.2.3", reader.Record.Scope().Version()) + require.Equal(t, "scope-schema", reader.Record.Scope().SchemaURL()) + require.EqualValues(t, 2, reader.Record.Scope().DroppedAttributesCount()) + + got := reader.Record.Log() + require.EqualValues(t, 100, got.TimeUnixNano()) + require.EqualValues(t, 200, got.ObservedTimeUnixNano()) + require.Equal(t, otelstef.SeverityNumberError, got.SeverityNumber()) + require.Equal(t, "ERROR", got.SeverityText()) + require.Equal(t, otelstef.AnyValueTypeKVList, got.Body().Type()) + require.Equal(t, 2, got.Body().KVList().Len()) + require.Equal(t, 1, got.Attributes().Len()) + require.EqualValues(t, 3, got.DroppedAttributesCount()) + require.EqualValues(t, logRecord.Flags(), got.Flags()) + require.Equal(t, pkg.Bytes(traceID[:]), got.TraceID()) + require.Equal(t, pkg.Bytes(spanID[:]), got.SpanID()) + require.Equal(t, "exception", got.EventName()) + require.ErrorIs(t, reader.Read(pkg.ReadOptions{}), io.EOF) +} + +func TestConvertSortedGroupsAndOrdersLogs(t *testing.T) { + src := plog.NewLogs() + for _, timestamp := range []pcommon.Timestamp{200, 100} { + rl := src.ResourceLogs().AppendEmpty() + rl.Resource().Attributes().PutStr("service.name", "checkout") + sl := rl.ScopeLogs().AppendEmpty() + sl.Scope().SetName("logger") + logRecord := sl.LogRecords().AppendEmpty() + logRecord.SetTimestamp(timestamp) + } + + buf := &pkg.MemChunkWriter{} + writer, err := otelstef.NewLogsWriter(buf, pkg.WriterOptions{}) + require.NoError(t, err) + require.NoError(t, NewOtlpToStef(true).Convert(src, writer)) + require.NoError(t, writer.Flush()) + require.EqualValues(t, 2, writer.RecordCount()) + + reader, err := otelstef.NewLogsReader(bytes.NewReader(buf.Bytes())) + require.NoError(t, err) + for _, timestamp := range []uint64{100, 200} { + require.NoError(t, reader.Read(pkg.ReadOptions{})) + require.Equal(t, timestamp, reader.Record.Log().TimeUnixNano()) + } + require.ErrorIs(t, reader.Read(pkg.ReadOptions{}), io.EOF) +} diff --git a/go/pdata/logs/sortedbyresource/converter.go b/go/pdata/logs/sortedbyresource/converter.go new file mode 100644 index 00000000..bde80f44 --- /dev/null +++ b/go/pdata/logs/sortedbyresource/converter.go @@ -0,0 +1,58 @@ +package sortedbyresource + +import ( + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/otel/otelstef" + "github.com/splunk/stef/go/pdata/internal/otlptools" + "github.com/splunk/stef/go/pkg" +) + +func OtlpToSortedTree(data plog.Logs) *SortedTree { + tree := NewSortedLogs() + converter := otlptools.Otlp2Stef{} + resourceLogs := data.ResourceLogs() + + for i := 0; i < resourceLogs.Len(); i++ { + srcResource := resourceLogs.At(i) + resource := otelstef.NewResource() + converter.ResourceSorted(resource, srcResource.Resource(), srcResource.SchemaUrl()) + resource.Freeze() + byResource := tree.ByResource(resource) + + scopeLogs := srcResource.ScopeLogs() + for j := 0; j < scopeLogs.Len(); j++ { + srcScope := scopeLogs.At(j) + scope := otelstef.NewScope() + converter.ScopeSorted(scope, srcScope.Scope(), srcScope.SchemaUrl()) + scope.Freeze() + byScope := byResource.ByScope(scope) + + logRecords := srcScope.LogRecords() + for k := 0; k < logRecords.Len(); k++ { + log := otelstef.NewLogRecord() + logRecordToStef(logRecords.At(k), log, &converter) + byScope.Append(log) + } + } + } + + tree.SortValues() + return tree +} + +func logRecordToStef(src plog.LogRecord, dst *otelstef.LogRecord, converter *otlptools.Otlp2Stef) { + dst.SetTimeUnixNano(uint64(src.Timestamp())) + dst.SetObservedTimeUnixNano(uint64(src.ObservedTimestamp())) + dst.SetSeverityNumber(otelstef.SeverityNumber(src.SeverityNumber())) + dst.SetSeverityText(src.SeverityText()) + converter.Value(src.Body(), dst.Body()) + converter.MapSorted(src.Attributes(), dst.Attributes()) + dst.SetDroppedAttributesCount(uint64(src.DroppedAttributesCount())) + dst.SetFlags(uint64(src.Flags())) + traceID := src.TraceID() + spanID := src.SpanID() + dst.SetTraceID(pkg.Bytes(traceID[:])) + dst.SetSpanID(pkg.Bytes(spanID[:])) + dst.SetEventName(src.EventName()) +} diff --git a/go/pdata/logs/sortedbyresource/converter_test.go b/go/pdata/logs/sortedbyresource/converter_test.go new file mode 100644 index 00000000..bb51e8f0 --- /dev/null +++ b/go/pdata/logs/sortedbyresource/converter_test.go @@ -0,0 +1,44 @@ +package sortedbyresource + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/otel/otelstef" + "github.com/splunk/stef/go/pkg" +) + +func TestOtlpToSortedTreeGroupsAndSortsLogs(t *testing.T) { + src := plog.NewLogs() + for _, timestamp := range []pcommon.Timestamp{200, 100} { + resourceLogs := src.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr("service.name", "checkout") + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("logger") + scopeLogs.LogRecords().AppendEmpty().SetTimestamp(timestamp) + } + + tree := OtlpToSortedTree(src) + output := &pkg.MemChunkWriter{} + writer, err := otelstef.NewLogsWriter(output, pkg.WriterOptions{}) + require.NoError(t, err) + require.NoError(t, tree.ToStef(writer)) + require.NoError(t, writer.Flush()) + + reader, err := otelstef.NewLogsReader(bytes.NewReader(output.Bytes())) + require.NoError(t, err) + for i, timestamp := range []uint64{100, 200} { + require.NoError(t, reader.Read(pkg.ReadOptions{})) + require.Equal(t, timestamp, reader.Record.Log().TimeUnixNano()) + if i > 0 { + require.False(t, reader.Record.IsResourceModified()) + require.False(t, reader.Record.IsScopeModified()) + } + } + require.ErrorIs(t, reader.Read(pkg.ReadOptions{}), io.EOF) +} diff --git a/go/pdata/logs/sortedbyresource/sortedlogs.go b/go/pdata/logs/sortedbyresource/sortedlogs.go new file mode 100644 index 00000000..991a7448 --- /dev/null +++ b/go/pdata/logs/sortedbyresource/sortedlogs.go @@ -0,0 +1,117 @@ +package sortedbyresource + +import ( + "io" + "slices" + + "modernc.org/b/v2" + + "github.com/splunk/stef/go/otel/otelstef" +) + +type SortedTree struct { + byResource *b.Tree[*otelstef.Resource, *ByResource] +} + +type ByResource struct { + byScope *b.Tree[*otelstef.Scope, *ByScope] +} + +type ByScope struct { + logs LogRecords +} + +type LogRecords []*otelstef.LogRecord + +func NewSortedLogs() *SortedTree { + return &SortedTree{ + byResource: b.TreeNew[*otelstef.Resource, *ByResource](otelstef.CmpResource), + } +} + +func (s *SortedTree) ByResource(resource *otelstef.Resource) *ByResource { + byResource, exists := s.byResource.Get(resource) + if !exists { + byResource = &ByResource{ + byScope: b.TreeNew[*otelstef.Scope, *ByScope](otelstef.CmpScope), + } + s.byResource.Set(resource, byResource) + } + return byResource +} + +func (r *ByResource) ByScope(scope *otelstef.Scope) *ByScope { + byScope, exists := r.byScope.Get(scope) + if !exists { + byScope = &ByScope{} + r.byScope.Set(scope, byScope) + } + return byScope +} + +func (s *ByScope) Append(log *otelstef.LogRecord) { + s.logs = append(s.logs, log) +} + +func (s *SortedTree) SortValues() { + _ = s.Iter(func(_ *otelstef.Resource, resource *ByResource) error { + return resource.Iter(func(_ *otelstef.Scope, scope *ByScope) error { + slices.SortFunc(scope.logs, otelstef.CmpLogRecord) + return nil + }) + }) +} + +func (s *SortedTree) ToStef(writer *otelstef.LogsWriter) error { + return s.Iter(func(resource *otelstef.Resource, byResource *ByResource) error { + writer.Record.SetResource(resource) + return byResource.Iter(func(scope *otelstef.Scope, byScope *ByScope) error { + writer.Record.SetScope(scope) + for _, log := range byScope.logs { + writer.Record.Log().CopyFrom(log) + if err := writer.Write(); err != nil { + return err + } + } + return nil + }) + }) +} + +func (s *SortedTree) Iter(f func(*otelstef.Resource, *ByResource) error) error { + iter, err := s.byResource.SeekFirst() + if err != nil { + return nil + } + for { + key, value, err := iter.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if err := f(key, value); err != nil { + return err + } + } +} + +func (r *ByResource) Iter(f func(*otelstef.Scope, *ByScope) error) error { + iter, err := r.byScope.SeekFirst() + if err != nil { + return nil + } + for { + key, value, err := iter.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if err := f(key, value); err != nil { + return err + } + } +} diff --git a/go/pdata/logs/stef2otlp.go b/go/pdata/logs/stef2otlp.go new file mode 100644 index 00000000..f71467cb --- /dev/null +++ b/go/pdata/logs/stef2otlp.go @@ -0,0 +1,14 @@ +package logs + +import ( + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/otel/otelstef" +) + +// StefToOtlp defines a converter from STEF format to OTLP logs. +type StefToOtlp interface { + // Convert reads STEF records and returns the corresponding OTLP logs. + // When untilEOF is false, it stops at the end of the current frame. + Convert(reader *otelstef.LogsReader, untilEOF bool) (plog.Logs, error) +} diff --git a/go/pdata/logs/stef2otlp_unsorted.go b/go/pdata/logs/stef2otlp_unsorted.go new file mode 100644 index 00000000..4d351826 --- /dev/null +++ b/go/pdata/logs/stef2otlp_unsorted.go @@ -0,0 +1,117 @@ +package logs + +import ( + "errors" + "io" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/otel/otelstef" + "github.com/splunk/stef/go/pdata/internal/otlptools" + "github.com/splunk/stef/go/pkg" +) + +// StefToOtlpUnsorted reads STEF records and converts them to OTLP logs. +type StefToOtlpUnsorted struct{} + +var _ StefToOtlp = (*StefToOtlpUnsorted)(nil) + +// Convert reads until EOF when untilEOF is true. Otherwise, it reads at least +// one record and stops at the end of the current frame. +func (c *StefToOtlpUnsorted) Convert(reader *otelstef.LogsReader, untilEOF bool) (plog.Logs, error) { + logs := plog.NewLogs() + + if err := reader.Read(pkg.ReadOptions{}); err != nil { + return logs, err + } + + var resourceLogs plog.ResourceLogs + var scopeLogs plog.ScopeLogs + record := &reader.Record + modified := true + + for { + if modified || record.IsResourceModified() { + modified = true + resourceLogs = logs.ResourceLogs().AppendEmpty() + if err := resourceToOtlp(record.Resource(), resourceLogs); err != nil { + return logs, err + } + } + + if modified || record.IsScopeModified() { + modified = true + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + if err := scopeToOtlp(record.Scope(), scopeLogs); err != nil { + return logs, err + } + } + + if err := logRecordToOtlp(record.Log(), scopeLogs.LogRecords().AppendEmpty()); err != nil { + return logs, err + } + modified = false + + var err error + if untilEOF { + err = reader.Read(pkg.ReadOptions{}) + if errors.Is(err, io.EOF) { + break + } + } else { + err = reader.Read(pkg.ReadOptions{TillEndOfFrame: true}) + if errors.Is(err, pkg.ErrEndOfFrame) { + break + } + } + if err != nil { + return logs, err + } + } + + return logs, nil +} + +func resourceToOtlp(src *otelstef.Resource, dst plog.ResourceLogs) error { + if src == nil { + return nil + } + dst.SetSchemaUrl(src.SchemaURL()) + dst.Resource().SetDroppedAttributesCount(uint32(src.DroppedAttributesCount())) + return otlptools.TefToOtlpMap(src.Attributes(), dst.Resource().Attributes()) +} + +func scopeToOtlp(src *otelstef.Scope, dst plog.ScopeLogs) error { + if src == nil { + return nil + } + dst.SetSchemaUrl(src.SchemaURL()) + dst.Scope().SetName(src.Name()) + dst.Scope().SetVersion(src.Version()) + dst.Scope().SetDroppedAttributesCount(uint32(src.DroppedAttributesCount())) + return otlptools.TefToOtlpMap(src.Attributes(), dst.Scope().Attributes()) +} + +func logRecordToOtlp(src *otelstef.LogRecord, dst plog.LogRecord) error { + dst.SetTimestamp(pcommon.Timestamp(src.TimeUnixNano())) + dst.SetObservedTimestamp(pcommon.Timestamp(src.ObservedTimeUnixNano())) + dst.SetSeverityNumber(plog.SeverityNumber(src.SeverityNumber())) + dst.SetSeverityText(src.SeverityText()) + if err := otlptools.TefAnyValueToOtlp(src.Body(), dst.Body()); err != nil { + return err + } + if err := otlptools.TefToOtlpMap(src.Attributes(), dst.Attributes()); err != nil { + return err + } + dst.SetDroppedAttributesCount(uint32(src.DroppedAttributesCount())) + dst.SetFlags(plog.LogRecordFlags(src.Flags())) + var traceID pcommon.TraceID + copy(traceID[:], src.TraceID()) + dst.SetTraceID(traceID) + var spanID pcommon.SpanID + copy(spanID[:], src.SpanID()) + dst.SetSpanID(spanID) + dst.SetEventName(src.EventName()) + return nil +} diff --git a/go/pdata/logs/testtools/normalize.go b/go/pdata/logs/testtools/normalize.go new file mode 100644 index 00000000..5753f38e --- /dev/null +++ b/go/pdata/logs/testtools/normalize.go @@ -0,0 +1,112 @@ +package testtools + +import ( + "slices" + "strings" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/splunk/stef/go/pdata/internal/otlptools" +) + +// NormalizeLogs canonicalizes grouping, record order, and map order so that +// semantically equivalent logs have identical protobuf encodings. +func NormalizeLogs(data plog.Logs) { + resourceLogs := data.ResourceLogs() + for i := 0; i < resourceLogs.Len(); i++ { + normalizeMap(resourceLogs.At(i).Resource().Attributes()) + } + resourceLogs.Sort(func(a, b plog.ResourceLogs) bool { + return otlptools.CmpResourceLogs(a, b) < 0 + }) + for i := 0; i < resourceLogs.Len()-1; { + if otlptools.CmpResourceLogs(resourceLogs.At(i), resourceLogs.At(i+1)) != 0 { + i++ + continue + } + resourceLogs.At(i + 1).ScopeLogs().MoveAndAppendTo(resourceLogs.At(i).ScopeLogs()) + removeResourceLogsAt(resourceLogs, i+1) + } + for i := 0; i < resourceLogs.Len(); i++ { + normalizeScopeLogs(resourceLogs.At(i).ScopeLogs()) + } +} + +func normalizeScopeLogs(scopeLogs plog.ScopeLogsSlice) { + for i := 0; i < scopeLogs.Len(); i++ { + normalizeMap(scopeLogs.At(i).Scope().Attributes()) + } + scopeLogs.Sort(func(a, b plog.ScopeLogs) bool { + return otlptools.CmpScopeLogs(a, b) < 0 + }) + for i := 0; i < scopeLogs.Len()-1; { + if otlptools.CmpScopeLogs(scopeLogs.At(i), scopeLogs.At(i+1)) != 0 { + i++ + continue + } + scopeLogs.At(i + 1).LogRecords().MoveAndAppendTo(scopeLogs.At(i).LogRecords()) + removeScopeLogsAt(scopeLogs, i+1) + } + for i := 0; i < scopeLogs.Len(); i++ { + records := scopeLogs.At(i).LogRecords() + for j := 0; j < records.Len(); j++ { + normalizeValue(records.At(j).Body()) + normalizeMap(records.At(j).Attributes()) + } + records.Sort(func(a, b plog.LogRecord) bool { + return otlptools.CmpLogRecords(a, b) < 0 + }) + } +} + +func normalizeMap(m pcommon.Map) { + type entry struct { + key string + value pcommon.Value + } + entries := make([]entry, 0, m.Len()) + m.Range(func(key string, value pcommon.Value) bool { + normalizeValue(value) + valueCopy := pcommon.NewValueEmpty() + value.CopyTo(valueCopy) + entries = append(entries, entry{key: key, value: valueCopy}) + return true + }) + slices.SortFunc(entries, func(a, b entry) int { return strings.Compare(a.key, b.key) }) + m.Clear() + m.EnsureCapacity(len(entries)) + for _, entry := range entries { + entry.value.CopyTo(m.PutEmpty(entry.key)) + } +} + +func normalizeValue(value pcommon.Value) { + switch value.Type() { + case pcommon.ValueTypeMap: + normalizeMap(value.Map()) + case pcommon.ValueTypeSlice: + values := value.Slice() + for i := 0; i < values.Len(); i++ { + normalizeValue(values.At(i)) + } + } +} + +func removeResourceLogsAt(logs plog.ResourceLogsSlice, index int) { + i := 0 + logs.RemoveIf(func(plog.ResourceLogs) bool { + remove := i == index + i++ + return remove + }) +} + +func removeScopeLogsAt(logs plog.ScopeLogsSlice, index int) { + i := 0 + logs.RemoveIf(func(plog.ScopeLogs) bool { + remove := i == index + i++ + return remove + }) +} diff --git a/go/pdata/logs/testtools/normalize_test.go b/go/pdata/logs/testtools/normalize_test.go new file mode 100644 index 00000000..2ed3f0b0 --- /dev/null +++ b/go/pdata/logs/testtools/normalize_test.go @@ -0,0 +1,64 @@ +package testtools + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/plog" +) + +func TestNormalizeLogsCanonicalizesLogRecordOrder(t *testing.T) { + tests := map[string]func(plog.LogRecord){ + "severity number": func(record plog.LogRecord) { + record.SetSeverityNumber(plog.SeverityNumberError) + }, + "severity text": func(record plog.LogRecord) { + record.SetSeverityText("error") + }, + "body": func(record plog.LogRecord) { + record.Body().SetStr("body") + }, + "attributes": func(record plog.LogRecord) { + record.Attributes().PutStr("key", "value") + }, + "dropped attribute count": func(record plog.LogRecord) { + record.SetDroppedAttributesCount(1) + }, + "flags": func(record plog.LogRecord) { + record.SetFlags(plog.DefaultLogRecordFlags.WithIsSampled(true)) + }, + "event name": func(record plog.LogRecord) { + record.SetEventName("event") + }, + } + + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + forward := newLogsWithRecordOrder(false, mutate) + reversed := newLogsWithRecordOrder(true, mutate) + + NormalizeLogs(forward) + NormalizeLogs(reversed) + + marshaler := plog.ProtoMarshaler{} + forwardBytes, err := marshaler.MarshalLogs(forward) + require.NoError(t, err) + reversedBytes, err := marshaler.MarshalLogs(reversed) + require.NoError(t, err) + require.Equal(t, forwardBytes, reversedBytes) + }) + } +} + +func newLogsWithRecordOrder(reversed bool, mutate func(plog.LogRecord)) plog.Logs { + logs := plog.NewLogs() + records := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + if reversed { + mutate(records.AppendEmpty()) + records.AppendEmpty() + } else { + records.AppendEmpty() + mutate(records.AppendEmpty()) + } + return logs +} diff --git a/go/pdata/metrics/pdatasort.go b/go/pdata/metrics/pdatasort.go index 444127d0..ded42302 100644 --- a/go/pdata/metrics/pdatasort.go +++ b/go/pdata/metrics/pdatasort.go @@ -1,6 +1,7 @@ package metrics import ( + "cmp" "strings" "go.opentelemetry.io/collector/pdata/pcommon" @@ -199,6 +200,10 @@ func (s *PDataSorter) cmpResourceMetrics(left, right pmetric.ResourceMetrics) in if c != 0 { return c } + c = cmp.Compare(left.Resource().DroppedAttributesCount(), right.Resource().DroppedAttributesCount()) + if c != 0 { + return c + } return s.cmpAttrs(left.Resource().Attributes(), right.Resource().Attributes()) } @@ -215,6 +220,10 @@ func (s *PDataSorter) cmpScopeMetrics(left, right pmetric.ScopeMetrics) int { if c != 0 { return c } + c = cmp.Compare(left.Scope().DroppedAttributesCount(), right.Scope().DroppedAttributesCount()) + if c != 0 { + return c + } return s.cmpAttrs(left.Scope().Attributes(), right.Scope().Attributes()) } diff --git a/go/pdata/metrics/sortedbyresource/sortedresource.go b/go/pdata/metrics/sortedbyresource/sortedresource.go index b13a4ecb..814d9e11 100644 --- a/go/pdata/metrics/sortedbyresource/sortedresource.go +++ b/go/pdata/metrics/sortedbyresource/sortedresource.go @@ -91,7 +91,9 @@ func (s *SortedTree) ByResource(resource *otelstef.Resource) *ByResource { elem, exists := s.byResource.Get(resource) if !exists { elem = &ByResource{byScope: b.TreeNew[*otelstef.Scope, *ByScope](otelstef.CmpScope), allocators: &s.allocators} - s.byResource.Set(resource.Clone(&s.allocators), elem) + clone := s.allocators.Resource.Alloc() + resource.CloneTo(clone, &s.allocators) + s.byResource.Set(clone, elem) } return elem } @@ -100,7 +102,9 @@ func (m *ByResource) ByScope(scope *otelstef.Scope) *ByScope { elem, exists := m.byScope.Get(scope) if !exists { elem = &ByScope{byMetrics: b.TreeNew[*otelstef.Metric, *ByMetric](otelstef.CmpMetric), allocators: m.allocators} - m.byScope.Set(scope.Clone(m.allocators), elem) + clone := m.allocators.Scope.Alloc() + scope.CloneTo(clone, m.allocators) + m.byScope.Set(clone, elem) } return elem } @@ -114,7 +118,9 @@ func (m *ByScope) ByMetric( byAttrs: b.TreeNew[*otelstef.Attributes, *Points](otelstef.CmpAttributes), allocators: m.allocators, } - m.byMetrics.Set(metric.Clone(m.allocators), elem) + clone := m.allocators.Metric.Alloc() + metric.CloneTo(clone, m.allocators) + m.byMetrics.Set(clone, elem) } return elem } diff --git a/go/pkg/types.go b/go/pkg/types.go index 9aa5d4e0..51369bc6 100644 --- a/go/pkg/types.go +++ b/go/pkg/types.go @@ -1,6 +1,7 @@ package pkg import ( + "cmp" "math/rand/v2" "strconv" "strings" @@ -42,13 +43,7 @@ func BoolCompare(left, right bool) int { } func Float64Compare(left, right float64) int { - if left > right { - return 1 - } - if left < right { - return -1 - } - return 0 + return cmp.Compare(left, right) } func StringCompare(left, right string) int { diff --git a/java/src/main/java/com/example/otelstef/AnyValue.java b/java/src/main/java/com/example/otelstef/AnyValue.java index 458617f0..4eaeed5b 100644 --- a/java/src/main/java/com/example/otelstef/AnyValue.java +++ b/java/src/main/java/com/example/otelstef/AnyValue.java @@ -219,14 +219,8 @@ public void setBytes(BytesValue v) { // Clone returns a deep copy of this oneof. public AnyValue clone() { AnyValue cpy = new AnyValue(); - cpy.typ = this.typ; - cpy.string = this.string; - cpy.bool = this.bool; - cpy.int64 = this.int64; - cpy.float64 = this.float64; - cpy.array = this.array.clone(); - cpy.kVList = this.kVList.clone(); - cpy.bytes = this.bytes; + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } @@ -493,15 +487,23 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash( - string, - bool, - int64, - float64, - array, - kVList, - bytes, - typ - ); + switch (this.typ) { + case TypeString: + return Objects.hash(typ, string); + case TypeBool: + return Objects.hash(typ, bool); + case TypeInt64: + return Objects.hash(typ, int64); + case TypeFloat64: + return Objects.hash(typ, float64); + case TypeArray: + return Objects.hash(typ, array); + case TypeKVList: + return Objects.hash(typ, kVList); + case TypeBytes: + return Objects.hash(typ, bytes); + default: + return Objects.hash(typ); + } } } diff --git a/java/src/main/java/com/example/otelstef/AnyValueArrayEncoder.java b/java/src/main/java/com/example/otelstef/AnyValueArrayEncoder.java index 774e8431..3a0530e6 100644 --- a/java/src/main/java/com/example/otelstef/AnyValueArrayEncoder.java +++ b/java/src/main/java/com/example/otelstef/AnyValueArrayEncoder.java @@ -17,6 +17,8 @@ class AnyValueArrayEncoder { private AnyValueEncoder elemEncoder; private WriterState state; private boolean isRecursive = false; + // Set after reset so every element in the first non-empty array is encoded in full. + private boolean forceElements; public void init(WriterState state, WriteColumnSet columns) throws IOException { this.state = state; @@ -43,20 +45,28 @@ public void init(WriterState state, WriteColumnSet columns) throws IOException { } public void reset() { + forceElements = true; if (!isRecursive) { elemEncoder.reset(); } } public void encode(AnyValueArray arr) throws IOException { + encode(arr, false); + } + + public void encode(AnyValueArray arr, boolean forceAllElements) throws IOException { int oldBitLen = buf.bitCount(); // Write the length of the array. int newLen = arr.elemsLen; buf.writeUvarintCompact(newLen); - + boolean forceElementsNow = forceAllElements || forceElements; for (int i = 0; i < newLen; i++) { - elemEncoder.encode(arr.elems[i]); + elemEncoder.encode(arr.elems[i], forceElementsNow); + } + if (newLen != 0) { + forceElements = false; } // Account written bits in the limiter. @@ -71,4 +81,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } } - diff --git a/java/src/main/java/com/example/otelstef/AnyValueEncoder.java b/java/src/main/java/com/example/otelstef/AnyValueEncoder.java index 429a5f77..ea9f670f 100644 --- a/java/src/main/java/com/example/otelstef/AnyValueEncoder.java +++ b/java/src/main/java/com/example/otelstef/AnyValueEncoder.java @@ -149,6 +149,12 @@ public void reset() { // Encode encodes val into buf public void encode(AnyValue val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf and propagates forced full encoding through + // the selected non-primitive value. + public void encode(AnyValue val, boolean forceAllFields) throws IOException { AnyValue.Type typ = val.typ; if (typ.getValue() > fieldCount) { // The current field type is not supported in target schema. Encode the type as None. @@ -181,11 +187,11 @@ public void encode(AnyValue val) throws IOException { break; case TypeArray: // Encode Array - arrayEncoder.encode(val.array); + arrayEncoder.encode(val.array, forceAllFields); break; case TypeKVList: // Encode KVList - kVListEncoder.encode(val.kVList); + kVListEncoder.encode(val.kVList, forceAllFields); break; case TypeBytes: // Encode Bytes diff --git a/java/src/main/java/com/example/otelstef/Attributes.java b/java/src/main/java/com/example/otelstef/Attributes.java index ae53eb57..fc18559f 100644 --- a/java/src/main/java/com/example/otelstef/Attributes.java +++ b/java/src/main/java/com/example/otelstef/Attributes.java @@ -123,6 +123,7 @@ void markUnmodifiedRecursively() { for (int i=0; i fieldCount) { // The current field type is not supported in target schema. Encode the type as None. diff --git a/java/src/main/java/com/example/otelstef/ExpHistogramBuckets.java b/java/src/main/java/com/example/otelstef/ExpHistogramBuckets.java index 46462d85..c2116ee4 100644 --- a/java/src/main/java/com/example/otelstef/ExpHistogramBuckets.java +++ b/java/src/main/java/com/example/otelstef/ExpHistogramBuckets.java @@ -111,8 +111,8 @@ void markUnmodifiedRecursively() { public ExpHistogramBuckets clone() { ExpHistogramBuckets cpy = new ExpHistogramBuckets(); - cpy.offset = this.offset; - cpy.bucketCounts = this.bucketCounts.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/ExpHistogramBucketsEncoder.java b/java/src/main/java/com/example/otelstef/ExpHistogramBucketsEncoder.java index 746dd822..2f35d206 100644 --- a/java/src/main/java/com/example/otelstef/ExpHistogramBucketsEncoder.java +++ b/java/src/main/java/com/example/otelstef/ExpHistogramBucketsEncoder.java @@ -87,18 +87,26 @@ public void reset() { // encode encodes val into buf public void encode(ExpHistogramBuckets val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(ExpHistogramBuckets val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = ExpHistogramBuckets.fieldModifiedOffset | ExpHistogramBuckets.fieldModifiedBucketCounts | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -116,7 +124,7 @@ public void encode(ExpHistogramBuckets val) throws IOException { if ((fieldMask & ExpHistogramBuckets.fieldModifiedBucketCounts) != 0) { // Encode BucketCounts - this.bucketCountsEncoder.encode(val.bucketCounts); + this.bucketCountsEncoder.encode(val.bucketCounts, forceFields); } // Account written bits in the limiter. @@ -152,4 +160,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/ExpHistogramValue.java b/java/src/main/java/com/example/otelstef/ExpHistogramValue.java index 9e80e51b..7647e1db 100644 --- a/java/src/main/java/com/example/otelstef/ExpHistogramValue.java +++ b/java/src/main/java/com/example/otelstef/ExpHistogramValue.java @@ -361,15 +361,8 @@ void markUnmodifiedRecursively() { public ExpHistogramValue clone() { ExpHistogramValue cpy = new ExpHistogramValue(); - cpy.count = this.count; - cpy.sum = this.sum; - cpy.min = this.min; - cpy.max = this.max; - cpy.scale = this.scale; - cpy.zeroCount = this.zeroCount; - cpy.positiveBuckets = this.positiveBuckets.clone(); - cpy.negativeBuckets = this.negativeBuckets.clone(); - cpy.zeroThreshold = this.zeroThreshold; + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/ExpHistogramValueEncoder.java b/java/src/main/java/com/example/otelstef/ExpHistogramValueEncoder.java index 912423ed..c2699a89 100644 --- a/java/src/main/java/com/example/otelstef/ExpHistogramValueEncoder.java +++ b/java/src/main/java/com/example/otelstef/ExpHistogramValueEncoder.java @@ -182,15 +182,22 @@ public void reset() { // encode encodes val into buf public void encode(ExpHistogramValue val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(ExpHistogramValue val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = ExpHistogramValue.fieldModifiedCount | ExpHistogramValue.fieldModifiedSum | @@ -201,6 +208,7 @@ public void encode(ExpHistogramValue val) throws IOException { ExpHistogramValue.fieldModifiedPositiveBuckets | ExpHistogramValue.fieldModifiedNegativeBuckets | ExpHistogramValue.fieldModifiedZeroThreshold | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -245,12 +253,12 @@ public void encode(ExpHistogramValue val) throws IOException { if ((fieldMask & ExpHistogramValue.fieldModifiedPositiveBuckets) != 0) { // Encode PositiveBuckets - this.positiveBucketsEncoder.encode(val.positiveBuckets); + this.positiveBucketsEncoder.encode(val.positiveBuckets, forceFields); } if ((fieldMask & ExpHistogramValue.fieldModifiedNegativeBuckets) != 0) { // Encode NegativeBuckets - this.negativeBucketsEncoder.encode(val.negativeBuckets); + this.negativeBucketsEncoder.encode(val.negativeBuckets, forceFields); } if ((fieldMask & ExpHistogramValue.fieldModifiedZeroThreshold) != 0) { @@ -348,4 +356,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/Float64ArrayEncoder.java b/java/src/main/java/com/example/otelstef/Float64ArrayEncoder.java index 230b361b..ec25d21f 100644 --- a/java/src/main/java/com/example/otelstef/Float64ArrayEncoder.java +++ b/java/src/main/java/com/example/otelstef/Float64ArrayEncoder.java @@ -33,12 +33,15 @@ public void reset() { } public void encode(Float64Array arr) throws IOException { + encode(arr, false); + } + + public void encode(Float64Array arr, boolean forceAllElements) throws IOException { int oldBitLen = buf.bitCount(); // Write the length of the array. int newLen = arr.elemsLen; buf.writeUvarintCompact(newLen); - for (int i = 0; i < newLen; i++) { elemEncoder.encode(arr.elems[i]); } @@ -55,4 +58,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } } - diff --git a/java/src/main/java/com/example/otelstef/HistogramValue.java b/java/src/main/java/com/example/otelstef/HistogramValue.java index 10eb6daa..51fd8894 100644 --- a/java/src/main/java/com/example/otelstef/HistogramValue.java +++ b/java/src/main/java/com/example/otelstef/HistogramValue.java @@ -247,11 +247,8 @@ void markUnmodifiedRecursively() { public HistogramValue clone() { HistogramValue cpy = new HistogramValue(); - cpy.count = this.count; - cpy.sum = this.sum; - cpy.min = this.min; - cpy.max = this.max; - cpy.bucketCounts = this.bucketCounts.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/HistogramValueEncoder.java b/java/src/main/java/com/example/otelstef/HistogramValueEncoder.java index 6e3a5bb0..ff2663de 100644 --- a/java/src/main/java/com/example/otelstef/HistogramValueEncoder.java +++ b/java/src/main/java/com/example/otelstef/HistogramValueEncoder.java @@ -124,21 +124,29 @@ public void reset() { // encode encodes val into buf public void encode(HistogramValue val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(HistogramValue val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = HistogramValue.fieldModifiedCount | HistogramValue.fieldModifiedSum | HistogramValue.fieldModifiedMin | HistogramValue.fieldModifiedMax | HistogramValue.fieldModifiedBucketCounts | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -173,7 +181,7 @@ public void encode(HistogramValue val) throws IOException { if ((fieldMask & HistogramValue.fieldModifiedBucketCounts) != 0) { // Encode BucketCounts - this.bucketCountsEncoder.encode(val.bucketCounts); + this.bucketCountsEncoder.encode(val.bucketCounts, forceFields); } // Account written bits in the limiter. @@ -233,4 +241,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/KeyValueList.java b/java/src/main/java/com/example/otelstef/KeyValueList.java index 44eb6a0d..4783c9a4 100644 --- a/java/src/main/java/com/example/otelstef/KeyValueList.java +++ b/java/src/main/java/com/example/otelstef/KeyValueList.java @@ -123,6 +123,7 @@ void markUnmodifiedRecursively() { for (int i=0; i right. + public static int compare(LogRecord left, LogRecord right) { + if (left == null) { + if (right == null) { + return 0; + } + return -1; + } + if (right == null) { + return 1; + } + int c; + + // Compare TimeUnixNano field. + c = Types.Uint64Compare(left.timeUnixNano, right.timeUnixNano); + if (c != 0) { + return c; + } + + // Compare ObservedTimeUnixNano field. + c = Types.Uint64Compare(left.observedTimeUnixNano, right.observedTimeUnixNano); + if (c != 0) { + return c; + } + + // Compare SeverityNumber field. + c = Types.Uint64Compare(left.severityNumber, right.severityNumber); + if (c != 0) { + return c; + } + + // Compare SeverityText field. + c = Types.StringCompare(left.severityText, right.severityText); + if (c != 0) { + return c; + } + + // Compare Body field. + c = AnyValue.compare(left.body, right.body); + if (c != 0) { + return c; + } + + // Compare Attributes field. + c = Attributes.compare(left.attributes, right.attributes); + if (c != 0) { + return c; + } + + // Compare DroppedAttributesCount field. + c = Types.Uint64Compare(left.droppedAttributesCount, right.droppedAttributesCount); + if (c != 0) { + return c; + } + + // Compare Flags field. + c = Types.Uint64Compare(left.flags, right.flags); + if (c != 0) { + return c; + } + + // Compare TraceID field. + c = Types.BytesCompare(left.traceID, right.traceID); + if (c != 0) { + return c; + } + + // Compare SpanID field. + c = Types.BytesCompare(left.spanID, right.spanID); + if (c != 0) { + return c; + } + + // Compare EventName field. + c = Types.StringCompare(left.eventName, right.eventName); + if (c != 0) { + return c; + } + + return 0; + } + + // mutateRandom mutates fields in a random, deterministic manner using random as a deterministic generator. + void mutateRandom(Random random, CommonMutateRandomLimiter limiter) { + final int fieldCount = Math.max(11,2); // At least 2 to ensure we don't recurse infinitely if there is only 1 field. + + if (random.nextInt(fieldCount) == 0) { + this.setTimeUnixNano(Types.Uint64Random(random)); + } + + if (random.nextInt(fieldCount) == 0) { + this.setObservedTimeUnixNano(Types.Uint64Random(random)); + } + + if (random.nextInt(fieldCount) == 0) { + this.setSeverityNumber((Types.Uint64Random(random) & 0x7FFFFFFFFFFFFFFFL) % 25); + } + + if (random.nextInt(fieldCount) == 0) { + this.setSeverityText(Types.StringRandom(random)); + } + + if (random.nextInt(fieldCount) == 0) { + this.body.mutateRandom(random, limiter); + } + + if (random.nextInt(fieldCount) == 0) { + this.attributes.mutateRandom(random, limiter); + } + + if (random.nextInt(fieldCount) == 0) { + this.setDroppedAttributesCount(Types.Uint64Random(random)); + } + + if (random.nextInt(fieldCount) == 0) { + this.setFlags(Types.Uint64Random(random)); + } + + if (random.nextInt(fieldCount) == 0) { + this.setTraceID(Types.BytesRandom(random)); + } + + if (random.nextInt(fieldCount) == 0) { + this.setSpanID(Types.BytesRandom(random)); + } + + if (random.nextInt(fieldCount) == 0) { + this.setEventName(Types.StringRandom(random)); + } + + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) { + return false; + } + return equals((LogRecord)o); + } + + @Override + public int hashCode() { + return Objects.hash( + timeUnixNano, + observedTimeUnixNano, + severityNumber, + severityText, + body, + attributes, + droppedAttributesCount, + flags, + traceID, + spanID, + eventName + ); + } + + +} diff --git a/java/src/main/java/com/example/otelstef/LogRecordDecoder.java b/java/src/main/java/com/example/otelstef/LogRecordDecoder.java new file mode 100644 index 00000000..4bce5854 --- /dev/null +++ b/java/src/main/java/com/example/otelstef/LogRecordDecoder.java @@ -0,0 +1,324 @@ +// Code generated by stefc. DO NOT EDIT. +// LogRecordDecoder implements decoding of LogRecord +package com.example.otelstef; + +import net.stef.BitsReader; +import net.stef.Helper; +import net.stef.ReadColumnSet; +import net.stef.ReadableColumn; +import net.stef.codecs.*; + +import java.io.IOException; + +class LogRecordDecoder { + private final BitsReader buf = new BitsReader(); + private ReadableColumn column; + private int fieldCount; + + + + private Uint64Decoder timeUnixNanoDecoder; + private boolean isTimeUnixNanoRecursive = false; // Indicates TimeUnixNano field's type is recursive. + private Uint64Decoder observedTimeUnixNanoDecoder; + private boolean isObservedTimeUnixNanoRecursive = false; // Indicates ObservedTimeUnixNano field's type is recursive. + private Uint64Decoder severityNumberDecoder; + private boolean isSeverityNumberRecursive = false; // Indicates SeverityNumber field's type is recursive. + private StringDictDecoder severityTextDecoder; + private boolean isSeverityTextRecursive = false; // Indicates SeverityText field's type is recursive. + private AnyValueDecoder bodyDecoder; + private boolean isBodyRecursive = false; // Indicates Body field's type is recursive. + private AttributesDecoder attributesDecoder; + private boolean isAttributesRecursive = false; // Indicates Attributes field's type is recursive. + private Uint64Decoder droppedAttributesCountDecoder; + private boolean isDroppedAttributesCountRecursive = false; // Indicates DroppedAttributesCount field's type is recursive. + private Uint64Decoder flagsDecoder; + private boolean isFlagsRecursive = false; // Indicates Flags field's type is recursive. + private BytesDecoder traceIDDecoder; + private boolean isTraceIDRecursive = false; // Indicates TraceID field's type is recursive. + private BytesDecoder spanIDDecoder; + private boolean isSpanIDRecursive = false; // Indicates SpanID field's type is recursive. + private StringDictDecoder eventNameDecoder; + private boolean isEventNameRecursive = false; // Indicates EventName field's type is recursive. + + + // Init is called once in the lifetime of the stream. + public void init(ReaderState state, ReadColumnSet columns) throws IOException { + // Remember this encoder in the state so that we can detect recursion. + if (state.LogRecordDecoder != null) { + throw new IllegalStateException("cannot initialize LogRecordDecoder: already initialized"); + } + state.LogRecordDecoder = this; + + try { + fieldCount = state.getStructFieldCounts().getLogRecordFieldCount(); + column = columns.getColumn(); + + if (this.fieldCount <= 0) { + return; // TimeUnixNano and subsequent fields are skipped. + } + timeUnixNanoDecoder = new Uint64Decoder(); + timeUnixNanoDecoder.init(columns.addSubColumn()); + if (this.fieldCount <= 1) { + return; // ObservedTimeUnixNano and subsequent fields are skipped. + } + observedTimeUnixNanoDecoder = new Uint64Decoder(); + observedTimeUnixNanoDecoder.init(columns.addSubColumn()); + if (this.fieldCount <= 2) { + return; // SeverityNumber and subsequent fields are skipped. + } + severityNumberDecoder = new Uint64Decoder(); + severityNumberDecoder.init(columns.addSubColumn()); + if (this.fieldCount <= 3) { + return; // SeverityText and subsequent fields are skipped. + } + severityTextDecoder = new StringDictDecoder(); + severityTextDecoder.init(state.LogSeverityText, columns.addSubColumn()); + if (this.fieldCount <= 4) { + return; // Body and subsequent fields are skipped. + } + if (state.AnyValueDecoder != null) { + // Recursion detected, use the existing decoder. + bodyDecoder = state.AnyValueDecoder; + isBodyRecursive = true; // Mark that we are using a recursive decoder. + } else { + bodyDecoder = new AnyValueDecoder(); + bodyDecoder.init(state, columns.addSubColumn()); + } + if (this.fieldCount <= 5) { + return; // Attributes and subsequent fields are skipped. + } + if (state.AttributesDecoder != null) { + // Recursion detected, use the existing decoder. + attributesDecoder = state.AttributesDecoder; + isAttributesRecursive = true; // Mark that we are using a recursive decoder. + } else { + attributesDecoder = new AttributesDecoder(); + attributesDecoder.init(state, columns.addSubColumn()); + } + if (this.fieldCount <= 6) { + return; // DroppedAttributesCount and subsequent fields are skipped. + } + droppedAttributesCountDecoder = new Uint64Decoder(); + droppedAttributesCountDecoder.init(columns.addSubColumn()); + if (this.fieldCount <= 7) { + return; // Flags and subsequent fields are skipped. + } + flagsDecoder = new Uint64Decoder(); + flagsDecoder.init(columns.addSubColumn()); + if (this.fieldCount <= 8) { + return; // TraceID and subsequent fields are skipped. + } + traceIDDecoder = new BytesDecoder(); + traceIDDecoder.init(columns.addSubColumn()); + if (this.fieldCount <= 9) { + return; // SpanID and subsequent fields are skipped. + } + spanIDDecoder = new BytesDecoder(); + spanIDDecoder.init(columns.addSubColumn()); + if (this.fieldCount <= 10) { + return; // EventName and subsequent fields are skipped. + } + eventNameDecoder = new StringDictDecoder(); + eventNameDecoder.init(state.LogEventName, columns.addSubColumn()); + } finally { + state.LogRecordDecoder = null; + } + } + + // continueDecoding is called at the start of the frame to continue decoding column data. + // This should set the decoder's source buffer, so the new decoding continues from + // the supplied column data. This should NOT reset the internal state of the decoder, + // since columns can cross frame boundaries and the new column data is considered + // continuation of that same column in the previous frame. + public void continueDecoding() { + this.buf.reset(this.column.getData()); + + if (this.fieldCount <= 0) { + return; // TimeUnixNano and subsequent fields are skipped. + } + timeUnixNanoDecoder.continueDecoding(); + if (this.fieldCount <= 1) { + return; // ObservedTimeUnixNano and subsequent fields are skipped. + } + observedTimeUnixNanoDecoder.continueDecoding(); + if (this.fieldCount <= 2) { + return; // SeverityNumber and subsequent fields are skipped. + } + severityNumberDecoder.continueDecoding(); + if (this.fieldCount <= 3) { + return; // SeverityText and subsequent fields are skipped. + } + severityTextDecoder.continueDecoding(); + if (this.fieldCount <= 4) { + return; // Body and subsequent fields are skipped. + } + + if (!isBodyRecursive) { + bodyDecoder.continueDecoding(); + } + + if (this.fieldCount <= 5) { + return; // Attributes and subsequent fields are skipped. + } + + if (!isAttributesRecursive) { + attributesDecoder.continueDecoding(); + } + + if (this.fieldCount <= 6) { + return; // DroppedAttributesCount and subsequent fields are skipped. + } + droppedAttributesCountDecoder.continueDecoding(); + if (this.fieldCount <= 7) { + return; // Flags and subsequent fields are skipped. + } + flagsDecoder.continueDecoding(); + if (this.fieldCount <= 8) { + return; // TraceID and subsequent fields are skipped. + } + traceIDDecoder.continueDecoding(); + if (this.fieldCount <= 9) { + return; // SpanID and subsequent fields are skipped. + } + spanIDDecoder.continueDecoding(); + if (this.fieldCount <= 10) { + return; // EventName and subsequent fields are skipped. + } + eventNameDecoder.continueDecoding(); + } + + public void reset() { + + if (fieldCount <= 0) { + // TimeUnixNano and all subsequent fields are skipped. + return; + } + timeUnixNanoDecoder.reset(); + if (fieldCount <= 1) { + // ObservedTimeUnixNano and all subsequent fields are skipped. + return; + } + observedTimeUnixNanoDecoder.reset(); + if (fieldCount <= 2) { + // SeverityNumber and all subsequent fields are skipped. + return; + } + severityNumberDecoder.reset(); + if (fieldCount <= 3) { + // SeverityText and all subsequent fields are skipped. + return; + } + severityTextDecoder.reset(); + if (fieldCount <= 4) { + // Body and all subsequent fields are skipped. + return; + } + if (!isBodyRecursive) { + bodyDecoder.reset(); + } + if (fieldCount <= 5) { + // Attributes and all subsequent fields are skipped. + return; + } + if (!isAttributesRecursive) { + attributesDecoder.reset(); + } + if (fieldCount <= 6) { + // DroppedAttributesCount and all subsequent fields are skipped. + return; + } + droppedAttributesCountDecoder.reset(); + if (fieldCount <= 7) { + // Flags and all subsequent fields are skipped. + return; + } + flagsDecoder.reset(); + if (fieldCount <= 8) { + // TraceID and all subsequent fields are skipped. + return; + } + traceIDDecoder.reset(); + if (fieldCount <= 9) { + // SpanID and all subsequent fields are skipped. + return; + } + spanIDDecoder.reset(); + if (fieldCount <= 10) { + // EventName and all subsequent fields are skipped. + return; + } + eventNameDecoder.reset(); + } + + public LogRecord decode(LogRecord dstPtr) throws IOException { + LogRecord val = dstPtr; + // Read bits that indicate which fields follow. + val.modifiedFields.mask = buf.readBits(fieldCount); + + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedTimeUnixNano) != 0) { + // Field is changed. + val.timeUnixNano = timeUnixNanoDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedObservedTimeUnixNano) != 0) { + // Field is changed. + val.observedTimeUnixNano = observedTimeUnixNanoDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedSeverityNumber) != 0) { + // Field is changed. + val.severityNumber = severityNumberDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedSeverityText) != 0) { + // Field is changed. + val.severityText = severityTextDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedBody) != 0) { + // Field is changed. + if (val.body == null) { + val.body = new AnyValue(val.modifiedFields, LogRecord.fieldModifiedBody); + } + val.body = bodyDecoder.decode(val.body); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedAttributes) != 0) { + // Field is changed. + if (val.attributes == null) { + val.attributes = new Attributes(val.modifiedFields, LogRecord.fieldModifiedAttributes); + } + val.attributes = attributesDecoder.decode(val.attributes); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedDroppedAttributesCount) != 0) { + // Field is changed. + val.droppedAttributesCount = droppedAttributesCountDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedFlags) != 0) { + // Field is changed. + val.flags = flagsDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedTraceID) != 0) { + // Field is changed. + val.traceID = traceIDDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedSpanID) != 0) { + // Field is changed. + val.spanID = spanIDDecoder.decode(); + } + + if ((val.modifiedFields.mask & LogRecord.fieldModifiedEventName) != 0) { + // Field is changed. + val.eventName = eventNameDecoder.decode(); + } + + + return val; + } +} + diff --git a/java/src/main/java/com/example/otelstef/LogRecordEncoder.java b/java/src/main/java/com/example/otelstef/LogRecordEncoder.java new file mode 100644 index 00000000..1a9d13f1 --- /dev/null +++ b/java/src/main/java/com/example/otelstef/LogRecordEncoder.java @@ -0,0 +1,407 @@ +// Code generated by stefc. DO NOT EDIT. +// LogRecordEncoder implements encoding of LogRecord +package com.example.otelstef; + +import net.stef.BitsWriter; +import net.stef.Helper; +import net.stef.SizeLimiter; +import net.stef.WriteColumnSet; +import net.stef.codecs.*; + +import java.io.IOException; + +class LogRecordEncoder { + private BitsWriter buf = new BitsWriter(); + private SizeLimiter limiter; + + // forceModifiedFields is set to true if the next encoding operation + // must write all fields, whether they are modified or not. + // This is used after frame restarts so that the data can be decoded + // from the frame start. + private boolean forceModifiedFields; + + + private Uint64Encoder timeUnixNanoEncoder; + private boolean isTimeUnixNanoRecursive = false; // Indicates TimeUnixNano field's type is recursive. + private Uint64Encoder observedTimeUnixNanoEncoder; + private boolean isObservedTimeUnixNanoRecursive = false; // Indicates ObservedTimeUnixNano field's type is recursive. + private Uint64Encoder severityNumberEncoder; + private boolean isSeverityNumberRecursive = false; // Indicates SeverityNumber field's type is recursive. + private StringDictEncoder severityTextEncoder; + private boolean isSeverityTextRecursive = false; // Indicates SeverityText field's type is recursive. + private AnyValueEncoder bodyEncoder; + private boolean isBodyRecursive = false; // Indicates Body field's type is recursive. + private AttributesEncoder attributesEncoder; + private boolean isAttributesRecursive = false; // Indicates Attributes field's type is recursive. + private Uint64Encoder droppedAttributesCountEncoder; + private boolean isDroppedAttributesCountRecursive = false; // Indicates DroppedAttributesCount field's type is recursive. + private Uint64Encoder flagsEncoder; + private boolean isFlagsRecursive = false; // Indicates Flags field's type is recursive. + private BytesEncoder traceIDEncoder; + private boolean isTraceIDRecursive = false; // Indicates TraceID field's type is recursive. + private BytesEncoder spanIDEncoder; + private boolean isSpanIDRecursive = false; // Indicates SpanID field's type is recursive. + private StringDictEncoder eventNameEncoder; + private boolean isEventNameRecursive = false; // Indicates EventName field's type is recursive. + + + private long keepFieldMask; + private int fieldCount; + + + public void init(WriterState state, WriteColumnSet columns) throws IOException { + // Remember this encoder in the state so that we can detect recursion. + if (state.LogRecordEncoder != null) { + throw new IllegalStateException("cannot initialize LogRecordEncoder: already initialized"); + } + state.LogRecordEncoder = this; + + try { + limiter = state.getLimiter(); + + fieldCount = state.getStructFieldCounts().getLogRecordFieldCount(); + keepFieldMask = ~((~0L) << fieldCount); + // Init encoder for TimeUnixNano field. + if (fieldCount <= 0) { + return; // TimeUnixNano and subsequent fields are skipped. + } + timeUnixNanoEncoder = new Uint64Encoder(); + timeUnixNanoEncoder.init(limiter, columns.addSubColumn()); + // Init encoder for ObservedTimeUnixNano field. + if (fieldCount <= 1) { + return; // ObservedTimeUnixNano and subsequent fields are skipped. + } + observedTimeUnixNanoEncoder = new Uint64Encoder(); + observedTimeUnixNanoEncoder.init(limiter, columns.addSubColumn()); + // Init encoder for SeverityNumber field. + if (fieldCount <= 2) { + return; // SeverityNumber and subsequent fields are skipped. + } + severityNumberEncoder = new Uint64Encoder(); + severityNumberEncoder.init(limiter, columns.addSubColumn()); + // Init encoder for SeverityText field. + if (fieldCount <= 3) { + return; // SeverityText and subsequent fields are skipped. + } + severityTextEncoder = new StringDictEncoder(); + severityTextEncoder.init(state.LogSeverityText, limiter, columns.addSubColumn()); + // Init encoder for Body field. + if (fieldCount <= 4) { + return; // Body and subsequent fields are skipped. + } + if (state.AnyValueEncoder != null) { + // Recursion detected, use the existing encoder. + bodyEncoder = state.AnyValueEncoder; + isBodyRecursive = true; + } else { + bodyEncoder = new AnyValueEncoder(); + bodyEncoder.init(state, columns.addSubColumn()); + } + // Init encoder for Attributes field. + if (fieldCount <= 5) { + return; // Attributes and subsequent fields are skipped. + } + if (state.AttributesEncoder != null) { + // Recursion detected, use the existing encoder. + attributesEncoder = state.AttributesEncoder; + isAttributesRecursive = true; + } else { + attributesEncoder = new AttributesEncoder(); + attributesEncoder.init(state, columns.addSubColumn()); + } + // Init encoder for DroppedAttributesCount field. + if (fieldCount <= 6) { + return; // DroppedAttributesCount and subsequent fields are skipped. + } + droppedAttributesCountEncoder = new Uint64Encoder(); + droppedAttributesCountEncoder.init(limiter, columns.addSubColumn()); + // Init encoder for Flags field. + if (fieldCount <= 7) { + return; // Flags and subsequent fields are skipped. + } + flagsEncoder = new Uint64Encoder(); + flagsEncoder.init(limiter, columns.addSubColumn()); + // Init encoder for TraceID field. + if (fieldCount <= 8) { + return; // TraceID and subsequent fields are skipped. + } + traceIDEncoder = new BytesEncoder(); + traceIDEncoder.init(limiter, columns.addSubColumn()); + // Init encoder for SpanID field. + if (fieldCount <= 9) { + return; // SpanID and subsequent fields are skipped. + } + spanIDEncoder = new BytesEncoder(); + spanIDEncoder.init(limiter, columns.addSubColumn()); + // Init encoder for EventName field. + if (fieldCount <= 10) { + return; // EventName and subsequent fields are skipped. + } + eventNameEncoder = new StringDictEncoder(); + eventNameEncoder.init(state.LogEventName, limiter, columns.addSubColumn()); + } finally { + state.LogRecordEncoder = null; + } + } + + public void reset() { + // Since we are resetting the state of encoder make sure the next encode() + // call forcefully writes all fields and does not attempt to skip. + this.forceModifiedFields = true; + + if (fieldCount <= 0) { + return; // TimeUnixNano and all subsequent fields are skipped. + } + timeUnixNanoEncoder.reset(); + if (fieldCount <= 1) { + return; // ObservedTimeUnixNano and all subsequent fields are skipped. + } + observedTimeUnixNanoEncoder.reset(); + if (fieldCount <= 2) { + return; // SeverityNumber and all subsequent fields are skipped. + } + severityNumberEncoder.reset(); + if (fieldCount <= 3) { + return; // SeverityText and all subsequent fields are skipped. + } + severityTextEncoder.reset(); + if (fieldCount <= 4) { + return; // Body and all subsequent fields are skipped. + } + + if (!isBodyRecursive) { + bodyEncoder.reset(); + } + + if (fieldCount <= 5) { + return; // Attributes and all subsequent fields are skipped. + } + + if (!isAttributesRecursive) { + attributesEncoder.reset(); + } + + if (fieldCount <= 6) { + return; // DroppedAttributesCount and all subsequent fields are skipped. + } + droppedAttributesCountEncoder.reset(); + if (fieldCount <= 7) { + return; // Flags and all subsequent fields are skipped. + } + flagsEncoder.reset(); + if (fieldCount <= 8) { + return; // TraceID and all subsequent fields are skipped. + } + traceIDEncoder.reset(); + if (fieldCount <= 9) { + return; // SpanID and all subsequent fields are skipped. + } + spanIDEncoder.reset(); + if (fieldCount <= 10) { + return; // EventName and all subsequent fields are skipped. + } + eventNameEncoder.reset(); + } + + // encode encodes val into buf + public void encode(LogRecord val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(LogRecord val, boolean forceAllFields) throws IOException { + int oldLen = this.buf.bitCount(); + + + + // Mask that describes what fields are encoded. Start with all modified fields. + long fieldMask = val.modifiedFields.mask; + // On an encoder restart or when requested by a containing collection, + // force writing of all fields. + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { + fieldMask = + LogRecord.fieldModifiedTimeUnixNano | + LogRecord.fieldModifiedObservedTimeUnixNano | + LogRecord.fieldModifiedSeverityNumber | + LogRecord.fieldModifiedSeverityText | + LogRecord.fieldModifiedBody | + LogRecord.fieldModifiedAttributes | + LogRecord.fieldModifiedDroppedAttributesCount | + LogRecord.fieldModifiedFlags | + LogRecord.fieldModifiedTraceID | + LogRecord.fieldModifiedSpanID | + LogRecord.fieldModifiedEventName | 0L; + this.forceModifiedFields = false; + } + + // Only write fields that we want to write. See init() for keepFieldMask. + fieldMask &= this.keepFieldMask; + + // Write bits to indicate which fields follow. + this.buf.writeBits(fieldMask, this.fieldCount); + + // Encode modified, present fields. + + if ((fieldMask & LogRecord.fieldModifiedTimeUnixNano) != 0) { + // Encode TimeUnixNano + this.timeUnixNanoEncoder.encode(val.timeUnixNano); + } + + if ((fieldMask & LogRecord.fieldModifiedObservedTimeUnixNano) != 0) { + // Encode ObservedTimeUnixNano + this.observedTimeUnixNanoEncoder.encode(val.observedTimeUnixNano); + } + + if ((fieldMask & LogRecord.fieldModifiedSeverityNumber) != 0) { + // Encode SeverityNumber + this.severityNumberEncoder.encode(val.severityNumber); + } + + if ((fieldMask & LogRecord.fieldModifiedSeverityText) != 0) { + // Encode SeverityText + this.severityTextEncoder.encode(val.severityText); + } + + if ((fieldMask & LogRecord.fieldModifiedBody) != 0) { + // Encode Body + this.bodyEncoder.encode(val.body, forceFields); + } + + if ((fieldMask & LogRecord.fieldModifiedAttributes) != 0) { + // Encode Attributes + this.attributesEncoder.encode(val.attributes, forceFields); + } + + if ((fieldMask & LogRecord.fieldModifiedDroppedAttributesCount) != 0) { + // Encode DroppedAttributesCount + this.droppedAttributesCountEncoder.encode(val.droppedAttributesCount); + } + + if ((fieldMask & LogRecord.fieldModifiedFlags) != 0) { + // Encode Flags + this.flagsEncoder.encode(val.flags); + } + + if ((fieldMask & LogRecord.fieldModifiedTraceID) != 0) { + // Encode TraceID + this.traceIDEncoder.encode(val.traceID); + } + + if ((fieldMask & LogRecord.fieldModifiedSpanID) != 0) { + // Encode SpanID + this.spanIDEncoder.encode(val.spanID); + } + + if ((fieldMask & LogRecord.fieldModifiedEventName) != 0) { + // Encode EventName + this.eventNameEncoder.encode(val.eventName); + } + + // Account written bits in the limiter. + int newLen = this.buf.bitCount(); + this.limiter.addFrameBits(newLen - oldLen); + + // Mark all fields non-modified so that next encode() correctly + // encodes only fields that change after this. + val.modifiedFields.mask = 0; + } + + // collectColumns collects all buffers from all encoders into buf. + public void collectColumns(WriteColumnSet columnSet) { + columnSet.setBits(this.buf); + int colIdx = 0; + + // Collect TimeUnixNano field. + if (this.fieldCount <= 0) { + return; // TimeUnixNano and subsequent fields are skipped. + } + + timeUnixNanoEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect ObservedTimeUnixNano field. + if (this.fieldCount <= 1) { + return; // ObservedTimeUnixNano and subsequent fields are skipped. + } + + observedTimeUnixNanoEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect SeverityNumber field. + if (this.fieldCount <= 2) { + return; // SeverityNumber and subsequent fields are skipped. + } + + severityNumberEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect SeverityText field. + if (this.fieldCount <= 3) { + return; // SeverityText and subsequent fields are skipped. + } + + severityTextEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect Body field. + if (this.fieldCount <= 4) { + return; // Body and subsequent fields are skipped. + } + if (!isBodyRecursive) { + bodyEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + } + + // Collect Attributes field. + if (this.fieldCount <= 5) { + return; // Attributes and subsequent fields are skipped. + } + if (!isAttributesRecursive) { + attributesEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + } + + // Collect DroppedAttributesCount field. + if (this.fieldCount <= 6) { + return; // DroppedAttributesCount and subsequent fields are skipped. + } + + droppedAttributesCountEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect Flags field. + if (this.fieldCount <= 7) { + return; // Flags and subsequent fields are skipped. + } + + flagsEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect TraceID field. + if (this.fieldCount <= 8) { + return; // TraceID and subsequent fields are skipped. + } + + traceIDEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect SpanID field. + if (this.fieldCount <= 9) { + return; // SpanID and subsequent fields are skipped. + } + + spanIDEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + // Collect EventName field. + if (this.fieldCount <= 10) { + return; // EventName and subsequent fields are skipped. + } + + eventNameEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + + } +} diff --git a/java/src/main/java/com/example/otelstef/Logs.java b/java/src/main/java/com/example/otelstef/Logs.java new file mode 100644 index 00000000..db4a141c --- /dev/null +++ b/java/src/main/java/com/example/otelstef/Logs.java @@ -0,0 +1,301 @@ +// Code generated by stefc. DO NOT EDIT. +// Logs Java class generated from template +package com.example.otelstef; + +import net.stef.BytesValue; +import net.stef.StringValue; +import net.stef.Types; +import net.stef.schema.WireSchema; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.*; + +public class Logs { + // Field values. + + Envelope envelope; + Resource resource; + Scope scope; + LogRecord log; + + // modifiedFields keeps track of which fields are modified. + final ModifiedFields modifiedFields = new ModifiedFields(); + + public static final String StructName = "Logs"; + + // Bitmasks for "modified" flags for each field. + + public static final long fieldModifiedEnvelope = 1 << 0; + public static final long fieldModifiedResource = 1 << 1; + public static final long fieldModifiedScope = 1 << 2; + public static final long fieldModifiedLog = 1 << 3; + + + + public Logs() { + init(null, 0); + } + + Logs(ModifiedFields parentModifiedFields, long parentModifiedBit) { + init(parentModifiedFields, parentModifiedBit); + } + + private void init(ModifiedFields parentModifiedFields, long parentModifiedBit) { + modifiedFields.parent = parentModifiedFields; + modifiedFields.parentBit = parentModifiedBit; + + envelope = new Envelope(modifiedFields, fieldModifiedEnvelope); + resource = new Resource(modifiedFields, fieldModifiedResource); + scope = new Scope(modifiedFields, fieldModifiedScope); + log = new LogRecord(modifiedFields, fieldModifiedLog); + } + + void reset() { + + if (envelope != null) { + envelope.reset(); + } + resource = new Resource(modifiedFields, fieldModifiedResource); + scope = new Scope(modifiedFields, fieldModifiedScope); + if (log != null) { + log.reset(); + } + } + + + public Envelope getEnvelope() { + return this.envelope; + } + + private void markEnvelopeModified() { + this.modifiedFields.markModified(fieldModifiedEnvelope); + } + + // isEnvelopeModified returns true if the value of Envelope field was modified since + // Logs was created, encoded or decoded. If the field is modified + // it will be encoded by the next Write() operation. If the field is decoded by the + // next Read() operation the modified flag will be set. + public boolean isEnvelopeModified() { + return (this.modifiedFields.mask & fieldModifiedEnvelope) != 0; + } + + public Resource getResource() { + return this.resource; + } + + private void markResourceModified() { + this.modifiedFields.markModified(fieldModifiedResource); + } + + // isResourceModified returns true if the value of Resource field was modified since + // Logs was created, encoded or decoded. If the field is modified + // it will be encoded by the next Write() operation. If the field is decoded by the + // next Read() operation the modified flag will be set. + public boolean isResourceModified() { + return (this.modifiedFields.mask & fieldModifiedResource) != 0; + } + + public Scope getScope() { + return this.scope; + } + + private void markScopeModified() { + this.modifiedFields.markModified(fieldModifiedScope); + } + + // isScopeModified returns true if the value of Scope field was modified since + // Logs was created, encoded or decoded. If the field is modified + // it will be encoded by the next Write() operation. If the field is decoded by the + // next Read() operation the modified flag will be set. + public boolean isScopeModified() { + return (this.modifiedFields.mask & fieldModifiedScope) != 0; + } + + public LogRecord getLog() { + return this.log; + } + + private void markLogModified() { + this.modifiedFields.markModified(fieldModifiedLog); + } + + // isLogModified returns true if the value of Log field was modified since + // Logs was created, encoded or decoded. If the field is modified + // it will be encoded by the next Write() operation. If the field is decoded by the + // next Read() operation the modified flag will be set. + public boolean isLogModified() { + return (this.modifiedFields.mask & fieldModifiedLog) != 0; + } + + + void markModifiedRecursively() { + envelope.markModifiedRecursively(); + resource.markModifiedRecursively(); + scope.markModifiedRecursively(); + log.markModifiedRecursively(); + modifiedFields.mask = + fieldModifiedEnvelope | + fieldModifiedResource | + fieldModifiedScope | + fieldModifiedLog | 0; + } + + void markUnmodifiedRecursively() { + if (isEnvelopeModified()) { + envelope.markUnmodifiedRecursively(); + } + if (isResourceModified()) { + resource.markUnmodifiedRecursively(); + } + if (isScopeModified()) { + scope.markUnmodifiedRecursively(); + } + if (isLogModified()) { + log.markUnmodifiedRecursively(); + } + modifiedFields.mask = 0; + } + + public Logs clone() { + Logs cpy = new Logs(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); + return cpy; + } + + // ByteSize returns approximate memory usage in bytes. Used to calculate memory used by dictionaries. + int byteSize() { + int size = 0; // TODO: calculate the size of this object. + size += this.envelope.byteSize(); + size += this.resource.byteSize(); + size += this.scope.byteSize(); + size += this.log.byteSize(); + return size; + } + + // Performs a deep copy from src to dst. + public void copyFrom(Logs src) { + envelope.copyFrom(src.envelope); + resource.copyFrom(src.resource); + scope.copyFrom(src.scope); + log.copyFrom(src.log); + } + + // equals performs deep comparison and returns true if struct is equal to val. + public boolean equals(Logs right) { + // Compare Envelope field. + if (!this.envelope.equals(right.envelope)) { + return false; + } + // Compare Resource field. + if (!this.resource.equals(right.resource)) { + return false; + } + // Compare Scope field. + if (!this.scope.equals(right.scope)) { + return false; + } + // Compare Log field. + if (!this.log.equals(right.log)) { + return false; + } + return true; + } + + public static boolean equals(Logs left, Logs right) { + return left.equals(right); + } + + // compare performs deep comparison and returns an integer that + // will be 0 if left == right, negative if left < right, positive if left > right. + public static int compare(Logs left, Logs right) { + if (left == null) { + if (right == null) { + return 0; + } + return -1; + } + if (right == null) { + return 1; + } + int c; + + // Compare Envelope field. + c = Envelope.compare(left.envelope, right.envelope); + if (c != 0) { + return c; + } + + // Compare Resource field. + c = Resource.compare(left.resource, right.resource); + if (c != 0) { + return c; + } + + // Compare Scope field. + c = Scope.compare(left.scope, right.scope); + if (c != 0) { + return c; + } + + // Compare Log field. + c = LogRecord.compare(left.log, right.log); + if (c != 0) { + return c; + } + + return 0; + } + + // mutateRandom mutates fields in a random, deterministic manner using random as a deterministic generator. + void mutateRandom(Random random, CommonMutateRandomLimiter limiter) { + final int fieldCount = Math.max(4,2); // At least 2 to ensure we don't recurse infinitely if there is only 1 field. + + if (random.nextInt(fieldCount) == 0) { + this.envelope.mutateRandom(random, limiter); + } + + if (random.nextInt(fieldCount) == 0) { + this.resource.mutateRandom(random, limiter); + } + + if (random.nextInt(fieldCount) == 0) { + this.scope.mutateRandom(random, limiter); + } + + if (random.nextInt(fieldCount) == 0) { + this.log.mutateRandom(random, limiter); + } + + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) { + return false; + } + return equals((Logs)o); + } + + @Override + public int hashCode() { + return Objects.hash( + envelope, + resource, + scope, + log + ); + } + + + static final byte[] wireSchemaBytes = new byte[]{ 0x06, 0x04, 0x01, 0x03, 0x07, 0x05, 0x0B }; + + static WireSchema wireSchema() throws IOException { + WireSchema w = new WireSchema(); + w.deserialize(new ByteArrayInputStream(wireSchemaBytes)); + return w; + } + +} diff --git a/java/src/main/java/com/example/otelstef/LogsDecoder.java b/java/src/main/java/com/example/otelstef/LogsDecoder.java new file mode 100644 index 00000000..46b07277 --- /dev/null +++ b/java/src/main/java/com/example/otelstef/LogsDecoder.java @@ -0,0 +1,207 @@ +// Code generated by stefc. DO NOT EDIT. +// LogsDecoder implements decoding of Logs +package com.example.otelstef; + +import net.stef.BitsReader; +import net.stef.Helper; +import net.stef.ReadColumnSet; +import net.stef.ReadableColumn; +import net.stef.codecs.*; + +import java.io.IOException; + +class LogsDecoder { + private final BitsReader buf = new BitsReader(); + private ReadableColumn column; + private int fieldCount; + + + + private EnvelopeDecoder envelopeDecoder; + private boolean isEnvelopeRecursive = false; // Indicates Envelope field's type is recursive. + private ResourceDecoder resourceDecoder; + private boolean isResourceRecursive = false; // Indicates Resource field's type is recursive. + private ScopeDecoder scopeDecoder; + private boolean isScopeRecursive = false; // Indicates Scope field's type is recursive. + private LogRecordDecoder logDecoder; + private boolean isLogRecursive = false; // Indicates Log field's type is recursive. + + + // Init is called once in the lifetime of the stream. + public void init(ReaderState state, ReadColumnSet columns) throws IOException { + // Remember this encoder in the state so that we can detect recursion. + if (state.LogsDecoder != null) { + throw new IllegalStateException("cannot initialize LogsDecoder: already initialized"); + } + state.LogsDecoder = this; + + try { + fieldCount = state.getStructFieldCounts().getLogsFieldCount(); + column = columns.getColumn(); + + if (this.fieldCount <= 0) { + return; // Envelope and subsequent fields are skipped. + } + if (state.EnvelopeDecoder != null) { + // Recursion detected, use the existing decoder. + envelopeDecoder = state.EnvelopeDecoder; + isEnvelopeRecursive = true; // Mark that we are using a recursive decoder. + } else { + envelopeDecoder = new EnvelopeDecoder(); + envelopeDecoder.init(state, columns.addSubColumn()); + } + if (this.fieldCount <= 1) { + return; // Resource and subsequent fields are skipped. + } + if (state.ResourceDecoder != null) { + // Recursion detected, use the existing decoder. + resourceDecoder = state.ResourceDecoder; + isResourceRecursive = true; // Mark that we are using a recursive decoder. + } else { + resourceDecoder = new ResourceDecoder(); + resourceDecoder.init(state, columns.addSubColumn()); + } + if (this.fieldCount <= 2) { + return; // Scope and subsequent fields are skipped. + } + if (state.ScopeDecoder != null) { + // Recursion detected, use the existing decoder. + scopeDecoder = state.ScopeDecoder; + isScopeRecursive = true; // Mark that we are using a recursive decoder. + } else { + scopeDecoder = new ScopeDecoder(); + scopeDecoder.init(state, columns.addSubColumn()); + } + if (this.fieldCount <= 3) { + return; // Log and subsequent fields are skipped. + } + if (state.LogRecordDecoder != null) { + // Recursion detected, use the existing decoder. + logDecoder = state.LogRecordDecoder; + isLogRecursive = true; // Mark that we are using a recursive decoder. + } else { + logDecoder = new LogRecordDecoder(); + logDecoder.init(state, columns.addSubColumn()); + } + } finally { + state.LogsDecoder = null; + } + } + + // continueDecoding is called at the start of the frame to continue decoding column data. + // This should set the decoder's source buffer, so the new decoding continues from + // the supplied column data. This should NOT reset the internal state of the decoder, + // since columns can cross frame boundaries and the new column data is considered + // continuation of that same column in the previous frame. + public void continueDecoding() { + this.buf.reset(this.column.getData()); + + if (this.fieldCount <= 0) { + return; // Envelope and subsequent fields are skipped. + } + + if (!isEnvelopeRecursive) { + envelopeDecoder.continueDecoding(); + } + + if (this.fieldCount <= 1) { + return; // Resource and subsequent fields are skipped. + } + + if (!isResourceRecursive) { + resourceDecoder.continueDecoding(); + } + + if (this.fieldCount <= 2) { + return; // Scope and subsequent fields are skipped. + } + + if (!isScopeRecursive) { + scopeDecoder.continueDecoding(); + } + + if (this.fieldCount <= 3) { + return; // Log and subsequent fields are skipped. + } + + if (!isLogRecursive) { + logDecoder.continueDecoding(); + } + + } + + public void reset() { + + if (fieldCount <= 0) { + // Envelope and all subsequent fields are skipped. + return; + } + if (!isEnvelopeRecursive) { + envelopeDecoder.reset(); + } + if (fieldCount <= 1) { + // Resource and all subsequent fields are skipped. + return; + } + if (!isResourceRecursive) { + resourceDecoder.reset(); + } + if (fieldCount <= 2) { + // Scope and all subsequent fields are skipped. + return; + } + if (!isScopeRecursive) { + scopeDecoder.reset(); + } + if (fieldCount <= 3) { + // Log and all subsequent fields are skipped. + return; + } + if (!isLogRecursive) { + logDecoder.reset(); + } + } + + public Logs decode(Logs dstPtr) throws IOException { + Logs val = dstPtr; + // Read bits that indicate which fields follow. + val.modifiedFields.mask = buf.readBits(fieldCount); + + + if ((val.modifiedFields.mask & Logs.fieldModifiedEnvelope) != 0) { + // Field is changed. + if (val.envelope == null) { + val.envelope = new Envelope(val.modifiedFields, Logs.fieldModifiedEnvelope); + } + val.envelope = envelopeDecoder.decode(val.envelope); + } + + if ((val.modifiedFields.mask & Logs.fieldModifiedResource) != 0) { + // Field is changed. + if (val.resource == null) { + val.resource = new Resource(val.modifiedFields, Logs.fieldModifiedResource); + } + val.resource = resourceDecoder.decode(val.resource); + } + + if ((val.modifiedFields.mask & Logs.fieldModifiedScope) != 0) { + // Field is changed. + if (val.scope == null) { + val.scope = new Scope(val.modifiedFields, Logs.fieldModifiedScope); + } + val.scope = scopeDecoder.decode(val.scope); + } + + if ((val.modifiedFields.mask & Logs.fieldModifiedLog) != 0) { + // Field is changed. + if (val.log == null) { + val.log = new LogRecord(val.modifiedFields, Logs.fieldModifiedLog); + } + val.log = logDecoder.decode(val.log); + } + + + return val; + } +} + diff --git a/java/src/main/java/com/example/otelstef/LogsEncoder.java b/java/src/main/java/com/example/otelstef/LogsEncoder.java new file mode 100644 index 00000000..d54a6af4 --- /dev/null +++ b/java/src/main/java/com/example/otelstef/LogsEncoder.java @@ -0,0 +1,247 @@ +// Code generated by stefc. DO NOT EDIT. +// LogsEncoder implements encoding of Logs +package com.example.otelstef; + +import net.stef.BitsWriter; +import net.stef.Helper; +import net.stef.SizeLimiter; +import net.stef.WriteColumnSet; +import net.stef.codecs.*; + +import java.io.IOException; + +class LogsEncoder { + private BitsWriter buf = new BitsWriter(); + private SizeLimiter limiter; + + // forceModifiedFields is set to true if the next encoding operation + // must write all fields, whether they are modified or not. + // This is used after frame restarts so that the data can be decoded + // from the frame start. + private boolean forceModifiedFields; + + + private EnvelopeEncoder envelopeEncoder; + private boolean isEnvelopeRecursive = false; // Indicates Envelope field's type is recursive. + private ResourceEncoder resourceEncoder; + private boolean isResourceRecursive = false; // Indicates Resource field's type is recursive. + private ScopeEncoder scopeEncoder; + private boolean isScopeRecursive = false; // Indicates Scope field's type is recursive. + private LogRecordEncoder logEncoder; + private boolean isLogRecursive = false; // Indicates Log field's type is recursive. + + + private long keepFieldMask; + private int fieldCount; + + + public void init(WriterState state, WriteColumnSet columns) throws IOException { + // Remember this encoder in the state so that we can detect recursion. + if (state.LogsEncoder != null) { + throw new IllegalStateException("cannot initialize LogsEncoder: already initialized"); + } + state.LogsEncoder = this; + + try { + limiter = state.getLimiter(); + + fieldCount = state.getStructFieldCounts().getLogsFieldCount(); + keepFieldMask = ~((~0L) << fieldCount); + // Init encoder for Envelope field. + if (fieldCount <= 0) { + return; // Envelope and subsequent fields are skipped. + } + if (state.EnvelopeEncoder != null) { + // Recursion detected, use the existing encoder. + envelopeEncoder = state.EnvelopeEncoder; + isEnvelopeRecursive = true; + } else { + envelopeEncoder = new EnvelopeEncoder(); + envelopeEncoder.init(state, columns.addSubColumn()); + } + // Init encoder for Resource field. + if (fieldCount <= 1) { + return; // Resource and subsequent fields are skipped. + } + if (state.ResourceEncoder != null) { + // Recursion detected, use the existing encoder. + resourceEncoder = state.ResourceEncoder; + isResourceRecursive = true; + } else { + resourceEncoder = new ResourceEncoder(); + resourceEncoder.init(state, columns.addSubColumn()); + } + // Init encoder for Scope field. + if (fieldCount <= 2) { + return; // Scope and subsequent fields are skipped. + } + if (state.ScopeEncoder != null) { + // Recursion detected, use the existing encoder. + scopeEncoder = state.ScopeEncoder; + isScopeRecursive = true; + } else { + scopeEncoder = new ScopeEncoder(); + scopeEncoder.init(state, columns.addSubColumn()); + } + // Init encoder for Log field. + if (fieldCount <= 3) { + return; // Log and subsequent fields are skipped. + } + if (state.LogRecordEncoder != null) { + // Recursion detected, use the existing encoder. + logEncoder = state.LogRecordEncoder; + isLogRecursive = true; + } else { + logEncoder = new LogRecordEncoder(); + logEncoder.init(state, columns.addSubColumn()); + } + } finally { + state.LogsEncoder = null; + } + } + + public void reset() { + // Since we are resetting the state of encoder make sure the next encode() + // call forcefully writes all fields and does not attempt to skip. + this.forceModifiedFields = true; + + if (fieldCount <= 0) { + return; // Envelope and all subsequent fields are skipped. + } + + if (!isEnvelopeRecursive) { + envelopeEncoder.reset(); + } + + if (fieldCount <= 1) { + return; // Resource and all subsequent fields are skipped. + } + + if (!isResourceRecursive) { + resourceEncoder.reset(); + } + + if (fieldCount <= 2) { + return; // Scope and all subsequent fields are skipped. + } + + if (!isScopeRecursive) { + scopeEncoder.reset(); + } + + if (fieldCount <= 3) { + return; // Log and all subsequent fields are skipped. + } + + if (!isLogRecursive) { + logEncoder.reset(); + } + + } + + // encode encodes val into buf + public void encode(Logs val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Logs val, boolean forceAllFields) throws IOException { + int oldLen = this.buf.bitCount(); + + + + // Mask that describes what fields are encoded. Start with all modified fields. + long fieldMask = val.modifiedFields.mask; + // On an encoder restart or when requested by a containing collection, + // force writing of all fields. + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { + fieldMask = + Logs.fieldModifiedEnvelope | + Logs.fieldModifiedResource | + Logs.fieldModifiedScope | + Logs.fieldModifiedLog | 0L; + this.forceModifiedFields = false; + } + + // Only write fields that we want to write. See init() for keepFieldMask. + fieldMask &= this.keepFieldMask; + + // Write bits to indicate which fields follow. + this.buf.writeBits(fieldMask, this.fieldCount); + + // Encode modified, present fields. + + if ((fieldMask & Logs.fieldModifiedEnvelope) != 0) { + // Encode Envelope + this.envelopeEncoder.encode(val.envelope, forceFields); + } + + if ((fieldMask & Logs.fieldModifiedResource) != 0) { + // Encode Resource + this.resourceEncoder.encode(val.resource, forceFields); + } + + if ((fieldMask & Logs.fieldModifiedScope) != 0) { + // Encode Scope + this.scopeEncoder.encode(val.scope, forceFields); + } + + if ((fieldMask & Logs.fieldModifiedLog) != 0) { + // Encode Log + this.logEncoder.encode(val.log, forceFields); + } + + // Account written bits in the limiter. + int newLen = this.buf.bitCount(); + this.limiter.addFrameBits(newLen - oldLen); + + // Mark all fields non-modified so that next encode() correctly + // encodes only fields that change after this. + val.modifiedFields.mask = 0; + } + + // collectColumns collects all buffers from all encoders into buf. + public void collectColumns(WriteColumnSet columnSet) { + columnSet.setBits(this.buf); + int colIdx = 0; + + // Collect Envelope field. + if (this.fieldCount <= 0) { + return; // Envelope and subsequent fields are skipped. + } + if (!isEnvelopeRecursive) { + envelopeEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + } + + // Collect Resource field. + if (this.fieldCount <= 1) { + return; // Resource and subsequent fields are skipped. + } + if (!isResourceRecursive) { + resourceEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + } + + // Collect Scope field. + if (this.fieldCount <= 2) { + return; // Scope and subsequent fields are skipped. + } + if (!isScopeRecursive) { + scopeEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + } + + // Collect Log field. + if (this.fieldCount <= 3) { + return; // Log and subsequent fields are skipped. + } + if (!isLogRecursive) { + logEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; + } + + } +} diff --git a/java/src/main/java/com/example/otelstef/LogsReader.java b/java/src/main/java/com/example/otelstef/LogsReader.java new file mode 100644 index 00000000..0485120e --- /dev/null +++ b/java/src/main/java/com/example/otelstef/LogsReader.java @@ -0,0 +1,108 @@ +// Code generated by stefc. DO NOT EDIT. +package com.example.otelstef; + +import net.stef.BaseReader; +import net.stef.FixedHeader; +import net.stef.FrameFlags; +import net.stef.ReadOptions; +import net.stef.ReadResult; +import net.stef.schema.WireSchema; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +// LogsReader allows reading records of Logs +// in STEF format from an input stream. +public class LogsReader { + private final BaseReader base; + private final LogsDecoder decoder = new LogsDecoder(); + private final ReaderState state = new ReaderState(); + + // Record contains the record that was just read by the last read() operation. + // The next read() will overwrite the record. + // Do not modify this field externally. Modifying this field will result in + // incorrect decoding of subsequent records. + public final Logs record = new Logs(); + + public LogsReader(InputStream source) throws IOException { + base = new BaseReader(new BufferedInputStream(source, 64 * 1024)); + initSchema(); + } + + private void initSchema() throws IOException { + WireSchema ownSchema = Logs.wireSchema(); + base.readVarHeader(ownSchema); + state.init(base.getSchema()); + decoder.init(state, base.readBufs.columns); + + if (!state.getStructFieldCounts().allFetched()) { + throw new IOException("override schema iterator is not done, decoded data schema is likely incompatible"); + } + } + + public FixedHeader getHeader() { + return base.fixedHeader; + } + + public Map getUserData() { + return base.varHeader.getUserData(); + } + + public long getRecordCount() { + return base.recordCount; + } + + private void nextFrame() throws IOException { + int frameFlags = base.nextFrame(); + if ((frameFlags & FrameFlags.RestartDictionaries) != 0) { + // The frame that has just started indicates that the dictionaries + // must be restarted. Reset all dictionaries. + state.resetDicts(); + } + if ((frameFlags & FrameFlags.RestartCodecs) != 0) { + // The frame that has just started indicates that the decoders + // must be restarted. + decoder.reset(); + } + decoder.continueDecoding(); + } + + // Read the next record. After read() returns successfully the record + // will be accessible in LogsReader.record field. + // + // If ReadOptions.tillEndOfFrame=true and end of the current + // frame is reached read() will return EndOfFrame. Calling read() + // after that with ReadOptions.tillEndOfFrame=false will correctly load + // the next frame (if any) and read the next record from the frame. + // The very first read() must use ReadOptions.tillEndOfFrame=false + // otherwise EndOfFrameException will be thrown since no frames are + // initially loaded in the Reader. See more details in ReadOptions. + // + // If read() encounters a decoding error a IOException will be thrown. + // If underlying source InputStream throws any error then read() will + // either throw that error or a decoding error if source InputStream + // returned an error prematurely while more data was expected in STEF stream. + // + // For well-formed streams that don't encounter decoding errors read() will + // throw EOFException once end of the underlying source InputStream is reached + // (assuming InputStream throws EOFException itself). + public ReadResult read(int opts) throws IOException { + while (base.frameRecordCount == 0) { + if ((opts&ReadOptions.tillEndOfFrame)!=0) { + return ReadResult.EndOfFrame; + } + nextFrame(); + } + base.frameRecordCount--; + base.recordCount++; + decoder.decode(record); + return ReadResult.Success; + } + + // Returns the schema of the STEF stream being read. + public WireSchema getSchema() { + return base.getSchema(); + } +} diff --git a/java/src/main/java/com/example/otelstef/LogsWriter.java b/java/src/main/java/com/example/otelstef/LogsWriter.java new file mode 100644 index 00000000..e94ff88c --- /dev/null +++ b/java/src/main/java/com/example/otelstef/LogsWriter.java @@ -0,0 +1,142 @@ +// Code generated by stefc. DO NOT EDIT. +package com.example.otelstef; + +import net.stef.Constants; +import net.stef.ChunkWriter; +import net.stef.FrameEncoder; +import net.stef.FrameFlags; +import net.stef.Serde; +import net.stef.VarHeader; +import net.stef.WriteBufs; +import net.stef.WriterOptions; +import net.stef.schema.Compatibility; +import net.stef.schema.WireSchema; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +// LogsWriter allows writing records of Logs +// in STEF format to an output stream. +public class LogsWriter { + // The next record to be written. Modify this record and call write() + // to write a record. + public final Logs record; + + private WriterOptions opts; + private ChunkWriter dst; + private FrameEncoder frameEncoder; + private LogsEncoder encoder; + private WriterState state; + private WriteBufs writeBufs; + private long frameRecordCount; + private long recordCount; + + public LogsWriter(ChunkWriter dst, WriterOptions opts) throws IOException { + this.dst = dst; + this.opts = opts; + this.record = new Logs(); + this.state = new WriterState(); + this.encoder = new LogsEncoder(); + this.writeBufs = new WriteBufs(); + this.frameEncoder = new FrameEncoder(); + + if (this.opts.getSchema() != null) { + WireSchema ownSchema = Logs.wireSchema(); + if (ownSchema.compatible(this.opts.getSchema()) == Compatibility.Incompatible) { + throw new IOException("schema is not compatible with Writer"); + } + } + + state.init(opts); + encoder.init(state, writeBufs.columns); + + if (!state.getStructFieldCounts().allFetched()) { + throw new IOException("override schema iterator is not done, likely supplied schema override does not match the generated code"); + } + + frameEncoder.init(dst, opts.getCompression()); + writeFixedHeader(); + writeVarHeader(); + } + + private void writeFixedHeader() throws IOException { + ByteArrayOutputStream hdrTail = new ByteArrayOutputStream(); + byte versionAndType = Constants.HdrFormatVersion; + hdrTail.write(versionAndType); + byte flags = (byte) (this.opts.getCompression().getValue() & Constants.HdrFlagsCompressionMethod); + hdrTail.write(flags); + long hdrTailSize = hdrTail.size(); + + ByteArrayOutputStream hdrFull = new ByteArrayOutputStream(); + hdrFull.write(Constants.HdrSignature); + Serde.writeUvarint(hdrTailSize, hdrFull); + hdrFull.write(hdrTail.toByteArray()); + + this.dst.writeChunk(hdrFull.toByteArray(), null); + } + + private void writeVarHeader() throws IOException { + VarHeader hdr = new VarHeader(); + if (opts.isIncludeDescriptor()) { + if (opts.getSchema() != null) { + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + opts.getSchema().serialize(buf); + hdr.setSchemaWireBytes(buf.toByteArray()); + } else { + hdr.setSchemaWireBytes(Logs.wireSchemaBytes); + } + } + if (opts.getUserData() != null && !opts.getUserData().isEmpty()) { + hdr.setUserData(opts.getUserData()); + } + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + hdr.serialize(buf); + frameEncoder.write(buf.toByteArray()); + frameEncoder.closeFrame(); + frameEncoder.openFrame(0); + } + + public void write() throws IOException { + encoder.encode(this.record); + frameRecordCount++; + int nextFrameFlags = this.opts.getFrameRestartFlags(); + boolean restartFrame = false; + if (state.getLimiter().isDictLimitReached() || (nextFrameFlags & FrameFlags.RestartDictionaries) != 0) { + state.resetDicts(); + nextFrameFlags = this.opts.getFrameRestartFlags() | FrameFlags.RestartDictionaries; + restartFrame = true; + } + if (state.getLimiter().isFrameLimitReached()) { + restartFrame = true; + } + if (restartFrame) { + restartFrame(nextFrameFlags); + } + recordCount++; + } + + public long getRecordCount() { + return this.recordCount; + } + + private void restartFrame(int nextFrameFlags) throws IOException { + if ((nextFrameFlags & FrameFlags.RestartCodecs) != 0) { + encoder.reset(); + } + + Serde.writeUvarint(frameRecordCount, frameEncoder); + frameRecordCount = 0; + encoder.collectColumns(writeBufs.columns); + writeBufs.writeTo(frameEncoder); + frameEncoder.closeFrame(); + frameEncoder.openFrame(nextFrameFlags); + state.getLimiter().resetFrameSize(); + } + + public void flush() throws IOException { + if (frameRecordCount == 0) { + return; + } + restartFrame(opts.getFrameRestartFlags()); + } +} diff --git a/java/src/main/java/com/example/otelstef/Metric.java b/java/src/main/java/com/example/otelstef/Metric.java index 374754a4..39e42bd4 100644 --- a/java/src/main/java/com/example/otelstef/Metric.java +++ b/java/src/main/java/com/example/otelstef/Metric.java @@ -283,14 +283,8 @@ void markUnmodifiedRecursively() { public Metric clone() { Metric cpy = new Metric(); - cpy.name = this.name; - cpy.description = this.description; - cpy.unit = this.unit; - cpy.type_ = this.type_; - cpy.metadata = this.metadata.clone(); - cpy.histogramBounds = this.histogramBounds.clone(); - cpy.aggregationTemporality = this.aggregationTemporality; - cpy.monotonic = this.monotonic; + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/MetricEncoder.java b/java/src/main/java/com/example/otelstef/MetricEncoder.java index 2290a3e6..bb0f3e0a 100644 --- a/java/src/main/java/com/example/otelstef/MetricEncoder.java +++ b/java/src/main/java/com/example/otelstef/MetricEncoder.java @@ -172,6 +172,12 @@ public void reset() { // encode encodes val into buf public void encode(Metric val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Metric val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); @@ -203,9 +209,10 @@ public void encode(Metric val) throws IOException { // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = Metric.fieldModifiedName | Metric.fieldModifiedDescription | @@ -215,6 +222,7 @@ public void encode(Metric val) throws IOException { Metric.fieldModifiedHistogramBounds | Metric.fieldModifiedAggregationTemporality | Metric.fieldModifiedMonotonic | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -247,12 +255,12 @@ public void encode(Metric val) throws IOException { if ((fieldMask & Metric.fieldModifiedMetadata) != 0) { // Encode Metadata - this.metadataEncoder.encode(val.metadata); + this.metadataEncoder.encode(val.metadata, forceFields); } if ((fieldMask & Metric.fieldModifiedHistogramBounds) != 0) { // Encode HistogramBounds - this.histogramBoundsEncoder.encode(val.histogramBounds); + this.histogramBoundsEncoder.encode(val.histogramBounds, forceFields); } if ((fieldMask & Metric.fieldModifiedAggregationTemporality) != 0) { @@ -347,4 +355,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/Metrics.java b/java/src/main/java/com/example/otelstef/Metrics.java index 34166579..27d96b7e 100644 --- a/java/src/main/java/com/example/otelstef/Metrics.java +++ b/java/src/main/java/com/example/otelstef/Metrics.java @@ -211,12 +211,8 @@ void markUnmodifiedRecursively() { public Metrics clone() { Metrics cpy = new Metrics(); - cpy.envelope = this.envelope.clone(); - cpy.metric = this.metric.clone(); - cpy.resource = this.resource.clone(); - cpy.scope = this.scope.clone(); - cpy.attributes = this.attributes.clone(); - cpy.point = this.point.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/MetricsEncoder.java b/java/src/main/java/com/example/otelstef/MetricsEncoder.java index 42fdad2e..4c25d289 100644 --- a/java/src/main/java/com/example/otelstef/MetricsEncoder.java +++ b/java/src/main/java/com/example/otelstef/MetricsEncoder.java @@ -185,15 +185,22 @@ public void reset() { // encode encodes val into buf public void encode(Metrics val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Metrics val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = Metrics.fieldModifiedEnvelope | Metrics.fieldModifiedMetric | @@ -201,6 +208,7 @@ public void encode(Metrics val) throws IOException { Metrics.fieldModifiedScope | Metrics.fieldModifiedAttributes | Metrics.fieldModifiedPoint | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -213,32 +221,32 @@ public void encode(Metrics val) throws IOException { if ((fieldMask & Metrics.fieldModifiedEnvelope) != 0) { // Encode Envelope - this.envelopeEncoder.encode(val.envelope); + this.envelopeEncoder.encode(val.envelope, forceFields); } if ((fieldMask & Metrics.fieldModifiedMetric) != 0) { // Encode Metric - this.metricEncoder.encode(val.metric); + this.metricEncoder.encode(val.metric, forceFields); } if ((fieldMask & Metrics.fieldModifiedResource) != 0) { // Encode Resource - this.resourceEncoder.encode(val.resource); + this.resourceEncoder.encode(val.resource, forceFields); } if ((fieldMask & Metrics.fieldModifiedScope) != 0) { // Encode Scope - this.scopeEncoder.encode(val.scope); + this.scopeEncoder.encode(val.scope, forceFields); } if ((fieldMask & Metrics.fieldModifiedAttributes) != 0) { // Encode Attributes - this.attributesEncoder.encode(val.attributes); + this.attributesEncoder.encode(val.attributes, forceFields); } if ((fieldMask & Metrics.fieldModifiedPoint) != 0) { // Encode Point - this.pointEncoder.encode(val.point); + this.pointEncoder.encode(val.point, forceFields); } // Account written bits in the limiter. @@ -311,4 +319,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/Point.java b/java/src/main/java/com/example/otelstef/Point.java index d39d3a94..2b6212d7 100644 --- a/java/src/main/java/com/example/otelstef/Point.java +++ b/java/src/main/java/com/example/otelstef/Point.java @@ -167,10 +167,8 @@ void markUnmodifiedRecursively() { public Point clone() { Point cpy = new Point(); - cpy.startTimestamp = this.startTimestamp; - cpy.timestamp = this.timestamp; - cpy.value = this.value.clone(); - cpy.exemplars = this.exemplars.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/PointEncoder.java b/java/src/main/java/com/example/otelstef/PointEncoder.java index cbb33af2..fd55172b 100644 --- a/java/src/main/java/com/example/otelstef/PointEncoder.java +++ b/java/src/main/java/com/example/otelstef/PointEncoder.java @@ -121,20 +121,28 @@ public void reset() { // encode encodes val into buf public void encode(Point val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Point val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = Point.fieldModifiedStartTimestamp | Point.fieldModifiedTimestamp | Point.fieldModifiedValue | Point.fieldModifiedExemplars | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -157,12 +165,12 @@ public void encode(Point val) throws IOException { if ((fieldMask & Point.fieldModifiedValue) != 0) { // Encode Value - this.valueEncoder.encode(val.value); + this.valueEncoder.encode(val.value, forceFields); } if ((fieldMask & Point.fieldModifiedExemplars) != 0) { // Encode Exemplars - this.exemplarsEncoder.encode(val.exemplars); + this.exemplarsEncoder.encode(val.exemplars, forceFields); } // Account written bits in the limiter. @@ -215,4 +223,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/PointValue.java b/java/src/main/java/com/example/otelstef/PointValue.java index 3a60e7f7..a4c9ee93 100644 --- a/java/src/main/java/com/example/otelstef/PointValue.java +++ b/java/src/main/java/com/example/otelstef/PointValue.java @@ -181,12 +181,8 @@ public SummaryValue getSummary() { // Clone returns a deep copy of this oneof. public PointValue clone() { PointValue cpy = new PointValue(); - cpy.typ = this.typ; - cpy.int64 = this.int64; - cpy.float64 = this.float64; - cpy.histogram = this.histogram.clone(); - cpy.expHistogram = this.expHistogram.clone(); - cpy.summary = this.summary.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } @@ -408,13 +404,19 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash( - int64, - float64, - histogram, - expHistogram, - summary, - typ - ); + switch (this.typ) { + case TypeInt64: + return Objects.hash(typ, int64); + case TypeFloat64: + return Objects.hash(typ, float64); + case TypeHistogram: + return Objects.hash(typ, histogram); + case TypeExpHistogram: + return Objects.hash(typ, expHistogram); + case TypeSummary: + return Objects.hash(typ, summary); + default: + return Objects.hash(typ); + } } } diff --git a/java/src/main/java/com/example/otelstef/PointValueEncoder.java b/java/src/main/java/com/example/otelstef/PointValueEncoder.java index 508abd9b..3ab36a76 100644 --- a/java/src/main/java/com/example/otelstef/PointValueEncoder.java +++ b/java/src/main/java/com/example/otelstef/PointValueEncoder.java @@ -135,6 +135,12 @@ public void reset() { // Encode encodes val into buf public void encode(PointValue val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf and propagates forced full encoding through + // the selected non-primitive value. + public void encode(PointValue val, boolean forceAllFields) throws IOException { PointValue.Type typ = val.typ; if (typ.getValue() > fieldCount) { // The current field type is not supported in target schema. Encode the type as None. @@ -159,15 +165,15 @@ public void encode(PointValue val) throws IOException { break; case TypeHistogram: // Encode Histogram - histogramEncoder.encode(val.histogram); + histogramEncoder.encode(val.histogram, forceAllFields); break; case TypeExpHistogram: // Encode ExpHistogram - expHistogramEncoder.encode(val.expHistogram); + expHistogramEncoder.encode(val.expHistogram, forceAllFields); break; case TypeSummary: // Encode Summary - summaryEncoder.encode(val.summary); + summaryEncoder.encode(val.summary, forceAllFields); break; } } diff --git a/java/src/main/java/com/example/otelstef/QuantileValue.java b/java/src/main/java/com/example/otelstef/QuantileValue.java index 14d17a63..bd574777 100644 --- a/java/src/main/java/com/example/otelstef/QuantileValue.java +++ b/java/src/main/java/com/example/otelstef/QuantileValue.java @@ -113,8 +113,8 @@ void markUnmodifiedRecursively() { public QuantileValue clone() { QuantileValue cpy = new QuantileValue(); - cpy.quantile = this.quantile; - cpy.value = this.value; + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/QuantileValueArrayEncoder.java b/java/src/main/java/com/example/otelstef/QuantileValueArrayEncoder.java index 18e952e1..a6533f23 100644 --- a/java/src/main/java/com/example/otelstef/QuantileValueArrayEncoder.java +++ b/java/src/main/java/com/example/otelstef/QuantileValueArrayEncoder.java @@ -17,6 +17,8 @@ class QuantileValueArrayEncoder { private QuantileValueEncoder elemEncoder; private WriterState state; private boolean isRecursive = false; + // Set after reset so every element in the first non-empty array is encoded in full. + private boolean forceElements; public void init(WriterState state, WriteColumnSet columns) throws IOException { this.state = state; @@ -43,20 +45,28 @@ public void init(WriterState state, WriteColumnSet columns) throws IOException { } public void reset() { + forceElements = true; if (!isRecursive) { elemEncoder.reset(); } } public void encode(QuantileValueArray arr) throws IOException { + encode(arr, false); + } + + public void encode(QuantileValueArray arr, boolean forceAllElements) throws IOException { int oldBitLen = buf.bitCount(); // Write the length of the array. int newLen = arr.elemsLen; buf.writeUvarintCompact(newLen); - + boolean forceElementsNow = forceAllElements || forceElements; for (int i = 0; i < newLen; i++) { - elemEncoder.encode(arr.elems[i]); + elemEncoder.encode(arr.elems[i], forceElementsNow); + } + if (newLen != 0) { + forceElements = false; } // Account written bits in the limiter. @@ -71,4 +81,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } } - diff --git a/java/src/main/java/com/example/otelstef/QuantileValueEncoder.java b/java/src/main/java/com/example/otelstef/QuantileValueEncoder.java index a335ea2c..72c394b2 100644 --- a/java/src/main/java/com/example/otelstef/QuantileValueEncoder.java +++ b/java/src/main/java/com/example/otelstef/QuantileValueEncoder.java @@ -77,18 +77,26 @@ public void reset() { // encode encodes val into buf public void encode(QuantileValue val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(QuantileValue val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = QuantileValue.fieldModifiedQuantile | QuantileValue.fieldModifiedValue | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -141,4 +149,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/ReaderState.java b/java/src/main/java/com/example/otelstef/ReaderState.java index efed76c6..09285152 100644 --- a/java/src/main/java/com/example/otelstef/ReaderState.java +++ b/java/src/main/java/com/example/otelstef/ReaderState.java @@ -10,6 +10,8 @@ public class ReaderState { // Dictionaries final StringDictDecoderDict AnyValueString = new StringDictDecoderDict(); final StringDictDecoderDict AttributeKey = new StringDictDecoderDict(); + final StringDictDecoderDict LogEventName = new StringDictDecoderDict(); + final StringDictDecoderDict LogSeverityText = new StringDictDecoderDict(); final MetricDecoderDict Metric = new MetricDecoderDict(); final StringDictDecoderDict MetricDescription = new StringDictDecoderDict(); final StringDictDecoderDict MetricName = new StringDictDecoderDict(); @@ -40,6 +42,8 @@ public class ReaderState { KeyValueListDecoder KeyValueListDecoder; LinkDecoder LinkDecoder; LinkArrayDecoder LinkArrayDecoder; + LogRecordDecoder LogRecordDecoder; + LogsDecoder LogsDecoder; MetricDecoder MetricDecoder; MetricsDecoder MetricsDecoder; PointDecoder PointDecoder; @@ -58,6 +62,8 @@ public void init(WireSchema overrideSchema) { structFieldCounts.init(overrideSchema); this.AnyValueString.init(); this.AttributeKey.init(); + this.LogEventName.init(); + this.LogSeverityText.init(); this.Metric.init(); this.MetricDescription.init(); this.MetricName.init(); @@ -77,6 +83,8 @@ public void init(WireSchema overrideSchema) { public void resetDicts() { this.AnyValueString.reset(); this.AttributeKey.reset(); + this.LogEventName.reset(); + this.LogSeverityText.reset(); this.Metric.reset(); this.MetricDescription.reset(); this.MetricName.reset(); diff --git a/java/src/main/java/com/example/otelstef/Resource.java b/java/src/main/java/com/example/otelstef/Resource.java index 42fee283..90432115 100644 --- a/java/src/main/java/com/example/otelstef/Resource.java +++ b/java/src/main/java/com/example/otelstef/Resource.java @@ -140,9 +140,8 @@ void markUnmodifiedRecursively() { public Resource clone() { Resource cpy = new Resource(); - cpy.schemaURL = this.schemaURL; - cpy.attributes = this.attributes.clone(); - cpy.droppedAttributesCount = this.droppedAttributesCount; + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/ResourceEncoder.java b/java/src/main/java/com/example/otelstef/ResourceEncoder.java index 1dcf18f1..c6c3820a 100644 --- a/java/src/main/java/com/example/otelstef/ResourceEncoder.java +++ b/java/src/main/java/com/example/otelstef/ResourceEncoder.java @@ -102,6 +102,12 @@ public void reset() { // encode encodes val into buf public void encode(Resource val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Resource val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); @@ -133,13 +139,15 @@ public void encode(Resource val) throws IOException { // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = Resource.fieldModifiedSchemaURL | Resource.fieldModifiedAttributes | Resource.fieldModifiedDroppedAttributesCount | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -157,7 +165,7 @@ public void encode(Resource val) throws IOException { if ((fieldMask & Resource.fieldModifiedAttributes) != 0) { // Encode Attributes - this.attributesEncoder.encode(val.attributes); + this.attributesEncoder.encode(val.attributes, forceFields); } if ((fieldMask & Resource.fieldModifiedDroppedAttributesCount) != 0) { @@ -206,4 +214,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/Scope.java b/java/src/main/java/com/example/otelstef/Scope.java index 0d73addf..a9d3d4f1 100644 --- a/java/src/main/java/com/example/otelstef/Scope.java +++ b/java/src/main/java/com/example/otelstef/Scope.java @@ -198,11 +198,8 @@ void markUnmodifiedRecursively() { public Scope clone() { Scope cpy = new Scope(); - cpy.name = this.name; - cpy.version = this.version; - cpy.schemaURL = this.schemaURL; - cpy.attributes = this.attributes.clone(); - cpy.droppedAttributesCount = this.droppedAttributesCount; + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/ScopeEncoder.java b/java/src/main/java/com/example/otelstef/ScopeEncoder.java index 8b596b78..d0b8ceaa 100644 --- a/java/src/main/java/com/example/otelstef/ScopeEncoder.java +++ b/java/src/main/java/com/example/otelstef/ScopeEncoder.java @@ -126,6 +126,12 @@ public void reset() { // encode encodes val into buf public void encode(Scope val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Scope val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); @@ -157,15 +163,17 @@ public void encode(Scope val) throws IOException { // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = Scope.fieldModifiedName | Scope.fieldModifiedVersion | Scope.fieldModifiedSchemaURL | Scope.fieldModifiedAttributes | Scope.fieldModifiedDroppedAttributesCount | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -193,7 +201,7 @@ public void encode(Scope val) throws IOException { if ((fieldMask & Scope.fieldModifiedAttributes) != 0) { // Encode Attributes - this.attributesEncoder.encode(val.attributes); + this.attributesEncoder.encode(val.attributes, forceFields); } if ((fieldMask & Scope.fieldModifiedDroppedAttributesCount) != 0) { @@ -258,4 +266,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/SeverityNumber.java b/java/src/main/java/com/example/otelstef/SeverityNumber.java new file mode 100644 index 00000000..0a2580e4 --- /dev/null +++ b/java/src/main/java/com/example/otelstef/SeverityNumber.java @@ -0,0 +1,33 @@ +// Code generated by stefc. DO NOT EDIT. +package com.example.otelstef; + +public class SeverityNumber { + public static final long Unspecified = 0; + public static final long Trace = 1; + public static final long Trace2 = 2; + public static final long Trace3 = 3; + public static final long Trace4 = 4; + public static final long Debug = 5; + public static final long Debug2 = 6; + public static final long Debug3 = 7; + public static final long Debug4 = 8; + public static final long Info = 9; + public static final long Info2 = 10; + public static final long Info3 = 11; + public static final long Info4 = 12; + public static final long Warn = 13; + public static final long Warn2 = 14; + public static final long Warn3 = 15; + public static final long Warn4 = 16; + public static final long Error = 17; + public static final long Error2 = 18; + public static final long Error3 = 19; + public static final long Error4 = 20; + public static final long Fatal = 21; + public static final long Fatal2 = 22; + public static final long Fatal3 = 23; + public static final long Fatal4 = 24; + + // Private constructor to prevent instantiation + private SeverityNumber() {} +} diff --git a/java/src/main/java/com/example/otelstef/Span.java b/java/src/main/java/com/example/otelstef/Span.java index 85e0169a..3a6471a1 100644 --- a/java/src/main/java/com/example/otelstef/Span.java +++ b/java/src/main/java/com/example/otelstef/Span.java @@ -453,20 +453,8 @@ void markUnmodifiedRecursively() { public Span clone() { Span cpy = new Span(); - cpy.traceID = this.traceID; - cpy.spanID = this.spanID; - cpy.traceState = this.traceState; - cpy.parentSpanID = this.parentSpanID; - cpy.flags = this.flags; - cpy.name = this.name; - cpy.kind = this.kind; - cpy.startTimeUnixNano = this.startTimeUnixNano; - cpy.endTimeUnixNano = this.endTimeUnixNano; - cpy.attributes = this.attributes.clone(); - cpy.droppedAttributesCount = this.droppedAttributesCount; - cpy.events = this.events.clone(); - cpy.links = this.links.clone(); - cpy.status = this.status.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/SpanEncoder.java b/java/src/main/java/com/example/otelstef/SpanEncoder.java index 41ff935f..ab53f123 100644 --- a/java/src/main/java/com/example/otelstef/SpanEncoder.java +++ b/java/src/main/java/com/example/otelstef/SpanEncoder.java @@ -261,15 +261,22 @@ public void reset() { // encode encodes val into buf public void encode(Span val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Span val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = Span.fieldModifiedTraceID | Span.fieldModifiedSpanID | @@ -285,6 +292,7 @@ public void encode(Span val) throws IOException { Span.fieldModifiedEvents | Span.fieldModifiedLinks | Span.fieldModifiedStatus | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -342,7 +350,7 @@ public void encode(Span val) throws IOException { if ((fieldMask & Span.fieldModifiedAttributes) != 0) { // Encode Attributes - this.attributesEncoder.encode(val.attributes); + this.attributesEncoder.encode(val.attributes, forceFields); } if ((fieldMask & Span.fieldModifiedDroppedAttributesCount) != 0) { @@ -352,17 +360,17 @@ public void encode(Span val) throws IOException { if ((fieldMask & Span.fieldModifiedEvents) != 0) { // Encode Events - this.eventsEncoder.encode(val.events); + this.eventsEncoder.encode(val.events, forceFields); } if ((fieldMask & Span.fieldModifiedLinks) != 0) { // Encode Links - this.linksEncoder.encode(val.links); + this.linksEncoder.encode(val.links, forceFields); } if ((fieldMask & Span.fieldModifiedStatus) != 0) { // Encode Status - this.statusEncoder.encode(val.status); + this.statusEncoder.encode(val.status, forceFields); } // Account written bits in the limiter. @@ -497,4 +505,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/SpanStatus.java b/java/src/main/java/com/example/otelstef/SpanStatus.java index a5030b14..d6e0beca 100644 --- a/java/src/main/java/com/example/otelstef/SpanStatus.java +++ b/java/src/main/java/com/example/otelstef/SpanStatus.java @@ -113,8 +113,8 @@ void markUnmodifiedRecursively() { public SpanStatus clone() { SpanStatus cpy = new SpanStatus(); - cpy.message = this.message; - cpy.code = this.code; + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/SpanStatusEncoder.java b/java/src/main/java/com/example/otelstef/SpanStatusEncoder.java index 5a81b236..2bb51056 100644 --- a/java/src/main/java/com/example/otelstef/SpanStatusEncoder.java +++ b/java/src/main/java/com/example/otelstef/SpanStatusEncoder.java @@ -77,18 +77,26 @@ public void reset() { // encode encodes val into buf public void encode(SpanStatus val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(SpanStatus val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = SpanStatus.fieldModifiedMessage | SpanStatus.fieldModifiedCode | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -141,4 +149,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/Spans.java b/java/src/main/java/com/example/otelstef/Spans.java index ca82f1f9..feb98eb9 100644 --- a/java/src/main/java/com/example/otelstef/Spans.java +++ b/java/src/main/java/com/example/otelstef/Spans.java @@ -159,10 +159,8 @@ void markUnmodifiedRecursively() { public Spans clone() { Spans cpy = new Spans(); - cpy.envelope = this.envelope.clone(); - cpy.resource = this.resource.clone(); - cpy.scope = this.scope.clone(); - cpy.span = this.span.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/SpansEncoder.java b/java/src/main/java/com/example/otelstef/SpansEncoder.java index bad31910..3fa3297b 100644 --- a/java/src/main/java/com/example/otelstef/SpansEncoder.java +++ b/java/src/main/java/com/example/otelstef/SpansEncoder.java @@ -141,20 +141,28 @@ public void reset() { // encode encodes val into buf public void encode(Spans val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(Spans val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = Spans.fieldModifiedEnvelope | Spans.fieldModifiedResource | Spans.fieldModifiedScope | Spans.fieldModifiedSpan | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -167,22 +175,22 @@ public void encode(Spans val) throws IOException { if ((fieldMask & Spans.fieldModifiedEnvelope) != 0) { // Encode Envelope - this.envelopeEncoder.encode(val.envelope); + this.envelopeEncoder.encode(val.envelope, forceFields); } if ((fieldMask & Spans.fieldModifiedResource) != 0) { // Encode Resource - this.resourceEncoder.encode(val.resource); + this.resourceEncoder.encode(val.resource, forceFields); } if ((fieldMask & Spans.fieldModifiedScope) != 0) { // Encode Scope - this.scopeEncoder.encode(val.scope); + this.scopeEncoder.encode(val.scope, forceFields); } if ((fieldMask & Spans.fieldModifiedSpan) != 0) { // Encode Span - this.spanEncoder.encode(val.span); + this.spanEncoder.encode(val.span, forceFields); } // Account written bits in the limiter. @@ -237,4 +245,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/SummaryValue.java b/java/src/main/java/com/example/otelstef/SummaryValue.java index 7a618506..d71f2b36 100644 --- a/java/src/main/java/com/example/otelstef/SummaryValue.java +++ b/java/src/main/java/com/example/otelstef/SummaryValue.java @@ -140,9 +140,8 @@ void markUnmodifiedRecursively() { public SummaryValue clone() { SummaryValue cpy = new SummaryValue(); - cpy.count = this.count; - cpy.sum = this.sum; - cpy.quantileValues = this.quantileValues.clone(); + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/java/src/main/java/com/example/otelstef/SummaryValueEncoder.java b/java/src/main/java/com/example/otelstef/SummaryValueEncoder.java index 5e3233db..212f8d19 100644 --- a/java/src/main/java/com/example/otelstef/SummaryValueEncoder.java +++ b/java/src/main/java/com/example/otelstef/SummaryValueEncoder.java @@ -99,19 +99,27 @@ public void reset() { // encode encodes val into buf public void encode(SummaryValue val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode(SummaryValue val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = SummaryValue.fieldModifiedCount | SummaryValue.fieldModifiedSum | SummaryValue.fieldModifiedQuantileValues | 0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -134,7 +142,7 @@ public void encode(SummaryValue val) throws IOException { if ((fieldMask & SummaryValue.fieldModifiedQuantileValues) != 0) { // Encode QuantileValues - this.quantileValuesEncoder.encode(val.quantileValues); + this.quantileValuesEncoder.encode(val.quantileValues, forceFields); } // Account written bits in the limiter. @@ -178,4 +186,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } - diff --git a/java/src/main/java/com/example/otelstef/Uint64ArrayEncoder.java b/java/src/main/java/com/example/otelstef/Uint64ArrayEncoder.java index b859e147..c7cfd943 100644 --- a/java/src/main/java/com/example/otelstef/Uint64ArrayEncoder.java +++ b/java/src/main/java/com/example/otelstef/Uint64ArrayEncoder.java @@ -33,12 +33,15 @@ public void reset() { } public void encode(Uint64Array arr) throws IOException { + encode(arr, false); + } + + public void encode(Uint64Array arr, boolean forceAllElements) throws IOException { int oldBitLen = buf.bitCount(); // Write the length of the array. int newLen = arr.elemsLen; buf.writeUvarintCompact(newLen); - for (int i = 0; i < newLen; i++) { elemEncoder.encode(arr.elems[i]); } @@ -55,4 +58,3 @@ public void collectColumns(WriteColumnSet columnSet) { } } } - diff --git a/java/src/main/java/com/example/otelstef/WriterState.java b/java/src/main/java/com/example/otelstef/WriterState.java index f54fbf2e..01bb20e9 100644 --- a/java/src/main/java/com/example/otelstef/WriterState.java +++ b/java/src/main/java/com/example/otelstef/WriterState.java @@ -14,6 +14,8 @@ public class WriterState { // Dictionaries final StringDictEncoderDict AnyValueString; final StringDictEncoderDict AttributeKey; + final StringDictEncoderDict LogEventName; + final StringDictEncoderDict LogSeverityText; final MetricEncoderDict Metric; final StringDictEncoderDict MetricDescription; final StringDictEncoderDict MetricName; @@ -45,6 +47,8 @@ public class WriterState { KeyValueListEncoder KeyValueListEncoder; LinkEncoder LinkEncoder; LinkArrayEncoder LinkArrayEncoder; + LogRecordEncoder LogRecordEncoder; + LogsEncoder LogsEncoder; MetricEncoder MetricEncoder; MetricsEncoder MetricsEncoder; PointEncoder PointEncoder; @@ -64,6 +68,8 @@ public WriterState() { limiter = new SizeLimiter(); AnyValueString = new StringDictEncoderDict(); AttributeKey = new StringDictEncoderDict(); + LogEventName = new StringDictEncoderDict(); + LogSeverityText = new StringDictEncoderDict(); Metric = new MetricEncoderDict(); MetricDescription = new StringDictEncoderDict(); MetricName = new StringDictEncoderDict(); @@ -85,6 +91,8 @@ public void init(WriterOptions opts) { // Init dictionaries AnyValueString.init(limiter); AttributeKey.init(limiter); + LogEventName.init(limiter); + LogSeverityText.init(limiter); Metric.init(limiter); MetricDescription.init(limiter); MetricName.init(limiter); @@ -103,6 +111,8 @@ public void resetDicts() { limiter.resetDict(); AnyValueString.reset(); AttributeKey.reset(); + LogEventName.reset(); + LogSeverityText.reset(); Metric.reset(); MetricDescription.reset(); MetricName.reset(); diff --git a/java/src/test/java/tests/CloneTest.java b/java/src/test/java/tests/CloneTest.java new file mode 100644 index 00000000..5ed3bbc0 --- /dev/null +++ b/java/src/test/java/tests/CloneTest.java @@ -0,0 +1,124 @@ +package tests; + +import com.example.otelstef.AnyValue; +import com.example.otelstef.Exemplar; +import com.example.otelstef.ExemplarValue; +import com.example.otelstef.LogRecord; +import com.example.otelstef.Logs; +import com.example.otelstef.Point; +import com.example.otelstef.PointValue; +import net.stef.StringValue; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.HashSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class CloneTest { + @Test + void logsCloneTracksExistingAttributeValueModifications() { + Logs source = new Logs(); + AnyValue attributeValue = new LogRecord().getBody(); + attributeValue.setInt64(1); + source.getLog().getAttributes().append(new StringValue("key"), attributeValue); + + Logs copy = source.clone(); + + assertFalse(copy.isLogModified()); + assertFalse(copy.getLog().isAttributesModified()); + + elementValue(copy.getLog().getAttributes().at(0)).setInt64(2); + + assertTrue(copy.isLogModified()); + assertTrue(copy.getLog().isAttributesModified()); + } + + @Test + void logsCloneTracksExistingNestedBodyMapValueModifications() { + Logs source = new Logs(); + source.getLog().getBody().setType(AnyValue.Type.TypeKVList); + AnyValue mapValue = new LogRecord().getBody(); + mapValue.setInt64(1); + source.getLog().getBody().getKVList().append(new StringValue("key"), mapValue); + + Logs copy = source.clone(); + + assertFalse(copy.isLogModified()); + assertFalse(copy.getLog().isBodyModified()); + + elementValue(copy.getLog().getBody().getKVList().at(0)).setInt64(2); + + assertTrue(copy.isLogModified()); + assertTrue(copy.getLog().isBodyModified()); + } + + @Test + void logsCloneTracksNestedLogModifications() { + Logs source = new Logs(); + source.getLog().setTimeUnixNano(1); + + Logs copy = source.clone(); + + assertFalse(copy.isLogModified()); + assertFalse(copy.getLog().isTimeUnixNanoModified()); + + copy.getLog().setTimeUnixNano(2); + + assertTrue(copy.isLogModified()); + assertTrue(copy.getLog().isTimeUnixNanoModified()); + } + + @Test + void logRecordCloneTracksNestedBodyModifications() { + LogRecord source = new LogRecord(); + source.getBody().setInt64(1); + + LogRecord copy = source.clone(); + + assertFalse(copy.isBodyModified()); + + copy.getBody().setInt64(2); + + assertTrue(copy.isBodyModified()); + } + + @Test + void oneofClonePreservesEqualityAndHashCodeAfterVariantSwitch() { + AnyValue anyValue = new LogRecord().getBody(); + anyValue.setInt64(42); + anyValue.setFloat64(1.0); + assertCloneHasEqualHashCode(anyValue, anyValue.clone()); + + PointValue pointValue = new Point().getValue(); + pointValue.setInt64(42); + pointValue.setFloat64(1.0); + assertCloneHasEqualHashCode(pointValue, pointValue.clone()); + + ExemplarValue exemplarValue = new Exemplar().getValue(); + exemplarValue.setInt64(42); + exemplarValue.setFloat64(1.0); + assertCloneHasEqualHashCode(exemplarValue, exemplarValue.clone()); + } + + private static void assertCloneHasEqualHashCode(T source, T copy) { + assertEquals(source, copy); + assertEquals(source.hashCode(), copy.hashCode()); + + HashSet values = new HashSet<>(); + values.add(source); + assertTrue(values.contains(copy)); + } + + private static AnyValue elementValue(Object elem) { + try { + Field valueField = elem.getClass().getDeclaredField("value"); + valueField.setAccessible(true); + return (AnyValue) valueField.get(elem); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } +} diff --git a/java/src/test/java/tests/EncoderResetTest.java b/java/src/test/java/tests/EncoderResetTest.java new file mode 100644 index 00000000..0a77334c --- /dev/null +++ b/java/src/test/java/tests/EncoderResetTest.java @@ -0,0 +1,118 @@ +package tests; + +import com.example.otelstef.AnyValue; +import com.example.otelstef.LogsWriter; +import com.example.otelstef.Metrics; +import com.example.otelstef.MetricsReader; +import com.example.otelstef.MetricsWriter; +import com.example.otelstef.PointValue; +import com.example.otelstef.QuantileValueArray; +import net.stef.ChunkWriter; +import net.stef.FrameFlags; +import net.stef.MemChunkWriter; +import net.stef.ReadOptions; +import net.stef.ReadResult; +import net.stef.StringValue; +import net.stef.WriterOptions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class EncoderResetTest { + private static AnyValue newAnyValue() throws Exception { + Constructor constructor = AnyValue.class.getDeclaredConstructor(); + constructor.setAccessible(true); + return constructor.newInstance(); + } + + private static class CapturingChunkWriter implements ChunkWriter { + private final List chunks = new ArrayList<>(); + + @Override + public void writeChunk(byte[] header, byte[] content) throws IOException { + ByteArrayOutputStream chunk = new ByteArrayOutputStream(); + chunk.write(header); + if (content != null) { + chunk.write(content); + } + chunks.add(chunk.toByteArray()); + } + + byte[] streamWithLastDataFrame() throws IOException { + // A standalone stream consists of the fixed header, variable-header frame, + // and one independently decodable data frame. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + stream.write(chunks.get(0)); + stream.write(chunks.get(1)); + stream.write(chunks.get(chunks.size() - 1)); + return stream.toByteArray(); + } + } + + @Test + void forceModifiedFieldsIsConsumedByNextWrite() throws Exception { + LogsWriter writer = new LogsWriter(new MemChunkWriter(), WriterOptions.builder().build()); + + Field encoderField = LogsWriter.class.getDeclaredField("encoder"); + encoderField.setAccessible(true); + Object encoder = encoderField.get(writer); + + Method reset = encoder.getClass().getDeclaredMethod("reset"); + reset.setAccessible(true); + reset.invoke(encoder); + + writer.write(); + + Field forceModifiedFields = encoder.getClass().getDeclaredField("forceModifiedFields"); + forceModifiedFields.setAccessible(true); + assertFalse(forceModifiedFields.getBoolean(encoder)); + } + + @Test + void restartedFrameFullyEncodesEveryCollectionElement() throws Exception { + CapturingChunkWriter output = new CapturingChunkWriter(); + int restartFlags = FrameFlags.RestartDictionaries | FrameFlags.RestartCodecs; + MetricsWriter writer = new MetricsWriter( + output, + WriterOptions.builder().frameRestartFlags(restartFlags).build() + ); + + writer.record.getPoint().getValue().setType(PointValue.Type.TypeSummary); + QuantileValueArray quantiles = writer.record.getPoint().getValue().getSummary().getQuantileValues(); + quantiles.ensureLen(2); + quantiles.at(0).setQuantile(0.5); + quantiles.at(0).setValue(10.0); + quantiles.at(1).setQuantile(0.99); + quantiles.at(1).setValue(20.0); + + AnyValue firstValue = newAnyValue(); + firstValue.setInt64(1); + writer.record.getAttributes().append(new StringValue("first"), firstValue); + AnyValue secondValue = newAnyValue(); + secondValue.setInt64(2); + writer.record.getAttributes().append(new StringValue("second"), secondValue); + + Metrics expected = writer.record.clone(); + writer.write(); + // Write the unchanged record again. Its modification masks are now clear, + // but the restarted frame must still contain a complete representation. + writer.write(); + + MetricsReader reader = new MetricsReader( + new ByteArrayInputStream(output.streamWithLastDataFrame()) + ); + assertEquals(ReadResult.Success, reader.read(ReadOptions.none)); + assertTrue(reader.record.equals(expected)); + } +} diff --git a/stefc/templates/go/array.go.tmpl b/stefc/templates/go/array.go.tmpl index b79e3ce5..d36aeb09 100644 --- a/stefc/templates/go/array.go.tmpl +++ b/stefc/templates/go/array.go.tmpl @@ -55,6 +55,23 @@ func (e *{{.ArrayName}}) fixParent(parentModifiedFields *modifiedFields) { e.parentModifiedFields = parentModifiedFields } +// attachParent establishes both parent pointer and bit in a newly copied array. +func (e *{{.ArrayName}}) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + e.parentModifiedFields = parentModifiedFields + e.parentModifiedBit = parentModifiedBit + {{- if not .ElemType.IsPrimitive}} + for i := range e.elems { + {{- if .ElemType.DictName}} + if !e.elems[i].canBeShared() { + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + } + {{- else}} + e.elems[i].attachParent(parentModifiedFields, parentModifiedBit) + {{- end}} + } + {{- end}} +} + func (e *{{.ArrayName}}) canBeShared() bool { // An array can never be shared. return false @@ -137,6 +154,21 @@ func (e* {{.ArrayName}}) setUnmodifiedRecursively() { {{end}} } +// clearModifiedRecursively clears modification state in all mutable elements. +func (e* {{.ArrayName}}) clearModifiedRecursively() { +{{if not .ElemType.IsPrimitive -}} + for i:=0; i < len(e.elems); i++ { + {{- if .ElemType.DictName}} + if !e.elems[i].canBeShared() { + e.elems[i].clearModifiedRecursively() + } + {{- else}} + e.elems[i].clearModifiedRecursively() + {{- end}} + } +{{end}} +} + // computeDiff compares e and val and returns true if they differ. // All fields that are different in e will be marked as modified. func (e *{{ .ArrayName }}) computeDiff(val *{{ .ArrayName }}) (ret bool) { @@ -384,7 +416,7 @@ func (a *{{ .ArrayName }}) mutateRandom(random *rand.Rand, schem *schema.Schema, {{- if .ElemType.DictName}} if a.elems[i].canBeShared() { // Elem may be shared by pointer. Clone it to have exclusive ownership. - a.elems[i] = a.elems[i].Clone(&Allocators{}) + a.elems[i] = a.elems[i].clone(&Allocators{}) } {{- end}} a.elems[i].mutateRandom(random, schem, limiter) @@ -579,4 +611,4 @@ func (d *{{ .ArrayName }}Decoder) Decode(dst *{{.ArrayName}}) error { } return d.buf.Error() -} \ No newline at end of file +} diff --git a/stefc/templates/go/modifiedfields.go.tmpl b/stefc/templates/go/modifiedfields.go.tmpl index 0bcdaa3e..2821cfd0 100644 --- a/stefc/templates/go/modifiedfields.go.tmpl +++ b/stefc/templates/go/modifiedfields.go.tmpl @@ -114,6 +114,13 @@ func (m *modifiedFieldsMultimap) fixParent(parentModifiedFields *modifiedFields) m.vals.parent = parentModifiedFields } +func (m *modifiedFieldsMultimap) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.keys.parent = parentModifiedFields + m.vals.parent = parentModifiedFields + m.keys.parentBit = parentModifiedBit + m.vals.parentBit = parentModifiedBit +} + // Returns a bitmask to be set in keys or vals to mark modification of // multimap element at index. func (m* modifiedFieldsMultimap) maskForIndex(index int) uint64 { diff --git a/stefc/templates/go/multimap.go.tmpl b/stefc/templates/go/multimap.go.tmpl index 6f0ecaee..87cd24c7 100644 --- a/stefc/templates/go/multimap.go.tmpl +++ b/stefc/templates/go/multimap.go.tmpl @@ -70,6 +70,20 @@ func (m *{{.MultimapName}}) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointers and bits in a newly copied multimap. +func (m *{{.MultimapName}}) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + m.modifiedElems.attachParent(parentModifiedFields, parentModifiedBit) + elems := m.elems[:m.initedCount] + for i:=0; i < len(elems); i++ { + {{- if not .Key.Type.IsPrimitive}} + elems[i].key.attachParent(&m.modifiedElems.keys, m.modifiedElems.maskForIndex(i)) + {{- end}} + {{- if not .Value.Type.IsPrimitive}} + elems[i].value.attachParent(&m.modifiedElems.vals, m.modifiedElems.maskForIndex(i)) + {{- end}} + } +} + func (m *{{.MultimapName}}) canBeShared() bool { // Multimap can never be shared. return false @@ -159,6 +173,19 @@ func (m* {{.MultimapName}}) setUnmodifiedRecursively() { m.modifiedElems.setUnmodifiedAll() } +// clearModifiedRecursively clears modification state in all mutable elements. +func (m* {{.MultimapName}}) clearModifiedRecursively() { + for i:=0; i < len(m.elems); i++ { + {{- if not .Key.Type.IsPrimitive}} + m.elems[i].key.clearModifiedRecursively() + {{- end}} + {{- if not .Value.Type.IsPrimitive}} + m.elems[i].value.clearModifiedRecursively() + {{- end}} + } + m.modifiedElems.setUnmodifiedAll() +} + // computeDiff compares m to val and returns true if they differ. // All fields that are different in m will be marked as modified. func (m *{{.MultimapName}}) computeDiff(val *{{.MultimapName}}) (ret bool) { @@ -236,7 +263,7 @@ func (m *{{.MultimapName}}) SetKey(i int, k {{if .Key.Type.Flags.PassByPtr}}*{{e } else { if !m.elems[i].key.IsEqual(k) { if m.elems[i].key.canBeShared() { - m.elems[i].key = m.elems[i].key.Clone(&Allocators{}) + m.elems[i].key = m.elems[i].key.clone(&Allocators{}) } m.elems[i].key.CopyFrom({{.Key.Type.ToStorage "k"}}) m.modifiedElems.markKeyModified(i) @@ -266,7 +293,7 @@ func (m *{{.MultimapName}}) SetValue(i int, v {{if.Value.Type.Flags.PassByPtr}}* } else { if !m.elems[i].value.IsEqual(v) { if m.elems[i].value.canBeShared() { - m.elems[i].value = m.elems[i].value.Clone(&Allocators{}) + m.elems[i].value = m.elems[i].value.clone(&Allocators{}) } m.elems[i].value.CopyFrom({{.Value.Type.ToStorage "v"}}) m.modifiedElems.markValModified(i) diff --git a/stefc/templates/go/oneof.go.tmpl b/stefc/templates/go/oneof.go.tmpl index 86afa5a1..e0d22298 100644 --- a/stefc/templates/go/oneof.go.tmpl +++ b/stefc/templates/go/oneof.go.tmpl @@ -114,6 +114,27 @@ func (s *{{ $.StructName }}) fixParent(parentModifiedFields *modifiedFields) { } } +// attachParent establishes both parent pointer and bit in a newly copied oneof. +func (s *{{ $.StructName }}) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.parentModifiedFields = parentModifiedFields + s.parentModifiedBit = parentModifiedBit + + switch s.Type() { + {{- range .Fields }} + {{- if not .Type.IsPrimitive }} + case {{ $.StructName }}Type{{.Name}}: + {{- if .Type.DictName}} + if !s.{{.name}}Ptr().canBeShared() { + s.{{.name}}Ptr().attachParent(parentModifiedFields, parentModifiedBit) + } + {{- else}} + s.{{.name}}Ptr().attachParent(parentModifiedFields, parentModifiedBit) + {{- end}} + {{- end }} + {{- end }} + } +} + type {{$.StructName }}Type byte const ( {{.StructName }}TypeNone = {{$.StructName }}Type(iota) @@ -238,39 +259,46 @@ func (s *{{ .StructName }}) canBeShared() bool { return false } -func (s *{{ .StructName }}) cloneShared(allocators *Allocators) {{if .Type.Flags.StoreByPtr}}*{{end}}{{.StructName}} { - // Oneof is not shareable, so cloneShared is just a Clone. - return s.Clone(allocators) -} - -func (s *{{ .StructName }}) Clone(allocators *Allocators) {{if .Type.Flags.StoreByPtr}}*{{end}}{{.StructName}} { -{{- if .Type.Flags.StoreByPtr}} +{{if .Type.Flags.StoreByPtr}} +func (s *{{ .StructName }}) clone(allocators *Allocators) *{{.StructName}} { allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof({{.StructName}}{}))) c := allocators.{{.StructName}}.Alloc() -{{- else}} - c := {{.StructName}}{} -{{- end}} - c.clearValSetType(s.Type()) + s.CloneTo(c, allocators) + return c +} + +func (s *{{ .StructName }}) cloneShared(allocators *Allocators) *{{.StructName}} { + // Oneof is not shareable, so cloneShared is just a clone. + return s.clone(allocators) +} +{{end}} + +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *{{ .StructName }}) CloneTo(dst *{{.StructName}}, allocators *Allocators) { + *dst = {{.StructName}}{} + dst.clearValSetType(s.Type()) switch s.Type() { {{- range .Fields }} {{- $isStringOrBytes := or (eq .Type.Storage "string") (eq .Type.Storage "pkg.Bytes") }} case {{ $.StructName }}Type{{.Name}}: {{- if .Type.IsPrimitive}} {{- if $isStringOrBytes}} - c.set{{.Name}}(s.{{.Name}}()) + dst.set{{.Name}}(s.{{.Name}}()) {{- else}} - *c.{{.name}}Ptr() = *s.{{.name}}Ptr() + *dst.{{.name}}Ptr() = *s.{{.name}}Ptr() {{- end}} {{- else if .Type.Flags.StoreByPtr}} - c.ptr = unsafe.Pointer(s.{{.name}}Ptr().cloneShared(allocators)) + dst.ptr = unsafe.Pointer(s.{{.name}}Ptr().cloneShared(allocators)) {{- else}} allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof({{.Type.Storage}}{}))) - c.alloc{{.Name}}Alloc(allocators) - copyToNew{{.Type.Storage}}(c.{{.name}}Ptr(), s.{{.name}}Ptr(), allocators) + dst.alloc{{.Name}}Alloc(allocators) + copyToNew{{.Type.Storage}}(dst.{{.name}}Ptr(), s.{{.name}}Ptr(), allocators) {{- end}} {{- end }} } - return c + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -376,6 +404,24 @@ func (s *{{ $.StructName }}) setUnmodifiedRecursively() { } } +// clearModifiedRecursively clears modification state in all mutable descendants. +func (s *{{ $.StructName }}) clearModifiedRecursively() { + switch s.Type() { + {{- range .Fields}} + {{- if not .Type.IsPrimitive}} + case {{ $.StructName }}Type{{.Name}}: + {{- if .Type.DictName}} + if !s.{{.name}}Ptr().canBeShared() { + s.{{.name}}Ptr().clearModifiedRecursively() + } + {{- else}} + s.{{.name}}Ptr().clearModifiedRecursively() + {{- end}} + {{- end}} + {{- end}} + } +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *{{ $.StructName }}) computeDiff(val *{{ $.StructName }}) (ret bool) { diff --git a/stefc/templates/go/struct.go.tmpl b/stefc/templates/go/struct.go.tmpl index 27a60cf6..2fef13bd 100644 --- a/stefc/templates/go/struct.go.tmpl +++ b/stefc/templates/go/struct.go.tmpl @@ -169,6 +169,39 @@ func (s *{{ $.StructName }}) fixParent(parentModifiedFields *modifiedFields) { {{- end -}} } +// attachParent establishes both parent pointers and bits in a newly copied struct. +func (s *{{ $.StructName }}) attachParent(parentModifiedFields *modifiedFields, parentModifiedBit uint64) { + s.modifiedFields.parent = parentModifiedFields + s.modifiedFields.parentBit = parentModifiedBit + {{- range .Fields }} + {{- if not .Type.IsPrimitive }} + {{- if .Type.Flags.StoreByPtr}} + {{- if .Optional}} + if s.Has{{.Name}}() && s.{{.name}} != nil { + {{- if .Type.DictName}} + if !s.{{.name}}.canBeShared() { + s.{{.name}}.attachParent(&s.modifiedFields, fieldModified{{ $.StructName }}{{.Name}}) + } + {{- else}} + s.{{.name}}.attachParent(&s.modifiedFields, fieldModified{{ $.StructName }}{{.Name}}) + {{- end}} + } + {{- else}} + {{- if .Type.DictName}} + if !s.{{.name}}.canBeShared() { + s.{{.name}}.attachParent(&s.modifiedFields, fieldModified{{ $.StructName }}{{.Name}}) + } + {{- else}} + s.{{.name}}.attachParent(&s.modifiedFields, fieldModified{{ $.StructName }}{{.Name}}) + {{- end}} + {{- end}} + {{- else }} + s.{{.name}}.attachParent(&s.modifiedFields, fieldModified{{ $.StructName }}{{.Name}}) + {{- end }} + {{- end -}} + {{- end -}} +} + {{if .DictName}} // Freeze the struct. Any attempt to modify it after this will panic. // This marks the struct as eligible for safely sharing by pointer without cloning, @@ -242,7 +275,7 @@ func (s *{{ $.StructName }}) Set{{.Name}}(v {{if .Type.Flags.PassByPtr}}*{{end}} } } else { if s.{{.name}}.canBeShared() { - s.{{.name}} = s.{{.name}}.Clone(&Allocators{}) + s.{{.name}} = s.{{.name}}.clone(&Allocators{}) } s.{{.name}}.CopyFrom({{.Type.ToStorage "v"}}) s.modifiedFields.markModified(fieldModified{{ $.StructName }}{{.Name}}) @@ -321,6 +354,29 @@ func (s *{{ $.StructName }}) setUnmodifiedRecursively() { s.modifiedFields.mask = 0 } +// clearModifiedRecursively clears modification state in s and all mutable descendants. +// Unlike setUnmodifiedRecursively it does not rely on parent modification bits being set. +func (s *{{ $.StructName }}) clearModifiedRecursively() { + {{- range .Fields }} + {{- if not .Type.IsPrimitive}} + {{- if .Optional}} + if s.Has{{.Name}}() { + {{- end}} + {{- if .Type.DictName}} + if !s.{{.name}}.canBeShared() { + s.{{.name}}.clearModifiedRecursively() + } + {{- else}} + s.{{.name}}.clearModifiedRecursively() + {{- end}} + {{- if .Optional}} + } + {{- end}} + {{- end}} + {{- end }} + s.modifiedFields.mask = 0 +} + // computeDiff compares s and val and returns true if they differ. // All fields that are different in s will be marked as modified. func (s *{{ $.StructName }}) computeDiff(val *{{ $.StructName }}) (ret bool) { @@ -368,40 +424,58 @@ func (s *{{ .StructName }}) canBeShared() bool { return {{if .DictName}}s.isFrozen(){{else}}false{{end}} } +{{if .Type.Flags.StoreByPtr}} +// clone returns a clone of s allocated using allocators. +func (s *{{ .StructName }}) clone(allocators *Allocators) *{{.StructName}} { + allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof({{.StructName}}{}))) + c := allocators.{{.StructName}}.Alloc() + s.CloneTo(c, allocators) + return c +} + // cloneShared returns a clone of s. It may return s if it is safe to share without cloning // (for example if s is frozen). -func (s *{{ .StructName }}) cloneShared(allocators *Allocators) {{if .Type.Flags.StoreByPtr}}*{{end}}{{.StructName}} { +func (s *{{ .StructName }}) cloneShared(allocators *Allocators) *{{.StructName}} { {{- if .DictName}} if s.isFrozen() { // If s is frozen it means it is safe to share without cloning. return s } {{- end}} - return s.Clone(allocators) + return s.clone(allocators) } +{{end}} -func (s *{{ .StructName }}) Clone(allocators *Allocators) {{if .Type.Flags.StoreByPtr}}*{{end}}{{.StructName}} { -{{- if .Type.Flags.StoreByPtr}} - allocators.allocSizeChecker.AddAllocSize(uint(unsafe.Sizeof({{.StructName}}{}))) - c := allocators.{{.StructName}}.Alloc() - *c = {{.StructName}}{ -{{- else}} - c := {{.StructName}}{ -{{- end}} +// CloneTo performs a deep copy from s to dst. dst does not need to be initialized +// and must not alias s. dst must remain at a stable address while it is modified. +func (s *{{ .StructName }}) CloneTo(dst *{{.StructName}}, allocators *Allocators) { + *dst = {{.StructName}}{ {{- range .Fields }} {{- if .Type.IsPrimitive}} {{.name}}: s.{{.name}}, - {{- else if .Type.Flags.StoreByPtr}} - {{.name}}: s.{{.name}}.cloneShared(allocators), {{- end}} {{- end }} } {{- range .Fields }} - {{- if and (not .Type.IsPrimitive) (not .Type.Flags.StoreByPtr)}} - copyToNew{{.Type.Storage}}({{if .Type.Flags.TakePtr}}&{{end}}c.{{.name}}, {{if .Type.Flags.TakePtr}}&{{end}}s.{{.name}}, allocators) + {{- if not .Type.IsPrimitive}} + {{- if .Optional}} + if s.Has{{.Name}}() { + {{- end}} + {{- if .Type.Flags.StoreByPtr}} + dst.{{.name}} = s.{{.name}}.cloneShared(allocators) + {{- else}} + copyToNew{{.Type.Storage}}({{if .Type.Flags.TakePtr}}&{{end}}dst.{{.name}}, {{if .Type.Flags.TakePtr}}&{{end}}s.{{.name}}, allocators) + {{- end}} + {{- if .Optional}} + } + {{- end}} {{- end}} {{- end }} - return c + {{- if .OptionalFieldCount}} + dst.optionalFieldsPresent = s.optionalFieldsPresent + {{- end}} + dst.attachParent(nil, 0) + dst.clearModifiedRecursively() } // ByteSize returns approximate memory usage in bytes. Used to calculate @@ -581,12 +655,12 @@ func (s *{{ .StructName }}) mutateRandom(random *rand.Rand, schem *schema.Schema s.{{.name}} = new({{ .Type.Storage }}) s.{{.name}}.init(&s.modifiedFields, fieldModified{{ $.StructName }}{{.Name}}) } else { - s.{{.name}} = s.{{.name}}.Clone(&Allocators{}) + s.{{.name}} = s.{{.name}}.clone(&Allocators{}) } } if s.{{.name}}.canBeShared() { // {{.name}} may be shared by pointer. Clone it to have exclusive ownership. - s.{{.name}} = s.{{.name}}.Clone(&Allocators{}) + s.{{.name}} = s.{{.name}}.clone(&Allocators{}) } {{end}} s.{{.name}}.mutateRandom(random, schem, limiter) @@ -726,7 +800,7 @@ func (d *{{ .StructName }}EncoderDict) Add(val *{{ .StructName }}) { d.dict.Set(val, refNum) return } - clone := val.Clone(d.allocators) // Clone before adding to dictionary. + clone := val.clone(d.allocators) // Clone before adding to dictionary. clone.Freeze() // Freeze the clone so that it can be safely shared by pointer. d.dict.Set(clone, refNum) } @@ -1036,7 +1110,7 @@ func (d *{{ .StructName }}Decoder) Decode(dstPtr {{if.DictName}}*{{end}}*{{.Stru // *dstPtr is pointing to a element in the dictionary. We are not allowed // to modify it. Make a clone of it and decode into the clone. - val := (*dstPtr).Clone(d.allocators) + val := (*dstPtr).clone(d.allocators) if d.allocators.allocSizeChecker.IsOverLimit() { return pkg.ErrRecordAllocLimitExceeded } @@ -1184,4 +1258,4 @@ func {{.StructName}}WireSchema() (schema.WireSchema, error) { } return w,nil } -{{end}} \ No newline at end of file +{{end}} diff --git a/stefc/templates/java/arrayEncoder.java.tmpl b/stefc/templates/java/arrayEncoder.java.tmpl index 96d2f562..83a8cb3b 100644 --- a/stefc/templates/java/arrayEncoder.java.tmpl +++ b/stefc/templates/java/arrayEncoder.java.tmpl @@ -16,6 +16,10 @@ class {{ .ArrayName }}Encoder { private {{.ElemType.EncoderType}}Encoder elemEncoder; private WriterState state; private boolean isRecursive = false; + {{- if not .ElemType.IsPrimitive}} + // Set after reset so every element in the first non-empty array is encoded in full. + private boolean forceElements; + {{- end}} public void init(WriterState state, WriteColumnSet columns) throws IOException { this.state = state; @@ -47,21 +51,36 @@ class {{ .ArrayName }}Encoder { } public void reset() { + {{- if not .ElemType.IsPrimitive}} + forceElements = true; + {{- end}} if (!isRecursive) { elemEncoder.reset(); } } public void encode({{ .ArrayName }} arr) throws IOException { + encode(arr, false); + } + + public void encode({{ .ArrayName }} arr, boolean forceAllElements) throws IOException { int oldBitLen = buf.bitCount(); // Write the length of the array. int newLen = arr.elemsLen; buf.writeUvarintCompact(newLen); + {{- if not .ElemType.IsPrimitive}} + boolean forceElementsNow = forceAllElements || forceElements; + {{- end}} for (int i = 0; i < newLen; i++) { - elemEncoder.encode(arr.elems[i]); + elemEncoder.encode(arr.elems[i]{{if not .ElemType.IsPrimitive}}, forceElementsNow{{end}}); } + {{- if not .ElemType.IsPrimitive}} + if (newLen != 0) { + forceElements = false; + } + {{- end}} // Account written bits in the limiter. int newBitLen = buf.bitCount(); @@ -75,4 +94,3 @@ class {{ .ArrayName }}Encoder { } } } - diff --git a/stefc/templates/java/multimap.java.tmpl b/stefc/templates/java/multimap.java.tmpl index 24ebb7ec..a1d5eac5 100644 --- a/stefc/templates/java/multimap.java.tmpl +++ b/stefc/templates/java/multimap.java.tmpl @@ -158,6 +158,7 @@ public class {{ .MultimapName }} { elems[i].value.markUnmodifiedRecursively(); {{- end}} } + modifiedElems.markUnmodifiedAll(); } // Append adds a key-value pair to the multimap. diff --git a/stefc/templates/java/multimapEncoder.java.tmpl b/stefc/templates/java/multimapEncoder.java.tmpl index 7b304706..7e695a44 100644 --- a/stefc/templates/java/multimapEncoder.java.tmpl +++ b/stefc/templates/java/multimapEncoder.java.tmpl @@ -18,6 +18,8 @@ class {{ .MultimapName }}Encoder { private {{.Value.Type.EncoderType}}Encoder valueEncoder; private boolean isKeyRecursive = false; private boolean isValueRecursive = false; + // Set after reset so the first non-empty multimap is encoded in full. + private boolean forceFull; public void init(WriterState state, WriteColumnSet columns) throws IOException { // Remember this encoder in the state so that we can detect recursion. @@ -60,6 +62,7 @@ class {{ .MultimapName }}Encoder { } public void reset() { + forceFull = true; if (!isKeyRecursive) { keyEncoder.reset(); } @@ -69,6 +72,10 @@ class {{ .MultimapName }}Encoder { } public void encode({{ .MultimapName }} list) throws IOException { + encode(list, false); + } + + public void encode({{ .MultimapName }} list, boolean forceAllElements) throws IOException { int oldLen = buf.size(); if (list.elemsLen == 0) { @@ -80,11 +87,13 @@ class {{ .MultimapName }}Encoder { return; } - if (!list.areKeysModified() && list.elemsLen < 63) { + boolean encodeAllElements = forceAllElements || forceFull; + if (!encodeAllElements && !list.areKeysModified() && list.elemsLen < 63) { encodeValuesOnly(list); } else { - encodeFull(list); + encodeFull(list, encodeAllElements); } + forceFull = false; limiter.addFrameBytes(buf.size() - oldLen); @@ -115,14 +124,14 @@ class {{ .MultimapName }}Encoder { } } - private void encodeFull({{ .MultimapName }} list) throws IOException { + private void encodeFull({{ .MultimapName }} list, boolean forceAllElements) throws IOException { // Record multimap len (LSB is 1 to indicate full encoding). buf.writeUvarint(((long)list.elemsLen << 1) | 0b1); - // Encode keys and values. + // Encode keys and values. for (int i = 0; i < list.elemsLen; i++) { - keyEncoder.encode(list.elems[i].key); - valueEncoder.encode(list.elems[i].value); + keyEncoder.encode(list.elems[i].key{{if not .Key.Type.IsPrimitive}}, forceAllElements{{end}}); + valueEncoder.encode(list.elems[i].value{{if not .Value.Type.IsPrimitive}}, forceAllElements{{end}}); } } @@ -138,4 +147,3 @@ class {{ .MultimapName }}Encoder { } } } - diff --git a/stefc/templates/java/oneof.java.tmpl b/stefc/templates/java/oneof.java.tmpl index 287d5517..33355ec8 100644 --- a/stefc/templates/java/oneof.java.tmpl +++ b/stefc/templates/java/oneof.java.tmpl @@ -133,10 +133,8 @@ public class {{ .StructName }} { // Clone returns a deep copy of this oneof. public {{.StructName}} clone() { {{.StructName}} cpy = new {{.StructName}}(); - cpy.typ = this.typ; - {{- range .Fields }} - cpy.{{.name}} = this.{{.name}}{{if .Type.MustClone}}.clone(){{end}}; - {{- end }} + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } @@ -290,11 +288,13 @@ public class {{ .StructName }} { @Override public int hashCode() { - return Objects.hash( - {{- range $index, $element := .Fields }} - {{.name}}, + switch (this.typ) { + {{- range .Fields }} + case Type{{.Name}}: + return Objects.hash(typ, {{.name}}); {{- end }} - typ - ); + default: + return Objects.hash(typ); + } } } diff --git a/stefc/templates/java/oneofEncoder.java.tmpl b/stefc/templates/java/oneofEncoder.java.tmpl index a946de58..e58016af 100644 --- a/stefc/templates/java/oneofEncoder.java.tmpl +++ b/stefc/templates/java/oneofEncoder.java.tmpl @@ -79,6 +79,12 @@ class {{ .StructName }}Encoder { // Encode encodes val into buf public void encode({{ .StructName }} val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf and propagates forced full encoding through + // the selected non-primitive value. + public void encode({{ .StructName }} val, boolean forceAllFields) throws IOException { {{ $.StructName }}.Type typ = val.typ; if (typ.getValue() > fieldCount) { // The current field type is not supported in target schema. Encode the type as None. @@ -96,7 +102,7 @@ class {{ .StructName }}Encoder { {{- range .Fields }} case Type{{.Name}}: // Encode {{.Name}} - {{.name}}Encoder.encode(val.{{.name}}); + {{.name}}Encoder.encode(val.{{.name}}{{if not .IsPrimitive}}, forceAllFields{{end}}); break; {{- end }} } diff --git a/stefc/templates/java/struct.java.tmpl b/stefc/templates/java/struct.java.tmpl index 244f633a..1bf79cba 100644 --- a/stefc/templates/java/struct.java.tmpl +++ b/stefc/templates/java/struct.java.tmpl @@ -161,9 +161,8 @@ public class {{ .StructName }} { public {{.StructName}} clone() { {{.StructName}} cpy = new {{.StructName}}(); - {{- range .Fields }} - cpy.{{.name}} = this.{{.name}}{{if .Type.MustClone}}.clone(){{end}}; - {{- end }} + cpy.copyFrom(this); + cpy.markUnmodifiedRecursively(); return cpy; } diff --git a/stefc/templates/java/structEncoder.java.tmpl b/stefc/templates/java/structEncoder.java.tmpl index 09b73a19..50733e31 100644 --- a/stefc/templates/java/structEncoder.java.tmpl +++ b/stefc/templates/java/structEncoder.java.tmpl @@ -93,6 +93,12 @@ class {{ .StructName }}Encoder { // encode encodes val into buf public void encode({{ .StructName }} val) throws IOException { + encode(val, false); + } + + // encode encodes val into buf. forceAllFields is used by collection encoders + // when every element must be independently reconstructable after a restart. + public void encode({{ .StructName }} val, boolean forceAllFields) throws IOException { int oldLen = this.buf.bitCount(); {{if .DictName}} @@ -124,12 +130,14 @@ class {{ .StructName }}Encoder { // Mask that describes what fields are encoded. Start with all modified fields. long fieldMask = val.modifiedFields.mask; - // If forceModifiedFields we need to set to 1 all bits so that we + // On an encoder restart or when requested by a containing collection, // force writing of all fields. - if (this.forceModifiedFields) { + boolean forceFields = forceAllFields || this.forceModifiedFields; + if (forceFields) { fieldMask = {{- range .Fields }} {{ $.StructName }}.fieldModified{{.Name}} | {{end}}0L; + this.forceModifiedFields = false; } // Only write fields that we want to write. See init() for keepFieldMask. @@ -146,7 +154,7 @@ class {{ .StructName }}Encoder { if ((fieldMask & {{ $.StructName }}.fieldModified{{.Name}}) != 0 {{- if .Optional}} && (val.optionalFieldsPresent & {{ $.StructName }}.fieldPresent{{.Name}}) != 0{{end}}) { // Encode {{.Name}} - this.{{.name}}Encoder.encode(val.{{.name}}); + this.{{.name}}Encoder.encode(val.{{.name}}{{if not .IsPrimitive}}, forceFields{{end}}); } {{ end }} // Account written bits in the limiter. @@ -181,4 +189,3 @@ class {{ .StructName }}Encoder { {{- end }} } } -