diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d3c62d2..71dc1380 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: - name: Run Gosec Security Scanner uses: securego/gosec@master with: - args: -exclude=G115 ./... + args: -exclude=G115 --exclude-generated ./... - name: Build run: go build -v ./... diff --git a/.gitignore b/.gitignore index 59c2172f..6caac796 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ go.work # Binary -dsync +/dsync java/target # Log files diff --git a/buf.gen.commands.yaml b/buf.gen.commands.yaml new file mode 100644 index 00000000..286a9867 --- /dev/null +++ b/buf.gen.commands.yaml @@ -0,0 +1,10 @@ +version: v2 +plugins: + - local: protoc-gen-go + out: gen + opt: paths=source_relative + - local: protoc-gen-commandargs-urfave + out: gen + opt: paths=source_relative +inputs: + - directory: proto/connectorcommands \ No newline at end of file diff --git a/buf.gen.yaml b/buf.gen.yaml index 0925a788..da16a685 100644 --- a/buf.gen.yaml +++ b/buf.gen.yaml @@ -14,3 +14,5 @@ plugins: out: java/src/main/java - remote: buf.build/protocolbuffers/java:v29.1 out: java/src/main/java +inputs: + - directory: proto/dsync \ No newline at end of file diff --git a/buf.lock b/buf.lock new file mode 100644 index 00000000..e1ba1b25 --- /dev/null +++ b/buf.lock @@ -0,0 +1,6 @@ +# Generated by buf. DO NOT EDIT. +version: v2 +deps: + - name: buf.build/adiom/commandargs + commit: 57609d4ad63a41cd86c89f5874199290 + digest: b5:72c714f13e7013a860cfd0dff8c720c0d7584ccb9446514e052166067c2bfdc47227d38b33a4b64daaacb2acbd5c5acbdc8509d9dbd563a5e6c3814f4e87dcc6 diff --git a/buf.yaml b/buf.yaml index 53da97ac..ea63a046 100644 --- a/buf.yaml +++ b/buf.yaml @@ -1,7 +1,11 @@ # For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml version: v2 modules: - - path: proto + - path: proto/dsync + - path: proto/connectorcommands + name: buf.build/adiom/connectorcommands +deps: + - buf.build/adiom/commandargs lint: use: - STANDARD diff --git a/gen/adiom/commands/connectors/v1/cosmos.pb.go b/gen/adiom/commands/connectors/v1/cosmos.pb.go new file mode 100644 index 00000000..d6fe4f78 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/cosmos.pb.go @@ -0,0 +1,180 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/cosmos.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type CosmosFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mongo *MongoBaseFlags `protobuf:"bytes,1,opt,name=mongo,proto3" json:"mongo,omitempty"` + CosmosReaderMaxNamespaces int32 `protobuf:"varint,2,opt,name=cosmos_reader_max_namespaces,json=cosmosReaderMaxNamespaces,proto3" json:"cosmos_reader_max_namespaces,omitempty"` + CosmosDeletesCdc string `protobuf:"bytes,3,opt,name=cosmos_deletes_cdc,json=cosmosDeletesCdc,proto3" json:"cosmos_deletes_cdc,omitempty"` + CosmosDeleteInterval *durationpb.Duration `protobuf:"bytes,4,opt,name=cosmos_delete_interval,json=cosmosDeleteInterval,proto3" json:"cosmos_delete_interval,omitempty"` + CosmosParallelPartitionWorkers int32 `protobuf:"varint,5,opt,name=cosmos_parallel_partition_workers,json=cosmosParallelPartitionWorkers,proto3" json:"cosmos_parallel_partition_workers,omitempty"` + CosmosStreamDeletesEnabled bool `protobuf:"varint,6,opt,name=cosmos_stream_deletes_enabled,json=cosmosStreamDeletesEnabled,proto3" json:"cosmos_stream_deletes_enabled,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CosmosFlags) Reset() { + *x = CosmosFlags{} + mi := &file_adiom_commands_connectors_v1_cosmos_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CosmosFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CosmosFlags) ProtoMessage() {} + +func (x *CosmosFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_cosmos_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CosmosFlags.ProtoReflect.Descriptor instead. +func (*CosmosFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_cosmos_proto_rawDescGZIP(), []int{0} +} + +func (x *CosmosFlags) GetMongo() *MongoBaseFlags { + if x != nil { + return x.Mongo + } + return nil +} + +func (x *CosmosFlags) GetCosmosReaderMaxNamespaces() int32 { + if x != nil { + return x.CosmosReaderMaxNamespaces + } + return 0 +} + +func (x *CosmosFlags) GetCosmosDeletesCdc() string { + if x != nil { + return x.CosmosDeletesCdc + } + return "" +} + +func (x *CosmosFlags) GetCosmosDeleteInterval() *durationpb.Duration { + if x != nil { + return x.CosmosDeleteInterval + } + return nil +} + +func (x *CosmosFlags) GetCosmosParallelPartitionWorkers() int32 { + if x != nil { + return x.CosmosParallelPartitionWorkers + } + return 0 +} + +func (x *CosmosFlags) GetCosmosStreamDeletesEnabled() bool { + if x != nil { + return x.CosmosStreamDeletesEnabled + } + return false +} + +var File_adiom_commands_connectors_v1_cosmos_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_cosmos_proto_rawDesc = "" + + "\n" + + ")adiom/commands/connectors/v1/cosmos.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\x1a\x1egoogle/protobuf/duration.proto\x1a(adiom/commands/connectors/v1/mongo.proto\"\x95\a\n" + + "\vCosmosFlags\x12B\n" + + "\x05mongo\x18\x01 \x01(\v2,.adiom.commands.connectors.v1.MongoBaseFlagsR\x05mongo\x12\xaa\x01\n" + + "\x1ccosmos_reader_max_namespaces\x18\x02 \x01(\x05Bi\x82\xb5\x18e\n" + + "\x1ccosmos-reader-max-namespaces\x12&max namespaces to copy (keep under 15)\x1a\x0182\x1aCosmos DB-specific OptionsR\x19cosmosReaderMaxNamespaces\x12\x92\x01\n" + + "\x12cosmos_deletes_cdc\x18\x03 \x01(\tBd\x82\xb5\x18`\n" + + "\x12cosmos-deletes-cdc\x12.witness connection string for CosmosDB deletes2\x1aCosmos DB-specific OptionsR\x10cosmosDeletesCdc\x12\x89\x01\n" + + "\x16cosmos_delete_interval\x18\x04 \x01(\v2\x19.google.protobuf.DurationB8\x82\xb5\x184\n" + + "\x16cosmos-delete-interval2\x1aCosmos DB-specific OptionsR\x14cosmosDeleteInterval\x12r\n" + + "!cosmos_parallel_partition_workers\x18\x05 \x01(\x05B'\x82\xb5\x18#\n" + + "!cosmos-parallel-partition-workersR\x1ecosmosParallelPartitionWorkers\x12\xc1\x01\n" + + "\x1dcosmos_stream_deletes_enabled\x18\x06 \x01(\bB~\x82\xb5\x18z\n" + + "\x1dcosmos-stream-deletes-enabled\x12=If this cosmos instance supports deletes in the change stream2\x1aCosmos DB-specific OptionsR\x1acosmosStreamDeletesEnabled:<\x82\xb5\x188\n" + + "\bCosmosDB\",mongodb://cosmos-connection-string [options]BKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_cosmos_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_cosmos_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_cosmos_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_cosmos_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_cosmos_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_cosmos_proto_rawDesc), len(file_adiom_commands_connectors_v1_cosmos_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_cosmos_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_cosmos_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_cosmos_proto_goTypes = []any{ + (*CosmosFlags)(nil), // 0: adiom.commands.connectors.v1.CosmosFlags + (*MongoBaseFlags)(nil), // 1: adiom.commands.connectors.v1.MongoBaseFlags + (*durationpb.Duration)(nil), // 2: google.protobuf.Duration +} +var file_adiom_commands_connectors_v1_cosmos_proto_depIdxs = []int32{ + 1, // 0: adiom.commands.connectors.v1.CosmosFlags.mongo:type_name -> adiom.commands.connectors.v1.MongoBaseFlags + 2, // 1: adiom.commands.connectors.v1.CosmosFlags.cosmos_delete_interval:type_name -> google.protobuf.Duration + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_cosmos_proto_init() } +func file_adiom_commands_connectors_v1_cosmos_proto_init() { + if File_adiom_commands_connectors_v1_cosmos_proto != nil { + return + } + file_adiom_commands_connectors_v1_mongo_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_cosmos_proto_rawDesc), len(file_adiom_commands_connectors_v1_cosmos_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_cosmos_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_cosmos_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_cosmos_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_cosmos_proto = out.File + file_adiom_commands_connectors_v1_cosmos_proto_goTypes = nil + file_adiom_commands_connectors_v1_cosmos_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/cosmos_commandargs.go b/gen/adiom/commands/connectors/v1/cosmos_commandargs.go new file mode 100644 index 00000000..b9983799 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/cosmos_commandargs.go @@ -0,0 +1,104 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import ( + durationpb "google.golang.org/protobuf/types/known/durationpb" +) + +import "github.com/urfave/cli/v2" + +func CosmosFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "skip-batch-overwrite", + }, + &cli.BoolFlag{ + Name: "full-document-key", + Usage: "uses the full document key instead of just _id (except for batch overwrites)", + }, + &cli.DurationFlag{ + Name: "server-timeout", + }, + &cli.DurationFlag{ + Name: "ping-timeout", + }, + &cli.IntFlag{ + Name: "writer-batch-size", + }, + &cli.Int64Flag{ + Name: "doc-partition", + Value: 50000, + }, + &cli.IntFlag{ + Name: "max-page-size", + }, + &cli.StringFlag{ + Name: "initial-sync-query", + Usage: "query filter for the initial data copy (v2 Extended JSON)", + Aliases: []string{"q"}, + }, + &cli.IntFlag{ + Name: "cosmos-reader-max-namespaces", + Usage: "max namespaces to copy (keep under 15)", + Value: 8, + Category: "Cosmos DB-specific Options", + }, + &cli.StringFlag{ + Name: "cosmos-deletes-cdc", + Usage: "witness connection string for CosmosDB deletes", + Category: "Cosmos DB-specific Options", + }, + &cli.DurationFlag{ + Name: "cosmos-delete-interval", + Category: "Cosmos DB-specific Options", + }, + &cli.IntFlag{ + Name: "cosmos-parallel-partition-workers", + }, + &cli.BoolFlag{ + Name: "cosmos-stream-deletes-enabled", + Usage: "If this cosmos instance supports deletes in the change stream", + Category: "Cosmos DB-specific Options", + }, + } +} + +func ParseCosmosFlags(c *cli.Context) (*CosmosFlags, error) { + cfg := &CosmosFlags{} + cfg.Mongo = &MongoBaseFlags{} + cfg.Mongo.SkipBatchOverwrite = c.Bool("skip-batch-overwrite") + cfg.Mongo.FullDocumentKey = c.Bool("full-document-key") + cfg.Mongo.ServerTimeout = durationpb.New(c.Duration("server-timeout")) + cfg.Mongo.PingTimeout = durationpb.New(c.Duration("ping-timeout")) + cfg.Mongo.WriterBatchSize = int32(c.Int("writer-batch-size")) + cfg.Mongo.DocPartition = c.Int64("doc-partition") + cfg.Mongo.MaxPageSize = int32(c.Int("max-page-size")) + cfg.Mongo.InitialSyncQuery = c.String("initial-sync-query") + cfg.CosmosReaderMaxNamespaces = int32(c.Int("cosmos-reader-max-namespaces")) + cfg.CosmosDeletesCdc = c.String("cosmos-deletes-cdc") + cfg.CosmosDeleteInterval = durationpb.New(c.Duration("cosmos-delete-interval")) + cfg.CosmosParallelPartitionWorkers = int32(c.Int("cosmos-parallel-partition-workers")) + cfg.CosmosStreamDeletesEnabled = c.Bool("cosmos-stream-deletes-enabled") + return cfg, nil +} + +func NewCosmosFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "CosmosDB", + Usage: "mongodb://cosmos-connection-string [options]", + Flags: CosmosFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewCosmosFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "CosmosDB", + Usage: "mongodb://cosmos-connection-string [options]", + Flags: CosmosFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/devnull.pb.go b/gen/adiom/commands/connectors/v1/devnull.pb.go new file mode 100644 index 00000000..1ad1498b --- /dev/null +++ b/gen/adiom/commands/connectors/v1/devnull.pb.go @@ -0,0 +1,157 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/devnull.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DevNullFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Sleep *durationpb.Duration `protobuf:"bytes,1,opt,name=sleep,proto3" json:"sleep,omitempty"` + SleepJitter *durationpb.Duration `protobuf:"bytes,2,opt,name=sleep_jitter,json=sleepJitter,proto3" json:"sleep_jitter,omitempty"` + LogJson bool `protobuf:"varint,3,opt,name=log_json,json=logJson,proto3" json:"log_json,omitempty"` + Id string `protobuf:"bytes,4,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DevNullFlags) Reset() { + *x = DevNullFlags{} + mi := &file_adiom_commands_connectors_v1_devnull_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DevNullFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DevNullFlags) ProtoMessage() {} + +func (x *DevNullFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_devnull_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DevNullFlags.ProtoReflect.Descriptor instead. +func (*DevNullFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_devnull_proto_rawDescGZIP(), []int{0} +} + +func (x *DevNullFlags) GetSleep() *durationpb.Duration { + if x != nil { + return x.Sleep + } + return nil +} + +func (x *DevNullFlags) GetSleepJitter() *durationpb.Duration { + if x != nil { + return x.SleepJitter + } + return nil +} + +func (x *DevNullFlags) GetLogJson() bool { + if x != nil { + return x.LogJson + } + return false +} + +func (x *DevNullFlags) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +var File_adiom_commands_connectors_v1_devnull_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_devnull_proto_rawDesc = "" + + "\n" + + "*adiom/commands/connectors/v1/devnull.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\x1a\x1egoogle/protobuf/duration.proto\"\xeb\x02\n" + + "\fDevNullFlags\x12R\n" + + "\x05sleep\x18\x01 \x01(\v2\x19.google.protobuf.DurationB!\x82\xb5\x18\x1d\x12\x1bSleep time between requestsR\x05sleep\x12t\n" + + "\fsleep_jitter\x18\x02 \x01(\v2\x19.google.protobuf.DurationB6\x82\xb5\x182\n" + + "\fsleep-jitter\x12\"If sleep is set, additional jitterR\vsleepJitter\x12L\n" + + "\blog_json\x18\x03 \x01(\bB1\x82\xb5\x18-\n" + + "\blog-json\x12!Convert data to json and log INFOR\alogJson\x122\n" + + "\x02id\x18\x04 \x01(\tB\"\x82\xb5\x18\x1e\x12\x1cA fixed id for the connectorR\x02id:\x0f\x82\xb5\x18\v\n" + + "\t/dev/nullBKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_devnull_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_devnull_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_devnull_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_devnull_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_devnull_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_devnull_proto_rawDesc), len(file_adiom_commands_connectors_v1_devnull_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_devnull_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_devnull_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_devnull_proto_goTypes = []any{ + (*DevNullFlags)(nil), // 0: adiom.commands.connectors.v1.DevNullFlags + (*durationpb.Duration)(nil), // 1: google.protobuf.Duration +} +var file_adiom_commands_connectors_v1_devnull_proto_depIdxs = []int32{ + 1, // 0: adiom.commands.connectors.v1.DevNullFlags.sleep:type_name -> google.protobuf.Duration + 1, // 1: adiom.commands.connectors.v1.DevNullFlags.sleep_jitter:type_name -> google.protobuf.Duration + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_devnull_proto_init() } +func file_adiom_commands_connectors_v1_devnull_proto_init() { + if File_adiom_commands_connectors_v1_devnull_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_devnull_proto_rawDesc), len(file_adiom_commands_connectors_v1_devnull_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_devnull_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_devnull_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_devnull_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_devnull_proto = out.File + file_adiom_commands_connectors_v1_devnull_proto_goTypes = nil + file_adiom_commands_connectors_v1_devnull_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/devnull_commandargs.go b/gen/adiom/commands/connectors/v1/devnull_commandargs.go new file mode 100644 index 00000000..79c6bb4c --- /dev/null +++ b/gen/adiom/commands/connectors/v1/devnull_commandargs.go @@ -0,0 +1,57 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import ( + durationpb "google.golang.org/protobuf/types/known/durationpb" +) + +import "github.com/urfave/cli/v2" + +func DevNullFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.DurationFlag{ + Name: "sleep", + Usage: "Sleep time between requests", + }, + &cli.DurationFlag{ + Name: "sleep-jitter", + Usage: "If sleep is set, additional jitter", + }, + &cli.BoolFlag{ + Name: "log-json", + Usage: "Convert data to json and log INFO", + }, + &cli.StringFlag{ + Name: "id", + Usage: "A fixed id for the connector", + }, + } +} + +func ParseDevNullFlags(c *cli.Context) (*DevNullFlags, error) { + cfg := &DevNullFlags{} + cfg.Sleep = durationpb.New(c.Duration("sleep")) + cfg.SleepJitter = durationpb.New(c.Duration("sleep-jitter")) + cfg.LogJson = c.Bool("log-json") + cfg.Id = c.String("id") + return cfg, nil +} + +func NewDevNullFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "/dev/null", + Flags: DevNullFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewDevNullFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "/dev/null", + Flags: DevNullFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/devrandom.pb.go b/gen/adiom/commands/connectors/v1/devrandom.pb.go new file mode 100644 index 00000000..e154e8b0 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/devrandom.pb.go @@ -0,0 +1,115 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/devrandom.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DevRandomFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DevRandomFlags) Reset() { + *x = DevRandomFlags{} + mi := &file_adiom_commands_connectors_v1_devrandom_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DevRandomFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DevRandomFlags) ProtoMessage() {} + +func (x *DevRandomFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_devrandom_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DevRandomFlags.ProtoReflect.Descriptor instead. +func (*DevRandomFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_devrandom_proto_rawDescGZIP(), []int{0} +} + +var File_adiom_commands_connectors_v1_devrandom_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_devrandom_proto_rawDesc = "" + + "\n" + + ",adiom/commands/connectors/v1/devrandom.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\"#\n" + + "\x0eDevRandomFlags:\x11\x82\xb5\x18\r\n" + + "\v/dev/randomBKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_devrandom_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_devrandom_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_devrandom_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_devrandom_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_devrandom_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_devrandom_proto_rawDesc), len(file_adiom_commands_connectors_v1_devrandom_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_devrandom_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_devrandom_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_devrandom_proto_goTypes = []any{ + (*DevRandomFlags)(nil), // 0: adiom.commands.connectors.v1.DevRandomFlags +} +var file_adiom_commands_connectors_v1_devrandom_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_devrandom_proto_init() } +func file_adiom_commands_connectors_v1_devrandom_proto_init() { + if File_adiom_commands_connectors_v1_devrandom_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_devrandom_proto_rawDesc), len(file_adiom_commands_connectors_v1_devrandom_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_devrandom_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_devrandom_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_devrandom_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_devrandom_proto = out.File + file_adiom_commands_connectors_v1_devrandom_proto_goTypes = nil + file_adiom_commands_connectors_v1_devrandom_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/devrandom_commandargs.go b/gen/adiom/commands/connectors/v1/devrandom_commandargs.go new file mode 100644 index 00000000..6b24b9fb --- /dev/null +++ b/gen/adiom/commands/connectors/v1/devrandom_commandargs.go @@ -0,0 +1,32 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import "github.com/urfave/cli/v2" + +func DevRandomFlagsCommandFlags() []cli.Flag { + return nil +} + +func ParseDevRandomFlags(c *cli.Context) (*DevRandomFlags, error) { + cfg := &DevRandomFlags{} + return cfg, nil +} + +func NewDevRandomFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "/dev/random", + Flags: DevRandomFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewDevRandomFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "/dev/random", + Flags: DevRandomFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/dynamodb.pb.go b/gen/adiom/commands/connectors/v1/dynamodb.pb.go new file mode 100644 index 00000000..5bf0e2b8 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/dynamodb.pb.go @@ -0,0 +1,144 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/dynamodb.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DynamoDBFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + DocPartition int32 `protobuf:"varint,1,opt,name=doc_partition,json=docPartition,proto3" json:"doc_partition,omitempty"` + PlanParallelism int32 `protobuf:"varint,2,opt,name=plan_parallelism,json=planParallelism,proto3" json:"plan_parallelism,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DynamoDBFlags) Reset() { + *x = DynamoDBFlags{} + mi := &file_adiom_commands_connectors_v1_dynamodb_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DynamoDBFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DynamoDBFlags) ProtoMessage() {} + +func (x *DynamoDBFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_dynamodb_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DynamoDBFlags.ProtoReflect.Descriptor instead. +func (*DynamoDBFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_dynamodb_proto_rawDescGZIP(), []int{0} +} + +func (x *DynamoDBFlags) GetDocPartition() int32 { + if x != nil { + return x.DocPartition + } + return 0 +} + +func (x *DynamoDBFlags) GetPlanParallelism() int32 { + if x != nil { + return x.PlanParallelism + } + return 0 +} + +func (x *DynamoDBFlags) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +var File_adiom_commands_connectors_v1_dynamodb_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_dynamodb_proto_rawDesc = "" + + "\n" + + "+adiom/commands/connectors/v1/dynamodb.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\"\xc4\x02\n" + + "\rDynamoDBFlags\x12i\n" + + "\rdoc_partition\x18\x01 \x01(\x05BD\x82\xb5\x18@\n" + + "\rdoc-partition\x12(Target number of documents per partition\x1a\x0550000R\fdocPartition\x12a\n" + + "\x10plan_parallelism\x18\x02 \x01(\x05B6\x82\xb5\x182\n" + + "\x10plan-parallelism\x12\x1bParallelism during planning\x1a\x014R\x0fplanParallelism\x122\n" + + "\x02id\x18\x03 \x01(\tB\"\x82\xb5\x18\x1e\x12\x1cA fixed id for the connectorR\x02id:1\x82\xb5\x18-\n" + + "\bDynamoDB\"!dynamodb OR dynamodb://localstackBKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_dynamodb_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_dynamodb_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_dynamodb_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_dynamodb_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_dynamodb_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_dynamodb_proto_rawDesc), len(file_adiom_commands_connectors_v1_dynamodb_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_dynamodb_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_dynamodb_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_dynamodb_proto_goTypes = []any{ + (*DynamoDBFlags)(nil), // 0: adiom.commands.connectors.v1.DynamoDBFlags +} +var file_adiom_commands_connectors_v1_dynamodb_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_dynamodb_proto_init() } +func file_adiom_commands_connectors_v1_dynamodb_proto_init() { + if File_adiom_commands_connectors_v1_dynamodb_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_dynamodb_proto_rawDesc), len(file_adiom_commands_connectors_v1_dynamodb_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_dynamodb_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_dynamodb_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_dynamodb_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_dynamodb_proto = out.File + file_adiom_commands_connectors_v1_dynamodb_proto_goTypes = nil + file_adiom_commands_connectors_v1_dynamodb_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/dynamodb_commandargs.go b/gen/adiom/commands/connectors/v1/dynamodb_commandargs.go new file mode 100644 index 00000000..1d6e21d7 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/dynamodb_commandargs.go @@ -0,0 +1,52 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import "github.com/urfave/cli/v2" + +func DynamoDBFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.IntFlag{ + Name: "doc-partition", + Usage: "Target number of documents per partition", + Value: 50000, + }, + &cli.IntFlag{ + Name: "plan-parallelism", + Usage: "Parallelism during planning", + Value: 4, + }, + &cli.StringFlag{ + Name: "id", + Usage: "A fixed id for the connector", + }, + } +} + +func ParseDynamoDBFlags(c *cli.Context) (*DynamoDBFlags, error) { + cfg := &DynamoDBFlags{} + cfg.DocPartition = int32(c.Int("doc-partition")) + cfg.PlanParallelism = int32(c.Int("plan-parallelism")) + cfg.Id = c.String("id") + return cfg, nil +} + +func NewDynamoDBFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "DynamoDB", + Usage: "dynamodb OR dynamodb://localstack", + Flags: DynamoDBFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewDynamoDBFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "DynamoDB", + Usage: "dynamodb OR dynamodb://localstack", + Flags: DynamoDBFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/fakesource.pb.go b/gen/adiom/commands/connectors/v1/fakesource.pb.go new file mode 100644 index 00000000..c6349952 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/fakesource.pb.go @@ -0,0 +1,262 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/fakesource.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FakeSourceFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Sleep *durationpb.Duration `protobuf:"bytes,1,opt,name=sleep,proto3" json:"sleep,omitempty"` + Jitter *durationpb.Duration `protobuf:"bytes,2,opt,name=jitter,proto3" json:"jitter,omitempty"` + UpdateDuration *durationpb.Duration `protobuf:"bytes,3,opt,name=update_duration,json=updateDuration,proto3" json:"update_duration,omitempty"` + StreamTick *durationpb.Duration `protobuf:"bytes,4,opt,name=stream_tick,json=streamTick,proto3" json:"stream_tick,omitempty"` + NumNamespaces int32 `protobuf:"varint,5,opt,name=num_namespaces,json=numNamespaces,proto3" json:"num_namespaces,omitempty"` + NumPartitionsPerNamespace int32 `protobuf:"varint,6,opt,name=num_partitions_per_namespace,json=numPartitionsPerNamespace,proto3" json:"num_partitions_per_namespace,omitempty"` + NumUpdatePartitionsPerNamespace int32 `protobuf:"varint,7,opt,name=num_update_partitions_per_namespace,json=numUpdatePartitionsPerNamespace,proto3" json:"num_update_partitions_per_namespace,omitempty"` + NumDocsPerPartition int64 `protobuf:"varint,8,opt,name=num_docs_per_partition,json=numDocsPerPartition,proto3" json:"num_docs_per_partition,omitempty"` + BatchSize int32 `protobuf:"varint,9,opt,name=batch_size,json=batchSize,proto3" json:"batch_size,omitempty"` + UpdateBatchSize int32 `protobuf:"varint,10,opt,name=update_batch_size,json=updateBatchSize,proto3" json:"update_batch_size,omitempty"` + MaxUpdatesPerTick int32 `protobuf:"varint,11,opt,name=max_updates_per_tick,json=maxUpdatesPerTick,proto3" json:"max_updates_per_tick,omitempty"` + NamespacePrefix string `protobuf:"bytes,12,opt,name=namespace_prefix,json=namespacePrefix,proto3" json:"namespace_prefix,omitempty"` + Payload []string `protobuf:"bytes,13,rep,name=payload,proto3" json:"payload,omitempty"` + PayloadJson string `protobuf:"bytes,14,opt,name=payload_json,json=payloadJson,proto3" json:"payload_json,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FakeSourceFlags) Reset() { + *x = FakeSourceFlags{} + mi := &file_adiom_commands_connectors_v1_fakesource_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FakeSourceFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FakeSourceFlags) ProtoMessage() {} + +func (x *FakeSourceFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_fakesource_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FakeSourceFlags.ProtoReflect.Descriptor instead. +func (*FakeSourceFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_fakesource_proto_rawDescGZIP(), []int{0} +} + +func (x *FakeSourceFlags) GetSleep() *durationpb.Duration { + if x != nil { + return x.Sleep + } + return nil +} + +func (x *FakeSourceFlags) GetJitter() *durationpb.Duration { + if x != nil { + return x.Jitter + } + return nil +} + +func (x *FakeSourceFlags) GetUpdateDuration() *durationpb.Duration { + if x != nil { + return x.UpdateDuration + } + return nil +} + +func (x *FakeSourceFlags) GetStreamTick() *durationpb.Duration { + if x != nil { + return x.StreamTick + } + return nil +} + +func (x *FakeSourceFlags) GetNumNamespaces() int32 { + if x != nil { + return x.NumNamespaces + } + return 0 +} + +func (x *FakeSourceFlags) GetNumPartitionsPerNamespace() int32 { + if x != nil { + return x.NumPartitionsPerNamespace + } + return 0 +} + +func (x *FakeSourceFlags) GetNumUpdatePartitionsPerNamespace() int32 { + if x != nil { + return x.NumUpdatePartitionsPerNamespace + } + return 0 +} + +func (x *FakeSourceFlags) GetNumDocsPerPartition() int64 { + if x != nil { + return x.NumDocsPerPartition + } + return 0 +} + +func (x *FakeSourceFlags) GetBatchSize() int32 { + if x != nil { + return x.BatchSize + } + return 0 +} + +func (x *FakeSourceFlags) GetUpdateBatchSize() int32 { + if x != nil { + return x.UpdateBatchSize + } + return 0 +} + +func (x *FakeSourceFlags) GetMaxUpdatesPerTick() int32 { + if x != nil { + return x.MaxUpdatesPerTick + } + return 0 +} + +func (x *FakeSourceFlags) GetNamespacePrefix() string { + if x != nil { + return x.NamespacePrefix + } + return "" +} + +func (x *FakeSourceFlags) GetPayload() []string { + if x != nil { + return x.Payload + } + return nil +} + +func (x *FakeSourceFlags) GetPayloadJson() string { + if x != nil { + return x.PayloadJson + } + return "" +} + +var File_adiom_commands_connectors_v1_fakesource_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_fakesource_proto_rawDesc = "" + + "\n" + + "-adiom/commands/connectors/v1/fakesource.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\x1a\x1egoogle/protobuf/duration.proto\"\xd8\f\n" + + "\x0fFakeSourceFlags\x12R\n" + + "\x05sleep\x18\x01 \x01(\v2\x19.google.protobuf.DurationB!\x82\xb5\x18\x1d\x12\x1bSleep time between requestsR\x05sleep\x12_\n" + + "\x06jitter\x18\x02 \x01(\v2\x19.google.protobuf.DurationB,\x82\xb5\x18(\x12&Additional jitter to certain durationsR\x06jitter\x12\x90\x01\n" + + "\x0fupdate_duration\x18\x03 \x01(\v2\x19.google.protobuf.DurationBL\x82\xb5\x18H\n" + + "\x0fupdate-duration\x12.Time for each update-batch-size to be produced\x1a\x05400msR\x0eupdateDuration\x12{\n" + + "\vstream_tick\x18\x04 \x01(\v2\x19.google.protobuf.DurationB?\x82\xb5\x18;\n" + + "\vstream-tick\x12(Polling time to check for stream updates\x1a\x021sR\n" + + "streamTick\x12h\n" + + "\x0enum_namespaces\x18\x05 \x01(\x05BA\x82\xb5\x18=\n" + + "\x0enum-namespaces\x12(Number of namespaces (index starts at 0)\x1a\x012R\rnumNamespaces\x12\x8a\x01\n" + + "\x1cnum_partitions_per_namespace\x18\x06 \x01(\x05BI\x82\xb5\x18E\n" + + "\x1cnum-partitions-per-namespace\x12\"Number of partitions per namespace\x1a\x013R\x19numPartitionsPerNamespace\x12\xcd\x01\n" + + "#num_update_partitions_per_namespace\x18\a \x01(\x05B\x7f\x82\xb5\x18{\n" + + "#num-update-partitions-per-namespace\x12TNumber of update partitions per namespace. If set to 0, use a single stream for all.R\x1fnumUpdatePartitionsPerNamespace\x12u\n" + + "\x16num_docs_per_partition\x18\b \x01(\x03B@\x82\xb5\x18<\n" + + "\x16num-docs-per-partition\x12\x1cNumber of docs per partition\x1a\x041000R\x13numDocsPerPartition\x12N\n" + + "\n" + + "batch_size\x18\t \x01(\x05B/\x82\xb5\x18+\n" + + "\n" + + "batch-size\x12\x18Number of docs per batch\x1a\x03150R\tbatchSize\x12b\n" + + "\x11update_batch_size\x18\n" + + " \x01(\x05B6\x82\xb5\x182\n" + + "\x11update-batch-size\x12\x19Number of docs per update\x1a\x0250R\x0fupdateBatchSize\x12k\n" + + "\x14max_updates_per_tick\x18\v \x01(\x05B:\x82\xb5\x186\n" + + "\x14max-updates-per-tick\x12\x19Number of docs per update\x1a\x03300R\x11maxUpdatesPerTick\x12[\n" + + "\x10namespace_prefix\x18\f \x01(\tB0\x82\xb5\x18,\n" + + "\x10namespace-prefix\x12\x14Prefix for namespace\x1a\x02nsR\x0fnamespacePrefix\x12A\n" + + "\apayload\x18\r \x03(\tB'\x82\xb5\x18#\x12!Payload for all items (key:value)R\apayload\x12j\n" + + "\fpayload_json\x18\x0e \x01(\tBG\x82\xb5\x18C\n" + + "\fpayload-json\x123Payload for all items as a json (overrides payload)R\vpayloadJson:\x15\x82\xb5\x18\x11\n" + + "\x0f/dev/fakesourceBKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_fakesource_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_fakesource_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_fakesource_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_fakesource_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_fakesource_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_fakesource_proto_rawDesc), len(file_adiom_commands_connectors_v1_fakesource_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_fakesource_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_fakesource_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_fakesource_proto_goTypes = []any{ + (*FakeSourceFlags)(nil), // 0: adiom.commands.connectors.v1.FakeSourceFlags + (*durationpb.Duration)(nil), // 1: google.protobuf.Duration +} +var file_adiom_commands_connectors_v1_fakesource_proto_depIdxs = []int32{ + 1, // 0: adiom.commands.connectors.v1.FakeSourceFlags.sleep:type_name -> google.protobuf.Duration + 1, // 1: adiom.commands.connectors.v1.FakeSourceFlags.jitter:type_name -> google.protobuf.Duration + 1, // 2: adiom.commands.connectors.v1.FakeSourceFlags.update_duration:type_name -> google.protobuf.Duration + 1, // 3: adiom.commands.connectors.v1.FakeSourceFlags.stream_tick:type_name -> google.protobuf.Duration + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_fakesource_proto_init() } +func file_adiom_commands_connectors_v1_fakesource_proto_init() { + if File_adiom_commands_connectors_v1_fakesource_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_fakesource_proto_rawDesc), len(file_adiom_commands_connectors_v1_fakesource_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_fakesource_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_fakesource_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_fakesource_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_fakesource_proto = out.File + file_adiom_commands_connectors_v1_fakesource_proto_goTypes = nil + file_adiom_commands_connectors_v1_fakesource_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/fakesource_commandargs.go b/gen/adiom/commands/connectors/v1/fakesource_commandargs.go new file mode 100644 index 00000000..750e12aa --- /dev/null +++ b/gen/adiom/commands/connectors/v1/fakesource_commandargs.go @@ -0,0 +1,117 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import ( + durationpb "google.golang.org/protobuf/types/known/durationpb" + time "time" +) + +import "github.com/urfave/cli/v2" + +func FakeSourceFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.DurationFlag{ + Name: "sleep", + Usage: "Sleep time between requests", + }, + &cli.DurationFlag{ + Name: "jitter", + Usage: "Additional jitter to certain durations", + }, + &cli.DurationFlag{ + Name: "update-duration", + Usage: "Time for each update-batch-size to be produced", + Value: func() time.Duration { d, _ := time.ParseDuration("400ms"); return d }(), + }, + &cli.DurationFlag{ + Name: "stream-tick", + Usage: "Polling time to check for stream updates", + Value: func() time.Duration { d, _ := time.ParseDuration("1s"); return d }(), + }, + &cli.IntFlag{ + Name: "num-namespaces", + Usage: "Number of namespaces (index starts at 0)", + Value: 2, + }, + &cli.IntFlag{ + Name: "num-partitions-per-namespace", + Usage: "Number of partitions per namespace", + Value: 3, + }, + &cli.IntFlag{ + Name: "num-update-partitions-per-namespace", + Usage: "Number of update partitions per namespace. If set to 0, use a single stream for all.", + }, + &cli.Int64Flag{ + Name: "num-docs-per-partition", + Usage: "Number of docs per partition", + Value: 1000, + }, + &cli.IntFlag{ + Name: "batch-size", + Usage: "Number of docs per batch", + Value: 150, + }, + &cli.IntFlag{ + Name: "update-batch-size", + Usage: "Number of docs per update", + Value: 50, + }, + &cli.IntFlag{ + Name: "max-updates-per-tick", + Usage: "Number of docs per update", + Value: 300, + }, + &cli.StringFlag{ + Name: "namespace-prefix", + Usage: "Prefix for namespace", + Value: "ns", + }, + &cli.StringSliceFlag{ + Name: "payload", + Usage: "Payload for all items (key:value)", + }, + &cli.StringFlag{ + Name: "payload-json", + Usage: "Payload for all items as a json (overrides payload)", + }, + } +} + +func ParseFakeSourceFlags(c *cli.Context) (*FakeSourceFlags, error) { + cfg := &FakeSourceFlags{} + cfg.Sleep = durationpb.New(c.Duration("sleep")) + cfg.Jitter = durationpb.New(c.Duration("jitter")) + cfg.UpdateDuration = durationpb.New(c.Duration("update-duration")) + cfg.StreamTick = durationpb.New(c.Duration("stream-tick")) + cfg.NumNamespaces = int32(c.Int("num-namespaces")) + cfg.NumPartitionsPerNamespace = int32(c.Int("num-partitions-per-namespace")) + cfg.NumUpdatePartitionsPerNamespace = int32(c.Int("num-update-partitions-per-namespace")) + cfg.NumDocsPerPartition = c.Int64("num-docs-per-partition") + cfg.BatchSize = int32(c.Int("batch-size")) + cfg.UpdateBatchSize = int32(c.Int("update-batch-size")) + cfg.MaxUpdatesPerTick = int32(c.Int("max-updates-per-tick")) + cfg.NamespacePrefix = c.String("namespace-prefix") + cfg.Payload = c.StringSlice("payload") + cfg.PayloadJson = c.String("payload-json") + return cfg, nil +} + +func NewFakeSourceFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "/dev/fakesource", + Flags: FakeSourceFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewFakeSourceFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "/dev/fakesource", + Flags: FakeSourceFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/file.pb.go b/gen/adiom/commands/connectors/v1/file.pb.go new file mode 100644 index 00000000..11883129 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/file.pb.go @@ -0,0 +1,145 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/file.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Format string `protobuf:"bytes,1,opt,name=format,proto3" json:"format,omitempty"` + Delimiter string `protobuf:"bytes,2,opt,name=delimiter,proto3" json:"delimiter,omitempty"` + BatchSize int32 `protobuf:"varint,3,opt,name=batch_size,json=batchSize,proto3" json:"batch_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileFlags) Reset() { + *x = FileFlags{} + mi := &file_adiom_commands_connectors_v1_file_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileFlags) ProtoMessage() {} + +func (x *FileFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_file_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileFlags.ProtoReflect.Descriptor instead. +func (*FileFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_file_proto_rawDescGZIP(), []int{0} +} + +func (x *FileFlags) GetFormat() string { + if x != nil { + return x.Format + } + return "" +} + +func (x *FileFlags) GetDelimiter() string { + if x != nil { + return x.Delimiter + } + return "" +} + +func (x *FileFlags) GetBatchSize() int32 { + if x != nil { + return x.BatchSize + } + return 0 +} + +var File_adiom_commands_connectors_v1_file_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_file_proto_rawDesc = "" + + "\n" + + "'adiom/commands/connectors/v1/file.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\"\xdb\x02\n" + + "\tFileFlags\x12X\n" + + "\x06format\x18\x01 \x01(\tB@\x82\xb5\x18<\x125Output format for stored files (only 'csv' supported)\x1a\x03csvR\x06format\x12>\n" + + "\tdelimiter\x18\x02 \x01(\tB \x82\xb5\x18\x1c\x12\x17CSV delimiter character\x1a\x01,R\tdelimiter\x12m\n" + + "\n" + + "batch_size\x18\x03 \x01(\x05BN\x82\xb5\x18J\n" + + "\n" + + "batch-size\x125Number of records to read per batch when listing data\x1a\x0510000R\tbatchSize:E\x82\xb5\x18A\n" + + "\x04file\"9file:///path/to/dir OR file:///path/to/file.csv [options]BKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_file_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_file_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_file_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_file_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_file_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_file_proto_rawDesc), len(file_adiom_commands_connectors_v1_file_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_file_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_file_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_file_proto_goTypes = []any{ + (*FileFlags)(nil), // 0: adiom.commands.connectors.v1.FileFlags +} +var file_adiom_commands_connectors_v1_file_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_file_proto_init() } +func file_adiom_commands_connectors_v1_file_proto_init() { + if File_adiom_commands_connectors_v1_file_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_file_proto_rawDesc), len(file_adiom_commands_connectors_v1_file_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_file_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_file_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_file_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_file_proto = out.File + file_adiom_commands_connectors_v1_file_proto_goTypes = nil + file_adiom_commands_connectors_v1_file_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/file_commandargs.go b/gen/adiom/commands/connectors/v1/file_commandargs.go new file mode 100644 index 00000000..b4406286 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/file_commandargs.go @@ -0,0 +1,53 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import "github.com/urfave/cli/v2" + +func FileFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "format", + Usage: "Output format for stored files (only 'csv' supported)", + Value: "csv", + }, + &cli.StringFlag{ + Name: "delimiter", + Usage: "CSV delimiter character", + Value: ",", + }, + &cli.IntFlag{ + Name: "batch-size", + Usage: "Number of records to read per batch when listing data", + Value: 10000, + }, + } +} + +func ParseFileFlags(c *cli.Context) (*FileFlags, error) { + cfg := &FileFlags{} + cfg.Format = c.String("format") + cfg.Delimiter = c.String("delimiter") + cfg.BatchSize = int32(c.Int("batch-size")) + return cfg, nil +} + +func NewFileFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "file", + Usage: "file:///path/to/dir OR file:///path/to/file.csv [options]", + Flags: FileFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewFileFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "file", + Usage: "file:///path/to/dir OR file:///path/to/file.csv [options]", + Flags: FileFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/grpc.pb.go b/gen/adiom/commands/connectors/v1/grpc.pb.go new file mode 100644 index 00000000..8504b47d --- /dev/null +++ b/gen/adiom/commands/connectors/v1/grpc.pb.go @@ -0,0 +1,133 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/grpc.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GrpcFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Insecure bool `protobuf:"varint,1,opt,name=insecure,proto3" json:"insecure,omitempty"` + Gzip bool `protobuf:"varint,2,opt,name=gzip,proto3" json:"gzip,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrpcFlags) Reset() { + *x = GrpcFlags{} + mi := &file_adiom_commands_connectors_v1_grpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrpcFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcFlags) ProtoMessage() {} + +func (x *GrpcFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_grpc_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcFlags.ProtoReflect.Descriptor instead. +func (*GrpcFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_grpc_proto_rawDescGZIP(), []int{0} +} + +func (x *GrpcFlags) GetInsecure() bool { + if x != nil { + return x.Insecure + } + return false +} + +func (x *GrpcFlags) GetGzip() bool { + if x != nil { + return x.Gzip + } + return false +} + +var File_adiom_commands_connectors_v1_grpc_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_grpc_proto_rawDesc = "" + + "\n" + + "'adiom/commands/connectors/v1/grpc.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\"\x9d\x01\n" + + "\tGrpcFlags\x125\n" + + "\binsecure\x18\x01 \x01(\bB\x19\x82\xb5\x18\x15\x12\x13Connect without TLSR\binsecure\x12.\n" + + "\x04gzip\x18\x02 \x01(\bB\x1a\x82\xb5\x18\x16\x12\x14Use gzip on requestsR\x04gzip:)\x82\xb5\x18%\n" + + "\x04grpc\"\x1dgrpc://address:port [options]BKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_grpc_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_grpc_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_grpc_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_grpc_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_grpc_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_grpc_proto_rawDesc), len(file_adiom_commands_connectors_v1_grpc_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_grpc_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_grpc_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_grpc_proto_goTypes = []any{ + (*GrpcFlags)(nil), // 0: adiom.commands.connectors.v1.GrpcFlags +} +var file_adiom_commands_connectors_v1_grpc_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_grpc_proto_init() } +func file_adiom_commands_connectors_v1_grpc_proto_init() { + if File_adiom_commands_connectors_v1_grpc_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_grpc_proto_rawDesc), len(file_adiom_commands_connectors_v1_grpc_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_grpc_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_grpc_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_grpc_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_grpc_proto = out.File + file_adiom_commands_connectors_v1_grpc_proto_goTypes = nil + file_adiom_commands_connectors_v1_grpc_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/grpc_commandargs.go b/gen/adiom/commands/connectors/v1/grpc_commandargs.go new file mode 100644 index 00000000..dbe36d02 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/grpc_commandargs.go @@ -0,0 +1,45 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import "github.com/urfave/cli/v2" + +func GrpcFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "insecure", + Usage: "Connect without TLS", + }, + &cli.BoolFlag{ + Name: "gzip", + Usage: "Use gzip on requests", + }, + } +} + +func ParseGrpcFlags(c *cli.Context) (*GrpcFlags, error) { + cfg := &GrpcFlags{} + cfg.Insecure = c.Bool("insecure") + cfg.Gzip = c.Bool("gzip") + return cfg, nil +} + +func NewGrpcFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "grpc", + Usage: "grpc://address:port [options]", + Flags: GrpcFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewGrpcFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "grpc", + Usage: "grpc://address:port [options]", + Flags: GrpcFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/kafka.pb.go b/gen/adiom/commands/connectors/v1/kafka.pb.go new file mode 100644 index 00000000..41da1ae9 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/kafka.pb.go @@ -0,0 +1,280 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/kafka.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type KafkaSrcFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Brokers []string `protobuf:"bytes,1,rep,name=brokers,proto3" json:"brokers,omitempty"` + Topics []string `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"` + TopicMappings []string `protobuf:"bytes,3,rep,name=topic_mappings,json=topicMappings,proto3" json:"topic_mappings,omitempty"` + SaslUser string `protobuf:"bytes,4,opt,name=sasl_user,json=saslUser,proto3" json:"sasl_user,omitempty"` + SaslPassword string `protobuf:"bytes,5,opt,name=sasl_password,json=saslPassword,proto3" json:"sasl_password,omitempty"` + Offset int64 `protobuf:"varint,6,opt,name=offset,proto3" json:"offset,omitempty"` + DataType string `protobuf:"bytes,7,opt,name=data_type,json=dataType,proto3" json:"data_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *KafkaSrcFlags) Reset() { + *x = KafkaSrcFlags{} + mi := &file_adiom_commands_connectors_v1_kafka_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *KafkaSrcFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*KafkaSrcFlags) ProtoMessage() {} + +func (x *KafkaSrcFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_kafka_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use KafkaSrcFlags.ProtoReflect.Descriptor instead. +func (*KafkaSrcFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_kafka_proto_rawDescGZIP(), []int{0} +} + +func (x *KafkaSrcFlags) GetBrokers() []string { + if x != nil { + return x.Brokers + } + return nil +} + +func (x *KafkaSrcFlags) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +func (x *KafkaSrcFlags) GetTopicMappings() []string { + if x != nil { + return x.TopicMappings + } + return nil +} + +func (x *KafkaSrcFlags) GetSaslUser() string { + if x != nil { + return x.SaslUser + } + return "" +} + +func (x *KafkaSrcFlags) GetSaslPassword() string { + if x != nil { + return x.SaslPassword + } + return "" +} + +func (x *KafkaSrcFlags) GetOffset() int64 { + if x != nil { + return x.Offset + } + return 0 +} + +func (x *KafkaSrcFlags) GetDataType() string { + if x != nil { + return x.DataType + } + return "" +} + +type KafkaDstFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Brokers []string `protobuf:"bytes,1,rep,name=brokers,proto3" json:"brokers,omitempty"` + DefaultTopic string `protobuf:"bytes,2,opt,name=default_topic,json=defaultTopic,proto3" json:"default_topic,omitempty"` + NamespaceTopic []string `protobuf:"bytes,3,rep,name=namespace_topic,json=namespaceTopic,proto3" json:"namespace_topic,omitempty"` + SaslUser string `protobuf:"bytes,4,opt,name=sasl_user,json=saslUser,proto3" json:"sasl_user,omitempty"` + SaslPassword string `protobuf:"bytes,5,opt,name=sasl_password,json=saslPassword,proto3" json:"sasl_password,omitempty"` + DataType string `protobuf:"bytes,6,opt,name=data_type,json=dataType,proto3" json:"data_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *KafkaDstFlags) Reset() { + *x = KafkaDstFlags{} + mi := &file_adiom_commands_connectors_v1_kafka_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *KafkaDstFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*KafkaDstFlags) ProtoMessage() {} + +func (x *KafkaDstFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_kafka_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use KafkaDstFlags.ProtoReflect.Descriptor instead. +func (*KafkaDstFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_kafka_proto_rawDescGZIP(), []int{1} +} + +func (x *KafkaDstFlags) GetBrokers() []string { + if x != nil { + return x.Brokers + } + return nil +} + +func (x *KafkaDstFlags) GetDefaultTopic() string { + if x != nil { + return x.DefaultTopic + } + return "" +} + +func (x *KafkaDstFlags) GetNamespaceTopic() []string { + if x != nil { + return x.NamespaceTopic + } + return nil +} + +func (x *KafkaDstFlags) GetSaslUser() string { + if x != nil { + return x.SaslUser + } + return "" +} + +func (x *KafkaDstFlags) GetSaslPassword() string { + if x != nil { + return x.SaslPassword + } + return "" +} + +func (x *KafkaDstFlags) GetDataType() string { + if x != nil { + return x.DataType + } + return "" +} + +var File_adiom_commands_connectors_v1_kafka_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_kafka_proto_rawDesc = "" + + "\n" + + "(adiom/commands/connectors/v1/kafka.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\"\x91\x03\n" + + "\rKafkaSrcFlags\x12\x18\n" + + "\abrokers\x18\x01 \x03(\tR\abrokers\x12\x16\n" + + "\x06topics\x18\x02 \x03(\tR\x06topics\x12;\n" + + "\x0etopic_mappings\x18\x03 \x03(\tB\x14\x82\xb5\x18\x10\n" + + "\x0etopic-mappingsR\rtopicMappings\x12,\n" + + "\tsasl_user\x18\x04 \x01(\tB\x0f\x82\xb5\x18\v\n" + + "\tsasl-userR\bsaslUser\x128\n" + + "\rsasl_password\x18\x05 \x01(\tB\x13\x82\xb5\x18\x0f\n" + + "\rsasl-passwordR\fsaslPassword\x12T\n" + + "\x06offset\x18\x06 \x01(\x03B<\x82\xb5\x188\x122Custom offset for kafka (a time, or -2 for oldest)\x1a\x02-1R\x06offset\x12B\n" + + "\tdata_type\x18\a \x01(\tB%\x82\xb5\x18!\n" + + "\tdata-type\x1a\x14DATA_TYPE_MONGO_BSONR\bdataType:\x0f\x82\xb5\x18\v\n" + + "\tkafka-src\"\xe0\x02\n" + + "\rKafkaDstFlags\x12\x18\n" + + "\abrokers\x18\x01 \x03(\tR\abrokers\x128\n" + + "\rdefault_topic\x18\x02 \x01(\tB\x13\x82\xb5\x18\x0f\n" + + "\rdefault-topicR\fdefaultTopic\x12>\n" + + "\x0fnamespace_topic\x18\x03 \x03(\tB\x15\x82\xb5\x18\x11\n" + + "\x0fnamespace-topicR\x0enamespaceTopic\x12,\n" + + "\tsasl_user\x18\x04 \x01(\tB\x0f\x82\xb5\x18\v\n" + + "\tsasl-userR\bsaslUser\x128\n" + + "\rsasl_password\x18\x05 \x01(\tB\x13\x82\xb5\x18\x0f\n" + + "\rsasl-passwordR\fsaslPassword\x12B\n" + + "\tdata_type\x18\x06 \x01(\tB%\x82\xb5\x18!\n" + + "\tdata-type\x1a\x14DATA_TYPE_MONGO_BSONR\bdataType:\x0f\x82\xb5\x18\v\n" + + "\tkafka-dstBKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_kafka_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_kafka_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_kafka_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_kafka_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_kafka_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_kafka_proto_rawDesc), len(file_adiom_commands_connectors_v1_kafka_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_kafka_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_kafka_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_adiom_commands_connectors_v1_kafka_proto_goTypes = []any{ + (*KafkaSrcFlags)(nil), // 0: adiom.commands.connectors.v1.KafkaSrcFlags + (*KafkaDstFlags)(nil), // 1: adiom.commands.connectors.v1.KafkaDstFlags +} +var file_adiom_commands_connectors_v1_kafka_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_kafka_proto_init() } +func file_adiom_commands_connectors_v1_kafka_proto_init() { + if File_adiom_commands_connectors_v1_kafka_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_kafka_proto_rawDesc), len(file_adiom_commands_connectors_v1_kafka_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_kafka_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_kafka_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_kafka_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_kafka_proto = out.File + file_adiom_commands_connectors_v1_kafka_proto_goTypes = nil + file_adiom_commands_connectors_v1_kafka_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/kafka_commandargs.go b/gen/adiom/commands/connectors/v1/kafka_commandargs.go new file mode 100644 index 00000000..485fff00 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/kafka_commandargs.go @@ -0,0 +1,117 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import "github.com/urfave/cli/v2" + +func KafkaSrcFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.StringSliceFlag{ + Name: "brokers", + }, + &cli.StringSliceFlag{ + Name: "topics", + }, + &cli.StringSliceFlag{ + Name: "topic-mappings", + }, + &cli.StringFlag{ + Name: "sasl-user", + }, + &cli.StringFlag{ + Name: "sasl-password", + }, + &cli.Int64Flag{ + Name: "offset", + Usage: "Custom offset for kafka (a time, or -2 for oldest)", + Value: -1, + }, + &cli.StringFlag{ + Name: "data-type", + Value: "DATA_TYPE_MONGO_BSON", + }, + } +} + +func ParseKafkaSrcFlags(c *cli.Context) (*KafkaSrcFlags, error) { + cfg := &KafkaSrcFlags{} + cfg.Brokers = c.StringSlice("brokers") + cfg.Topics = c.StringSlice("topics") + cfg.TopicMappings = c.StringSlice("topic-mappings") + cfg.SaslUser = c.String("sasl-user") + cfg.SaslPassword = c.String("sasl-password") + cfg.Offset = c.Int64("offset") + cfg.DataType = c.String("data-type") + return cfg, nil +} + +func NewKafkaSrcFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "kafka-src", + Flags: KafkaSrcFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewKafkaSrcFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "kafka-src", + Flags: KafkaSrcFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} + +func KafkaDstFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.StringSliceFlag{ + Name: "brokers", + }, + &cli.StringFlag{ + Name: "default-topic", + }, + &cli.StringSliceFlag{ + Name: "namespace-topic", + }, + &cli.StringFlag{ + Name: "sasl-user", + }, + &cli.StringFlag{ + Name: "sasl-password", + }, + &cli.StringFlag{ + Name: "data-type", + Value: "DATA_TYPE_MONGO_BSON", + }, + } +} + +func ParseKafkaDstFlags(c *cli.Context) (*KafkaDstFlags, error) { + cfg := &KafkaDstFlags{} + cfg.Brokers = c.StringSlice("brokers") + cfg.DefaultTopic = c.String("default-topic") + cfg.NamespaceTopic = c.StringSlice("namespace-topic") + cfg.SaslUser = c.String("sasl-user") + cfg.SaslPassword = c.String("sasl-password") + cfg.DataType = c.String("data-type") + return cfg, nil +} + +func NewKafkaDstFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "kafka-dst", + Flags: KafkaDstFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewKafkaDstFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "kafka-dst", + Flags: KafkaDstFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/mongo.pb.go b/gen/adiom/commands/connectors/v1/mongo.pb.go new file mode 100644 index 00000000..7ded388f --- /dev/null +++ b/gen/adiom/commands/connectors/v1/mongo.pb.go @@ -0,0 +1,269 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/mongo.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Base flags shared between MongoDB and CosmosDB connectors +type MongoBaseFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + SkipBatchOverwrite bool `protobuf:"varint,1,opt,name=skip_batch_overwrite,json=skipBatchOverwrite,proto3" json:"skip_batch_overwrite,omitempty"` + FullDocumentKey bool `protobuf:"varint,2,opt,name=full_document_key,json=fullDocumentKey,proto3" json:"full_document_key,omitempty"` + ServerTimeout *durationpb.Duration `protobuf:"bytes,3,opt,name=server_timeout,json=serverTimeout,proto3" json:"server_timeout,omitempty"` + PingTimeout *durationpb.Duration `protobuf:"bytes,4,opt,name=ping_timeout,json=pingTimeout,proto3" json:"ping_timeout,omitempty"` + WriterBatchSize int32 `protobuf:"varint,5,opt,name=writer_batch_size,json=writerBatchSize,proto3" json:"writer_batch_size,omitempty"` + DocPartition int64 `protobuf:"varint,6,opt,name=doc_partition,json=docPartition,proto3" json:"doc_partition,omitempty"` + MaxPageSize int32 `protobuf:"varint,7,opt,name=max_page_size,json=maxPageSize,proto3" json:"max_page_size,omitempty"` + InitialSyncQuery string `protobuf:"bytes,8,opt,name=initial_sync_query,json=initialSyncQuery,proto3" json:"initial_sync_query,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MongoBaseFlags) Reset() { + *x = MongoBaseFlags{} + mi := &file_adiom_commands_connectors_v1_mongo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MongoBaseFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MongoBaseFlags) ProtoMessage() {} + +func (x *MongoBaseFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_mongo_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MongoBaseFlags.ProtoReflect.Descriptor instead. +func (*MongoBaseFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_mongo_proto_rawDescGZIP(), []int{0} +} + +func (x *MongoBaseFlags) GetSkipBatchOverwrite() bool { + if x != nil { + return x.SkipBatchOverwrite + } + return false +} + +func (x *MongoBaseFlags) GetFullDocumentKey() bool { + if x != nil { + return x.FullDocumentKey + } + return false +} + +func (x *MongoBaseFlags) GetServerTimeout() *durationpb.Duration { + if x != nil { + return x.ServerTimeout + } + return nil +} + +func (x *MongoBaseFlags) GetPingTimeout() *durationpb.Duration { + if x != nil { + return x.PingTimeout + } + return nil +} + +func (x *MongoBaseFlags) GetWriterBatchSize() int32 { + if x != nil { + return x.WriterBatchSize + } + return 0 +} + +func (x *MongoBaseFlags) GetDocPartition() int64 { + if x != nil { + return x.DocPartition + } + return 0 +} + +func (x *MongoBaseFlags) GetMaxPageSize() int32 { + if x != nil { + return x.MaxPageSize + } + return 0 +} + +func (x *MongoBaseFlags) GetInitialSyncQuery() string { + if x != nil { + return x.InitialSyncQuery + } + return "" +} + +// Full MongoDB flags = base + MongoDB-specific extras +type MongoFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Base *MongoBaseFlags `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + SampleFactor int32 `protobuf:"varint,2,opt,name=sample_factor,json=sampleFactor,proto3" json:"sample_factor,omitempty"` + PerNamespaceStreams bool `protobuf:"varint,3,opt,name=per_namespace_streams,json=perNamespaceStreams,proto3" json:"per_namespace_streams,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MongoFlags) Reset() { + *x = MongoFlags{} + mi := &file_adiom_commands_connectors_v1_mongo_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MongoFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MongoFlags) ProtoMessage() {} + +func (x *MongoFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_mongo_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MongoFlags.ProtoReflect.Descriptor instead. +func (*MongoFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_mongo_proto_rawDescGZIP(), []int{1} +} + +func (x *MongoFlags) GetBase() *MongoBaseFlags { + if x != nil { + return x.Base + } + return nil +} + +func (x *MongoFlags) GetSampleFactor() int32 { + if x != nil { + return x.SampleFactor + } + return 0 +} + +func (x *MongoFlags) GetPerNamespaceStreams() bool { + if x != nil { + return x.PerNamespaceStreams + } + return false +} + +var File_adiom_commands_connectors_v1_mongo_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_mongo_proto_rawDesc = "" + + "\n" + + "(adiom/commands/connectors/v1/mongo.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\x1a\x1egoogle/protobuf/duration.proto\"\xc3\x05\n" + + "\x0eMongoBaseFlags\x120\n" + + "\x14skip_batch_overwrite\x18\x01 \x01(\bR\x12skipBatchOverwrite\x12~\n" + + "\x11full_document_key\x18\x02 \x01(\bBR\x82\xb5\x18N\x12Luses the full document key instead of just _id (except for batch overwrites)R\x0ffullDocumentKey\x12V\n" + + "\x0eserver_timeout\x18\x03 \x01(\v2\x19.google.protobuf.DurationB\x14\x82\xb5\x18\x10\n" + + "\x0eserver-timeoutR\rserverTimeout\x12P\n" + + "\fping_timeout\x18\x04 \x01(\v2\x19.google.protobuf.DurationB\x12\x82\xb5\x18\x0e\n" + + "\fping-timeoutR\vpingTimeout\x12C\n" + + "\x11writer_batch_size\x18\x05 \x01(\x05B\x17\x82\xb5\x18\x13\n" + + "\x11writer-batch-sizeR\x0fwriterBatchSize\x12?\n" + + "\rdoc_partition\x18\x06 \x01(\x03B\x1a\x82\xb5\x18\x16\n" + + "\rdoc-partition\x1a\x0550000R\fdocPartition\x127\n" + + "\rmax_page_size\x18\a \x01(\x05B\x13\x82\xb5\x18\x0f\n" + + "\rmax-page-sizeR\vmaxPageSize\x12\x84\x01\n" + + "\x12initial_sync_query\x18\b \x01(\tBV\x82\xb5\x18R\n" + + "\x12initial-sync-query\x129query filter for the initial data copy (v2 Extended JSON):\x01qR\x10initialSyncQuery:\x0f\x82\xb5\x18\v\n" + + "\tMongoBase\"\xe0\x02\n" + + "\n" + + "MongoFlags\x12@\n" + + "\x04base\x18\x01 \x01(\v2,.adiom.commands.connectors.v1.MongoBaseFlagsR\x04base\x12c\n" + + "\rsample_factor\x18\x02 \x01(\x05B>\x82\xb5\x18:\n" + + "\rsample-factor\x12%Number of extra samples per partition\x1a\x0210R\fsampleFactor\x12u\n" + + "\x15per_namespace_streams\x18\x03 \x01(\bBA\x82\xb5\x18=\n" + + "\x15per-namespace-streams\x12$Each namespace has a separate streamR\x13perNamespaceStreams:4\x82\xb5\x180\n" + + "\aMongoDB\"%mongodb://connection-string [options]BKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_mongo_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_mongo_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_mongo_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_mongo_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_mongo_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_mongo_proto_rawDesc), len(file_adiom_commands_connectors_v1_mongo_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_mongo_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_mongo_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_adiom_commands_connectors_v1_mongo_proto_goTypes = []any{ + (*MongoBaseFlags)(nil), // 0: adiom.commands.connectors.v1.MongoBaseFlags + (*MongoFlags)(nil), // 1: adiom.commands.connectors.v1.MongoFlags + (*durationpb.Duration)(nil), // 2: google.protobuf.Duration +} +var file_adiom_commands_connectors_v1_mongo_proto_depIdxs = []int32{ + 2, // 0: adiom.commands.connectors.v1.MongoBaseFlags.server_timeout:type_name -> google.protobuf.Duration + 2, // 1: adiom.commands.connectors.v1.MongoBaseFlags.ping_timeout:type_name -> google.protobuf.Duration + 0, // 2: adiom.commands.connectors.v1.MongoFlags.base:type_name -> adiom.commands.connectors.v1.MongoBaseFlags + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_mongo_proto_init() } +func file_adiom_commands_connectors_v1_mongo_proto_init() { + if File_adiom_commands_connectors_v1_mongo_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_mongo_proto_rawDesc), len(file_adiom_commands_connectors_v1_mongo_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_mongo_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_mongo_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_mongo_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_mongo_proto = out.File + file_adiom_commands_connectors_v1_mongo_proto_goTypes = nil + file_adiom_commands_connectors_v1_mongo_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/mongo_commandargs.go b/gen/adiom/commands/connectors/v1/mongo_commandargs.go new file mode 100644 index 00000000..cf290a98 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/mongo_commandargs.go @@ -0,0 +1,151 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import ( + durationpb "google.golang.org/protobuf/types/known/durationpb" +) + +import "github.com/urfave/cli/v2" + +func MongoBaseFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "skip-batch-overwrite", + }, + &cli.BoolFlag{ + Name: "full-document-key", + Usage: "uses the full document key instead of just _id (except for batch overwrites)", + }, + &cli.DurationFlag{ + Name: "server-timeout", + }, + &cli.DurationFlag{ + Name: "ping-timeout", + }, + &cli.IntFlag{ + Name: "writer-batch-size", + }, + &cli.Int64Flag{ + Name: "doc-partition", + Value: 50000, + }, + &cli.IntFlag{ + Name: "max-page-size", + }, + &cli.StringFlag{ + Name: "initial-sync-query", + Usage: "query filter for the initial data copy (v2 Extended JSON)", + Aliases: []string{"q"}, + }, + } +} + +func ParseMongoBaseFlags(c *cli.Context) (*MongoBaseFlags, error) { + cfg := &MongoBaseFlags{} + cfg.SkipBatchOverwrite = c.Bool("skip-batch-overwrite") + cfg.FullDocumentKey = c.Bool("full-document-key") + cfg.ServerTimeout = durationpb.New(c.Duration("server-timeout")) + cfg.PingTimeout = durationpb.New(c.Duration("ping-timeout")) + cfg.WriterBatchSize = int32(c.Int("writer-batch-size")) + cfg.DocPartition = c.Int64("doc-partition") + cfg.MaxPageSize = int32(c.Int("max-page-size")) + cfg.InitialSyncQuery = c.String("initial-sync-query") + return cfg, nil +} + +func NewMongoBaseFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "MongoBase", + Flags: MongoBaseFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewMongoBaseFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "MongoBase", + Flags: MongoBaseFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} + +func MongoFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "skip-batch-overwrite", + }, + &cli.BoolFlag{ + Name: "full-document-key", + Usage: "uses the full document key instead of just _id (except for batch overwrites)", + }, + &cli.DurationFlag{ + Name: "server-timeout", + }, + &cli.DurationFlag{ + Name: "ping-timeout", + }, + &cli.IntFlag{ + Name: "writer-batch-size", + }, + &cli.Int64Flag{ + Name: "doc-partition", + Value: 50000, + }, + &cli.IntFlag{ + Name: "max-page-size", + }, + &cli.StringFlag{ + Name: "initial-sync-query", + Usage: "query filter for the initial data copy (v2 Extended JSON)", + Aliases: []string{"q"}, + }, + &cli.IntFlag{ + Name: "sample-factor", + Usage: "Number of extra samples per partition", + Value: 10, + }, + &cli.BoolFlag{ + Name: "per-namespace-streams", + Usage: "Each namespace has a separate stream", + }, + } +} + +func ParseMongoFlags(c *cli.Context) (*MongoFlags, error) { + cfg := &MongoFlags{} + cfg.Base = &MongoBaseFlags{} + cfg.Base.SkipBatchOverwrite = c.Bool("skip-batch-overwrite") + cfg.Base.FullDocumentKey = c.Bool("full-document-key") + cfg.Base.ServerTimeout = durationpb.New(c.Duration("server-timeout")) + cfg.Base.PingTimeout = durationpb.New(c.Duration("ping-timeout")) + cfg.Base.WriterBatchSize = int32(c.Int("writer-batch-size")) + cfg.Base.DocPartition = c.Int64("doc-partition") + cfg.Base.MaxPageSize = int32(c.Int("max-page-size")) + cfg.Base.InitialSyncQuery = c.String("initial-sync-query") + cfg.SampleFactor = int32(c.Int("sample-factor")) + cfg.PerNamespaceStreams = c.Bool("per-namespace-streams") + return cfg, nil +} + +func NewMongoFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "MongoDB", + Usage: "mongodb://connection-string [options]", + Flags: MongoFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewMongoFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "MongoDB", + Usage: "mongodb://connection-string [options]", + Flags: MongoFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/postgres.pb.go b/gen/adiom/commands/connectors/v1/postgres.pb.go new file mode 100644 index 00000000..8a6d4400 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/postgres.pb.go @@ -0,0 +1,220 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/postgres.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type PostgresFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + StreamMaxBatchWait *durationpb.Duration `protobuf:"bytes,2,opt,name=stream_max_batch_wait,json=streamMaxBatchWait,proto3" json:"stream_max_batch_wait,omitempty"` + StreamMaxBatchSize int32 `protobuf:"varint,3,opt,name=stream_max_batch_size,json=streamMaxBatchSize,proto3" json:"stream_max_batch_size,omitempty"` + StreamFlushDelay *durationpb.Duration `protobuf:"bytes,4,opt,name=stream_flush_delay,json=streamFlushDelay,proto3" json:"stream_flush_delay,omitempty"` + Manual bool `protobuf:"varint,5,opt,name=manual,proto3" json:"manual,omitempty"` + ReplicationSlot string `protobuf:"bytes,6,opt,name=replication_slot,json=replicationSlot,proto3" json:"replication_slot,omitempty"` + PublicationName string `protobuf:"bytes,7,opt,name=publication_name,json=publicationName,proto3" json:"publication_name,omitempty"` + EstimatedCountThreshold int64 `protobuf:"varint,8,opt,name=estimated_count_threshold,json=estimatedCountThreshold,proto3" json:"estimated_count_threshold,omitempty"` + DocPartition int64 `protobuf:"varint,9,opt,name=doc_partition,json=docPartition,proto3" json:"doc_partition,omitempty"` + EnableReplicaMode bool `protobuf:"varint,10,opt,name=enable_replica_mode,json=enableReplicaMode,proto3" json:"enable_replica_mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PostgresFlags) Reset() { + *x = PostgresFlags{} + mi := &file_adiom_commands_connectors_v1_postgres_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PostgresFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PostgresFlags) ProtoMessage() {} + +func (x *PostgresFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_postgres_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PostgresFlags.ProtoReflect.Descriptor instead. +func (*PostgresFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_postgres_proto_rawDescGZIP(), []int{0} +} + +func (x *PostgresFlags) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *PostgresFlags) GetStreamMaxBatchWait() *durationpb.Duration { + if x != nil { + return x.StreamMaxBatchWait + } + return nil +} + +func (x *PostgresFlags) GetStreamMaxBatchSize() int32 { + if x != nil { + return x.StreamMaxBatchSize + } + return 0 +} + +func (x *PostgresFlags) GetStreamFlushDelay() *durationpb.Duration { + if x != nil { + return x.StreamFlushDelay + } + return nil +} + +func (x *PostgresFlags) GetManual() bool { + if x != nil { + return x.Manual + } + return false +} + +func (x *PostgresFlags) GetReplicationSlot() string { + if x != nil { + return x.ReplicationSlot + } + return "" +} + +func (x *PostgresFlags) GetPublicationName() string { + if x != nil { + return x.PublicationName + } + return "" +} + +func (x *PostgresFlags) GetEstimatedCountThreshold() int64 { + if x != nil { + return x.EstimatedCountThreshold + } + return 0 +} + +func (x *PostgresFlags) GetDocPartition() int64 { + if x != nil { + return x.DocPartition + } + return 0 +} + +func (x *PostgresFlags) GetEnableReplicaMode() bool { + if x != nil { + return x.EnableReplicaMode + } + return false +} + +var File_adiom_commands_connectors_v1_postgres_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_postgres_proto_rawDesc = "" + + "\n" + + "+adiom/commands/connectors/v1/postgres.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\x1a\x1egoogle/protobuf/duration.proto\"\xde\v\n" + + "\rPostgresFlags\x12k\n" + + "\tpage_size\x18\x01 \x01(\x05BN\x82\xb5\x18J\n" + + "\tpage-size\x127Specify pagination limit when fetching for initial sync\x1a\x041000R\bpageSize\x12\x9d\x01\n" + + "\x15stream_max_batch_wait\x18\x02 \x01(\v2\x19.google.protobuf.DurationBO\x82\xb5\x18K\n" + + "\x15stream-max-batch-wait\x12.Force flush a stream batch after this interval\x1a\x025sR\x12streamMaxBatchWait\x12}\n" + + "\x15stream_max_batch_size\x18\x03 \x01(\x05BJ\x82\xb5\x18F\n" + + "\x15stream-max-batch-size\x12(Force flush a stream batch at this limit\x1a\x03100R\x12streamMaxBatchSize\x12\xed\x01\n" + + "\x12stream_flush_delay\x18\x04 \x01(\v2\x19.google.protobuf.DurationB\xa3\x01\x82\xb5\x18\x9e\x01\n" + + "\x12stream-flush-delay\x12\x83\x01Delay before notifying postgres backend of stream progress. This should be comfortably larger than saving streaming cursor updates.\x1a\x023mR\x10streamFlushDelay\x12\xca\x01\n" + + "\x06manual\x18\x05 \x01(\bB\xb1\x01\x82\xb5\x18\xac\x01\x12\xa9\x01Use to not recreate replication slot and publication (e.g. if you are managing these outside). You will still need to clean up the slot later even if you don't use this.R\x06manual\x12M\n" + + "\x10replication_slot\x18\x06 \x01(\tB\"\x82\xb5\x18\x1e\n" + + "\x10replication-slot\x1a\n" + + "dsync_slotR\x0freplicationSlot\x12L\n" + + "\x10publication_name\x18\a \x01(\tB!\x82\xb5\x18\x1d\n" + + "\x10publication-name\x1a\tdsync_pubR\x0fpublicationName\x12\x9d\x01\n" + + "\x19estimated_count_threshold\x18\b \x01(\x03Ba\x82\xb5\x18]\n" + + "\x19estimated-count-threshold\x127If estimated count is less than this, try a full count.\x1a\a1000000R\x17estimatedCountThreshold\x12@\n" + + "\rdoc_partition\x18\t \x01(\x03B\x1b\x82\xb5\x18\x17\n" + + "\rdoc-partition\x1a\x06100000R\fdocPartition\x12\xc8\x01\n" + + "\x13enable_replica_mode\x18\n" + + " \x01(\bB\x97\x01\x82\xb5\x18\x92\x01\n" + + "\x13enable-replica-mode\x12uEnable replica mode (SET session_replication_role = 'replica') for sink operations. This disables triggers and rules.\x1a\x04trueR\x11enableReplicaMode::\x82\xb5\x186\n" + + "\bPostgres\"*postgresql://user:pass@host:port [options]BKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_postgres_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_postgres_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_postgres_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_postgres_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_postgres_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_postgres_proto_rawDesc), len(file_adiom_commands_connectors_v1_postgres_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_postgres_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_postgres_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_postgres_proto_goTypes = []any{ + (*PostgresFlags)(nil), // 0: adiom.commands.connectors.v1.PostgresFlags + (*durationpb.Duration)(nil), // 1: google.protobuf.Duration +} +var file_adiom_commands_connectors_v1_postgres_proto_depIdxs = []int32{ + 1, // 0: adiom.commands.connectors.v1.PostgresFlags.stream_max_batch_wait:type_name -> google.protobuf.Duration + 1, // 1: adiom.commands.connectors.v1.PostgresFlags.stream_flush_delay:type_name -> google.protobuf.Duration + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_postgres_proto_init() } +func file_adiom_commands_connectors_v1_postgres_proto_init() { + if File_adiom_commands_connectors_v1_postgres_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_postgres_proto_rawDesc), len(file_adiom_commands_connectors_v1_postgres_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_postgres_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_postgres_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_postgres_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_postgres_proto = out.File + file_adiom_commands_connectors_v1_postgres_proto_goTypes = nil + file_adiom_commands_connectors_v1_postgres_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/postgres_commandargs.go b/gen/adiom/commands/connectors/v1/postgres_commandargs.go new file mode 100644 index 00000000..9bbe36a1 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/postgres_commandargs.go @@ -0,0 +1,96 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import ( + durationpb "google.golang.org/protobuf/types/known/durationpb" + time "time" +) + +import "github.com/urfave/cli/v2" + +func PostgresFlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.IntFlag{ + Name: "page-size", + Usage: "Specify pagination limit when fetching for initial sync", + Value: 1000, + }, + &cli.DurationFlag{ + Name: "stream-max-batch-wait", + Usage: "Force flush a stream batch after this interval", + Value: func() time.Duration { d, _ := time.ParseDuration("5s"); return d }(), + }, + &cli.IntFlag{ + Name: "stream-max-batch-size", + Usage: "Force flush a stream batch at this limit", + Value: 100, + }, + &cli.DurationFlag{ + Name: "stream-flush-delay", + Usage: "Delay before notifying postgres backend of stream progress. This should be comfortably larger than saving streaming cursor updates.", + Value: func() time.Duration { d, _ := time.ParseDuration("3m"); return d }(), + }, + &cli.BoolFlag{ + Name: "manual", + Usage: "Use to not recreate replication slot and publication (e.g. if you are managing these outside). You will still need to clean up the slot later even if you don't use this.", + }, + &cli.StringFlag{ + Name: "replication-slot", + Value: "dsync_slot", + }, + &cli.StringFlag{ + Name: "publication-name", + Value: "dsync_pub", + }, + &cli.Int64Flag{ + Name: "estimated-count-threshold", + Usage: "If estimated count is less than this, try a full count.", + Value: 1000000, + }, + &cli.Int64Flag{ + Name: "doc-partition", + Value: 100000, + }, + &cli.BoolFlag{ + Name: "enable-replica-mode", + Usage: "Enable replica mode (SET session_replication_role = 'replica') for sink operations. This disables triggers and rules.", + Value: true, + }, + } +} + +func ParsePostgresFlags(c *cli.Context) (*PostgresFlags, error) { + cfg := &PostgresFlags{} + cfg.PageSize = int32(c.Int("page-size")) + cfg.StreamMaxBatchWait = durationpb.New(c.Duration("stream-max-batch-wait")) + cfg.StreamMaxBatchSize = int32(c.Int("stream-max-batch-size")) + cfg.StreamFlushDelay = durationpb.New(c.Duration("stream-flush-delay")) + cfg.Manual = c.Bool("manual") + cfg.ReplicationSlot = c.String("replication-slot") + cfg.PublicationName = c.String("publication-name") + cfg.EstimatedCountThreshold = c.Int64("estimated-count-threshold") + cfg.DocPartition = c.Int64("doc-partition") + cfg.EnableReplicaMode = c.Bool("enable-replica-mode") + return cfg, nil +} + +func NewPostgresFlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "Postgres", + Usage: "postgresql://user:pass@host:port [options]", + Flags: PostgresFlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewPostgresFlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "Postgres", + Usage: "postgresql://user:pass@host:port [options]", + Flags: PostgresFlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/s3.pb.go b/gen/adiom/commands/connectors/v1/s3.pb.go new file mode 100644 index 00000000..9f4b2a32 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/s3.pb.go @@ -0,0 +1,233 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/s3.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type S3Flags struct { + state protoimpl.MessageState `protogen:"open.v1"` + PrettyJson bool `protobuf:"varint,1,opt,name=pretty_json,json=prettyJson,proto3" json:"pretty_json,omitempty"` + Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` + Prefix string `protobuf:"bytes,3,opt,name=prefix,proto3" json:"prefix,omitempty"` + OutputFormat string `protobuf:"bytes,4,opt,name=output_format,json=outputFormat,proto3" json:"output_format,omitempty"` + Profile string `protobuf:"bytes,5,opt,name=profile,proto3" json:"profile,omitempty"` + Endpoint string `protobuf:"bytes,6,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + AccessKeyId string `protobuf:"bytes,7,opt,name=access_key_id,json=accessKeyId,proto3" json:"access_key_id,omitempty"` + SecretAccessKey string `protobuf:"bytes,8,opt,name=secret_access_key,json=secretAccessKey,proto3" json:"secret_access_key,omitempty"` + SessionToken string `protobuf:"bytes,9,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"` + UsePathStyle bool `protobuf:"varint,10,opt,name=use_path_style,json=usePathStyle,proto3" json:"use_path_style,omitempty"` + MaxFileSize int64 `protobuf:"varint,11,opt,name=max_file_size,json=maxFileSize,proto3" json:"max_file_size,omitempty"` + MaxTotalMemory int64 `protobuf:"varint,12,opt,name=max_total_memory,json=maxTotalMemory,proto3" json:"max_total_memory,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *S3Flags) Reset() { + *x = S3Flags{} + mi := &file_adiom_commands_connectors_v1_s3_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *S3Flags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3Flags) ProtoMessage() {} + +func (x *S3Flags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_s3_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3Flags.ProtoReflect.Descriptor instead. +func (*S3Flags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_s3_proto_rawDescGZIP(), []int{0} +} + +func (x *S3Flags) GetPrettyJson() bool { + if x != nil { + return x.PrettyJson + } + return false +} + +func (x *S3Flags) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *S3Flags) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +func (x *S3Flags) GetOutputFormat() string { + if x != nil { + return x.OutputFormat + } + return "" +} + +func (x *S3Flags) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *S3Flags) GetEndpoint() string { + if x != nil { + return x.Endpoint + } + return "" +} + +func (x *S3Flags) GetAccessKeyId() string { + if x != nil { + return x.AccessKeyId + } + return "" +} + +func (x *S3Flags) GetSecretAccessKey() string { + if x != nil { + return x.SecretAccessKey + } + return "" +} + +func (x *S3Flags) GetSessionToken() string { + if x != nil { + return x.SessionToken + } + return "" +} + +func (x *S3Flags) GetUsePathStyle() bool { + if x != nil { + return x.UsePathStyle + } + return false +} + +func (x *S3Flags) GetMaxFileSize() int64 { + if x != nil { + return x.MaxFileSize + } + return 0 +} + +func (x *S3Flags) GetMaxTotalMemory() int64 { + if x != nil { + return x.MaxTotalMemory + } + return 0 +} + +var File_adiom_commands_connectors_v1_s3_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_s3_proto_rawDesc = "" + + "\n" + + "%adiom/commands/connectors/v1/s3.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\"\xb9\t\n" + + "\aS3Flags\x12W\n" + + "\vpretty_json\x18\x01 \x01(\bB6\x82\xb5\x182\n" + + "\vpretty-json\x12\x1dPretty-print JSON output data\x1a\x04trueR\n" + + "prettyJson\x12@\n" + + "\x06region\x18\x02 \x01(\tB(\x82\xb5\x18$\x12 AWS region for the target bucket \x01R\x06region\x12e\n" + + "\x06prefix\x18\x03 \x01(\tBM\x82\xb5\x18I\x12GOverride or append to the key prefix derived from the connection stringR\x06prefix\x12x\n" + + "\routput_format\x18\x04 \x01(\tBS\x82\xb5\x18O\n" + + "\routput-format\x128Output format for stored objects (only 'json' supported)\x1a\x04jsonR\foutputFormat\x125\n" + + "\aprofile\x18\x05 \x01(\tB\x1b\x82\xb5\x18\x17\x12\x15Shared config profileR\aprofile\x12_\n" + + "\bendpoint\x18\x06 \x01(\tBC\x82\xb5\x18?\x12=Custom S3 endpoint (for Localstack or S3-compatible services)R\bendpoint\x12Q\n" + + "\raccess_key_id\x18\a \x01(\tB-\x82\xb5\x18)\n" + + "\raccess-key-id\x12\x18Static AWS access key IDR\vaccessKeyId\x12a\n" + + "\x11secret_access_key\x18\b \x01(\tB5\x82\xb5\x181\n" + + "\x11secret-access-key\x12\x1cStatic AWS secret access keyR\x0fsecretAccessKey\x12R\n" + + "\rsession_token\x18\t \x01(\tB-\x82\xb5\x18)\n" + + "\rsession-token\x12\x18Static AWS session tokenR\fsessionToken\x12s\n" + + "\x0euse_path_style\x18\n" + + " \x01(\bBM\x82\xb5\x18I\n" + + "\x0euse-path-style\x127Use path-style addressing (useful for Localstack/minio)R\fusePathStyle\x12h\n" + + "\rmax_file_size\x18\v \x01(\x03BD\x82\xb5\x18@\n" + + "\rmax-file-size\x12+Maximum size of a single file in S3 (in MB)\x1a\x0210R\vmaxFileSize\x12\x86\x01\n" + + "\x10max_total_memory\x18\f \x01(\x03B\\\x82\xb5\x18X\n" + + "\x10max-total-memory\x12?Maximum total memory for batching before flushing to S3 (in MB)\x1a\x03100R\x0emaxTotalMemory:(\x82\xb5\x18$\n" + + "\x02s3\"\x1es3://bucket[/prefix] [options]BKZIgithub.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1b\x06proto3" + +var ( + file_adiom_commands_connectors_v1_s3_proto_rawDescOnce sync.Once + file_adiom_commands_connectors_v1_s3_proto_rawDescData []byte +) + +func file_adiom_commands_connectors_v1_s3_proto_rawDescGZIP() []byte { + file_adiom_commands_connectors_v1_s3_proto_rawDescOnce.Do(func() { + file_adiom_commands_connectors_v1_s3_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_s3_proto_rawDesc), len(file_adiom_commands_connectors_v1_s3_proto_rawDesc))) + }) + return file_adiom_commands_connectors_v1_s3_proto_rawDescData +} + +var file_adiom_commands_connectors_v1_s3_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_adiom_commands_connectors_v1_s3_proto_goTypes = []any{ + (*S3Flags)(nil), // 0: adiom.commands.connectors.v1.S3Flags +} +var file_adiom_commands_connectors_v1_s3_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_adiom_commands_connectors_v1_s3_proto_init() } +func file_adiom_commands_connectors_v1_s3_proto_init() { + if File_adiom_commands_connectors_v1_s3_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_commands_connectors_v1_s3_proto_rawDesc), len(file_adiom_commands_connectors_v1_s3_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_adiom_commands_connectors_v1_s3_proto_goTypes, + DependencyIndexes: file_adiom_commands_connectors_v1_s3_proto_depIdxs, + MessageInfos: file_adiom_commands_connectors_v1_s3_proto_msgTypes, + }.Build() + File_adiom_commands_connectors_v1_s3_proto = out.File + file_adiom_commands_connectors_v1_s3_proto_goTypes = nil + file_adiom_commands_connectors_v1_s3_proto_depIdxs = nil +} diff --git a/gen/adiom/commands/connectors/v1/s3_commandargs.go b/gen/adiom/commands/connectors/v1/s3_commandargs.go new file mode 100644 index 00000000..a48c24c9 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/s3_commandargs.go @@ -0,0 +1,100 @@ +// Code generated by protoc-gen-commandargs-urfave. DO NOT EDIT. + +package connectorsv1 + +import "github.com/urfave/cli/v2" + +func S3FlagsCommandFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "pretty-json", + Usage: "Pretty-print JSON output data", + Value: true, + }, + &cli.StringFlag{ + Name: "region", + Usage: "AWS region for the target bucket", + Required: true, + }, + &cli.StringFlag{ + Name: "prefix", + Usage: "Override or append to the key prefix derived from the connection string", + }, + &cli.StringFlag{ + Name: "output-format", + Usage: "Output format for stored objects (only 'json' supported)", + Value: "json", + }, + &cli.StringFlag{ + Name: "profile", + Usage: "Shared config profile", + }, + &cli.StringFlag{ + Name: "endpoint", + Usage: "Custom S3 endpoint (for Localstack or S3-compatible services)", + }, + &cli.StringFlag{ + Name: "access-key-id", + Usage: "Static AWS access key ID", + }, + &cli.StringFlag{ + Name: "secret-access-key", + Usage: "Static AWS secret access key", + }, + &cli.StringFlag{ + Name: "session-token", + Usage: "Static AWS session token", + }, + &cli.BoolFlag{ + Name: "use-path-style", + Usage: "Use path-style addressing (useful for Localstack/minio)", + }, + &cli.Int64Flag{ + Name: "max-file-size", + Usage: "Maximum size of a single file in S3 (in MB)", + Value: 10, + }, + &cli.Int64Flag{ + Name: "max-total-memory", + Usage: "Maximum total memory for batching before flushing to S3 (in MB)", + Value: 100, + }, + } +} + +func ParseS3Flags(c *cli.Context) (*S3Flags, error) { + cfg := &S3Flags{} + cfg.PrettyJson = c.Bool("pretty-json") + cfg.Region = c.String("region") + cfg.Prefix = c.String("prefix") + cfg.OutputFormat = c.String("output-format") + cfg.Profile = c.String("profile") + cfg.Endpoint = c.String("endpoint") + cfg.AccessKeyId = c.String("access-key-id") + cfg.SecretAccessKey = c.String("secret-access-key") + cfg.SessionToken = c.String("session-token") + cfg.UsePathStyle = c.Bool("use-path-style") + cfg.MaxFileSize = c.Int64("max-file-size") + cfg.MaxTotalMemory = c.Int64("max-total-memory") + return cfg, nil +} + +func NewS3FlagsCommand(action cli.ActionFunc, subcommands ...*cli.Command) *cli.Command { + return &cli.Command{ + Name: "s3", + Usage: "s3://bucket[/prefix] [options]", + Flags: S3FlagsCommandFlags(), + Subcommands: subcommands, + Action: action, + } +} + +func NewS3FlagsApp(action cli.ActionFunc, subcommands ...*cli.Command) *cli.App { + return &cli.App{ + Name: "s3", + Usage: "s3://bucket[/prefix] [options]", + Flags: S3FlagsCommandFlags(), + Commands: subcommands, + Action: action, + } +} diff --git a/gen/adiom/commands/connectors/v1/s3vectors.pb.go b/gen/adiom/commands/connectors/v1/s3vectors.pb.go new file mode 100644 index 00000000..65a54ec5 --- /dev/null +++ b/gen/adiom/commands/connectors/v1/s3vectors.pb.go @@ -0,0 +1,170 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: adiom/commands/connectors/v1/s3vectors.proto + +package connectorsv1 + +import ( + _ "github.com/adiom-data/commandargs/gen/commandargs/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type S3VectorsFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + Bucket string `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + VectorKey string `protobuf:"bytes,2,opt,name=vector_key,json=vectorKey,proto3" json:"vector_key,omitempty"` + MaxParallelism int32 `protobuf:"varint,3,opt,name=max_parallelism,json=maxParallelism,proto3" json:"max_parallelism,omitempty"` + RateLimit int32 `protobuf:"varint,4,opt,name=rate_limit,json=rateLimit,proto3" json:"rate_limit,omitempty"` + BatchSize int32 `protobuf:"varint,5,opt,name=batch_size,json=batchSize,proto3" json:"batch_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *S3VectorsFlags) Reset() { + *x = S3VectorsFlags{} + mi := &file_adiom_commands_connectors_v1_s3vectors_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *S3VectorsFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3VectorsFlags) ProtoMessage() {} + +func (x *S3VectorsFlags) ProtoReflect() protoreflect.Message { + mi := &file_adiom_commands_connectors_v1_s3vectors_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3VectorsFlags.ProtoReflect.Descriptor instead. +func (*S3VectorsFlags) Descriptor() ([]byte, []int) { + return file_adiom_commands_connectors_v1_s3vectors_proto_rawDescGZIP(), []int{0} +} + +func (x *S3VectorsFlags) GetBucket() string { + if x != nil { + return x.Bucket + } + return "" +} + +func (x *S3VectorsFlags) GetVectorKey() string { + if x != nil { + return x.VectorKey + } + return "" +} + +func (x *S3VectorsFlags) GetMaxParallelism() int32 { + if x != nil { + return x.MaxParallelism + } + return 0 +} + +func (x *S3VectorsFlags) GetRateLimit() int32 { + if x != nil { + return x.RateLimit + } + return 0 +} + +func (x *S3VectorsFlags) GetBatchSize() int32 { + if x != nil { + return x.BatchSize + } + return 0 +} + +var File_adiom_commands_connectors_v1_s3vectors_proto protoreflect.FileDescriptor + +const file_adiom_commands_connectors_v1_s3vectors_proto_rawDesc = "" + + "\n" + + ",adiom/commands/connectors/v1/s3vectors.proto\x12\x1cadiom.commands.connectors.v1\x1a\x1ccommandargs/v1/options.proto\"\xd9\x04\n" + + "\x0eS3VectorsFlags\x12;\n" + + "\x06bucket\x18\x01 \x01(\tB#\x82\xb5\x18\x1f\x12\x1bthe s3 vector bucket to use \x01R\x06bucket\x12V\n" + + "\n" + + "vector_key\x18\x02 \x01(\tB7\x82\xb5\x183\n" + + "\n" + + "vector-key\x12\x1fThe field containing the vector\x1a\x04dataR\tvectorKey\x12\x88\x01\n" + + "\x0fmax_parallelism\x18\x03 \x01(\x05B_\x82\xb5\x18[\n" + + "\x0fmax-parallelism\x12ELimits how many concurrent requests are made to s3vectors per worker.\x1a\x011R\x0emaxParallelism\x12\x8a\x01\n" + + "\n" + + "rate_limit\x18\x04 \x01(\x05Bk\x82\xb5\x18g\n" + + "\n" + + "rate-limit\x12SMax vectors per second across workers on this process per namespace (vector index).\x1a\x042500R\trateLimit\x12r\n" + + "\n" + + "batch_size\x18\x05 \x01(\x05BS\x82\xb5\x18O\n" + + "\n" + + "batch-size\x12\n" + + "\aGetInfo\x12\x18.adiom.v1.GetInfoRequest\x1a\x19.adiom.v1.GetInfoResponse\x12e\n" + + "\x14GetNamespaceMetadata\x12%.adiom.v1.GetNamespaceMetadataRequest\x1a&.adiom.v1.GetNamespaceMetadataResponse\x12D\n" + + "\tWriteData\x12\x1a.adiom.v1.WriteDataRequest\x1a\x1b.adiom.v1.WriteDataResponse\x12M\n" + + "\fWriteUpdates\x12\x1d.adiom.v1.WriteUpdatesRequest\x1a\x1e.adiom.v1.WriteUpdatesResponse\x12M\n" + + "\fGeneratePlan\x12\x1d.adiom.v1.GeneratePlanRequest\x1a\x1e.adiom.v1.GeneratePlanResponse\x12A\n" + + "\bListData\x12\x19.adiom.v1.ListDataRequest\x1a\x1a.adiom.v1.ListDataResponse\x12R\n" + + "\rStreamUpdates\x12\x1e.adiom.v1.StreamUpdatesRequest\x1a\x1f.adiom.v1.StreamUpdatesResponse0\x01\x12F\n" + + "\tStreamLSN\x12\x1a.adiom.v1.StreamLSNRequest\x1a\x1b.adiom.v1.StreamLSNResponse0\x01\x12A\n" + + "\bGetByIds\x12\x19.adiom.v1.GetByIdsRequest\x1a\x1a.adiom.v1.GetByIdsResponse2\xbc\x01\n" + + "\x10TransformService\x12Y\n" + + "\x10GetTransformInfo\x12!.adiom.v1.GetTransformInfoRequest\x1a\".adiom.v1.GetTransformInfoResponse\x12M\n" + + "\fGetTransform\x12\x1d.adiom.v1.GetTransformRequest\x1a\x1e.adiom.v1.GetTransformResponseB2Z0github.com/adiom-data/dsync/gen/adiom/v1;adiomv1b\x06proto3" var file_adiom_v1_adiom_proto_goTypes = []any{ (*GetInfoRequest)(nil), // 0: adiom.v1.GetInfoRequest @@ -152,7 +103,7 @@ func file_adiom_v1_adiom_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_adiom_v1_adiom_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_v1_adiom_proto_rawDesc), len(file_adiom_v1_adiom_proto_rawDesc)), NumEnums: 0, NumMessages: 0, NumExtensions: 0, @@ -162,7 +113,6 @@ func file_adiom_v1_adiom_proto_init() { DependencyIndexes: file_adiom_v1_adiom_proto_depIdxs, }.Build() File_adiom_v1_adiom_proto = out.File - file_adiom_v1_adiom_proto_rawDesc = nil file_adiom_v1_adiom_proto_goTypes = nil file_adiom_v1_adiom_proto_depIdxs = nil } diff --git a/gen/adiom/v1/messages.pb.go b/gen/adiom/v1/messages.pb.go index d3438147..679c0f6f 100644 --- a/gen/adiom/v1/messages.pb.go +++ b/gen/adiom/v1/messages.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.11 // protoc (unknown) // source: adiom/v1/messages.proto @@ -12,6 +12,7 @@ import ( timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -1967,270 +1968,159 @@ func (x *GetTransformInfoResponse_TransformInfo) GetResponseTypes() []DataType { var File_adiom_v1_messages_proto protoreflect.FileDescriptor -var file_adiom_v1_messages_proto_rawDesc = []byte{ - 0x0a, 0x17, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x61, 0x64, 0x69, 0x6f, 0x6d, - 0x2e, 0x76, 0x31, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x47, 0x0a, 0x09, 0x42, 0x73, 0x6f, 0x6e, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x9e, 0x01, - 0x0a, 0x10, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x27, 0x0a, - 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x22, 0xbe, - 0x01, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, - 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, - 0x74, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x0e, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, - 0xa7, 0x03, 0x0a, 0x0c, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, - 0x12, 0x35, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1d, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x70, 0x61, - 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x73, 0x69, 0x6e, 0x6b, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x53, 0x69, - 0x6e, 0x6b, 0x52, 0x04, 0x73, 0x69, 0x6e, 0x6b, 0x1a, 0xe0, 0x01, 0x0a, 0x06, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x12, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x73, 0x6e, - 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6c, - 0x73, 0x6e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x30, 0x0a, 0x14, 0x6d, 0x75, 0x6c, 0x74, - 0x69, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x6c, 0x61, 0x6e, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x4e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x65, - 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0b, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x12, 0x1c, 0x0a, - 0x0a, 0x67, 0x65, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x08, 0x67, 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x73, 0x1a, 0x4c, 0x0a, 0x04, 0x53, - 0x69, 0x6e, 0x6b, 0x12, 0x44, 0x0a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x12, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x10, 0x0a, 0x0e, 0x47, 0x65, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xa4, 0x01, 0x0a, 0x0f, - 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x17, 0x0a, 0x07, 0x64, 0x62, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x64, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x61, - 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, - 0x74, 0x69, 0x65, 0x73, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, - 0x65, 0x73, 0x22, 0x3b, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, - 0x34, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x72, 0x0a, 0x13, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, - 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x12, - 0x18, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x14, 0x47, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x33, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, - 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, - 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x49, 0x0a, 0x12, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x73, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x11, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0x9d, 0x01, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x64, 0x69, 0x6f, - 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x73, 0x6f, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x28, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x14, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x12, 0x30, 0x0a, 0x14, 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x5f, 0x75, 0x6e, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x55, 0x6e, 0x73, - 0x65, 0x74, 0x22, 0x84, 0x01, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x64, 0x69, 0x6f, - 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x0a, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, - 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x22, 0x47, 0x0a, 0x10, 0x4c, 0x69, 0x73, - 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x43, 0x75, 0x72, 0x73, - 0x6f, 0x72, 0x22, 0x6c, 0x0a, 0x10, 0x57, 0x72, 0x69, 0x74, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x26, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x22, 0x13, 0x0a, 0x11, 0x57, 0x72, 0x69, 0x74, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x87, 0x01, 0x0a, 0x13, 0x57, 0x72, 0x69, 0x74, 0x65, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, - 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, - 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, - 0x16, 0x0a, 0x14, 0x57, 0x72, 0x69, 0x74, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x76, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, - 0x26, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, - 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, - 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x22, - 0xb2, 0x01, 0x0a, 0x15, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x64, 0x69, - 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x75, 0x72, 0x73, - 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x43, 0x75, - 0x72, 0x73, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, - 0x74, 0x69, 0x6d, 0x65, 0x22, 0x4a, 0x0a, 0x10, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x53, - 0x4e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, - 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x75, 0x72, 0x73, - 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, - 0x22, 0x46, 0x0a, 0x11, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x53, 0x4e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x73, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x03, 0x6c, 0x73, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, - 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, 0x65, - 0x78, 0x74, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x22, 0xc0, 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, - 0x42, 0x79, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x35, 0x0a, 0x03, 0x69, 0x64, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, - 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x03, 0x69, 0x64, - 0x73, 0x12, 0x26, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x1a, 0x30, 0x0a, 0x09, 0x49, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x73, - 0x6f, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x02, 0x69, 0x64, 0x22, 0x73, 0x0a, 0x10, 0x47, - 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3b, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, - 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x22, 0x0a, 0x0c, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x12, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x22, 0x19, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xa6, 0x02, 0x0a, 0x18, - 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0a, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x61, - 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x75, 0x73, - 0x65, 0x5f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x75, 0x73, 0x65, 0x4d, - 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x73, - 0x1a, 0x81, 0x01, 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x49, 0x6e, - 0x66, 0x6f, 0x12, 0x35, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, - 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x0e, 0x72, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, - 0x79, 0x70, 0x65, 0x73, 0x22, 0xe3, 0x01, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x64, - 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x35, 0x0a, 0x0c, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x37, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, - 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x22, 0x71, 0x0a, 0x11, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2a, 0x0a, - 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, - 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xaf, 0x01, - 0x0a, 0x14, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, - 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, - 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x12, 0x39, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x73, 0x2a, - 0x52, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, 0x0a, 0x11, 0x44, - 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, - 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x4d, 0x4f, 0x4e, 0x47, 0x4f, 0x5f, 0x42, 0x53, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, - 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x49, - 0x44, 0x10, 0x02, 0x2a, 0x8d, 0x01, 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x55, - 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x53, 0x45, 0x52, - 0x54, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x55, - 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, - 0x45, 0x10, 0x03, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x50, 0x41, 0x52, 0x54, 0x49, 0x41, 0x4c, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, - 0x45, 0x10, 0x04, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2d, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x64, 0x73, 0x79, - 0x6e, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2f, 0x76, 0x31, 0x3b, - 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_adiom_v1_messages_proto_rawDesc = "" + + "\n" + + "\x17adiom/v1/messages.proto\x12\badiom.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"G\n" + + "\tBsonValue\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data\x12\x12\n" + + "\x04type\x18\x02 \x01(\rR\x04type\x12\x12\n" + + "\x04name\x18\x03 \x01(\tR\x04name\"\x9e\x01\n" + + "\x10UpdatesPartition\x12\x1e\n" + + "\n" + + "namespaces\x18\x01 \x03(\tR\n" + + "namespaces\x12)\n" + + "\x10partition_header\x18\x02 \x01(\tR\x0fpartitionHeader\x12'\n" + + "\x0fpartition_value\x18\x03 \x01(\tR\x0epartitionValue\x12\x16\n" + + "\x06cursor\x18\x04 \x01(\fR\x06cursor\"\xbe\x01\n" + + "\tPartition\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12)\n" + + "\x10partition_header\x18\x02 \x01(\tR\x0fpartitionHeader\x12'\n" + + "\x0fpartition_value\x18\x03 \x01(\tR\x0epartitionValue\x12\x16\n" + + "\x06cursor\x18\x04 \x01(\fR\x06cursor\x12'\n" + + "\x0festimated_count\x18\x05 \x01(\x04R\x0eestimatedCount\"\xa7\x03\n" + + "\fCapabilities\x125\n" + + "\x06source\x18\x01 \x01(\v2\x1d.adiom.v1.Capabilities.SourceR\x06source\x12/\n" + + "\x04sink\x18\x02 \x01(\v2\x1b.adiom.v1.Capabilities.SinkR\x04sink\x1a\xe0\x01\n" + + "\x06Source\x12D\n" + + "\x14supported_data_types\x18\x01 \x03(\x0e2\x12.adiom.v1.DataTypeR\x12supportedDataTypes\x12\x1d\n" + + "\n" + + "lsn_stream\x18\x02 \x01(\bR\tlsnStream\x120\n" + + "\x14multi_namespace_plan\x18\x03 \x01(\bR\x12multiNamespacePlan\x12!\n" + + "\fdefault_plan\x18\x04 \x01(\bR\vdefaultPlan\x12\x1c\n" + + "\n" + + "get_by_ids\x18\x05 \x01(\bR\bgetByIds\x1aL\n" + + "\x04Sink\x12D\n" + + "\x14supported_data_types\x18\x01 \x03(\x0e2\x12.adiom.v1.DataTypeR\x12supportedDataTypes\"\x10\n" + + "\x0eGetInfoRequest\"\xa4\x01\n" + + "\x0fGetInfoResponse\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x17\n" + + "\adb_type\x18\x02 \x01(\tR\x06dbType\x12\x18\n" + + "\aversion\x18\x03 \x01(\tR\aversion\x12\x12\n" + + "\x04spec\x18\x04 \x01(\tR\x04spec\x12:\n" + + "\fcapabilities\x18\x05 \x01(\v2\x16.adiom.v1.CapabilitiesR\fcapabilities\";\n" + + "\x1bGetNamespaceMetadataRequest\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\"4\n" + + "\x1cGetNamespaceMetadataResponse\x12\x14\n" + + "\x05count\x18\x01 \x01(\x04R\x05count\"r\n" + + "\x13GeneratePlanRequest\x12\x1e\n" + + "\n" + + "namespaces\x18\x01 \x03(\tR\n" + + "namespaces\x12!\n" + + "\finitial_sync\x18\x02 \x01(\bR\vinitialSync\x12\x18\n" + + "\aupdates\x18\x03 \x01(\bR\aupdates\"\x96\x01\n" + + "\x14GeneratePlanResponse\x123\n" + + "\n" + + "partitions\x18\x01 \x03(\v2\x13.adiom.v1.PartitionR\n" + + "partitions\x12I\n" + + "\x12updates_partitions\x18\x02 \x03(\v2\x1a.adiom.v1.UpdatesPartitionR\x11updatesPartitions\"\x9d\x01\n" + + "\x06Update\x12#\n" + + "\x02id\x18\x01 \x03(\v2\x13.adiom.v1.BsonValueR\x02id\x12(\n" + + "\x04type\x18\x02 \x01(\x0e2\x14.adiom.v1.UpdateTypeR\x04type\x12\x12\n" + + "\x04data\x18\x03 \x01(\fR\x04data\x120\n" + + "\x14partial_update_unset\x18\x04 \x03(\tR\x12partialUpdateUnset\"\x84\x01\n" + + "\x0fListDataRequest\x121\n" + + "\tpartition\x18\x01 \x01(\v2\x13.adiom.v1.PartitionR\tpartition\x12&\n" + + "\x04type\x18\x02 \x01(\x0e2\x12.adiom.v1.DataTypeR\x04type\x12\x16\n" + + "\x06cursor\x18\x03 \x01(\fR\x06cursor\"G\n" + + "\x10ListDataResponse\x12\x12\n" + + "\x04data\x18\x01 \x03(\fR\x04data\x12\x1f\n" + + "\vnext_cursor\x18\x02 \x01(\fR\n" + + "nextCursor\"l\n" + + "\x10WriteDataRequest\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12\x12\n" + + "\x04data\x18\x02 \x03(\fR\x04data\x12&\n" + + "\x04type\x18\x03 \x01(\x0e2\x12.adiom.v1.DataTypeR\x04type\"\x13\n" + + "\x11WriteDataResponse\"\x87\x01\n" + + "\x13WriteUpdatesRequest\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12*\n" + + "\aupdates\x18\x02 \x03(\v2\x10.adiom.v1.UpdateR\aupdates\x12&\n" + + "\x04type\x18\x03 \x01(\x0e2\x12.adiom.v1.DataTypeR\x04type\"\x16\n" + + "\x14WriteUpdatesResponse\"v\n" + + "\x14StreamUpdatesRequest\x12\x1e\n" + + "\n" + + "namespaces\x18\x01 \x03(\tR\n" + + "namespaces\x12&\n" + + "\x04type\x18\x02 \x01(\x0e2\x12.adiom.v1.DataTypeR\x04type\x12\x16\n" + + "\x06cursor\x18\x03 \x01(\fR\x06cursor\"\xb2\x01\n" + + "\x15StreamUpdatesResponse\x12*\n" + + "\aupdates\x18\x01 \x03(\v2\x10.adiom.v1.UpdateR\aupdates\x12\x1c\n" + + "\tnamespace\x18\x02 \x01(\tR\tnamespace\x12\x1f\n" + + "\vnext_cursor\x18\x03 \x01(\fR\n" + + "nextCursor\x12.\n" + + "\x04time\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x04time\"J\n" + + "\x10StreamLSNRequest\x12\x1e\n" + + "\n" + + "namespaces\x18\x01 \x03(\tR\n" + + "namespaces\x12\x16\n" + + "\x06cursor\x18\x02 \x01(\fR\x06cursor\"F\n" + + "\x11StreamLSNResponse\x12\x10\n" + + "\x03lsn\x18\x01 \x01(\x04R\x03lsn\x12\x1f\n" + + "\vnext_cursor\x18\x02 \x01(\fR\n" + + "nextCursor\"\xc0\x01\n" + + "\x0fGetByIdsRequest\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x125\n" + + "\x03ids\x18\x02 \x03(\v2#.adiom.v1.GetByIdsRequest.IdRequestR\x03ids\x12&\n" + + "\x04type\x18\x03 \x01(\x0e2\x12.adiom.v1.DataTypeR\x04type\x1a0\n" + + "\tIdRequest\x12#\n" + + "\x02id\x18\x01 \x03(\v2\x13.adiom.v1.BsonValueR\x02id\"s\n" + + "\x10GetByIdsResponse\x12;\n" + + "\x04data\x18\x01 \x03(\v2'.adiom.v1.GetByIdsResponse.ResponseItemR\x04data\x1a\"\n" + + "\fResponseItem\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data\"\x19\n" + + "\x17GetTransformInfoRequest\"\xa6\x02\n" + + "\x18GetTransformInfoResponse\x12P\n" + + "\n" + + "transforms\x18\x01 \x03(\v20.adiom.v1.GetTransformInfoResponse.TransformInfoR\n" + + "transforms\x124\n" + + "\x16use_multiple_responses\x18\x02 \x01(\bR\x14useMultipleResponses\x1a\x81\x01\n" + + "\rTransformInfo\x125\n" + + "\frequest_type\x18\x01 \x01(\x0e2\x12.adiom.v1.DataTypeR\vrequestType\x129\n" + + "\x0eresponse_types\x18\x02 \x03(\x0e2\x12.adiom.v1.DataTypeR\rresponseTypes\"\xe3\x01\n" + + "\x13GetTransformRequest\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12*\n" + + "\aupdates\x18\x02 \x03(\v2\x10.adiom.v1.UpdateR\aupdates\x12\x12\n" + + "\x04data\x18\x03 \x03(\fR\x04data\x125\n" + + "\frequest_type\x18\x04 \x01(\x0e2\x12.adiom.v1.DataTypeR\vrequestType\x127\n" + + "\rresponse_type\x18\x05 \x01(\x0e2\x12.adiom.v1.DataTypeR\fresponseType\"q\n" + + "\x11TransformResponse\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12*\n" + + "\aupdates\x18\x02 \x03(\v2\x10.adiom.v1.UpdateR\aupdates\x12\x12\n" + + "\x04data\x18\x03 \x03(\fR\x04data\"\xaf\x01\n" + + "\x14GetTransformResponse\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12*\n" + + "\aupdates\x18\x02 \x03(\v2\x10.adiom.v1.UpdateR\aupdates\x12\x12\n" + + "\x04data\x18\x03 \x03(\fR\x04data\x129\n" + + "\tresponses\x18\x04 \x03(\v2\x1b.adiom.v1.TransformResponseR\tresponses*R\n" + + "\bDataType\x12\x15\n" + + "\x11DATA_TYPE_UNKNOWN\x10\x00\x12\x18\n" + + "\x14DATA_TYPE_MONGO_BSON\x10\x01\x12\x15\n" + + "\x11DATA_TYPE_JSON_ID\x10\x02*\x8d\x01\n" + + "\n" + + "UpdateType\x12\x17\n" + + "\x13UPDATE_TYPE_UNKNOWN\x10\x00\x12\x16\n" + + "\x12UPDATE_TYPE_INSERT\x10\x01\x12\x16\n" + + "\x12UPDATE_TYPE_UPDATE\x10\x02\x12\x16\n" + + "\x12UPDATE_TYPE_DELETE\x10\x03\x12\x1e\n" + + "\x1aUPDATE_TYPE_PARTIAL_UPDATE\x10\x04B2Z0github.com/adiom-data/dsync/gen/adiom/v1;adiomv1b\x06proto3" var ( file_adiom_v1_messages_proto_rawDescOnce sync.Once - file_adiom_v1_messages_proto_rawDescData = file_adiom_v1_messages_proto_rawDesc + file_adiom_v1_messages_proto_rawDescData []byte ) func file_adiom_v1_messages_proto_rawDescGZIP() []byte { file_adiom_v1_messages_proto_rawDescOnce.Do(func() { - file_adiom_v1_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_adiom_v1_messages_proto_rawDescData) + file_adiom_v1_messages_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_v1_messages_proto_rawDesc), len(file_adiom_v1_messages_proto_rawDesc))) }) return file_adiom_v1_messages_proto_rawDescData } @@ -2322,7 +2212,7 @@ func file_adiom_v1_messages_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_adiom_v1_messages_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_v1_messages_proto_rawDesc), len(file_adiom_v1_messages_proto_rawDesc)), NumEnums: 2, NumMessages: 33, NumExtensions: 0, @@ -2334,7 +2224,6 @@ func file_adiom_v1_messages_proto_init() { MessageInfos: file_adiom_v1_messages_proto_msgTypes, }.Build() File_adiom_v1_messages_proto = out.File - file_adiom_v1_messages_proto_rawDesc = nil file_adiom_v1_messages_proto_goTypes = nil file_adiom_v1_messages_proto_depIdxs = nil } diff --git a/gen/adiom/v1/vector.pb.go b/gen/adiom/v1/vector.pb.go index e88cc977..1a6b3e4f 100644 --- a/gen/adiom/v1/vector.pb.go +++ b/gen/adiom/v1/vector.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.11 // protoc (unknown) // source: adiom/v1/vector.proto @@ -11,6 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -382,79 +383,42 @@ func (x *GetSupportedDataTypesResponse) GetTypes() []DataType { var File_adiom_v1_vector_proto protoreflect.FileDescriptor -var file_adiom_v1_vector_proto_rawDesc = []byte{ - 0x0a, 0x15, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2f, 0x76, 0x31, 0x2f, 0x76, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, - 0x31, 0x1a, 0x17, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x4f, 0x0a, 0x11, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x12, 0x26, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x1d, 0x0a, 0x07, 0x43, - 0x68, 0x75, 0x6e, 0x6b, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x3b, 0x0a, 0x12, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x25, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, - 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x65, - 0x64, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x51, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x45, 0x6d, - 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, - 0x74, 0x61, 0x12, 0x26, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x12, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x1f, 0x0a, 0x09, 0x45, 0x6d, - 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x01, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x3f, 0x0a, 0x14, 0x47, - 0x65, 0x74, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x13, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6d, 0x62, - 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x1e, 0x0a, 0x1c, - 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x49, 0x0a, 0x1d, - 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, - 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x61, - 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x32, 0xc4, 0x01, 0x0a, 0x0f, 0x43, 0x68, 0x75, 0x6e, - 0x6b, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x68, 0x0a, 0x15, 0x47, - 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x54, - 0x79, 0x70, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x61, - 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, - 0x72, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, - 0x6b, 0x65, 0x64, 0x12, 0x1b, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1c, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, - 0x68, 0x75, 0x6e, 0x6b, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xcb, - 0x01, 0x0a, 0x10, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x68, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, - 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x61, - 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, - 0x72, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, - 0x0c, 0x47, 0x65, 0x74, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, - 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x6d, 0x62, 0x65, - 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x61, - 0x64, 0x69, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x6d, 0x62, 0x65, 0x64, - 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x32, 0x5a, 0x30, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x64, 0x69, 0x6f, 0x6d, - 0x2d, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x64, 0x73, 0x79, 0x6e, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x2f, - 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x64, 0x69, 0x6f, 0x6d, 0x76, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_adiom_v1_vector_proto_rawDesc = "" + + "\n" + + "\x15adiom/v1/vector.proto\x12\badiom.v1\x1a\x17adiom/v1/messages.proto\"O\n" + + "\x11GetChunkedRequest\x12\x12\n" + + "\x04data\x18\x01 \x03(\fR\x04data\x12&\n" + + "\x04type\x18\x02 \x01(\x0e2\x12.adiom.v1.DataTypeR\x04type\"\x1d\n" + + "\aChunked\x12\x12\n" + + "\x04data\x18\x01 \x03(\fR\x04data\";\n" + + "\x12GetChunkedResponse\x12%\n" + + "\x04data\x18\x01 \x03(\v2\x11.adiom.v1.ChunkedR\x04data\"Q\n" + + "\x13GetEmbeddingRequest\x12\x12\n" + + "\x04data\x18\x01 \x03(\fR\x04data\x12&\n" + + "\x04type\x18\x02 \x01(\x0e2\x12.adiom.v1.DataTypeR\x04type\"\x1f\n" + + "\tEmbedding\x12\x12\n" + + "\x04data\x18\x01 \x03(\x01R\x04data\"?\n" + + "\x14GetEmbeddingResponse\x12'\n" + + "\x04data\x18\x01 \x03(\v2\x13.adiom.v1.EmbeddingR\x04data\"\x1e\n" + + "\x1cGetSupportedDataTypesRequest\"I\n" + + "\x1dGetSupportedDataTypesResponse\x12(\n" + + "\x05types\x18\x01 \x03(\x0e2\x12.adiom.v1.DataTypeR\x05types2\xc4\x01\n" + + "\x0fChunkingService\x12h\n" + + "\x15GetSupportedDataTypes\x12&.adiom.v1.GetSupportedDataTypesRequest\x1a'.adiom.v1.GetSupportedDataTypesResponse\x12G\n" + + "\n" + + "GetChunked\x12\x1b.adiom.v1.GetChunkedRequest\x1a\x1c.adiom.v1.GetChunkedResponse2\xcb\x01\n" + + "\x10EmbeddingService\x12h\n" + + "\x15GetSupportedDataTypes\x12&.adiom.v1.GetSupportedDataTypesRequest\x1a'.adiom.v1.GetSupportedDataTypesResponse\x12M\n" + + "\fGetEmbedding\x12\x1d.adiom.v1.GetEmbeddingRequest\x1a\x1e.adiom.v1.GetEmbeddingResponseB2Z0github.com/adiom-data/dsync/gen/adiom/v1;adiomv1b\x06proto3" var ( file_adiom_v1_vector_proto_rawDescOnce sync.Once - file_adiom_v1_vector_proto_rawDescData = file_adiom_v1_vector_proto_rawDesc + file_adiom_v1_vector_proto_rawDescData []byte ) func file_adiom_v1_vector_proto_rawDescGZIP() []byte { file_adiom_v1_vector_proto_rawDescOnce.Do(func() { - file_adiom_v1_vector_proto_rawDescData = protoimpl.X.CompressGZIP(file_adiom_v1_vector_proto_rawDescData) + file_adiom_v1_vector_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_adiom_v1_vector_proto_rawDesc), len(file_adiom_v1_vector_proto_rawDesc))) }) return file_adiom_v1_vector_proto_rawDescData } @@ -502,7 +466,7 @@ func file_adiom_v1_vector_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_adiom_v1_vector_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_adiom_v1_vector_proto_rawDesc), len(file_adiom_v1_vector_proto_rawDesc)), NumEnums: 0, NumMessages: 8, NumExtensions: 0, @@ -513,7 +477,6 @@ func file_adiom_v1_vector_proto_init() { MessageInfos: file_adiom_v1_vector_proto_msgTypes, }.Build() File_adiom_v1_vector_proto = out.File - file_adiom_v1_vector_proto_rawDesc = nil file_adiom_v1_vector_proto_goTypes = nil file_adiom_v1_vector_proto_depIdxs = nil } diff --git a/go.mod b/go.mod index 1cbebc4d..1c9668b0 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,12 @@ module github.com/adiom-data/dsync -go 1.24.0 - -toolchain go1.24.3 +go 1.24.3 require ( connectrpc.com/connect v1.18.1 connectrpc.com/grpcreflect v1.3.0 github.com/IBM/sarama v1.46.3 + github.com/adiom-data/commandargs v0.0.0-20260415173909-157c91bb5601 github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.29.17 github.com/aws/aws-sdk-go-v2/credentials v1.17.70 @@ -43,7 +42,7 @@ require ( golang.org/x/net v0.46.0 golang.org/x/time v0.12.0 google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 + google.golang.org/protobuf v1.36.11 ) require ( diff --git a/go.sum b/go.sum index 72213352..7cfae0b9 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,12 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 h1:fMi9ZZ/it4orHj3xWrM6cLkVFcCbkXQALFUiNtHtCPs= github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249/go.mod h1:iU1PxQMQwoHZZWmMKrMkrNlY+3+p9vxIjpZOVyxWa0g= +github.com/adiom-data/commandargs v0.0.0-20260414165004-3548c27b82b0 h1:doJzLkecFo6Ix7IRK5JwARfiiatJD24I0dGbZdsQXoY= +github.com/adiom-data/commandargs v0.0.0-20260414165004-3548c27b82b0/go.mod h1:c34lSdtbMEJjOETeP139JwqdSI/n72P9I3a35Dbg9rY= +github.com/adiom-data/commandargs v0.0.0-20260414220334-d5577dbfd3b6 h1:BRtwf2zYCwNIeCWcpKCjW9vQy7x+kyGDTHl8e4iOLHM= +github.com/adiom-data/commandargs v0.0.0-20260414220334-d5577dbfd3b6/go.mod h1:c34lSdtbMEJjOETeP139JwqdSI/n72P9I3a35Dbg9rY= +github.com/adiom-data/commandargs v0.0.0-20260415173909-157c91bb5601 h1:ECk1FunPS0SxPv9H+th7/uC/XO5R9stcqm8d0hyIxD8= +github.com/adiom-data/commandargs v0.0.0-20260415173909-157c91bb5601/go.mod h1:c34lSdtbMEJjOETeP139JwqdSI/n72P9I3a35Dbg9rY= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -529,8 +535,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/app/options/connectorflags.go b/internal/app/options/connectorflags.go index 61fcb478..1c6e38db 100644 --- a/internal/app/options/connectorflags.go +++ b/internal/app/options/connectorflags.go @@ -8,9 +8,7 @@ import ( "net" "net/http" "strings" - "time" - "github.com/IBM/sarama" "github.com/adiom-data/dsync/connectors/airbyte" "github.com/adiom-data/dsync/connectors/cosmos" "github.com/adiom-data/dsync/connectors/dynamodb" @@ -26,6 +24,7 @@ import ( "github.com/adiom-data/dsync/connectors/testconn" "github.com/adiom-data/dsync/connectors/vector" adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + connectorsv1 "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1" "github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect" "github.com/urfave/cli/v2" "github.com/urfave/cli/v2/altsrc" @@ -180,7 +179,7 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "/dev/random") }, - Create: CreateHelper("/dev/random", "/dev/random", nil, func(*cli.Context, []string, AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + Create: CreateHelper("/dev/random", "/dev/random", connectorsv1.DevRandomFlagsCommandFlags(), func(_ *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { return random.NewConn(random.ConnectorSettings{}), nil }), }, @@ -189,103 +188,12 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "/dev/fakesource") }, - Create: CreateHelper("/dev/fakesource", "/dev/fakesource", []cli.Flag{ - &cli.DurationFlag{ - Name: "sleep", - Usage: "Sleep time between requests", - }, - &cli.DurationFlag{ - Name: "jitter", - Usage: "Additional jitter to certain durations", - }, - &cli.DurationFlag{ - Name: "update-duration", - Usage: "Time for each update-batch-size to be produced", - Value: time.Millisecond * 400, - }, - &cli.DurationFlag{ - Name: "stream-tick", - Usage: "Polling time to check for stream updates", - Value: time.Second, - }, - &cli.IntFlag{ - Name: "num-namespaces", - Usage: "Number of namespaces (index starts at 0)", - Value: 2, - }, - &cli.IntFlag{ - Name: "num-partitions-per-namespace", - Usage: "Number of partitions per namespace", - Value: 3, - }, - &cli.IntFlag{ - Name: "num-update-partitions-per-namespace", - Usage: "Number of update partitions per namespace. If set to 0, use a single stream for all.", - }, - &cli.Int64Flag{ - Name: "num-docs-per-partition", - Usage: "Number of docs per partition", - Value: 1000, - }, - &cli.IntFlag{ - Name: "batch-size", - Usage: "Number of docs per batch", - Value: 150, - }, - &cli.IntFlag{ - Name: "update-batch-size", - Usage: "Number of docs per update", - Value: 50, - }, - &cli.IntFlag{ - Name: "max-updates-per-tick", - Usage: "Number of docs per update", - Value: 300, - }, - &cli.StringFlag{ - Name: "namespace-prefix", - Usage: "Prefix for namespace", - Value: "ns", - }, - &cli.StringSliceFlag{ - Name: "payload", - Usage: "Payload for all items (`key:value`)", - }, - &cli.StringFlag{ - Name: "payload-json", - Usage: "Payload for all items as a json (overrides payload)", - }, - }, func(c *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - var m map[string]any - payload := c.StringSlice("payload") - if len(payload) > 0 { - m = map[string]any{} - } - for _, p := range payload { - k, v, _ := strings.Cut(p, ":") - m[k] = v - } - payloadJSON := c.String("payload-json") - if payloadJSON != "" { - if err := json.Unmarshal([]byte(payloadJSON), &m); err != nil { - return nil, fmt.Errorf("err unmarshalling payload-json: %w", err) - } + Create: CreateHelper("/dev/fakesource", "/dev/fakesource", connectorsv1.FakeSourceFlagsCommandFlags(), func(c *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseFakeSourceFlags(c) + if err != nil { + return nil, err } - return random.NewConnV2(random.ConnV2Input{ - NamespacePrefix: c.String("namespace-prefix"), - NumNamespaces: c.Int("num-namespaces"), - NumPartitionsPerNamespace: c.Int("num-partitions-per-namespace"), - NumUpdatePartitionsPerNamespace: c.Int("num-update-partitions-per-namespace"), - BatchSize: c.Int("batch-size"), - UpdateBatchSize: c.Int("update-batch-size"), - MaxUpdatesPerTick: c.Int("max-updates-per-tick"), - NumDocsPerPartition: c.Int64("num-docs-per-partition"), - Payload: m, - Sleep: c.Duration("sleep"), - Jitter: c.Duration("jitter"), - UpdateDuration: c.Duration("update-duration"), - StreamTick: c.Duration("stream-tick"), - }), nil + return fakeSourceFromFlags(flags) }), }, { @@ -293,25 +201,12 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "/dev/null") }, - Create: CreateHelper("/dev/null", "/dev/null", []cli.Flag{ - &cli.DurationFlag{ - Name: "sleep", - Usage: "Sleep time between requests", - }, - &cli.DurationFlag{ - Name: "sleep-jitter", - Usage: "If sleep is set, additional jitter", - }, - &cli.BoolFlag{ - Name: "log-json", - Usage: "Convert data to json and log INFO", - }, - &cli.StringFlag{ - Name: "id", - Usage: "A fixed id for the connector", - }, - }, func(c *cli.Context, args []string, as AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - return null.NewConn(c.String("id"), c.Bool("log-json"), c.Duration("sleep"), c.Duration("sleep-jitter")), nil + Create: CreateHelper("/dev/null", "/dev/null", connectorsv1.DevNullFlagsCommandFlags(), func(c *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseDevNullFlags(c) + if err != nil { + return nil, err + } + return null.NewConn(flags.Id, flags.LogJson, flags.Sleep.AsDuration(), flags.SleepJitter.AsDuration()), nil }), }, { @@ -323,11 +218,12 @@ func GetRegisteredConnectors() []RegisteredConnector { if len(args) == 0 { return nil, nil, fmt.Errorf("missing s3 connection string: %w", ErrMissingConnector) } - settings := s3connector.ConnectorSettings{ - Uri: args[0], - } - return CreateHelper("s3", "s3://bucket[/prefix] [options]", S3Flags(&settings), func(_ *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - return s3connector.NewConn(settings) + return CreateHelper("s3", "s3://bucket[/prefix] [options]", connectorsv1.S3FlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseS3Flags(c) + if err != nil { + return nil, err + } + return s3connector.NewConn(s3SettingsFromFlags(flags, args[0])) })(args, as) }, }, @@ -340,10 +236,15 @@ func GetRegisteredConnectors() []RegisteredConnector { if len(args) == 0 { return nil, nil, fmt.Errorf("missing file connection string: %w", ErrMissingConnector) } - settings := fileconnector.ConnectorSettings{ - Uri: args[0], - } - return CreateHelper("file", "file:///path/to/dir OR file:///path/to/file.csv [options]", FileFlags(&settings), func(_ *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + return CreateHelper("file", "file:///path/to/dir OR file:///path/to/file.csv [options]", connectorsv1.FileFlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseFileFlags(c) + if err != nil { + return nil, err + } + settings, err := fileSettingsFromFlags(flags, args[0]) + if err != nil { + return nil, err + } return fileconnector.NewConn(settings) })(args, as) }, @@ -366,13 +267,12 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "sqlbatch") }, - Create: CreateHelper("sqlbatch", "sqlbatch", []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Required: true, - Value: "config.yml", - }}, func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - return sqlbatch.NewConn(c.String("config")) + Create: CreateHelper("sqlbatch", "sqlbatch", connectorsv1.SqlbatchFlagsCommandFlags(), func(c *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseSqlbatchFlags(c) + if err != nil { + return nil, err + } + return sqlbatch.NewConn(flags.Config) }), }, { @@ -380,39 +280,24 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "dynamodb") || strings.EqualFold(s, "dynamodb://localstack") }, - Create: CreateHelper("DynamoDB", "dynamodb OR dynamodb://localstack", []cli.Flag{ - &cli.IntFlag{ - Name: "doc-partition", - Usage: "Target number of documents per partition", - Value: 50000, - }, - &cli.IntFlag{ - Name: "plan-parallelism", - Usage: "Parallelism during planning", - Value: 4, - }, - &cli.StringFlag{ - Name: "id", - Usage: "A fixed id for the connector", - }, - }, func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + Create: CreateHelper("DynamoDB", "dynamodb OR dynamodb://localstack", connectorsv1.DynamoDBFlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseDynamoDBFlags(c) + if err != nil { + return nil, err + } + connString := "" if strings.EqualFold(args[0], "dynamodb://localstack") { - _, connString, ok := strings.Cut(args[0], "://") + _, cs, ok := strings.Cut(args[0], "://") if !ok { return nil, fmt.Errorf("invalid connection string %v", args[0]) } - return dynamodb.NewConn(connString, - dynamodb.WithDocsPerSegment(c.Int("doc-partition")), - dynamodb.WithPlanParallelism(c.Int("plan-parallelism")), - dynamodb.WithID(c.String("id")), - ), nil - } else { - return dynamodb.NewConn("", - dynamodb.WithDocsPerSegment(c.Int("doc-partition")), - dynamodb.WithPlanParallelism(c.Int("plan-parallelism")), - dynamodb.WithID(c.String("id")), - ), nil + connString = cs } + return dynamodb.NewConn(connString, + dynamodb.WithDocsPerSegment(int(flags.DocPartition)), + dynamodb.WithPlanParallelism(int(flags.PlanParallelism)), + dynamodb.WithID(flags.Id), + ), nil }), }, { @@ -420,39 +305,12 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "s3vector") || strings.EqualFold(s, "s3vectors") }, - Create: CreateHelper("s3vectors", "s3vector or s3vectors", []cli.Flag{ - &cli.StringFlag{ - Name: "bucket", - Usage: "the s3 vector bucket to use", - Required: true, - }, - &cli.StringFlag{ - Name: "vector-key", - Value: "data", - Usage: "The field containing the vector", - }, - &cli.IntFlag{ - Name: "max-parallelism", - Value: 1, - Usage: "Limits how many concurrent requests are made to s3vectors per worker.", - }, - &cli.IntFlag{ - Name: "rate-limit", - Value: 2500, - Usage: "Max vectors per second across workers on this process per namespace (vector index).", - }, - &cli.IntFlag{ - Name: "batch-size", - Value: 500, - Usage: "Max size of each PutVector requests (aws hard limit is 500).", - }, - }, func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - bucket := c.String("bucket") - vectorKey := c.String("vector-key") - maxParallelism := c.Int("max-parallelism") - rateLimit := c.Int("rate-limit") - batchSize := c.Int("batch-size") - return s3vector.NewConn(bucket, vectorKey, maxParallelism, batchSize, rateLimit) + Create: CreateHelper("s3vectors", "s3vector or s3vectors", connectorsv1.S3VectorsFlagsCommandFlags(), func(c *cli.Context, _ []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseS3VectorsFlags(c) + if err != nil { + return nil, err + } + return s3vector.NewConn(flags.Bucket, flags.VectorKey, int(flags.MaxParallelism), int(flags.BatchSize), int(flags.RateLimit)) }), }, { @@ -464,13 +322,12 @@ func GetRegisteredConnectors() []RegisteredConnector { return false }, Create: func(args []string, as AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, []string, error) { - settings := cosmos.ConnectorSettings{} - settings.ConnectionString = args[0] - if as.BaseThreadCount != 0 { - settings.NumParallelPartitionWorkers = as.BaseThreadCount / 2 - } - return CreateHelper("CosmosDB", "mongodb://cosmos-connection-string [options]", CosmosFlags(&settings), func(_ *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - return cosmos.NewConn(settings), nil + return CreateHelper("CosmosDB", "mongodb://cosmos-connection-string [options]", connectorsv1.CosmosFlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseCosmosFlags(c) + if err != nil { + return nil, err + } + return cosmos.NewConn(cosmosSettingsFromFlags(flags, args[0], as)), nil })(args, as) }, }, @@ -484,21 +341,12 @@ func GetRegisteredConnectors() []RegisteredConnector { return false }, Create: func(args []string, as AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, []string, error) { - settings := mongo.ConnectorSettings{ConnectionString: args[0]} - return CreateHelper("MongoDB", "mongodb://connection-string [options]", append(MongoFlags(&settings), []cli.Flag{ - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "sample-factor", - Destination: &settings.SampleFactor, - Usage: "Number of extra samples per partition", - Value: 10, - }), - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "per-namespace-streams", - Usage: "Each namespace has a separate stream", - Destination: &settings.PerNamespaceStreams, - }), - }...), func(_ *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - return mongo.NewConn(settings) + return CreateHelper("MongoDB", "mongodb://connection-string [options]", connectorsv1.MongoFlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseMongoFlags(c) + if err != nil { + return nil, err + } + return mongo.NewConn(mongoSettingsFromFlags(flags, args[0])) })(args, as) }, }, @@ -508,13 +356,12 @@ func GetRegisteredConnectors() []RegisteredConnector { return strings.HasPrefix(s, "postgres://") || strings.HasPrefix(s, "postgresql://") }, Create: func(args []string, as AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, []string, error) { - settings := postgresSettingsDefault - return CreateHelper("Postgres", postgresUsage, PostgresFlags(&settings), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - if c.Bool("manual") { - settings.Force = false + return CreateHelper("Postgres", postgresUsage, connectorsv1.PostgresFlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParsePostgresFlags(c) + if err != nil { + return nil, err } - settings.URL = args[0] - return postgres.NewConn(c.Context, settings) + return postgres.NewConn(c.Context, postgresSettingsFromFlags(flags, args[0])) })(args, as) }, }, @@ -641,43 +488,12 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "kafka-dst") }, - Create: CreateHelper("kafka-dst", "kafka-dst", []cli.Flag{ - altsrc.NewStringSliceFlag(&cli.StringSliceFlag{ - Name: "brokers", - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "default-topic", - }), - altsrc.NewStringSliceFlag(&cli.StringSliceFlag{ - Name: "namespace-topic", - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "sasl-user", - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "sasl-password", - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "data-type", - Value: adiomv1.DataType_DATA_TYPE_MONGO_BSON.String(), - }), - }, func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - brokers := c.StringSlice("brokers") - defaultTopic := c.String("default-topic") - namespaceTopic := c.StringSlice("namespace-topic") - user := c.String("sasl-user") - password := c.String("sasl-password") - dataType := adiomv1.DataType(adiomv1.DataType_value[c.String("data-type")]) - - tm := map[string]string{} - for _, m := range namespaceTopic { - ns, topic, ok := strings.Cut(m, ":") - if !ok { - return nil, fmt.Errorf("invalid topic mapping %v", m) - } - tm[ns] = topic + Create: CreateHelper("kafka-dst", "kafka-dst", connectorsv1.KafkaDstFlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseKafkaDstFlags(c) + if err != nil { + return nil, err } - return kafka.NewDestKafka(brokers, defaultTopic, tm, user, password, dataType) + return kafkaDstFromFlags(flags) }), }, { @@ -685,8 +501,12 @@ func GetRegisteredConnectors() []RegisteredConnector { IsConnector: func(s string) bool { return strings.EqualFold(s, "kafka-src") }, - Create: CreateHelper("kafka-src", "kafka-src", KafkaSrcFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { - return ParseKafkaSrcFlags(c) + Create: CreateHelper("kafka-src", "kafka-src", connectorsv1.KafkaSrcFlagsCommandFlags(), func(c *cli.Context, args []string, _ AdditionalSettings) (adiomv1connect.ConnectorServiceHandler, error) { + flags, err := connectorsv1.ParseKafkaSrcFlags(c) + if err != nil { + return nil, err + } + return kafkaSrcFromFlags(flags) }), }, { @@ -719,11 +539,14 @@ func KafkaWrap(args []string, as AdditionalSettings) (adiomv1connect.ConnectorSe HelpName: "kafka-wrap", Usage: "Connector", UsageText: "kafka-wrap [options] [source-connector] [connector-options]", - Flags: KafkaSrcFlags(), + Flags: connectorsv1.KafkaSrcFlagsCommandFlags(), Action: func(c *cli.Context) error { - var err error restArgs = c.Args().Slice() - conn, err = ParseKafkaSrcFlags(c) + flags, err := connectorsv1.ParseKafkaSrcFlags(c) + if err != nil { + return err + } + conn, err = kafkaSrcFromFlags(flags) if err != nil { return err } @@ -767,56 +590,33 @@ func KafkaWrap(args []string, as AdditionalSettings) (adiomv1connect.ConnectorSe return nil, nil, ErrHelp } -func ParseKafkaSrcFlags(c *cli.Context) (adiomv1connect.ConnectorServiceHandler, error) { - brokers := c.StringSlice("brokers") - topics := c.StringSlice("topics") - topicMappings := c.StringSlice("topic-mappings") - user := c.String("sasl-user") - password := c.String("sasl-password") - kafkaOffset := c.Int64("offset") - dataType := adiomv1.DataType(adiomv1.DataType_value[c.String("data-type")]) - +func kafkaSrcFromFlags(f *connectorsv1.KafkaSrcFlags) (adiomv1connect.ConnectorServiceHandler, error) { + dataType := adiomv1.DataType(adiomv1.DataType_value[f.DataType]) tm := map[string][]string{} - for _, topic := range topics { + for _, topic := range f.Topics { tm[topic] = nil } - for _, m := range topicMappings { + for _, m := range f.TopicMappings { topic, ns, ok := strings.Cut(m, ":") if !ok { return nil, fmt.Errorf("invalid topic mapping %v", m) } tm[topic] = append(tm[topic], ns) } - return kafka.NewKafkaConn(brokers, tm, kafka.DsyncMessageToUpdate, kafka.DsyncMessageToNamespace, user, password, kafkaOffset, dataType), nil + return kafka.NewKafkaConn(f.Brokers, tm, kafka.DsyncMessageToUpdate, kafka.DsyncMessageToNamespace, f.SaslUser, f.SaslPassword, f.Offset, dataType), nil } -func KafkaSrcFlags() []cli.Flag { - return []cli.Flag{ - altsrc.NewStringSliceFlag(&cli.StringSliceFlag{ - Name: "brokers", - }), - altsrc.NewStringSliceFlag(&cli.StringSliceFlag{ - Name: "topics", - }), - altsrc.NewStringSliceFlag(&cli.StringSliceFlag{ - Name: "topic-mappings", - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "sasl-user", - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "sasl-password", - }), - altsrc.NewInt64Flag(&cli.Int64Flag{ - Name: "offset", - Usage: "Custom offset for kafka (a time, or -2 for oldest)", - Value: sarama.OffsetNewest, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "data-type", - Value: adiomv1.DataType_DATA_TYPE_MONGO_BSON.String(), - }), +func kafkaDstFromFlags(f *connectorsv1.KafkaDstFlags) (adiomv1connect.ConnectorServiceHandler, error) { + dataType := adiomv1.DataType(adiomv1.DataType_value[f.DataType]) + tm := map[string]string{} + for _, m := range f.NamespaceTopic { + ns, topic, ok := strings.Cut(m, ":") + if !ok { + return nil, fmt.Errorf("invalid topic mapping %v", m) + } + tm[ns] = topic } + return kafka.NewDestKafka(f.Brokers, f.DefaultTopic, tm, f.SaslUser, f.SaslPassword, dataType) } func WeaviateFlags() []cli.Flag { @@ -849,189 +649,111 @@ func WeaviateFlags() []cli.Flag { } } -func FileFlags(settings *fileconnector.ConnectorSettings) []cli.Flag { - var delimiterStr string - return []cli.Flag{ - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "format", - Usage: "Output format for stored files (only 'csv' supported)", - Value: "csv", - Destination: &settings.Format, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "delimiter", - Usage: "CSV delimiter character", - Value: ",", - Destination: &delimiterStr, - Action: func(_ *cli.Context, v string) error { - if len(v) != 1 { - return fileconnector.ErrInvalidDelimiter - } - settings.Delimiter = rune(v[0]) - return nil - }, - }), - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "batch-size", - Usage: "Number of records to read per batch when listing data", - Value: fileconnector.DefaultBatchSize, - Destination: &settings.BatchSize, - }), +func fileSettingsFromFlags(f *connectorsv1.FileFlags, uri string) (fileconnector.ConnectorSettings, error) { + if f.Delimiter != "" && len(f.Delimiter) != 1 { + return fileconnector.ConnectorSettings{}, fileconnector.ErrInvalidDelimiter } + s := fileconnector.ConnectorSettings{ + Uri: uri, + Format: f.Format, + BatchSize: int(f.BatchSize), + } + if len(f.Delimiter) == 1 { + s.Delimiter = rune(f.Delimiter[0]) + } + return s, nil } -func S3Flags(settings *s3connector.ConnectorSettings) []cli.Flag { - return []cli.Flag{ - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "pretty-json", - Usage: "Pretty-print JSON output data", - Value: true, - Destination: &settings.PrettyJSON, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "region", - Usage: "AWS region for the target bucket", - Required: true, - Destination: &settings.Region, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "prefix", - Usage: "Override or append to the key prefix derived from the connection string", - Destination: &settings.Prefix, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "output-format", - Usage: "Output format for stored objects (only 'json' supported)", - Value: "json", - Destination: &settings.OutputFormat, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "profile", - Usage: "Shared config profile", - Destination: &settings.Profile, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "endpoint", - Usage: "Custom S3 endpoint (for Localstack or S3-compatible services)", - Destination: &settings.Endpoint, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "access-key-id", - Usage: "Static AWS access key ID", - Destination: &settings.AccessKeyID, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "secret-access-key", - Usage: "Static AWS secret access key", - Destination: &settings.SecretAccessKey, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "session-token", - Usage: "Static AWS session token", - Destination: &settings.SessionToken, - }), - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "use-path-style", - Usage: "Use path-style addressing (useful for Localstack/minio)", - Destination: &settings.UsePathStyle, - }), - altsrc.NewInt64Flag(&cli.Int64Flag{ - Name: "max-file-size", - Usage: "Maximum size of a single file in S3 (in MB)", - Value: 10, - Destination: &settings.MaxFileSizeMB, - }), - altsrc.NewInt64Flag(&cli.Int64Flag{ - Name: "max-total-memory", - Usage: "Maximum total memory for batching before flushing to S3 (in MB)", - Value: 100, - Destination: &settings.MaxTotalMemoryMB, - }), +func s3SettingsFromFlags(f *connectorsv1.S3Flags, uri string) s3connector.ConnectorSettings { + return s3connector.ConnectorSettings{ + Uri: uri, + PrettyJSON: f.PrettyJson, + Region: f.Region, + Prefix: f.Prefix, + OutputFormat: f.OutputFormat, + Profile: f.Profile, + Endpoint: f.Endpoint, + AccessKeyID: f.AccessKeyId, + SecretAccessKey: f.SecretAccessKey, + SessionToken: f.SessionToken, + UsePathStyle: f.UsePathStyle, + MaxFileSizeMB: f.MaxFileSize, + MaxTotalMemoryMB: f.MaxTotalMemory, } } -func CosmosFlags(settings *cosmos.ConnectorSettings) []cli.Flag { - return append(MongoFlags(&settings.ConnectorSettings), []cli.Flag{ - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "cosmos-reader-max-namespaces", - Usage: "maximum number of namespaces that can be copied from the CosmosDB connector. Recommended to keep this number under 15 to avoid performance issues.", - Value: cosmosDefaultMaxNumNamespaces, - Required: false, - Destination: &settings.MaxNumNamespaces, - Category: "Cosmos DB-specific Options", - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "cosmos-deletes-cdc", - Usage: "witness connection string to generate CDC events for CosmosDB deletes", - Category: "Cosmos DB-specific Options", - Action: func(_ *cli.Context, v string) error { - settings.EmulateDeletes = true - settings.WitnessMongoConnString = v - return nil - }, - }), - altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "cosmos-delete-interval", - Required: false, - Destination: &settings.DeletesCheckInterval, - }), - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "cosmos-parallel-partition-workers", - Required: false, - Destination: &settings.NumParallelPartitionWorkers, - }), - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "cosmos-stream-deletes-enabled", - Usage: "If this cosmos instance supports deletes in the change stream (currently a preview feature)", - Destination: &settings.WithDelete, - }), - }...) +func mongoBaseSettingsFromFlags(f *connectorsv1.MongoBaseFlags, connString string) mongo.ConnectorSettings { + s := mongo.ConnectorSettings{ConnectionString: connString} + if f == nil { + return s + } + s.SkipBatchOverwrite = f.SkipBatchOverwrite + s.FullDocumentKey = f.FullDocumentKey + s.ServerConnectTimeout = f.ServerTimeout.AsDuration() + s.PingTimeout = f.PingTimeout.AsDuration() + s.WriterMaxBatchSize = int(f.WriterBatchSize) + s.TargetDocCountPerPartition = f.DocPartition + s.MaxPageSize = int(f.MaxPageSize) + s.Query = f.InitialSyncQuery + return s } -func MongoFlags(settings *mongo.ConnectorSettings) []cli.Flag { - return []cli.Flag{ - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "skip-batch-overwrite", - Destination: &settings.SkipBatchOverwrite, - }), - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "full-document-key", - Usage: "uses the full document key instead of just _id (except for batch overwrites)", - Destination: &settings.FullDocumentKey, - }), - altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "server-timeout", - Required: false, - Destination: &settings.ServerConnectTimeout, - }), - altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "ping-timeout", - Required: false, - Destination: &settings.PingTimeout, - }), - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "writer-batch-size", - Required: false, - Destination: &settings.WriterMaxBatchSize, - }), - altsrc.NewInt64Flag(&cli.Int64Flag{ - Name: "doc-partition", - Required: false, - Destination: &settings.TargetDocCountPerPartition, - Value: 50 * 1000, - }), - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "max-page-size", - Destination: &settings.MaxPageSize, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "initial-sync-query", - Usage: "query filter for the initial data copy (v2 Extended JSON)", - Aliases: []string{"q"}, - Required: false, - Destination: &settings.Query, - }), +func mongoSettingsFromFlags(f *connectorsv1.MongoFlags, connString string) mongo.ConnectorSettings { + s := mongoBaseSettingsFromFlags(f.Base, connString) + s.SampleFactor = int(f.SampleFactor) + s.PerNamespaceStreams = f.PerNamespaceStreams + return s +} + +func cosmosSettingsFromFlags(f *connectorsv1.CosmosFlags, connString string, as AdditionalSettings) cosmos.ConnectorSettings { + s := cosmos.ConnectorSettings{ + ConnectorSettings: mongoBaseSettingsFromFlags(f.Mongo, connString), + MaxNumNamespaces: int(f.CosmosReaderMaxNamespaces), + NumParallelPartitionWorkers: int(f.CosmosParallelPartitionWorkers), + WithDelete: f.CosmosStreamDeletesEnabled, + } + if as.BaseThreadCount != 0 && s.NumParallelPartitionWorkers == 0 { + s.NumParallelPartitionWorkers = as.BaseThreadCount / 2 + } + if f.CosmosDeleteInterval != nil { + s.DeletesCheckInterval = f.CosmosDeleteInterval.AsDuration() + } + if f.CosmosDeletesCdc != "" { + s.EmulateDeletes = true + s.WitnessMongoConnString = f.CosmosDeletesCdc + } + return s +} + +func fakeSourceFromFlags(f *connectorsv1.FakeSourceFlags) (adiomv1connect.ConnectorServiceHandler, error) { + var m map[string]any + if len(f.Payload) > 0 { + m = map[string]any{} + } + for _, p := range f.Payload { + k, v, _ := strings.Cut(p, ":") + m[k] = v + } + if f.PayloadJson != "" { + if err := json.Unmarshal([]byte(f.PayloadJson), &m); err != nil { + return nil, fmt.Errorf("err unmarshalling payload-json: %w", err) + } } + return random.NewConnV2(random.ConnV2Input{ + NamespacePrefix: f.NamespacePrefix, + NumNamespaces: int(f.NumNamespaces), + NumPartitionsPerNamespace: int(f.NumPartitionsPerNamespace), + NumUpdatePartitionsPerNamespace: int(f.NumUpdatePartitionsPerNamespace), + BatchSize: int(f.BatchSize), + UpdateBatchSize: int(f.UpdateBatchSize), + MaxUpdatesPerTick: int(f.MaxUpdatesPerTick), + NumDocsPerPartition: f.NumDocsPerPartition, + Payload: m, + Sleep: f.Sleep.AsDuration(), + Jitter: f.Jitter.AsDuration(), + UpdateDuration: f.UpdateDuration.AsDuration(), + StreamTick: f.StreamTick.AsDuration(), + }), nil } var postgresUsage = `postgresql://user:pass@host:port [options] @@ -1049,78 +771,25 @@ Destination: Not currently supported ` -var postgresSettingsDefault = postgres.PostgresSettings{ - Force: true, - SlotName: "dsync_slot", - PublicationName: "dsync_pub", - Limit: 1000, - StreamMaxBatchWait: time.Second * 5, - StreamMaxBatchSize: 100, - StreamFlushDelay: time.Minute * 3, - EstimatedCountThreshold: 1000000, - TargetDocCountPerPartition: 100000, - EnableReplicaMode: true, -} - -func PostgresFlags(settings *postgres.PostgresSettings) []cli.Flag { - return []cli.Flag{ - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "page-size", - Usage: "Specify pagination limit when fetching for initial sync", - Value: postgresSettingsDefault.Limit, - Destination: &settings.Limit, - }), - altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "stream-max-batch-wait", - Usage: "Force flush a stream batch after this interval", - Value: postgresSettingsDefault.StreamMaxBatchWait, - Destination: &settings.StreamMaxBatchWait, - }), - altsrc.NewIntFlag(&cli.IntFlag{ - Name: "stream-max-batch-size", - Usage: "Force flush a stream batch at this limit", - Value: postgresSettingsDefault.StreamMaxBatchSize, - Destination: &settings.StreamMaxBatchSize, - }), - altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "stream-flush-delay", - Usage: "Delay before notifying postgres backend of stream progress. This should be comfortably larger than saving streaming cursor updates.", - Value: postgresSettingsDefault.StreamFlushDelay, - Destination: &settings.StreamFlushDelay, - }), - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "manual", - Usage: "Use to not recreate replication slot and publication (e.g. if you are managing these outside). You will still need to clean up the slot later even if you don't use this.", - Value: !postgresSettingsDefault.Force, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "replication-slot", - Value: postgresSettingsDefault.SlotName, - Destination: &postgresSettingsDefault.SlotName, - }), - altsrc.NewStringFlag(&cli.StringFlag{ - Name: "publication-name", - Value: postgresSettingsDefault.PublicationName, - Destination: &postgresSettingsDefault.PublicationName, - }), - altsrc.NewInt64Flag(&cli.Int64Flag{ - Name: "estimated-count-threshold", - Usage: "If estimated count is less than this, try a full count.", - Value: postgresSettingsDefault.EstimatedCountThreshold, - Destination: &settings.EstimatedCountThreshold, - }), - altsrc.NewInt64Flag(&cli.Int64Flag{ - Name: "doc-partition", - Value: postgresSettingsDefault.TargetDocCountPerPartition, - Destination: &settings.TargetDocCountPerPartition, - }), - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "enable-replica-mode", - Usage: "Enable replica mode (SET session_replication_role = 'replica') for sink operations. This disables triggers and rules.", - Value: postgresSettingsDefault.EnableReplicaMode, - Destination: &settings.EnableReplicaMode, - }), +func postgresSettingsFromFlags(f *connectorsv1.PostgresFlags, url string) postgres.PostgresSettings { + s := postgres.PostgresSettings{ + URL: url, + Force: !f.Manual, + SlotName: f.ReplicationSlot, + PublicationName: f.PublicationName, + Limit: int(f.PageSize), + StreamMaxBatchSize: int(f.StreamMaxBatchSize), + EstimatedCountThreshold: f.EstimatedCountThreshold, + TargetDocCountPerPartition: f.DocPartition, + EnableReplicaMode: f.EnableReplicaMode, + } + if f.StreamMaxBatchWait != nil { + s.StreamMaxBatchWait = f.StreamMaxBatchWait.AsDuration() + } + if f.StreamFlushDelay != nil { + s.StreamFlushDelay = f.StreamFlushDelay.AsDuration() } + return s } func insecureClient() *http.Client { diff --git a/internal/app/options/flags.go b/internal/app/options/flags.go index 6d5f92c7..59f1836d 100644 --- a/internal/app/options/flags.go +++ b/internal/app/options/flags.go @@ -29,8 +29,7 @@ const ( DefaultPprofPort = 8081 // DefaultWebPort is the default port for the web server. DefaultWebPort = 8080 - // DefaultMaxNumNamespaces is the default maximum number of namespaces that can be copied from the CosmosDB connector. - cosmosDefaultMaxNumNamespaces = 8 + ) var validModes = []string{iface.SyncModeFull, iface.SyncModeCDC, iface.SyncModeInitialSync} diff --git a/internal/app/options/grpc.go b/internal/app/options/grpc.go index 1d024a02..9d6d6380 100644 --- a/internal/app/options/grpc.go +++ b/internal/app/options/grpc.go @@ -7,9 +7,9 @@ import ( "strings" "connectrpc.com/connect" + connectorsv1 "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1" "github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect" "github.com/urfave/cli/v2" - "github.com/urfave/cli/v2/altsrc" ) var GRPCConnector = ConfigureGRPCFactory("Connector", nil, func(c connect.HTTPClient, s string, opts ...connect.ClientOption) interface{} { @@ -69,29 +69,24 @@ func ConfigureGRPCFactory(usage string, missingErr error, f func(connect.HTTPCli HelpName: "grpc", Usage: usage, UsageText: "grpc://address:port [options]", - Flags: []cli.Flag{ - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "insecure", - Usage: "Connect without TLS", - }), - altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "gzip", - Usage: "Use gzip on requests", - }), - }, + Flags: connectorsv1.GrpcFlagsCommandFlags(), Action: func(c *cli.Context) error { restArgs = c.Args().Slice() + flags, err := connectorsv1.ParseGrpcFlags(c) + if err != nil { + return err + } _, endpoint, ok := strings.Cut(args[0], "://") if !ok { return fmt.Errorf("invalid connection string %v", args[0]) } options := []connect.ClientOption{connect.WithGRPC()} - if c.Bool("gzip") { + if flags.Gzip { options = append(options, connect.WithSendGzip()) } finalEndpoint := "https://" + endpoint httpClient := http.DefaultClient - if c.Bool("insecure") { + if flags.Insecure { httpClient = insecureClient() finalEndpoint = "http://" + endpoint } diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/cosmos.proto b/proto/connectorcommands/adiom/commands/connectors/v1/cosmos.proto new file mode 100644 index 00000000..383c383b --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/cosmos.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; +import "google/protobuf/duration.proto"; +import "adiom/commands/connectors/v1/mongo.proto"; + +message CosmosFlags { + option (commandargs.v1.command) = { + name: "CosmosDB" + usage: "mongodb://cosmos-connection-string [options]" + }; + + MongoBaseFlags mongo = 1; + + int32 cosmos_reader_max_namespaces = 2 [(commandargs.v1.flag) = { + name: "cosmos-reader-max-namespaces" + default: "8" + usage: "max namespaces to copy (keep under 15)" + category: "Cosmos DB-specific Options" + }]; + string cosmos_deletes_cdc = 3 [(commandargs.v1.flag) = { + name: "cosmos-deletes-cdc" + usage: "witness connection string for CosmosDB deletes" + category: "Cosmos DB-specific Options" + }]; + google.protobuf.Duration cosmos_delete_interval = 4 [(commandargs.v1.flag) = { + name: "cosmos-delete-interval" + category: "Cosmos DB-specific Options" + }]; + int32 cosmos_parallel_partition_workers = 5 [(commandargs.v1.flag) = { + name: "cosmos-parallel-partition-workers" + }]; + bool cosmos_stream_deletes_enabled = 6 [(commandargs.v1.flag) = { + name: "cosmos-stream-deletes-enabled" + usage: "If this cosmos instance supports deletes in the change stream" + category: "Cosmos DB-specific Options" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/devnull.proto b/proto/connectorcommands/adiom/commands/connectors/v1/devnull.proto new file mode 100644 index 00000000..3b3801d1 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/devnull.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; +import "google/protobuf/duration.proto"; + +message DevNullFlags { + option (commandargs.v1.command) = { + name: "/dev/null" + }; + + google.protobuf.Duration sleep = 1 [(commandargs.v1.flag) = { + usage: "Sleep time between requests" + }]; + google.protobuf.Duration sleep_jitter = 2 [(commandargs.v1.flag) = { + name: "sleep-jitter" + usage: "If sleep is set, additional jitter" + }]; + bool log_json = 3 [(commandargs.v1.flag) = { + name: "log-json" + usage: "Convert data to json and log INFO" + }]; + string id = 4 [(commandargs.v1.flag) = { + usage: "A fixed id for the connector" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/devrandom.proto b/proto/connectorcommands/adiom/commands/connectors/v1/devrandom.proto new file mode 100644 index 00000000..706cf475 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/devrandom.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message DevRandomFlags { + option (commandargs.v1.command) = { + name: "/dev/random" + }; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/dynamodb.proto b/proto/connectorcommands/adiom/commands/connectors/v1/dynamodb.proto new file mode 100644 index 00000000..8afd2cc3 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/dynamodb.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message DynamoDBFlags { + option (commandargs.v1.command) = { + name: "DynamoDB" + usage: "dynamodb OR dynamodb://localstack" + }; + + int32 doc_partition = 1 [(commandargs.v1.flag) = { + name: "doc-partition" + usage: "Target number of documents per partition" + default: "50000" + }]; + int32 plan_parallelism = 2 [(commandargs.v1.flag) = { + name: "plan-parallelism" + usage: "Parallelism during planning" + default: "4" + }]; + string id = 3 [(commandargs.v1.flag) = { + usage: "A fixed id for the connector" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/fakesource.proto b/proto/connectorcommands/adiom/commands/connectors/v1/fakesource.proto new file mode 100644 index 00000000..2172f843 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/fakesource.proto @@ -0,0 +1,75 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; +import "google/protobuf/duration.proto"; + +message FakeSourceFlags { + option (commandargs.v1.command) = { + name: "/dev/fakesource" + }; + + google.protobuf.Duration sleep = 1 [(commandargs.v1.flag) = { + usage: "Sleep time between requests" + }]; + google.protobuf.Duration jitter = 2 [(commandargs.v1.flag) = { + usage: "Additional jitter to certain durations" + }]; + google.protobuf.Duration update_duration = 3 [(commandargs.v1.flag) = { + name: "update-duration" + usage: "Time for each update-batch-size to be produced" + default: "400ms" + }]; + google.protobuf.Duration stream_tick = 4 [(commandargs.v1.flag) = { + name: "stream-tick" + usage: "Polling time to check for stream updates" + default: "1s" + }]; + int32 num_namespaces = 5 [(commandargs.v1.flag) = { + name: "num-namespaces" + usage: "Number of namespaces (index starts at 0)" + default: "2" + }]; + int32 num_partitions_per_namespace = 6 [(commandargs.v1.flag) = { + name: "num-partitions-per-namespace" + usage: "Number of partitions per namespace" + default: "3" + }]; + int32 num_update_partitions_per_namespace = 7 [(commandargs.v1.flag) = { + name: "num-update-partitions-per-namespace" + usage: "Number of update partitions per namespace. If set to 0, use a single stream for all." + }]; + int64 num_docs_per_partition = 8 [(commandargs.v1.flag) = { + name: "num-docs-per-partition" + usage: "Number of docs per partition" + default: "1000" + }]; + int32 batch_size = 9 [(commandargs.v1.flag) = { + name: "batch-size" + usage: "Number of docs per batch" + default: "150" + }]; + int32 update_batch_size = 10 [(commandargs.v1.flag) = { + name: "update-batch-size" + usage: "Number of docs per update" + default: "50" + }]; + int32 max_updates_per_tick = 11 [(commandargs.v1.flag) = { + name: "max-updates-per-tick" + usage: "Number of docs per update" + default: "300" + }]; + string namespace_prefix = 12 [(commandargs.v1.flag) = { + name: "namespace-prefix" + usage: "Prefix for namespace" + default: "ns" + }]; + repeated string payload = 13 [(commandargs.v1.flag) = { + usage: "Payload for all items (key:value)" + }]; + string payload_json = 14 [(commandargs.v1.flag) = { + name: "payload-json" + usage: "Payload for all items as a json (overrides payload)" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/file.proto b/proto/connectorcommands/adiom/commands/connectors/v1/file.proto new file mode 100644 index 00000000..0769056d --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/file.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message FileFlags { + option (commandargs.v1.command) = { + name: "file" + usage: "file:///path/to/dir OR file:///path/to/file.csv [options]" + }; + + string format = 1 [(commandargs.v1.flag) = { + usage: "Output format for stored files (only 'csv' supported)" + default: "csv" + }]; + string delimiter = 2 [(commandargs.v1.flag) = { + usage: "CSV delimiter character" + default: "," + }]; + int32 batch_size = 3 [(commandargs.v1.flag) = { + name: "batch-size" + usage: "Number of records to read per batch when listing data" + default: "10000" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/grpc.proto b/proto/connectorcommands/adiom/commands/connectors/v1/grpc.proto new file mode 100644 index 00000000..b781aed8 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/grpc.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message GrpcFlags { + option (commandargs.v1.command) = { + name: "grpc" + usage: "grpc://address:port [options]" + }; + + bool insecure = 1 [(commandargs.v1.flag) = { + usage: "Connect without TLS" + }]; + bool gzip = 2 [(commandargs.v1.flag) = { + usage: "Use gzip on requests" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/kafka.proto b/proto/connectorcommands/adiom/commands/connectors/v1/kafka.proto new file mode 100644 index 00000000..bb32fd17 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/kafka.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message KafkaSrcFlags { + option (commandargs.v1.command) = { + name: "kafka-src" + }; + + repeated string brokers = 1; + repeated string topics = 2; + repeated string topic_mappings = 3 [(commandargs.v1.flag) = { + name: "topic-mappings" + }]; + string sasl_user = 4 [(commandargs.v1.flag) = { + name: "sasl-user" + }]; + string sasl_password = 5 [(commandargs.v1.flag) = { + name: "sasl-password" + }]; + int64 offset = 6 [(commandargs.v1.flag) = { + usage: "Custom offset for kafka (a time, or -2 for oldest)" + default: "-1" + }]; + string data_type = 7 [(commandargs.v1.flag) = { + name: "data-type" + default: "DATA_TYPE_MONGO_BSON" + }]; +} + +message KafkaDstFlags { + option (commandargs.v1.command) = { + name: "kafka-dst" + }; + + repeated string brokers = 1; + string default_topic = 2 [(commandargs.v1.flag) = { + name: "default-topic" + }]; + repeated string namespace_topic = 3 [(commandargs.v1.flag) = { + name: "namespace-topic" + }]; + string sasl_user = 4 [(commandargs.v1.flag) = { + name: "sasl-user" + }]; + string sasl_password = 5 [(commandargs.v1.flag) = { + name: "sasl-password" + }]; + string data_type = 6 [(commandargs.v1.flag) = { + name: "data-type" + default: "DATA_TYPE_MONGO_BSON" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/mongo.proto b/proto/connectorcommands/adiom/commands/connectors/v1/mongo.proto new file mode 100644 index 00000000..10497716 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/mongo.proto @@ -0,0 +1,59 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; +import "google/protobuf/duration.proto"; + +// Base flags shared between MongoDB and CosmosDB connectors +message MongoBaseFlags { + option (commandargs.v1.command) = { + name: "MongoBase" + }; + + bool skip_batch_overwrite = 1; + bool full_document_key = 2 [(commandargs.v1.flag) = { + usage: "uses the full document key instead of just _id (except for batch overwrites)" + }]; + google.protobuf.Duration server_timeout = 3 [(commandargs.v1.flag) = { + name: "server-timeout" + }]; + google.protobuf.Duration ping_timeout = 4 [(commandargs.v1.flag) = { + name: "ping-timeout" + }]; + int32 writer_batch_size = 5 [(commandargs.v1.flag) = { + name: "writer-batch-size" + }]; + int64 doc_partition = 6 [(commandargs.v1.flag) = { + name: "doc-partition" + default: "50000" + }]; + int32 max_page_size = 7 [(commandargs.v1.flag) = { + name: "max-page-size" + }]; + string initial_sync_query = 8 [(commandargs.v1.flag) = { + name: "initial-sync-query" + usage: "query filter for the initial data copy (v2 Extended JSON)" + aliases: ["q"] + }]; +} + +// Full MongoDB flags = base + MongoDB-specific extras +message MongoFlags { + option (commandargs.v1.command) = { + name: "MongoDB" + usage: "mongodb://connection-string [options]" + }; + + MongoBaseFlags base = 1; + + int32 sample_factor = 2 [(commandargs.v1.flag) = { + name: "sample-factor" + usage: "Number of extra samples per partition" + default: "10" + }]; + bool per_namespace_streams = 3 [(commandargs.v1.flag) = { + name: "per-namespace-streams" + usage: "Each namespace has a separate stream" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/postgres.proto b/proto/connectorcommands/adiom/commands/connectors/v1/postgres.proto new file mode 100644 index 00000000..4b3ce18e --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/postgres.proto @@ -0,0 +1,59 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; +import "google/protobuf/duration.proto"; + +message PostgresFlags { + option (commandargs.v1.command) = { + name: "Postgres" + usage: "postgresql://user:pass@host:port [options]" + }; + + int32 page_size = 1 [(commandargs.v1.flag) = { + name: "page-size" + usage: "Specify pagination limit when fetching for initial sync" + default: "1000" + }]; + google.protobuf.Duration stream_max_batch_wait = 2 [(commandargs.v1.flag) = { + name: "stream-max-batch-wait" + usage: "Force flush a stream batch after this interval" + default: "5s" + }]; + int32 stream_max_batch_size = 3 [(commandargs.v1.flag) = { + name: "stream-max-batch-size" + usage: "Force flush a stream batch at this limit" + default: "100" + }]; + google.protobuf.Duration stream_flush_delay = 4 [(commandargs.v1.flag) = { + name: "stream-flush-delay" + usage: "Delay before notifying postgres backend of stream progress. This should be comfortably larger than saving streaming cursor updates." + default: "3m" + }]; + bool manual = 5 [(commandargs.v1.flag) = { + usage: "Use to not recreate replication slot and publication (e.g. if you are managing these outside). You will still need to clean up the slot later even if you don't use this." + }]; + string replication_slot = 6 [(commandargs.v1.flag) = { + name: "replication-slot" + default: "dsync_slot" + }]; + string publication_name = 7 [(commandargs.v1.flag) = { + name: "publication-name" + default: "dsync_pub" + }]; + int64 estimated_count_threshold = 8 [(commandargs.v1.flag) = { + name: "estimated-count-threshold" + usage: "If estimated count is less than this, try a full count." + default: "1000000" + }]; + int64 doc_partition = 9 [(commandargs.v1.flag) = { + name: "doc-partition" + default: "100000" + }]; + bool enable_replica_mode = 10 [(commandargs.v1.flag) = { + name: "enable-replica-mode" + usage: "Enable replica mode (SET session_replication_role = 'replica') for sink operations. This disables triggers and rules." + default: "true" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/s3.proto b/proto/connectorcommands/adiom/commands/connectors/v1/s3.proto new file mode 100644 index 00000000..1c9a8100 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/s3.proto @@ -0,0 +1,62 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message S3Flags { + option (commandargs.v1.command) = { + name: "s3" + usage: "s3://bucket[/prefix] [options]" + }; + + bool pretty_json = 1 [(commandargs.v1.flag) = { + name: "pretty-json" + usage: "Pretty-print JSON output data" + default: "true" + }]; + string region = 2 [(commandargs.v1.flag) = { + usage: "AWS region for the target bucket" + required: true + }]; + string prefix = 3 [(commandargs.v1.flag) = { + usage: "Override or append to the key prefix derived from the connection string" + }]; + string output_format = 4 [(commandargs.v1.flag) = { + name: "output-format" + usage: "Output format for stored objects (only 'json' supported)" + default: "json" + }]; + string profile = 5 [(commandargs.v1.flag) = { + usage: "Shared config profile" + }]; + string endpoint = 6 [(commandargs.v1.flag) = { + usage: "Custom S3 endpoint (for Localstack or S3-compatible services)" + }]; + string access_key_id = 7 [(commandargs.v1.flag) = { + name: "access-key-id" + usage: "Static AWS access key ID" + }]; + string secret_access_key = 8 [(commandargs.v1.flag) = { + name: "secret-access-key" + usage: "Static AWS secret access key" + }]; + string session_token = 9 [(commandargs.v1.flag) = { + name: "session-token" + usage: "Static AWS session token" + }]; + bool use_path_style = 10 [(commandargs.v1.flag) = { + name: "use-path-style" + usage: "Use path-style addressing (useful for Localstack/minio)" + }]; + int64 max_file_size = 11 [(commandargs.v1.flag) = { + name: "max-file-size" + usage: "Maximum size of a single file in S3 (in MB)" + default: "10" + }]; + int64 max_total_memory = 12 [(commandargs.v1.flag) = { + name: "max-total-memory" + usage: "Maximum total memory for batching before flushing to S3 (in MB)" + default: "100" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/s3vectors.proto b/proto/connectorcommands/adiom/commands/connectors/v1/s3vectors.proto new file mode 100644 index 00000000..25ed56f3 --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/s3vectors.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message S3VectorsFlags { + option (commandargs.v1.command) = { + name: "s3vectors" + usage: "s3vector or s3vectors" + }; + + string bucket = 1 [(commandargs.v1.flag) = { + usage: "the s3 vector bucket to use" + required: true + }]; + string vector_key = 2 [(commandargs.v1.flag) = { + name: "vector-key" + usage: "The field containing the vector" + default: "data" + }]; + int32 max_parallelism = 3 [(commandargs.v1.flag) = { + name: "max-parallelism" + usage: "Limits how many concurrent requests are made to s3vectors per worker." + default: "1" + }]; + int32 rate_limit = 4 [(commandargs.v1.flag) = { + name: "rate-limit" + usage: "Max vectors per second across workers on this process per namespace (vector index)." + default: "2500" + }]; + int32 batch_size = 5 [(commandargs.v1.flag) = { + name: "batch-size" + usage: "Max size of each PutVector requests (aws hard limit is 500)." + default: "500" + }]; +} diff --git a/proto/connectorcommands/adiom/commands/connectors/v1/sqlbatch.proto b/proto/connectorcommands/adiom/commands/connectors/v1/sqlbatch.proto new file mode 100644 index 00000000..f6240a7f --- /dev/null +++ b/proto/connectorcommands/adiom/commands/connectors/v1/sqlbatch.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package adiom.commands.connectors.v1; +option go_package = "github.com/adiom-data/dsync/gen/adiom/commands/connectors/v1;connectorsv1"; + +import "commandargs/v1/options.proto"; + +message SqlbatchFlags { + option (commandargs.v1.command) = { + name: "sqlbatch" + }; + + string config = 1 [(commandargs.v1.flag) = { + required: true + default: "config.yml" + }]; +} diff --git a/proto/adiom/v1/adiom.proto b/proto/dsync/adiom/v1/adiom.proto similarity index 100% rename from proto/adiom/v1/adiom.proto rename to proto/dsync/adiom/v1/adiom.proto diff --git a/proto/adiom/v1/messages.proto b/proto/dsync/adiom/v1/messages.proto similarity index 100% rename from proto/adiom/v1/messages.proto rename to proto/dsync/adiom/v1/messages.proto diff --git a/proto/adiom/v1/vector.proto b/proto/dsync/adiom/v1/vector.proto similarity index 100% rename from proto/adiom/v1/vector.proto rename to proto/dsync/adiom/v1/vector.proto diff --git a/scripts/devsetup.sh b/scripts/devsetup.sh index 85008cc3..6d291554 100755 --- a/scripts/devsetup.sh +++ b/scripts/devsetup.sh @@ -3,4 +3,5 @@ go install github.com/bufbuild/buf/cmd/buf@latest go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest go install google.golang.org/protobuf/cmd/protoc-gen-go@latest go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest -go install connectrpc.com/connect/cmd/protoc-gen-connect-go@latest \ No newline at end of file +go install connectrpc.com/connect/cmd/protoc-gen-connect-go@latest +go install github.com/adiom-data/commandargs/cmd/protoc-gen-commandargs-urfave@latest