diff --git a/backend/modules/llm/application/convertor/manage.go b/backend/modules/llm/application/convertor/manage.go index 035788dbae..6c3d1af071 100644 --- a/backend/modules/llm/application/convertor/manage.go +++ b/backend/modules/llm/application/convertor/manage.go @@ -175,6 +175,8 @@ func AbilityMultiModalDO2DTO(a *entity.AbilityMultiModal) *manage.AbilityMultiMo return &manage.AbilityMultiModal{ Image: ptr.Of(a.Image), AbilityImage: AbilityImageDO2DTO(a.AbilityImage), + Video: ptr.Of(a.Video), + AbilityVideo: AbilityVideoDO2DTO(a.AbilityVideo), } } @@ -190,6 +192,18 @@ func AbilityImageDO2DTO(a *entity.AbilityImage) *manage.AbilityImage { } } +func AbilityVideoDO2DTO(a *entity.AbilityVideo) *manage.AbilityVideo { + if a == nil { + return nil + } + return &manage.AbilityVideo{ + MaxVideoSizeInMb: ptr.Of(a.MaxVideoSizeInMB), + SupportedVideoFormats: slices.Transform(a.SupportedVideoFormats, func(format entity.VideoFormat, _ int) manage.VideoFormat { + return manage.VideoFormat(format) + }), + } +} + func ProtocolConfigDO2DTO(p *entity.ProtocolConfig) *manage.ProtocolConfig { if p == nil { return nil diff --git a/backend/modules/llm/application/convertor/manage_test.go b/backend/modules/llm/application/convertor/manage_test.go index 6d2b8dc482..4b0992cbbd 100644 --- a/backend/modules/llm/application/convertor/manage_test.go +++ b/backend/modules/llm/application/convertor/manage_test.go @@ -283,10 +283,18 @@ func TestAbilityMultiModalDO2DTO(t *testing.T) { AbilityImage: &entity.AbilityImage{ URLEnabled: true, }, + Video: true, + AbilityVideo: &entity.AbilityVideo{ + MaxVideoSizeInMB: 200, + SupportedVideoFormats: []entity.VideoFormat{"mp4", "webm"}, + }, } got := AbilityMultiModalDO2DTO(a) assert.True(t, *got.Image) assert.True(t, *got.AbilityImage.URLEnabled) + assert.True(t, *got.Video) + assert.Equal(t, int32(200), *got.AbilityVideo.MaxVideoSizeInMb) + assert.Equal(t, []manage.VideoFormat{"mp4", "webm"}, got.AbilityVideo.SupportedVideoFormats) }) } diff --git a/backend/modules/llm/application/convertor/runtime.go b/backend/modules/llm/application/convertor/runtime.go index 18222fce76..619fafae0a 100644 --- a/backend/modules/llm/application/convertor/runtime.go +++ b/backend/modules/llm/application/convertor/runtime.go @@ -96,6 +96,7 @@ func ChatMessagePartDTO2DO(dto *druntime.ChatMessagePart) (do *entity.ChatMessag Type: entity.ChatMessagePartType(dto.GetType()), Text: dto.GetText(), ImageURL: ChatMessageImageURLDTO2DO(dto.GetImageURL()), + VideoURL: ChatMessageVideoURLDTO2DO(dto.GetVideoURL()), } } @@ -110,6 +111,24 @@ func ChatMessageImageURLDTO2DO(dto *druntime.ChatMessageImageURL) (do *entity.Ch } } +func ChatMessageVideoURLDTO2DO(dto *druntime.ChatMessageVideoURL) (do *entity.ChatMessageVideoURL) { + if dto == nil { + return nil + } + return &entity.ChatMessageVideoURL{ + URL: dto.GetURL(), + Detail: VideoURLDetailDTO2DO(dto.GetDetail()), + MIMEType: dto.GetMimeType(), + } +} + +func VideoURLDetailDTO2DO(dto *druntime.VideoURLDetail) (do *entity.VideoURLDetail) { + if dto == nil { + return nil + } + return &entity.VideoURLDetail{FPS: dto.GetFps()} +} + func MessageDO2DTO(do *entity.Message) (dto *druntime.Message) { if do == nil { return nil @@ -188,6 +207,7 @@ func ChatMessagePartDO2DTO(do *entity.ChatMessagePart) (dto *druntime.ChatMessag Type: ptr.Of(druntime.ChatMessagePartType(do.Type)), Text: ptr.Of(do.Text), ImageURL: ChatMessageImageURLDO2DTO(do.ImageURL), + VideoURL: ChatMessageVideoURLDO2DTO(do.VideoURL), } } @@ -201,3 +221,21 @@ func ChatMessageImageURLDO2DTO(do *entity.ChatMessageImageURL) (dto *druntime.Ch MimeType: ptr.Of(do.MIMEType), } } + +func ChatMessageVideoURLDO2DTO(do *entity.ChatMessageVideoURL) (dto *druntime.ChatMessageVideoURL) { + if do == nil { + return nil + } + return &druntime.ChatMessageVideoURL{ + URL: ptr.Of(do.URL), + Detail: VideoURLDetailDO2DTO(do.Detail), + MimeType: ptr.Of(do.MIMEType), + } +} + +func VideoURLDetailDO2DTO(do *entity.VideoURLDetail) (dto *druntime.VideoURLDetail) { + if do == nil { + return nil + } + return &druntime.VideoURLDetail{Fps: ptr.Of(do.FPS)} +} diff --git a/backend/modules/llm/application/convertor/runtime_test.go b/backend/modules/llm/application/convertor/runtime_test.go index 3a9f397d1c..051ff29b89 100644 --- a/backend/modules/llm/application/convertor/runtime_test.go +++ b/backend/modules/llm/application/convertor/runtime_test.go @@ -128,21 +128,44 @@ func TestToolCallConvert(t *testing.T) { } func TestChatMessagePartConvert(t *testing.T) { - dto := &druntime.ChatMessagePart{ - Type: gptr.Of(druntime.ChatMessagePartTypeImageURL), - ImageURL: &druntime.ChatMessageImageURL{ - URL: gptr.Of("http://img.com"), - Detail: gptr.Of(druntime.ImageURLDetailHigh), - MimeType: gptr.Of("image/png"), - }, - } - do := ChatMessagePartDTO2DO(dto) - assert.Equal(t, entity.ChatMessagePartTypeImageURL, do.Type) - assert.Equal(t, "http://img.com", do.ImageURL.URL) - assert.Equal(t, entity.ImageURLDetailHigh, do.ImageURL.Detail) + t.Run("image", func(t *testing.T) { + dto := &druntime.ChatMessagePart{ + Type: gptr.Of(druntime.ChatMessagePartTypeImageURL), + ImageURL: &druntime.ChatMessageImageURL{ + URL: gptr.Of("http://img.com"), + Detail: gptr.Of(druntime.ImageURLDetailHigh), + MimeType: gptr.Of("image/png"), + }, + } + do := ChatMessagePartDTO2DO(dto) + assert.Equal(t, entity.ChatMessagePartTypeImageURL, do.Type) + assert.Equal(t, "http://img.com", do.ImageURL.URL) + assert.Equal(t, entity.ImageURLDetailHigh, do.ImageURL.Detail) + + dto2 := ChatMessagePartDO2DTO(do) + assert.Equal(t, druntime.ChatMessagePartTypeImageURL, *dto2.Type) + assert.Equal(t, "http://img.com", *dto2.ImageURL.URL) + assert.Equal(t, druntime.ImageURLDetailHigh, *dto2.ImageURL.Detail) + }) - dto2 := ChatMessagePartDO2DTO(do) - assert.Equal(t, druntime.ChatMessagePartTypeImageURL, *dto2.Type) - assert.Equal(t, "http://img.com", *dto2.ImageURL.URL) - assert.Equal(t, druntime.ImageURLDetailHigh, *dto2.ImageURL.Detail) + t.Run("video", func(t *testing.T) { + dto := &druntime.ChatMessagePart{ + Type: gptr.Of(druntime.ChatMessagePartTypeVideoURL), + VideoURL: &druntime.ChatMessageVideoURL{ + URL: gptr.Of("https://example.com/video.mp4"), + Detail: &druntime.VideoURLDetail{Fps: gptr.Of(2.5)}, + MimeType: gptr.Of("video/mp4"), + }, + } + do := ChatMessagePartDTO2DO(dto) + assert.Equal(t, entity.ChatMessagePartTypeVideoURL, do.Type) + assert.Equal(t, "https://example.com/video.mp4", do.VideoURL.URL) + assert.Equal(t, 2.5, do.VideoURL.Detail.FPS) + + dto2 := ChatMessagePartDO2DTO(do) + assert.Equal(t, druntime.ChatMessagePartTypeVideoURL, *dto2.Type) + assert.Equal(t, "https://example.com/video.mp4", *dto2.VideoURL.URL) + assert.Equal(t, 2.5, *dto2.VideoURL.Detail.Fps) + assert.Equal(t, "video/mp4", *dto2.VideoURL.MimeType) + }) } diff --git a/backend/modules/llm/domain/entity/eino_convertor.go b/backend/modules/llm/domain/entity/eino_convertor.go index 5326be9417..192c41e9ba 100644 --- a/backend/modules/llm/domain/entity/eino_convertor.go +++ b/backend/modules/llm/domain/entity/eino_convertor.go @@ -52,6 +52,7 @@ func FromDOChatMsgPart(p *ChatMessagePart) schema.ChatMessagePart { Type: schema.ChatMessagePartType(p.Type), Text: p.Text, ImageURL: FromDOImageURL(p.ImageURL), + VideoURL: FromDOVideoURL(p.VideoURL), } } @@ -66,6 +67,16 @@ func FromDOImageURL(p *ChatMessageImageURL) *schema.ChatMessageImageURL { } } +func FromDOVideoURL(p *ChatMessageVideoURL) *schema.ChatMessageVideoURL { + if p == nil { + return nil + } + return &schema.ChatMessageVideoURL{ + URL: p.URL, + MIMEType: p.MIMEType, + } +} + func FromDOToolCalls(ts []*ToolCall) []schema.ToolCall { return slices.Transform(ts, func(t *ToolCall, _ int) schema.ToolCall { return FromDOToolCall(t) @@ -265,6 +276,7 @@ func ToDOMultiContent(cm schema.ChatMessagePart) *ChatMessagePart { Type: ChatMessagePartType(cm.Type), Text: cm.Text, ImageURL: ToDOImageURL(cm.ImageURL), + VideoURL: ToDOVideoURL(cm.VideoURL), } } @@ -279,6 +291,16 @@ func ToDOImageURL(cm *schema.ChatMessageImageURL) *ChatMessageImageURL { } } +func ToDOVideoURL(cm *schema.ChatMessageVideoURL) *ChatMessageVideoURL { + if cm == nil { + return nil + } + return &ChatMessageVideoURL{ + URL: cm.URL, + MIMEType: cm.MIMEType, + } +} + func ToDORespMeta(rm *schema.ResponseMeta) *ResponseMeta { if rm == nil { return nil diff --git a/backend/modules/llm/domain/entity/eino_convertor_test.go b/backend/modules/llm/domain/entity/eino_convertor_test.go index 974e209623..8b7bdbbf2e 100644 --- a/backend/modules/llm/domain/entity/eino_convertor_test.go +++ b/backend/modules/llm/domain/entity/eino_convertor_test.go @@ -245,12 +245,15 @@ func TestEinoConvertor_MoreFromDO(t *testing.T) { Content: "hello", MultiModalContent: []*ChatMessagePart{ {Type: ChatMessagePartTypeImageURL, ImageURL: &ChatMessageImageURL{URL: "url"}}, + {Type: ChatMessagePartTypeVideoURL, VideoURL: &ChatMessageVideoURL{URL: "video-url", MIMEType: "video/mp4"}}, }, }, } res := FromDOMessages(dos) assert.Len(t, res, 1) assert.Equal(t, schema.User, res[0].Role) + assert.Equal(t, "video-url", res[0].MultiContent[1].VideoURL.URL) + assert.Equal(t, "video/mp4", res[0].MultiContent[1].VideoURL.MIMEType) }) t.Run("FromDOImageURL_nil", func(t *testing.T) { @@ -299,11 +302,15 @@ func TestEinoConvertor_MoreToDO(t *testing.T) { cms := []schema.ChatMessagePart{ {Type: schema.ChatMessagePartTypeText, Text: "txt"}, {Type: schema.ChatMessagePartTypeImageURL, ImageURL: &schema.ChatMessageImageURL{URL: "url"}}, + {Type: schema.ChatMessagePartTypeVideoURL, VideoURL: &schema.ChatMessageVideoURL{URL: "video-url", MIMEType: "video/mp4"}}, } res := ToDOMultiContents(cms) - assert.Len(t, res, 2) + assert.Len(t, res, 3) assert.Equal(t, ChatMessagePartTypeText, res[0].Type) assert.Equal(t, ChatMessagePartTypeImageURL, res[1].Type) + assert.Equal(t, ChatMessagePartTypeVideoURL, res[2].Type) + assert.Equal(t, "video-url", res[2].VideoURL.URL) + assert.Equal(t, "video/mp4", res[2].VideoURL.MIMEType) }) t.Run("GetReasoningContent", func(t *testing.T) { diff --git a/backend/modules/llm/domain/entity/manage.go b/backend/modules/llm/domain/entity/manage.go index e7ce589c8a..41bf921b82 100644 --- a/backend/modules/llm/domain/entity/manage.go +++ b/backend/modules/llm/domain/entity/manage.go @@ -71,6 +71,9 @@ func (a *Ability) ValidAbility() error { return errors.Errorf("multi modal Image is true but ability multi modal ability image is nil") } } + if a.AbilityMultiModal.Video && a.AbilityMultiModal.AbilityVideo == nil { + return errors.Errorf("multi modal video is true but ability multi modal ability video is nil") + } } return nil } @@ -114,6 +117,13 @@ func (m *Model) SupportImageBinary() (bool, int64, int64) { m.Ability.AbilityMultiModal.AbilityImage.MaxImageCount, m.Ability.AbilityMultiModal.AbilityImage.MaxImageSize } +func (m *Model) SupportVideoInput() bool { + if m == nil || m.Ability == nil || m.Ability.AbilityMultiModal == nil { + return false + } + return m.Ability.AbilityMultiModal.Video && m.Ability.AbilityMultiModal.AbilityVideo != nil +} + func (m *Model) SupportFunctionCall() bool { if m == nil || m.Ability == nil { return false @@ -181,6 +191,8 @@ func (a *Ability) GetAbilityEnums() []AbilityEnum { type AbilityMultiModal struct { Image bool `json:"image" yaml:"image" mapstructure:"image"` AbilityImage *AbilityImage `json:"ability_image" yaml:"ability_image" mapstructure:"ability_image"` + Video bool `json:"video" yaml:"video" mapstructure:"video"` + AbilityVideo *AbilityVideo `json:"ability_video" yaml:"ability_video" mapstructure:"ability_video"` } type AbilityImage struct { @@ -190,6 +202,13 @@ type AbilityImage struct { MaxImageCount int64 `json:"max_image_count" yaml:"max_image_count" mapstructure:"max_image_count"` } +type AbilityVideo struct { + MaxVideoSizeInMB int32 `json:"max_video_size_in_mb" yaml:"max_video_size_in_mb" mapstructure:"max_video_size_in_mb"` + SupportedVideoFormats []VideoFormat `json:"supported_video_formats" yaml:"supported_video_formats" mapstructure:"supported_video_formats"` +} + +type VideoFormat string + type ProtocolConfig struct { BaseURL string `json:"base_url" yaml:"base_url" mapstructure:"base_url"` APIKey string `json:"api_key" yaml:"api_key" mapstructure:"api_key"` diff --git a/backend/modules/llm/domain/entity/manage_test.go b/backend/modules/llm/domain/entity/manage_test.go index 8c1de4aee6..0a23843940 100644 --- a/backend/modules/llm/domain/entity/manage_test.go +++ b/backend/modules/llm/domain/entity/manage_test.go @@ -174,6 +174,22 @@ func TestModel_Valid(t *testing.T) { }, wantErr: true, }, + { + name: "model video ability is invalid", + fields: fields{ + model: &Model{ + ID: 1, Name: "name", + Ability: &Ability{ + MultiModal: true, + AbilityMultiModal: &AbilityMultiModal{ + Video: true, + AbilityVideo: nil, + }, + }, + }, + }, + wantErr: true, + }, { name: "model ability is invalid", fields: fields{ @@ -320,6 +336,18 @@ func TestSupportImageURL(t *testing.T) { } } +func TestSupportVideoInput(t *testing.T) { + assert.False(t, (*Model)(nil).SupportVideoInput()) + assert.False(t, (&Model{Ability: &Ability{MultiModal: true, AbilityMultiModal: &AbilityMultiModal{}}}).SupportVideoInput()) + assert.True(t, (&Model{Ability: &Ability{ + MultiModal: true, + AbilityMultiModal: &AbilityMultiModal{ + Video: true, + AbilityVideo: &AbilityVideo{}, + }, + }}).SupportVideoInput()) +} + func TestParamConfig_GetCommonParamDefaultVal(t *testing.T) { type fields struct { ParamSchemas []*ParamSchema diff --git a/backend/modules/llm/domain/entity/runtime.go b/backend/modules/llm/domain/entity/runtime.go index e03db1bb86..943be5a9a1 100644 --- a/backend/modules/llm/domain/entity/runtime.go +++ b/backend/modules/llm/domain/entity/runtime.go @@ -93,6 +93,7 @@ type ChatMessagePart struct { Type ChatMessagePartType `json:"type"` Text string `json:"text"` ImageURL *ChatMessageImageURL `json:"image_url"` + VideoURL *ChatMessageVideoURL `json:"video_url"` } func (p *ChatMessagePart) IsMultiModal() bool { @@ -129,6 +130,28 @@ type ChatMessageImageURL struct { MIMEType string `json:"mime_type,omitempty"` } +type ChatMessageVideoURL struct { + URL string `json:"url,omitempty"` + Detail *VideoURLDetail `json:"detail,omitempty"` + MIMEType string `json:"mime_type,omitempty"` +} + +type VideoURLDetail struct { + FPS float64 `json:"fps,omitempty"` +} + +func (m *Message) HasVideoContent() bool { + if m == nil { + return false + } + for _, part := range m.MultiModalContent { + if part != nil && part.Type == ChatMessagePartTypeVideoURL && part.VideoURL != nil { + return true + } + } + return false +} + // ImageURLDetail is the detail of the image url. type ImageURLDetail string diff --git a/backend/modules/llm/domain/entity/runtime_test.go b/backend/modules/llm/domain/entity/runtime_test.go index 53aa7c9233..4513685fc7 100644 --- a/backend/modules/llm/domain/entity/runtime_test.go +++ b/backend/modules/llm/domain/entity/runtime_test.go @@ -60,6 +60,19 @@ func TestMessage_MultiModal(t *testing.T) { assert.True(t, m.HasMultiModalContent()) }) + t.Run("has_video_content", func(t *testing.T) { + assert.False(t, (*Message)(nil).HasVideoContent()) + assert.False(t, (&Message{}).HasVideoContent()) + + m := &Message{MultiModalContent: []*ChatMessagePart{ + {Type: ChatMessagePartTypeVideoURL}, + }} + assert.False(t, m.HasVideoContent()) + + m.MultiModalContent[0].VideoURL = &ChatMessageVideoURL{URL: "https://example.com/video.mp4"} + assert.True(t, m.HasVideoContent()) + }) + t.Run("get_image_count_and_max_size", func(t *testing.T) { m := &Message{ MultiModalContent: []*ChatMessagePart{ diff --git a/backend/modules/llm/domain/service/runtime.go b/backend/modules/llm/domain/service/runtime.go index 5d3f46ee10..1e48f903a6 100644 --- a/backend/modules/llm/domain/service/runtime.go +++ b/backend/modules/llm/domain/service/runtime.go @@ -105,11 +105,14 @@ func (r *RuntimeImpl) HandleMsgsPreCallModel(ctx context.Context, model *entity. func (r *RuntimeImpl) ValidModelAndRequest(ctx context.Context, model *entity.Model, input []*entity.Message, opts ...entity.Option) error { // 如果msg中有多模态输入，看模型是否支持多模态 - var hasMultiModal, hasImageURL, hasImageBinary bool + var hasMultiModal, hasImageURL, hasImageBinary, hasVideo bool var maxImageCnt, maxImageSizeInByte int64 for _, msg := range input { if msg.HasMultiModalContent() { hasMultiModal = true + if msg.HasVideoContent() { + hasVideo = true + } tmpHasImageURL, tmpHasImageBinary, tmpMaxImageCnt, tmpMaxImageSizeInByte := msg.GetImageCountAndMaxSize() if tmpHasImageURL { hasImageURL = true @@ -128,6 +131,9 @@ func (r *RuntimeImpl) ValidModelAndRequest(ctx context.Context, model *entity.Mo if hasMultiModal && !model.SupportMultiModalInput() { return errorx.NewByCode(llm_errorx.RequestNotCompatibleWithModelAbilityCode, errorx.WithExtraMsg("messages have multi modal content, but this model does not support multi modal")) } + if hasVideo && !model.SupportVideoInput() { + return errorx.NewByCode(llm_errorx.RequestNotCompatibleWithModelAbilityCode, errorx.WithExtraMsg("messages have video input, but this model does not support video input")) + } if hasImageURL { s, cnt := model.SupportImageURL() if !s { diff --git a/backend/modules/llm/domain/service/runtime_test.go b/backend/modules/llm/domain/service/runtime_test.go index 6ead5b3ff4..2738cb0909 100644 --- a/backend/modules/llm/domain/service/runtime_test.go +++ b/backend/modules/llm/domain/service/runtime_test.go @@ -28,6 +28,32 @@ import ( "github.com/coze-dev/coze-loop/backend/pkg/unittest" ) +func TestRuntimeImpl_ValidModelAndRequest_Video(t *testing.T) { + input := []*entity.Message{{ + Role: entity.RoleUser, + MultiModalContent: []*entity.ChatMessagePart{{ + Type: entity.ChatMessagePartTypeVideoURL, + VideoURL: &entity.ChatMessageVideoURL{URL: "https://example.com/video.mp4"}, + }}, + }} + runtime := &RuntimeImpl{} + + unsupported := &entity.Model{Ability: &entity.Ability{ + MultiModal: true, + AbilityMultiModal: &entity.AbilityMultiModal{}, + }} + assert.Error(t, runtime.ValidModelAndRequest(context.Background(), unsupported, input)) + + supported := &entity.Model{Ability: &entity.Ability{ + MultiModal: true, + AbilityMultiModal: &entity.AbilityMultiModal{ + Video: true, + AbilityVideo: &entity.AbilityVideo{}, + }, + }} + assert.NoError(t, runtime.ValidModelAndRequest(context.Background(), supported, input)) +} + func TestRuntimeImpl_Generate(t *testing.T) { var opts []entity.Option opts = append(opts, entity.WithTools([]*entity.ToolInfo{