[fix][model] preserve video input messages

backend/modules/llm/application/convertor/manage.go

@@ -175,6 +175,8 @@ func AbilityMultiModalDO2DTO(a *entity.AbilityMultiModal) *manage.AbilityMultiMo
 	return &manage.AbilityMultiModal{
 		Image:        ptr.Of(a.Image),
 		AbilityImage: AbilityImageDO2DTO(a.AbilityImage),
+		Video:        ptr.Of(a.Video),
+		AbilityVideo: AbilityVideoDO2DTO(a.AbilityVideo),
 	}
 }
 
@@ -190,6 +192,18 @@ func AbilityImageDO2DTO(a *entity.AbilityImage) *manage.AbilityImage {
 	}
 }
 
+func AbilityVideoDO2DTO(a *entity.AbilityVideo) *manage.AbilityVideo {
+	if a == nil {
+		return nil
+	}
+	return &manage.AbilityVideo{
+		MaxVideoSizeInMb: ptr.Of(a.MaxVideoSizeInMB),
+		SupportedVideoFormats: slices.Transform(a.SupportedVideoFormats, func(format entity.VideoFormat, _ int) manage.VideoFormat {
+			return manage.VideoFormat(format)
+		}),
+	}
+}
+
 func ProtocolConfigDO2DTO(p *entity.ProtocolConfig) *manage.ProtocolConfig {
 	if p == nil {
 		return nil

backend/modules/llm/application/convertor/manage_test.go

@@ -283,10 +283,18 @@ func TestAbilityMultiModalDO2DTO(t *testing.T) {
 			AbilityImage: &entity.AbilityImage{
 				URLEnabled: true,
 			},
+			Video: true,
+			AbilityVideo: &entity.AbilityVideo{
+				MaxVideoSizeInMB:      200,
+				SupportedVideoFormats: []entity.VideoFormat{"mp4", "webm"},
+			},
 		}
 		got := AbilityMultiModalDO2DTO(a)
 		assert.True(t, *got.Image)
 		assert.True(t, *got.AbilityImage.URLEnabled)
+		assert.True(t, *got.Video)
+		assert.Equal(t, int32(200), *got.AbilityVideo.MaxVideoSizeInMb)
+		assert.Equal(t, []manage.VideoFormat{"mp4", "webm"}, got.AbilityVideo.SupportedVideoFormats)
 	})
 }
 

backend/modules/llm/application/convertor/runtime.go

@@ -96,6 +96,7 @@ func ChatMessagePartDTO2DO(dto *druntime.ChatMessagePart) (do *entity.ChatMessag
 		Type:     entity.ChatMessagePartType(dto.GetType()),
 		Text:     dto.GetText(),
 		ImageURL: ChatMessageImageURLDTO2DO(dto.GetImageURL()),
+		VideoURL: ChatMessageVideoURLDTO2DO(dto.GetVideoURL()),
 	}
 }
 
@@ -110,6 +111,24 @@ func ChatMessageImageURLDTO2DO(dto *druntime.ChatMessageImageURL) (do *entity.Ch
 	}
 }
 
+func ChatMessageVideoURLDTO2DO(dto *druntime.ChatMessageVideoURL) (do *entity.ChatMessageVideoURL) {
+	if dto == nil {
+		return nil
+	}
+	return &entity.ChatMessageVideoURL{
+		URL:      dto.GetURL(),
+		Detail:   VideoURLDetailDTO2DO(dto.GetDetail()),
+		MIMEType: dto.GetMimeType(),
+	}
+}
+
+func VideoURLDetailDTO2DO(dto *druntime.VideoURLDetail) (do *entity.VideoURLDetail) {
+	if dto == nil {
+		return nil
+	}
+	return &entity.VideoURLDetail{FPS: dto.GetFps()}
+}
+
 func MessageDO2DTO(do *entity.Message) (dto *druntime.Message) {
 	if do == nil {
 		return nil
@@ -188,6 +207,7 @@ func ChatMessagePartDO2DTO(do *entity.ChatMessagePart) (dto *druntime.ChatMessag
 		Type:     ptr.Of(druntime.ChatMessagePartType(do.Type)),
 		Text:     ptr.Of(do.Text),
 		ImageURL: ChatMessageImageURLDO2DTO(do.ImageURL),
+		VideoURL: ChatMessageVideoURLDO2DTO(do.VideoURL),
 	}
 }
 
@@ -201,3 +221,21 @@ func ChatMessageImageURLDO2DTO(do *entity.ChatMessageImageURL) (dto *druntime.Ch
 		MimeType: ptr.Of(do.MIMEType),
 	}
 }
+
+func ChatMessageVideoURLDO2DTO(do *entity.ChatMessageVideoURL) (dto *druntime.ChatMessageVideoURL) {
+	if do == nil {
+		return nil
+	}
+	return &druntime.ChatMessageVideoURL{
+		URL:      ptr.Of(do.URL),
+		Detail:   VideoURLDetailDO2DTO(do.Detail),
+		MimeType: ptr.Of(do.MIMEType),
+	}
+}
+
+func VideoURLDetailDO2DTO(do *entity.VideoURLDetail) (dto *druntime.VideoURLDetail) {
+	if do == nil {
+		return nil
+	}
+	return &druntime.VideoURLDetail{Fps: ptr.Of(do.FPS)}
+}

backend/modules/llm/application/convertor/runtime_test.go

@@ -128,21 +128,44 @@ func TestToolCallConvert(t *testing.T) {
 }
 
 func TestChatMessagePartConvert(t *testing.T) {
-	dto := &druntime.ChatMessagePart{
-		Type: gptr.Of(druntime.ChatMessagePartTypeImageURL),
-		ImageURL: &druntime.ChatMessageImageURL{
-			URL:      gptr.Of("http://img.com"),
-			Detail:   gptr.Of(druntime.ImageURLDetailHigh),
-			MimeType: gptr.Of("image/png"),
-		},
-	}
-	do := ChatMessagePartDTO2DO(dto)
-	assert.Equal(t, entity.ChatMessagePartTypeImageURL, do.Type)
-	assert.Equal(t, "http://img.com", do.ImageURL.URL)
-	assert.Equal(t, entity.ImageURLDetailHigh, do.ImageURL.Detail)
+	t.Run("image", func(t *testing.T) {
+		dto := &druntime.ChatMessagePart{
+			Type: gptr.Of(druntime.ChatMessagePartTypeImageURL),
+			ImageURL: &druntime.ChatMessageImageURL{
+				URL:      gptr.Of("http://img.com"),
+				Detail:   gptr.Of(druntime.ImageURLDetailHigh),
+				MimeType: gptr.Of("image/png"),
+			},
+		}
+		do := ChatMessagePartDTO2DO(dto)
+		assert.Equal(t, entity.ChatMessagePartTypeImageURL, do.Type)
+		assert.Equal(t, "http://img.com", do.ImageURL.URL)
+		assert.Equal(t, entity.ImageURLDetailHigh, do.ImageURL.Detail)
+
+		dto2 := ChatMessagePartDO2DTO(do)
+		assert.Equal(t, druntime.ChatMessagePartTypeImageURL, *dto2.Type)
+		assert.Equal(t, "http://img.com", *dto2.ImageURL.URL)
+		assert.Equal(t, druntime.ImageURLDetailHigh, *dto2.ImageURL.Detail)
+	})
 
-	dto2 := ChatMessagePartDO2DTO(do)
-	assert.Equal(t, druntime.ChatMessagePartTypeImageURL, *dto2.Type)
-	assert.Equal(t, "http://img.com", *dto2.ImageURL.URL)
-	assert.Equal(t, druntime.ImageURLDetailHigh, *dto2.ImageURL.Detail)
+	t.Run("video", func(t *testing.T) {
+		dto := &druntime.ChatMessagePart{
+			Type: gptr.Of(druntime.ChatMessagePartTypeVideoURL),
+			VideoURL: &druntime.ChatMessageVideoURL{
+				URL:      gptr.Of("https://example.com/video.mp4"),
+				Detail:   &druntime.VideoURLDetail{Fps: gptr.Of(2.5)},
+				MimeType: gptr.Of("video/mp4"),
+			},
+		}
+		do := ChatMessagePartDTO2DO(dto)
+		assert.Equal(t, entity.ChatMessagePartTypeVideoURL, do.Type)
+		assert.Equal(t, "https://example.com/video.mp4", do.VideoURL.URL)
+		assert.Equal(t, 2.5, do.VideoURL.Detail.FPS)
+
+		dto2 := ChatMessagePartDO2DTO(do)
+		assert.Equal(t, druntime.ChatMessagePartTypeVideoURL, *dto2.Type)
+		assert.Equal(t, "https://example.com/video.mp4", *dto2.VideoURL.URL)
+		assert.Equal(t, 2.5, *dto2.VideoURL.Detail.Fps)
+		assert.Equal(t, "video/mp4", *dto2.VideoURL.MimeType)
+	})
 }

backend/modules/llm/domain/entity/eino_convertor.go

@@ -52,6 +52,7 @@ func FromDOChatMsgPart(p *ChatMessagePart) schema.ChatMessagePart {
 		Type:     schema.ChatMessagePartType(p.Type),
 		Text:     p.Text,
 		ImageURL: FromDOImageURL(p.ImageURL),
+		VideoURL: FromDOVideoURL(p.VideoURL),
 	}
 }
 
@@ -66,6 +67,16 @@ func FromDOImageURL(p *ChatMessageImageURL) *schema.ChatMessageImageURL {
 	}
 }
 
+func FromDOVideoURL(p *ChatMessageVideoURL) *schema.ChatMessageVideoURL {
+	if p == nil {
+		return nil
+	}
+	return &schema.ChatMessageVideoURL{
+		URL:      p.URL,
+		MIMEType: p.MIMEType,
+	}
+}
+
 func FromDOToolCalls(ts []*ToolCall) []schema.ToolCall {
 	return slices.Transform(ts, func(t *ToolCall, _ int) schema.ToolCall {
 		return FromDOToolCall(t)
@@ -265,6 +276,7 @@ func ToDOMultiContent(cm schema.ChatMessagePart) *ChatMessagePart {
 		Type:     ChatMessagePartType(cm.Type),
 		Text:     cm.Text,
 		ImageURL: ToDOImageURL(cm.ImageURL),
+		VideoURL: ToDOVideoURL(cm.VideoURL),
 	}
 }
 
@@ -279,6 +291,16 @@ func ToDOImageURL(cm *schema.ChatMessageImageURL) *ChatMessageImageURL {
 	}
 }
 
+func ToDOVideoURL(cm *schema.ChatMessageVideoURL) *ChatMessageVideoURL {
+	if cm == nil {
+		return nil
+	}
+	return &ChatMessageVideoURL{
+		URL:      cm.URL,
+		MIMEType: cm.MIMEType,
+	}
+}
+
 func ToDORespMeta(rm *schema.ResponseMeta) *ResponseMeta {
 	if rm == nil {
 		return nil

backend/modules/llm/domain/entity/eino_convertor_test.go

@@ -245,12 +245,15 @@ func TestEinoConvertor_MoreFromDO(t *testing.T) {
 				Content: "hello",
 				MultiModalContent: []*ChatMessagePart{
 					{Type: ChatMessagePartTypeImageURL, ImageURL: &ChatMessageImageURL{URL: "url"}},
+					{Type: ChatMessagePartTypeVideoURL, VideoURL: &ChatMessageVideoURL{URL: "video-url", MIMEType: "video/mp4"}},
 				},
 			},
 		}
 		res := FromDOMessages(dos)
 		assert.Len(t, res, 1)
 		assert.Equal(t, schema.User, res[0].Role)
+		assert.Equal(t, "video-url", res[0].MultiContent[1].VideoURL.URL)
+		assert.Equal(t, "video/mp4", res[0].MultiContent[1].VideoURL.MIMEType)
 	})
 
 	t.Run("FromDOImageURL_nil", func(t *testing.T) {
@@ -299,11 +302,15 @@ func TestEinoConvertor_MoreToDO(t *testing.T) {
 		cms := []schema.ChatMessagePart{
 			{Type: schema.ChatMessagePartTypeText, Text: "txt"},
 			{Type: schema.ChatMessagePartTypeImageURL, ImageURL: &schema.ChatMessageImageURL{URL: "url"}},
+			{Type: schema.ChatMessagePartTypeVideoURL, VideoURL: &schema.ChatMessageVideoURL{URL: "video-url", MIMEType: "video/mp4"}},
 		}
 		res := ToDOMultiContents(cms)
-		assert.Len(t, res, 2)
+		assert.Len(t, res, 3)
 		assert.Equal(t, ChatMessagePartTypeText, res[0].Type)
 		assert.Equal(t, ChatMessagePartTypeImageURL, res[1].Type)
+		assert.Equal(t, ChatMessagePartTypeVideoURL, res[2].Type)
+		assert.Equal(t, "video-url", res[2].VideoURL.URL)
+		assert.Equal(t, "video/mp4", res[2].VideoURL.MIMEType)
 	})
 
 	t.Run("GetReasoningContent", func(t *testing.T) {

backend/modules/llm/domain/entity/manage.go

@@ -71,6 +71,9 @@ func (a *Ability) ValidAbility() error {
 				return errors.Errorf("multi modal Image is true but ability multi modal ability image is nil")
 			}
 		}
+		if a.AbilityMultiModal.Video && a.AbilityMultiModal.AbilityVideo == nil {
+			return errors.Errorf("multi modal video is true but ability multi modal ability video is nil")
+		}
 	}
 	return nil
 }
@@ -114,6 +117,13 @@ func (m *Model) SupportImageBinary() (bool, int64, int64) {
 		m.Ability.AbilityMultiModal.AbilityImage.MaxImageCount, m.Ability.AbilityMultiModal.AbilityImage.MaxImageSize
 }
 
+func (m *Model) SupportVideoInput() bool {
+	if m == nil || m.Ability == nil || m.Ability.AbilityMultiModal == nil {
+		return false
+	}
+	return m.Ability.AbilityMultiModal.Video && m.Ability.AbilityMultiModal.AbilityVideo != nil
+}
+
 func (m *Model) SupportFunctionCall() bool {
 	if m == nil || m.Ability == nil {
 		return false
@@ -181,6 +191,8 @@ func (a *Ability) GetAbilityEnums() []AbilityEnum {
 type AbilityMultiModal struct {
 	Image        bool          `json:"image" yaml:"image" mapstructure:"image"`
 	AbilityImage *AbilityImage `json:"ability_image" yaml:"ability_image" mapstructure:"ability_image"`
+	Video        bool          `json:"video" yaml:"video" mapstructure:"video"`
+	AbilityVideo *AbilityVideo `json:"ability_video" yaml:"ability_video" mapstructure:"ability_video"`
 }
 
 type AbilityImage struct {
@@ -190,6 +202,13 @@ type AbilityImage struct {
 	MaxImageCount int64 `json:"max_image_count" yaml:"max_image_count" mapstructure:"max_image_count"`
 }
 
+type AbilityVideo struct {
+	MaxVideoSizeInMB      int32         `json:"max_video_size_in_mb" yaml:"max_video_size_in_mb" mapstructure:"max_video_size_in_mb"`
+	SupportedVideoFormats []VideoFormat `json:"supported_video_formats" yaml:"supported_video_formats" mapstructure:"supported_video_formats"`
+}
+
+type VideoFormat string
+
 type ProtocolConfig struct {
 	BaseURL                string                  `json:"base_url" yaml:"base_url" mapstructure:"base_url"`
 	APIKey                 string                  `json:"api_key" yaml:"api_key" mapstructure:"api_key"`

backend/modules/llm/domain/entity/manage_test.go

@@ -174,6 +174,22 @@ func TestModel_Valid(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "model video ability is invalid",
+			fields: fields{
+				model: &Model{
+					ID: 1, Name: "name",
+					Ability: &Ability{
+						MultiModal: true,
+						AbilityMultiModal: &AbilityMultiModal{
+							Video:        true,
+							AbilityVideo: nil,
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "model ability is invalid",
 			fields: fields{
@@ -320,6 +336,18 @@ func TestSupportImageURL(t *testing.T) {
 	}
 }
 
+func TestSupportVideoInput(t *testing.T) {
+	assert.False(t, (*Model)(nil).SupportVideoInput())
+	assert.False(t, (&Model{Ability: &Ability{MultiModal: true, AbilityMultiModal: &AbilityMultiModal{}}}).SupportVideoInput())
+	assert.True(t, (&Model{Ability: &Ability{
+		MultiModal: true,
+		AbilityMultiModal: &AbilityMultiModal{
+			Video:        true,
+			AbilityVideo: &AbilityVideo{},
+		},
+	}}).SupportVideoInput())
+}
+
 func TestParamConfig_GetCommonParamDefaultVal(t *testing.T) {
 	type fields struct {
 		ParamSchemas []*ParamSchema

backend/modules/llm/domain/entity/runtime.go

@@ -93,6 +93,7 @@ type ChatMessagePart struct {
 	Type     ChatMessagePartType  `json:"type"`
 	Text     string               `json:"text"`
 	ImageURL *ChatMessageImageURL `json:"image_url"`
+	VideoURL *ChatMessageVideoURL `json:"video_url"`
 }
 
 func (p *ChatMessagePart) IsMultiModal() bool {
@@ -129,6 +130,28 @@ type ChatMessageImageURL struct {
 	MIMEType string `json:"mime_type,omitempty"`
 }
 
+type ChatMessageVideoURL struct {
+	URL      string          `json:"url,omitempty"`
+	Detail   *VideoURLDetail `json:"detail,omitempty"`
+	MIMEType string          `json:"mime_type,omitempty"`
+}
+
+type VideoURLDetail struct {
+	FPS float64 `json:"fps,omitempty"`
+}
+
+func (m *Message) HasVideoContent() bool {
+	if m == nil {
+		return false
+	}
+	for _, part := range m.MultiModalContent {
+		if part != nil && part.Type == ChatMessagePartTypeVideoURL && part.VideoURL != nil {
+			return true
+		}
+	}
+	return false
+}
+
 // ImageURLDetail is the detail of the image url.
 type ImageURLDetail string
 

backend/modules/llm/domain/entity/runtime_test.go

@@ -60,6 +60,19 @@ func TestMessage_MultiModal(t *testing.T) {
 		assert.True(t, m.HasMultiModalContent())
 	})
 
+	t.Run("has_video_content", func(t *testing.T) {
+		assert.False(t, (*Message)(nil).HasVideoContent())
+		assert.False(t, (&Message{}).HasVideoContent())
+
+		m := &Message{MultiModalContent: []*ChatMessagePart{
+			{Type: ChatMessagePartTypeVideoURL},
+		}}
+		assert.False(t, m.HasVideoContent())
+
+		m.MultiModalContent[0].VideoURL = &ChatMessageVideoURL{URL: "https://example.com/video.mp4"}
+		assert.True(t, m.HasVideoContent())
+	})
+
 	t.Run("get_image_count_and_max_size", func(t *testing.T) {
 		m := &Message{
 			MultiModalContent: []*ChatMessagePart{