fix(ingestion): block video parse while keeping visual upload routing (#21245)

This commit is contained in:
jay77721 authored and GitHub committed 2026-10-21 16:16:39 +08:00
1 parent d6ad954298
commit b84cadea97
16 files changed
+524 -123

No files matched your search

@@ -1101,3 +1101,35 @@ func picturePNG(t *testing.T) []byte {
}
return data.Bytes()
}
// TestMaybeDispatchVideoRejectsExplicitly pins the runtime backstop behind the
// frontend parse block: a video file whose setups declare the video family
// gets an explicit "not yet supported" error (not a silent fallback), while
// configs without a video family fall through to the generic dispatch path.
func TestMaybeDispatchVideoRejectsExplicitly(t *testing.T) {
ctx := t.Context()
inputs := map[string]any{"tenant_id": "t1"}
// Main path: standard component defaults declare the video family, so the
// branch handles the file and fails loudly.
_, handled, err := maybeDispatchVideo(ctx, nil, utility.FileTypeVIDEO, "clip.mov", []byte("mov-bytes"), inputs, defaultSetups())
if !handled {
t.Fatal("maybeDispatchVideo did not handle a video file with a video setup")
}
if err == nil || !strings.Contains(err.Error(), "not yet supported") {
t.Fatalf("video error = %v, want explicit not-yet-supported", err)
}
// Without a video family the branch stays out of the way; the generic
// dispatch path (GetParser "unsupported file type") answers instead.
_, handled, err = maybeDispatchVideo(ctx, nil, utility.FileTypeVIDEO, "clip.mov", []byte("mov-bytes"), inputs, map[string]schema.ParserSetup{})
if handled || err != nil {
t.Fatalf("handled = %v, err = %v, want pass-through without a video setup", handled, err)
}
// Non-video files never enter the video branch.
_, handled, err = maybeDispatchVideo(ctx, nil, utility.FileTypeVISUAL, "photo.jpg", []byte("jpeg-bytes"), inputs, defaultSetups())
if handled || err != nil {
t.Fatalf("handled = %v, err = %v, want pass-through for images", handled, err)
}
}
@@ -251,6 +251,9 @@ func familyToExt(family string) utility.FileType {
// Python family identifier used by ParserComponent setups.
// Returns "" when the hint is unknown.
func pythonFamilyName(raw string) string {
if utility.IsVideoExtension(raw) || raw == "video" {
return "video"
}
switch raw {
case "pdf":
return "pdf"
@@ -275,9 +278,6 @@ func pythonFamilyName(raw string) string {
case "text&code", "txt", "py", "js", "java", "c", "cpp", "h", "php",
"go", "ts", "sh", "cs", "kt", "sql":
return "text&code"
case "mp4", "avi", "mkv", "mov", "webm", "flv",
"mpeg", "mpg", "wmv", "3gp", "3gpp", "video":
return "video"
case "eml", "msg", "email":
return "email"
case "da", "wave", "wav", "mp3", "aac", "flac", "ogg",
@@ -638,6 +638,63 @@ func TestUploadLocalDocuments_FallsBackWhenSelectionLacksFamily(t *testing.T) {
parserFamilySetup(t, got[1], "text&code")
}
// TestUploadLocalDocuments_SwitchesVideoToPicture pins the upload policy:
// video uploads are accepted (visual bucket) and switched to the picture
// pipeline; parsing stays blocked later because no parser implements video
// analysis.
func TestUploadLocalDocuments_SwitchesVideoToPicture(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
mockStorage := newFakeUploadStorage()
factory := storage.GetStorageFactory()
origStorage := factory.GetStorage()
factory.SetStorage(mockStorage)
t.Cleanup(func() { factory.SetStorage(origStorage) })
if err := dao.DB.Create(&entity.Tenant{
ID: "tenant-1", LLMID: "llm-default", EmbdID: "embd-default", ASRID: "asr-default",
}).Error; err != nil {
t.Fatalf("insert tenant: %v", err)
}
kb := &entity.Knowledgebase{
ID: "kb-video-block",
TenantID: "tenant-1",
Name: "kb-video-block",
CreatedBy: "user-1",
ParserID: "naive",
ParserConfig: presentationParserConfig(),
}
if err := dao.DB.Create(kb).Error; err != nil {
t.Fatalf("insert kb: %v", err)
}
ctx := t.Context()
svc := testDocumentService(t)
got, errs := svc.UploadLocalDocuments(ctx, kb, "user-1", []*multipart.FileHeader{
makeTestFileHeader(t, "file", "clip.mov", []byte("mov-bytes")),
makeTestFileHeader(t, "file", "movie.mp4", []byte("mp4-bytes")),
makeTestFileHeader(t, "file", "legacy.rmvb", []byte("rmvb-bytes")),
makeTestFileHeader(t, "file", "notes.txt", []byte("plain text")),
}, "", nil)
if len(errs) != 0 {
t.Fatalf("unexpected errs: %v", errs)
}
if len(got) != 4 {
t.Fatalf("uploaded docs = %d, want 4", len(got))
}
for _, doc := range got[:3] {
if parserID, _ := doc["parser_id"].(string); parserID != "picture" {
t.Fatalf("%v parser_id = %q, want picture", doc["name"], parserID)
}
if docType, _ := doc["type"].(string); docType != string(utility.FileTypeVISUAL) {
t.Fatalf("%v type = %q, want %q", doc["name"], docType, utility.FileTypeVISUAL)
}
parserFamilySetup(t, doc, "video")
}
}
// TestInsertSyncDocument_FallsBackWhenSelectionLacksFamily covers the connector
// path: it shares resolveDocumentParser with the upload path, so a synced text
// file lands on general too.
+15
View File
@@ -139,6 +139,13 @@ func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDicts
}
ft := utility.FilenameType(name)
if utility.IsVideoFilename(name) {
// Uploads accept video, but no parser implements video analysis.
// Report that instead of inlining the container as a bogus image
// data URI (the old visual classification) or dumping raw bytes.
texts = append(texts, fmt.Sprintf("\n -----------------\nFile: %s\nContent as following: \n(video content is not supported)\n", name))
continue
}
if ft == utility.FileTypeVISUAL {
mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))
if mediaType == "" {
@@ -183,6 +190,14 @@ func (s *FileService) ParseAgentUploads(ctx context.Context, userID string, file
return nil, fmt.Errorf("file %q: upload is empty", name)
}
// Reject video filenames before the MIME-based image branch: a
// mislabeled video (e.g. clip.mp4 declaring image/png) would otherwise
// be inlined as a bogus image data URI without ever reaching the
// parser-level video check.
if utility.IsVideoFilename(name) {
return nil, fmt.Errorf("file %q: video content is not supported", name)
}
mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))
if strings.HasPrefix(mediaType, "image/") {
contents = append(contents, "data:"+mediaType+";base64,"+base64.StdEncoding.EncodeToString(data))
+53
View File
@@ -233,6 +233,40 @@ func TestFileService_GetFileContents_ImageAttachmentAsDataURI(t *testing.T) {
}
}
// Video attachments are accepted but unparsable: GetFileContents reports a
// fixed notice as the file's text (so the model sees why there is no content)
// instead of inlining the container as a bogus image or failing the batch.
// A mixed batch keeps the valid attachments alongside the notice.
func TestFileService_GetFileContents_VideoAttachmentReportsNotice(t *testing.T) {
memory := storage.NewMemoryStorage()
ctx := t.Context()
if err := memory.Put(ctx, "user-1-downloads", "vid-1", []byte("mp4-bytes")); err != nil {
t.Fatalf("put video: %v", err)
}
if err := memory.Put(ctx, "user-1-downloads", "img-1", []byte("jpeg-bytes")); err != nil {
t.Fatalf("put image: %v", err)
}
factory := storage.GetStorageFactory()
originalStorage := factory.GetStorage()
factory.SetStorage(memory)
t.Cleanup(func() { factory.SetStorage(originalStorage) })
svc := testFileService()
texts, images, err := svc.GetFileContents(ctx, "user-1", []map[string]interface{}{
{"id": "vid-1", "name": "clip.mp4", "mime_type": "video/mp4", "created_by": "user-1"},
{"id": "img-1", "name": "photo.jpeg", "mime_type": "image/jpeg", "created_by": "user-1"},
})
if err != nil {
t.Fatalf("GetFileContents failed: %v", err)
}
if len(texts) != 1 || !strings.Contains(texts[0], "clip.mp4") || !strings.Contains(texts[0], "video content is not supported") {
t.Fatalf("video notice = %v, want one text entry naming the file as unsupported", texts)
}
if len(images) != 1 {
t.Fatalf("expected the valid image to survive alongside the notice, got %v", images)
}
}
func TestFileService_ParseAgentUploads_TextAndImageInRequestOrder(t *testing.T) {
ctx := t.Context()
memory := storage.NewMemoryStorage()
@@ -265,6 +299,25 @@ func TestFileService_ParseAgentUploads_TextAndImageInRequestOrder(t *testing.T)
}
}
func TestFileService_ParseAgentUploads_RejectsVideoWithImageMime(t *testing.T) {
ctx := t.Context()
memory := storage.NewMemoryStorage()
if err := memory.Put(ctx, "user-1-downloads", "video-id", []byte("mp4-bytes")); err != nil {
t.Fatalf("put video: %v", err)
}
factory := storage.GetStorageFactory()
originalStorage := factory.GetStorage()
factory.SetStorage(memory)
t.Cleanup(func() { factory.SetStorage(originalStorage) })
_, err := testFileService().ParseAgentUploads(ctx, "user-1", []map[string]interface{}{
{"id": "video-id", "name": "clip.mp4", "mime_type": "image/png", "created_by": "user-1"},
}, "")
if err == nil || !strings.Contains(err.Error(), "video content is not supported") {
t.Fatalf("error = %v, want video rejection even with an image MIME type", err)
}
}
func TestFileService_ParseAgentUploads_RejectsForeignOwner(t *testing.T) {
ctx := t.Context()
memory := storage.NewMemoryStorage()
+38 -5
View File
@@ -52,8 +52,33 @@ const (
var (
filenameLenLimit = 255
// videoExtensions is the single source of truth for video containers.
// GetFileType (runtime dispatch) and the ingestion parser-family mapping
// read this set. FilenameType keeps the legacy upload bucket (video is
// admitted as visual and routed to the picture pipeline), so every video
// container accepted below must also be present here.
videoExtensions = map[string]struct{}{
"mp4": {}, "avi": {}, "mkv": {}, "mov": {}, "webm": {}, "flv": {},
"mpeg": {}, "mpg": {}, "wmv": {}, "3gp": {}, "3gpp": {},
"rm": {}, "rmvb": {}, "asf": {}, "dat": {}, "asx": {}, "wvx": {},
"mpe": {}, "mpa": {},
}
)
// IsVideoExtension reports whether ext (with or without a leading dot,
// case-insensitive) is a video container.
func IsVideoExtension(ext string) bool {
normalized := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(ext), "."))
_, ok := videoExtensions[normalized]
return ok
}
// IsVideoFilename reports whether filename has a video extension.
func IsVideoFilename(filename string) bool {
return IsVideoExtension(filepath.Ext(filename))
}
func init() {
}
@@ -81,6 +106,10 @@ func GetFileType(filename string) FileType {
suffix = strings.ToLower(ext)
}
if IsVideoExtension(suffix) {
return FileTypeVIDEO
}
switch suffix {
case "pdf":
return FileTypePDF
@@ -114,9 +143,6 @@ func GetFileType(filename string) FileType {
case "da", "wave", "wav", "mp3", "aac", "flac", "ogg",
"aiff", "au", "midi", "wma", "ape", "alac", "wv", "opus":
return FileTypeAURAL
case "mp4", "avi", "mkv", "mov", "webm", "flv", "mpeg",
"mpg", "wmv", "3gp", "3gpp":
return FileTypeVIDEO
case "png", "jpg", "jpeg", "gif", "bmp":
return FileTypeVISUAL
case "tiff", "tif", "webp", "svg", "ico":
@@ -137,6 +163,15 @@ func FilenameType(filename string) FileType {
return FileTypeOTHER
}
// Video is admitted at upload in the legacy visual bucket and routed to the
// picture pipeline; parsing is blocked later (no parser implements video
// analysis). Keep the admission here so every video container in the shared
// set passes the upload gate, including webm/flv/3gp/3gpp which the old
// bucket missed.
if IsVideoFilename(normalized) {
return FileTypeVISUAL
}
if matched, _ := regexp.MatchString(`.*\.pdf$`, normalized); matched {
return FileTypePDF
}
@@ -165,8 +200,6 @@ func FilenameType(filename string) FileType {
visualExtensions := []string{
"jpg", "jpeg", "png", "tif", "gif", "bmp", "pcx", "tga", "exif", "fpx", "svg", "psd", "cdr",
"pcd", "dxf", "ufo", "eps", "ai", "raw", "wmf", "webp", "avif", "apng", "icon", "ico",
"mpg", "mpeg", "avi", "rm", "rmvb", "mov", "wmv", "asf", "dat", "asx", "wvx", "mpe",
"mpa", "mp4", "mkv",
}
for _, ext := range visualExtensions {
if strings.HasSuffix(normalized, "."+ext) {
+55
View File
@@ -37,3 +37,58 @@ func TestFilenameTypeBMP(t *testing.T) {
}
}
}
// TestFilenameTypeVideo covers the upload bucket: video is admitted as visual
// (legacy categorization) and routed to the picture pipeline; parsing is
// blocked later. Every extension in the shared video set must classify as
// visual here, while GetFileType reports them as video for runtime dispatch.
func TestFilenameTypeVideo(t *testing.T) {
cases := []struct {
name string
want FileType
}{
{"clip.mp4", FileTypeVISUAL},
{"clip.mov", FileTypeVISUAL},
{"CLIP.MOV", FileTypeVISUAL},
{"clip.mkv", FileTypeVISUAL},
{"clip.webm", FileTypeVISUAL},
{"clip.flv", FileTypeVISUAL},
{"clip.wmv", FileTypeVISUAL},
{"clip.mpeg", FileTypeVISUAL},
{"clip.mpg", FileTypeVISUAL},
{"clip.3gp", FileTypeVISUAL},
{"clip.3gpp", FileTypeVISUAL},
{"clip.rm", FileTypeVISUAL},
{"clip.rmvb", FileTypeVISUAL},
{"clip.asf", FileTypeVISUAL},
{"clip.dat", FileTypeVISUAL},
{"clip.asx", FileTypeVISUAL},
{"clip.wvx", FileTypeVISUAL},
{"clip.mpe", FileTypeVISUAL},
{"clip.mpa", FileTypeVISUAL},
{"photo.jpg", FileTypeVISUAL},
}
for _, tc := range cases {
if got := FilenameType(tc.name); got != tc.want {
t.Fatalf("FilenameType(%q) = %q, want %q", tc.name, got, tc.want)
}
}
}
// TestIsVideoExtension pins the shared predicate that upload admission, runtime
// dispatch and the parser-family mapping read.
func TestIsVideoExtension(t *testing.T) {
for _, ext := range []string{"mp4", ".MOV", "rmvb", "3gpp"} {
if !IsVideoExtension(ext) {
t.Fatalf("IsVideoExtension(%q) = false, want true", ext)
}
}
for _, ext := range []string{"", "jpg", "pdf", "mp4x"} {
if IsVideoExtension(ext) {
t.Fatalf("IsVideoExtension(%q) = true, want false", ext)
}
}
if !IsVideoFilename("path/to/CLIP.MOV") {
t.Fatal("IsVideoFilename(\"path/to/CLIP.MOV\") = false, want true")
}
}
+38 -1
View File
@@ -55,7 +55,29 @@ export const FileTypeSuffixMap = {
[FileType.Doc]: ['doc'],
[FileType.Docx]: ['docx'],
[FileType.PowerPoint]: ['pptx', 'ppt'],
[FileType.Video]: ['mp4', 'avi', 'mkv'],
[FileType.Video]: [
// Mirrors utility.videoExtensions (internal/utility/file.go): upload
// admission, runtime dispatch and the parser-family mapping read one set.
'mp4',
'avi',
'mkv',
'mov',
'webm',
'flv',
'mpeg',
'mpg',
'wmv',
'3gp',
'3gpp',
'rm',
'rmvb',
'asf',
'dat',
'asx',
'wvx',
'mpe',
'mpa',
],
[FileType.Audio]: [
'da',
'wave',
@@ -93,7 +115,22 @@ export const FileIconMap = {
mp4: 'mp4',
avi: 'avi',
mkv: 'mkv',
mov: 'mp4',
webm: 'mp4',
flv: 'mp4',
wmv: 'mp4',
mpeg: 'mp4',
mpg: 'mp4',
'3gp': 'mp4',
'3gpp': 'mp4',
rm: 'mp4',
rmvb: 'rmvb',
asf: 'mp4',
dat: 'mp4',
asx: 'mp4',
wvx: 'mp4',
mpe: 'mp4',
mpa: 'mp4',
wav: 'wav',
html: 'html',
json: 'json',
+6
View File
@@ -416,6 +416,12 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
'{{name}} ({{fileType}}): not supported by the current parser configuration; it will automatically switch to the built-in {{parser}} configuration',
fileTypeUnsupportedManualSwitch:
'{{name}} ({{fileType}}): not supported by the current parser configuration; switch its parsing method manually after upload',
fileTypeVideoUnsupported:
'{{name}} ({{fileType}}): video files cannot be parsed yet',
videoUnsupportedAfterUploadHint:
'You can continue uploading, but video files cannot be parsed yet and will not be processed.',
videoUnsupportedParseHint:
'Video files cannot be parsed yet. Delete this file or convert it to a supported format.',
reselectParserAfterUploadHint:
'Continue uploading, then system will automatically switch these files to a supported built-in parsing configuration.',
manualSwitchAfterUploadHint:
+6
View File
@@ -372,6 +372,12 @@ export default {
'{{name}}({{fileType}}):当前解析配置不支持该文件类型,将自动切换为内置 {{parser}} 配置',
fileTypeUnsupportedManualSwitch:
'{{name}}({{fileType}}):当前解析配置不支持该文件类型,上传后可手动切换解析配置',
fileTypeVideoUnsupported:
'{{name}}({{fileType}}):视频文件暂不支持解析',
videoUnsupportedAfterUploadHint:
'可继续上传,但视频文件暂不支持解析,上传后不会被处理。',
videoUnsupportedParseHint:
'视频文件暂不支持解析,请删除该文件或转换为受支持的格式。',
reselectParserAfterUploadHint:
'可继续上传,系统将自动切换至支持这些文件类型的内置解析配置。',
manualSwitchAfterUploadHint:
@@ -1,5 +1,6 @@
import type { TFunction } from 'i18next';
import type { ReactNode } from 'react';
import { FileType } from '@/constants/file';
import { ParserGapReason, ParserModelKind } from './constant';
import {
BuiltinSwitchTargetLabels,
@@ -30,7 +31,21 @@ type UploadGapHintKeys = {
mixedSwitch: string;
};
// Video uploads are accepted but parsing is unsupported (no parser implements
// video analysis). The generic "reselect the parser" guidance is wrong for
// them: no selectable parser can process the file, so they get their own copy.
const isVideoGap = (gap: FileParserGap) => gap.fileType === FileType.Video;
const onlyVideoGaps = (gaps: FileParserGap[]) =>
gaps.length > 0 && gaps.every(isVideoGap);
function renderGapLine(t: TFunction, gap: FileParserGap): ReactNode {
if (gap.fileType === FileType.Video) {
return t('knowledgeDetails.fileTypeVideoUnsupported', {
name: gap.name,
fileType: t(`flow.fileFormatOptions.${gap.fileType}`),
});
}
if (gap.reason === ParserGapReason.MissingModel) {
return t('knowledgeDetails.fileModelMissing', {
name: gap.name,
@@ -52,6 +67,9 @@ function renderUploadGapLine(
t: TFunction,
gap: UploadFileParserGap,
): ReactNode {
if (gap.fileType === FileType.Video) {
return renderGapLine(t, gap);
}
if (gap.reason === ParserGapReason.MissingModel) {
return renderGapLine(t, gap);
}
@@ -91,7 +109,14 @@ export function buildParserGapModalContent(
gaps: FileParserGap[],
hintKeys: GapHintKeys,
): ReactNode {
return buildContent(t, gaps, pickByGapKind(gaps, hintKeys), renderGapLine);
return buildContent(
t,
gaps,
onlyVideoGaps(gaps)
? 'knowledgeDetails.videoUnsupportedParseHint'
: pickByGapKind(gaps, hintKeys),
renderGapLine,
);
}
export function buildUploadGapModalContent(
@@ -102,7 +127,9 @@ export function buildUploadGapModalContent(
return buildContent(
t,
gaps,
pickBySwitchFate(gaps, hintKeys),
onlyVideoGaps(gaps)
? 'knowledgeDetails.videoUnsupportedAfterUploadHint'
: pickBySwitchFate(gaps, hintKeys),
renderUploadGapLine,
);
}
@@ -1,4 +1,5 @@
import { Modal } from '@/components/ui/modal/modal';
import { FileType } from '@/constants/file';
import {
UseRowSelectionType,
useSelectedIds,
@@ -21,7 +22,7 @@ import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { useParams } from 'react-router';
import { toast } from 'sonner';
import { DocumentType } from './constant';
import { DocumentType, ParserGapReason } from './constant';
import { buildParserGapModalContent } from './parser-gap-content';
import { useParserGapValidation } from './use-parser-gap-validation';
import { isDocumentProcessing } from './utils';
@@ -72,14 +73,26 @@ export function useBulkOperateDataset({
.map((x) => x.id);
if (validIds.length === 0) {
// Pure-video selections have no parser that can process them, so
// offering anything that implies a fix would mislead. Acknowledge
// instead, mirroring the single-document run path. The reason
// check keeps this narrow: a future gap kind on a parseable file
// must keep the actionable dialog.
const videoOnly =
gaps.length > 0 &&
gaps.every(
(gap) =>
gap.fileType === FileType.Video &&
gap.reason === ParserGapReason.UnsupportedType,
);
Modal.error({
title: t('knowledgeDetails.parseBlockedTitle'),
content: buildParserGapModalContent(t, gaps, {
missingModel: 'knowledgeDetails.addModelToParseHint',
unsupportedType: 'knowledgeDetails.reselectParserToParseHint',
}),
showCancel: false,
okText: t('common.cancel'),
showCancel: !videoOnly,
okText: t(videoOnly ? 'common.ok' : 'common.cancel'),
closable: false,
});
return;
@@ -1,10 +1,12 @@
import { Modal } from '@/components/ui/modal/modal';
import { FileType } from '@/constants/file';
import { useSetModalState } from '@/hooks/common-hooks';
import { useRunDocument } from '@/hooks/use-document-request';
import { IDocumentInfo } from '@/interfaces/database/document';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { buildParserGapModalContent } from './parser-gap-content';
import { ParserGapReason } from './constant';
import { UseChangeDocumentParserShowType } from './use-change-document-parser';
import { useParserGapValidation } from './use-parser-gap-validation';
import { pickByGapKind } from './utils';
@@ -35,6 +37,18 @@ export const useHandleRunDocumentByIds = (
const gaps = findDocumentParseGaps([record]);
if (gaps.length > 0) {
hideModal();
// Video has no parser that can process it, so offering "reselect parse
// method" would send the user into a dialog that cannot fix anything.
// Acknowledge instead; the change-parser route stays for the other
// gap kinds. The reason check keeps this narrow: a future gap kind on
// a parseable file must keep the actionable dialog.
const videoOnly =
gaps.length > 0 &&
gaps.every(
(gap) =>
gap.fileType === FileType.Video &&
gap.reason === ParserGapReason.UnsupportedType,
);
Modal.error({
title: t('knowledgeDetails.parseBlockedTitle'),
content: buildParserGapModalContent(t, gaps, {
@@ -42,16 +56,17 @@ export const useHandleRunDocumentByIds = (
unsupportedType: 'knowledgeDetails.reselectParserToParseHint',
}),
okText: t(
pickByGapKind(gaps, {
missingModel: 'knowledgeDetails.goAddModel',
unsupportedType: 'knowledgeDetails.reselectParser',
}),
videoOnly
? 'common.ok'
: pickByGapKind(gaps, {
missingModel: 'knowledgeDetails.goAddModel',
unsupportedType: 'knowledgeDetails.reselectParser',
}),
),
cancelText: t('common.cancel'),
showCancel: !videoOnly,
closable: false,
onOk: () => {
showChangeParserModal(record);
},
onOk: videoOnly ? undefined : () => showChangeParserModal(record),
});
return;
}
+89 -36
View File
@@ -124,11 +124,37 @@ describe('getFileTypeByExtension', () => {
expect(getFileTypeByExtension('mp3')).toBe(FileType.Audio);
expect(getFileTypeByExtension('wav')).toBe(FileType.Audio);
expect(getFileTypeByExtension('mp4')).toBe(FileType.Video);
expect(getFileTypeByExtension('mov')).toBe(FileType.Video);
expect(getFileTypeByExtension('webm')).toBe(FileType.Video);
expect(getFileTypeByExtension('flv')).toBe(FileType.Video);
expect(getFileTypeByExtension('wmv')).toBe(FileType.Video);
// Legacy containers share the same backend video set, so the upload warning
// and parse block must recognize them too.
expect(getFileTypeByExtension('rm')).toBe(FileType.Video);
expect(getFileTypeByExtension('rmvb')).toBe(FileType.Video);
expect(getFileTypeByExtension('asf')).toBe(FileType.Video);
expect(getFileTypeByExtension('dat')).toBe(FileType.Video);
expect(getFileTypeByExtension('asx')).toBe(FileType.Video);
expect(getFileTypeByExtension('wvx')).toBe(FileType.Video);
expect(getFileTypeByExtension('mpe')).toBe(FileType.Video);
expect(getFileTypeByExtension('mpa')).toBe(FileType.Video);
expect(getFileTypeByExtension('jpeg')).toBe(FileType.Image);
expect(getFileTypeByExtension('PNG')).toBe(FileType.Image);
expect(getFileTypeByExtension('pdf')).toBe(FileType.PDF);
});
it('ignores surrounding whitespace like the backend does', () => {
// getExtension trims (mirroring utility.normalizeFilename); without it a
// name like 'clip.mov ' maps to nothing and skips both the upload warning
// and the parse block while the backend still admits it as visual.
expect(getFileTypeByExtension(getExtension('clip.mov '))).toBe(
FileType.Video,
);
expect(switchTarget('clip.mov ')).toBe('picture');
// The trim is shared: other consumers resolve trailing-space names too.
expect(getExtension('report.pdf ')).toBe('pdf');
});
it('returns undefined for unknown extensions', () => {
expect(getFileTypeByExtension('xyz')).toBeUndefined();
expect(getFileTypeByExtension('')).toBeUndefined();
@@ -136,9 +162,8 @@ describe('getFileTypeByExtension', () => {
});
// Saved config shaped like a pipeline that declares pdf/image plus audio
// (ASR model missing) and video, with the Parser-level vision enhancement
// enabled and its model configured — the current contract, where media
// families other than audio carry no model of their own.
// (ASR model missing) and video. Video needs no model: parsing it is
// unsupported regardless of the configured vision enhancement.
const SavedConfig: SavedParserConfig = {
setups: [
{ fileFormat: FileType.PDF, parse_method: 'DeepDOC' },
@@ -147,7 +172,6 @@ const SavedConfig: SavedParserConfig = {
{ fileFormat: FileType.Audio, vlm: { llm_id: '' } },
{ fileFormat: FileType.Video },
],
visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },
};
describe('findParserGap', () => {
@@ -180,29 +204,16 @@ describe('findParserGap', () => {
});
});
it('passes video when the Parser-level vision enhancement has a model', () => {
expect(findParserGap(FileType.Video, SavedConfig)).toBeNull();
});
it('flags declared video when the vision enhancement is off or has no model', () => {
const disabled: SavedParserConfig = {
setups: [{ fileFormat: FileType.Video }],
visionEnhancement: { enabled: false, modelId: 'glm-4v@ZHIPU-AI' },
};
const noModel: SavedParserConfig = {
setups: [{ fileFormat: FileType.Video }],
visionEnhancement: { enabled: true, modelId: '' },
};
const expected = {
reason: ParserGapReason.MissingModel,
it('flags video as unsupported even when a vision model is configured', () => {
// Video parsing is not implemented yet: a declared family with a vision
// model still stays unsupported so upload warns and parse-click blocks.
expect(findParserGap(FileType.Video, SavedConfig)).toEqual({
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
modelKind: ParserModelKind.Vision,
};
expect(findParserGap(FileType.Video, disabled)).toEqual(expected);
expect(findParserGap(FileType.Video, noModel)).toEqual(expected);
});
});
it('reads the video model from the family setup for configs saved before the move', () => {
it('flags declared video as unsupported regardless of its setup', () => {
const withModel: SavedParserConfig = {
setups: [
{ fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
@@ -211,12 +222,12 @@ describe('findParserGap', () => {
const withoutModel: SavedParserConfig = {
setups: [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }],
};
expect(findParserGap(FileType.Video, withModel)).toBeNull();
expect(findParserGap(FileType.Video, withoutModel)).toEqual({
reason: ParserGapReason.MissingModel,
const expected = {
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
modelKind: ParserModelKind.Vision,
});
};
expect(findParserGap(FileType.Video, withModel)).toEqual(expected);
expect(findParserGap(FileType.Video, withoutModel)).toEqual(expected);
});
it('passes image/email with a general-like setup — no switch needed', () => {
@@ -259,6 +270,11 @@ describe('findFilesParserGaps', () => {
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Docx,
},
{
name: 'movie.mp4',
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
},
{
name: 'deck.pptx',
reason: ParserGapReason.UnsupportedType,
@@ -295,10 +311,16 @@ describe('hasUnsupportedTypeGap', () => {
describe('builtinParserSwitchTarget', () => {
it('maps owned file types to their built-in pipelines', () => {
// The visual bucket includes the video extensions, mirroring the backend.
// Images switch to picture; video uploads are accepted into the visual
// bucket and switched to picture as well, but parsing stays blocked
// because no parser implements video analysis.
expect(switchTarget('photo.JPG')).toBe('picture');
expect(switchTarget('movie.mp4')).toBe('picture');
expect(switchTarget('clip.mov')).toBe('picture');
expect(switchTarget('clip.webm')).toBe('picture');
expect(switchTarget('clip.rmvb')).toBe('picture');
expect(switchTarget('clip.asf')).toBe('picture');
expect(switchTarget('clip.mpa')).toBe('picture');
expect(switchTarget('song.mp3')).toBe('audio');
expect(switchTarget('song.opus')).toBe('audio');
expect(switchTarget('deck.ppt')).toBe('presentation');
@@ -408,6 +430,22 @@ describe('pickBySwitchFate', () => {
pickBySwitchFate([unsupportedGap(null), missingModelGap], Options),
).toBe('manual');
});
it('does not count video as switchable even though it targets picture', () => {
// Video uploads switch to picture but stay unparsable, so a batch of a
// video plus a non-switchable file must read as manual, not mixed; a
// video plus an auto-switchable file reads as mixed, not auto.
const videoGap: UploadFileParserGap = {
name: 'clip.mov',
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
autoSwitchTarget: 'picture',
};
expect(pickBySwitchFate([videoGap], Options)).toBe('manual');
expect(
pickBySwitchFate([videoGap, unsupportedGap('general')], Options),
).toBe('mixed');
});
});
describe('getSavedParserConfig', () => {
@@ -443,7 +481,7 @@ describe('getSavedParserConfig', () => {
]);
});
it('reads the vision enhancement globals from the Parser entry top level', () => {
it('reads only the saved Parser setups from the Parser entry', () => {
const knowledgeBase = {
parser_config: {
'Parser:1': {
@@ -455,11 +493,10 @@ describe('getSavedParserConfig', () => {
};
expect(getSavedParserConfig(knowledgeBase)).toEqual({
setups: [{ fileFormat: FileType.Video }],
visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },
});
});
it('leaves the vision enhancement undefined for configs saved before the move', () => {
it('keeps family-local settings for configs saved before the vision move', () => {
const knowledgeBase = {
parser_config: {
'Parser:1': {
@@ -468,7 +505,6 @@ describe('getSavedParserConfig', () => {
},
};
const config = getSavedParserConfig(knowledgeBase);
expect(config?.visionEnhancement).toBeUndefined();
expect(config?.setups).toEqual([
{ fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
]);
@@ -508,8 +544,18 @@ describe('findDocumentsParserGaps', () => {
modelKind: ParserModelKind.Asr,
},
]);
// No fallback either: nothing can be determined, so nothing is flagged.
// No fallback either: nothing can be determined for other types, so
// nothing is flagged — except video, which is never parsable.
expect(findDocumentsParserGaps([legacyRow], null)).toEqual([]);
expect(
findDocumentsParserGaps([{ name: 'clip.mov', parser_config: {} }], null),
).toEqual([
{
name: 'clip.mov',
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
},
]);
});
it('evaluates each document against its own config in a mixed batch', () => {
@@ -544,6 +590,13 @@ describe('findDocumentsParserGaps', () => {
},
},
};
expect(findDocumentsParserGaps([videoDoc], null)).toEqual([]);
// Video stays unsupported even with a vision model configured.
expect(findDocumentsParserGaps([videoDoc], null)).toEqual([
{
name: 'movie.mp4',
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
},
]);
});
});
+61 -65
View File
@@ -129,13 +129,13 @@ export const isDocumentProcessing = (
// A file cannot be parsed under the dataset's current Parser operator when
// the operator does not declare the file's type family at all
// (unsupportedType), or when the family is declared but the model it needs
// is not set (missingModel): audio reads its per-setup ASR model, while
// video reads the Parser-level vision enhancement (enable_vision_enhancement
// + vlm.llm_id) — its model no longer lives on the family setup. The tenant
// default is deliberately not consulted: a global default does not make the
// file parsable here. Image files only need their family declared: the image
// OCR switch runs local extraction independently, and the vision model is a
// global supplement for it.
// is not set (missingModel): audio reads its per-setup ASR model. Video is
// temporarily blocked as unsupported until the video-analysis capability
// lands (maybeDispatchVideo still returns "not yet supported"), so it never
// reports missingModel even when its family is declared.
// Image files only need their family declared: the image OCR switch runs
// local extraction independently, and the vision model is a global supplement
// for it.
// These helpers power the upload warning and the parse-click validation.
export type ParserModelGap = {
@@ -209,8 +209,11 @@ const GeneralFamilyFileTypes = new Set<FileType>([
FileType.Docx,
]);
// Extension sets mirror utility.FilenameType (internal/utility/file.go); the
// visual bucket intentionally includes the video extensions.
// Extension sets mirror the backend upload buckets (utility.FilenameType):
// the visual bucket admits image containers plus every video container in the
// shared backend set, so video keeps the picture switch the backend performs.
// Parsability is judged separately: pickBySwitchFate excludes video from the
// switchable count because no parser implements video analysis.
const VisualSwitchSuffixes = new Set([
'jpg',
'jpeg',
@@ -237,21 +240,26 @@ const VisualSwitchSuffixes = new Set([
'apng',
'icon',
'ico',
'mpg',
'mpeg',
// Video containers (mirrors utility.videoExtensions).
'mp4',
'avi',
'mkv',
'mov',
'webm',
'flv',
'mpeg',
'mpg',
'wmv',
'3gp',
'3gpp',
'rm',
'rmvb',
'mov',
'wmv',
'asf',
'dat',
'asx',
'wvx',
'mpe',
'mpa',
'mp4',
'mkv',
]);
const AuralSwitchSuffixes = new Set([
@@ -279,6 +287,12 @@ export function builtinParserSwitchTarget(
name: string,
fileType: FileType | undefined,
): BuiltinSwitchTarget | null {
// Video uploads are accepted (visual bucket) but unsupported for parsing (no
// parser implements video analysis). The picture switch below mirrors what
// the backend performs; the row is still unparsable, so the user is warned
// at upload and blocked at parse-click. pickBySwitchFate separately excludes
// video from the switchable count so mixed batches never promise an
// automatic switch that cannot make the file parsable.
const suffix = getExtension(name);
if (VisualSwitchSuffixes.has(suffix)) {
return 'picture';
@@ -327,7 +341,12 @@ export function pickBySwitchFate<T>(
if (unsupported.length === 0) {
return options.missingModel;
}
const switchable = unsupported.filter((gap) => gap.autoSwitchTarget !== null);
// Video uploads switch to picture but stay unparsable, so they are not
// counted as switchable: counting them would promise an automatic switch
// that cannot make the file parsable.
const switchable = unsupported.filter(
(gap) => gap.autoSwitchTarget !== null && gap.fileType !== FileType.Video,
);
if (switchable.length === 0) {
return options.manualSwitch;
}
@@ -338,20 +357,8 @@ export function pickBySwitchFate<T>(
type ParserSetup = Record<string, any> & { fileFormat?: string };
/**
* The Parser-level vision enhancement globals (enable_vision_enhancement +
* vlm.llm_id): one switch and model serve every media family. Undefined marks
* a config saved before the model moved off the per-family setups — those
* still carry the video model on the video setup itself.
*/
export type ParserVisionEnhancement = {
enabled: boolean;
modelId: string;
};
export type SavedParserConfig = {
setups: ParserSetup[];
visionEnhancement?: ParserVisionEnhancement;
};
const ExtensionToFileTypeMap: Record<string, FileType> = Object.entries(
@@ -369,32 +376,12 @@ export function getFileTypeByExtension(
return ExtensionToFileTypeMap[extension.toLowerCase()];
}
// Reads the Parser-level vision globals from a saved Parser entry. A boolean
// enable_vision_enhancement marks the current contract; configs saved before
// the model moved to the top level do not have it, and their video setup
// still holds the model — so the absence is meaningful, not a default-off.
function readVisionEnhancement(
parserEntry: unknown,
): ParserVisionEnhancement | undefined {
if (typeof parserEntry !== 'object' || parserEntry === null) {
return undefined;
}
const params = parserEntry as Record<string, any>;
const enabled = params.enable_vision_enhancement;
if (typeof enabled !== 'boolean') {
return undefined;
}
const modelId = params.vlm?.llm_id;
return { enabled, modelId: typeof modelId === 'string' ? modelId : '' };
}
/**
* Parser config exactly as saved: the per-family setups plus the Parser-level
* vision enhancement globals. Accepts either a dataset or a document row: a
* document's parser_config snapshots the parser it actually runs with
* (dataset config at upload, the override after a parser change). Returns
* null when parser_config has no Parser operator entry (legacy flat-shape
* configs, pipelines without a Parser stage): type support cannot be
* Parser config exactly as saved: the per-family setups. Accepts either a
* dataset or a document row: a document's parser_config snapshots the parser it
* actually runs with (dataset config at upload, the override after a parser
* change). Returns null when parser_config has no Parser operator entry (legacy
* flat-shape configs, pipelines without a Parser stage): type support cannot be
* determined then and validation is skipped.
*/
export function getSavedParserConfig(
@@ -411,7 +398,6 @@ export function getSavedParserConfig(
}
return {
setups: transformParserConfigSetups(parserEntry),
visionEnhancement: readVisionEnhancement(parserEntry),
};
}
@@ -441,16 +427,12 @@ export function findParserGap(
};
}
case FileType.Video: {
const hasVisionModel = config.visionEnhancement
? config.visionEnhancement.enabled && !!config.visionEnhancement.modelId
: !!setup.vlm?.llm_id;
return hasVisionModel
? null
: {
reason: ParserGapReason.MissingModel,
fileType,
modelKind: ParserModelKind.Vision,
};
// Video parsing is not implemented yet (the backend dispatch returns
// "video parsing is not yet supported"), so a declared family or a
// configured vision model does not make the file parsable. Report it
// as unsupported so upload warns and parse-click blocks instead of
// prompting for a model that cannot unblock it.
return { reason: ParserGapReason.UnsupportedType, fileType };
}
default:
return null;
@@ -480,6 +462,20 @@ export function findDocumentsParserGaps(
): FileParserGap[] {
return documents.flatMap((doc) => {
const config = getSavedParserConfig(doc) ?? fallbackConfig;
return config ? findFilesParserGaps([doc.name], config) : [];
if (!config) {
// Video is never parsable regardless of configuration, so it stays
// blocked even when type support cannot be determined for other files.
const fileType = getFileTypeByExtension(getExtension(doc.name));
return fileType === FileType.Video
? [
{
reason: ParserGapReason.UnsupportedType,
fileType,
name: doc.name,
},
]
: [];
}
return findFilesParserGaps([doc.name], config);
});
}
+4 -1
View File
@@ -60,7 +60,10 @@ export const buildChunkHighlights = (
export const isFileUploadDone = (file: UploadFile) => file.status === 'done';
export const getExtension = (name: string) =>
name?.slice(name.lastIndexOf('.') + 1).toLowerCase() ?? '';
name
?.slice(name.lastIndexOf('.') + 1)
.trim()
.toLowerCase() ?? '';
export const isPdf = (name: string) => {
return getExtension(name) === 'pdf';