From 0aa00d0d9da0a6ecfff79c7f01492129dcf43e48 Mon Sep 17 00:00:00 2001 From: balibabu Date: Sat, 10 Oct 2026 13:43:34 +0800 Subject: [PATCH] Fix: Limit the pipeline canvas to a maximum of one extractor operator. (#20711) --- .../advanced_ingestion_pipeline.json | 433 +++++------------- web/AGENTS.md | 1 + web/src/pages/agent/constant/pipeline.tsx | 11 + .../__tests__/pipeline-connection.test.ts | 59 +++ .../pages/agent/utils/pipeline-connection.ts | 26 +- 5 files changed, 203 insertions(+), 327 deletions(-) create mode 100644 web/src/pages/agent/utils/__tests__/pipeline-connection.test.ts diff --git a/internal/agent/templates/advanced_ingestion_pipeline.json b/internal/agent/templates/advanced_ingestion_pipeline.json index 68ac23a2a8..f4215787c2 100644 --- a/internal/agent/templates/advanced_ingestion_pipeline.json +++ b/internal/agent/templates/advanced_ingestion_pipeline.json @@ -14,162 +14,83 @@ "canvas_category": "dataflow_canvas", "dsl": { "components": { - "Extractor:CurlyEmusJam": { + "Extractor:LazyCarpetsKiss": { "downstream": [ "Tokenizer:WittySunsListen" ], "obj": { "component_name": "Extractor", "params": { - "field_name": "metadata", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, "llm_id": "", - "maxTokensEnabled": false, + "temperature": 0.1, + "top_p": 0.3, + "frequency_penalty": 0.7, + "presence_penalty": 0.4, "max_tokens": 256, + "temperatureEnabled": true, + "topPEnabled": true, + "presencePenaltyEnabled": true, + "frequencyPenaltyEnabled": true, + "maxTokensEnabled": false, + "keywords": { + "top_n": 5, + "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top {{ topn }} important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY." + }, + "questions": { + "top_n": 3, + "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose {{ topn }} questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top {{ topn }} important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY." + }, + "tags": { + "top_n": 0, + "tag_file_id": "" + }, + "summary": { + "enabled": true, + "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff." + }, + "metadata": { + "enabled": true, + "metadata": [ + { + "key": "names", + "type": "string", + "description": "Names of people, organizations, or products mentioned in the content" + }, + { + "key": "dates", + "type": "string", + "description": "Important dates or time periods mentioned in the content" + }, + { + "key": "locations", + "type": "string", + "description": "Geographic locations mentioned in the content" + }, + { + "key": "events", + "type": "string", + "description": "Key events described in the content" + }, + { + "key": "key_facts", + "type": "string", + "description": "Important facts or numerical data stated in the content" + } + ], + "built_in_metadata": [] + }, "outputs": { "chunks": { "type": "Array", "value": [] } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Content:\n{Extractor:SmartWindowsHammer@chunks}", - "role": "user" - } - ], - "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 - } - }, - "upstream": [ - "Extractor:SmartWindowsHammer" - ] - }, - "Extractor:LazyCarpetsKiss": { - "downstream": [ - "Extractor:LovelyPearsRest" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "summary", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}", - "role": "user" - } - ], - "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 + } } }, "upstream": [ "TokenChunker:BumpyStarsPress" ] }, - "Extractor:LovelyPearsRest": { - "downstream": [ - "Extractor:SmartWindowsHammer" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "keywords", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}", - "role": "user" - } - ], - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 - } - }, - "upstream": [ - "Extractor:LazyCarpetsKiss" - ] - }, - "Extractor:SmartWindowsHammer": { - "downstream": [ - "Extractor:CurlyEmusJam" - ], - "obj": { - "component_name": "Extractor", - "params": { - "field_name": "questions", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": [ - { - "content": "Text Content\n{Extractor:LovelyPearsRest@chunks}", - "role": "user" - } - ], - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 - } - }, - "upstream": [ - "Extractor:LovelyPearsRest" - ] - }, "File": { "downstream": [ "Parser:HipSignsRhyme" @@ -368,7 +289,7 @@ } }, "upstream": [ - "Extractor:CurlyEmusJam" + "Extractor:LazyCarpetsKiss" ] } }, @@ -402,41 +323,9 @@ "data": { "isHovered": false }, - "id": "xy-edge__Extractor:LazyCarpetsKissstart-Extractor:LovelyPearsRestend", + "id": "xy-edge__Extractor:LazyCarpetsKissstart-Tokenizer:WittySunsListenend", "source": "Extractor:LazyCarpetsKiss", "sourceHandle": "start", - "target": "Extractor:LovelyPearsRest", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:LovelyPearsReststart-Extractor:SmartWindowsHammerend", - "selected": false, - "source": "Extractor:LovelyPearsRest", - "sourceHandle": "start", - "target": "Extractor:SmartWindowsHammer", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:SmartWindowsHammerstart-Extractor:CurlyEmusJamend", - "selected": false, - "source": "Extractor:SmartWindowsHammer", - "sourceHandle": "start", - "target": "Extractor:CurlyEmusJam", - "targetHandle": "end" - }, - { - "data": { - "isHovered": false - }, - "id": "xy-edge__Extractor:CurlyEmusJamstart-Tokenizer:WittySunsListenend", - "source": "Extractor:CurlyEmusJam", - "sourceHandle": "start", "target": "Tokenizer:WittySunsListen", "targetHandle": "end" } @@ -611,30 +500,73 @@ { "data": { "form": { - "field_name": "summary", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, "llm_id": "", - "maxTokensEnabled": false, + "temperature": 0.1, + "top_p": 0.3, + "frequency_penalty": 0.7, + "presence_penalty": 0.4, "max_tokens": 256, + "temperatureEnabled": true, + "topPEnabled": true, + "presencePenaltyEnabled": true, + "frequencyPenaltyEnabled": true, + "maxTokensEnabled": false, + "keywords": { + "top_n": 5, + "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top {{ topn }} important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY." + }, + "questions": { + "top_n": 3, + "system_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose {{ topn }} questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top {{ topn }} important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY." + }, + "tags": { + "top_n": 0, + "tag_file_id": "" + }, + "summary": { + "enabled": true, + "system_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff." + }, + "metadata": { + "enabled": true, + "metadata": [ + { + "key": "names", + "type": "string", + "description": "Names of people, organizations, or products mentioned in the content" + }, + { + "key": "dates", + "type": "string", + "description": "Important dates or time periods mentioned in the content" + }, + { + "key": "locations", + "type": "string", + "description": "Geographic locations mentioned in the content" + }, + { + "key": "events", + "type": "string", + "description": "Key events described in the content" + }, + { + "key": "key_facts", + "type": "string", + "description": "Important facts or numerical data stated in the content" + } + ], + "built_in_metadata": [] + }, "outputs": { "chunks": { "type": "Array", "value": [] } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Text to Summarize:\n{TokenChunker:BumpyStarsPress@chunks}", - "sys_prompt": "Act as a precise summarizer. Your task is to create a summary of the provided content that is both concise and faithful to the original.\n\nKey Instructions:\n1. Accuracy: Strictly base the summary on the information given. Do not introduce any new facts, conclusions, or interpretations that are not explicitly stated.\n2. Language: Write the summary in the same language as the source text.\n3. Objectivity: Present the key points without bias, preserving the original intent and tone of the content. Do not editorialize.\n4. Conciseness: Focus on the most important ideas, omitting minor details and fluff.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 + } }, "label": "Extractor", - "name": "Summarization" + "name": "Auto Extract" }, "id": "Extractor:LazyCarpetsKiss", "measured": { @@ -650,135 +582,6 @@ "targetPosition": "left", "type": "contextNode" }, - { - "data": { - "form": { - "field_name": "keywords", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Text Content\n{Extractor:LazyCarpetsKiss@chunks}", - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Auto Keyword" - }, - "dragging": false, - "id": "Extractor:LovelyPearsRest", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 983.5410692821999, - "y": 301.1557383781162 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - }, - { - "data": { - "form": { - "field_name": "questions", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Text Content\n{Extractor:LovelyPearsRest@chunks}", - "sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Auto Question" - }, - "dragging": false, - "id": "Extractor:SmartWindowsHammer", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 1021.1009769800036, - "y": 421.67760363913044 - }, - "selected": false, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - }, - { - "data": { - "form": { - "field_name": "metadata", - "frequencyPenaltyEnabled": true, - "frequency_penalty": 0.7, - "llm_id": "", - "maxTokensEnabled": false, - "max_tokens": 256, - "outputs": { - "chunks": { - "type": "Array", - "value": [] - } - }, - "presencePenaltyEnabled": true, - "presence_penalty": 0.4, - "prompts": "Content:\n{Extractor:SmartWindowsHammer@chunks}", - "sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.", - "temperature": 0.1, - "temperatureEnabled": true, - "tenant_llm_id": null, - "topPEnabled": true, - "top_p": 0.3 - }, - "label": "Extractor", - "name": "Auto Metadata" - }, - "dragging": false, - "id": "Extractor:CurlyEmusJam", - "measured": { - "height": 90, - "width": 200 - }, - "position": { - "x": 1065.7115140232393, - "y": 527.4370438206126 - }, - "selected": true, - "sourcePosition": "right", - "targetPosition": "left", - "type": "contextNode" - }, { "data": { "form": { diff --git a/web/AGENTS.md b/web/AGENTS.md index 9511f4b6e5..8d379255fe 100644 --- a/web/AGENTS.md +++ b/web/AGENTS.md @@ -134,6 +134,7 @@ The folder `src/components/ui/` is the project's **shared UI library** — it co - Components: `EditableTextarea`, `RAGFlowFormItem` - Constants: `InitialMockData`, `DefaultPlaceholder` - Avoid camelCase or SCREAMING_SNAKE_CASE for components and top-level constants. +- **Place domain constant lists in the page's `constant/` directory** (e.g. `pages/agent/constant/pipeline.tsx`) when they belong to an existing family of constants there (like `ChunkerOperators`, `SingleOperators`), instead of declaring them locally next to their consumer. Keeping related lists in one file makes their overlaps and differences visible and gives new entries a single obvious home. - **Name things semantically, not generically.** A name should say what the value holds or what the callback does, so call sites read naturally. Don't leak a generic underlying-primitive API into your own component's public props — rename at the wrapper boundary (e.g., a tabs wrapper exposes `activeTab` / `onTabChange`, mapped internally to Radix's `value` / `onValueChange`), and align with what callers already name the state. Same for variables and functions: `hiddenNodeIds` over `nodeIdSet`, `pickByBackend` over `resolveVariant`. - Avoid duplicating component structures in JSX; favor render props or reusable components. diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx index 4852cb1590..aa8652cae3 100644 --- a/web/src/pages/agent/constant/pipeline.tsx +++ b/web/src/pages/agent/constant/pipeline.tsx @@ -429,12 +429,23 @@ export const NoDebugOperatorsList = [Operator.File]; // legacy Python operator and is deliberately excluded. export const ChunkerOperators = [Operator.TokenChunker, Operator.TitleChunker]; +// Operators offered in the flat top-level list of the pipeline "next step" +// menu, distinct from the chunker group. Single-instance filtering happens +// per entry in buildPipelineNextOperators. +export const TopLevelOperators = [ + Operator.Parser, + Operator.Tokenizer, + Operator.Compiler, + Operator.Extractor, +]; + // Pipeline operators limited to a single instance per canvas. Title/Token // chunkers are separate entries here (each blocks its own duplication) but // also share one slot: see the chunker group rule in buildPipelineNextOperators. export const SingleOperators = [ Operator.Tokenizer, Operator.Compiler, + Operator.Extractor, Operator.GeneralChunker, Operator.TokenChunker, Operator.TitleChunker, diff --git a/web/src/pages/agent/utils/__tests__/pipeline-connection.test.ts b/web/src/pages/agent/utils/__tests__/pipeline-connection.test.ts new file mode 100644 index 0000000000..9c3d572d61 --- /dev/null +++ b/web/src/pages/agent/utils/__tests__/pipeline-connection.test.ts @@ -0,0 +1,59 @@ +import { Operator } from '@/constants/agent'; +import { + buildPipelineNextOperators, + hasPipelineNextOperators, + isSingleInstanceOperator, +} from '../pipeline-connection'; + +const noOperators = () => false; +const onlyExtractor = (operator: Operator) => operator === Operator.Extractor; + +describe('buildPipelineNextOperators', () => { + it('offers Extractor when the canvas has none', () => { + const { operators } = buildPipelineNextOperators( + Operator.TokenChunker, + noOperators, + ); + expect(operators).toContain(Operator.Extractor); + }); + + it('hides Extractor when one is already on the canvas', () => { + const { operators } = buildPipelineNextOperators( + Operator.TokenChunker, + onlyExtractor, + ); + expect(operators).not.toContain(Operator.Extractor); + // Other operators are unaffected. + expect(operators).toEqual( + expect.arrayContaining([ + Operator.Parser, + Operator.Tokenizer, + Operator.Compiler, + ]), + ); + }); + + it('still hides the chunker group from an Extractor source', () => { + const { operators, showChunker } = buildPipelineNextOperators( + Operator.Extractor, + onlyExtractor, + ); + expect(operators).not.toContain(Operator.Extractor); + expect(showChunker).toBe(false); + }); +}); + +describe('hasPipelineNextOperators', () => { + it('reports an empty menu when only single-instance operators remain', () => { + const everythingTaken = () => true; + expect(hasPipelineNextOperators(Operator.Extractor, everythingTaken)).toBe( + false, + ); + }); +}); + +describe('isSingleInstanceOperator', () => { + it('treats Extractor as single-instance so duplication is disabled', () => { + expect(isSingleInstanceOperator(Operator.Extractor)).toBe(true); + }); +}); diff --git a/web/src/pages/agent/utils/pipeline-connection.ts b/web/src/pages/agent/utils/pipeline-connection.ts index ba1647f924..04dc99098a 100644 --- a/web/src/pages/agent/utils/pipeline-connection.ts +++ b/web/src/pages/agent/utils/pipeline-connection.ts @@ -1,5 +1,9 @@ import { Operator } from '@/constants/agent'; -import { ChunkerOperators, SingleOperators } from '../constant/pipeline'; +import { + ChunkerOperators, + SingleOperators, + TopLevelOperators, +} from '../constant/pipeline'; export function isChunkerOperator(operator: Operator): boolean { return (ChunkerOperators as Operator[]).includes(operator); @@ -15,8 +19,8 @@ export function isSingleInstanceOperator(operator: Operator): boolean { } export interface PipelineNextOperators { - // Operators offered in the flat top-level list (Parser, Tokenizer, - // Extractor, Compiler), minus the single-instance ones already on canvas. + // Operators offered in the flat top-level list, minus the single-instance + // ones already on canvas. operators: Operator[]; // The chunker variants, offered only while the single chunker slot is free. chunkerOperators: Operator[]; @@ -38,16 +42,14 @@ export function buildPipelineNextOperators( const operators: Operator[] = []; // Pipelines require a Parser to feed a chunker, so from a Parser node the - // menu offers only the chunker group. + // menu offers only the chunker group. Extractor joins the filtered list + // because the Go pipeline allows at most one per canvas (ValidatePipeline). if (source !== Operator.Parser) { - [Operator.Parser, Operator.Tokenizer, Operator.Compiler].forEach( - (operator) => { - if (!hasOperator(operator)) { - operators.push(operator); - } - }, - ); - operators.push(Operator.Extractor); + TopLevelOperators.forEach((operator) => { + if (!hasOperator(operator)) { + operators.push(operator); + } + }); } // The chunker group occupies a single slot: any chunker already on the