From 99e804e70dc32ae0d0a69c0eb18b06b3df5afe61 Mon Sep 17 00:00:00 2001 From: MaxtuneLee Date: Mon, 3 Jun 2024 16:45:28 +0800 Subject: [PATCH 001/128] docs: add auth header when use openai model --- .../docs-demo-openai-header_2024-06-03-08-17.json | 10 ++++++++++ .../vmind/__tests__/browser/src/pages/DataInput.tsx | 12 +++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 common/changes/@visactor/vmind/docs-demo-openai-header_2024-06-03-08-17.json diff --git a/common/changes/@visactor/vmind/docs-demo-openai-header_2024-06-03-08-17.json b/common/changes/@visactor/vmind/docs-demo-openai-header_2024-06-03-08-17.json new file mode 100644 index 00000000..9f9d6260 --- /dev/null +++ b/common/changes/@visactor/vmind/docs-demo-openai-header_2024-06-03-08-17.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@visactor/vmind", + "comment": "send authorization header when use openai models", + "type": "none" + } + ], + "packageName": "@visactor/vmind" +} \ No newline at end of file diff --git a/packages/vmind/__tests__/browser/src/pages/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/DataInput.tsx index 8712ed95..46b97eaa 100644 --- a/packages/vmind/__tests__/browser/src/pages/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/DataInput.tsx @@ -118,9 +118,15 @@ export function DataInput(props: IPropsType) { model, cache, showThoughts: showThoughts, - headers: { - 'api-key': apiKey - } + headers: + model.includes(Model.GPT3_5) || model.includes(Model.GPT4) + ? { + //must has Authorization: `Bearer ${openAIKey}` if use openai api + Authorization: `Bearer ${apiKey}` + } + : { + 'api-key': apiKey + } }); }, [apiKey, cache, model, showThoughts, url]); From ed0b9d62c140e5fe030cf15a2f70597057d53199 Mon Sep 17 00:00:00 2001 From: MaxtuneLee Date: Wed, 3 Jul 2024 10:56:02 +0800 Subject: [PATCH 002/128] docs: set all headers in all cases --- .../docs-demo-openai-header_2024-07-03-02-55.json | 10 ++++++++++ .../__tests__/browser/src/pages/DataInput.tsx | 14 +++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-02-55.json diff --git a/common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-02-55.json b/common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-02-55.json new file mode 100644 index 00000000..0945a5c3 --- /dev/null +++ b/common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-02-55.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@visactor/vmind", + "comment": "set all headers in all cases", + "type": "none" + } + ], + "packageName": "@visactor/vmind" +} \ No newline at end of file diff --git a/packages/vmind/__tests__/browser/src/pages/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/DataInput.tsx index 46b97eaa..f245e1e4 100644 --- a/packages/vmind/__tests__/browser/src/pages/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/DataInput.tsx @@ -118,15 +118,11 @@ export function DataInput(props: IPropsType) { model, cache, showThoughts: showThoughts, - headers: - model.includes(Model.GPT3_5) || model.includes(Model.GPT4) - ? { - //must has Authorization: `Bearer ${openAIKey}` if use openai api - Authorization: `Bearer ${apiKey}` - } - : { - 'api-key': apiKey - } + headers: { + //must has Authorization: `Bearer ${openAIKey}` if use openai api + Authorization: `Bearer ${apiKey}`, + 'api-key': apiKey + } }); }, [apiKey, cache, model, showThoughts, url]); From c6392df5441b095aaf73f843cfe708ef28027403 Mon Sep 17 00:00:00 2001 From: MaxtuneLee Date: Wed, 3 Jul 2024 11:06:36 +0800 Subject: [PATCH 003/128] docs: add bearer header in smart insight page --- .../docs-demo-openai-header_2024-07-03-03-05.json | 10 ++++++++++ .../__tests__/browser/src/pages/Insight/DataInput.tsx | 1 + 2 files changed, 11 insertions(+) create mode 100644 common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-03-05.json diff --git a/common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-03-05.json b/common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-03-05.json new file mode 100644 index 00000000..3ef3b3d9 --- /dev/null +++ b/common/changes/@visactor/vmind/docs-demo-openai-header_2024-07-03-03-05.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@visactor/vmind", + "comment": "add bearer header in smart insight page", + "type": "none" + } + ], + "packageName": "@visactor/vmind" +} \ No newline at end of file diff --git a/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx index 8b89b781..d20d6e98 100644 --- a/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx @@ -90,6 +90,7 @@ export function DataInput(props: IPropsType) { cache, showThoughts: showThoughts, headers: { + Authorization: `Bearer ${apiKey}`, 'api-key': apiKey } }); From fee5db74f07ae4eebada1529c87ee735ceda28f8 Mon Sep 17 00:00:00 2001 From: xile611 Date: Thu, 11 Jul 2024 11:08:43 +0800 Subject: [PATCH 004/128] feat: support liquid, linearProgress and circlularProgress --- .../browser/src/constants/mockData.ts | 12 ++ .../pages/ChartGeneration/ChartPreview.tsx | 4 +- .../src/pages/ChartGeneration/DataInput.tsx | 8 +- .../skylark/prompt/knowledge.ts | 13 ++ .../GPT/prompt/knowledges.ts | 24 +++ .../getChartSpec/VChart/chartPipeline.ts | 42 ++++- .../getChartSpec/VChart/transformers.ts | 150 +++++++++++++++++- packages/vmind/src/common/typings/index.ts | 5 +- 8 files changed, 246 insertions(+), 12 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 01b815f4..dfa24217 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -4855,3 +4855,15 @@ export const mockUserTextInput10 = { 如果你需要了解更多股市指数波动信息，可以访问财经新闻网站或咨询专业的投资顾问。` }; + +export const mockProgressData = { + csv: `年份,进度 + 2024,0.56`, + input: '帮我展示今年的进度数据' +}; + +export const liquidData = { + csv: `进度 + 0.56`, + input: '展示进度数据' +}; diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/ChartPreview.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/ChartPreview.tsx index 2c022bb9..69603b3b 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/ChartPreview.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/ChartPreview.tsx @@ -1,7 +1,7 @@ import React, { useState, useCallback, useEffect, useRef } from 'react'; import './index.scss'; import { Button, Input, Card, Space, Modal, Spin } from '@arco-design/web-react'; -import VChart from '@visactor/vchart'; +import VChart, { registerLiquidChart } from '@visactor/vchart'; import { ManualTicker, defaultTimeline } from '@visactor/vrender-core'; import VMind from '../../../../../src'; import { createFFmpeg, fetchFile } from '@ffmpeg/ffmpeg'; @@ -36,6 +36,8 @@ function downloadVideo(link: string, filename = 'out') { a.dispatchEvent(new MouseEvent('click')); } +registerLiquidChart(); + export function ChartPreview(props: IPropsType) { const [generating, setGenerating] = useState(false); const [outType, setOutType] = useState<'gif' | 'video' | ''>(''); diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index 49ee5dbf..036beecb 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -38,7 +38,9 @@ import { mockUserInput17, mockUserInput18, SalesRecordsData, - gmvData + gmvData, + mockProgressData, + liquidData } from '../../constants/mockData'; import VMind, { ArcoTheme, builtinThemeMap, BuiltinThemeType } from '../../../../../src/index'; import { Model } from '../../../../../src/index'; @@ -83,7 +85,9 @@ const demoDataList: { [key: string]: any } = { 'Sales of different drinkings': mockUserInput3Eng, 'Multi measure': mockUserInput17, DataQuery: mockUserInput18, - salesData: SalesRecordsData + salesData: SalesRecordsData, + progress: mockProgressData, + liquid: liquidData }; const globalVariables = (import.meta as any).env; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts index a46b60ae..f1a55f98 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts @@ -60,5 +60,18 @@ export const chartKnowledgeBase: ChartKnowledgeBase = { 'Dynamic Bar Chart can only be used when data has a field that is date type.' ], constraints: ['Use Dynamic Bar Chart if user want to show changes of rankings in data.'] + }, + + [ChartType.LinearProgress]: { + knowledge: ['Linear progress chart shows progress value of one or more categories '] + }, + + [ChartType.CircularProgress]: { + knowledge: ['Circular progress chart shows progress value of one or more categories'] + }, + + [ChartType.LiquidChart]: { + knowledge: ['Liquid chart show a percent value'], + constraints: ['Use Liquid Chart if user want to show a percent value'] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts index 9bec80d4..cd945b33 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts @@ -117,6 +117,30 @@ export const chartKnowledgeDict: ChartKnowledge = { knowledge: [ 'Dynamic Bar Chart is a dynamic chart that is suitable for displaying changing data and can be used to show ranking, comparisons or data changes over time. It usually has a time field. It updates the data dynamically according to the time field.' ] + }, + [ChartType.LiquidChart]: { + index: 14, + visualChannels: ['x', 'y'], + examples: [], + knowledge: [ + 'Liquid chart is used to display a single value, with the value range typically from 0 to 1. The value usually represents progress, completion, or percentage, and is associated with only one field' + ] + }, + [ChartType.LinearProgress]: { + index: 15, + visualChannels: ['x', 'y'], + examples: [], + knowledge: [ + 'Linear Progress chart is typically used to display progress data, which is usually a value between 0 and 1. Linear progress bars can show single progress values as well as multiple progress values. By default, the left Y-axis of the linear progress bar is the categorical field, and the bottom X-axis is the numerical field.' + ] + }, + [ChartType.CircularProgress]: { + index: 16, + visualChannels: ['x', 'y'], + examples: [], + knowledge: [ + 'Circular progress chart is also used to display progress data, presented in a circular form, with the values on the numerical axis typically ranging from 0 to 1.' + ] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts index 8df5a9b8..8b6b23cf 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts @@ -1,3 +1,4 @@ +import { ChartType } from '../../../../../common/typings'; import type { Transformer } from '../../../../../base/tools/transformer'; import type { GetChartSpecContext, GetChartSpecOutput } from '../types'; import { @@ -25,10 +26,6 @@ import { rankingBarField, customMark, scatterAxis, - animationOneByOne, - animationCartesianBar, - animationCartisianLine, - animationCartesianPie, wordCloudData, displayConfBar, displayConfLine, @@ -46,7 +43,15 @@ import { waterfallStackLabel, boxPlotField, boxPlotStyle, - theme + theme, + liquidField, + liquidStyle, + linearProgressField, + circularProgressField, + circularProgressStyle, + linearProgressStyle, + linearProgressAxes, + indicator } from './transformers'; const pipelineBar = [ @@ -145,6 +150,28 @@ const pipelineWaterfall = [chartType, data, color, waterfallField, waterfallAxes const pipelineBoxPlot = [chartType, data, color, boxPlotField, boxPlotStyle, legend, theme]; +const pipelineLiquid = [chartType, data, color, liquidField, liquidStyle, indicator, theme]; + +const pipelineLinearProgress = [ + chartType, + data, + color, + linearProgressField, + linearProgressAxes, + linearProgressStyle, + theme +]; + +const pipelineCircularProgress = [ + chartType, + data, + color, + circularProgressField, + circularProgressStyle, + indicator, + theme +]; + const pipelineMap: { [chartType: string]: any } = { 'BAR CHART': pipelineBar, 'LINE CHART': pipelineLine, @@ -158,7 +185,10 @@ const pipelineMap: { [chartType: string]: any } = { 'RADAR CHART': pipelineRadar, 'SANKEY CHART': pipelineSankey, 'WATERFALL CHART': pipelineWaterfall, - 'BOX PLOT': pipelineBoxPlot + 'BOX PLOT': pipelineBoxPlot, + [ChartType.LiquidChart.toUpperCase()]: pipelineLiquid, + [ChartType.LinearProgress.toUpperCase()]: pipelineLinearProgress, + [ChartType.CircularProgress.toUpperCase()]: pipelineCircularProgress }; export const beforePipe: Transformer = ( diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 45acb99d..3f665797 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -18,7 +18,7 @@ import { import { getFieldByDataType } from '../../../../../common/utils/utils'; import { array, isArray } from '@visactor/vutils'; import { isValidDataset } from '../../../../../common/dataProcess'; -import { DataType, ChartTheme } from '../../../../../common/typings'; +import { DataType, ChartType } from '../../../../../common/typings'; import { builtinThemeMap } from '../../../../../common/builtinTheme'; import { COLOR_FIELD } from '@visactor/chart-advisor'; @@ -37,7 +37,10 @@ const chartTypeMap: { [chartName: string]: string } = { 'RADAR CHART': 'radar', 'SANKEY CHART': 'sankey', 'WATERFALL CHART': 'waterfall', - 'BOX PLOT': 'boxPlot' + 'BOX PLOT': 'boxPlot', + [ChartType.LiquidChart.toUpperCase()]: 'liquid', + [ChartType.LinearProgress.toUpperCase()]: 'linearProgress', + [ChartType.CircularProgress.toUpperCase()]: 'circularProgress' }; export const chartType: Transformer = (context: Context) => { @@ -1239,3 +1242,146 @@ export const theme: Transformer = (context: Context return { spec }; }; + +export const liquidField: Transformer = (context: Context) => { + const { cell, dataset, spec } = context; + + spec.valueField = cell.value; + spec.indicatorSmartInvert = true; + + return { spec }; +}; + +export const liquidStyle: Transformer = (context: Context) => { + const { spec } = context; + spec.liquid = { + ...spec.liquid, + style: {} + }; + return { spec }; +}; + +export const linearProgressField: Transformer = (context: Context) => { + //assign field in spec according to cell + const { cell, spec } = context; + + spec.xField = cell.y; + spec.yField = cell.x; + if (cell.color) { + spec.seriesField = cell.color; + } + spec.cornerRadius = 20; + + return { spec }; +}; + +export const linearProgressAxes: Transformer = (context: Context) => { + //assign field in spec according to cell + const { cell, spec } = context; + const hasSingleData = spec.data.values && spec.data.values.length === 1; + + spec.axes = [ + { + orient: 'left', + type: 'band', + domainLine: { visible: false }, + tick: { visible: false }, + label: { + formatMethod: hasSingleData ? val => `${cell.x}: ${val}` : null, + style: { + fontSize: 16 + } + } + }, + { + orient: 'bottom', + type: 'linear', + visible: true, + grid: { + visible: false + }, + label: { + formatMethod: (val: number) => { + return val >= 0 && val <= 1 ? `${val * 100}%` : val; + }, + flush: true + } + } + ]; + + return { spec }; +}; + +export const linearProgressStyle: Transformer = (context: Context) => { + const { spec } = context; + spec.progress = { + ...spec.progress, + style: {} + }; + return { spec }; +}; + +export const circularProgressField: Transformer = (context: Context) => { + //assign field in spec according to cell + const { cell, spec } = context; + + spec.categoryField = cell.radius; + spec.valueField = cell.value; + if (cell.color) { + spec.seriesField = cell.color; + } + + spec.radius = 0.8; + spec.innerRadius = 0.7; + spec.roundCap = true; + spec.cornerRadius = 20; + + return { spec }; +}; + +export const circularProgressStyle: Transformer = (context: Context) => { + const { spec } = context; + spec.progress = { + ...spec.progress, + style: {} + }; + return { spec }; +}; + +export const indicator: Transformer = (context: Context) => { + const { spec, cell } = context; + const firstEntry = spec.data.values[0]; + if (!firstEntry) { + return { spec }; + } + const valueField = (cell.value ?? cell.y) as string; + const value = firstEntry[valueField]; + const cat = firstEntry[cell.radius ?? cell.x]; + + spec.indicator = { + visible: true, + fixed: true, + trigger: 'none', + title: { + visible: true, + autoLimit: true, + space: 12, + style: { + fontSize: 16, + fill: 'gray', + text: cat ?? valueField + } + }, + content: [ + { + visible: true, + style: { + fontSize: 20, + fill: '#000', + text: value >= 0 && value <= 1 ? `${(value * 100).toFixed(2)}%` : `${value}` + } + } + ] + }; + return { spec }; +}; diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index f709af11..73bc22a0 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -72,7 +72,10 @@ export enum ChartType { FunnelChart = 'Funnel Chart', DualAxisChart = 'Dual Axis Chart', WaterFallChart = 'Waterfall Chart', - BoxPlot = 'Box Plot' + BoxPlot = 'Box Plot', + LinearProgress = 'Linear Progress chart', + CircularProgress = 'Circular Progress chart', + LiquidChart = 'Liquid Chart' } export type GPTChartAdvisorResult = { From 7ceb9055eb24c6d2271204ac67f6355ad253f24e Mon Sep 17 00:00:00 2001 From: xile611 Date: Thu, 11 Jul 2024 22:53:11 +0800 Subject: [PATCH 005/128] feat: update knowledge of liquid, linear progress chart, circular progress chart in skylark promit --- .../skylark/prompt/knowledge.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts index d7e6b9ee..807d957e 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts @@ -1,4 +1,5 @@ /* eslint-disable max-len */ +import { ChartType } from '../../../../../../common/typings'; import type { ChannelInfo } from './types'; export const ChartFieldInfo: ChannelInfo = { @@ -204,5 +205,56 @@ export const ChartFieldInfo: ChannelInfo = { "time channel can't be empty", 'Only date fields can be used in time channel.' ] + }, + + [ChartType.LinearProgress.toUpperCase()]: { + visualChannels: { + x: "x-axis of bar chart. Can't be empty. Can only use categorical field.", + y: "y-axis of bar chart. Can't be empty. Only number fields", + color: + 'color channel of bar. Used to distinguish different bars. Only categorical fields. Can be empty if no suitable field.' + }, + responseDescription: { + x: 'field assigned to x channel', + y: 'field assigned to y channel', + color: 'field assigned to color channel' + }, + knowledge: [ + "x-axis in linear progress chart can only be a categorical field. Don't use time field", + 'Only use categorical field can be used in x channel', + 'y field can not be empty' + ] + }, + + [ChartType.CircularProgress.toUpperCase()]: { + visualChannels: { + x: "x-axis of bar chart. Can't be empty. Can only use categorical field.", + y: "y-axis of bar chart. Can't be empty. Only number fields", + color: + 'color channel of bar. Used to distinguish different bars. Only categorical fields. Can be empty if no suitable field.' + }, + responseDescription: { + x: 'field assigned to x channel', + y: 'field assigned to y channel', + color: 'field assigned to color channel' + }, + knowledge: [ + "x-axis in linear progress chart can only be a categorical field. Don't use time field", + 'Only use categorical field can be used in x channel', + 'y field can not be empty' + ] + }, + + [ChartType.LiquidChart.toUpperCase()]: { + visualChannels: { + value: "value field of liquid chart. Can't be empty. Only number fields" + }, + responseDescription: { + value: 'field assigned to value channel' + }, + knowledge: [ + 'Liquid chart is used to display a single value, with the value range typically from 0 to 1.', + 'The value usually represents progress, completion, or percentage, and is associated with only one field' + ] } }; From e244c9abe7f8d55f94fa6d8c169d8a1c22c2ff43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Mon, 15 Jul 2024 17:09:47 +0800 Subject: [PATCH 006/128] Based on GPT, covers VChart basic chart types --- .../browser/src/constants/mockData.ts | 465 +++++++++++++++++ .../src/pages/ChartGeneration/DataInput.tsx | 22 +- packages/vmind/package.json | 2 +- .../applications/chartGeneration/constants.ts | 12 +- .../generateTypeAndFieldMap/GPT/index.ts | 12 + .../GPT/patcher/index.ts | 169 ++++++- .../GPT/prompt/index.ts | 3 +- .../GPT/prompt/knowledges.ts | 102 +++- .../GPT/prompt/template.ts | 5 +- .../getChartSpec/VChart/chartPipeline.ts | 86 +++- .../getChartSpec/VChart/constants.ts | 2 + .../getChartSpec/VChart/transformers.ts | 466 +++++++++++++++++- .../src/applications/chartGeneration/types.ts | 3 +- packages/vmind/src/applications/types.ts | 12 +- .../src/base/taskNode/ruleBasedTaskNode.ts | 13 +- packages/vmind/src/common/typings/index.ts | 55 ++- packages/vmind/src/common/utils/utils.ts | 3 + packages/vmind/src/core/VMind.ts | 28 +- 18 files changed, 1409 insertions(+), 51 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index dfa24217..1f5fe7e5 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -4751,6 +4751,471 @@ export const gmvData = { input: '帮我绘制按城市的gmv和增长率，使用双轴图' }; +export const bubbleCirclePackingData = { + csv: `industry,gross product +第一产业,88345.1 +第二产业,483164.5 +第三产业,638697.6 +农林牧渔业,92582.4 +工业,401644.3 +建筑业,83383.1 +批发和零售业,114517.7 +交通运输、仓储和邮政业,49673.7 +住宿和餐饮业,17855.3 +金融业,96811.0 +房地产业,73821.3 +其他,279918.4 +`, + input: '请使用气泡图帮我绘制' +}; + +export const mapChartData = { + csv: `name,value +Russia,17234034 +Canada,9984670 +China,9596960 +United States of America,9525067 +Brazil,8515167 +`, + input: '请将数据渲染到地图图表中' +}; + +export const rangeColumnChartData = { + csv: `categories,min,max +Category One,76,100 +Category Two,56,108 +Category Three,38,129 +Category Four,58,155 +Category Five,45,120 +Category Six,23,99 +Category Seven,18,56 +Category Eight,18,34 +`, + input: '请使用区间柱图渲染数据' +}; + +export const sunburstChartData = { + csv: `Country,Region,Category,Value +Country A,Region1,Office Supplies,824 +Country A,Region1,Furniture,920 +Country A,Region1,Electronic equipment,936 +Country A,Region2,Office Supplies,1270 +Country A,Region2,Furniture,1399 +Country A,Region2,Electronic equipment,1466 +Country A,Region3,Office Supplies,1408 +Country A,Region3,Furniture,1676 +Country A,Region3,Electronic equipment,1559 +Country A,Region4,Office Supplies,745 +Country A,Region4,Furniture,919 +Country A,Region4,Electronic equipment,781 +Country A,Region5,Office Supplies,267 +Country A,Region5,Furniture,316 +Country A,Region5,Electronic equipment,230 +Country A,Region6,Office Supplies,347 +Country A,Region6,Furniture,501 +Country A,Region6,Electronic equipment,453 +Country B,Region1,Office Supplies,824 +Country B,Region1,Furniture,920 +Country B,Region1,Electronic equipment,936 +Country B,Region2,Office Supplies,1270 +Country B,Region2,Furniture,1399 +Country B,Region2,Electronic equipment,1466 +Country B,Region3,Office Supplies,1408 +Country B,Region3,Furniture,1676 +Country B,Region3,Electronic equipment,1559 +Country B,Region4,Office Supplies,745 +Country B,Region4,Furniture,919 +Country B,Region4,Electronic equipment,781 +Country B,Region5,Office Supplies,267 +Country B,Region5,Furniture,316 +Country B,Region5,Electronic equipment,230 +Country B,Region6,Office Supplies,347 +Country B,Region6,Furniture,501 +Country B,Region6,Electronic equipment,453 +Country C,Region1,Office Supplies,824 +Country C,Region1,Furniture,920 +Country C,Region1,Electronic equipment,936 +Country C,Region2,Office Supplies,1270 +Country C,Region2,Furniture,1399 +Country C,Region2,Electronic equipment,1466 +Country C,Region3,Office Supplies,1408 +Country C,Region3,Furniture,1676 +Country C,Region3,Electronic equipment,1559 +Country C,Region4,Office Supplies,745 +Country C,Region4,Furniture,919 +Country C,Region4,Electronic equipment,781 +Country C,Region5,Office Supplies,267 +Country C,Region5,Furniture,316 +Country C,Region5,Electronic equipment,230 +Country C,Region6,Office Supplies,347 +Country C,Region6,Furniture,501 +Country C,Region6,Electronic equipment,453 +`, + input: '请使用旭日图来渲染数据' +}; + +export const treemapChartData = { + csv: `Category-0,Category-1,Category-2,Category-3,value +query,methods,add,,593 +query,methods,and,,330 +query,methods,average,,287 +query,methods,count,,277 +query,methods,distinct,,292 +query,methods,div,,595 +query,methods,eq,,594 +query,methods,fn,,460 +query,methods,gt,,603 +query,methods,gte,,625 +query,methods,iff,,748 +query,methods,isa,,461 +query,methods,lt,,597 +query,methods,lte,,619 +query,methods,max,,283 +query,methods,min,,283 +query,methods,mod,,591 +query,methods,mul,,603 +query,methods,neq,,599 +query,methods,not,,386 +query,methods,or,,323 +query,methods,orderby,,307 +query,methods,range,,772 +query,methods,select,,296 +query,methods,stddev,,363 +query,methods,sub,,600 +query,methods,sum,,280 +query,methods,update,,307 +query,methods,variance,,335 +query,methods,where,,299 +query,methods,xor,,354 +query,methods,_,,264 +query,AggregateExpression,,,1616 +query,And,,,1027 +query,Arithmetic,,,3891 +query,Average,,,891 +query,BinaryExpression,,,2893 +query,Comparison,,,5103 +query,CompositeExpression,,,3677 +query,Count,,,781 +query,DateUtil,,,4141 +query,Distinct,,,933 +query,Expression,,,5130 +query,ExpressionIterator,,,3617 +query,Fn,,,3240 +query,If,,,2732 +query,IsA,,,2039 +query,Literal,,,1214 +query,Match,,,3748 +query,Maximum,,,843 +query,Minimum,,,843 +query,Not,,,1554 +query,Or,,,970 +query,Query,,,13896 +query,Range,,,1594 +query,StringUtil,,,4130 +query,Sum,,,791 +query,Variable,,,1124 +query,Variance,,,1876 +query,Xor,,,1101 +util,palette,ColorPalette,,6367 +util,palette,Palette,,1229 +util,palette,ShapePalette,,2059 +util,palette,valuePalette,,2291 +util,math,DenseMatrix,,3165 +util,math,IMatrix,,2815 +util,math,SparseMatrix,,3366 +util,heap,FibonacciHeap,,9354 +util,heap,HeapNode,,1233 +util,Arrays,,,8258 +util,Colors,,,10001 +util,Dates,,,8217 +util,Displays,,,12555 +util,Filter,,,2324 +util,Geometry,,,10993 +util,IEvaluable,,,335 +util,IPredicate,,,383 +util,IValueProxy,,,874 +util,Maths,,,17705 +util,Orientation,,,1486 +util,Property,,,5559 +util,Shapes,,,19118 +util,Sort,,,6887 +util,Stats,,,6557 +util,Strings,,,22026 +animate,interpolate,ArrayInterpolator,,1983 +animate,interpolate,ColorInterpolator,,2047 +animate,interpolate,DateInterpolator,,1375 +animate,interpolate,Interpolator,,8746 +animate,interpolate,MatrixInterpolator,,2202 +animate,interpolate,NumberInterpolator,,1382 +animate,interpolate,ObjectInterpolator,,1629 +animate,interpolate,PointInterpolator,,1675 +animate,interpolate,RectangleInterpolator,,2042 +animate,Easing,,,17010 +animate,FunctionSequence,,,5842 +animate,ISchedulable,,,1041 +animate,Parallel,,,5176 +animate,Pause,,,449 +animate,Scheduler,,,5593 +animate,Sequence,,,5534 +animate,Transition,,,9201 +animate,Transitioner,,,19975 +animate,TransitionEvent,,,1116 +animate,Tween,,,6006 +scale,IScaleMap,,,2105 +scale,LinearScale,,,1316 +scale,LogScale,,,3151 +scale,OrdinalScale,,,3770 +scale,QuantileScale,,,2435 +scale,QuantitativeScale,,,4839 +scale,RootScale,,,1756 +scale,Scale,,,4268 +scale,ScaleType,,,1821 +scale,TimeScale,,,5833 +physics,DragForce,,,1082 +physics,GravityForce,,,1336 +physics,IForce,,,319 +physics,NBodyForce,,,10498 +physics,Particle,,,2822 +physics,Simulation,,,9983 +physics,Spring,,,2213 +physics,SpringForce,,,1681 +data,converters,Converters,,721 +data,converters,DelimitedTextConverter,,4294 +data,converters,GraphMLConverter,,9800 +data,converters,IDataConverter,,1314 +data,converters,JSONConverter,,2220 +data,DataField,,,1759 +data,DataSchema,,,2165 +data,DataSet,,,586 +data,DataSource,,,3331 +data,DataTable,,,772 +data,DataUtil,,,3322 +vis,controls,AnchorControl,,2138 +vis,controls,ClickControl,,3824 +vis,controls,Control,,1353 +vis,controls,ControlList,,4665 +vis,controls,DragControl,,2649 +vis,controls,ExpandControl,,2832 +vis,controls,HoverControl,,4896 +vis,controls,IControl,,763 +vis,controls,PanZoomControl,,5222 +vis,controls,SelectionControl,,7862 +vis,controls,TooltipControl,,8435 +vis,operator,layout,AxisLayout,6725 +vis,operator,layout,BundledEdgeRouter,3727 +vis,operator,layout,CircleLayout,9317 +vis,operator,layout,CirclePackingLayout,12003 +vis,operator,layout,DendrogramLayout,4853 +vis,operator,layout,ForceDirectedLayout,8411 +vis,operator,layout,IcicleTreeLayout,4864 +vis,operator,layout,IndentedTreeLayout,3174 +vis,operator,layout,Layout,7881 +vis,operator,layout,NodeLinkTreeLayout,12870 +vis,operator,layout,PieLayout,2728 +vis,operator,layout,RadialTreeLayout,12348 +vis,operator,layout,RandomLayout,870 +vis,operator,layout,StackedAreaLayout,9121 +vis,operator,layout,TreeMapLayout,9191 +vis,operator,encoder,ColorEncoder,3179 +vis,operator,encoder,Encoder,4060 +vis,operator,encoder,PropertyEncoder,4138 +vis,operator,encoder,ShapeEncoder,1690 +vis,operator,encoder,valueEncoder,1830 +vis,operator,distortion,BifocalDistortion,4461 +vis,operator,distortion,Distortion,6314 +vis,operator,distortion,FisheyeDistortion,3444 +vis,operator,filter,FisheyeTreeFilter,5219 +vis,operator,filter,GraphDistanceFilter,3165 +vis,operator,filter,VisibilityFilter,3509 +vis,operator,label,Labeler,9956 +vis,operator,label,RadialLabeler,3899 +vis,operator,label,StackedAreaLabeler,3202 +vis,operator,IOperator,,1286 +vis,operator,Operator,,2490 +vis,operator,OperatorList,,5248 +vis,operator,OperatorSequence,,4190 +vis,operator,OperatorSwitch,,2581 +vis,operator,SortOperator,,2023 +vis,data,render,ArrowType,698 +vis,data,render,EdgeRenderer,5569 +vis,data,render,IRenderer,353 +vis,data,render,ShapeRenderer,2247 +vis,data,Data,,20544 +vis,data,DataList,,19788 +vis,data,DataSprite,,10349 +vis,data,EdgeSprite,,3301 +vis,data,NodeSprite,,19382 +vis,data,ScaleBinding,,11275 +vis,data,Tree,,7147 +vis,data,TreeBuilder,,9930 +vis,axis,Axes,,1302 +vis,axis,Axis,,24593 +vis,axis,AxisGridLine,,652 +vis,axis,AxisLabel,,636 +vis,axis,CartesianAxes,,6703 +vis,events,DataEvent,,2313 +vis,events,SelectionEvent,,1880 +vis,events,TooltipEvent,,1701 +vis,events,VisualizationEvent,,1117 +vis,legend,Legend,,20859 +vis,legend,LegendItem,,4614 +vis,legend,LegendRange,,10530 +vis,Visualization,,,16540 +display,DirtySprite,,,8833 +display,LineSprite,,,1732 +display,RectSprite,,,3623 +display,TextSprite,,,10066 +analytics,graph,BetweennessCentrality,,3534 +analytics,graph,LinkDistance,,5731 +analytics,graph,MaxFlowMinCut,,7840 +analytics,graph,ShortestPaths,,5914 +analytics,graph,SpanningTree,,3416 +analytics,cluster,AgglomerativeCluster,,3938 +analytics,cluster,CommunityStructure,,3812 +analytics,cluster,HierarchicalCluster,,6714 +analytics,cluster,MergeEdge,,743 +analytics,optimization,AspectRatioBanker,,7074 +flex,FlareVis,,,4116`, + input: '请使用矩形树图渲染数据' +}; + +export const gaugeChartData = { + csv: `name,value,description + 目标A,0.6,目标A的描述`, + input: '请使用仪表盘来完成数据渲染' +}; + +export const linearProgressChartData = { + csv: `type,value,text +Tradition Industries,0.795,79.5% +Business Companies,0.25,25% +Customer-facing Companies,0.065,6.5%`, + input: '请使用线形进度图来完成数据渲染' +}; + +export const basicHeatMapChartData = { + csv: `val1,val2,value +Asset Liability Ratio,Asset Liability Ratio,1 +Asset Liability Ratio,Asset Liability Ratio (Deducting Advance Payments),0.594527 +Asset Liability Ratio,Debt-to-long Capital Ratio,0.492963 +Asset Liability Ratio,Long Term Asset Suitability Ratio,-0.160995 +Asset Liability Ratio,Equity Multiplier,0.723664 +Asset Liability Ratio,Equity Ratio of Current Liability,0.658646 +Asset Liability Ratio,Interest Bearing Debt / Fully Invested Capital,-0.857474 +Asset Liability Ratio,Current Liability / Total Liabilities,0.320706 +Asset Liability Ratio,Capital Fixation Ratio,-0.284634 +Asset Liability Ratio,Expected Default Frequency,-0.091423 +Asset Liability Ratio (Deducting Advance Payments),Asset Liability Ratio,0.594527 +Asset Liability Ratio (Deducting Advance Payments),Asset Liability Ratio (Deducting Advance Payments),1 +Asset Liability Ratio (Deducting Advance Payments),Debt-to-long Capital Ratio,0.724546 +Asset Liability Ratio (Deducting Advance Payments),Long Term Asset Suitability Ratio,-0.099318 +Asset Liability Ratio (Deducting Advance Payments),Equity Multiplier,0.540639 +Asset Liability Ratio (Deducting Advance Payments),Equity Ratio of Current Liability,0.49214 +Asset Liability Ratio (Deducting Advance Payments),Interest Bearing Debt / Fully Invested Capital,-0.554039 +Asset Liability Ratio (Deducting Advance Payments),Current Liability / Total Liabilities,0.17127 +Asset Liability Ratio (Deducting Advance Payments),Capital Fixation Ratio,-0.265259 +Asset Liability Ratio (Deducting Advance Payments),Expected Default Frequency,0.068577 +Debt-to-long Capital Ratio,Asset Liability Ratio,0.492963 +Debt-to-long Capital Ratio,Asset Liability Ratio (Deducting Advance Payments),0.724546 +Debt-to-long Capital Ratio,Debt-to-long Capital Ratio,1 +Debt-to-long Capital Ratio,Long Term Asset Suitability Ratio,-0.091338 +Debt-to-long Capital Ratio,Equity Multiplier,0.450542 +Debt-to-long Capital Ratio,Equity Ratio of Current Liability,0.375839 +Debt-to-long Capital Ratio,Interest Bearing Debt / Fully Invested Capital,-0.524955 +Debt-to-long Capital Ratio,Current Liability / Total Liabilities,0.300627 +Debt-to-long Capital Ratio,Capital Fixation Ratio,-0.198362 +Debt-to-long Capital Ratio,Expected Default Frequency,0.033209 +Long Term Asset Suitability Ratio,Asset Liability Ratio,-0.160995 +Long Term Asset Suitability Ratio,Asset Liability Ratio (Deducting Advance Payments),-0.099318 +Long Term Asset Suitability Ratio,Debt-to-long Capital Ratio,-0.091338 +Long Term Asset Suitability Ratio,Long Term Asset Suitability Ratio,1 +Long Term Asset Suitability Ratio,Equity Multiplier,-0.049872 +Long Term Asset Suitability Ratio,Equity Ratio of Current Liability,-0.028452 +Long Term Asset Suitability Ratio,Interest Bearing Debt / Fully Invested Capital,0.157157 +Long Term Asset Suitability Ratio,Current Liability / Total Liabilities,0.009742 +Long Term Asset Suitability Ratio,Capital Fixation Ratio,-0.162374 +Long Term Asset Suitability Ratio,Expected Default Frequency,0.155095 +Equity Multiplier,Asset Liability Ratio,0.723664 +Equity Multiplier,Asset Liability Ratio (Deducting Advance Payments),0.540639 +Equity Multiplier,Debt-to-long Capital Ratio,0.450542 +Equity Multiplier,Long Term Asset Suitability Ratio,-0.049872 +Equity Multiplier,Equity Multiplier,1 +Equity Multiplier,Equity Ratio of Current Liability,0.951933 +Equity Multiplier,Interest Bearing Debt / Fully Invested Capital,-0.651767 +Equity Multiplier,Current Liability / Total Liabilities,0.079052 +Equity Multiplier,Capital Fixation Ratio,-0.535984 +Equity Multiplier,Expected Default Frequency,0.00798 +Equity Ratio of Current Liability,Asset Liability Ratio,0.658646 +Equity Ratio of Current Liability,Asset Liability Ratio (Deducting Advance Payments),0.49214 +Equity Ratio of Current Liability,Debt-to-long Capital Ratio,0.375839 +Equity Ratio of Current Liability,Long Term Asset Suitability Ratio,-0.028452 +Equity Ratio of Current Liability,Equity Multiplier,0.951933 +Equity Ratio of Current Liability,Equity Ratio of Current Liability,1 +Equity Ratio of Current Liability,Interest Bearing Debt / Fully Invested Capital,-0.543147 +Equity Ratio of Current Liability,Current Liability / Total Liabilities,-0.106139 +Equity Ratio of Current Liability,Capital Fixation Ratio,-0.52232 +Equity Ratio of Current Liability,Expected Default Frequency,0.011466 +Interest Bearing Debt / Fully Invested Capital,Asset Liability Ratio,-0.857474 +Interest Bearing Debt / Fully Invested Capital,Asset Liability Ratio (Deducting Advance Payments),-0.554039 +Interest Bearing Debt / Fully Invested Capital,Debt-to-long Capital Ratio,-0.524955 +Interest Bearing Debt / Fully Invested Capital,Long Term Asset Suitability Ratio,0.157157 +Interest Bearing Debt / Fully Invested Capital,Equity Multiplier,-0.651767 +Interest Bearing Debt / Fully Invested Capital,Equity Ratio of Current Liability,-0.543147 +Interest Bearing Debt / Fully Invested Capital,Interest Bearing Debt / Fully Invested Capital,1 +Interest Bearing Debt / Fully Invested Capital,Current Liability / Total Liabilities,-0.595016 +Interest Bearing Debt / Fully Invested Capital,Capital Fixation Ratio,0.310521 +Interest Bearing Debt / Fully Invested Capital,Expected Default Frequency,0.066397 +Current Liability / Total Liabilities,Asset Liability Ratio,0.320706 +Current Liability / Total Liabilities,Asset Liability Ratio (Deducting Advance Payments),0.17127 +Current Liability / Total Liabilities,Debt-to-long Capital Ratio,0.300627 +Current Liability / Total Liabilities,Long Term Asset Suitability Ratio,0.009742 +Current Liability / Total Liabilities,Equity Multiplier,0.079052 +Current Liability / Total Liabilities,Equity Ratio of Current Liability,-0.106139 +Current Liability / Total Liabilities,Interest Bearing Debt / Fully Invested Capital,-0.595016 +Current Liability / Total Liabilities,Current Liability / Total Liabilities,1 +Current Liability / Total Liabilities,Capital Fixation Ratio,-0.105199 +Current Liability / Total Liabilities,Expected Default Frequency,-0.064886 +Capital Fixation Ratio,Asset Liability Ratio,-0.284634 +Capital Fixation Ratio,Asset Liability Ratio (Deducting Advance Payments),-0.265259 +Capital Fixation Ratio,Debt-to-long Capital Ratio,-0.198362 +Capital Fixation Ratio,Long Term Asset Suitability Ratio,-0.162374 +Capital Fixation Ratio,Equity Multiplier,-0.535984 +Capital Fixation Ratio,Equity Ratio of Current Liability,-0.52232 +Capital Fixation Ratio,Interest Bearing Debt / Fully Invested Capital,0.310521 +Capital Fixation Ratio,Current Liability / Total Liabilities,-0.105199 +Capital Fixation Ratio,Capital Fixation Ratio,1 +Capital Fixation Ratio,Expected Default Frequency,-0.080153 +Expected Default Frequency,Asset Liability Ratio,-0.091423 +Expected Default Frequency,Asset Liability Ratio (Deducting Advance Payments),0.068577 +Expected Default Frequency,Debt-to-long Capital Ratio,0.033209 +Expected Default Frequency,Long Term Asset Suitability Ratio,0.155095 +Expected Default Frequency,Equity Multiplier,0.00798 +Expected Default Frequency,Equity Ratio of Current Liability,0.011466 +Expected Default Frequency,Interest Bearing Debt / Fully Invested Capital,0.066397 +Expected Default Frequency,Current Liability / Total Liabilities,-0.064886 +Expected Default Frequency,Capital Fixation Ratio,-0.080153 +Expected Default Frequency,Expected Default Frequency,1`, + input: '请使用热图完成数据渲染' +}; + +export const vennChartData = { + csv: `sets,name,value +0,A,8 +1,D,8 +2,B,10 +3,C,12 +4,A,4 +4,B,4 +5,A,4 +5,C,4 +6,B,4 +6,C,4 +7,A,2 +7,B,2 +7,C,2`, + input: '请使用韦恩图渲染数据' +}; + export const mockUserTextInput0 = { text: `快手消失了。快手上市后，市值一度超过2000亿美元，现在只剩200多亿美元。去年快手的营收破了千亿，公司也赚钱了，但市场不买账了。 滴滴也不见了。滴滴去年营收接近2000亿元，并首次实现年度盈利，不过滴滴从美股退市后，一直没在港股上市，所以没有市值参考。`, diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index 0ba31f56..c14daebb 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -40,7 +40,16 @@ import { SalesRecordsData, gmvData, mockProgressData, - liquidData + liquidData, + bubbleCirclePackingData, + rangeColumnChartData, + sunburstChartData, + treemapChartData, + gaugeChartData, + linearProgressChartData, + basicHeatMapChartData, + vennChartData, + mapChartData } from '../../constants/mockData'; import VMind, { ArcoTheme, builtinThemeMap, BuiltinThemeType } from '../../../../../src/index'; import { Model } from '../../../../../src/index'; @@ -87,7 +96,16 @@ const demoDataList: { [key: string]: any } = { DataQuery: mockUserInput18, salesData: SalesRecordsData, progress: mockProgressData, - liquid: liquidData + liquid: liquidData, + BubbleCirclePacking: bubbleCirclePackingData, + Map: mapChartData, + RangeColumn: rangeColumnChartData, + Sunburst: sunburstChartData, + TreeMap: treemapChartData, + Gauge: gaugeChartData, + LinearProgress: linearProgressChartData, + BasicHeatMap: basicHeatMapChartData, + Venn: vennChartData }; const globalVariables = (import.meta as any).env; diff --git a/packages/vmind/package.json b/packages/vmind/package.json index e773ba67..ff7ef369 100644 --- a/packages/vmind/package.json +++ b/packages/vmind/package.json @@ -77,7 +77,7 @@ "react": "^18.0.0", "react-dom": "^18.0.0", "react-router-dom": "6.9.0", - "@visactor/vchart": "^1.10.4", + "@visactor/vchart": "^1.11.4", "@rollup/plugin-dynamic-import-vars": "~2.1.0", "@types/node": "*", "dotenv": "~16.3.1", diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index 951d4cac..07b0ec82 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -1,3 +1,13 @@ -import { ChartType } from '../../common/typings'; +import type { BasemapOption } from '../../common/typings'; +import { ChartType, MapRegionCoordinate, UncommonChartType } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); +export const SUPPORTED_UNCOMMON_CHART_LIST = Object.values(UncommonChartType); + +export const DEFAULT_MAP_OPTION: BasemapOption = { + jsonUrl: 'https://lf9-dp-fe-cms-tos.byteorg.com/obj/bit-cloud/geojson/world.json', + regionProjectType: null, + regionCoordinate: MapRegionCoordinate.GEO, + zoom: 1, + center: null +}; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts index ef7168d5..bd2fbd4d 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts @@ -6,13 +6,19 @@ import { chartGenerationRequestLLM, parseChartGenerationResponse } from './utils import { GPTChartGenerationPrompt } from './prompt'; import { patchAxisField, + patchBasicHeatMapChart, patchBoxPlot, patchCartesianXField, patchColorField, patchDualAxis, patchDynamicBarChart, patchLabelField, + patchLinearProgressChart, + patchNeedColor, + patchNeedSize, patchPieChart, + patchRangeColumnChart, + patchSunburstAndTreemapChart, patchWordCloud, patchYField } from './patcher'; @@ -34,11 +40,17 @@ const ChartGenerationTaskNodeGPTMeta: LLMBasedTaskNodeMeta< patchColorField, patchLabelField, patchYField, + patchNeedColor, + patchNeedSize, patchBoxPlot, patchDualAxis, patchPieChart, patchWordCloud, patchDynamicBarChart, + patchRangeColumnChart, + patchLinearProgressChart, + patchSunburstAndTreemapChart, + patchBasicHeatMapChart, patchCartesianXField ], requester: chartGenerationRequestLLM, diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts index 3ae1eb1a..1b6504a4 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts @@ -6,12 +6,13 @@ import { foldDatasetByYField, getFieldByDataType, getFieldByRole, + getFieldsByDataType, getRemainedFields } from '../../../../../../common/utils/utils'; -import { ChartType } from '../../../../../../common/typings'; -import { DataType, ROLE } from '../../../../../../common/typings'; +import { ChartType, DataType, ROLE } from '../../../../../../common/typings'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../../types'; import { isValidDataset } from '../../../../../../common/dataProcess'; +import { needSizeFieldChartList, needColorFieldChartList } from '../prompt/knowledges'; const CARTESIAN_CHART_LIST = [ 'Dynamic Bar Chart', @@ -21,7 +22,9 @@ const CARTESIAN_CHART_LIST = [ 'Funnel Chart', 'Dual Axis Chart', 'Waterfall Chart', - 'Box Plot' + 'Box Plot', + 'Range Column Chart', + 'linear Progress Chart' ]; export const patchAxisField: Transformer< @@ -96,7 +99,8 @@ export const patchYField: Transformer< if (y && isArray(y) && y.length > 1) { if ( chartTypeNew === ('BOX PLOT' as ChartType) || - (chartTypeNew === ('DUAL AXIS CHART' as ChartType) && y.length === 2) + (chartTypeNew === ('DUAL AXIS CHART' as ChartType) && y.length === 2) || + (chartTypeNew === ('RANGE COLUMN CHART' as ChartType) && y.length === 2) ) { return { ...context @@ -367,3 +371,160 @@ export const patchCartesianXField: Transformer< cell: cellNew }; }; + +export const patchNeedColor: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + const { chartType, cell, fieldInfo } = context; + const cellNew: any = { ...cell }; + if (needColorFieldChartList.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType)) { + const colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean).flat(); + if (colorField.length !== 0) { + cellNew.color = colorField[0]; + } else { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const colorField = getFieldByRole(remainedFields, ROLE.DIMENSION); + if (colorField) { + cellNew.color = colorField.fieldName; + } else { + cellNew.color = remainedFields[0].fieldName; + } + } + } + return { + cell: cellNew + }; +}; + +export const patchNeedSize: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + const { chartType, cell, fieldInfo } = context; + const cellNew: any = { ...cell }; + if (needSizeFieldChartList.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType)) { + const sizeField = [cellNew.size, cellNew.value, cellNew.y, cellNew.radius, cellNew.angle].filter(Boolean).flat(); + if (sizeField.length !== 0) { + cellNew.size = sizeField[0]; + } else { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const sizeField = getFieldByRole(remainedFields, ROLE.MEASURE); + if (sizeField) { + cellNew.size = sizeField.fieldName; + } else { + cellNew.size = remainedFields[0].fieldName; + } + } + } + return { + cell: cellNew + }; +}; + +export const patchRangeColumnChart: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + // Range Column Chart's y field must length == 2 + const { chartType, cell, fieldInfo } = context; + const cellNew = { ...cell }; + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const numericFields = getFieldsByDataType(remainedFields, [DataType.FLOAT, DataType.INT]); + if (chartType === ('RANGE COLUMN CHART' as ChartType)) { + if (cellNew.y && cellNew.y instanceof Array && cellNew.y.length === 2) { + return { cell: cellNew }; + } + if (numericFields.length >= 2) { + cellNew.y = [numericFields[0].fieldName, numericFields[1].fieldName]; + } else { + throw Error( + 'The y-axis of the range column chart requires two numeric fields, ' + + 'but the result of data aggregation does not have two numeric fields' + ); + } + } + return { + //...context, + cell: cellNew + }; +}; + +export const patchLinearProgressChart: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + const { chartType, cell, fieldInfo } = context; + const cellNew = { ...cell }; + if (chartType === ('Linear Progress Chart' as ChartType)) { + const yField = [cellNew.y, cellNew.size, cellNew.value, cellNew.radius, cellNew.angle].filter(Boolean).flat(); + if (yField.length !== 0) { + cellNew.y = yField[0]; + } else { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const yField = getFieldByRole(remainedFields, ROLE.MEASURE); + if (yField) { + cellNew.y = yField.fieldName; + } else { + cellNew.y = remainedFields[0].fieldName; + } + } + } + return { + //...context, + cell: cellNew + }; +}; + +export const patchSunburstAndTreemapChart: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + const { chartType, cell } = context; + const cellNew: any = { ...cell }; + + if (chartType === ('SUNBURST CHART' as ChartType) || chartType === ('TREEMAP CHART' as ChartType)) { + cellNew.y = [cellNew.y, cellNew.value, cellNew.radius, cellNew.size, cellNew.angle].filter(Boolean).flat(); + if (cellNew.y.length !== 0) { + cellNew.y = cellNew.y[0]; + } else { + throw Error( + 'The y-axis of the sunburst chart and the treemap chart requires a numeric field,' + + 'but the result of data aggregation does not have a numeric field' + ); + } + } + + return { + //...context, + cell: cellNew + }; +}; + +export const patchBasicHeatMapChart: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + const { chartType, cell, fieldInfo } = context; + const cellNew: any = { ...cell }; + if (chartType === ('BASIC HEAT MAP' as ChartType)) { + const colorField = [cellNew.x, cellNew.y, cellNew.label, cellNew.color].filter(Boolean).flat(); + if (colorField.length >= 2) { + cellNew.x = colorField[0]; + cellNew.y = colorField[1]; + } else { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const colorField = getFieldsByDataType(remainedFields, [DataType.STRING]); + if (colorField.length >= 2) { + cellNew.x = colorField[0]; + cellNew.y = colorField[1]; + } else { + cellNew.x = remainedFields[0].fieldName; + cellNew.y = remainedFields[1].fieldName; + } + } + } + return { + cell: cellNew + }; +}; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts index 47a16d96..8c162331 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts @@ -39,7 +39,7 @@ export class GPTChartGenerationPrompt extends Prompt chartKnowledgeDict[a].index - chartKnowledgeDict[b].index); @@ -75,6 +75,7 @@ export class GPTChartGenerationPrompt extends Prompt { - if ( - chartTypeList.includes(ChartType.WordCloud) || - chartTypeList.includes(ChartType.PieChart) || - chartTypeList.includes(ChartType.RoseChart) - ) { - const validChartList = [ChartType.WordCloud, ChartType.PieChart, ChartType.RoseChart]; - const includedCharts = chartTypeList.filter(chart => validChartList.includes(chart)); + const validSet = new Set(chartTypeList); + if (needColorFieldChartList.some(chartType => validSet.has(chartType))) { + const includedCharts = chartTypeList.filter(chart => needColorFieldChartList.includes(chart)); return ", can't be empty in " + includedCharts.join(', ') + '.'; } return '.'; }; const getSizeKnowledge = (chartTypeList: ChartType[]) => { - if (chartTypeList.includes(ChartType.ScatterPlot) || chartTypeList.includes(ChartType.WordCloud)) { - const validChartList = [ChartType.ScatterPlot, ChartType.WordCloud]; - const includedCharts = chartTypeList.filter(chart => validChartList.includes(chart)); - return ' Only used in ' + includedCharts.join(' and ') + '.'; + const validSet = new Set(chartTypeList); + if (needSizeFieldChartList.some(chartType => validSet.has(chartType))) { + const includedCharts = chartTypeList.filter(chart => needSizeFieldChartList.includes(chart)); + return ' Only used in ' + includedCharts.join(' , ') + '.'; } return '.'; }; @@ -32,7 +48,7 @@ export const visualChannelInfoMap = { color: (chartTypeList: ChartType[]) => `the field mapped to the color channel. Must use a string field${getColorKnowledge(chartTypeList)}`, size: (chartTypeList: ChartType[]) => - `the field mapped to the size channel. Must use a number field.${getSizeKnowledge(chartTypeList)}`, + `the field mapped to the size channel. Must use a number field${getSizeKnowledge(chartTypeList)}`, angle: (chartTypeList: ChartType[]) => "the field mapped to the angle channel of the pie chart. Can't be empty in Pie Chart.", radius: (chartTypeList: ChartType[]) => @@ -44,7 +60,9 @@ export const visualChannelInfoMap = { target: (chartTypeList: ChartType[]) => "the field mapped to the target channel. Only used in Sankey Chart. Can't be empty in Sankey Chart.", value: (chartTypeList: ChartType[]) => - "the field mapped to the value channel. Only used in Sankey Chart. Can't be empty in Sankey Chart." + "the field mapped to the value channel. Only used in Sankey Chart. Can't be empty in Sankey Chart.", + group: (chartTypeList: ChartType[]) => + "the field mapped to the value channel. Only used in Venn Chart. Can't be empty in Venn Chart. Digital IDs are often used to distinguish whether data is in the same group." }; export const chartKnowledgeDict: ChartKnowledge = { [ChartType.BarChart]: { @@ -141,6 +159,64 @@ export const chartKnowledgeDict: ChartKnowledge = { knowledge: [ 'Circular progress chart is also used to display progress data, presented in a circular form, with the values on the numerical axis typically ranging from 0 to 1.' ] + }, + [ChartType.BubbleCirclePacking]: { + index: 17, + visualChannels: ['color', 'size'], + examples: [], + knowledge: [] + }, + [ChartType.MapChart]: { + index: 18, + visualChannels: ['color', 'size'], + examples: [], + knowledge: [] + }, + [ChartType.RangeColumnChart]: { + index: 19, + visualChannels: ['y', 'x'], + examples: [], + knowledge: [] + }, + [ChartType.SunburstChart]: { + index: 20, + visualChannels: ['x', 'y'], + examples: [], + knowledge: [ + 'The x field of a sunburst chart can be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + ] + }, + [ChartType.TreemapChart]: { + index: 21, + visualChannels: ['x', 'y'], + examples: [], + knowledge: [ + 'The x field of a treemap chart can be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + ] + }, + [ChartType.Gauge]: { + index: 22, + visualChannels: ['color', 'size'], + examples: [], + knowledge: ['The gauge chart must contain two fields: size and color.'] + }, + // [ChartType.LinearProgressChart]: { + // index: 23, + // visualChannels: ['y', 'x'], + // examples: [], + // knowledge: [] + // }, + [ChartType.BasicHeatMap]: { + index: 24, + visualChannels: ['y', 'x', 'size'], + examples: [], + knowledge: [] + }, + [ChartType.VennChart]: { + index: 25, + visualChannels: ['size', 'color', 'group'], + examples: [], + knowledge: ['The venn chart must contain three fields: group, size and color.'] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts index a713eef4..bf51a4af 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts @@ -2,6 +2,7 @@ export const ChartAdvisorPromptEnglish = ( showThoughts: boolean, supportedChartList: string[], + supportedUncommonChartList: string[], knowledge: string, visualChannels: string, constraints: string, @@ -10,9 +11,9 @@ export const ChartAdvisorPromptEnglish = ( User want to create an visualization chart for data video using data from a csv file. Ignore the duration in User Input. Your task is: 1. Based on the user's input, infer the user's intention, such as comparison, ranking, trend display, proportion, distribution, etc. If user did not show their intention, just ignore and do the next steps. -2. Select the single chart type that best suites the data from the list of supported charts. Supported chart types: ${JSON.stringify( +2. Select the single chart type that best suites the data from the list of supported charts. If the user's input does not specify a chart type, do not use uncommon chart types. Supported chart types: ${JSON.stringify( supportedChartList -)}. +)}. Uncommon chart types: ${JSON.stringify(supportedUncommonChartList)}. 3. Map all the fields in the data to the visual channels according to user input and the chart type you choose. Don't use non-existent fields. Only use existing fields without further processing. If the existing fields can't meet user's intention, just use the most related fields. ${knowledge.length > 0 ? '\nKnowledge' : ''} ${knowledge} diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts index 8b6b23cf..f03b9876 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts @@ -51,7 +51,33 @@ import { circularProgressStyle, linearProgressStyle, linearProgressAxes, - indicator + indicator, + bubbleCirclePackingField, + bubbleCirclePackingDisplayConf, + bubbleCirclePackingData, + rangeColumnField, + rangeColumnDisplayConf, + sunburstData, + sunburstDisplayConf, + treemapData, + sunburstOrTreemapField, + treemapDisplayConf, + gaugeField, + gaugeDisplayConf, + // linearProgressField, + // linearProgressDisplayConf, + arrayData, + vennData, + vennField, + basicHeatMapSeries, + basicHeatMapRegion, + basicHeatMapColor, + basicHeatMapAxes, + basicHeatMapLegend, + basemap, + mapField, + mapDisplayConf, + registerChart } from './transformers'; const pipelineBar = [ @@ -153,24 +179,52 @@ const pipelineBoxPlot = [chartType, data, color, boxPlotField, boxPlotStyle, leg const pipelineLiquid = [chartType, data, color, liquidField, liquidStyle, indicator, theme]; const pipelineLinearProgress = [ + chartType, + data, + color, + linearProgressField, + linearProgressAxes, + linearProgressStyle, + theme +]; + +const pipelineCircularProgress = [ + chartType, + data, + color, + circularProgressField, + circularProgressStyle, + indicator, + theme +]; + +const pipelineBubbleCirclePacking = [ chartType, + bubbleCirclePackingData, data, color, - linearProgressField, - linearProgressAxes, - linearProgressStyle, + bubbleCirclePackingField, + bubbleCirclePackingDisplayConf, theme ]; -const pipelineCircularProgress = [ +const pipelineMapChart = [chartType, basemap, arrayData, mapField, mapDisplayConf, theme]; +const pipelineRangeColumn = [chartType, data, rangeColumnField, rangeColumnDisplayConf, theme]; +const pipelineSunburst = [chartType, sunburstData, sunburstOrTreemapField, sunburstDisplayConf, theme]; +const pipelineTreemap = [chartType, treemapData, sunburstOrTreemapField, treemapDisplayConf, theme]; +const pipelineGauge = [chartType, arrayData, gaugeField, gaugeDisplayConf, theme]; +// const pipelineLinearProgress = [chartType, arrayData, linearProgressField, linearProgressDisplayConf, theme]; +const pipelineBasicHeatMap = [ chartType, - data, - color, - circularProgressField, - circularProgressStyle, - indicator, + arrayData, + basicHeatMapSeries, + basicHeatMapRegion, + basicHeatMapColor, + basicHeatMapAxes, + basicHeatMapLegend, theme ]; +const pipelineVenn = [chartType, registerChart, vennData, vennField, legend, theme]; const pipelineMap: { [chartType: string]: any } = { 'BAR CHART': pipelineBar, @@ -188,7 +242,17 @@ const pipelineMap: { [chartType: string]: any } = { 'BOX PLOT': pipelineBoxPlot, [ChartType.LiquidChart.toUpperCase()]: pipelineLiquid, [ChartType.LinearProgress.toUpperCase()]: pipelineLinearProgress, - [ChartType.CircularProgress.toUpperCase()]: pipelineCircularProgress + [ChartType.CircularProgress.toUpperCase()]: pipelineCircularProgress, + 'BOX PLOT': pipelineBoxPlot, + 'BUBBLE CIRCLE PACKING': pipelineBubbleCirclePacking, + 'MAP CHART': pipelineMapChart, + 'RANGE COLUMN CHART': pipelineRangeColumn, + 'SUNBURST CHART': pipelineSunburst, + 'TREEMAP CHART': pipelineTreemap, + 'GAUGE CHART': pipelineGauge, + // 'LINEAR PROGRESS CHART': pipelineLinearProgress, + 'BASIC HEAT MAP': pipelineBasicHeatMap, + 'VENN CHART': pipelineVenn }; export const beforePipe: Transformer = ( diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/constants.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/constants.ts index 9642dfd2..7653ac80 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/constants.ts @@ -12,6 +12,8 @@ export const LINEAR_COLOR_THEMES = [ ['#1DD0F3', '#CB2BC6'] ]; +export const BASIC_HEAT_MAP_COLOR_THEMES = ['#feedde', '#fdbe85', '#fd8d3c', '#e6550d', '#a63603']; + export const animationDuration = 500; export const oneByOneGroupSize = 10; export const DEFAULT_VIDEO_LENGTH = 2000; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 3f665797..811c0673 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -1,4 +1,6 @@ import type { Transformer } from '../../../../../base/tools/transformer'; +import { registerVennChart } from '@visactor/vchart'; +import VChart from '@visactor/vchart'; import type { GetChartSpecContext, GetChartSpecOutput } from '../types'; import { COLOR_THEMES, @@ -13,13 +15,15 @@ import { SUB_SERIES_ID, WORDCLOUD_NUM_LIMIT, animationDuration, - oneByOneGroupSize + oneByOneGroupSize, + BASIC_HEAT_MAP_COLOR_THEMES } from './constants'; import { getFieldByDataType } from '../../../../../common/utils/utils'; import { array, isArray } from '@visactor/vutils'; import { isValidDataset } from '../../../../../common/dataProcess'; import { DataType, ChartType } from '../../../../../common/typings'; import { builtinThemeMap } from '../../../../../common/builtinTheme'; +import type { VMindDataset } from '../../../../../common/typings'; import { COLOR_FIELD } from '@visactor/chart-advisor'; type Context = GetChartSpecContext & GetChartSpecOutput; @@ -41,6 +45,15 @@ const chartTypeMap: { [chartName: string]: string } = { [ChartType.LiquidChart.toUpperCase()]: 'liquid', [ChartType.LinearProgress.toUpperCase()]: 'linearProgress', [ChartType.CircularProgress.toUpperCase()]: 'circularProgress' + 'BUBBLE CIRCLE PACKING': 'circlePacking', + 'MAP CHART': 'map', + 'RANGE COLUMN CHART': 'rangeColumn', + 'SUNBURST CHART': 'sunburst', + 'TREEMAP CHART': 'treemap', + 'GAUGE CHART': 'gauge', + // 'LINEAR PROGRESS CHART': 'linearProgress', + 'BASIC HEAT MAP': 'common', + 'VENN CHART': 'venn' }; export const chartType: Transformer = (context: Context) => { @@ -60,6 +73,17 @@ export const data: Transformer = (context: Context) return { spec }; }; +export const arrayData: Transformer = (context: Context) => { + const { dataset, spec } = context; + spec.data = [ + { + id: 'data', + values: isValidDataset(dataset) ? dataset.flat(4) : [] + } + ]; + return { spec }; +}; + export const funnelData: Transformer = (context: Context) => { const { dataset, cell, spec } = context; // spec.data = [dataset] @@ -1385,3 +1409,443 @@ export const indicator: Transformer = (context: Con }; return { spec }; }; + +export const bubbleCirclePackingData: Transformer = (context: Context) => { + const { dataset, spec, cell } = context; + if (cell.size) { + dataset.forEach(data => { + data.value = data[cell.size]; + delete data[cell.size]; + }); + } + return { spec }; +}; + +export const bubbleCirclePackingField: Transformer = (context: Context) => { + //assign field in spec according to cell + const { cell, spec } = context; + spec.categoryField = cell.color || cell.x; + + if (cell.size) { + spec.valueField = cell.size; + } + + return { spec }; +}; + +export const bubbleCirclePackingDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.drill = true; + spec.layoutPadding = 5; + spec.animationEnter = { + easing: 'cubicInOut' + }; + spec.animationExit = { + easing: 'cubicInOut' + }; + spec.animationUpdate = { + easing: 'cubicInOut' + }; + return { spec }; +}; + +export const rangeColumnField: Transformer = (context: Context) => { + //assign field in spec according to cell + const { cell, spec } = context; + spec.yField = cell.x; + + spec.xField = [cell.y[0], cell.y[1]]; + + return { spec }; +}; + +export const rangeColumnDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.direction = 'horizontal'; + spec.label = { + visible: true + }; + return { spec }; +}; + +export const sunburstData: Transformer = (context: Context) => { + const { dataset, cell, spec } = context; + spec.data = { id: 'data', values: getSunburstData(dataset, cell.x, 0, cell.y) }; + return { spec }; +}; + +export const getSunburstData: any = ( + dataset: VMindDataset, + xField: string[] | string, + index: number, + yField: string +) => { + if (xField.length - 1 === index) { + return Array.from( + new Set( + dataset.map(data => { + return { name: data[xField[index]], value: data[yField] }; + }) + ) + ); + } + // Get the value range of this layer + const values = Array.from( + new Set( + dataset.map(data => { + return data[xField[index]]; + }) + ) + ); + return values.map(value => { + const currentDataset = dataset.filter(data => { + return data[xField[index]] === value; + }); + return { name: value, children: getSunburstData(currentDataset, xField, index + 1, yField) }; + }); +}; + +export const sunburstOrTreemapField: Transformer = (context: Context) => { + //assign field in spec according to cell + const { spec } = context; + spec.categoryField = 'name'; + + spec.valueField = 'value'; + + return { spec }; +}; + +export const sunburstDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.offsetX = 0; + spec.offsetY = 0; + spec.outerRadius = 1; + spec.innerRadius = 0; + spec.gap = 5; + spec.drill = true; + spec.sunburst = { + visible: true, + style: { + fillOpacity: (datum: { isLeaf: any }) => { + return datum.isLeaf ? 0.4 : 0.8; + } + } + }; + spec.label = { + visible: true, + style: { + fontSize: 12, + fillOpacity: (datum: { isLeaf: any }) => { + return datum.isLeaf ? 0.4 : 0.8; + } + } + }; + spec.tooltip = { + mark: { + title: { + value: (val: { datum: any[] }) => { + return val?.datum?.map(data => data.name).join(' / '); + } + } + } + }; + spec.animationEnter = { + easing: 'cubicInOut', + duration: 1000 + }; + spec.animationExit = { + easing: 'cubicInOut', + duration: 1000 + }; + spec.animationUpdate = { + easing: 'cubicInOut', + duration: 1000 + }; + return { spec }; +}; + +export const treemapData: Transformer = (context: Context) => { + const { dataset, cell, spec } = context; + spec.data = { id: 'data', values: getTreemapData(dataset, cell.x, 0, cell.y) }; + return { spec }; +}; + +export const getTreemapData: any = ( + dataset: VMindDataset, + xField: string[] | string, + index: number, + yField: string +) => { + if (xField.length - 1 === index) { + return Array.from( + new Set( + dataset.map(data => { + return { name: data[xField[index]], value: data[yField] }; + }) + ) + ); + } + // Get the value range of this layer + const values = Array.from( + new Set( + dataset.map(data => { + return data[xField[index]]; + }) + ) + ); + return values.map(value => { + const currentDataset = dataset.filter(data => { + return data[xField[index]] === value; + }); + if (currentDataset[0] && currentDataset[0][xField[index + 1]] === '') { + return { name: value, value: currentDataset[0][yField] }; + } + return { name: value, children: getTreemapData(currentDataset, xField, index + 1, yField) }; + }); +}; + +export const treemapDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.label = { + visible: true, + style: { + fontSize: 12 + } + }; + return { spec }; +}; + +export const gaugeField: Transformer = (context: Context) => { + const { spec, cell } = context; + spec.valueField = cell.size; + spec.categoryField = cell.color; + return { spec }; +}; + +export const gaugeDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.outerRadius = 0.8; + spec.innerRadius = 0.5; + spec.startAngle = -180; + spec.endAngle = 0; + return { spec }; +}; + +export const linearProgressField: Transformer = (context: Context) => { + const { spec, cell } = context; + spec.yField = cell.x; + spec.xField = cell.y; + + spec.seriesField = cell.x; + return { spec }; +}; + +export const linearProgressDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.direction = 'horizontal'; + + spec.cornerRadius = 20; + spec.bandWidth = 30; + spec.axes = [ + { + orient: 'left', + label: { visible: true }, + type: 'band', + domainLine: { visible: false }, + tick: { visible: false } + }, + { orient: 'bottom', label: { visible: true }, type: 'linear', visible: false } + ]; + return { spec }; +}; + +export const vennData: Transformer = (context: Context) => { + const { dataset, spec, cell } = context; + const id2dataMap = {}; + dataset.forEach(data => { + if (id2dataMap[data[cell.group]]) { + id2dataMap[data[cell.group]].sets.push(data[cell.color]); + } else { + id2dataMap[data[cell.group]] = { sets: [data[cell.color]], value: data[cell.size] }; + } + }); + spec.data = { + values: Object.values(id2dataMap) + }; + + return { spec }; +}; + +export const vennField: Transformer = (context: Context) => { + const { spec } = context; + spec.valueField = 'value'; + spec.categoryField = 'sets'; + spec.seriesField = 'sets'; + return { spec }; +}; +export const registerChart: Transformer = (context: Context) => { + const { spec } = context; + if (spec.type === 'venn') { + registerVennChart(); + } + return { spec }; +}; + +export const basicHeatMapSeries: Transformer = (context: Context) => { + const { spec, cell } = context; + spec.series = [ + { + type: 'heatmap', + regionId: 'region0', + xField: cell.x, + yField: cell.y, + valueField: cell.size, + cell: { + style: { + fill: { + field: cell.size, + scale: 'color' + } + } + } + } + ]; + return { spec }; +}; +export const basicHeatMapRegion: Transformer = (context: Context) => { + const { spec } = context; + spec.region = [ + { + id: 'region0', + width: 200, // limit the width of the region + height: 200, // limit the height of the region + padding: { + top: 40 + } + } + ]; + return { spec }; +}; +export const basicHeatMapColor: Transformer = (context: Context) => { + const { spec, cell } = context; + spec.color = { + type: 'linear', + domain: [ + { + dataId: 'data', + fields: [cell.size] + } + ], + range: BASIC_HEAT_MAP_COLOR_THEMES + }; + return { spec }; +}; +export const basicHeatMapAxes: Transformer = (context: Context) => { + const { spec } = context; + spec.axes = [ + { + orient: 'bottom', + type: 'band', + grid: { + visible: false + }, + domainLine: { + visible: false + }, + label: { + space: 10, + style: { + textAlign: 'left', + textBaseline: 'middle', + angle: 90, + fontSize: 8 + } + }, + bandPadding: 0, + height: (layoutRect: any) => { + // canvas height - region height - paddingTop - paddingBottom + return layoutRect.height - 314; + } + }, + { + orient: 'left', + type: 'band', + grid: { + visible: false + }, + domainLine: { + visible: false + }, + label: { + space: 10, + style: { + fontSize: 8 + } + }, + bandPadding: 0 + } + ]; + return { spec }; +}; + +export const basicHeatMapLegend: Transformer = (context: Context) => { + const { spec } = context; + spec.legends = { + visible: true, + orient: 'right', + position: 'start', + type: 'color', + field: 'value' + }; + return { spec }; +}; + +export const basemap: Transformer> = async (context: Context) => { + const { basemapOption, spec } = context; + const jsonUrl = basemapOption.jsonUrl; + const response = await fetch(jsonUrl); + const geoJson = await response.json(); + VChart.registerMap('map', geoJson); + if (basemapOption.regionProjectType) { + spec.region = [ + { + roam: true, + projection: { type: basemapOption.regionProjectType }, + coordinate: basemapOption.regionCoordinate + } + ]; + } else { + spec.region = [ + { + roam: true, + coordinate: basemapOption.regionCoordinate + } + ]; + } + + spec.map = 'map'; + return { spec }; +}; + +export const mapField: Transformer = (context: Context) => { + const { spec, cell } = context; + + spec.nameField = cell.color; + spec.valueField = cell.size; + spec.nameProperty = cell.color; + return { spec }; +}; + +export const mapDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.legends = [ + { + visible: true, + type: 'color', + field: 'value', + orient: 'bottom', + position: 'start' + } + ]; + return { spec }; +}; diff --git a/packages/vmind/src/applications/chartGeneration/types.ts b/packages/vmind/src/applications/chartGeneration/types.ts index ede8319c..f3656d90 100644 --- a/packages/vmind/src/applications/chartGeneration/types.ts +++ b/packages/vmind/src/applications/chartGeneration/types.ts @@ -1,6 +1,6 @@ export type Cell = { //字段映射，可用的视觉通道：["x","y","color","size","angle","time"] - x?: string; + x?: string | string[]; y?: string | string[]; color?: string; size?: string; @@ -11,4 +11,5 @@ export type Cell = { target?: string; value?: string; category?: string; + group?: string; }; diff --git a/packages/vmind/src/applications/types.ts b/packages/vmind/src/applications/types.ts index a71caba2..3fb577ce 100644 --- a/packages/vmind/src/applications/types.ts +++ b/packages/vmind/src/applications/types.ts @@ -1,6 +1,14 @@ -import type { ChartType, ILLMOptions, SimpleFieldInfo, VMindDataset, ChartTheme } from '../common/typings'; +import type { + BasemapOption, + ChartType, + ILLMOptions, + SimpleFieldInfo, + VMindDataset, + ChartTheme +} from '../common/typings'; import type { Cell } from './chartGeneration/types'; import type { InsightAlgorithm, VMindInsight } from './IngelligentInsight/types'; +import type { UncommonChartType } from '../common/typings'; //context of the DataExtraction Application export type DataExtractionContext = { @@ -38,6 +46,8 @@ export type ChartGenerationContext = { fieldInfo: SimpleFieldInfo[]; dataset?: VMindDataset; chartTypeList: ChartType[]; //supported chart list + uncommonChartTypeList: UncommonChartType[]; //supported uncommon chart list + basemapOption: BasemapOption; // only use in map chart } & { totalTime?: number; colors?: string[]; diff --git a/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts b/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts index 92168c9e..3bb46a87 100644 --- a/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts +++ b/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts @@ -32,7 +32,7 @@ export class RuleBasedTaskNode extends BaseTaskNode { this.updateContext({ ...this.context, ...context }); let pipelines = this.pipelines; if (isFunction(this.pipelines)) { @@ -40,13 +40,10 @@ export class RuleBasedTaskNode extends BaseTaskNode[]).reduce( - (pre: any, transformer: Transformer) => { - const res = transformer(pre); - return { ...pre, ...res }; - }, - context - ); + let result: any = context; + for (const transformer of pipelines as Transformer[]) { + result = { ...result, ...(await transformer(result)) }; + } return result; } catch (e: any) { console.error(`${this.name} error!`); diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index 73bc22a0..f6bd1027 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -75,7 +75,28 @@ export enum ChartType { BoxPlot = 'Box Plot', LinearProgress = 'Linear Progress chart', CircularProgress = 'Circular Progress chart', - LiquidChart = 'Liquid Chart' + LiquidChart = 'Liquid Chart', + BubbleCirclePacking = 'Bubble Circle Packing', + MapChart = 'Map Chart', + RangeColumnChart = 'Range Column Chart', + SunburstChart = 'Sunburst Chart', + TreemapChart = 'Treemap Chart', + Gauge = 'Gauge Chart', + // LinearProgressChart = 'Linear Progress Chart', + BasicHeatMap = 'Basic Heat Map', + VennChart = 'Venn Chart' +} + +export enum UncommonChartType { + BubbleCirclePacking = 'Bubble Circle Packing', + MapChart = 'Map Chart', + RangeColumnChart = 'Range Column Chart', + SunburstChart = 'Sunburst Chart', + TreemapChart = 'Treemap Chart', + Gauge = 'Gauge Chart', + LinearProgressChart = 'Linear Progress Chart', + BasicHeatMap = 'Basic Heat Map', + VennChart = 'Venn Chart' } export type GPTChartAdvisorResult = { @@ -181,3 +202,35 @@ export type VMindTheme = { }; export type { ITheme as ChartTheme } from '@visactor/vchart'; + +export enum mapRegionProjectionType { + ALBERS = 'albers', + ALBERS_USA = 'albersUsa', + AZIMUTHAL_EQUAL_AREA = 'azimuthalEqualArea', + AZIMUTHAL_EQUIDISTANT = 'azimuthalEquidistant', + CONIC_CONFORMAL = 'conicConformal', + CONIC_EQUAL_AREA = 'conicEqualArea', + CONIC_EQUIDISTANT = 'conicEquidistant', + EQUAL_EARTH = 'equalEarth', + EQUIRECTANGULAR = 'equirectangular', + GNOMONIC = 'gnomonic', + MERCATOR = 'mercator', + NATURAL_EARTH1 = 'naturalEarth1', + ORTHOGRAPHIC = 'orthographic', + STEREOGRAPHIC = 'stereographic', + TRANSVERSE_MERCATOR = 'transverseMercator' +} + +export enum MapRegionCoordinate { + CARTESIAN = 'cartesian', + POLAR = 'polar', + GEO = 'geo' +} + +export type BasemapOption = { + jsonUrl: string; + regionProjectType: mapRegionProjectionType; + regionCoordinate: MapRegionCoordinate; + zoom: number; + center: number[]; +}; diff --git a/packages/vmind/src/common/utils/utils.ts b/packages/vmind/src/common/utils/utils.ts index a92981a3..b1411b29 100644 --- a/packages/vmind/src/common/utils/utils.ts +++ b/packages/vmind/src/common/utils/utils.ts @@ -78,6 +78,9 @@ export const getFieldByRole = (fields: SimpleFieldInfo[], role: ROLE) => { export const getFieldByDataType = (fields: SimpleFieldInfo[], dataTypeList: DataType[]) => { return fields.find(f => dataTypeList.includes(f.type)); }; +export const getFieldsByDataType = (fields: SimpleFieldInfo[], dataTypeList: DataType[]) => { + return fields.filter(f => dataTypeList.includes(f.type)); +}; export const foldDatasetByYField = ( dataset: DataItem[], diff --git a/packages/vmind/src/core/VMind.ts b/packages/vmind/src/core/VMind.ts index 89163647..303cd865 100644 --- a/packages/vmind/src/core/VMind.ts +++ b/packages/vmind/src/core/VMind.ts @@ -7,7 +7,9 @@ import type { OuterPackages, VMindDataset, ChartType, - ChartTheme + ChartTheme, + UncommonChartType, + BasemapOption } from '../common/typings'; import { Model, ModelType } from '../common/typings'; import { getFieldInfoFromDataset, parseCSVData as parseCSVDataWithRule } from '../common/dataProcess'; @@ -23,7 +25,11 @@ import type { import applicationMetaList, { ApplicationType } from '../applications'; import { calculateTokenUsage } from '../common/utils/utils'; import { isNil } from '@visactor/vutils'; -import { SUPPORTED_CHART_LIST } from '../applications/chartGeneration/constants'; +import { + DEFAULT_MAP_OPTION, + SUPPORTED_CHART_LIST, + SUPPORTED_UNCOMMON_CHART_LIST +} from '../applications/chartGeneration/constants'; import { BaseApplication } from '../base/application'; import { fillSpecTemplateWithData } from '../common/specUtils'; import type { ApplicationMeta, TaskNode } from '../base/metaTypes'; @@ -170,6 +176,8 @@ class VMind { * @param colorPalette color palette of the chart * @param animationDuration duration of chart animation. * @param chartTypeList supported chart list. VMind will generate a chart among this list. + * @param uncommonChartTypeList uncommon chart types. If the user does not specify a type, uncommon types are not considered. + * @param basemapOption map chart's base map. Only use in map chart. * @returns spec and time duration of the chart. */ async generateChart( @@ -182,6 +190,8 @@ class VMind { animationDuration?: number; enableDataQuery?: boolean; theme?: ChartTheme | string; + uncommonChartTypeList?: UncommonChartType[]; + basemapOption?: BasemapOption; } ): Promise { const modelType = this.getModelType(); @@ -189,7 +199,15 @@ class VMind { let finalFieldInfo = fieldInfo; let queryDatasetUsage; - const { enableDataQuery, colorPalette, animationDuration, theme, chartTypeList } = options ?? {}; + const { + enableDataQuery, + colorPalette, + animationDuration, + theme, + chartTypeList, + uncommonChartTypeList, + basemapOption + } = options ?? {}; try { if (!isNil(dataset) && (isNil(enableDataQuery) || enableDataQuery) && modelType !== ModelType.CHART_ADVISOR) { //run data aggregation first @@ -223,7 +241,9 @@ class VMind { colors: colorPalette, totalTime: animationDuration, chartTheme: theme, - chartTypeList: chartTypeList ?? SUPPORTED_CHART_LIST + chartTypeList: chartTypeList ?? SUPPORTED_CHART_LIST, + uncommonChartTypeList: uncommonChartTypeList ?? SUPPORTED_UNCOMMON_CHART_LIST, + basemapOption: basemapOption ?? DEFAULT_MAP_OPTION }; const chartGenerationResult = await this.runApplication(ApplicationType.ChartGeneration, modelType, context); From 87c8f2b8629d4b6db929330ae44066a9fdb42d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Mon, 22 Jul 2024 15:57:08 +0800 Subject: [PATCH 007/128] Remove uncommon chart types --- .../applications/chartGeneration/constants.ts | 3 +-- .../GPT/prompt/index.ts | 3 +-- .../GPT/prompt/template.ts | 5 ++--- packages/vmind/src/applications/types.ts | 2 -- packages/vmind/src/common/typings/index.ts | 12 ----------- packages/vmind/src/core/VMind.ts | 20 ++----------------- 6 files changed, 6 insertions(+), 39 deletions(-) diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index 07b0ec82..15376fb0 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -1,8 +1,7 @@ import type { BasemapOption } from '../../common/typings'; -import { ChartType, MapRegionCoordinate, UncommonChartType } from '../../common/typings'; +import { ChartType, MapRegionCoordinate } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); -export const SUPPORTED_UNCOMMON_CHART_LIST = Object.values(UncommonChartType); export const DEFAULT_MAP_OPTION: BasemapOption = { jsonUrl: 'https://lf9-dp-fe-cms-tos.byteorg.com/obj/bit-cloud/geojson/world.json', diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts index 8c162331..47a16d96 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts @@ -39,7 +39,7 @@ export class GPTChartGenerationPrompt extends Prompt chartKnowledgeDict[a].index - chartKnowledgeDict[b].index); @@ -75,7 +75,6 @@ export class GPTChartGenerationPrompt extends Prompt 0 ? '\nKnowledge' : ''} ${knowledge} diff --git a/packages/vmind/src/applications/types.ts b/packages/vmind/src/applications/types.ts index 3fb577ce..ec7fe39c 100644 --- a/packages/vmind/src/applications/types.ts +++ b/packages/vmind/src/applications/types.ts @@ -8,7 +8,6 @@ import type { } from '../common/typings'; import type { Cell } from './chartGeneration/types'; import type { InsightAlgorithm, VMindInsight } from './IngelligentInsight/types'; -import type { UncommonChartType } from '../common/typings'; //context of the DataExtraction Application export type DataExtractionContext = { @@ -46,7 +45,6 @@ export type ChartGenerationContext = { fieldInfo: SimpleFieldInfo[]; dataset?: VMindDataset; chartTypeList: ChartType[]; //supported chart list - uncommonChartTypeList: UncommonChartType[]; //supported uncommon chart list basemapOption: BasemapOption; // only use in map chart } & { totalTime?: number; diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index f6bd1027..22dedde0 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -87,18 +87,6 @@ export enum ChartType { VennChart = 'Venn Chart' } -export enum UncommonChartType { - BubbleCirclePacking = 'Bubble Circle Packing', - MapChart = 'Map Chart', - RangeColumnChart = 'Range Column Chart', - SunburstChart = 'Sunburst Chart', - TreemapChart = 'Treemap Chart', - Gauge = 'Gauge Chart', - LinearProgressChart = 'Linear Progress Chart', - BasicHeatMap = 'Basic Heat Map', - VennChart = 'Venn Chart' -} - export type GPTChartAdvisorResult = { CHART_TYPE: ChartType; DOUBLE_CHECK: string; diff --git a/packages/vmind/src/core/VMind.ts b/packages/vmind/src/core/VMind.ts index 303cd865..5059145c 100644 --- a/packages/vmind/src/core/VMind.ts +++ b/packages/vmind/src/core/VMind.ts @@ -8,7 +8,6 @@ import type { VMindDataset, ChartType, ChartTheme, - UncommonChartType, BasemapOption } from '../common/typings'; import { Model, ModelType } from '../common/typings'; @@ -25,11 +24,7 @@ import type { import applicationMetaList, { ApplicationType } from '../applications'; import { calculateTokenUsage } from '../common/utils/utils'; import { isNil } from '@visactor/vutils'; -import { - DEFAULT_MAP_OPTION, - SUPPORTED_CHART_LIST, - SUPPORTED_UNCOMMON_CHART_LIST -} from '../applications/chartGeneration/constants'; +import { DEFAULT_MAP_OPTION, SUPPORTED_CHART_LIST } from '../applications/chartGeneration/constants'; import { BaseApplication } from '../base/application'; import { fillSpecTemplateWithData } from '../common/specUtils'; import type { ApplicationMeta, TaskNode } from '../base/metaTypes'; @@ -176,7 +171,6 @@ class VMind { * @param colorPalette color palette of the chart * @param animationDuration duration of chart animation. * @param chartTypeList supported chart list. VMind will generate a chart among this list. - * @param uncommonChartTypeList uncommon chart types. If the user does not specify a type, uncommon types are not considered. * @param basemapOption map chart's base map. Only use in map chart. * @returns spec and time duration of the chart. */ @@ -190,7 +184,6 @@ class VMind { animationDuration?: number; enableDataQuery?: boolean; theme?: ChartTheme | string; - uncommonChartTypeList?: UncommonChartType[]; basemapOption?: BasemapOption; } ): Promise { @@ -199,15 +192,7 @@ class VMind { let finalFieldInfo = fieldInfo; let queryDatasetUsage; - const { - enableDataQuery, - colorPalette, - animationDuration, - theme, - chartTypeList, - uncommonChartTypeList, - basemapOption - } = options ?? {}; + const { enableDataQuery, colorPalette, animationDuration, theme, chartTypeList, basemapOption } = options ?? {}; try { if (!isNil(dataset) && (isNil(enableDataQuery) || enableDataQuery) && modelType !== ModelType.CHART_ADVISOR) { //run data aggregation first @@ -242,7 +227,6 @@ class VMind { totalTime: animationDuration, chartTheme: theme, chartTypeList: chartTypeList ?? SUPPORTED_CHART_LIST, - uncommonChartTypeList: uncommonChartTypeList ?? SUPPORTED_UNCOMMON_CHART_LIST, basemapOption: basemapOption ?? DEFAULT_MAP_OPTION }; From e8f45c856aeeb6710c6040e387d8c8c6241a471e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Mon, 22 Jul 2024 17:04:48 +0800 Subject: [PATCH 008/128] Migrate needSizeFieldChatTypeList, it has nothing to do with prompt. --- .../applications/chartGeneration/constants.ts | 20 +++++++++++++ .../GPT/patcher/index.ts | 8 +++-- .../GPT/prompt/knowledges.ts | 29 ++++--------------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index 15376fb0..1a1c4041 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -3,6 +3,26 @@ import { ChartType, MapRegionCoordinate } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); +export const NEED_COLOR_FIELD_CHART_LIST = [ + ChartType.WordCloud, + ChartType.PieChart, + ChartType.RoseChart, + ChartType.MapChart, + ChartType.BubbleCirclePacking, + ChartType.VennChart, + ChartType.Gauge +]; + +export const NEED_SIZE_FIELD_CHART_LIST = [ + ChartType.ScatterPlot, + ChartType.WordCloud, + ChartType.MapChart, + ChartType.BubbleCirclePacking, + ChartType.VennChart, + ChartType.Gauge, + ChartType.BasicHeatMap +]; + export const DEFAULT_MAP_OPTION: BasemapOption = { jsonUrl: 'https://lf9-dp-fe-cms-tos.byteorg.com/obj/bit-cloud/geojson/world.json', regionProjectType: null, diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts index 1b6504a4..68f543c6 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts @@ -12,7 +12,7 @@ import { import { ChartType, DataType, ROLE } from '../../../../../../common/typings'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../../types'; import { isValidDataset } from '../../../../../../common/dataProcess'; -import { needSizeFieldChartList, needColorFieldChartList } from '../prompt/knowledges'; +import { NEED_SIZE_FIELD_CHART_LIST, NEED_COLOR_FIELD_CHART_LIST } from '../../../../constants'; const CARTESIAN_CHART_LIST = [ 'Dynamic Bar Chart', @@ -378,7 +378,9 @@ export const patchNeedColor: Transformer< > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { const { chartType, cell, fieldInfo } = context; const cellNew: any = { ...cell }; - if (needColorFieldChartList.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType)) { + if ( + NEED_COLOR_FIELD_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) + ) { const colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean).flat(); if (colorField.length !== 0) { cellNew.color = colorField[0]; @@ -403,7 +405,7 @@ export const patchNeedSize: Transformer< > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { const { chartType, cell, fieldInfo } = context; const cellNew: any = { ...cell }; - if (needSizeFieldChartList.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType)) { + if (NEED_SIZE_FIELD_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType)) { const sizeField = [cellNew.size, cellNew.value, cellNew.y, cellNew.radius, cellNew.angle].filter(Boolean).flat(); if (sizeField.length !== 0) { cellNew.size = sizeField[0]; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts index 76db8a3e..e9746388 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts @@ -2,31 +2,12 @@ import { ChartType } from '../../../../../../common/typings'; import { barChartExample1, dynamicBarChart1, lineChartExample1, lineChartExample2, pieChartExample1 } from './examples'; import type { ChartKnowledge } from './types'; - -export const needColorFieldChartList = [ - ChartType.WordCloud, - ChartType.PieChart, - ChartType.RoseChart, - ChartType.MapChart, - ChartType.BubbleCirclePacking, - ChartType.VennChart, - ChartType.Gauge -]; - -export const needSizeFieldChartList = [ - ChartType.ScatterPlot, - ChartType.WordCloud, - ChartType.MapChart, - ChartType.BubbleCirclePacking, - ChartType.VennChart, - ChartType.Gauge, - ChartType.BasicHeatMap -]; +import { NEED_SIZE_FIELD_CHART_LIST, NEED_COLOR_FIELD_CHART_LIST } from '../../../../constants'; const getColorKnowledge = (chartTypeList: ChartType[]) => { const validSet = new Set(chartTypeList); - if (needColorFieldChartList.some(chartType => validSet.has(chartType))) { - const includedCharts = chartTypeList.filter(chart => needColorFieldChartList.includes(chart)); + if (NEED_COLOR_FIELD_CHART_LIST.some(chartType => validSet.has(chartType))) { + const includedCharts = chartTypeList.filter(chart => NEED_COLOR_FIELD_CHART_LIST.includes(chart)); return ", can't be empty in " + includedCharts.join(', ') + '.'; } return '.'; @@ -34,8 +15,8 @@ const getColorKnowledge = (chartTypeList: ChartType[]) => { const getSizeKnowledge = (chartTypeList: ChartType[]) => { const validSet = new Set(chartTypeList); - if (needSizeFieldChartList.some(chartType => validSet.has(chartType))) { - const includedCharts = chartTypeList.filter(chart => needSizeFieldChartList.includes(chart)); + if (NEED_SIZE_FIELD_CHART_LIST.some(chartType => validSet.has(chartType))) { + const includedCharts = chartTypeList.filter(chart => NEED_SIZE_FIELD_CHART_LIST.includes(chart)); return ' Only used in ' + includedCharts.join(' , ') + '.'; } return '.'; From 3d0df648279e705996f068daf0ac544e237e4061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Tue, 23 Jul 2024 10:30:58 +0800 Subject: [PATCH 009/128] Adjust visual channels for Sunburst, Treemap, and Venn --- .../browser/src/constants/mockData.ts | 2 +- .../applications/chartGeneration/constants.ts | 29 +++++--- .../generateTypeAndFieldMap/GPT/index.ts | 2 - .../GPT/patcher/index.ts | 74 ++++++++----------- .../GPT/prompt/index.ts | 20 +++-- .../GPT/prompt/knowledges.ts | 58 +++++++++++---- .../getChartSpec/VChart/chartPipeline.ts | 15 ++-- .../getChartSpec/VChart/transformers.ts | 48 ++++++------ .../src/applications/chartGeneration/types.ts | 5 +- 9 files changed, 142 insertions(+), 111 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 1f5fe7e5..e60bd7ad 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -4795,7 +4795,7 @@ Category Eight,18,34 }; export const sunburstChartData = { - csv: `Country,Region,Category,Value + csv: `Category0,Category1,Category2,Value Country A,Region1,Office Supplies,824 Country A,Region1,Furniture,920 Country A,Region1,Electronic equipment,936 diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index 1a1c4041..00bb380e 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -3,24 +3,31 @@ import { ChartType, MapRegionCoordinate } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); -export const NEED_COLOR_FIELD_CHART_LIST = [ +export const NEED_COLOR_FIELD_CHART_LIST = [ChartType.PieChart, ChartType.RoseChart]; + +export const NEED_SIZE_FIELD_CHART_LIST = [ChartType.ScatterPlot, ChartType.BasicHeatMap]; + +export const NEED_COLOR_AND_SIZE_CHART_LIST = [ ChartType.WordCloud, - ChartType.PieChart, - ChartType.RoseChart, ChartType.MapChart, ChartType.BubbleCirclePacking, ChartType.VennChart, - ChartType.Gauge + ChartType.Gauge, + ChartType.SunburstChart, + ChartType.TreemapChart ]; -export const NEED_SIZE_FIELD_CHART_LIST = [ +export const CARTESIAN_CHART_LIST = [ + ChartType.DynamicBarChart, + ChartType.BarChart, + ChartType.LineChart, ChartType.ScatterPlot, - ChartType.WordCloud, - ChartType.MapChart, - ChartType.BubbleCirclePacking, - ChartType.VennChart, - ChartType.Gauge, - ChartType.BasicHeatMap + ChartType.FunnelChart, + ChartType.DualAxisChart, + ChartType.WaterFallChart, + ChartType.BoxPlot, + ChartType.RangeColumnChart, + ChartType.LinearProgressChart ]; export const DEFAULT_MAP_OPTION: BasemapOption = { diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts index bd2fbd4d..2ba28beb 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts @@ -18,7 +18,6 @@ import { patchNeedSize, patchPieChart, patchRangeColumnChart, - patchSunburstAndTreemapChart, patchWordCloud, patchYField } from './patcher'; @@ -49,7 +48,6 @@ const ChartGenerationTaskNodeGPTMeta: LLMBasedTaskNodeMeta< patchDynamicBarChart, patchRangeColumnChart, patchLinearProgressChart, - patchSunburstAndTreemapChart, patchBasicHeatMapChart, patchCartesianXField ], diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts index 68f543c6..59f4dd9c 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts @@ -12,20 +12,12 @@ import { import { ChartType, DataType, ROLE } from '../../../../../../common/typings'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../../types'; import { isValidDataset } from '../../../../../../common/dataProcess'; -import { NEED_SIZE_FIELD_CHART_LIST, NEED_COLOR_FIELD_CHART_LIST } from '../../../../constants'; - -const CARTESIAN_CHART_LIST = [ - 'Dynamic Bar Chart', - 'Bar Chart', - 'Line Chart', - 'Scatter Plot', - 'Funnel Chart', - 'Dual Axis Chart', - 'Waterfall Chart', - 'Box Plot', - 'Range Column Chart', - 'linear Progress Chart' -]; +import { + NEED_COLOR_FIELD_CHART_LIST, + NEED_SIZE_FIELD_CHART_LIST, + CARTESIAN_CHART_LIST, + NEED_COLOR_AND_SIZE_CHART_LIST +} from '../../../../constants'; export const patchAxisField: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, @@ -379,9 +371,10 @@ export const patchNeedColor: Transformer< const { chartType, cell, fieldInfo } = context; const cellNew: any = { ...cell }; if ( - NEED_COLOR_FIELD_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) + NEED_COLOR_FIELD_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) || + NEED_COLOR_AND_SIZE_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) ) { - const colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean).flat(); + const colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean); if (colorField.length !== 0) { cellNew.color = colorField[0]; } else { @@ -405,7 +398,10 @@ export const patchNeedSize: Transformer< > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { const { chartType, cell, fieldInfo } = context; const cellNew: any = { ...cell }; - if (NEED_SIZE_FIELD_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType)) { + if ( + NEED_SIZE_FIELD_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType) || + NEED_COLOR_AND_SIZE_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType) + ) { const sizeField = [cellNew.size, cellNew.value, cellNew.y, cellNew.radius, cellNew.angle].filter(Boolean).flat(); if (sizeField.length !== 0) { cellNew.size = sizeField[0]; @@ -433,7 +429,7 @@ export const patchRangeColumnChart: Transformer< const cellNew = { ...cell }; const remainedFields = getRemainedFields(cellNew, fieldInfo); const numericFields = getFieldsByDataType(remainedFields, [DataType.FLOAT, DataType.INT]); - if (chartType === ('RANGE COLUMN CHART' as ChartType)) { + if (chartType === ChartType.RangeColumnChart.toUpperCase()) { if (cellNew.y && cellNew.y instanceof Array && cellNew.y.length === 2) { return { cell: cellNew }; } @@ -458,7 +454,20 @@ export const patchLinearProgressChart: Transformer< > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { const { chartType, cell, fieldInfo } = context; const cellNew = { ...cell }; - if (chartType === ('Linear Progress Chart' as ChartType)) { + if (chartType === ChartType.LinearProgressChart.toUpperCase()) { + const xField = [cellNew.x, cellNew.color].filter(Boolean).flat(); + if (xField.length !== 0) { + cellNew.x = xField[0]; + } else { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const xField = getFieldByRole(remainedFields, ROLE.DIMENSION); + if (xField) { + cellNew.x = xField.fieldName; + } else { + cellNew.x = remainedFields[0].fieldName; + } + } + const yField = [cellNew.y, cellNew.size, cellNew.value, cellNew.radius, cellNew.angle].filter(Boolean).flat(); if (yField.length !== 0) { cellNew.y = yField[0]; @@ -478,38 +487,13 @@ export const patchLinearProgressChart: Transformer< }; }; -export const patchSunburstAndTreemapChart: Transformer< - GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, - Partial -> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell } = context; - const cellNew: any = { ...cell }; - - if (chartType === ('SUNBURST CHART' as ChartType) || chartType === ('TREEMAP CHART' as ChartType)) { - cellNew.y = [cellNew.y, cellNew.value, cellNew.radius, cellNew.size, cellNew.angle].filter(Boolean).flat(); - if (cellNew.y.length !== 0) { - cellNew.y = cellNew.y[0]; - } else { - throw Error( - 'The y-axis of the sunburst chart and the treemap chart requires a numeric field,' + - 'but the result of data aggregation does not have a numeric field' - ); - } - } - - return { - //...context, - cell: cellNew - }; -}; - export const patchBasicHeatMapChart: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { const { chartType, cell, fieldInfo } = context; const cellNew: any = { ...cell }; - if (chartType === ('BASIC HEAT MAP' as ChartType)) { + if (chartType === ChartType.BasicHeatMap.toUpperCase()) { const colorField = [cellNew.x, cellNew.y, cellNew.label, cellNew.color].filter(Boolean).flat(); if (colorField.length >= 2) { cellNew.x = colorField[0]; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts index 47a16d96..aaf6b2c7 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts @@ -3,7 +3,14 @@ import { ChartAdvisorPromptEnglish } from './template'; import type { GenerateChartAndFieldMapContext } from '../../types'; import { pick } from '@visactor/vutils'; import { getStrFromArray } from '../../../../../../common/utils/utils'; -import { chartGenerationConstraints, chartKnowledgeDict, defaultExamples, visualChannelInfoMap } from './knowledges'; +import { + chartGenerationConstraints, + chartKnowledgeDict, + defaultExamples, + getCartesianCoordinateSystemKnowledge, + getNeedColorAndSizeKnowledge, + visualChannelInfoMap +} from './knowledges'; import { uniqArray } from '@visactor/vutils'; const patchUserInput = (userInput: string) => { @@ -44,10 +51,13 @@ export class GPTChartGenerationPrompt extends Prompt chartKnowledgeDict[a].index - chartKnowledgeDict[b].index); - const chartKnowledge = sortedChartTypeList.reduce((res, chartType) => { - const { knowledge } = chartKnowledgeDict[chartType]; - return [...res, ...(knowledge ?? [])]; - }, []); + const chartKnowledge = sortedChartTypeList.reduce( + (res, chartType) => { + const { knowledge } = chartKnowledgeDict[chartType]; + return [...res, ...(knowledge ?? [])]; + }, + [getNeedColorAndSizeKnowledge(chartTypeList), getCartesianCoordinateSystemKnowledge(chartTypeList)] + ); const knowledgeStr = getStrFromArray(chartKnowledge); diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts index e9746388..16e4cfa5 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts @@ -2,7 +2,12 @@ import { ChartType } from '../../../../../../common/typings'; import { barChartExample1, dynamicBarChart1, lineChartExample1, lineChartExample2, pieChartExample1 } from './examples'; import type { ChartKnowledge } from './types'; -import { NEED_SIZE_FIELD_CHART_LIST, NEED_COLOR_FIELD_CHART_LIST } from '../../../../constants'; +import { + CARTESIAN_CHART_LIST, + NEED_COLOR_FIELD_CHART_LIST, + NEED_SIZE_FIELD_CHART_LIST, + NEED_COLOR_AND_SIZE_CHART_LIST +} from '../../../../constants'; const getColorKnowledge = (chartTypeList: ChartType[]) => { const validSet = new Set(chartTypeList); @@ -17,7 +22,31 @@ const getSizeKnowledge = (chartTypeList: ChartType[]) => { const validSet = new Set(chartTypeList); if (NEED_SIZE_FIELD_CHART_LIST.some(chartType => validSet.has(chartType))) { const includedCharts = chartTypeList.filter(chart => NEED_SIZE_FIELD_CHART_LIST.includes(chart)); - return ' Only used in ' + includedCharts.join(' , ') + '.'; + return ", can't be empty in " + includedCharts.join(' , ') + '.'; + } + return '.'; +}; + +export const getNeedColorAndSizeKnowledge = (chartTypeList: ChartType[]) => { + const validSet = new Set(chartTypeList); + if (NEED_COLOR_AND_SIZE_CHART_LIST.some(chartType => validSet.has(chartType))) { + const includedCharts = chartTypeList.filter(chart => NEED_COLOR_AND_SIZE_CHART_LIST.includes(chart)); + return ( + includedCharts.join(', ') + + ' all commonly use color to represent different categories and size to represent the magnitude of the values. Map your data fields to the color (representing the category) and size (representing the value) visual channels instead of using the x-axis of the Cartesian coordinate system.' + ); + } + return '.'; +}; + +export const getCartesianCoordinateSystemKnowledge = (chartTypeList: ChartType[]) => { + const validSet = new Set(chartTypeList); + if (CARTESIAN_CHART_LIST.some(chartType => validSet.has(chartType))) { + const includedCharts = chartTypeList.filter(chart => CARTESIAN_CHART_LIST.includes(chart)); + return ( + includedCharts.join(', ') + + ' are visualizations based on the Cartesian coordinate system. The Cartesian coordinate system is usually used to show linear relationships and numerical changes. Please map the data fields to the x-axis (independent variable) and the y-axis (dependent variable).' + ); } return '.'; }; @@ -41,9 +70,7 @@ export const visualChannelInfoMap = { target: (chartTypeList: ChartType[]) => "the field mapped to the target channel. Only used in Sankey Chart. Can't be empty in Sankey Chart.", value: (chartTypeList: ChartType[]) => - "the field mapped to the value channel. Only used in Sankey Chart. Can't be empty in Sankey Chart.", - group: (chartTypeList: ChartType[]) => - "the field mapped to the value channel. Only used in Venn Chart. Can't be empty in Venn Chart. Digital IDs are often used to distinguish whether data is in the same group." + "the field mapped to the value channel. Only used in Sankey Chart. Can't be empty in Sankey Chart." }; export const chartKnowledgeDict: ChartKnowledge = { [ChartType.BarChart]: { @@ -69,7 +96,10 @@ export const chartKnowledgeDict: ChartKnowledge = { [ChartType.WordCloud]: { index: 6, visualChannels: ['color', 'size'], - examples: [] + examples: [], + knowledge: [ + 'Word cloud is an effective visualization tool that can intuitively display the frequency of keywords. The size of a keyword is proportional to its popularity, which is suitable for displaying the importance and trends in text data.' + ] }, [ChartType.RoseChart]: { index: 7, @@ -161,19 +191,17 @@ export const chartKnowledgeDict: ChartKnowledge = { }, [ChartType.SunburstChart]: { index: 20, - visualChannels: ['x', 'y'], + visualChannels: ['color', 'size'], examples: [], knowledge: [ - 'The x field of a sunburst chart can be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + 'The colors field for sunburst chart and treemap chart must be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' ] }, [ChartType.TreemapChart]: { index: 21, - visualChannels: ['x', 'y'], + visualChannels: ['color', 'size'], examples: [], - knowledge: [ - 'The x field of a treemap chart can be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' - ] + knowledge: [] }, [ChartType.Gauge]: { index: 22, @@ -195,9 +223,11 @@ export const chartKnowledgeDict: ChartKnowledge = { }, [ChartType.VennChart]: { index: 25, - visualChannels: ['size', 'color', 'group'], + visualChannels: ['size', 'color'], examples: [], - knowledge: ['The venn chart must contain three fields: group, size and color.'] + knowledge: [ + 'The color field of the Venn diagram requires an array of length 2. The field with subscript 0 maps to the sets, and the field with subscript 1 maps to the name.' + ] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts index f03b9876..fe96ceff 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts @@ -208,15 +208,16 @@ const pipelineBubbleCirclePacking = [ theme ]; -const pipelineMapChart = [chartType, basemap, arrayData, mapField, mapDisplayConf, theme]; -const pipelineRangeColumn = [chartType, data, rangeColumnField, rangeColumnDisplayConf, theme]; -const pipelineSunburst = [chartType, sunburstData, sunburstOrTreemapField, sunburstDisplayConf, theme]; -const pipelineTreemap = [chartType, treemapData, sunburstOrTreemapField, treemapDisplayConf, theme]; -const pipelineGauge = [chartType, arrayData, gaugeField, gaugeDisplayConf, theme]; -// const pipelineLinearProgress = [chartType, arrayData, linearProgressField, linearProgressDisplayConf, theme]; +const pipelineMapChart = [chartType, basemap, color, arrayData, mapField, mapDisplayConf, theme]; +const pipelineRangeColumn = [chartType, data, color, rangeColumnField, rangeColumnDisplayConf, theme]; +const pipelineSunburst = [chartType, sunburstData, color, sunburstOrTreemapField, sunburstDisplayConf, theme]; +const pipelineTreemap = [chartType, treemapData, color, sunburstOrTreemapField, treemapDisplayConf, theme]; +const pipelineGauge = [chartType, arrayData, color, gaugeField, gaugeDisplayConf, theme]; +// const pipelineLinearProgress = [chartType, arrayData, color, linearProgressField, linearProgressDisplayConf, theme]; const pipelineBasicHeatMap = [ chartType, arrayData, + color, basicHeatMapSeries, basicHeatMapRegion, basicHeatMapColor, @@ -224,7 +225,7 @@ const pipelineBasicHeatMap = [ basicHeatMapLegend, theme ]; -const pipelineVenn = [chartType, registerChart, vennData, vennField, legend, theme]; +const pipelineVenn = [chartType, registerChart, vennData, color, vennField, legend, theme]; const pipelineMap: { [chartType: string]: any } = { 'BAR CHART': pipelineBar, diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 811c0673..07f14524 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -753,7 +753,7 @@ export const cartesianBar: Transformer = (context: const cellNew = { ...cell }; const flattenedXField = Array.isArray(cell.x) ? cell.x : [cell.x]; if (cell.color && cell.color.length > 0 && cell.color !== cell.x) { - flattenedXField.push(cell.color); + flattenedXField.push(cell.color as string); } spec.xField = flattenedXField; spec.yField = cell.y; @@ -1155,7 +1155,7 @@ export const animationCartesianPie: Transformer = ( const { spec } = context; const totalTime = context.totalTime ?? DEFAULT_PIE_VIDEO_LENGTH; - const groupKey = context.cell.color; + const groupKey = context.cell.color as string; const dataValues = spec.data.values as any[]; const groupNum = dataValues.map(d => d[groupKey!]).filter(onlyUnique).length; //const delay = totalTime / groupNum - 1000; @@ -1470,21 +1470,21 @@ export const rangeColumnDisplayConf: Transformer = export const sunburstData: Transformer = (context: Context) => { const { dataset, cell, spec } = context; - spec.data = { id: 'data', values: getSunburstData(dataset, cell.x, 0, cell.y) }; + spec.data = { id: 'data', values: getSunburstData(dataset, cell.color, 0, cell.size) }; return { spec }; }; export const getSunburstData: any = ( dataset: VMindDataset, - xField: string[] | string, + colorField: string[] | string, index: number, - yField: string + sizeField: string ) => { - if (xField.length - 1 === index) { + if (colorField.length - 1 === index) { return Array.from( new Set( dataset.map(data => { - return { name: data[xField[index]], value: data[yField] }; + return { name: data[colorField[index]], value: data[sizeField] }; }) ) ); @@ -1493,15 +1493,15 @@ export const getSunburstData: any = ( const values = Array.from( new Set( dataset.map(data => { - return data[xField[index]]; + return data[colorField[index]]; }) ) ); return values.map(value => { const currentDataset = dataset.filter(data => { - return data[xField[index]] === value; + return data[colorField[index]] === value; }); - return { name: value, children: getSunburstData(currentDataset, xField, index + 1, yField) }; + return { name: value, children: getSunburstData(currentDataset, colorField, index + 1, sizeField) }; }); }; @@ -1566,21 +1566,21 @@ export const sunburstDisplayConf: Transformer = (co export const treemapData: Transformer = (context: Context) => { const { dataset, cell, spec } = context; - spec.data = { id: 'data', values: getTreemapData(dataset, cell.x, 0, cell.y) }; + spec.data = { id: 'data', values: getTreemapData(dataset, cell.color, 0, cell.size) }; return { spec }; }; export const getTreemapData: any = ( dataset: VMindDataset, - xField: string[] | string, + colorField: string[] | string, index: number, - yField: string + sizeField: string ) => { - if (xField.length - 1 === index) { + if (colorField.length - 1 === index) { return Array.from( new Set( dataset.map(data => { - return { name: data[xField[index]], value: data[yField] }; + return { name: data[colorField[index]], value: data[sizeField] }; }) ) ); @@ -1589,18 +1589,18 @@ export const getTreemapData: any = ( const values = Array.from( new Set( dataset.map(data => { - return data[xField[index]]; + return data[colorField[index]]; }) ) ); return values.map(value => { const currentDataset = dataset.filter(data => { - return data[xField[index]] === value; + return data[colorField[index]] === value; }); - if (currentDataset[0] && currentDataset[0][xField[index + 1]] === '') { - return { name: value, value: currentDataset[0][yField] }; + if (currentDataset[0] && currentDataset[0][colorField[index + 1]] === '') { + return { name: value, value: currentDataset[0][sizeField] }; } - return { name: value, children: getTreemapData(currentDataset, xField, index + 1, yField) }; + return { name: value, children: getTreemapData(currentDataset, colorField, index + 1, sizeField) }; }); }; @@ -1662,11 +1662,13 @@ export const linearProgressDisplayConf: Transformer export const vennData: Transformer = (context: Context) => { const { dataset, spec, cell } = context; const id2dataMap = {}; + const setsField = cell.color[0]; + const nameField = cell.color[1]; dataset.forEach(data => { - if (id2dataMap[data[cell.group]]) { - id2dataMap[data[cell.group]].sets.push(data[cell.color]); + if (id2dataMap[data[setsField]]) { + id2dataMap[data[setsField]].sets.push(data[nameField]); } else { - id2dataMap[data[cell.group]] = { sets: [data[cell.color]], value: data[cell.size] }; + id2dataMap[data[setsField]] = { sets: [data[nameField]], value: data[cell.size] }; } }); spec.data = { diff --git a/packages/vmind/src/applications/chartGeneration/types.ts b/packages/vmind/src/applications/chartGeneration/types.ts index f3656d90..5103fed2 100644 --- a/packages/vmind/src/applications/chartGeneration/types.ts +++ b/packages/vmind/src/applications/chartGeneration/types.ts @@ -1,8 +1,8 @@ export type Cell = { //字段映射，可用的视觉通道：["x","y","color","size","angle","time"] - x?: string | string[]; + x?: string; y?: string | string[]; - color?: string; + color?: string | string[]; size?: string; angle?: string; radius?: string; @@ -11,5 +11,4 @@ export type Cell = { target?: string; value?: string; category?: string; - group?: string; }; From 41484b1a0b152840fda56ceea8de0aebf8a9f591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Tue, 23 Jul 2024 11:23:56 +0800 Subject: [PATCH 010/128] Adjust the registration timing of the map basemap --- .../src/pages/ChartGeneration/DataInput.tsx | 9 +++++++++ .../applications/chartGeneration/constants.ts | 1 - .../getChartSpec/VChart/transformers.ts | 20 +++++++++++-------- .../src/base/taskNode/ruleBasedTaskNode.ts | 13 +++++++----- packages/vmind/src/common/typings/index.ts | 1 - 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index c14daebb..b4614f9e 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -54,6 +54,7 @@ import { import VMind, { ArcoTheme, builtinThemeMap, BuiltinThemeType } from '../../../../../src/index'; import { Model } from '../../../../../src/index'; import { isArray } from '@visactor/vutils'; +import VChart from '@visactor/vchart'; const TextArea = Input.TextArea; const Option = Select.Option; @@ -117,6 +118,7 @@ const ModelConfigMap: any = { }; const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions'; +const MAP_CHART_BASEMAP = 'https://lf9-dp-fe-cms-tos.byteorg.com/obj/bit-cloud/geojson/world.json'; const specTemplateTest = false; export function DataInput(props: IPropsType) { const defaultDataKey = Object.keys(demoDataList)[3]; @@ -168,6 +170,13 @@ export function DataInput(props: IPropsType) { const finalDataset = specTemplateTest && model !== Model.CHART_ADVISOR ? undefined : dataset; const startTime = new Date().getTime(); + + // The map chart requires the user to register a base map named map in advance using VChart.registerMap + if (csv === demoDataList.Map.csv) { + const response = await fetch(MAP_CHART_BASEMAP); + const geoJson = await response.json(); + VChart.registerMap('map', geoJson); + } const chartGenerationRes = await vmind.generateChart(describe, finalFieldInfo, finalDataset, { //enableDataQuery: false, //chartTypeList: [ChartType.BarChart, ChartType.LineChart], diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index 00bb380e..ff6af9f6 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -31,7 +31,6 @@ export const CARTESIAN_CHART_LIST = [ ]; export const DEFAULT_MAP_OPTION: BasemapOption = { - jsonUrl: 'https://lf9-dp-fe-cms-tos.byteorg.com/obj/bit-cloud/geojson/world.json', regionProjectType: null, regionCoordinate: MapRegionCoordinate.GEO, zoom: 1, diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 07f14524..27d25c24 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -1,6 +1,5 @@ import type { Transformer } from '../../../../../base/tools/transformer'; import { registerVennChart } from '@visactor/vchart'; -import VChart from '@visactor/vchart'; import type { GetChartSpecContext, GetChartSpecOutput } from '../types'; import { COLOR_THEMES, @@ -1802,12 +1801,8 @@ export const basicHeatMapLegend: Transformer = (con return { spec }; }; -export const basemap: Transformer> = async (context: Context) => { +export const basemap: Transformer = (context: Context) => { const { basemapOption, spec } = context; - const jsonUrl = basemapOption.jsonUrl; - const response = await fetch(jsonUrl); - const geoJson = await response.json(); - VChart.registerMap('map', geoJson); if (basemapOption.regionProjectType) { spec.region = [ { @@ -1839,15 +1834,24 @@ export const mapField: Transformer = (context: Cont }; export const mapDisplayConf: Transformer = (context: Context) => { - const { spec } = context; + const { spec, cell } = context; spec.legends = [ { visible: true, type: 'color', - field: 'value', + field: cell.size, orient: 'bottom', position: 'start' } ]; + spec.area = { + style: { + fill: { + field: cell.size, + scale: 'color', + changeDomain: 'replace' + } + } + }; return { spec }; }; diff --git a/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts b/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts index 3bb46a87..92168c9e 100644 --- a/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts +++ b/packages/vmind/src/base/taskNode/ruleBasedTaskNode.ts @@ -32,7 +32,7 @@ export class RuleBasedTaskNode extends BaseTaskNode { + executeTask(context: Context): Result | TaskError { this.updateContext({ ...this.context, ...context }); let pipelines = this.pipelines; if (isFunction(this.pipelines)) { @@ -40,10 +40,13 @@ export class RuleBasedTaskNode extends BaseTaskNode[]) { - result = { ...result, ...(await transformer(result)) }; - } + const result: Result = (pipelines as Transformer[]).reduce( + (pre: any, transformer: Transformer) => { + const res = transformer(pre); + return { ...pre, ...res }; + }, + context + ); return result; } catch (e: any) { console.error(`${this.name} error!`); diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index 22dedde0..2afca682 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -216,7 +216,6 @@ export enum MapRegionCoordinate { } export type BasemapOption = { - jsonUrl: string; regionProjectType: mapRegionProjectionType; regionCoordinate: MapRegionCoordinate; zoom: number; From 3d13bcbef530bbdea8056d7033939a49284a42f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Tue, 23 Jul 2024 14:56:01 +0800 Subject: [PATCH 011/128] Use enumerations instead of chart type strings; merge linear progress charts --- .../browser/src/constants/mockData.ts | 4 +- .../applications/chartGeneration/constants.ts | 8 +- .../GPT/patcher/index.ts | 24 +++--- .../GPT/prompt/knowledges.ts | 14 +--- .../getChartSpec/VChart/chartPipeline.ts | 75 ++++++++---------- .../getChartSpec/VChart/transformers.ts | 79 ++++++------------- 6 files changed, 81 insertions(+), 123 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index e60bd7ad..25de468e 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -5324,11 +5324,11 @@ export const mockUserTextInput10 = { export const mockProgressData = { csv: `年份,进度 2024,0.56`, - input: '帮我展示今年的进度数据' + input: '请使用环形进度图帮我展示今年的进度数据' }; export const liquidData = { csv: `进度 0.56`, - input: '展示进度数据' + input: '请使用水波图展示进度数据' }; diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index ff6af9f6..4326f34e 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -3,7 +3,7 @@ import { ChartType, MapRegionCoordinate } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); -export const NEED_COLOR_FIELD_CHART_LIST = [ChartType.PieChart, ChartType.RoseChart]; +export const NEED_COLOR_FIELD_CHART_LIST = [ChartType.PieChart, ChartType.RoseChart, ChartType.LinearProgress]; export const NEED_SIZE_FIELD_CHART_LIST = [ChartType.ScatterPlot, ChartType.BasicHeatMap]; @@ -14,7 +14,9 @@ export const NEED_COLOR_AND_SIZE_CHART_LIST = [ ChartType.VennChart, ChartType.Gauge, ChartType.SunburstChart, - ChartType.TreemapChart + ChartType.TreemapChart, + ChartType.CircularProgress, + ChartType.LiquidChart ]; export const CARTESIAN_CHART_LIST = [ @@ -27,7 +29,7 @@ export const CARTESIAN_CHART_LIST = [ ChartType.WaterFallChart, ChartType.BoxPlot, ChartType.RangeColumnChart, - ChartType.LinearProgressChart + ChartType.LinearProgress ]; export const DEFAULT_MAP_OPTION: BasemapOption = { diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts index 59f4dd9c..6cab9455 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts @@ -90,9 +90,9 @@ export const patchYField: Transformer< if (y && isArray(y) && y.length > 1) { if ( - chartTypeNew === ('BOX PLOT' as ChartType) || - (chartTypeNew === ('DUAL AXIS CHART' as ChartType) && y.length === 2) || - (chartTypeNew === ('RANGE COLUMN CHART' as ChartType) && y.length === 2) + chartTypeNew === ChartType.BoxPlot.toUpperCase() || + (chartTypeNew === ChartType.DualAxisChart.toUpperCase() && y.length === 2) || + (chartTypeNew === ChartType.RangeColumnChart.toUpperCase() && y.length === 2) ) { return { ...context @@ -112,7 +112,7 @@ export const patchYField: Transformer< cellNew.color = FOLD_NAME.toString(); } } else { - chartTypeNew = 'SCATTER PLOT' as ChartType; + chartTypeNew = ChartType.ScatterPlot.toUpperCase(); cellNew = { ...cell, x: y[0], @@ -139,7 +139,7 @@ export const patchBoxPlot: Transformer< ...cell }; const { y } = cellNew; - if (chartType === ('BOX PLOT' as ChartType)) { + if (chartType === ChartType.BoxPlot.toUpperCase()) { if (typeof y === 'string' && y.split(',').length > 1) { cellNew.y = y.split(',').map(str => str.trim()); } else if (isNil(y) || y.length === 0) { @@ -205,7 +205,7 @@ export const patchDualAxis: Transformer< const cellNew: any = { ...cell }; //Dual-axis drawing yLeft and yRight - if (chartType === ('DUAL AXIS CHART' as ChartType)) { + if (chartType === ChartType.DualAxisChart.toUpperCase()) { cellNew.y = [cellNew.y, cellNew.yLeft, cellNew.yRight, cellNew.y1, cellNew.y2].filter(Boolean).flat(); } @@ -222,12 +222,12 @@ export const patchPieChart: Transformer< const { chartType, cell, fieldInfo } = context; const cellNew = { ...cell }; - if (chartType === ('ROSE CHART' as ChartType)) { + if (chartType === ChartType.RoseChart.toUpperCase()) { cellNew.angle = cellNew.radius ?? cellNew.size ?? cellNew.angle; } //Pie chart must have color field and the angle field - if (chartType === ('PIE CHART' as ChartType) || chartType === ('ROSE CHART' as ChartType)) { + if (chartType === ChartType.PieChart.toUpperCase() || chartType === ChartType.RoseChart.toUpperCase()) { if (!cellNew.color || !cellNew.angle) { const remainedFields = getRemainedFields(cellNew, fieldInfo); @@ -265,7 +265,7 @@ export const patchWordCloud: Transformer< const { chartType, cell, fieldInfo } = context; const cellNew = { ...cell }; - if (chartType === ('WORD CLOUD' as ChartType)) { + if (chartType === ChartType.WordCloud.toUpperCase()) { if (!cellNew.size || !cellNew.color || cellNew.color === cellNew.size) { const remainedFields = getRemainedFields(cellNew, fieldInfo); @@ -311,7 +311,7 @@ export const patchDynamicBarChart: Transformer< const cellNew = { ...cell }; let chartTypeNew = chartType; - if (chartType === ('DYNAMIC BAR CHART' as ChartType)) { + if (chartType === ChartType.DynamicBarChart.toUpperCase()) { if (!cell.time || cell.time === '' || cell.time.length === 0) { const remainedFields = getRemainedFields(cellNew, fieldInfo); @@ -325,7 +325,7 @@ export const patchDynamicBarChart: Transformer< cellNew.time = stringField.fieldName; } else { //no available field, set chart type to bar chart - chartTypeNew = 'BAR CHART' as ChartType; + chartTypeNew = ChartType.BarChart.toUpperCase(); } } } @@ -454,7 +454,7 @@ export const patchLinearProgressChart: Transformer< > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { const { chartType, cell, fieldInfo } = context; const cellNew = { ...cell }; - if (chartType === ChartType.LinearProgressChart.toUpperCase()) { + if (chartType === ChartType.LinearProgress.toUpperCase()) { const xField = [cellNew.x, cellNew.color].filter(Boolean).flat(); if (xField.length !== 0) { cellNew.x = xField[0]; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts index 16e4cfa5..f35c59f4 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts @@ -159,9 +159,7 @@ export const chartKnowledgeDict: ChartKnowledge = { index: 15, visualChannels: ['x', 'y'], examples: [], - knowledge: [ - 'Linear Progress chart is typically used to display progress data, which is usually a value between 0 and 1. Linear progress bars can show single progress values as well as multiple progress values. By default, the left Y-axis of the linear progress bar is the categorical field, and the bottom X-axis is the numerical field.' - ] + knowledge: [] }, [ChartType.CircularProgress]: { index: 16, @@ -209,20 +207,14 @@ export const chartKnowledgeDict: ChartKnowledge = { examples: [], knowledge: ['The gauge chart must contain two fields: size and color.'] }, - // [ChartType.LinearProgressChart]: { - // index: 23, - // visualChannels: ['y', 'x'], - // examples: [], - // knowledge: [] - // }, [ChartType.BasicHeatMap]: { - index: 24, + index: 23, visualChannels: ['y', 'x', 'size'], examples: [], knowledge: [] }, [ChartType.VennChart]: { - index: 25, + index: 24, visualChannels: ['size', 'color'], examples: [], knowledge: [ diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts index fe96ceff..79657aed 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts @@ -64,8 +64,6 @@ import { treemapDisplayConf, gaugeField, gaugeDisplayConf, - // linearProgressField, - // linearProgressDisplayConf, arrayData, vennData, vennField, @@ -179,23 +177,23 @@ const pipelineBoxPlot = [chartType, data, color, boxPlotField, boxPlotStyle, leg const pipelineLiquid = [chartType, data, color, liquidField, liquidStyle, indicator, theme]; const pipelineLinearProgress = [ - chartType, - data, - color, - linearProgressField, - linearProgressAxes, - linearProgressStyle, - theme + chartType, + data, + color, + linearProgressField, + linearProgressAxes, + linearProgressStyle, + theme ]; const pipelineCircularProgress = [ - chartType, - data, - color, - circularProgressField, - circularProgressStyle, - indicator, - theme + chartType, + data, + color, + circularProgressField, + circularProgressStyle, + indicator, + theme ]; const pipelineBubbleCirclePacking = [ @@ -213,7 +211,6 @@ const pipelineRangeColumn = [chartType, data, color, rangeColumnField, rangeColu const pipelineSunburst = [chartType, sunburstData, color, sunburstOrTreemapField, sunburstDisplayConf, theme]; const pipelineTreemap = [chartType, treemapData, color, sunburstOrTreemapField, treemapDisplayConf, theme]; const pipelineGauge = [chartType, arrayData, color, gaugeField, gaugeDisplayConf, theme]; -// const pipelineLinearProgress = [chartType, arrayData, color, linearProgressField, linearProgressDisplayConf, theme]; const pipelineBasicHeatMap = [ chartType, arrayData, @@ -228,32 +225,30 @@ const pipelineBasicHeatMap = [ const pipelineVenn = [chartType, registerChart, vennData, color, vennField, legend, theme]; const pipelineMap: { [chartType: string]: any } = { - 'BAR CHART': pipelineBar, - 'LINE CHART': pipelineLine, - 'PIE CHART': pipelinePie, - 'WORD CLOUD': pipelineWordCloud, - 'SCATTER PLOT': pipelineScatterPlot, - 'DYNAMIC BAR CHART': pipelineRankingBar, - 'FUNNEL CHART': pipelineFunnel, - 'DUAL AXIS CHART': pipelineDualAxis, - 'ROSE CHART': pipelineRose, - 'RADAR CHART': pipelineRadar, - 'SANKEY CHART': pipelineSankey, - 'WATERFALL CHART': pipelineWaterfall, - 'BOX PLOT': pipelineBoxPlot, + [ChartType.BarChart.toUpperCase()]: pipelineBar, + [ChartType.LineChart.toUpperCase()]: pipelineLine, + [ChartType.PieChart.toUpperCase()]: pipelinePie, + [ChartType.WordCloud.toUpperCase()]: pipelineWordCloud, + [ChartType.ScatterPlot.toUpperCase()]: pipelineScatterPlot, + [ChartType.DynamicBarChart.toUpperCase()]: pipelineRankingBar, + [ChartType.FunnelChart.toUpperCase()]: pipelineFunnel, + [ChartType.DualAxisChart.toUpperCase()]: pipelineDualAxis, + [ChartType.RoseChart.toUpperCase()]: pipelineRose, + [ChartType.RadarChart.toUpperCase()]: pipelineRadar, + [ChartType.SankeyChart.toUpperCase()]: pipelineSankey, + [ChartType.WaterFallChart.toUpperCase()]: pipelineWaterfall, + [ChartType.BoxPlot.toUpperCase()]: pipelineBoxPlot, [ChartType.LiquidChart.toUpperCase()]: pipelineLiquid, [ChartType.LinearProgress.toUpperCase()]: pipelineLinearProgress, [ChartType.CircularProgress.toUpperCase()]: pipelineCircularProgress, - 'BOX PLOT': pipelineBoxPlot, - 'BUBBLE CIRCLE PACKING': pipelineBubbleCirclePacking, - 'MAP CHART': pipelineMapChart, - 'RANGE COLUMN CHART': pipelineRangeColumn, - 'SUNBURST CHART': pipelineSunburst, - 'TREEMAP CHART': pipelineTreemap, - 'GAUGE CHART': pipelineGauge, - // 'LINEAR PROGRESS CHART': pipelineLinearProgress, - 'BASIC HEAT MAP': pipelineBasicHeatMap, - 'VENN CHART': pipelineVenn + [ChartType.BubbleCirclePacking.toUpperCase()]: pipelineBubbleCirclePacking, + [ChartType.MapChart.toUpperCase()]: pipelineMapChart, + [ChartType.RangeColumnChart.toUpperCase()]: pipelineRangeColumn, + [ChartType.SunburstChart.toUpperCase()]: pipelineSunburst, + [ChartType.TreemapChart.toUpperCase()]: pipelineTreemap, + [ChartType.Gauge.toUpperCase()]: pipelineGauge, + [ChartType.BasicHeatMap.toUpperCase()]: pipelineBasicHeatMap, + [ChartType.VennChart.toUpperCase()]: pipelineVenn }; export const beforePipe: Transformer = ( diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 27d25c24..3b1e4bd5 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -28,31 +28,30 @@ import { COLOR_FIELD } from '@visactor/chart-advisor'; type Context = GetChartSpecContext & GetChartSpecOutput; const chartTypeMap: { [chartName: string]: string } = { - 'BAR CHART': 'bar', - 'LINE CHART': 'line', - 'PIE CHART': 'pie', - 'WORD CLOUD': 'wordCloud', - 'SCATTER PLOT': 'scatter', - 'DYNAMIC BAR CHART': 'bar', - 'FUNNEL CHART': 'funnel', - 'DUAL AXIS CHART': 'common', - 'ROSE CHART': 'rose', - 'RADAR CHART': 'radar', - 'SANKEY CHART': 'sankey', - 'WATERFALL CHART': 'waterfall', - 'BOX PLOT': 'boxPlot', + [ChartType.BarChart.toUpperCase()]: 'bar', + [ChartType.LineChart.toUpperCase()]: 'line', + [ChartType.PieChart.toUpperCase()]: 'pie', + [ChartType.WordCloud.toUpperCase()]: 'wordCloud', + [ChartType.ScatterPlot.toUpperCase()]: 'scatter', + [ChartType.DynamicBarChart.toUpperCase()]: 'bar', + [ChartType.FunnelChart.toUpperCase()]: 'funnel', + [ChartType.DualAxisChart.toUpperCase()]: 'common', + [ChartType.RoseChart.toUpperCase()]: 'rose', + [ChartType.RadarChart.toUpperCase()]: 'radar', + [ChartType.SankeyChart.toUpperCase()]: 'sankey', + [ChartType.WaterFallChart.toUpperCase()]: 'waterfall', + [ChartType.BoxPlot.toUpperCase()]: 'boxPlot', [ChartType.LiquidChart.toUpperCase()]: 'liquid', [ChartType.LinearProgress.toUpperCase()]: 'linearProgress', - [ChartType.CircularProgress.toUpperCase()]: 'circularProgress' - 'BUBBLE CIRCLE PACKING': 'circlePacking', - 'MAP CHART': 'map', - 'RANGE COLUMN CHART': 'rangeColumn', - 'SUNBURST CHART': 'sunburst', - 'TREEMAP CHART': 'treemap', - 'GAUGE CHART': 'gauge', - // 'LINEAR PROGRESS CHART': 'linearProgress', - 'BASIC HEAT MAP': 'common', - 'VENN CHART': 'venn' + [ChartType.CircularProgress.toUpperCase()]: 'circularProgress', + [ChartType.BubbleCirclePacking.toUpperCase()]: 'circlePacking', + [ChartType.MapChart.toUpperCase()]: 'map', + [ChartType.RangeColumnChart.toUpperCase()]: 'rangeColumn', + [ChartType.SunburstChart.toUpperCase()]: 'sunburst', + [ChartType.TreemapChart.toUpperCase()]: 'treemap', + [ChartType.Gauge.toUpperCase()]: 'gauge', + [ChartType.BasicHeatMap.toUpperCase()]: 'common', + [ChartType.VennChart.toUpperCase()]: 'venn' }; export const chartType: Transformer = (context: Context) => { @@ -1348,11 +1347,9 @@ export const circularProgressField: Transformer = ( //assign field in spec according to cell const { cell, spec } = context; - spec.categoryField = cell.radius; + spec.categoryField = cell.color; spec.valueField = cell.value; - if (cell.color) { - spec.seriesField = cell.color; - } + spec.seriesField = cell.color; spec.radius = 0.8; spec.innerRadius = 0.7; @@ -1630,34 +1627,6 @@ export const gaugeDisplayConf: Transformer = (conte return { spec }; }; -export const linearProgressField: Transformer = (context: Context) => { - const { spec, cell } = context; - spec.yField = cell.x; - spec.xField = cell.y; - - spec.seriesField = cell.x; - return { spec }; -}; - -export const linearProgressDisplayConf: Transformer = (context: Context) => { - const { spec } = context; - spec.direction = 'horizontal'; - - spec.cornerRadius = 20; - spec.bandWidth = 30; - spec.axes = [ - { - orient: 'left', - label: { visible: true }, - type: 'band', - domainLine: { visible: false }, - tick: { visible: false } - }, - { orient: 'bottom', label: { visible: true }, type: 'linear', visible: false } - ]; - return { spec }; -}; - export const vennData: Transformer = (context: Context) => { const { dataset, spec, cell } = context; const id2dataMap = {}; From 4d91a80e7ee8bcf939142337de469668217f2789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Sun, 11 Aug 2024 00:33:38 +0800 Subject: [PATCH 012/128] feat: Expanding a single column combination chart --- .../browser/src/constants/mockData.ts | 588 +++++++++++++++++- .../src/pages/ChartGeneration/DataInput.tsx | 16 +- .../applications/chartGeneration/constants.ts | 14 +- .../taskNodes/generateChartType/types.ts | 2 + .../taskNodes/generateFieldMap/types.ts | 9 +- .../generateTypeAndFieldMap/GPT/index.ts | 4 +- .../GPT/patcher/index.ts | 69 +- .../GPT/prompt/examples.ts | 20 +- .../GPT/prompt/index.ts | 24 +- .../GPT/prompt/knowledges.ts | 92 ++- .../GPT/prompt/template.ts | 56 +- .../generateTypeAndFieldMap/GPT/utils.ts | 22 +- .../getChartSpec/VChart/chartPipeline.ts | 19 +- .../getChartSpec/VChart/transformers.ts | 240 ++++++- .../taskNodes/getChartSpec/types.ts | 2 +- packages/vmind/src/applications/types.ts | 6 +- packages/vmind/src/common/typings/index.ts | 14 +- packages/vmind/src/core/VMind.ts | 4 +- 18 files changed, 1134 insertions(+), 67 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 25de468e..e00e559d 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -5094,7 +5094,7 @@ Customer-facing Companies,0.065,6.5%`, }; export const basicHeatMapChartData = { - csv: `val1,val2,value + csv: `Financial Ratio Indicator,Comparison Metric,Correlation Coefficient Asset Liability Ratio,Asset Liability Ratio,1 Asset Liability Ratio,Asset Liability Ratio (Deducting Advance Payments),0.594527 Asset Liability Ratio,Debt-to-long Capital Ratio,0.492963 @@ -5195,7 +5195,7 @@ Expected Default Frequency,Interest Bearing Debt / Fully Invested Capital,0.0663 Expected Default Frequency,Current Liability / Total Liabilities,-0.064886 Expected Default Frequency,Capital Fixation Ratio,-0.080153 Expected Default Frequency,Expected Default Frequency,1`, - input: '请使用热图完成数据渲染' + input: '请使用热图帮我展示各财务比率指标的相关系数' }; export const vennChartData = { @@ -5213,7 +5213,589 @@ export const vennChartData = { 7,A,2 7,B,2 7,C,2`, - input: '请使用韦恩图渲染数据' + input: '帮我展示各元素集合重叠区域，以及分布情况。请使用sets字段来区分不同区域。' +}; + +export const singleColumnLineCombinationChartData = { + csv: `date,Social Penetration,Engagement - Socialization,Penetration of Private Messages,Number of Private Messages per User +2022-03-08,1.2020804451630671,0.7782279444864411,0.21493020207806002,0.26002007727513005 +2022-03-09,1.911162758594358,0.6970763116149991,0.31807068769079905,1.791161354352144 +2022-03-10,0.919293523406533,0.6754510401577041,0.9774597853017851,1.410289280738493 +2022-03-11,1.613617931982911,4.391274420824463,0.662196540154462,2.374084690062705 +2022-03-12,1.121445896148114,0.5827973501093221,1.547704220622789,0.42545370377775604 +2022-03-13,0.527369749651032,0.321443063009525,0.52384329811226,0.45119753243186805 +2022-03-14,0.07991167814791,0.9838146243033221,1.242674929077965,1.276345431333586 +2022-03-15,0.29341332300751205,0.225492098355163,1.133076821811672,0.22045142924559902 +2022-03-16,1.103150404211427,0.580792120371675,1.261446251860683,0.061775812702848 +2022-03-17,0.38285701953593604,0.510579334822959,0.908573021920086,0.429194248082692 +2022-03-18,0.8908005339482421,1.656998380732747,2.003638269088651,1.194511111230195 +2022-03-19,0.8379933951719881,1.372238730132033,1.285574252451237,0.659056833995271 +2022-03-20,2.913059895355856,3.375041306488803,1.683935730207101,0.40361011204949004 +2022-03-21,1.245523794769467,0.5770535812918991,0.518960506573594,0.8441533797310441 +2022-03-22,1.801955367373218,2.562767955961243,2.076490748024919,1.8682050366367142 +2022-03-23,30.25170429439274,60.10096481599735,31.918470633985613,70.34237647545662 +2022-03-24,2.084831170978463,2.824195229461953,1.735738814795219,3.854924132202267 +2022-03-25,0.8300684679922911,0.37108301055861304,1.072942910906809,0.868965201778456 +2022-03-26,5.057522939533747,6.8131452572139555,10.31725741409195,0.6724835487477681 +2022-03-27,1.364245041731511,0.993289299105189,1.288157916081268,1.290710012048575 +2022-03-28,0.429699282814745,0.27225365805664004,0.24488975421611903,0.944202324553962 +2022-03-29,8.636960050981939,2.663368864789215,3.030888929145336,2.191548298261417 +2022-03-30,0.31233481070398905,1.136502929086737,1.4304534277803231,0.9398051933719991 +2022-03-31,0.8979576483673781,0.8268293005130081,1.32475784490384,0.212383798504713 +2022-04-01,1.187326287404249,1.2824253532703072,0.9289553576582981,1.328758303758777 +2022-04-02,0.013251966153457001,1.580091614771022,0.8470948142995741,1.061518688804395 +2022-04-03,0.150440921215681,0.7793382242254321,0.5908683646183901,1.021428732507816 +2022-04-04,1.437865757864698,0.34207119701235905,1.006103497118929,0.8496566144677861 +2022-04-05,19.70534531285437,4.144335050792541,13.880860231019398,19.78289952089912 +2022-04-06,1.068708512330189,1.336025709304907,7.121018094960213,0.032847589287524 +2022-04-07,3.239083103313066,0.9037359185383671,2.010136943230478,2.758258304057329 +2022-04-08,3.609669038457516,13.698767079174356,14.457401852798982,11.6964360828172 +2022-04-09,0.8581543303200471,1.11771091972957,0.9593788300193951,1.121066838670219 +2022-04-10,1.235338424517758,0.5546236200208831,1.138776207683746,0.5694312841615821 +2022-04-11,0.615560300356359,0.133505664394376,0.575680679073131,0.502679162912634 +2022-04-12,3.5445165395901572,8.458105165611851,2.90160000840929,3.084080663501847 +2022-04-13,1.037776502324676,1.412321112593621,1.416876192652653,0.270647644667579 +2022-04-14,0.534866532613768,0.577163861285873,0.034172330313952004,0.268470350264293 +2022-04-15,3.225952049483959,5.112791338669762,3.745099757241069,2.612195295941399 +2022-04-16,12.278592894400722,13.301401633991011,6.202762040985617,18.098567324029407 +2022-04-17,0.411460847135642,0.8194099242420241,0.6882380919620981,0.645388563624089 +2022-04-18,0.304818623987116,0.601409275693432,0.07904084882979401,0.16144847919011002 +2022-04-19,0.8929797309154061,0.932019259560644,0.013117573233606001,0.36918835535862005 +2022-04-20,4.074891060179412,3.6166710941152713,3.143179675624189,5.283666616893828 +2022-04-21,0.902423656543305,2.682237126638688,0.6180860661837431,1.631115014312209 +2022-04-22,0.868968791964932,0.329339479566271,0.001536382167654,0.807551152019862 +2022-04-23,0.598986391584271,0.923756524041317,0.27081187550692803,0.22998130489182 +2022-04-24,0.27570721725353503,0.6601580208066361,0.7774852739576821,1.107108162852159 +2022-04-25,0.38518667923696703,2.037027697748363,0.859959904015784,0.641738277028276 +2022-04-26,0.622049495688147,0.803863499657206,0.22244045641286703,1.320954321806144 +2022-04-27,3.540727101774814,0.651469202290988,0.509144429951741,0.20073460039620403 +2022-04-28,2.742513433824975,2.6893975459595643,0.13894846116144902,0.7415989059349031 +2022-04-29,1.621566806821747,0.9147556633717211,0.19686640023465302,1.66475441380813 +2022-04-30,2.352493741253538,3.463127996997362,2.931184732430775,1.792534569741266 +2022-05-01,0.023487873849160002,0.5573480476128151,0.544054544143018,0.35016352980012605 +2022-05-02,9.85187760463177,35.95040240058573,10.530894431942714,4.323328789531124 +2022-05-03,6.3835439206943,5.577920982650984,4.483541728043783,3.7796731596206943 +2022-05-04,2.897597920598689,5.106082630695584,8.400221891051206,8.018922971992662 +2022-05-05,0.147546385656207,0.040771286126692,0.6862921980279431,0.44745062012103404 +2022-05-06,4.668438202775402,0.23681078820699203,4.557825514906153,3.46073691075845 +2022-05-07,1.868897716467347,1.727448921279463,0.527497153125138,0.120868915583745 +2022-05-08,1.356875902669512,1.434740383567053,1.25226701190186,0.42909048781311704 +2022-05-09,1.029882775112869,0.9377192673786531,0.643077700672811,0.539461571476962 +2022-05-10,1.079419585437788,0.5840348488590831,0.031193331319320002,0.341612265371036 +2022-05-11,0.43374835866466704,0.01643698693241,0.39325414355653304,0.918926460102595 +2022-05-12,0.6884004244125861,1.538028843999251,0.863652647240271,1.249717355592488 +2022-05-13,0.5513219132650721,1.063801968030616,0.65309749602536,1.244646630025216 +2022-05-14,0.26075436773203503,0.16980099784180802,0.502507081730036,0.026734770744615003 +2022-05-15,3.534068370760441,2.144086432206532,1.989727381483406,1.510725845099207 +2022-05-16,1.051684273070894,1.152664833294427,0.747391405264186,0.283792059973869 +2022-05-17,0.8574030868970931,0.8751363341872681,0.901758308356248,0.41795473321335 +2022-05-18,0.515497463894879,0.8615187586650901,0.43634054682577406,1.895212820019219 +2022-05-19,0.9592457438221671,0.126354125202296,2.8539304550226543,0.77105240170235 +2022-05-20,0.069882250377393,0.7098481312166951,0.5071020864029481,2.543277547025898 +2022-05-21,10.360352568974609,17.706030110118455,2.848137759152337,1.260088014509166 +2022-05-22,1.112796751785197,0.9176470910577301,0.43003615544265306,0.910155627802492 +2022-05-23,4.891916669440483,16.47452431770853,2.744192036280677,1.124249996351633 +2022-05-24,0.09534096170855301,0.7854768368577231,0.6263662592501831,0.19409864063613502 +2022-05-25,17.21160548866725,34.373177238596014,7.564216276703391,17.06769270843336 +2022-05-26,0.9890377076819251,0.789535076533531,0.8798790088553111,0.592040366427117 +2022-05-27,1.207578684423282,0.5312851260232421,1.055373428020675,0.84736634769268 +2022-05-28,15.265265813052475,12.028735185645875,14.171624745847268,35.787490090751945 +2022-05-29,1.667919015486645,1.439873268216485,1.078931580631559,0.9907508552960291 +2022-05-30,2.232083847244808,3.827653889831506,1.429842282923971,3.76351508114206 +2022-05-31,1.113011174076138,1.4295696272440521,1.141933003322777,1.22202135537901 +2022-06-01,0.9839213173080511,1.325995589464272,2.006909504383298,1.7803529363753632 +2022-06-02,3.618368394111176,0.627748977148721,1.176985400742372,1.267728131617331 +2022-06-03,1.5240830060491182,0.83151391784571,1.356692075891955,3.227340286052029 +2022-06-04,0.7108355579615261,0.011530713721830002,0.52485737802621,0.661668615725415 +2022-06-05,1.842738679408876,1.130871471288951,1.189107233207374,3.246025623322408 +2022-06-06,2.968986726243696,2.9958102236757282,0.47619309340284505,1.3335764819663671 +2022-06-07,0.30957106038382504,0.44026116440941004,0.42982954277120905,1.741430660250139 +2022-06-08,3.077870353397237,4.156128784010092,1.021512025143279,4.351094223144934 +2022-06-09,0.460474051903736,1.2828566287818242,0.22198861960947303,0.085629074470341 +2022-06-10,0.9248894668614721,0.23152297348937,0.18500236928827601,0.5925681882904861 +2022-06-11,0.5170229174882011,1.344093064443273,0.6923204827335401,1.013151233597585 +2022-06-12,1.343310704782682,0.42108731154009404,1.547163500435265,0.999128791000602 +2022-06-13,4.213284444567541,2.448703044357372,0.13900479867391702,1.556248326348268 +2022-06-14,0.718083714760927,1.912454128957958,0.24702549770442003,2.037101388253879 +2022-06-15,2.443370806389791,0.010368540480550002,0.133167401519793,2.42241572339565 +2022-06-16,3.594231845735735,0.6677966564200211,4.009094569074379,5.5253640045926335 +2022-06-17,1.113951398490715,0.11111864432229301,0.9730098334869651,0.9908309309721931 +2022-06-18,0.5614439322266921,0.806546956486733,0.32013564834811403,0.8486918887953661 +2022-06-19,1.243458727745012,1.135192118212662,5.021267051723119,4.233329079605447 +2022-06-20,1.31005897849395,1.045144309632654,4.014478741193049,2.239616259510069 +2022-06-21,0.572744269459053,1.076475248295982,0.7240389412682761,0.10908279405984601 +2022-06-22,24.581669047338767,24.02021682920061,37.05462241129788,23.51884099646193 +2022-06-23,0.47413969565051606,0.027051327125384002,0.16824850993617801,0.41568301223291604 +2022-06-24,0.43416765009011604,2.878601660092219,1.9317197675603461,1.840590020030168 +2022-06-25,0.943163264857342,0.019465090936308,0.285177476177572,0.5532414004550751 +2022-06-26,0.233205152479134,1.2557471231721191,1.260075716991639,2.715172232506665 +2022-06-27,3.9663605369632933,0.5849212901405391,3.452510757244162,2.679757899115613 +2022-06-28,0.016605232459512002,0.280224773927022,0.729523367081325,1.105137522496868 +2022-06-29,1.374134760412724,0.751739223993092,0.7538963415227941,1.347929408272255 +2022-06-30,0.715486103147415,0.555444809676618,0.08646395240381301,0.5491110504617991 +2022-07-01,0.747532331413809,1.043273887199711,0.49779032149353303,0.885618995495125 +2022-07-02,0.8670766655477491,1.8724388596735242,0.391981409036455,1.040394051716164 +2022-07-03,0.392542368674532,1.204834846410795,1.103562611606201,0.387838034146796 +2022-07-04,2.354765436840968,0.147183599628016,1.798064276315111,0.33504623462675404 +2022-07-05,0.25059554847566,0.259954349350673,0.407469349519874,3.278091213717644 +2022-07-06,4.720665637767317,2.071271452074246,4.502139668053687,5.134166854414464 +2022-07-07,0.646476205877439,0.186466306558064,0.499133613586184,0.15787863433545202 +2022-07-08,4.428354777598469,7.997827905255312,5.845195208267132,3.695856221993309 +2022-07-09,1.502467804494858,0.33362085354833704,1.26505092951483,0.9204718402028381 +2022-07-10,11.706734521216822,36.550441521853145,19.095991246701065,12.292686689327372 +2022-07-11,1.153083534578196,0.337559642243531,1.154651513638781,0.22569823789057203 +2022-07-12,1.4863362021332551,0.48307852351030106,1.5365158530965122,0.057003071478754004 +2022-07-13,1.61896686341972,0.9387594299650791,1.092431878065284,1.886305858547745 +2022-07-14,49.60818750100216,22.940238837323303,13.579325095732914,38.67416013838181 +2022-07-15,1.785205883890873,0.631304013521069,0.62869350368483,0.34417591219448 +2022-07-16,0.715594243310775,0.984028805544615,0.967817534607272,0.153576028704162 +2022-07-17,0.8147430610328651,0.26284189122074103,2.154865858339681,1.283391109459497 +2022-07-18,1.24545618431753,1.082826448671828,0.45266260254768803,0.084950117524903 +2022-07-19,0.083915091025848,0.5880718113215341,1.007179376918947,0.865719136965563 +2022-07-20,0.32701962372413,0.17041520446782002,1.773905435681336,1.104677623595627 +2022-07-21,0.722041044814348,0.822598941600809,0.9593067917284711,0.41221237294221 +2022-07-22,10.662811760936936,0.183634003906628,13.34157103707886,12.159275730149757 +2022-07-23,0.364677770561312,0.559247300094862,0.40653075149155804,0.9694788164354201 +2022-07-24,0.9157754804402811,1.091835974661574,0.8199135063617131,1.062993342271482 +2022-07-25,0.67849525511577,3.048531280160542,1.471231045142114,0.591416124923933 +2022-07-26,1.330066578741473,1.582638082613582,1.789945856913414,2.192979954983851 +2022-07-27,5.686664774275858,6.436173309221049,4.7854634352690155,1.63427958962943 +2022-07-28,0.22324041916141402,1.428578028660132,1.458686942176205,1.311736714260567 +2022-07-29,2.476037411183793,0.7249411790805861,0.42039676295059203,2.995565219170837 +2022-07-30,0.773637653847835,0.9265499876312321,0.32494629315872703,0.324341917122049 +2022-07-31,0.169152723468269,0.9014047551998651,0.46982772647431204,0.5534444716525361 +2022-08-01,1.519627178758216,1.610068186750942,3.309159861747862,0.529330910775445 +2022-08-02,3.545652606242915,2.330031081649702,3.9640154638953513,3.856552869039815 +2022-08-03,1.3404062798488199,0.462815961543229,0.9670138622828691,1.234951845374218 +2022-08-04,0.6530263834012761,0.08597037675761801,0.9707805498085981,0.16192153323070702 +2022-08-05,0.708729075421144,3.581243043200601,2.651870015387795,2.173110201699371 +2022-08-06,11.986631123295103,10.930286916614767,22.78182877052594,11.135752238663327 +2022-08-07,1.232939713701664,1.8029857915415932,1.476317514220094,0.958792084262276 +2022-08-08,0.530813431405317,2.014631665755229,2.7977280500624433,2.039075329513739 +2022-08-09,2.510188659097596,2.983415297830412,0.126105493389976,2.121594836643942 +2022-08-10,1.622456744339298,1.289582374551249,2.5475660513369442,3.282862603860423 +2022-08-11,0.25962103068208003,0.37516694909806303,0.5828588359439041,0.139231895746797 +2022-08-12,0.884113805589634,1.603259620438238,1.8751590471743,0.24122279792940302 +2022-08-13,8.624685230360152,3.3791466219930433,9.375119750363103,0.7506624109540441 +2022-08-14,1.6498217904306332,2.627126072551769,0.57102589263998,2.4024472876658223 +2022-08-15,5.840124155766275,3.354483317899366,1.075590808840468,2.3650336028113492 +2022-08-16,2.405030143824614,1.444879535922397,0.942851527789979,1.423465983705226 +2022-08-17,1.034391306438299,0.6369673400603011,0.473080126014195,0.8585472144660361 +2022-08-18,4.703412413763946,0.5990722583580821,0.911179769434693,1.189643421133775 +2022-08-19,0.22138733877979702,0.24079634787311602,0.045592515857608006,0.9837464357223601 +2022-08-20,0.14621922149563202,0.096627004274573,0.7910843611370051,0.059870034265504 +2022-08-21,5.702011116768989,11.782408392531364,4.022381962144387,12.55656527594101 +2022-08-22,2.289251748782675,1.846139752930779,1.302218066447416,1.195140815195466 +2022-08-23,0.40156922050687605,2.44670175499675,2.701336009126233,2.5846045653914143 +2022-08-24,16.368686321533847,9.285256494681935,14.17816605948454,6.813674272547164 +2022-08-25,17.325634435909638,40.63547963350666,52.36627420733006,22.12581500864155 +2022-08-26,2.771994472181008,1.432898057334553,0.500652317272802,0.29065103217099203 +2022-08-27,2.705080627435551,1.140228551443707,2.535947091285437,2.068004826572326 +2022-08-28,0.366184071525715,0.37561791103581305,1.087399555743408,1.3431857507096732 +2022-08-29,0.192969279119436,1.203078886727172,0.8966719923838661,1.07779591413233 +2022-08-30,0.248116517168076,0.901524977902984,0.9660257636136701,0.13247782324235202 +2022-08-31,0.7172605174280461,0.22721632248943202,0.9247009404760361,0.28572976740870504 +2022-09-01,0.259653267575217,2.040817156148029,2.19857288799284,2.229208371156883 +2022-09-02,1.398428414171018,0.071469482611002,0.9048807067534731,0.0022491420541680004 +2022-09-03,1.7166677805176591,1.903668070163285,1.866568462888393,1.8648831840830011`, + input: '请使用组合图展示不同类别的权重随着时间的变化' +}; + +export const singleColumnLineCombinationChartData1 = { + csv: `date,Social Penetration,Engagement - Socialization,Penetration of Private Messages,Number of Private Messages per User +2022-03-08,1.2020804451630671,0.7782279444864411,0.21493020207806002,0.26002007727513005 +2022-03-09,1.911162758594358,0.6970763116149991,0.31807068769079905,1.791161354352144 +2022-03-10,0.919293523406533,0.6754510401577041,0.9774597853017851,1.410289280738493 +2022-03-11,1.613617931982911,4.391274420824463,0.662196540154462,2.374084690062705 +2022-03-12,1.121445896148114,0.5827973501093221,1.547704220622789,0.42545370377775604 +2022-03-13,0.527369749651032,0.321443063009525,0.52384329811226,0.45119753243186805 +2022-03-14,0.07991167814791,0.9838146243033221,1.242674929077965,1.276345431333586 +2022-03-15,0.29341332300751205,0.225492098355163,1.133076821811672,0.22045142924559902 +2022-03-16,1.103150404211427,0.580792120371675,1.261446251860683,0.061775812702848 +2022-03-17,0.38285701953593604,0.510579334822959,0.908573021920086,0.429194248082692 +2022-03-18,0.8908005339482421,1.656998380732747,2.003638269088651,1.194511111230195 +2022-03-19,0.8379933951719881,1.372238730132033,1.285574252451237,0.659056833995271 +2022-03-20,2.913059895355856,3.375041306488803,1.683935730207101,0.40361011204949004 +2022-03-21,1.245523794769467,0.5770535812918991,0.518960506573594,0.8441533797310441 +2022-03-22,1.801955367373218,2.562767955961243,2.076490748024919,1.8682050366367142 +2022-03-23,30.25170429439274,60.10096481599735,31.918470633985613,70.34237647545662 +2022-03-24,2.084831170978463,2.824195229461953,1.735738814795219,3.854924132202267 +2022-03-25,0.8300684679922911,0.37108301055861304,1.072942910906809,0.868965201778456 +2022-03-26,5.057522939533747,6.8131452572139555,10.31725741409195,0.6724835487477681 +2022-03-27,1.364245041731511,0.993289299105189,1.288157916081268,1.290710012048575 +2022-03-28,0.429699282814745,0.27225365805664004,0.24488975421611903,0.944202324553962 +2022-03-29,8.636960050981939,2.663368864789215,3.030888929145336,2.191548298261417 +2022-03-30,0.31233481070398905,1.136502929086737,1.4304534277803231,0.9398051933719991 +2022-03-31,0.8979576483673781,0.8268293005130081,1.32475784490384,0.212383798504713 +2022-04-01,1.187326287404249,1.2824253532703072,0.9289553576582981,1.328758303758777 +2022-04-02,0.013251966153457001,1.580091614771022,0.8470948142995741,1.061518688804395 +2022-04-03,0.150440921215681,0.7793382242254321,0.5908683646183901,1.021428732507816 +2022-04-04,1.437865757864698,0.34207119701235905,1.006103497118929,0.8496566144677861 +2022-04-05,19.70534531285437,4.144335050792541,13.880860231019398,19.78289952089912 +2022-04-06,1.068708512330189,1.336025709304907,7.121018094960213,0.032847589287524 +2022-04-07,3.239083103313066,0.9037359185383671,2.010136943230478,2.758258304057329 +2022-04-08,3.609669038457516,13.698767079174356,14.457401852798982,11.6964360828172 +2022-04-09,0.8581543303200471,1.11771091972957,0.9593788300193951,1.121066838670219 +2022-04-10,1.235338424517758,0.5546236200208831,1.138776207683746,0.5694312841615821 +2022-04-11,0.615560300356359,0.133505664394376,0.575680679073131,0.502679162912634 +2022-04-12,3.5445165395901572,8.458105165611851,2.90160000840929,3.084080663501847 +2022-04-13,1.037776502324676,1.412321112593621,1.416876192652653,0.270647644667579 +2022-04-14,0.534866532613768,0.577163861285873,0.034172330313952004,0.268470350264293 +2022-04-15,3.225952049483959,5.112791338669762,3.745099757241069,2.612195295941399 +2022-04-16,12.278592894400722,13.301401633991011,6.202762040985617,18.098567324029407 +2022-04-17,0.411460847135642,0.8194099242420241,0.6882380919620981,0.645388563624089 +2022-04-18,0.304818623987116,0.601409275693432,0.07904084882979401,0.16144847919011002 +2022-04-19,0.8929797309154061,0.932019259560644,0.013117573233606001,0.36918835535862005 +2022-04-20,4.074891060179412,3.6166710941152713,3.143179675624189,5.283666616893828 +2022-04-21,0.902423656543305,2.682237126638688,0.6180860661837431,1.631115014312209 +2022-04-22,0.868968791964932,0.329339479566271,0.001536382167654,0.807551152019862 +2022-04-23,0.598986391584271,0.923756524041317,0.27081187550692803,0.22998130489182 +2022-04-24,0.27570721725353503,0.6601580208066361,0.7774852739576821,1.107108162852159 +2022-04-25,0.38518667923696703,2.037027697748363,0.859959904015784,0.641738277028276 +2022-04-26,0.622049495688147,0.803863499657206,0.22244045641286703,1.320954321806144 +2022-04-27,3.540727101774814,0.651469202290988,0.509144429951741,0.20073460039620403 +2022-04-28,2.742513433824975,2.6893975459595643,0.13894846116144902,0.7415989059349031 +2022-04-29,1.621566806821747,0.9147556633717211,0.19686640023465302,1.66475441380813 +2022-04-30,2.352493741253538,3.463127996997362,2.931184732430775,1.792534569741266 +2022-05-01,0.023487873849160002,0.5573480476128151,0.544054544143018,0.35016352980012605 +2022-05-02,9.85187760463177,35.95040240058573,10.530894431942714,4.323328789531124 +2022-05-03,6.3835439206943,5.577920982650984,4.483541728043783,3.7796731596206943 +2022-05-04,2.897597920598689,5.106082630695584,8.400221891051206,8.018922971992662 +2022-05-05,0.147546385656207,0.040771286126692,0.6862921980279431,0.44745062012103404 +2022-05-06,4.668438202775402,0.23681078820699203,4.557825514906153,3.46073691075845 +2022-05-07,1.868897716467347,1.727448921279463,0.527497153125138,0.120868915583745 +2022-05-08,1.356875902669512,1.434740383567053,1.25226701190186,0.42909048781311704 +2022-05-09,1.029882775112869,0.9377192673786531,0.643077700672811,0.539461571476962 +2022-05-10,1.079419585437788,0.5840348488590831,0.031193331319320002,0.341612265371036 +2022-05-11,0.43374835866466704,0.01643698693241,0.39325414355653304,0.918926460102595 +2022-05-12,0.6884004244125861,1.538028843999251,0.863652647240271,1.249717355592488 +2022-05-13,0.5513219132650721,1.063801968030616,0.65309749602536,1.244646630025216 +2022-05-14,0.26075436773203503,0.16980099784180802,0.502507081730036,0.026734770744615003 +2022-05-15,3.534068370760441,2.144086432206532,1.989727381483406,1.510725845099207 +2022-05-16,1.051684273070894,1.152664833294427,0.747391405264186,0.283792059973869 +2022-05-17,0.8574030868970931,0.8751363341872681,0.901758308356248,0.41795473321335 +2022-05-18,0.515497463894879,0.8615187586650901,0.43634054682577406,1.895212820019219 +2022-05-19,0.9592457438221671,0.126354125202296,2.8539304550226543,0.77105240170235 +2022-05-20,0.069882250377393,0.7098481312166951,0.5071020864029481,2.543277547025898 +2022-05-21,10.360352568974609,17.706030110118455,2.848137759152337,1.260088014509166 +2022-05-22,1.112796751785197,0.9176470910577301,0.43003615544265306,0.910155627802492 +2022-05-23,4.891916669440483,16.47452431770853,2.744192036280677,1.124249996351633 +2022-05-24,0.09534096170855301,0.7854768368577231,0.6263662592501831,0.19409864063613502 +2022-05-25,17.21160548866725,34.373177238596014,7.564216276703391,17.06769270843336 +2022-05-26,0.9890377076819251,0.789535076533531,0.8798790088553111,0.592040366427117 +2022-05-27,1.207578684423282,0.5312851260232421,1.055373428020675,0.84736634769268 +2022-05-28,15.265265813052475,12.028735185645875,14.171624745847268,35.787490090751945 +2022-05-29,1.667919015486645,1.439873268216485,1.078931580631559,0.9907508552960291 +2022-05-30,2.232083847244808,3.827653889831506,1.429842282923971,3.76351508114206 +2022-05-31,1.113011174076138,1.4295696272440521,1.141933003322777,1.22202135537901 +2022-06-01,0.9839213173080511,1.325995589464272,2.006909504383298,1.7803529363753632 +2022-06-02,3.618368394111176,0.627748977148721,1.176985400742372,1.267728131617331 +2022-06-03,1.5240830060491182,0.83151391784571,1.356692075891955,3.227340286052029 +2022-06-04,0.7108355579615261,0.011530713721830002,0.52485737802621,0.661668615725415 +2022-06-05,1.842738679408876,1.130871471288951,1.189107233207374,3.246025623322408 +2022-06-06,2.968986726243696,2.9958102236757282,0.47619309340284505,1.3335764819663671 +2022-06-07,0.30957106038382504,0.44026116440941004,0.42982954277120905,1.741430660250139 +2022-06-08,3.077870353397237,4.156128784010092,1.021512025143279,4.351094223144934 +2022-06-09,0.460474051903736,1.2828566287818242,0.22198861960947303,0.085629074470341 +2022-06-10,0.9248894668614721,0.23152297348937,0.18500236928827601,0.5925681882904861 +2022-06-11,0.5170229174882011,1.344093064443273,0.6923204827335401,1.013151233597585 +2022-06-12,1.343310704782682,0.42108731154009404,1.547163500435265,0.999128791000602 +2022-06-13,4.213284444567541,2.448703044357372,0.13900479867391702,1.556248326348268 +2022-06-14,0.718083714760927,1.912454128957958,0.24702549770442003,2.037101388253879 +2022-06-15,2.443370806389791,0.010368540480550002,0.133167401519793,2.42241572339565 +2022-06-16,3.594231845735735,0.6677966564200211,4.009094569074379,5.5253640045926335 +2022-06-17,1.113951398490715,0.11111864432229301,0.9730098334869651,0.9908309309721931 +2022-06-18,0.5614439322266921,0.806546956486733,0.32013564834811403,0.8486918887953661 +2022-06-19,1.243458727745012,1.135192118212662,5.021267051723119,4.233329079605447 +2022-06-20,1.31005897849395,1.045144309632654,4.014478741193049,2.239616259510069 +2022-06-21,0.572744269459053,1.076475248295982,0.7240389412682761,0.10908279405984601 +2022-06-22,24.581669047338767,24.02021682920061,37.05462241129788,23.51884099646193 +2022-06-23,0.47413969565051606,0.027051327125384002,0.16824850993617801,0.41568301223291604 +2022-06-24,0.43416765009011604,2.878601660092219,1.9317197675603461,1.840590020030168 +2022-06-25,0.943163264857342,0.019465090936308,0.285177476177572,0.5532414004550751 +2022-06-26,0.233205152479134,1.2557471231721191,1.260075716991639,2.715172232506665 +2022-06-27,3.9663605369632933,0.5849212901405391,3.452510757244162,2.679757899115613 +2022-06-28,0.016605232459512002,0.280224773927022,0.729523367081325,1.105137522496868 +2022-06-29,1.374134760412724,0.751739223993092,0.7538963415227941,1.347929408272255 +2022-06-30,0.715486103147415,0.555444809676618,0.08646395240381301,0.5491110504617991 +2022-07-01,0.747532331413809,1.043273887199711,0.49779032149353303,0.885618995495125 +2022-07-02,0.8670766655477491,1.8724388596735242,0.391981409036455,1.040394051716164 +2022-07-03,0.392542368674532,1.204834846410795,1.103562611606201,0.387838034146796 +2022-07-04,2.354765436840968,0.147183599628016,1.798064276315111,0.33504623462675404 +2022-07-05,0.25059554847566,0.259954349350673,0.407469349519874,3.278091213717644 +2022-07-06,4.720665637767317,2.071271452074246,4.502139668053687,5.134166854414464 +2022-07-07,0.646476205877439,0.186466306558064,0.499133613586184,0.15787863433545202 +2022-07-08,4.428354777598469,7.997827905255312,5.845195208267132,3.695856221993309 +2022-07-09,1.502467804494858,0.33362085354833704,1.26505092951483,0.9204718402028381 +2022-07-10,11.706734521216822,36.550441521853145,19.095991246701065,12.292686689327372 +2022-07-11,1.153083534578196,0.337559642243531,1.154651513638781,0.22569823789057203 +2022-07-12,1.4863362021332551,0.48307852351030106,1.5365158530965122,0.057003071478754004 +2022-07-13,1.61896686341972,0.9387594299650791,1.092431878065284,1.886305858547745 +2022-07-14,49.60818750100216,22.940238837323303,13.579325095732914,38.67416013838181 +2022-07-15,1.785205883890873,0.631304013521069,0.62869350368483,0.34417591219448 +2022-07-16,0.715594243310775,0.984028805544615,0.967817534607272,0.153576028704162 +2022-07-17,0.8147430610328651,0.26284189122074103,2.154865858339681,1.283391109459497 +2022-07-18,1.24545618431753,1.082826448671828,0.45266260254768803,0.084950117524903 +2022-07-19,0.083915091025848,0.5880718113215341,1.007179376918947,0.865719136965563 +2022-07-20,0.32701962372413,0.17041520446782002,1.773905435681336,1.104677623595627 +2022-07-21,0.722041044814348,0.822598941600809,0.9593067917284711,0.41221237294221 +2022-07-22,10.662811760936936,0.183634003906628,13.34157103707886,12.159275730149757 +2022-07-23,0.364677770561312,0.559247300094862,0.40653075149155804,0.9694788164354201 +2022-07-24,0.9157754804402811,1.091835974661574,0.8199135063617131,1.062993342271482 +2022-07-25,0.67849525511577,3.048531280160542,1.471231045142114,0.591416124923933 +2022-07-26,1.330066578741473,1.582638082613582,1.789945856913414,2.192979954983851 +2022-07-27,5.686664774275858,6.436173309221049,4.7854634352690155,1.63427958962943 +2022-07-28,0.22324041916141402,1.428578028660132,1.458686942176205,1.311736714260567 +2022-07-29,2.476037411183793,0.7249411790805861,0.42039676295059203,2.995565219170837 +2022-07-30,0.773637653847835,0.9265499876312321,0.32494629315872703,0.324341917122049 +2022-07-31,0.169152723468269,0.9014047551998651,0.46982772647431204,0.5534444716525361 +2022-08-01,1.519627178758216,1.610068186750942,3.309159861747862,0.529330910775445 +2022-08-02,3.545652606242915,2.330031081649702,3.9640154638953513,3.856552869039815 +2022-08-03,1.3404062798488199,0.462815961543229,0.9670138622828691,1.234951845374218 +2022-08-04,0.6530263834012761,0.08597037675761801,0.9707805498085981,0.16192153323070702 +2022-08-05,0.708729075421144,3.581243043200601,2.651870015387795,2.173110201699371 +2022-08-06,11.986631123295103,10.930286916614767,22.78182877052594,11.135752238663327 +2022-08-07,1.232939713701664,1.8029857915415932,1.476317514220094,0.958792084262276 +2022-08-08,0.530813431405317,2.014631665755229,2.7977280500624433,2.039075329513739 +2022-08-09,2.510188659097596,2.983415297830412,0.126105493389976,2.121594836643942 +2022-08-10,1.622456744339298,1.289582374551249,2.5475660513369442,3.282862603860423 +2022-08-11,0.25962103068208003,0.37516694909806303,0.5828588359439041,0.139231895746797 +2022-08-12,0.884113805589634,1.603259620438238,1.8751590471743,0.24122279792940302 +2022-08-13,8.624685230360152,3.3791466219930433,9.375119750363103,0.7506624109540441 +2022-08-14,1.6498217904306332,2.627126072551769,0.57102589263998,2.4024472876658223 +2022-08-15,5.840124155766275,3.354483317899366,1.075590808840468,2.3650336028113492 +2022-08-16,2.405030143824614,1.444879535922397,0.942851527789979,1.423465983705226 +2022-08-17,1.034391306438299,0.6369673400603011,0.473080126014195,0.8585472144660361 +2022-08-18,4.703412413763946,0.5990722583580821,0.911179769434693,1.189643421133775 +2022-08-19,0.22138733877979702,0.24079634787311602,0.045592515857608006,0.9837464357223601 +2022-08-20,0.14621922149563202,0.096627004274573,0.7910843611370051,0.059870034265504 +2022-08-21,5.702011116768989,11.782408392531364,4.022381962144387,12.55656527594101 +2022-08-22,2.289251748782675,1.846139752930779,1.302218066447416,1.195140815195466 +2022-08-23,0.40156922050687605,2.44670175499675,2.701336009126233,2.5846045653914143 +2022-08-24,16.368686321533847,9.285256494681935,14.17816605948454,6.813674272547164 +2022-08-25,17.325634435909638,40.63547963350666,52.36627420733006,22.12581500864155 +2022-08-26,2.771994472181008,1.432898057334553,0.500652317272802,0.29065103217099203 +2022-08-27,2.705080627435551,1.140228551443707,2.535947091285437,2.068004826572326 +2022-08-28,0.366184071525715,0.37561791103581305,1.087399555743408,1.3431857507096732 +2022-08-29,0.192969279119436,1.203078886727172,0.8966719923838661,1.07779591413233 +2022-08-30,0.248116517168076,0.901524977902984,0.9660257636136701,0.13247782324235202 +2022-08-31,0.7172605174280461,0.22721632248943202,0.9247009404760361,0.28572976740870504 +2022-09-01,0.259653267575217,2.040817156148029,2.19857288799284,2.229208371156883 +2022-09-02,1.398428414171018,0.071469482611002,0.9048807067534731,0.0022491420541680004 +2022-09-03,1.7166677805176591,1.903668070163285,1.866568462888393,1.8648831840830011`, + input: + '请使用组合图分别展示Social Penetration和Engagement - Socialization类别的权重随着时间的变化，以及Penetration of Private Messages和Number of Private Messages per User类别的权重随着时间的变化' +}; + +export const singleColumnPieCombinationChartData = { + csv: `姓名,时间投入量,资金投入量,劳动投入量 +小明,120,80,123,10 +小李,200,20,456,8 +小呆,100,400,789,1 +`, + input: '分开展示每位同学在各时间、资金、劳动投入中的占比情况。使用饼图组合绘制图表。' +}; + +export const singleColumnPieAndLineCombinationChartData = { + csv: `性别,数量,平均分 +男,10,93, +女,20,89`, + input: '使用饼图和柱图的组合图分别展示班级内男生女生的人数占比率，以及班级男生和女生各自的平均分' +}; + +export const singleColumnBarCombinationChartData = { + csv: `region,可乐销售额,雪碧销售额,芬达销售额,醒目销售额 +south,2350,215,345,1476 +east,1027,654,654,830 +west,1027,159,2100,532 +north,1027,28,1679,498 +`, + input: '帮我使用柱状组合图展示不同区域各商品销售额' +}; + +export const singleColumnBarCombinationChartData1 = { + csv: `date,Social Penetration,Engagement - Socialization,Penetration of Private Messages,Number of Private Messages per User +2022-03-08,1.2020804451630671,0.7782279444864411,0.21493020207806002,0.26002007727513005 +2022-03-09,1.911162758594358,0.6970763116149991,0.31807068769079905,1.791161354352144 +2022-03-10,0.919293523406533,0.6754510401577041,0.9774597853017851,1.410289280738493 +2022-03-11,1.613617931982911,4.391274420824463,0.662196540154462,2.374084690062705 +2022-03-12,1.121445896148114,0.5827973501093221,1.547704220622789,0.42545370377775604 +2022-03-13,0.527369749651032,0.321443063009525,0.52384329811226,0.45119753243186805 +2022-03-14,0.07991167814791,0.9838146243033221,1.242674929077965,1.276345431333586 +2022-03-15,0.29341332300751205,0.225492098355163,1.133076821811672,0.22045142924559902 +2022-03-16,1.103150404211427,0.580792120371675,1.261446251860683,0.061775812702848 +2022-03-17,0.38285701953593604,0.510579334822959,0.908573021920086,0.429194248082692 +2022-03-18,0.8908005339482421,1.656998380732747,2.003638269088651,1.194511111230195 +2022-03-19,0.8379933951719881,1.372238730132033,1.285574252451237,0.659056833995271 +2022-03-20,2.913059895355856,3.375041306488803,1.683935730207101,0.40361011204949004 +2022-03-21,1.245523794769467,0.5770535812918991,0.518960506573594,0.8441533797310441 +2022-03-22,1.801955367373218,2.562767955961243,2.076490748024919,1.8682050366367142 +2022-03-23,30.25170429439274,60.10096481599735,31.918470633985613,70.34237647545662 +2022-03-24,2.084831170978463,2.824195229461953,1.735738814795219,3.854924132202267 +2022-03-25,0.8300684679922911,0.37108301055861304,1.072942910906809,0.868965201778456 +2022-03-26,5.057522939533747,6.8131452572139555,10.31725741409195,0.6724835487477681 +2022-03-27,1.364245041731511,0.993289299105189,1.288157916081268,1.290710012048575 +2022-03-28,0.429699282814745,0.27225365805664004,0.24488975421611903,0.944202324553962 +2022-03-29,8.636960050981939,2.663368864789215,3.030888929145336,2.191548298261417 +2022-03-30,0.31233481070398905,1.136502929086737,1.4304534277803231,0.9398051933719991 +2022-03-31,0.8979576483673781,0.8268293005130081,1.32475784490384,0.212383798504713 +2022-04-01,1.187326287404249,1.2824253532703072,0.9289553576582981,1.328758303758777 +2022-04-02,0.013251966153457001,1.580091614771022,0.8470948142995741,1.061518688804395 +2022-04-03,0.150440921215681,0.7793382242254321,0.5908683646183901,1.021428732507816 +2022-04-04,1.437865757864698,0.34207119701235905,1.006103497118929,0.8496566144677861 +2022-04-05,19.70534531285437,4.144335050792541,13.880860231019398,19.78289952089912 +2022-04-06,1.068708512330189,1.336025709304907,7.121018094960213,0.032847589287524 +2022-04-07,3.239083103313066,0.9037359185383671,2.010136943230478,2.758258304057329 +2022-04-08,3.609669038457516,13.698767079174356,14.457401852798982,11.6964360828172 +2022-04-09,0.8581543303200471,1.11771091972957,0.9593788300193951,1.121066838670219 +2022-04-10,1.235338424517758,0.5546236200208831,1.138776207683746,0.5694312841615821 +2022-04-11,0.615560300356359,0.133505664394376,0.575680679073131,0.502679162912634 +2022-04-12,3.5445165395901572,8.458105165611851,2.90160000840929,3.084080663501847 +2022-04-13,1.037776502324676,1.412321112593621,1.416876192652653,0.270647644667579 +2022-04-14,0.534866532613768,0.577163861285873,0.034172330313952004,0.268470350264293 +2022-04-15,3.225952049483959,5.112791338669762,3.745099757241069,2.612195295941399 +2022-04-16,12.278592894400722,13.301401633991011,6.202762040985617,18.098567324029407 +2022-04-17,0.411460847135642,0.8194099242420241,0.6882380919620981,0.645388563624089 +2022-04-18,0.304818623987116,0.601409275693432,0.07904084882979401,0.16144847919011002 +2022-04-19,0.8929797309154061,0.932019259560644,0.013117573233606001,0.36918835535862005 +2022-04-20,4.074891060179412,3.6166710941152713,3.143179675624189,5.283666616893828 +2022-04-21,0.902423656543305,2.682237126638688,0.6180860661837431,1.631115014312209 +2022-04-22,0.868968791964932,0.329339479566271,0.001536382167654,0.807551152019862 +2022-04-23,0.598986391584271,0.923756524041317,0.27081187550692803,0.22998130489182 +2022-04-24,0.27570721725353503,0.6601580208066361,0.7774852739576821,1.107108162852159 +2022-04-25,0.38518667923696703,2.037027697748363,0.859959904015784,0.641738277028276 +2022-04-26,0.622049495688147,0.803863499657206,0.22244045641286703,1.320954321806144 +2022-04-27,3.540727101774814,0.651469202290988,0.509144429951741,0.20073460039620403 +2022-04-28,2.742513433824975,2.6893975459595643,0.13894846116144902,0.7415989059349031 +2022-04-29,1.621566806821747,0.9147556633717211,0.19686640023465302,1.66475441380813 +2022-04-30,2.352493741253538,3.463127996997362,2.931184732430775,1.792534569741266 +2022-05-01,0.023487873849160002,0.5573480476128151,0.544054544143018,0.35016352980012605 +2022-05-02,9.85187760463177,35.95040240058573,10.530894431942714,4.323328789531124 +2022-05-03,6.3835439206943,5.577920982650984,4.483541728043783,3.7796731596206943 +2022-05-04,2.897597920598689,5.106082630695584,8.400221891051206,8.018922971992662 +2022-05-05,0.147546385656207,0.040771286126692,0.6862921980279431,0.44745062012103404 +2022-05-06,4.668438202775402,0.23681078820699203,4.557825514906153,3.46073691075845 +2022-05-07,1.868897716467347,1.727448921279463,0.527497153125138,0.120868915583745 +2022-05-08,1.356875902669512,1.434740383567053,1.25226701190186,0.42909048781311704 +2022-05-09,1.029882775112869,0.9377192673786531,0.643077700672811,0.539461571476962 +2022-05-10,1.079419585437788,0.5840348488590831,0.031193331319320002,0.341612265371036 +2022-05-11,0.43374835866466704,0.01643698693241,0.39325414355653304,0.918926460102595 +2022-05-12,0.6884004244125861,1.538028843999251,0.863652647240271,1.249717355592488 +2022-05-13,0.5513219132650721,1.063801968030616,0.65309749602536,1.244646630025216 +2022-05-14,0.26075436773203503,0.16980099784180802,0.502507081730036,0.026734770744615003 +2022-05-15,3.534068370760441,2.144086432206532,1.989727381483406,1.510725845099207 +2022-05-16,1.051684273070894,1.152664833294427,0.747391405264186,0.283792059973869 +2022-05-17,0.8574030868970931,0.8751363341872681,0.901758308356248,0.41795473321335 +2022-05-18,0.515497463894879,0.8615187586650901,0.43634054682577406,1.895212820019219 +2022-05-19,0.9592457438221671,0.126354125202296,2.8539304550226543,0.77105240170235 +2022-05-20,0.069882250377393,0.7098481312166951,0.5071020864029481,2.543277547025898 +2022-05-21,10.360352568974609,17.706030110118455,2.848137759152337,1.260088014509166 +2022-05-22,1.112796751785197,0.9176470910577301,0.43003615544265306,0.910155627802492 +2022-05-23,4.891916669440483,16.47452431770853,2.744192036280677,1.124249996351633 +2022-05-24,0.09534096170855301,0.7854768368577231,0.6263662592501831,0.19409864063613502 +2022-05-25,17.21160548866725,34.373177238596014,7.564216276703391,17.06769270843336 +2022-05-26,0.9890377076819251,0.789535076533531,0.8798790088553111,0.592040366427117 +2022-05-27,1.207578684423282,0.5312851260232421,1.055373428020675,0.84736634769268 +2022-05-28,15.265265813052475,12.028735185645875,14.171624745847268,35.787490090751945 +2022-05-29,1.667919015486645,1.439873268216485,1.078931580631559,0.9907508552960291 +2022-05-30,2.232083847244808,3.827653889831506,1.429842282923971,3.76351508114206 +2022-05-31,1.113011174076138,1.4295696272440521,1.141933003322777,1.22202135537901 +2022-06-01,0.9839213173080511,1.325995589464272,2.006909504383298,1.7803529363753632 +2022-06-02,3.618368394111176,0.627748977148721,1.176985400742372,1.267728131617331 +2022-06-03,1.5240830060491182,0.83151391784571,1.356692075891955,3.227340286052029 +2022-06-04,0.7108355579615261,0.011530713721830002,0.52485737802621,0.661668615725415 +2022-06-05,1.842738679408876,1.130871471288951,1.189107233207374,3.246025623322408 +2022-06-06,2.968986726243696,2.9958102236757282,0.47619309340284505,1.3335764819663671 +2022-06-07,0.30957106038382504,0.44026116440941004,0.42982954277120905,1.741430660250139 +2022-06-08,3.077870353397237,4.156128784010092,1.021512025143279,4.351094223144934 +2022-06-09,0.460474051903736,1.2828566287818242,0.22198861960947303,0.085629074470341 +2022-06-10,0.9248894668614721,0.23152297348937,0.18500236928827601,0.5925681882904861 +2022-06-11,0.5170229174882011,1.344093064443273,0.6923204827335401,1.013151233597585 +2022-06-12,1.343310704782682,0.42108731154009404,1.547163500435265,0.999128791000602 +2022-06-13,4.213284444567541,2.448703044357372,0.13900479867391702,1.556248326348268 +2022-06-14,0.718083714760927,1.912454128957958,0.24702549770442003,2.037101388253879 +2022-06-15,2.443370806389791,0.010368540480550002,0.133167401519793,2.42241572339565 +2022-06-16,3.594231845735735,0.6677966564200211,4.009094569074379,5.5253640045926335 +2022-06-17,1.113951398490715,0.11111864432229301,0.9730098334869651,0.9908309309721931 +2022-06-18,0.5614439322266921,0.806546956486733,0.32013564834811403,0.8486918887953661 +2022-06-19,1.243458727745012,1.135192118212662,5.021267051723119,4.233329079605447 +2022-06-20,1.31005897849395,1.045144309632654,4.014478741193049,2.239616259510069 +2022-06-21,0.572744269459053,1.076475248295982,0.7240389412682761,0.10908279405984601 +2022-06-22,24.581669047338767,24.02021682920061,37.05462241129788,23.51884099646193 +2022-06-23,0.47413969565051606,0.027051327125384002,0.16824850993617801,0.41568301223291604 +2022-06-24,0.43416765009011604,2.878601660092219,1.9317197675603461,1.840590020030168 +2022-06-25,0.943163264857342,0.019465090936308,0.285177476177572,0.5532414004550751 +2022-06-26,0.233205152479134,1.2557471231721191,1.260075716991639,2.715172232506665 +2022-06-27,3.9663605369632933,0.5849212901405391,3.452510757244162,2.679757899115613 +2022-06-28,0.016605232459512002,0.280224773927022,0.729523367081325,1.105137522496868 +2022-06-29,1.374134760412724,0.751739223993092,0.7538963415227941,1.347929408272255 +2022-06-30,0.715486103147415,0.555444809676618,0.08646395240381301,0.5491110504617991 +2022-07-01,0.747532331413809,1.043273887199711,0.49779032149353303,0.885618995495125 +2022-07-02,0.8670766655477491,1.8724388596735242,0.391981409036455,1.040394051716164 +2022-07-03,0.392542368674532,1.204834846410795,1.103562611606201,0.387838034146796 +2022-07-04,2.354765436840968,0.147183599628016,1.798064276315111,0.33504623462675404 +2022-07-05,0.25059554847566,0.259954349350673,0.407469349519874,3.278091213717644 +2022-07-06,4.720665637767317,2.071271452074246,4.502139668053687,5.134166854414464 +2022-07-07,0.646476205877439,0.186466306558064,0.499133613586184,0.15787863433545202 +2022-07-08,4.428354777598469,7.997827905255312,5.845195208267132,3.695856221993309 +2022-07-09,1.502467804494858,0.33362085354833704,1.26505092951483,0.9204718402028381 +2022-07-10,11.706734521216822,36.550441521853145,19.095991246701065,12.292686689327372 +2022-07-11,1.153083534578196,0.337559642243531,1.154651513638781,0.22569823789057203 +2022-07-12,1.4863362021332551,0.48307852351030106,1.5365158530965122,0.057003071478754004 +2022-07-13,1.61896686341972,0.9387594299650791,1.092431878065284,1.886305858547745 +2022-07-14,49.60818750100216,22.940238837323303,13.579325095732914,38.67416013838181 +2022-07-15,1.785205883890873,0.631304013521069,0.62869350368483,0.34417591219448 +2022-07-16,0.715594243310775,0.984028805544615,0.967817534607272,0.153576028704162 +2022-07-17,0.8147430610328651,0.26284189122074103,2.154865858339681,1.283391109459497 +2022-07-18,1.24545618431753,1.082826448671828,0.45266260254768803,0.084950117524903 +2022-07-19,0.083915091025848,0.5880718113215341,1.007179376918947,0.865719136965563 +2022-07-20,0.32701962372413,0.17041520446782002,1.773905435681336,1.104677623595627 +2022-07-21,0.722041044814348,0.822598941600809,0.9593067917284711,0.41221237294221 +2022-07-22,10.662811760936936,0.183634003906628,13.34157103707886,12.159275730149757 +2022-07-23,0.364677770561312,0.559247300094862,0.40653075149155804,0.9694788164354201 +2022-07-24,0.9157754804402811,1.091835974661574,0.8199135063617131,1.062993342271482 +2022-07-25,0.67849525511577,3.048531280160542,1.471231045142114,0.591416124923933 +2022-07-26,1.330066578741473,1.582638082613582,1.789945856913414,2.192979954983851 +2022-07-27,5.686664774275858,6.436173309221049,4.7854634352690155,1.63427958962943 +2022-07-28,0.22324041916141402,1.428578028660132,1.458686942176205,1.311736714260567 +2022-07-29,2.476037411183793,0.7249411790805861,0.42039676295059203,2.995565219170837 +2022-07-30,0.773637653847835,0.9265499876312321,0.32494629315872703,0.324341917122049 +2022-07-31,0.169152723468269,0.9014047551998651,0.46982772647431204,0.5534444716525361 +2022-08-01,1.519627178758216,1.610068186750942,3.309159861747862,0.529330910775445 +2022-08-02,3.545652606242915,2.330031081649702,3.9640154638953513,3.856552869039815 +2022-08-03,1.3404062798488199,0.462815961543229,0.9670138622828691,1.234951845374218 +2022-08-04,0.6530263834012761,0.08597037675761801,0.9707805498085981,0.16192153323070702 +2022-08-05,0.708729075421144,3.581243043200601,2.651870015387795,2.173110201699371 +2022-08-06,11.986631123295103,10.930286916614767,22.78182877052594,11.135752238663327 +2022-08-07,1.232939713701664,1.8029857915415932,1.476317514220094,0.958792084262276 +2022-08-08,0.530813431405317,2.014631665755229,2.7977280500624433,2.039075329513739 +2022-08-09,2.510188659097596,2.983415297830412,0.126105493389976,2.121594836643942 +2022-08-10,1.622456744339298,1.289582374551249,2.5475660513369442,3.282862603860423 +2022-08-11,0.25962103068208003,0.37516694909806303,0.5828588359439041,0.139231895746797 +2022-08-12,0.884113805589634,1.603259620438238,1.8751590471743,0.24122279792940302 +2022-08-13,8.624685230360152,3.3791466219930433,9.375119750363103,0.7506624109540441 +2022-08-14,1.6498217904306332,2.627126072551769,0.57102589263998,2.4024472876658223 +2022-08-15,5.840124155766275,3.354483317899366,1.075590808840468,2.3650336028113492 +2022-08-16,2.405030143824614,1.444879535922397,0.942851527789979,1.423465983705226 +2022-08-17,1.034391306438299,0.6369673400603011,0.473080126014195,0.8585472144660361 +2022-08-18,4.703412413763946,0.5990722583580821,0.911179769434693,1.189643421133775 +2022-08-19,0.22138733877979702,0.24079634787311602,0.045592515857608006,0.9837464357223601 +2022-08-20,0.14621922149563202,0.096627004274573,0.7910843611370051,0.059870034265504 +2022-08-21,5.702011116768989,11.782408392531364,4.022381962144387,12.55656527594101 +2022-08-22,2.289251748782675,1.846139752930779,1.302218066447416,1.195140815195466 +2022-08-23,0.40156922050687605,2.44670175499675,2.701336009126233,2.5846045653914143 +2022-08-24,16.368686321533847,9.285256494681935,14.17816605948454,6.813674272547164 +2022-08-25,17.325634435909638,40.63547963350666,52.36627420733006,22.12581500864155 +2022-08-26,2.771994472181008,1.432898057334553,0.500652317272802,0.29065103217099203 +2022-08-27,2.705080627435551,1.140228551443707,2.535947091285437,2.068004826572326 +2022-08-28,0.366184071525715,0.37561791103581305,1.087399555743408,1.3431857507096732 +2022-08-29,0.192969279119436,1.203078886727172,0.8966719923838661,1.07779591413233 +2022-08-30,0.248116517168076,0.901524977902984,0.9660257636136701,0.13247782324235202 +2022-08-31,0.7172605174280461,0.22721632248943202,0.9247009404760361,0.28572976740870504 +2022-09-01,0.259653267575217,2.040817156148029,2.19857288799284,2.229208371156883 +2022-09-02,1.398428414171018,0.071469482611002,0.9048807067534731,0.0022491420541680004 +2022-09-03,1.7166677805176591,1.903668070163285,1.866568462888393,1.8648831840830011`, + input: '请使用组合图展示不同类别的权重随着时间的变化，用四个柱图。' }; export const mockUserTextInput0 = { diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index b4614f9e..efb68e99 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -49,7 +49,13 @@ import { linearProgressChartData, basicHeatMapChartData, vennChartData, - mapChartData + mapChartData, + singleColumnLineCombinationChartData, + singleColumnLineCombinationChartData1, + singleColumnPieCombinationChartData, + singleColumnPieAndLineCombinationChartData, + singleColumnBarCombinationChartData1, + singleColumnBarCombinationChartData } from '../../constants/mockData'; import VMind, { ArcoTheme, builtinThemeMap, BuiltinThemeType } from '../../../../../src/index'; import { Model } from '../../../../../src/index'; @@ -106,7 +112,13 @@ const demoDataList: { [key: string]: any } = { Gauge: gaugeChartData, LinearProgress: linearProgressChartData, BasicHeatMap: basicHeatMapChartData, - Venn: vennChartData + Venn: vennChartData, + SingleColumnLineCommon: singleColumnLineCombinationChartData, + SingleColumnLineCommon1: singleColumnLineCombinationChartData1, + SingleColumnPieCommon: singleColumnPieCombinationChartData, + SingleColumnBarCommon: singleColumnBarCombinationChartData, + SingleColumnBarCommon1: singleColumnBarCombinationChartData1, + SingleColumnPieAndLineCommon: singleColumnPieAndLineCombinationChartData }; const globalVariables = (import.meta as any).env; diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index 4326f34e..330d0cbd 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -1,9 +1,16 @@ import type { BasemapOption } from '../../common/typings'; -import { ChartType, MapRegionCoordinate } from '../../common/typings'; +import { ChartType, BasicChartType, CombinationChartType, MapRegionCoordinate } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); +export const BASIC_CHART_LIST = Object.values(BasicChartType); +export const COMBINATION_CHART_LIST = Object.values(CombinationChartType); -export const NEED_COLOR_FIELD_CHART_LIST = [ChartType.PieChart, ChartType.RoseChart, ChartType.LinearProgress]; +export const NEED_COLOR_FIELD_CHART_LIST = [ + ChartType.PieChart, + ChartType.RoseChart, + ChartType.LinearProgress, + ChartType.LineChart +]; export const NEED_SIZE_FIELD_CHART_LIST = [ChartType.ScatterPlot, ChartType.BasicHeatMap]; @@ -29,7 +36,8 @@ export const CARTESIAN_CHART_LIST = [ ChartType.WaterFallChart, ChartType.BoxPlot, ChartType.RangeColumnChart, - ChartType.LinearProgress + ChartType.LinearProgress, + ChartType.BasicHeatMap ]; export const DEFAULT_MAP_OPTION: BasemapOption = { diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts index 40d00b80..71b2bfa4 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts @@ -1,10 +1,12 @@ import type { ChartType } from '../../../../common/typings'; import type { GetVizSchemaContext, GetVizSchemaOutput } from '../getVizSchema/types'; +import type { BasicChartType } from '../../../../common/typings'; export type GenerateChartTypeContext = GetVizSchemaContext & GetVizSchemaOutput; export type GenerateChartTypeOutput = { chartType: ChartType; + subChartType?: BasicChartType[]; chartSource: string; chartTypeTokenUsage: any; }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts index 84b3105a..fa1e7c74 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts @@ -1,9 +1,12 @@ -import { Cell } from '../../types'; -import { GenerateChartTypeContext, GenerateChartTypeOutput } from '../generateChartType/types'; +import type { Cell } from '../../types'; +import type { GenerateChartTypeContext, GenerateChartTypeOutput } from '../generateChartType/types'; +import type { VMindDataset } from '../../../../common/typings'; export type GenerateFieldMapContext = GenerateChartTypeContext & GenerateChartTypeOutput; export type GenerateFieldMapOutput = { - cell: Cell; + cell?: Cell; + cells?: Cell[]; + datasets?: VMindDataset[]; fieldMapTokenUsage: any; }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts index 2ba28beb..966f7d02 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/index.ts @@ -18,6 +18,7 @@ import { patchNeedSize, patchPieChart, patchRangeColumnChart, + patchSingleColumnCombinationChart, patchWordCloud, patchYField } from './patcher'; @@ -49,7 +50,8 @@ const ChartGenerationTaskNodeGPTMeta: LLMBasedTaskNodeMeta< patchRangeColumnChart, patchLinearProgressChart, patchBasicHeatMapChart, - patchCartesianXField + patchCartesianXField, + patchSingleColumnCombinationChart ], requester: chartGenerationRequestLLM, prompt: new GPTChartGenerationPrompt() diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts index 6cab9455..c50ab3eb 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts @@ -1,5 +1,7 @@ import { FOLD_NAME, FOLD_VALUE } from '@visactor/chart-advisor'; import { isArray, isNil } from '@visactor/vutils'; +import ChartGenerationTaskNodeGPTMeta from '../index'; +import type { Cell } from '../../../../types'; import type { Transformer } from '../../../../../../base/tools/transformer'; import { @@ -9,6 +11,7 @@ import { getFieldsByDataType, getRemainedFields } from '../../../../../../common/utils/utils'; +import type { BasicChartType, VMindDataset } from '../../../../../../common/typings'; import { ChartType, DataType, ROLE } from '../../../../../../common/typings'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../../types'; import { isValidDataset } from '../../../../../../common/dataProcess'; @@ -16,7 +19,8 @@ import { NEED_COLOR_FIELD_CHART_LIST, NEED_SIZE_FIELD_CHART_LIST, CARTESIAN_CHART_LIST, - NEED_COLOR_AND_SIZE_CHART_LIST + NEED_COLOR_AND_SIZE_CHART_LIST, + COMBINATION_CHART_LIST } from '../../../../constants'; export const patchAxisField: Transformer< @@ -374,7 +378,12 @@ export const patchNeedColor: Transformer< NEED_COLOR_FIELD_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) || NEED_COLOR_AND_SIZE_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) ) { - const colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean); + let colorField; + if (CARTESIAN_CHART_LIST.every(cartesianChartType => cartesianChartType.toUpperCase() !== chartType)) { + colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean); + } else { + colorField = [cellNew.color]; + } if (colorField.length !== 0) { cellNew.color = colorField[0]; } else { @@ -514,3 +523,59 @@ export const patchBasicHeatMapChart: Transformer< cell: cellNew }; }; + +export const patchSingleColumnCombinationChart: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + const { chartType, cells, subChartType } = context; + if ( + COMBINATION_CHART_LIST.some(combinationChartType => { + return chartType.toUpperCase() === combinationChartType.toUpperCase(); + }) + ) { + const cellsNew: Cell[] = [...cells]; + const subChartTypeNew: BasicChartType[] = [...subChartType]; + const datasetNew: VMindDataset[] = []; + + // Sort by subChartType so that the Cartesian chart is placed under the combined chart + const combined = cellsNew.map((cell, index) => ({ + cell, + subChartType: subChartTypeNew[index] + })); + combined.sort((a, b) => { + if ( + CARTESIAN_CHART_LIST.some(cartesianChartType => { + return cartesianChartType.toUpperCase() === a.subChartType.toUpperCase(); + }) + ) { + return 1; + } + return -1; + }); + const sortedCellsNew = combined.map(item => item.cell); + const sortedSubChartTypeNew = combined.map(item => item.subChartType); + + const patchers = ChartGenerationTaskNodeGPTMeta.patcher.filter(patch => { + return patch.name !== 'patchSingleColumnCombinationChart'; + }); + + sortedSubChartTypeNew.forEach((chartType, index) => { + const input = { ...context, chartType, cell: sortedCellsNew[index] }; + const result = patchers.reduce((pre, pipeline) => { + const res = pipeline(pre); + return { ...pre, ...res } as any; + }, input); + subChartTypeNew[index] = result.chartType; + cellsNew[index] = result.cell; + datasetNew[index] = result.dataset; + }); + + return { + subChartType: subChartTypeNew, + cells: cellsNew, + datasets: datasetNew + }; + } + return {}; +}; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/examples.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/examples.ts index 818900aa..ca5f3e45 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/examples.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/examples.ts @@ -19,10 +19,10 @@ Response: \`\`\` {${showThoughts ? '\n"thought": "Your thoughts",' : ''} "CHART_TYPE": "Line Chart", -"FIELD_MAP": { +"FIELD_MAP": [{ "x": "日期", "y": "降雨量" -}${ +}]${ showThoughts ? ',\n"REASON": "User wants to show the trend of the rainfall, which is suitable for displaying with a line chart. The \'日期\' is used as the x-axis because it\'s a date, and the 降雨量 is used as the y-axis because it\'s a number. This chart can show the trend of rainfall."' : '' @@ -49,10 +49,10 @@ Data field description: [ Response: \`\`\` {${showThoughts ? '\n"thought": "Your thoughts",' : ''}"CHART_TYPE": "Line Chart", -"FIELD_MAP": { +"FIELD_MAP": [{ "x": "日期", "y": "降雨量" -}${ +}]${ showThoughts ? ',\n"REASON": "User did not show their intention about the data in their input. The data has two fields and it contains a date field, so Line Chart is best suitable to show the data. The field \'日期\' is used as the x-axis because it\'s a date, and the 降雨量 is used as the y-axis because it\'s a number. The duration is 20s but we just ignore it."' : '' @@ -82,10 +82,10 @@ Response: \`\`\` {${showThoughts ? '\n"thought": "Your thoughts",' : ''} "CHART_TYPE": "Pie Chart", -"FIELD_MAP": { +"FIELD_MAP": [{ "angle": "市场份额", "color": "品牌名称" -}${ +}]${ showThoughts ? ',\n"REASON": "The data contains the market share, and the user wants to show percentage data, which is suitable for displaying with a pie chart. The 市场份额 is used as the angle of the pie chart to show the market share of each brand. The 品牌名称 is used as the color to distinguish different brands. The duration is 5s but we just ignore it."' : '' @@ -119,11 +119,11 @@ Response: \`\`\` {${showThoughts ? '\n"thought": "Your thoughts",' : ''} "CHART_TYPE": "Dynamic Bar Chart", -"FIELD_MAP": { +"FIELD_MAP": [{ "x": "country", "y": "金牌数量", "time": "year" -}${ +}]${ showThoughts ? ",\n\"REASON\": \"The data contains the year, country, and medal count, and the user's intention contains 'comparison', which is suitable for drawing a dynamic bar chart that changes over time to show the comparison of gold medal counts of various countries in each Olympic Games.The 'country' field is used as the x-axis of the bar chart, and '金牌数量' is used as the y-axis to show the comparison of gold medal counts of various countries in the current year.The 'year' field is used as the time field of the dynamic bar chart to show the comparison of gold medal counts of various countries at different years.\"" : '' @@ -154,10 +154,10 @@ Response: \`\`\` {${showThoughts ? '\n"thought": "Your thoughts",' : ''} "CHART_TYPE": "Bar Chart", -"FIELD_MAP": { +"FIELD_MAP": [{ "x": "商品名称", "y": "销售额" -}${ +}]${ showThoughts ? ",\n\"REASON\": \"User wants to show the sales of different products in different regions, which is suitable for displaying with a bar chart. The '商品名称' is used as the x-axis because it's a string field, and the '销售额' is used as the y - axis because it's a number field. The 'region' field can be used to distinguish different regions, but since the user did not specify the color channel, we will not use it.\"" : '' diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts index aaf6b2c7..8b724e0f 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts @@ -12,6 +12,7 @@ import { visualChannelInfoMap } from './knowledges'; import { uniqArray } from '@visactor/vutils'; +import { BASIC_CHART_LIST, COMBINATION_CHART_LIST } from '../../../../constants'; const patchUserInput = (userInput: string) => { const FULL_WIDTH_SYMBOLS = ['，', '。']; @@ -67,8 +68,25 @@ export class GPTChartGenerationPrompt extends Prompt `"${channel}": ${visualChannelInfoMap[channel](sortedChartTypeList)}`) - .join('\n'); + .map((channel: string) => { + const visualChannelInfo = visualChannelInfoMap[channel](sortedChartTypeList); + if (visualChannelInfo.multipleFieldsInfo) { + return `"${channel}": { + "oneOf": [ + {"type": "string", "description": "${visualChannelInfo.singleFieldInfo}"}, + { + "type": "array", + "items": { + "type": "string", + "description": "${visualChannelInfo.multipleFieldsInfo}" + } + } + ] + }`; + } + return `"${channel}": {"type": "string", "description": "${visualChannelInfo.singleFieldInfo}"}`; + }) + .join(',\n '); const constraintsStr = getStrFromArray(chartGenerationConstraints); @@ -85,6 +103,8 @@ export class GPTChartGenerationPrompt extends Prompt 'the field mapped to the x-axis, can be empty. Can Only has one field.', - y: (chartTypeList: ChartType[]) => - 'the field mapped to the y-axis. Only number fields, can be empty. Use array if there are more than one fields.', - color: (chartTypeList: ChartType[]) => - `the field mapped to the color channel. Must use a string field${getColorKnowledge(chartTypeList)}`, - size: (chartTypeList: ChartType[]) => - `the field mapped to the size channel. Must use a number field${getSizeKnowledge(chartTypeList)}`, - angle: (chartTypeList: ChartType[]) => - "the field mapped to the angle channel of the pie chart. Can't be empty in Pie Chart.", - radius: (chartTypeList: ChartType[]) => - "the field mapped to the radius channel of the rose chart. Only used in Rose Chart. Can't be empty in Rose Chart.", - time: (chartTypeList: ChartType[]) => - "This is usually a date field and can be used only in Dynamic Bar Chart. Can't be empty in Dynamic Bar Chart.", - source: (chartTypeList: ChartType[]) => - "the field mapped to the source channel. Only used in Sankey Chart. Can't be empty in Sankey Chart.", - target: (chartTypeList: ChartType[]) => - "the field mapped to the target channel. Only used in Sankey Chart. Can't be empty in Sankey Chart.", - value: (chartTypeList: ChartType[]) => - "the field mapped to the value channel. Only used in Sankey Chart. Can't be empty in Sankey Chart." + x: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: 'the field mapped to the x-axis, can be empty. Can Only has one field.' + }; + }, + y: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: 'the field mapped to the y-axis. Only number fields, can be empty.', + multipleFieldsInfo: + 'the field mapped to the y-axis. Only number fields, can be empty. Use array if there are more than one fields.' + }; + }, + color: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: `the field mapped to the color channel. Must use a string field${getColorKnowledge( + chartTypeList + )}`, + multipleFieldsInfo: `the field mapped to the color channel. Must use a string field${getColorKnowledge( + chartTypeList + )} Use array if there are more than one fields.` + }; + }, + size: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: `the field mapped to the size channel. Must use a number field${getSizeKnowledge(chartTypeList)}` + }; + }, + angle: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: "the field mapped to the angle channel of the pie chart. Can't be empty in Pie Chart." + }; + }, + radius: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: + "the field mapped to the radius channel of the rose chart. Only used in Rose Chart. Can't be empty in Rose Chart." + }; + }, + time: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: + "This is usually a date field and can be used only in Dynamic Bar Chart. Can't be empty in Dynamic Bar Chart." + }; + }, + source: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: + "the field mapped to the source channel. Only used in Sankey Chart. Can't be empty in Sankey Chart." + }; + }, + target: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: + "the field mapped to the target channel. Only used in Sankey Chart. Can't be empty in Sankey Chart." + }; + }, + value: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: + "the field mapped to the value channel. Only used in Sankey Chart. Can't be empty in Sankey Chart." + }; + } }; export const chartKnowledgeDict: ChartKnowledge = { [ChartType.BarChart]: { @@ -159,7 +202,9 @@ export const chartKnowledgeDict: ChartKnowledge = { index: 15, visualChannels: ['x', 'y'], examples: [], - knowledge: [] + knowledge: [ + 'Linear Progress chart is typically used to display progress data, which is usually a value between 0 and 1. Linear progress bars can show single progress values as well as multiple progress values. By default, the left Y-axis of the linear progress bar is the categorical field, and the bottom X-axis is the numerical field.' + ] }, [ChartType.CircularProgress]: { index: 16, @@ -220,6 +265,12 @@ export const chartKnowledgeDict: ChartKnowledge = { knowledge: [ 'The color field of the Venn diagram requires an array of length 2. The field with subscript 0 maps to the sets, and the field with subscript 1 maps to the name.' ] + }, + [ChartType.SingleColumnCombinationChart]: { + index: 27, + visualChannels: [], + examples: [], + knowledge: ['Single column combination charts can be combined with a variety of different basic chart types'] } }; @@ -233,5 +284,6 @@ export const chartGenerationConstraints = [ 'A number field can not be used as a color field. A string field can not be used as y-axis', "Ignore requests unrelated to chart visualization in the user's request.", `The keys in FIELD_MAP must be selected from the list of available visual channels. Don't use visual channels that do not exist.`, + `The response must pass the JSONSchema validation.`, `Wrap the reply content using \`\`\`, and the returned content must be directly parsed by JSON.parse() in JavaScript.` ]; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts index a713eef4..47280c54 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts @@ -2,6 +2,8 @@ export const ChartAdvisorPromptEnglish = ( showThoughts: boolean, supportedChartList: string[], + basicChartList: string[], + combinationChartList: string[], knowledge: string, visualChannels: string, constraints: string, @@ -19,14 +21,56 @@ ${knowledge} Let's think step by step. ${showThoughts ? 'Fill your thoughts in {thought}.' : ''} -Response in the following format: +Please return the response according to the following JsonSchema: \`\`\` -{${showThoughts ? '\n"thought" : your thoughts' : ''} -"CHART_TYPE": the chart type you choose. Supported chart types: ${JSON.stringify(supportedChartList)}. -"FIELD_MAP": { // Visual channels and the fields mapped to them -${visualChannels} -}${showThoughts ? ',\n"Reason": the reason for selecting the chart type and visual mapping.' : ''} +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties":{ + ${showThoughts ? '\n"thought": {"type": "string","description": "your thoughts"},' : ''} + "CHART_TYPE": { + "type": "string", + "enum": ${JSON.stringify(supportedChartList)}, + "description": "Specify the chart type based on user instructions and data" + }, + "SUB_CHART_TYPE": { + "type": "array", + "items": { + "enum": ${JSON.stringify(basicChartList)} + }, + "description": "The default is an empty array, and only relevant results are returned when there are multiple indicators. For example, combination charts, dual-axis charts, etc." + }, + "FIELD_MAP": { + "type": "array", + "items": { + "type": "object", + "properties": { + ${visualChannels} + } + }, + "description": "By default, the array length is 1, and only one unique visual channel field mapping is returned. Multiple visual channel field mappings are returned only when a combined chart is needed, and the order needs to correspond to the order of the CHART_TYPE array." + } + ${ + showThoughts + ? ',\n"REASON": {"type": "string","description": "the reason for selecting the chart type and visual mapping."},' + : '' + } + }, + "required": ["CHART_TYPE", "FIELD_MAP"], + "if": { + "properties": { + "CHART_TYPE": { "enum": ${JSON.stringify(combinationChartList)} } + } + }, + "then": { + "required": ["CHART_TYPE", "FIELD_MAP", "SUB_CHART_TYPE"] + }, + "else": { + "not": { + "required": ["SUB_CHART_TYPE"] + } + } } \`\`\` diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts index 5fafe040..5ee508bf 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts @@ -3,12 +3,15 @@ import type { Parser } from '../../../../../base/tools/parser'; import type { Requester } from '../../../../../base/tools/requester'; import { parseGPTResponse, requestGPT } from '../../../../../common/utils/gpt'; import type { ChartType } from '../../../../../common/typings'; +import { COMBINATION_CHART_LIST } from '../../../constants'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../types'; +import type { BasicChartType } from '../../../../../common/typings'; type GPTChartAdvisorResult = { CHART_TYPE: ChartType; + SUB_CHART_TYPE: BasicChartType[]; DOUBLE_CHECK: string; - FIELD_MAP: Cell; + FIELD_MAP: Cell[]; THOUGHT: string; VIDEO_DURATION?: number; COLOR_PALETTE?: string[]; @@ -25,9 +28,20 @@ export const parseChartGenerationResponse: Parser< throw Error((advisorResJson as any).message); } - const { CHART_TYPE, FIELD_MAP } = advisorResJson; - - return { chartType: CHART_TYPE as ChartType, cell: FIELD_MAP, usage: advisorRes.usage }; + const { CHART_TYPE, FIELD_MAP, SUB_CHART_TYPE } = advisorResJson; + if ( + COMBINATION_CHART_LIST.every(chartType => { + return chartType.toUpperCase() !== CHART_TYPE.toUpperCase(); + }) + ) { + return { chartType: CHART_TYPE as ChartType, cell: FIELD_MAP[0], usage: advisorRes.usage }; + } + return { + chartType: CHART_TYPE as ChartType, + cells: FIELD_MAP, + subChartType: SUB_CHART_TYPE, + usage: advisorRes.usage + }; }; export const chartGenerationRequestLLM: Requester = async ( diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts index 79657aed..0e41920c 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts @@ -75,7 +75,12 @@ import { basemap, mapField, mapDisplayConf, - registerChart + registerChart, + commonSingleColumnRegion, + commonSingleColumnSeries, + commonSingleColumnLegend, + commonSingleColumnAxes, + commonSingleColumnLayout } from './transformers'; const pipelineBar = [ @@ -223,6 +228,15 @@ const pipelineBasicHeatMap = [ theme ]; const pipelineVenn = [chartType, registerChart, vennData, color, vennField, legend, theme]; +const pipelineSingleColumnCombinationChart = [ + chartType, + commonSingleColumnRegion, + commonSingleColumnSeries, + commonSingleColumnLegend, + commonSingleColumnAxes, + commonSingleColumnLayout, + theme +]; const pipelineMap: { [chartType: string]: any } = { [ChartType.BarChart.toUpperCase()]: pipelineBar, @@ -248,7 +262,8 @@ const pipelineMap: { [chartType: string]: any } = { [ChartType.TreemapChart.toUpperCase()]: pipelineTreemap, [ChartType.Gauge.toUpperCase()]: pipelineGauge, [ChartType.BasicHeatMap.toUpperCase()]: pipelineBasicHeatMap, - [ChartType.VennChart.toUpperCase()]: pipelineVenn + [ChartType.VennChart.toUpperCase()]: pipelineVenn, + [ChartType.SingleColumnCombinationChart.toUpperCase()]: pipelineSingleColumnCombinationChart }; export const beforePipe: Transformer = ( diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 3b1e4bd5..387a6500 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -20,10 +20,13 @@ import { import { getFieldByDataType } from '../../../../../common/utils/utils'; import { array, isArray } from '@visactor/vutils'; import { isValidDataset } from '../../../../../common/dataProcess'; -import { DataType, ChartType } from '../../../../../common/typings'; -import { builtinThemeMap } from '../../../../../common/builtinTheme'; import type { VMindDataset } from '../../../../../common/typings'; -import { COLOR_FIELD } from '@visactor/chart-advisor'; +import { BasicChartType, ChartType, DataType } from '../../../../../common/typings'; +import { builtinThemeMap } from '../../../../../common/builtinTheme'; +import { FOLD_NAME, FOLD_VALUE, COLOR_FIELD } from '@visactor/chart-advisor'; +import { CARTESIAN_CHART_LIST } from '../../../constants'; + +const CARTESIAN_CHART_LIST_UPPER = CARTESIAN_CHART_LIST.map(chartType => chartType.toUpperCase() as ChartType); type Context = GetChartSpecContext & GetChartSpecOutput; @@ -51,7 +54,8 @@ const chartTypeMap: { [chartName: string]: string } = { [ChartType.TreemapChart.toUpperCase()]: 'treemap', [ChartType.Gauge.toUpperCase()]: 'gauge', [ChartType.BasicHeatMap.toUpperCase()]: 'common', - [ChartType.VennChart.toUpperCase()]: 'venn' + [ChartType.VennChart.toUpperCase()]: 'venn', + [ChartType.SingleColumnCombinationChart.toUpperCase()]: 'common' }; export const chartType: Transformer = (context: Context) => { @@ -1824,3 +1828,231 @@ export const mapDisplayConf: Transformer = (context }; return { spec }; }; + +export const commonSingleColumnRegion: Transformer = (context: Context) => { + const { cells, spec, datasets, subChartType } = context; + + const regionList: string[] = []; + + cells.forEach((cell, index) => { + const sizeField = [cell.y, cell.size, cell.angle, cell.radius, cell.value].filter(Boolean).flat(); + const dataset = datasets[index]; + if (sizeField.length !== 0) { + if (sizeField[0] === FOLD_VALUE.toString()) { + const sizeFieldName = [...new Set(dataset.map(data => data[FOLD_NAME.toString()]))]; + regionList.push(sizeFieldName.join(' & ') + '-' + subChartType[index]); + } else { + regionList.push(sizeField[0] + '-' + subChartType[index]); + } + } + }); + + spec.region = regionList.map((region, index) => { + return { id: region }; + }); + spec.regionList = regionList; + return { spec }; +}; + +export const commonSingleColumnSeries: Transformer = (context: Context) => { + const { cells, spec, datasets, subChartType } = context; + const commonData = {}; + spec.seriesField = 'type'; + spec.region.forEach((region: { [id: string]: string }) => { + commonData[region.id] = []; + }); + cells.forEach((cell, index) => { + const regionId = spec.regionList[index]; + const dataset = datasets[index]; + + dataset.forEach(data => { + const subData = { + type: regionId + }; + for (const visualChannel in cell) { + subData[cell[visualChannel]] = data[cell[visualChannel]]; + } + commonData[regionId].push(subData); + }); + }); + + spec.series = spec.region.map((region: { [id: string]: string }, index: number) => { + const regionId = region.id; + const serie = { + id: regionId, + regionId: regionId, + type: chartTypeMap[subChartType[index].toUpperCase()], + data: { id: regionId, values: commonData[regionId] } + }; + switch (subChartType[index].toUpperCase()) { + case BasicChartType.BarChart.toUpperCase(): + if (cells[index].color) { + return { + ...serie, + barMinWidth: 20, + xField: cells[index].x, + yField: cells[index].y, + seriesField: cells[index].color + }; + } + return { + ...serie, + barMinWidth: 20, + xField: cells[index].x, + yField: cells[index].y + }; + case BasicChartType.PieChart.toUpperCase(): + return { + ...serie, + valueField: [cells[index].angle, cells[index].value, cells[index].radius, cells[index].size].filter( + Boolean + )[0], + categoryField: cells[index].color, + seriesField: cells[index].color + }; + case BasicChartType.LineChart.toUpperCase(): + if (cells[index].color) { + return { + ...serie, + xField: cells[index].x, + yField: cells[index].y, + seriesField: cells[index].color + }; + } + return { + ...serie, + xField: cells[index].x, + yField: cells[index].y + }; + default: + return { + ...serie, + xField: cells[index].x, + yField: cells[index].y + }; + } + }); + + return { spec }; +}; + +export const commonSingleColumnLegend: Transformer = (context: Context) => { + const { spec } = context; + spec.legends = { + padding: { + top: 10 + }, + visible: true, + orient: 'bottom', + id: 'legend', + regionId: spec.region.map((region: { [id: string]: string }) => { + return region.id; + }) + }; + return { spec }; +}; +export const commonSingleColumnAxes: Transformer = (context: Context) => { + const { spec, subChartType } = context; + + const cartesianRegion: string[] = spec.legends.regionId.filter((item: string, index: number) => { + return CARTESIAN_CHART_LIST_UPPER.includes(subChartType[index].toUpperCase() as ChartType); + }); + + const leftAxesCommonSpec = { + expand: { max: 0.2 }, + label: { flush: true, visible: true }, + tick: { visible: false }, + forceTickCount: 3 + }; + if (cartesianRegion.length !== 0) { + // Y-axes + spec.axes = cartesianRegion.map((region: string) => { + return { + id: region + '-left', + regionId: region, + orient: 'left', + title: { visible: true, text: region }, + ...leftAxesCommonSpec + }; + }); + + // X-axes + spec.axes.push({ + id: 'bottom', + regionId: cartesianRegion, + orient: 'bottom', + label: { + firstVisible: true, + lastVisible: true, + visible: true + }, + tick: { visible: false }, + paddingInner: 0.99, + paddingOuter: 0 + }); + } + return { spec }; +}; +export const commonSingleColumnLayout: Transformer = (context: Context) => { + const { spec, subChartType } = context; + const regionLength = spec.legends.regionId.length; + const elements = []; + + const existCartesianChart = subChartType.some(chartType => + CARTESIAN_CHART_LIST_UPPER.includes(chartType.toUpperCase() as ChartType) + ); + + spec.region.forEach((region: { [id: string]: string }, index: number) => { + if (CARTESIAN_CHART_LIST_UPPER.includes(subChartType[index].toUpperCase() as ChartType)) { + elements.push({ + modelId: region.id + '-left', + col: 0, + row: index + }); + elements.push({ + modelId: region.id, + col: 1, + row: index + }); + } else if (!existCartesianChart) { + elements.push({ + modelId: region.id, + col: 0, + row: index + }); + } else { + elements.push({ + modelId: region.id, + col: 0, + colSpan: 2, + row: index + }); + } + }); + if ( + subChartType.some(item => { + return CARTESIAN_CHART_LIST_UPPER.includes(item.toUpperCase() as ChartType); + }) + ) { + elements.push({ + modelId: 'legend', + col: 0, + colSpan: 2, + row: regionLength + 1 + }); + elements.push({ + modelId: 'bottom', + col: 1, + row: regionLength + }); + spec.layout = { type: 'grid', col: 2, row: regionLength + 2, elements: elements }; + } else { + elements.push({ + modelId: 'legend', + col: 0, + row: regionLength + }); + spec.layout = { type: 'grid', col: 1, row: regionLength + 1, elements: elements }; + } + return { spec }; +}; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/types.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/types.ts index c92a3bcd..6c35667d 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/types.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/types.ts @@ -1,4 +1,4 @@ -import { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../generateTypeAndFieldMap/types'; +import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../generateTypeAndFieldMap/types'; export type GetChartSpecContext = GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput; export type Spec = any; diff --git a/packages/vmind/src/applications/types.ts b/packages/vmind/src/applications/types.ts index ec7fe39c..c65e97b2 100644 --- a/packages/vmind/src/applications/types.ts +++ b/packages/vmind/src/applications/types.ts @@ -8,6 +8,7 @@ import type { } from '../common/typings'; import type { Cell } from './chartGeneration/types'; import type { InsightAlgorithm, VMindInsight } from './IngelligentInsight/types'; +import type { BasicChartType } from '../common/typings'; //context of the DataExtraction Application export type DataExtractionContext = { @@ -54,7 +55,10 @@ export type ChartGenerationContext = { export type ChartGenerationOutput = { chartType: ChartType; - cell: Cell; + subChartType?: BasicChartType[]; + cell?: Cell; + cells?: Cell[]; + datasets?: VMindDataset[]; spec: any; chartSource: string; usage: any; //token usage of the LLM diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index 2afca682..b1237e26 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -82,9 +82,19 @@ export enum ChartType { SunburstChart = 'Sunburst Chart', TreemapChart = 'Treemap Chart', Gauge = 'Gauge Chart', - // LinearProgressChart = 'Linear Progress Chart', BasicHeatMap = 'Basic Heat Map', - VennChart = 'Venn Chart' + VennChart = 'Venn Chart', + SingleColumnCombinationChart = 'Single Column Combination Chart' +} + +export enum CombinationChartType { + SingleColumnCombinationChart = 'Single Column Combination Chart' +} + +export enum BasicChartType { + BarChart = 'Bar Chart', + LineChart = 'Line Chart', + PieChart = 'Pie Chart' } export type GPTChartAdvisorResult = { diff --git a/packages/vmind/src/core/VMind.ts b/packages/vmind/src/core/VMind.ts index 5059145c..805f755c 100644 --- a/packages/vmind/src/core/VMind.ts +++ b/packages/vmind/src/core/VMind.ts @@ -236,7 +236,7 @@ class VMind { return chartGenerationResult.advisedList; } - const { chartType, spec, cell, chartSource, time } = chartGenerationResult; + const { chartType, subChartType, spec, cell, cells, chartSource, time } = chartGenerationResult; const usage = calculateTokenUsage([queryDatasetUsage, chartGenerationResult.usage]); return { //...chartGenerationResult, @@ -245,6 +245,8 @@ class VMind { cell, chartSource, chartType, + subChartType, + cells, time }; } From fd10e386cf2a0051dec45f91afd6aa09463e9385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Tue, 13 Aug 2024 10:57:52 +0800 Subject: [PATCH 013/128] Reuse the field pipeline of the basic chart --- .../getChartSpec/VChart/transformers.ts | 48 +++++++------------ 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 387a6500..bda8bb92 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -1855,7 +1855,7 @@ export const commonSingleColumnRegion: Transformer }; export const commonSingleColumnSeries: Transformer = (context: Context) => { - const { cells, spec, datasets, subChartType } = context; + const { cells, spec, datasets, subChartType, fieldInfo } = context; const commonData = {}; spec.seriesField = 'type'; spec.region.forEach((region: { [id: string]: string }) => { @@ -1878,55 +1878,39 @@ export const commonSingleColumnSeries: Transformer spec.series = spec.region.map((region: { [id: string]: string }, index: number) => { const regionId = region.id; - const serie = { + const seriesSubset = { id: regionId, regionId: regionId, type: chartTypeMap[subChartType[index].toUpperCase()], data: { id: regionId, values: commonData[regionId] } }; + let specNew: { spec: any }; + switch (subChartType[index].toUpperCase()) { case BasicChartType.BarChart.toUpperCase(): - if (cells[index].color) { - return { - ...serie, - barMinWidth: 20, - xField: cells[index].x, - yField: cells[index].y, - seriesField: cells[index].color - }; - } + specNew = cartesianBar({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); + return { - ...serie, + ...seriesSubset, barMinWidth: 20, - xField: cells[index].x, - yField: cells[index].y + ...specNew.spec }; case BasicChartType.PieChart.toUpperCase(): + specNew = pieField({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); return { - ...serie, - valueField: [cells[index].angle, cells[index].value, cells[index].radius, cells[index].size].filter( - Boolean - )[0], - categoryField: cells[index].color, - seriesField: cells[index].color + ...seriesSubset, + ...specNew.spec, + seriesField: specNew.spec.categoryField }; case BasicChartType.LineChart.toUpperCase(): - if (cells[index].color) { - return { - ...serie, - xField: cells[index].x, - yField: cells[index].y, - seriesField: cells[index].color - }; - } + specNew = cartesianLine({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); return { - ...serie, - xField: cells[index].x, - yField: cells[index].y + ...seriesSubset, + ...specNew.spec }; default: return { - ...serie, + ...seriesSubset, xField: cells[index].x, yField: cells[index].y }; From 6a73014aa65a92675d4614e0ffe8f7d3131ec51d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Tue, 13 Aug 2024 13:37:39 +0800 Subject: [PATCH 014/128] fix:Modify the default style of the X-axis of a single column combination chart;Modify the mockData for singleColumnPieAndLineCombinationChartData --- .../vmind/__tests__/browser/src/constants/mockData.ts | 8 ++++---- .../taskNodes/getChartSpec/VChart/transformers.ts | 5 +---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index e00e559d..1597fe54 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -5597,10 +5597,10 @@ export const singleColumnPieCombinationChartData = { }; export const singleColumnPieAndLineCombinationChartData = { - csv: `性别,数量,平均分 -男,10,93, -女,20,89`, - input: '使用饼图和柱图的组合图分别展示班级内男生女生的人数占比率，以及班级男生和女生各自的平均分' + csv: `性别,数量,期中平均分,期末平均分 +男,10,93,86 +女,20,89,87`, + input: '使用饼图和柱图的组合图分别展示班级内男生女生的人数占比率，以及班级男生和女生各自的期中和期末的平均分' }; export const singleColumnBarCombinationChartData = { diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index bda8bb92..8b5b4ee1 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -1892,7 +1892,6 @@ export const commonSingleColumnSeries: Transformer return { ...seriesSubset, - barMinWidth: 20, ...specNew.spec }; case BasicChartType.PieChart.toUpperCase(): @@ -1970,9 +1969,7 @@ export const commonSingleColumnAxes: Transformer = lastVisible: true, visible: true }, - tick: { visible: false }, - paddingInner: 0.99, - paddingOuter: 0 + tick: { visible: false } }); } return { spec }; From d242b777e036227df9fceaf91d713d9e93158698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Wed, 14 Aug 2024 18:36:35 +0800 Subject: [PATCH 015/128] fix:The bug of inconsistent length of sub_chart_type and field_map --- .../taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts index 47280c54..528c0010 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/template.ts @@ -39,7 +39,7 @@ Please return the response according to the following JsonSchema: "items": { "enum": ${JSON.stringify(basicChartList)} }, - "description": "The default is an empty array, and only relevant results are returned when there are multiple indicators. For example, combination charts, dual-axis charts, etc." + "description": "The default is an empty array, and only relevant results are returned when there are multiple indicators. For example, combination charts, etc. The length of SUB_CHART_TYPE must be the same as the length of FIELD_MAP" }, "FIELD_MAP": { "type": "array", @@ -49,7 +49,7 @@ Please return the response according to the following JsonSchema: ${visualChannels} } }, - "description": "By default, the array length is 1, and only one unique visual channel field mapping is returned. Multiple visual channel field mappings are returned only when a combined chart is needed, and the order needs to correspond to the order of the CHART_TYPE array." + "description": "By default, the array length is 1, and only one unique visual channel field mapping is returned; Multiple visual channel field mappings are returned only when a combined chart is needed. The length needs to be consistent with the length of SUB_CHART_TYPE and the order needs to correspond to the order of the SUB_CHART_TYPE array." } ${ showThoughts From 337131e0d713ef4d58ae02c3f53b487f58f51a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Fri, 16 Aug 2024 01:03:43 +0800 Subject: [PATCH 016/128] fix:Adjust the name of the base chart of the combination chart and remove the pie chart. Remove the sorting of the sub-chart types of the combination chart. --- .../browser/src/constants/mockData.ts | 18 +------------ .../src/pages/ChartGeneration/DataInput.tsx | 6 +---- .../applications/chartGeneration/constants.ts | 4 +-- .../taskNodes/generateChartType/types.ts | 4 +-- .../GPT/patcher/index.ts | 26 +++---------------- .../generateTypeAndFieldMap/GPT/utils.ts | 4 +-- .../getChartSpec/VChart/transformers.ts | 13 +++------- packages/vmind/src/applications/types.ts | 4 +-- packages/vmind/src/common/typings/index.ts | 5 ++-- 9 files changed, 19 insertions(+), 65 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 1597fe54..ae7c464c 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -5587,22 +5587,6 @@ export const singleColumnLineCombinationChartData1 = { '请使用组合图分别展示Social Penetration和Engagement - Socialization类别的权重随着时间的变化，以及Penetration of Private Messages和Number of Private Messages per User类别的权重随着时间的变化' }; -export const singleColumnPieCombinationChartData = { - csv: `姓名,时间投入量,资金投入量,劳动投入量 -小明,120,80,123,10 -小李,200,20,456,8 -小呆,100,400,789,1 -`, - input: '分开展示每位同学在各时间、资金、劳动投入中的占比情况。使用饼图组合绘制图表。' -}; - -export const singleColumnPieAndLineCombinationChartData = { - csv: `性别,数量,期中平均分,期末平均分 -男,10,93,86 -女,20,89,87`, - input: '使用饼图和柱图的组合图分别展示班级内男生女生的人数占比率，以及班级男生和女生各自的期中和期末的平均分' -}; - export const singleColumnBarCombinationChartData = { csv: `region,可乐销售额,雪碧销售额,芬达销售额,醒目销售额 south,2350,215,345,1476 @@ -5610,7 +5594,7 @@ east,1027,654,654,830 west,1027,159,2100,532 north,1027,28,1679,498 `, - input: '帮我使用柱状组合图展示不同区域各商品销售额' + input: '帮我使用四个柱图的组合展示不同区域各商品销售额' }; export const singleColumnBarCombinationChartData1 = { diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index efb68e99..1f3aae53 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -52,8 +52,6 @@ import { mapChartData, singleColumnLineCombinationChartData, singleColumnLineCombinationChartData1, - singleColumnPieCombinationChartData, - singleColumnPieAndLineCombinationChartData, singleColumnBarCombinationChartData1, singleColumnBarCombinationChartData } from '../../constants/mockData'; @@ -115,10 +113,8 @@ const demoDataList: { [key: string]: any } = { Venn: vennChartData, SingleColumnLineCommon: singleColumnLineCombinationChartData, SingleColumnLineCommon1: singleColumnLineCombinationChartData1, - SingleColumnPieCommon: singleColumnPieCombinationChartData, SingleColumnBarCommon: singleColumnBarCombinationChartData, - SingleColumnBarCommon1: singleColumnBarCombinationChartData1, - SingleColumnPieAndLineCommon: singleColumnPieAndLineCombinationChartData + SingleColumnBarCommon1: singleColumnBarCombinationChartData1 }; const globalVariables = (import.meta as any).env; diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index 330d0cbd..bba57823 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -1,8 +1,8 @@ import type { BasemapOption } from '../../common/typings'; -import { ChartType, BasicChartType, CombinationChartType, MapRegionCoordinate } from '../../common/typings'; +import { ChartType, CombinationBasicChartType, CombinationChartType, MapRegionCoordinate } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); -export const BASIC_CHART_LIST = Object.values(BasicChartType); +export const BASIC_CHART_LIST = Object.values(CombinationBasicChartType); export const COMBINATION_CHART_LIST = Object.values(CombinationChartType); export const NEED_COLOR_FIELD_CHART_LIST = [ diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts index 71b2bfa4..06859ffd 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/types.ts @@ -1,12 +1,12 @@ import type { ChartType } from '../../../../common/typings'; import type { GetVizSchemaContext, GetVizSchemaOutput } from '../getVizSchema/types'; -import type { BasicChartType } from '../../../../common/typings'; +import type { CombinationBasicChartType } from '../../../../common/typings'; export type GenerateChartTypeContext = GetVizSchemaContext & GetVizSchemaOutput; export type GenerateChartTypeOutput = { chartType: ChartType; - subChartType?: BasicChartType[]; + subChartType?: CombinationBasicChartType[]; chartSource: string; chartTypeTokenUsage: any; }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts index c50ab3eb..2d2836ef 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts @@ -11,7 +11,7 @@ import { getFieldsByDataType, getRemainedFields } from '../../../../../../common/utils/utils'; -import type { BasicChartType, VMindDataset } from '../../../../../../common/typings'; +import type { CombinationBasicChartType, VMindDataset } from '../../../../../../common/typings'; import { ChartType, DataType, ROLE } from '../../../../../../common/typings'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../../types'; import { isValidDataset } from '../../../../../../common/dataProcess'; @@ -535,33 +535,15 @@ export const patchSingleColumnCombinationChart: Transformer< }) ) { const cellsNew: Cell[] = [...cells]; - const subChartTypeNew: BasicChartType[] = [...subChartType]; + const subChartTypeNew: CombinationBasicChartType[] = [...subChartType]; const datasetNew: VMindDataset[] = []; - // Sort by subChartType so that the Cartesian chart is placed under the combined chart - const combined = cellsNew.map((cell, index) => ({ - cell, - subChartType: subChartTypeNew[index] - })); - combined.sort((a, b) => { - if ( - CARTESIAN_CHART_LIST.some(cartesianChartType => { - return cartesianChartType.toUpperCase() === a.subChartType.toUpperCase(); - }) - ) { - return 1; - } - return -1; - }); - const sortedCellsNew = combined.map(item => item.cell); - const sortedSubChartTypeNew = combined.map(item => item.subChartType); - const patchers = ChartGenerationTaskNodeGPTMeta.patcher.filter(patch => { return patch.name !== 'patchSingleColumnCombinationChart'; }); - sortedSubChartTypeNew.forEach((chartType, index) => { - const input = { ...context, chartType, cell: sortedCellsNew[index] }; + subChartType.forEach((chartType, index) => { + const input = { ...context, chartType, cell: cells[index] }; const result = patchers.reduce((pre, pipeline) => { const res = pipeline(pre); return { ...pre, ...res } as any; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts index 5ee508bf..00fa53e1 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts @@ -5,11 +5,11 @@ import { parseGPTResponse, requestGPT } from '../../../../../common/utils/gpt'; import type { ChartType } from '../../../../../common/typings'; import { COMBINATION_CHART_LIST } from '../../../constants'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from '../types'; -import type { BasicChartType } from '../../../../../common/typings'; +import type { CombinationBasicChartType } from '../../../../../common/typings'; type GPTChartAdvisorResult = { CHART_TYPE: ChartType; - SUB_CHART_TYPE: BasicChartType[]; + SUB_CHART_TYPE: CombinationBasicChartType[]; DOUBLE_CHECK: string; FIELD_MAP: Cell[]; THOUGHT: string; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 8b5b4ee1..30857353 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -21,7 +21,7 @@ import { getFieldByDataType } from '../../../../../common/utils/utils'; import { array, isArray } from '@visactor/vutils'; import { isValidDataset } from '../../../../../common/dataProcess'; import type { VMindDataset } from '../../../../../common/typings'; -import { BasicChartType, ChartType, DataType } from '../../../../../common/typings'; +import { CombinationBasicChartType, ChartType, DataType } from '../../../../../common/typings'; import { builtinThemeMap } from '../../../../../common/builtinTheme'; import { FOLD_NAME, FOLD_VALUE, COLOR_FIELD } from '@visactor/chart-advisor'; import { CARTESIAN_CHART_LIST } from '../../../constants'; @@ -1887,21 +1887,14 @@ export const commonSingleColumnSeries: Transformer let specNew: { spec: any }; switch (subChartType[index].toUpperCase()) { - case BasicChartType.BarChart.toUpperCase(): + case CombinationBasicChartType.BarChart.toUpperCase(): specNew = cartesianBar({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); return { ...seriesSubset, ...specNew.spec }; - case BasicChartType.PieChart.toUpperCase(): - specNew = pieField({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); - return { - ...seriesSubset, - ...specNew.spec, - seriesField: specNew.spec.categoryField - }; - case BasicChartType.LineChart.toUpperCase(): + case CombinationBasicChartType.LineChart.toUpperCase(): specNew = cartesianLine({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); return { ...seriesSubset, diff --git a/packages/vmind/src/applications/types.ts b/packages/vmind/src/applications/types.ts index c65e97b2..dad9b8b4 100644 --- a/packages/vmind/src/applications/types.ts +++ b/packages/vmind/src/applications/types.ts @@ -8,7 +8,7 @@ import type { } from '../common/typings'; import type { Cell } from './chartGeneration/types'; import type { InsightAlgorithm, VMindInsight } from './IngelligentInsight/types'; -import type { BasicChartType } from '../common/typings'; +import type { CombinationBasicChartType } from '../common/typings'; //context of the DataExtraction Application export type DataExtractionContext = { @@ -55,7 +55,7 @@ export type ChartGenerationContext = { export type ChartGenerationOutput = { chartType: ChartType; - subChartType?: BasicChartType[]; + subChartType?: CombinationBasicChartType[]; cell?: Cell; cells?: Cell[]; datasets?: VMindDataset[]; diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index b1237e26..12dfb1a6 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -91,10 +91,9 @@ export enum CombinationChartType { SingleColumnCombinationChart = 'Single Column Combination Chart' } -export enum BasicChartType { +export enum CombinationBasicChartType { BarChart = 'Bar Chart', - LineChart = 'Line Chart', - PieChart = 'Pie Chart' + LineChart = 'Line Chart' } export type GPTChartAdvisorResult = { From e08d118320ae0d06d4302127cfcd16796eaefeb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Fri, 16 Aug 2024 12:18:56 +0800 Subject: [PATCH 017/128] Merge cells and cells --- .../taskNodes/chartAdvisor/transformers.ts | 8 +- .../taskNodes/generateFieldMap/types.ts | 1 - .../GPT/patcher/index.ts | 144 ++++++++++++------ .../generateTypeAndFieldMap/GPT/utils.ts | 7 - .../getChartSpec/VChart/transformers.ts | 109 ++++++++----- .../chartGeneration/taskNodes/utils.ts | 7 + packages/vmind/src/applications/types.ts | 1 - packages/vmind/src/core/VMind.ts | 3 +- 8 files changed, 178 insertions(+), 102 deletions(-) diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts index 5f355a71..7b9fa89c 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts @@ -92,7 +92,7 @@ const getTop1AdvisedChart: Transformer if (advisedList.length === 0) { return { chartType: VMindChartType.BarChart.toUpperCase() as VMindChartType, - cell: {}, + cells: [{}], dataset: undefined, chartSource, usage @@ -101,7 +101,7 @@ const getTop1AdvisedChart: Transformer const result = advisedList[0]; return { chartType: result.chartType as VMindChartType, - cell: getCell(result.cell), + cells: [getCell(result.cell)], dataset: result.dataset, chartSource, usage @@ -116,8 +116,8 @@ const chartAdvisorHandler = (context: ChartAdvisorContext & ChartAdvisorOutput) export const chartGenerationErrorWrapper: Transformer = ( context: ChartAdvisorContext & ChartAdvisorOutput ) => { - const { error, chartType, fieldInfo, cell } = context as any; - if (error || !checkChartTypeAndCell(chartType, cell, fieldInfo)) { + const { error, chartType, fieldInfo, cells } = context as any; + if (error || cells.some((cell: Cell) => !checkChartTypeAndCell(chartType, cell, fieldInfo))) { console.warn('LLM generation error, use rule generation.'); return chartAdvisorHandler(context); } diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts index fa1e7c74..01dc6509 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/types.ts @@ -5,7 +5,6 @@ import type { VMindDataset } from '../../../../common/typings'; export type GenerateFieldMapContext = GenerateChartTypeContext & GenerateChartTypeOutput; export type GenerateFieldMapOutput = { - cell?: Cell; cells?: Cell[]; datasets?: VMindDataset[]; fieldMapTokenUsage: any; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts index 2d2836ef..4be2634b 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/patcher/index.ts @@ -22,14 +22,18 @@ import { NEED_COLOR_AND_SIZE_CHART_LIST, COMBINATION_CHART_LIST } from '../../../../constants'; +import { isCombinationChartType } from '../../../utils'; export const patchAxisField: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { cell } = context; + const { cells, chartType } = context; + if (isCombinationChartType(chartType)) { + return {}; + } - const cellNew: any = { ...cell }; + const cellNew: any = { ...cells[0] }; // patch the "axis" field to x if (cellNew.axis && (!cellNew.x || !cellNew.y)) { @@ -42,7 +46,7 @@ export const patchAxisField: Transformer< return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -50,12 +54,15 @@ export const patchColorField: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { cell } = context; - const cellNew = { ...cell, color: cell.color ?? cell.category }; + const { cells, chartType } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...cells[0], color: cells[0].color ?? cells[0].category }; return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -63,9 +70,12 @@ export const patchLabelField: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { cell } = context; + const { cells, chartType } = context; + if (isCombinationChartType(chartType)) { + return {}; + } - const cellNew: any = { ...cell }; + const cellNew: any = { ...cells[0] }; //patch the "label" fields to color if (cellNew.label && (!cellNew.color || cellNew.color.length === 0)) { cellNew.color = cellNew.label; @@ -73,7 +83,7 @@ export const patchLabelField: Transformer< return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -81,8 +91,11 @@ export const patchYField: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, dataset, fieldInfo } = context; - let cellNew = { ...cell }; + const { chartType, cells, dataset, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + let cellNew = { ...cells[0] }; const { x, y } = cellNew; let chartTypeNew = chartType; let datasetNew = dataset; @@ -118,7 +131,7 @@ export const patchYField: Transformer< } else { chartTypeNew = ChartType.ScatterPlot.toUpperCase(); cellNew = { - ...cell, + ...cells[0], x: y[0], y: y[1], color: typeof x === 'string' ? x : x[0] @@ -129,7 +142,7 @@ export const patchYField: Transformer< return { //...context, chartType: chartTypeNew, - cell: cellNew, + cells: [cellNew], dataset: datasetNew }; }; @@ -138,9 +151,12 @@ export const patchBoxPlot: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell } = context; + const { chartType, cells } = context; + if (isCombinationChartType(chartType)) { + return {}; + } const cellNew = { - ...cell + ...cells[0] }; const { y } = cellNew; if (chartType === ChartType.BoxPlot.toUpperCase()) { @@ -197,7 +213,7 @@ export const patchBoxPlot: Transformer< return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -205,8 +221,11 @@ export const patchDualAxis: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell } = context; - const cellNew: any = { ...cell }; + const { chartType, cells } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew: any = { ...cells[0] }; //Dual-axis drawing yLeft and yRight if (chartType === ChartType.DualAxisChart.toUpperCase()) { @@ -215,7 +234,7 @@ export const patchDualAxis: Transformer< return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -223,8 +242,11 @@ export const patchPieChart: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, fieldInfo } = context; - const cellNew = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...cells[0] }; if (chartType === ChartType.RoseChart.toUpperCase()) { cellNew.angle = cellNew.radius ?? cellNew.size ?? cellNew.angle; @@ -257,7 +279,7 @@ export const patchPieChart: Transformer< } return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -266,8 +288,11 @@ export const patchWordCloud: Transformer< Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { //Word cloud must have color fields and size fields - const { chartType, cell, fieldInfo } = context; - const cellNew = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...cells[0] }; if (chartType === ChartType.WordCloud.toUpperCase()) { if (!cellNew.size || !cellNew.color || cellNew.color === cellNew.size) { @@ -303,7 +328,7 @@ export const patchWordCloud: Transformer< } return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -311,12 +336,15 @@ export const patchDynamicBarChart: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, fieldInfo } = context; - const cellNew = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...cells[0] }; let chartTypeNew = chartType; if (chartType === ChartType.DynamicBarChart.toUpperCase()) { - if (!cell.time || cell.time === '' || cell.time.length === 0) { + if (!cellNew.time || cellNew.time === '' || cellNew.time.length === 0) { const remainedFields = getRemainedFields(cellNew, fieldInfo); //Dynamic bar chart does not have a time field, choose a discrete field as time. @@ -337,7 +365,7 @@ export const patchDynamicBarChart: Transformer< return { //...context, - cell: cellNew, + cells: [cellNew], chartType: chartTypeNew }; }; @@ -346,8 +374,11 @@ export const patchCartesianXField: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, fieldInfo } = context; - const cellNew = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...cells[0] }; //Cartesian chart must have X field if (CARTESIAN_CHART_LIST.map(chart => chart.toUpperCase()).includes(chartType)) { @@ -364,7 +395,7 @@ export const patchCartesianXField: Transformer< } return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -372,8 +403,11 @@ export const patchNeedColor: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, fieldInfo } = context; - const cellNew: any = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew: any = { ...cells[0] }; if ( NEED_COLOR_FIELD_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) || NEED_COLOR_AND_SIZE_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) @@ -397,7 +431,7 @@ export const patchNeedColor: Transformer< } } return { - cell: cellNew + cells: [cellNew] }; }; @@ -405,8 +439,11 @@ export const patchNeedSize: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, fieldInfo } = context; - const cellNew: any = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew: any = { ...cells[0] }; if ( NEED_SIZE_FIELD_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType) || NEED_COLOR_AND_SIZE_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType) @@ -425,7 +462,7 @@ export const patchNeedSize: Transformer< } } return { - cell: cellNew + cells: [cellNew] }; }; @@ -434,8 +471,11 @@ export const patchRangeColumnChart: Transformer< Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { // Range Column Chart's y field must length == 2 - const { chartType, cell, fieldInfo } = context; - const cellNew = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...cells[0] }; const remainedFields = getRemainedFields(cellNew, fieldInfo); const numericFields = getFieldsByDataType(remainedFields, [DataType.FLOAT, DataType.INT]); if (chartType === ChartType.RangeColumnChart.toUpperCase()) { @@ -453,7 +493,7 @@ export const patchRangeColumnChart: Transformer< } return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -461,8 +501,11 @@ export const patchLinearProgressChart: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, fieldInfo } = context; - const cellNew = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...cells[0] }; if (chartType === ChartType.LinearProgress.toUpperCase()) { const xField = [cellNew.x, cellNew.color].filter(Boolean).flat(); if (xField.length !== 0) { @@ -492,7 +535,7 @@ export const patchLinearProgressChart: Transformer< } return { //...context, - cell: cellNew + cells: [cellNew] }; }; @@ -500,8 +543,11 @@ export const patchBasicHeatMapChart: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, Partial > = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { - const { chartType, cell, fieldInfo } = context; - const cellNew: any = { ...cell }; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew: any = { ...cells[0] }; if (chartType === ChartType.BasicHeatMap.toUpperCase()) { const colorField = [cellNew.x, cellNew.y, cellNew.label, cellNew.color].filter(Boolean).flat(); if (colorField.length >= 2) { @@ -520,7 +566,7 @@ export const patchBasicHeatMapChart: Transformer< } } return { - cell: cellNew + cells: [cellNew] }; }; @@ -543,13 +589,13 @@ export const patchSingleColumnCombinationChart: Transformer< }); subChartType.forEach((chartType, index) => { - const input = { ...context, chartType, cell: cells[index] }; + const input = { ...context, chartType, cells: [cells[index]] }; const result = patchers.reduce((pre, pipeline) => { const res = pipeline(pre); return { ...pre, ...res } as any; }, input); subChartTypeNew[index] = result.chartType; - cellsNew[index] = result.cell; + cellsNew[index] = result.cells[0]; datasetNew[index] = result.dataset; }); diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts index 00fa53e1..d17ddcfe 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/utils.ts @@ -29,13 +29,6 @@ export const parseChartGenerationResponse: Parser< } const { CHART_TYPE, FIELD_MAP, SUB_CHART_TYPE } = advisorResJson; - if ( - COMBINATION_CHART_LIST.every(chartType => { - return chartType.toUpperCase() !== CHART_TYPE.toUpperCase(); - }) - ) { - return { chartType: CHART_TYPE as ChartType, cell: FIELD_MAP[0], usage: advisorRes.usage }; - } return { chartType: CHART_TYPE as ChartType, cells: FIELD_MAP, diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 30857353..16e1e671 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -87,7 +87,8 @@ export const arrayData: Transformer = (context: Con }; export const funnelData: Transformer = (context: Context) => { - const { dataset, cell, spec } = context; + const { dataset, cells, spec } = context; + const cell = cells[0]; // spec.data = [dataset] spec.data = { id: 'data', @@ -110,7 +111,8 @@ export const wordCloudData: Transformer = (context: export const sequenceData: Transformer = ( context: Context & { totalTime: number } ) => { - const { dataset, cell, totalTime, spec } = context; + const { dataset, cells, totalTime, spec } = context; + const cell = cells[0]; const timeField = cell.time as string; const latestData = isValidDataset(dataset) ? dataset : []; @@ -239,7 +241,8 @@ export const sequenceData: Transformer = (context: Context) => { - const { dataset, cell, spec } = context; + const { dataset, cells, spec } = context; + const cell = cells[0]; const { source, target } = cell; const linkData = isValidDataset(dataset) ? dataset : []; const nodes = [ @@ -379,7 +382,8 @@ export const colorLine: Transformer = (context: Con export const cartesianLine: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec, fieldInfo } = context; + const { cells, spec, fieldInfo } = context; + const cell = cells[0]; const cellNew = { ...cell }; spec.xField = cell.x; spec.yField = cell.y; @@ -396,12 +400,13 @@ export const cartesianLine: Transformer = (context: cellNew.color = colorField.fieldName; } } - return { spec, cell: cellNew }; + return { spec, cells: [cellNew] }; }; export const pieField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.valueField = cell.angle || cell.value; if (cell.color || (cell as any).category) { spec.categoryField = cell.color || (cell as any).category; @@ -411,7 +416,8 @@ export const pieField: Transformer = (context: Cont export const scatterField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.xField = cell.x; spec.yField = cell.y; if (cell.color) { @@ -429,7 +435,8 @@ export const scatterField: Transformer = (context: export const wordCloudField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.nameField = cell.color; if (cell.size) { @@ -443,7 +450,8 @@ export const wordCloudField: Transformer = (context export const funnelField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.categoryField = cell.color || cell.x; spec.valueField = cell.value || cell.y; @@ -452,7 +460,8 @@ export const funnelField: Transformer = (context: C export const waterfallField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.xField = cell.x; spec.yField = cell.y; spec.total = { @@ -501,7 +510,8 @@ export const waterfallStackLabel: Transformer = (co export const dualAxisSeries: Transformer = (context: Context) => { //assign series in dual-axis chart - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; const { color } = cell; const dataValues = spec.data.values; @@ -598,7 +608,8 @@ export const wordCloudDisplayConf: Transformer = (c }; export const radarField: Transformer = (context: Context) => { - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; if (cell.x || cell.angle) { spec.categoryField = cell.x ?? cell.angle; } @@ -673,7 +684,8 @@ export const radarAxis: Transformer = (context: Con }; export const sankeyField: Transformer = (context: Context) => { - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.sourceField = cell.source; spec.targetField = cell.target; spec.valueField = cell.value; @@ -684,7 +696,8 @@ export const sankeyField: Transformer = (context: C }; export const boxPlotField: Transformer = (context: Context) => { - const { cell, dataset, spec } = context; + const { cells, dataset, spec } = context; + const cell = cells[0]; const { x, y } = cell; const data = isValidDataset(dataset) ? (dataset as { [key: string]: number }[]) : []; // assign x field @@ -751,7 +764,8 @@ export const sankeyLink: Transformer = (context: Co export const cartesianBar: Transformer = (context: Context) => { //assign fields according to cell - const { cell, fieldInfo, spec } = context; + const { cells, fieldInfo, spec } = context; + const cell = cells[0]; const cellNew = { ...cell }; const flattenedXField = Array.isArray(cell.x) ? cell.x : [cell.x]; if (cell.color && cell.color.length > 0 && cell.color !== cell.x) { @@ -773,12 +787,13 @@ export const cartesianBar: Transformer = (context: cellNew.color = colorField.fieldName; } } - return { spec, cell: cellNew }; + return { spec, cells: [cellNew] }; }; export const rankingBarField: Transformer = (context: Context) => { //折线图根据cell分配字段 - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.xField = cell.y; spec.yField = cell.x; if (cell.color) { @@ -791,7 +806,8 @@ export const rankingBarField: Transformer = (contex }; export const roseField: Transformer = (context: Context) => { - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.valueField = cell.radius || cell.angle; if (cell.color) { spec.categoryField = cell.color; @@ -917,7 +933,8 @@ export const axis: Transformer = (context: Context) }; export const legend: Transformer = (context: Context) => { - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; if (!(cell.color || cell.category) && !spec.seriesField && spec.type !== 'common') { return { spec }; } @@ -1157,7 +1174,7 @@ export const animationCartesianPie: Transformer = ( const { spec } = context; const totalTime = context.totalTime ?? DEFAULT_PIE_VIDEO_LENGTH; - const groupKey = context.cell.color as string; + const groupKey = context.cells[0].color as string; const dataValues = spec.data.values as any[]; const groupNum = dataValues.map(d => d[groupKey!]).filter(onlyUnique).length; //const delay = totalTime / groupNum - 1000; @@ -1270,7 +1287,8 @@ export const theme: Transformer = (context: Context }; export const liquidField: Transformer = (context: Context) => { - const { cell, dataset, spec } = context; + const { cells, dataset, spec } = context; + const cell = cells[0]; spec.valueField = cell.value; spec.indicatorSmartInvert = true; @@ -1289,7 +1307,8 @@ export const liquidStyle: Transformer = (context: C export const linearProgressField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.xField = cell.y; spec.yField = cell.x; @@ -1303,7 +1322,8 @@ export const linearProgressField: Transformer = (co export const linearProgressAxes: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; const hasSingleData = spec.data.values && spec.data.values.length === 1; spec.axes = [ @@ -1349,7 +1369,8 @@ export const linearProgressStyle: Transformer = (co export const circularProgressField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.categoryField = cell.color; spec.valueField = cell.value; @@ -1373,7 +1394,8 @@ export const circularProgressStyle: Transformer = ( }; export const indicator: Transformer = (context: Context) => { - const { spec, cell } = context; + const { spec, cells } = context; + const cell = cells[0]; const firstEntry = spec.data.values[0]; if (!firstEntry) { return { spec }; @@ -1411,7 +1433,8 @@ export const indicator: Transformer = (context: Con }; export const bubbleCirclePackingData: Transformer = (context: Context) => { - const { dataset, spec, cell } = context; + const { dataset, spec, cells } = context; + const cell = cells[0]; if (cell.size) { dataset.forEach(data => { data.value = data[cell.size]; @@ -1423,7 +1446,8 @@ export const bubbleCirclePackingData: Transformer = export const bubbleCirclePackingField: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.categoryField = cell.color || cell.x; if (cell.size) { @@ -1451,7 +1475,8 @@ export const bubbleCirclePackingDisplayConf: Transformer = (context: Context) => { //assign field in spec according to cell - const { cell, spec } = context; + const { cells, spec } = context; + const cell = cells[0]; spec.yField = cell.x; spec.xField = [cell.y[0], cell.y[1]]; @@ -1469,7 +1494,8 @@ export const rangeColumnDisplayConf: Transformer = }; export const sunburstData: Transformer = (context: Context) => { - const { dataset, cell, spec } = context; + const { dataset, cells, spec } = context; + const cell = cells[0]; spec.data = { id: 'data', values: getSunburstData(dataset, cell.color, 0, cell.size) }; return { spec }; }; @@ -1565,7 +1591,8 @@ export const sunburstDisplayConf: Transformer = (co }; export const treemapData: Transformer = (context: Context) => { - const { dataset, cell, spec } = context; + const { dataset, cells, spec } = context; + const cell = cells[0]; spec.data = { id: 'data', values: getTreemapData(dataset, cell.color, 0, cell.size) }; return { spec }; }; @@ -1616,7 +1643,8 @@ export const treemapDisplayConf: Transformer = (con }; export const gaugeField: Transformer = (context: Context) => { - const { spec, cell } = context; + const { spec, cells } = context; + const cell = cells[0]; spec.valueField = cell.size; spec.categoryField = cell.color; return { spec }; @@ -1632,7 +1660,8 @@ export const gaugeDisplayConf: Transformer = (conte }; export const vennData: Transformer = (context: Context) => { - const { dataset, spec, cell } = context; + const { dataset, spec, cells } = context; + const cell = cells[0]; const id2dataMap = {}; const setsField = cell.color[0]; const nameField = cell.color[1]; @@ -1666,7 +1695,8 @@ export const registerChart: Transformer = (context: }; export const basicHeatMapSeries: Transformer = (context: Context) => { - const { spec, cell } = context; + const { spec, cells } = context; + const cell = cells[0]; spec.series = [ { type: 'heatmap', @@ -1701,7 +1731,8 @@ export const basicHeatMapRegion: Transformer = (con return { spec }; }; export const basicHeatMapColor: Transformer = (context: Context) => { - const { spec, cell } = context; + const { spec, cells } = context; + const cell = cells[0]; spec.color = { type: 'linear', domain: [ @@ -1798,7 +1829,8 @@ export const basemap: Transformer = (context: Conte }; export const mapField: Transformer = (context: Context) => { - const { spec, cell } = context; + const { spec, cells } = context; + const cell = cells[0]; spec.nameField = cell.color; spec.valueField = cell.size; @@ -1807,7 +1839,8 @@ export const mapField: Transformer = (context: Cont }; export const mapDisplayConf: Transformer = (context: Context) => { - const { spec, cell } = context; + const { spec, cells } = context; + const cell = cells[0]; spec.legends = [ { visible: true, @@ -1888,14 +1921,14 @@ export const commonSingleColumnSeries: Transformer switch (subChartType[index].toUpperCase()) { case CombinationBasicChartType.BarChart.toUpperCase(): - specNew = cartesianBar({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); + specNew = cartesianBar({ ...context, cells: [cells[index]], spec: {}, fieldInfo: fieldInfo }); return { ...seriesSubset, ...specNew.spec }; case CombinationBasicChartType.LineChart.toUpperCase(): - specNew = cartesianLine({ ...context, cell: cells[index], spec: {}, fieldInfo: fieldInfo }); + specNew = cartesianLine({ ...context, cells: [cells[index]], spec: {}, fieldInfo: fieldInfo }); return { ...seriesSubset, ...specNew.spec diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts index 2a439e77..e035f299 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts @@ -2,6 +2,7 @@ import type { Transformer } from '../../../base/tools/transformer'; import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } from './generateTypeAndFieldMap/types'; import type { ChartType } from '../../../common/typings'; import { replaceAll } from '../../../common/utils/utils'; +import { COMBINATION_CHART_LIST } from '../constants'; export const addChartSource: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, @@ -24,3 +25,9 @@ export const patchChartType: Transformer< return { chartType: chartTypeNew.toUpperCase() as ChartType }; }; + +export const isCombinationChartType = (chartType: ChartType) => { + return COMBINATION_CHART_LIST.some( + combinationChartType => combinationChartType.toUpperCase() === chartType.toUpperCase() + ); +}; diff --git a/packages/vmind/src/applications/types.ts b/packages/vmind/src/applications/types.ts index dad9b8b4..1c3dae12 100644 --- a/packages/vmind/src/applications/types.ts +++ b/packages/vmind/src/applications/types.ts @@ -56,7 +56,6 @@ export type ChartGenerationContext = { export type ChartGenerationOutput = { chartType: ChartType; subChartType?: CombinationBasicChartType[]; - cell?: Cell; cells?: Cell[]; datasets?: VMindDataset[]; spec: any; diff --git a/packages/vmind/src/core/VMind.ts b/packages/vmind/src/core/VMind.ts index 805f755c..14b18d30 100644 --- a/packages/vmind/src/core/VMind.ts +++ b/packages/vmind/src/core/VMind.ts @@ -236,13 +236,12 @@ class VMind { return chartGenerationResult.advisedList; } - const { chartType, subChartType, spec, cell, cells, chartSource, time } = chartGenerationResult; + const { chartType, subChartType, spec, cells, chartSource, time } = chartGenerationResult; const usage = calculateTokenUsage([queryDatasetUsage, chartGenerationResult.usage]); return { //...chartGenerationResult, spec, usage, - cell, chartSource, chartType, subChartType, From cbcc52f1598685926cc4fd5e3e0ca99e37f41060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Mon, 2 Sep 2024 16:59:19 +0800 Subject: [PATCH 018/128] feat: Skylark model simultaneously expands basic chart types and combination charts; unifies the way to obtain cells. --- .../browser/src/constants/mockData.ts | 6 +- .../src/pages/ChartGeneration/DataInput.tsx | 6 +- .../applications/chartGeneration/constants.ts | 14 +- .../taskNodes/chartAdvisor/transformers.ts | 6 +- .../generateChartType/skylark/prompt/index.ts | 2 + .../skylark/prompt/knowledge.ts | 49 ++++++ .../skylark/prompt/template.ts | 2 + .../generateChartType/skylark/utils.ts | 4 +- .../generateFieldMap/skylark/patcher/index.ts | 163 ++++++++++++++---- .../generateFieldMap/skylark/prompt/index.ts | 29 +++- .../skylark/prompt/knowledge.ts | 132 ++++++++++++-- .../skylark/prompt/template.ts | 13 +- .../generateFieldMap/skylark/utils.ts | 2 +- .../taskNodes/generateFieldMap/types.ts | 2 +- .../GPT/patcher/index.ts | 58 +++---- .../GPT/prompt/index.ts | 4 +- .../GPT/prompt/template.ts | 4 +- .../getChartSpec/VChart/transformers.ts | 92 +++++----- .../chartGeneration/taskNodes/utils.ts | 9 + packages/vmind/src/common/specUtils/index.ts | 6 +- packages/vmind/src/common/typings/index.ts | 4 +- packages/vmind/src/common/utils/skylark.ts | 4 +- packages/vmind/src/common/utils/utils.ts | 9 + 23 files changed, 460 insertions(+), 160 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index ae7c464c..5d327eac 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -5213,7 +5213,7 @@ export const vennChartData = { 7,A,2 7,B,2 7,C,2`, - input: '帮我展示各元素集合重叠区域，以及分布情况。请使用sets字段来区分不同区域。' + input: '帮我展示各元素集合重叠区域，以及分布情况。请使用数据中的sets字段来划分不同集合。' }; export const singleColumnLineCombinationChartData = { @@ -5398,7 +5398,7 @@ export const singleColumnLineCombinationChartData = { 2022-09-01,0.259653267575217,2.040817156148029,2.19857288799284,2.229208371156883 2022-09-02,1.398428414171018,0.071469482611002,0.9048807067534731,0.0022491420541680004 2022-09-03,1.7166677805176591,1.903668070163285,1.866568462888393,1.8648831840830011`, - input: '请使用组合图展示不同类别的权重随着时间的变化' + input: '请使用四个折线的组合图展示不同类别的权重随着时间的变化' }; export const singleColumnLineCombinationChartData1 = { @@ -5889,7 +5889,7 @@ export const mockUserTextInput10 = { export const mockProgressData = { csv: `年份,进度 - 2024,0.56`, + 2024年,0.56`, input: '请使用环形进度图帮我展示今年的进度数据' }; diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index 1f3aae53..c093aa0b 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -195,14 +195,14 @@ export function DataInput(props: IPropsType) { console.log(chartGenerationRes); if (isArray(chartGenerationRes)) { const resNew = chartGenerationRes.map(res => { - const { spec, cell } = res; + const { spec, cells } = res; specTemplateTest && (spec.data = undefined); - const finalSpec = specTemplateTest ? vmind.fillSpecWithData(spec, dataset, cell) : spec; + const finalSpec = specTemplateTest ? vmind.fillSpecWithData(spec, dataset, cells) : spec; return finalSpec; }); props.onSpecListGenerate(resNew); } else { - const { spec, time, cell } = chartGenerationRes; + const { spec, time, cells } = chartGenerationRes; const finalSpec = specTemplateTest ? vmind.fillSpecWithData(spec, dataset) : spec; diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index bba57823..f40c6bca 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -2,17 +2,12 @@ import type { BasemapOption } from '../../common/typings'; import { ChartType, CombinationBasicChartType, CombinationChartType, MapRegionCoordinate } from '../../common/typings'; export const SUPPORTED_CHART_LIST = Object.values(ChartType); -export const BASIC_CHART_LIST = Object.values(CombinationBasicChartType); +export const COMBINATION_BASIC_CHART_LIST = Object.values(CombinationBasicChartType); export const COMBINATION_CHART_LIST = Object.values(CombinationChartType); -export const NEED_COLOR_FIELD_CHART_LIST = [ - ChartType.PieChart, - ChartType.RoseChart, - ChartType.LinearProgress, - ChartType.LineChart -]; +export const NEED_COLOR_FIELD_CHART_LIST = [ChartType.PieChart, ChartType.RoseChart, ChartType.LinearProgress]; -export const NEED_SIZE_FIELD_CHART_LIST = [ChartType.ScatterPlot, ChartType.BasicHeatMap]; +export const NEED_SIZE_FIELD_CHART_LIST = [ChartType.ScatterPlot, ChartType.BasicHeatMap, ChartType.LiquidChart]; export const NEED_COLOR_AND_SIZE_CHART_LIST = [ ChartType.WordCloud, @@ -22,8 +17,7 @@ export const NEED_COLOR_AND_SIZE_CHART_LIST = [ ChartType.Gauge, ChartType.SunburstChart, ChartType.TreemapChart, - ChartType.CircularProgress, - ChartType.LiquidChart + ChartType.CircularProgress ]; export const CARTESIAN_CHART_LIST = [ diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts index 7b9fa89c..db93c95c 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/chartAdvisor/transformers.ts @@ -36,7 +36,7 @@ type getAdvisedListOutput = { chartSource: string; advisedList: { chartType: string; - cell: Cell; + cells: Cell[]; dataset: VMindDataset; score: number; }[]; @@ -70,7 +70,7 @@ export const getAdvisedListTransformer: Transformer availableChartTypeList.includes(d.chartType) && d.score - 0 >= 0.00000001) .map((result: any) => ({ chartType: chartTypeMap(result.chartType as unknown as ChartType).toUpperCase(), - cell: getCell(result.cell), + cells: [getCell(result.cell)], dataset: result.dataset, score: result.score })); @@ -101,7 +101,7 @@ const getTop1AdvisedChart: Transformer const result = advisedList[0]; return { chartType: result.chartType as VMindChartType, - cells: [getCell(result.cell)], + cells: [getCell(result.cells[0])], dataset: result.dataset, chartSource, usage diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/index.ts index f087b4e2..0e1f0d0b 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/index.ts @@ -3,6 +3,7 @@ import type { GenerateChartTypeContext } from '../../types'; import { getChartRecommendPrompt } from './template'; import { getStrFromArray } from '../../../../../../common/utils/utils'; import { chartKnowledgeBase } from './knowledge'; +import { COMBINATION_CHART_LIST } from '../../../../constants'; export class ChartTypeGenerationPrompt extends Prompt { constructor() { @@ -25,6 +26,7 @@ export class ChartTypeGenerationPrompt extends Prompt const chartRecommendPrompt = getChartRecommendPrompt( chartTypeList, + COMBINATION_CHART_LIST, chartRecommendKnowledgeStr, chartRecommendConstraintsStr, llmOptions.showThoughts ?? true diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts index f1a55f98..38fb8e62 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts @@ -1,6 +1,7 @@ /* eslint-disable max-len */ import { ChartType } from '../../../../../../common/typings'; +import { COMBINATION_BASIC_CHART_LIST } from '../../../../constants'; type ChartKnowledgeBase = { [chartType: string]: { @@ -73,5 +74,53 @@ export const chartKnowledgeBase: ChartKnowledgeBase = { [ChartType.LiquidChart]: { knowledge: ['Liquid chart show a percent value'], constraints: ['Use Liquid Chart if user want to show a percent value'] + }, + [ChartType.BubbleCirclePacking]: { + knowledge: [ + 'Bubble Circle Packing is useful for visualizing hierarchical data with circles representing nodes, where the size and color can convey additional information about the data points.' + ] + }, + [ChartType.MapChart]: { + knowledge: [ + 'Map Charts are used to visualize geographical data, allowing for the identification of spatial patterns and trends.' + ] + }, + [ChartType.RangeColumnChart]: { + knowledge: [ + 'Range Column Charts are designed to show data that has a minimum and a maximum value, making them effective for displaying variability or uncertainty within data sets.' + ] + }, + [ChartType.SunburstChart]: { + knowledge: [ + 'Sunburst Charts are excellent for visualizing hierarchical data, allowing users to see relationships between categories and subcategories at varying levels of detail.', + 'The colors field for sunburst chart and treemap chart must be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + ] + }, + [ChartType.TreemapChart]: { + knowledge: [ + 'Treemap Charts are effective for displaying large amounts of hierarchical data in a compact space, where areas represent the size of each category.', + 'The colors field for sunburst chart and treemap chart must be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + ] + }, + [ChartType.Gauge]: { + knowledge: [ + 'Gauge Charts are useful for displaying performance metrics against a target, providing a quick visual summary at a glance.', + 'The gauge chart must contain two fields: size and color.' + ] + }, + [ChartType.BasicHeatMap]: { + knowledge: [ + 'Basic Heat Maps are great for visualizing data intensity over two dimensions, helping to identify patterns through color coding.' + ] + }, + [ChartType.VennChart]: { + knowledge: [ + 'Venn Charts are useful for displaying the relationships between different groups, emphasizing similarities and differences visually.', + 'The color field of the Venn diagram requires an array of length 2. The field with subscript 0 maps to the sets, and the field with subscript 1 maps to the name.' + ] + }, + [ChartType.SingleColumnCombinationChart]: { + knowledge: ['Single column combination charts can be combined with a variety of different basic chart types'], + constraints: [`subChartType cannot be empty, it is an array of values in ${COMBINATION_BASIC_CHART_LIST}.`] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/template.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/template.ts index b9dda506..091482a1 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/template.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/template.ts @@ -3,6 +3,7 @@ import type { ChartType } from '../../../../../../common/typings'; export const getChartRecommendPrompt = ( chartTypeList: ChartType[], + combinationChartList: string[], knowledgeStr: string, constraintsStr: string, showThoughts: boolean @@ -25,4 +26,5 @@ Response in the following format: ${ showThoughts ? 'thoughts: //Your thoughts\n' : '' }chartType: //chart type you choose based on data and user's command. Only one chart type can be used. +subChartType: //The default value is an empty array. It is assigned only when chartType is in ${combinationChartList}. `; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/utils.ts index 25eec0d7..803bc599 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/utils.ts @@ -23,7 +23,7 @@ export const parseChartTypeResponse: Parser> = (context: PatchContext) => { - const { fieldInfo, cell } = context; - const cellNew = { ...cell }; + const { fieldInfo, cells, chartType } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...getCell(cells) }; const columns = fieldInfo.map(field => field.fieldName); @@ -31,14 +46,17 @@ const patchNullField: Transformer> }); return { - cell: cellNew + cells: [cellNew] }; }; const patchField: Transformer> = (context: PatchContext) => { - const { fieldInfo, cell } = context; + const { fieldInfo, cells, chartType } = context; + if (isCombinationChartType(chartType)) { + return {}; + } const fieldNames = fieldInfo.map(field => field.fieldName); - const cellNew = { ...cell }; + const cellNew = { ...getCell(cells) }; Object.keys(cellNew).forEach(key => { const value = cellNew[key]; if (isString(value) && (value ?? '').includes(',')) { @@ -49,13 +67,16 @@ const patchField: Transformer> = ( } }); return { - cell: cellNew + cells: [cellNew] }; }; const patchColorField: Transformer> = (context: PatchContext) => { - const { chartType, fieldInfo, cell } = context; - const cellNew = { ...cell }; + const { chartType, fieldInfo, cells } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...getCell(cells) }; const { color } = cellNew; let chartTypeNew = chartType; if (color) { @@ -72,38 +93,45 @@ const patchColorField: Transformer } return { - cell: cellNew + cells: [cellNew], + chartType: chartTypeNew }; }; const patchRadarChart: Transformer> = (context: PatchContext) => { - const { chartType, cell } = context; + const { chartType, cells } = context; + if (isCombinationChartType(chartType)) { + return {}; + } if (chartType === ChartType.RadarChart.toUpperCase()) { const cellNew = { - x: cell.angle, - y: cell.value, - color: cell.color + x: getCell(cells).angle, + y: getCell(cells).value, + color: getCell(cells).color }; return { - cell: cellNew + cells: [cellNew] }; } return context; }; const patchBoxPlot: Transformer> = (context: PatchContext) => { - const { chartType, cell } = context; + const { chartType, cells } = context; + if (isCombinationChartType(chartType)) { + return {}; + } if (chartType === ChartType.BoxPlot.toUpperCase()) { - const { x, min, q1, median, q3, max } = cell as any; + const { x, min, q1, median, q3, max } = getCell(cells) as any; const cellNew = { x, y: [min, q1, median, q3, max].filter(Boolean) }; return { - cell: cellNew + cells: [cellNew] }; } return context; @@ -111,9 +139,12 @@ const patchBoxPlot: Transformer> = const patchDifferentMeasureLevel: Transformer = (context: PatchContext) => { //if the difference in the level of two measures in cartesian chart is enormous, use Dual-axis chart - const { chartType, cell, dataset } = context; + const { chartType, cells, dataset } = context; + if (isCombinationChartType(chartType)) { + return {}; + } const chartTypeNew = chartType; - const cellNew = { ...cell }; + const cellNew = { ...getCell(cells) }; const datasetNew = dataset; if (chartTypeNew === ChartType.BarChart.toUpperCase()) { if (isValidDataset(datasetNew) && isArray(cellNew.y) && cellNew.y.length === 2) { @@ -147,9 +178,12 @@ const patchDifferentMeasureLevel: Transformer = (context: Pat }; const patchFoldField: Transformer> = (context: PatchContext) => { - const { chartType, cell, fieldInfo, dataset } = context; - const chartTypeNew = chartType; - const cellNew = { ...cell }; + const { chartType, cells, fieldInfo, dataset } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const chartTypeNew = chartType.toUpperCase(); + const cellNew = { ...getCell(cells) }; let datasetNew = dataset; if ( chartTypeNew === ChartType.BarChart.toUpperCase() || @@ -164,14 +198,17 @@ const patchFoldField: Transformer> } return { chartType: chartTypeNew, - cell: cellNew, + cells: [cellNew], dataset: datasetNew }; }; const patchDualAxisChart: Transformer> = (context: PatchContext) => { - const { chartType, cell } = context; - const cellNew: any = { ...cell }; + const { chartType, cells } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew: any = { ...getCell(cells) }; //Dual-axis drawing yLeft and yRight if (chartType === ChartType.DualAxisChart.toUpperCase()) { @@ -184,13 +221,16 @@ const patchDualAxisChart: Transformer> = (context: PatchContext) => { - const { chartType, cell, fieldInfo } = context; + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } const cellNew = { - ...cell + ...getCell(cells) }; let chartTypeNew = chartType; @@ -221,25 +261,28 @@ const patchDynamicBarChart: Transformer> = (context: PatchContext) => { - const { cell } = context; + const { cells, chartType } = context; + if (isCombinationChartType(chartType)) { + return {}; + } const cellNew = { - ...cell + ...getCell(cells) }; - //only x and y field can be array + //only x, y and color field can be array Object.keys(cellNew).forEach(key => { - if (key !== 'x' && key !== 'y' && isArray(cellNew[key])) { + if (key !== 'x' && key !== 'y' && key !== 'color' && isArray(cellNew[key])) { cellNew[key] = cellNew[key][0]; } }); return { - cell: cellNew + cells: [cellNew] }; }; @@ -252,10 +295,53 @@ const calculateUsage: Transformer = (context: PatchContext) = }; }; +const patchSingleColumnCombinationChart: Transformer> = ( + context: PatchContext +) => { + const { chartType, cells, subChartType } = context; + if ( + COMBINATION_CHART_LIST.some(combinationChartType => { + return chartType.toUpperCase() === combinationChartType.toUpperCase(); + }) + ) { + const cellsNew: Cell[] = [...cells]; + const subChartTypeNew: CombinationBasicChartType[] = [...subChartType]; + const datasetNew: VMindDataset[] = []; + + const patchers = GenerateFieldMapTaskNodeMeta.patcher.filter(patch => { + return patch.name !== 'patchSingleColumnCombinationChart'; + }); + const minLength = Math.min(cells.length, subChartType.length); + for (let index = 0; index < minLength; index++) { + const input = { + ...context, + chartType: subChartType[index].toString() as ChartType, + cells: [getCell(cells, index)] + }; + const result = patchers.reduce((pre, pipeline) => { + const res = pipeline(pre); + return { ...pre, ...res } as any; + }, input); + subChartTypeNew[index] = result.chartType.toString() as CombinationBasicChartType; + cellsNew[index] = getCell(result.cells); + datasetNew[index] = result.dataset; + } + + return { + subChartType: subChartTypeNew, + cells: cellsNew, + datasetsForCombinationChart: datasetNew + }; + } + return {}; +}; + export const patchPipelines: Transformer>[] = [ addChartSource as any, calculateUsage, patchNullField, + patchNeedColor, + patchNeedSize, patchField, patchColorField, patchRadarChart, @@ -264,5 +350,10 @@ export const patchPipelines: Transformer { @@ -9,15 +14,27 @@ export class FieldMapGenerationPrompt extends Prompt { super(''); } getSystemPrompt(context: GenerateFieldMapContext) { - const { chartType, llmOptions } = context; + const { chartType, llmOptions, subChartType } = context; + const chartTypeList = subChartType && subChartType.length !== 0 ? subChartType : [chartType]; + const infoMap: any = { + visualChannelInfoStrMap: {}, + channelResponseStrArray: [], + fieldMapKnowledgeStrArray: [] + }; + chartTypeList.forEach(chartType => { + const { visualChannels, responseDescription, knowledge } = ChartFieldInfo[chartType.toUpperCase()]; + infoMap.visualChannelInfoStrMap[chartType] = getStrFromDict(visualChannels); + infoMap.channelResponseStrArray.push(getYAMLArrayStrFromDict(responseDescription)); + infoMap.fieldMapKnowledgeStrArray.push(...knowledge); + }); //call skylark to get field map result. - const { visualChannels, responseDescription, knowledge } = ChartFieldInfo[chartType.toUpperCase()]; - const visualChannelInfoStr = getStrFromDict(visualChannels); - const channelResponseStr = getStrFromDict(responseDescription); - const fieldMapKnowledgeStr = getStrFromArray(knowledge); + const visualChannelInfoStr = getStrFromDict(infoMap.visualChannelInfoStrMap); + const channelResponseStr = getYAMLStrFromArray(infoMap.channelResponseStrArray); + const fieldMapKnowledgeStr = getStrFromArray(Array.from(new Set(infoMap.fieldMapKnowledgeStrArray))); const fieldMapPrompt = getFieldMapPrompt( chartType, + subChartType, visualChannelInfoStr, channelResponseStr, fieldMapKnowledgeStr, diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts index 807d957e..568dac64 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts @@ -1,6 +1,7 @@ /* eslint-disable max-len */ import { ChartType } from '../../../../../../common/typings'; import type { ChannelInfo } from './types'; +import { COMBINATION_BASIC_CHART_LIST } from '../../../../constants'; export const ChartFieldInfo: ChannelInfo = { 'BAR CHART': { @@ -228,20 +229,15 @@ export const ChartFieldInfo: ChannelInfo = { [ChartType.CircularProgress.toUpperCase()]: { visualChannels: { - x: "x-axis of bar chart. Can't be empty. Can only use categorical field.", - y: "y-axis of bar chart. Can't be empty. Only number fields", - color: - 'color channel of bar. Used to distinguish different bars. Only categorical fields. Can be empty if no suitable field.' + color: 'color channel of Circular Progress Chart is used to mark the title.', + value: 'value channel of Circular Progress Chart.' }, responseDescription: { - x: 'field assigned to x channel', - y: 'field assigned to y channel', - color: 'field assigned to color channel' + color: 'field assigned to color channel', + value: 'field assigned to value channel' }, knowledge: [ - "x-axis in linear progress chart can only be a categorical field. Don't use time field", - 'Only use categorical field can be used in x channel', - 'y field can not be empty' + 'Circular progress chart is also used to display progress data, presented in a circular form, with the values on the numerical axis typically ranging from 0 to 1.' ] }, @@ -256,5 +252,121 @@ export const ChartFieldInfo: ChannelInfo = { 'Liquid chart is used to display a single value, with the value range typically from 0 to 1.', 'The value usually represents progress, completion, or percentage, and is associated with only one field' ] + }, + + [ChartType.BubbleCirclePacking.toUpperCase()]: { + visualChannels: { + color: 'color channel of bubble circle packing. Used to differentiate between bubbles.', + size: 'size channel of bubble circle packing. Mapped to the size of each bubble. Only number fields.' + }, + responseDescription: { + color: 'field assigned to color channel', + size: 'field assigned to size channel' + }, + knowledge: ['Only number fields can be used in size channel.'] + }, + [ChartType.MapChart.toUpperCase()]: { + visualChannels: { + color: 'color channel of map chart. Used to represent different regions/distinctions.', + size: 'size channel of map chart. Represents a numeric value for each region. Only number fields.' + }, + responseDescription: { + color: 'field assigned to color channel', + size: 'field assigned to size channel' + }, + knowledge: ['Color fields should represent categorical variables, while size fields represent numerical data.'] + }, + [ChartType.RangeColumnChart.toUpperCase()]: { + visualChannels: { + y: "y-axis of range column chart. An array of length 2 is required to map to the numeric field, representing the maximum and minimum values respectively. Can't be empty.", + x: "x-axis of range column chart. Represents the categorical field for grouping. Can't be empty." + }, + responseDescription: { + y: 'field assigned to y channel', + x: 'field assigned to x channel' + }, + knowledge: [] + }, + [ChartType.SunburstChart.toUpperCase()]: { + visualChannels: { + color: + "color channel of sunburst chart. Used to represent different layers in the hierarchy. Must be an array. Can't be empty. The elements in the array are sorted from the highest level to the lowest level.", + size: 'size channel of sunburst chart. Mapped to the relative size of each segment.' + }, + responseDescription: { + color: 'field assigned to color channel', + size: 'field assigned to size channel' + }, + knowledge: [ + 'The color field must be an array sorted from large to small according to the coverage described by the data field.' + ] + }, + [ChartType.TreemapChart.toUpperCase()]: { + visualChannels: { + color: + "The color channel of the treemap is used to distinguish different areas. It must be an array. Can't be empty. The elements in the array are sorted from the highest level to the lowest level.", + size: 'size channel of treemap chart. Represents the area occupied by each segment.' + }, + responseDescription: { + color: 'field assigned to color channel', + size: 'field assigned to size channel' + }, + knowledge: [ + 'The color field for treemap chart must be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + ] + }, + [ChartType.Gauge.toUpperCase()]: { + visualChannels: { + color: + 'color channel of gauge chart. Used to represent the current value against a range. Must be a string field.', + size: "size channel of gauge chart. Represents a numeric value indicating the current state. Can't be empty." + }, + responseDescription: { + color: 'field assigned to color channel', + size: 'field assigned to size channel' + }, + knowledge: [ + 'The gauge chart must contain two fields: size and color. They are essential for displaying the gauge effectively.' + ] + }, + [ChartType.BasicHeatMap.toUpperCase()]: { + visualChannels: { + y: "y-axis of basic heat map. Represents categorical values along the vertical axis. Can't be empty.", + x: "x-axis of basic heat map. Represents categorical values along the horizontal axis. Can't be empty.", + size: 'size channel of basic heat map. Represents the intensity of each cell in the heat map. Only number fields. Can be empty if no suitable field.' + }, + responseDescription: { + y: 'field assigned to y channel', + x: 'field assigned to x channel', + size: 'field assigned to size channel' + }, + knowledge: ['All visual channels must reflect the structure of the data presented.'] + }, + [ChartType.VennChart.toUpperCase()]: { + visualChannels: { + size: "size channel of Venn chart. Represents the area of each segment. Only number fields. Can't be empty.", + color: + 'color channel of Venn chart. Represents different sets. Must be an array of length 2, with the first field for the sets and the second for the labels.' + }, + responseDescription: { + size: 'field assigned to size channel', + color: 'field assigned to color channel' + }, + knowledge: [ + 'The color field of the Venn Chart requires an array of length 2. The field with subscript 0 maps to the sets, and the field with subscript 1 maps to the name.' + ] + }, + [ChartType.SingleColumnCombinationChart.toUpperCase()]: { + visualChannels: { + // 该图表类型没有特定的视觉通道描述 + }, + responseDescription: { + // 响应描述不适用 + }, + knowledge: [ + `Single column combination charts can be combined with a variety of different basic chart types. such as ${COMBINATION_BASIC_CHART_LIST.join( + ',' + )}.` + ] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts index 4d266ee7..8325c590 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts @@ -1,15 +1,24 @@ /* eslint-disable max-len */ export const getFieldMapPrompt = ( chartType: string, + subChartType: string[], availableChannels: string, channelsInResponse: string, channelKnowledge: string, showThoughts: boolean -) => `You are an export in data visualization. User wants to generate a ${chartType.toLocaleLowerCase()} using the fields provided. +) => `You are an expert in data visualization. User wants to generate a ${ + subChartType && subChartType.length !== 0 + ? 'combination of ' + + subChartType.slice(0, subChartType.length - 1).join() + + 'and' + + subChartType[subChartType.length - 1] + : chartType.toLocaleLowerCase() +} using the fields provided. Your task is: 1. Filter out useful fields related to user's command. 2. Assign the useful fields to the available visual channels according to field name and type. -3. Response in YAML format without any additional descriptions +3. If the chart type is a combination chart, the above two steps need to be repeated for each sub-chart generation task of the combination chart. The number of mapping relationships of visual channels in the response needs to be consistent with the number of sub-charts generated. +4. The response is an array in YAML format without any additional descriptions. Available visual channels: ${availableChannels} diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/utils.ts index 050e890f..0d49c68f 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/utils.ts @@ -25,7 +25,7 @@ export const parseFieldMapResponse: ParserChartType.ScatterPlot.toUpperCase(); cellNew = { - ...cells[0], + ...getCell(cells), x: y[0], y: y[1], color: typeof x === 'string' ? x : x[0] @@ -156,7 +156,7 @@ export const patchBoxPlot: Transformer< return {}; } const cellNew = { - ...cells[0] + ...getCell(cells) }; const { y } = cellNew; if (chartType === ChartType.BoxPlot.toUpperCase()) { @@ -225,7 +225,7 @@ export const patchDualAxis: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew: any = { ...cells[0] }; + const cellNew: any = { ...getCell(cells) }; //Dual-axis drawing yLeft and yRight if (chartType === ChartType.DualAxisChart.toUpperCase()) { @@ -246,7 +246,7 @@ export const patchPieChart: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew = { ...cells[0] }; + const cellNew = { ...getCell(cells) }; if (chartType === ChartType.RoseChart.toUpperCase()) { cellNew.angle = cellNew.radius ?? cellNew.size ?? cellNew.angle; @@ -292,7 +292,7 @@ export const patchWordCloud: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew = { ...cells[0] }; + const cellNew = { ...getCell(cells) }; if (chartType === ChartType.WordCloud.toUpperCase()) { if (!cellNew.size || !cellNew.color || cellNew.color === cellNew.size) { @@ -340,7 +340,7 @@ export const patchDynamicBarChart: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew = { ...cells[0] }; + const cellNew = { ...getCell(cells) }; let chartTypeNew = chartType; if (chartType === ChartType.DynamicBarChart.toUpperCase()) { @@ -378,7 +378,7 @@ export const patchCartesianXField: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew = { ...cells[0] }; + const cellNew = { ...getCell(cells) }; //Cartesian chart must have X field if (CARTESIAN_CHART_LIST.map(chart => chart.toUpperCase()).includes(chartType)) { @@ -407,17 +407,12 @@ export const patchNeedColor: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew: any = { ...cells[0] }; + const cellNew: any = { ...getCell(cells) }; if ( NEED_COLOR_FIELD_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) || NEED_COLOR_AND_SIZE_CHART_LIST.some(needColorFieldChartType => needColorFieldChartType.toUpperCase() === chartType) ) { - let colorField; - if (CARTESIAN_CHART_LIST.every(cartesianChartType => cartesianChartType.toUpperCase() !== chartType)) { - colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean); - } else { - colorField = [cellNew.color]; - } + const colorField = [cellNew.color, cellNew.x, cellNew.label, (cellNew as any).sets].filter(Boolean); if (colorField.length !== 0) { cellNew.color = colorField[0]; } else { @@ -443,7 +438,7 @@ export const patchNeedSize: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew: any = { ...cells[0] }; + const cellNew: any = { ...getCell(cells) }; if ( NEED_SIZE_FIELD_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType) || NEED_COLOR_AND_SIZE_CHART_LIST.some(needSizeFieldChartType => needSizeFieldChartType.toUpperCase() === chartType) @@ -475,7 +470,7 @@ export const patchRangeColumnChart: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew = { ...cells[0] }; + const cellNew = { ...getCell(cells) }; const remainedFields = getRemainedFields(cellNew, fieldInfo); const numericFields = getFieldsByDataType(remainedFields, [DataType.FLOAT, DataType.INT]); if (chartType === ChartType.RangeColumnChart.toUpperCase()) { @@ -505,7 +500,7 @@ export const patchLinearProgressChart: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew = { ...cells[0] }; + const cellNew = { ...getCell(cells) }; if (chartType === ChartType.LinearProgress.toUpperCase()) { const xField = [cellNew.x, cellNew.color].filter(Boolean).flat(); if (xField.length !== 0) { @@ -547,7 +542,7 @@ export const patchBasicHeatMapChart: Transformer< if (isCombinationChartType(chartType)) { return {}; } - const cellNew: any = { ...cells[0] }; + const cellNew: any = { ...getCell(cells) }; if (chartType === ChartType.BasicHeatMap.toUpperCase()) { const colorField = [cellNew.x, cellNew.y, cellNew.label, cellNew.color].filter(Boolean).flat(); if (colorField.length >= 2) { @@ -588,21 +583,26 @@ export const patchSingleColumnCombinationChart: Transformer< return patch.name !== 'patchSingleColumnCombinationChart'; }); - subChartType.forEach((chartType, index) => { - const input = { ...context, chartType, cells: [cells[index]] }; + const minLength = Math.min(cells.length, subChartType.length); + for (let index = 0; index < minLength; index++) { + const input = { + ...context, + chartType: subChartType[index].toString() as ChartType, + cells: [getCell(cells, index)] + }; const result = patchers.reduce((pre, pipeline) => { const res = pipeline(pre); return { ...pre, ...res } as any; }, input); - subChartTypeNew[index] = result.chartType; - cellsNew[index] = result.cells[0]; + subChartTypeNew[index] = result.chartType.toString() as CombinationBasicChartType; + cellsNew[index] = getCell(result.cells); datasetNew[index] = result.dataset; - }); + } return { subChartType: subChartTypeNew, cells: cellsNew, - datasets: datasetNew + datasetsForCombinationChart: datasetNew }; } return {}; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts index 8b724e0f..3203f364 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts @@ -12,7 +12,7 @@ import { visualChannelInfoMap } from './knowledges'; import { uniqArray } from '@visactor/vutils'; -import { BASIC_CHART_LIST, COMBINATION_CHART_LIST } from '../../../../constants'; +import { COMBINATION_BASIC_CHART_LIST, COMBINATION_CHART_LIST } from '../../../../constants'; const patchUserInput = (userInput: string) => { const FULL_WIDTH_SYMBOLS = ['，', '。']; @@ -103,7 +103,7 @@ export class GPTChartGenerationPrompt extends Prompt chartType.toUpperCase() as ChartType); @@ -88,7 +89,7 @@ export const arrayData: Transformer = (context: Con export const funnelData: Transformer = (context: Context) => { const { dataset, cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); // spec.data = [dataset] spec.data = { id: 'data', @@ -112,7 +113,7 @@ export const sequenceData: Transformer { const { dataset, cells, totalTime, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); const timeField = cell.time as string; const latestData = isValidDataset(dataset) ? dataset : []; @@ -242,7 +243,7 @@ export const sequenceData: Transformer = (context: Context) => { const { dataset, cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); const { source, target } = cell; const linkData = isValidDataset(dataset) ? dataset : []; const nodes = [ @@ -383,7 +384,7 @@ export const colorLine: Transformer = (context: Con export const cartesianLine: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec, fieldInfo } = context; - const cell = cells[0]; + const cell = getCell(cells); const cellNew = { ...cell }; spec.xField = cell.x; spec.yField = cell.y; @@ -406,7 +407,7 @@ export const cartesianLine: Transformer = (context: export const pieField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.valueField = cell.angle || cell.value; if (cell.color || (cell as any).category) { spec.categoryField = cell.color || (cell as any).category; @@ -417,7 +418,7 @@ export const pieField: Transformer = (context: Cont export const scatterField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.xField = cell.x; spec.yField = cell.y; if (cell.color) { @@ -436,7 +437,7 @@ export const scatterField: Transformer = (context: export const wordCloudField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.nameField = cell.color; if (cell.size) { @@ -451,7 +452,7 @@ export const wordCloudField: Transformer = (context export const funnelField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.categoryField = cell.color || cell.x; spec.valueField = cell.value || cell.y; @@ -461,7 +462,7 @@ export const funnelField: Transformer = (context: C export const waterfallField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.xField = cell.x; spec.yField = cell.y; spec.total = { @@ -511,7 +512,7 @@ export const waterfallStackLabel: Transformer = (co export const dualAxisSeries: Transformer = (context: Context) => { //assign series in dual-axis chart const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); const { color } = cell; const dataValues = spec.data.values; @@ -609,7 +610,7 @@ export const wordCloudDisplayConf: Transformer = (c export const radarField: Transformer = (context: Context) => { const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); if (cell.x || cell.angle) { spec.categoryField = cell.x ?? cell.angle; } @@ -685,7 +686,7 @@ export const radarAxis: Transformer = (context: Con export const sankeyField: Transformer = (context: Context) => { const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.sourceField = cell.source; spec.targetField = cell.target; spec.valueField = cell.value; @@ -697,7 +698,7 @@ export const sankeyField: Transformer = (context: C export const boxPlotField: Transformer = (context: Context) => { const { cells, dataset, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); const { x, y } = cell; const data = isValidDataset(dataset) ? (dataset as { [key: string]: number }[]) : []; // assign x field @@ -765,7 +766,7 @@ export const sankeyLink: Transformer = (context: Co export const cartesianBar: Transformer = (context: Context) => { //assign fields according to cell const { cells, fieldInfo, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); const cellNew = { ...cell }; const flattenedXField = Array.isArray(cell.x) ? cell.x : [cell.x]; if (cell.color && cell.color.length > 0 && cell.color !== cell.x) { @@ -793,7 +794,7 @@ export const cartesianBar: Transformer = (context: export const rankingBarField: Transformer = (context: Context) => { //折线图根据cell分配字段 const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.xField = cell.y; spec.yField = cell.x; if (cell.color) { @@ -807,7 +808,7 @@ export const rankingBarField: Transformer = (contex export const roseField: Transformer = (context: Context) => { const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.valueField = cell.radius || cell.angle; if (cell.color) { spec.categoryField = cell.color; @@ -934,7 +935,7 @@ export const axis: Transformer = (context: Context) export const legend: Transformer = (context: Context) => { const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); if (!(cell.color || cell.category) && !spec.seriesField && spec.type !== 'common') { return { spec }; } @@ -1174,7 +1175,7 @@ export const animationCartesianPie: Transformer = ( const { spec } = context; const totalTime = context.totalTime ?? DEFAULT_PIE_VIDEO_LENGTH; - const groupKey = context.cells[0].color as string; + const groupKey = getCell(context.cells).color as string; const dataValues = spec.data.values as any[]; const groupNum = dataValues.map(d => d[groupKey!]).filter(onlyUnique).length; //const delay = totalTime / groupNum - 1000; @@ -1288,7 +1289,7 @@ export const theme: Transformer = (context: Context export const liquidField: Transformer = (context: Context) => { const { cells, dataset, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.valueField = cell.value; spec.indicatorSmartInvert = true; @@ -1308,7 +1309,7 @@ export const liquidStyle: Transformer = (context: C export const linearProgressField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.xField = cell.y; spec.yField = cell.x; @@ -1323,7 +1324,7 @@ export const linearProgressField: Transformer = (co export const linearProgressAxes: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); const hasSingleData = spec.data.values && spec.data.values.length === 1; spec.axes = [ @@ -1370,7 +1371,7 @@ export const linearProgressStyle: Transformer = (co export const circularProgressField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.categoryField = cell.color; spec.valueField = cell.value; @@ -1395,7 +1396,7 @@ export const circularProgressStyle: Transformer = ( export const indicator: Transformer = (context: Context) => { const { spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); const firstEntry = spec.data.values[0]; if (!firstEntry) { return { spec }; @@ -1434,7 +1435,7 @@ export const indicator: Transformer = (context: Con export const bubbleCirclePackingData: Transformer = (context: Context) => { const { dataset, spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); if (cell.size) { dataset.forEach(data => { data.value = data[cell.size]; @@ -1447,7 +1448,7 @@ export const bubbleCirclePackingData: Transformer = export const bubbleCirclePackingField: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.categoryField = cell.color || cell.x; if (cell.size) { @@ -1476,7 +1477,7 @@ export const bubbleCirclePackingDisplayConf: Transformer = (context: Context) => { //assign field in spec according to cell const { cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.yField = cell.x; spec.xField = [cell.y[0], cell.y[1]]; @@ -1495,7 +1496,7 @@ export const rangeColumnDisplayConf: Transformer = export const sunburstData: Transformer = (context: Context) => { const { dataset, cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.data = { id: 'data', values: getSunburstData(dataset, cell.color, 0, cell.size) }; return { spec }; }; @@ -1592,7 +1593,7 @@ export const sunburstDisplayConf: Transformer = (co export const treemapData: Transformer = (context: Context) => { const { dataset, cells, spec } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.data = { id: 'data', values: getTreemapData(dataset, cell.color, 0, cell.size) }; return { spec }; }; @@ -1644,7 +1645,7 @@ export const treemapDisplayConf: Transformer = (con export const gaugeField: Transformer = (context: Context) => { const { spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.valueField = cell.size; spec.categoryField = cell.color; return { spec }; @@ -1661,7 +1662,7 @@ export const gaugeDisplayConf: Transformer = (conte export const vennData: Transformer = (context: Context) => { const { dataset, spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); const id2dataMap = {}; const setsField = cell.color[0]; const nameField = cell.color[1]; @@ -1696,7 +1697,7 @@ export const registerChart: Transformer = (context: export const basicHeatMapSeries: Transformer = (context: Context) => { const { spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.series = [ { type: 'heatmap', @@ -1732,7 +1733,7 @@ export const basicHeatMapRegion: Transformer = (con }; export const basicHeatMapColor: Transformer = (context: Context) => { const { spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.color = { type: 'linear', domain: [ @@ -1830,7 +1831,7 @@ export const basemap: Transformer = (context: Conte export const mapField: Transformer = (context: Context) => { const { spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.nameField = cell.color; spec.valueField = cell.size; @@ -1840,7 +1841,7 @@ export const mapField: Transformer = (context: Cont export const mapDisplayConf: Transformer = (context: Context) => { const { spec, cells } = context; - const cell = cells[0]; + const cell = getCell(cells); spec.legends = [ { visible: true, @@ -1863,13 +1864,16 @@ export const mapDisplayConf: Transformer = (context }; export const commonSingleColumnRegion: Transformer = (context: Context) => { - const { cells, spec, datasets, subChartType } = context; + const { cells, spec, datasetsForCombinationChart, subChartType } = context; const regionList: string[] = []; cells.forEach((cell, index) => { + if (subChartType[index] === undefined) { + return; + } const sizeField = [cell.y, cell.size, cell.angle, cell.radius, cell.value].filter(Boolean).flat(); - const dataset = datasets[index]; + const dataset = datasetsForCombinationChart[index]; if (sizeField.length !== 0) { if (sizeField[0] === FOLD_VALUE.toString()) { const sizeFieldName = [...new Set(dataset.map(data => data[FOLD_NAME.toString()]))]; @@ -1888,16 +1892,18 @@ export const commonSingleColumnRegion: Transformer }; export const commonSingleColumnSeries: Transformer = (context: Context) => { - const { cells, spec, datasets, subChartType, fieldInfo } = context; + const { cells, spec, datasetsForCombinationChart, subChartType, fieldInfo } = context; const commonData = {}; spec.seriesField = 'type'; spec.region.forEach((region: { [id: string]: string }) => { commonData[region.id] = []; }); cells.forEach((cell, index) => { + if (subChartType[index] === undefined) { + return; + } const regionId = spec.regionList[index]; - const dataset = datasets[index]; - + const dataset = datasetsForCombinationChart[index]; dataset.forEach(data => { const subData = { type: regionId @@ -1921,14 +1927,14 @@ export const commonSingleColumnSeries: Transformer switch (subChartType[index].toUpperCase()) { case CombinationBasicChartType.BarChart.toUpperCase(): - specNew = cartesianBar({ ...context, cells: [cells[index]], spec: {}, fieldInfo: fieldInfo }); + specNew = cartesianBar({ ...context, cells: [getCell(cells, index)], spec: {}, fieldInfo: fieldInfo }); return { ...seriesSubset, ...specNew.spec }; case CombinationBasicChartType.LineChart.toUpperCase(): - specNew = cartesianLine({ ...context, cells: [cells[index]], spec: {}, fieldInfo: fieldInfo }); + specNew = cartesianLine({ ...context, cells: [getCell(cells, index)], spec: {}, fieldInfo: fieldInfo }); return { ...seriesSubset, ...specNew.spec @@ -1936,8 +1942,8 @@ export const commonSingleColumnSeries: Transformer default: return { ...seriesSubset, - xField: cells[index].x, - yField: cells[index].y + xField: getCell(cells, index).x, + yField: getCell(cells, index).y }; } }); diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts index e035f299..5a22fb29 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/utils.ts @@ -3,6 +3,7 @@ import type { GenerateChartAndFieldMapContext, GenerateChartAndFieldMapOutput } import type { ChartType } from '../../../common/typings'; import { replaceAll } from '../../../common/utils/utils'; import { COMBINATION_CHART_LIST } from '../constants'; +import type { Cell } from '../types'; export const addChartSource: Transformer< GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, @@ -31,3 +32,11 @@ export const isCombinationChartType = (chartType: ChartType) => { combinationChartType => combinationChartType.toUpperCase() === chartType.toUpperCase() ); }; + +export const getCell = (cells: Cell[], index?: number) => { + if (index && cells.length >= index) { + return cells[index]; + } + // By default, the element with index 0 is returned + return cells[0]; +}; diff --git a/packages/vmind/src/common/specUtils/index.ts b/packages/vmind/src/common/specUtils/index.ts index fcec701b..8993bdf7 100644 --- a/packages/vmind/src/common/specUtils/index.ts +++ b/packages/vmind/src/common/specUtils/index.ts @@ -173,7 +173,7 @@ export const fillSpecTemplateWithData = ( const context: any = { spec: template, dataset: datasetNew, - cell: cellNew, + cells: [cellNew], totalTime }; @@ -205,7 +205,7 @@ export const fillSpecTemplateWithData = ( const contextNew: any = { spec: template, dataset: datasetNew, - cell: cellNew, + cells: [cellNew], totalTime }; const { spec: spec1 } = data(contextNew); @@ -233,7 +233,7 @@ export const fillSpecTemplateWithData = ( const contextNew: any = { spec: template, dataset: datasetNew, - cell: cellNew, + cells: [cellNew], totalTime }; const { spec } = data(contextNew); diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index 12dfb1a6..d2fb2b52 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -181,7 +181,7 @@ export type VMindDataset = DataItem[]; export type PatchContext = { chartType: string; - cell: Cell; + cells: Cell[]; dataset: DataItem[]; fieldInfo: SimpleFieldInfo[]; }; @@ -189,7 +189,7 @@ export type PatchContext = { export type PatchPipeline = ( context: PatchContext, _originalContext: PatchContext -) => { chartType: string; cell: Cell; dataset: DataItem[]; fieldInfo: SimpleFieldInfo[] }; +) => { chartType: string; cells: Cell[]; dataset: DataItem[]; fieldInfo: SimpleFieldInfo[] }; export type TaskError = { error: boolean }; diff --git a/packages/vmind/src/common/utils/skylark.ts b/packages/vmind/src/common/utils/skylark.ts index 762851a7..07ebb500 100644 --- a/packages/vmind/src/common/utils/skylark.ts +++ b/packages/vmind/src/common/utils/skylark.ts @@ -44,7 +44,7 @@ export const requestSkyLark = async (prompt: string, message: string, options: I }; const startsWithTextAndColon = (str: string) => { - const regex = /^.+\:/; + const regex = /^(.+:| +\-)/; return regex.test(str); }; @@ -71,7 +71,7 @@ export const parseSkylarkResponse = (larkResponse: LLMResponse): Record startsWithTextAndColon(str)) //remove blank space at the start of each line - .map((str: string) => str.replace(/^\s+/, '')) + // .map((str: string) => str.replace(/^\s+/, '')) //wrap string list with [] .map((str: string) => { if (isStringArray(str)) { diff --git a/packages/vmind/src/common/utils/utils.ts b/packages/vmind/src/common/utils/utils.ts index b1411b29..3617fb8d 100644 --- a/packages/vmind/src/common/utils/utils.ts +++ b/packages/vmind/src/common/utils/utils.ts @@ -113,6 +113,15 @@ export const getStrFromDict = (dict: Record) => .map(key => `${key}: ${dict[key]}`) .join('\n'); +export const getYAMLArrayStrFromDict = (dict: Record) => + '-' + + Object.keys(dict) + .map(key => ` ${key}: ${dict[key]}`) + .join('\n') + .substring(1); + +export const getYAMLStrFromArray = (array: string[]) => Object.values(array).join('\n'); + export const uniqBy = (array: any, key: string) => { const seen = new Set(); return array.filter((item: any) => { From 83638b254e2688660d83b61f529277f62a7b9966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Wed, 11 Sep 2024 23:02:55 +0800 Subject: [PATCH 019/128] feat: Expanded chart types: dynamic scatter plot chart and dynamic rose chart. --- common/config/rush/pnpm-lock.yaml | 225 +- .../browser/src/constants/mockData.ts | 3955 +++++++++++++++++ .../src/pages/ChartGeneration/DataInput.tsx | 10 +- packages/vmind/package.json | 1 + .../applications/chartGeneration/constants.ts | 6 + .../skylark/prompt/knowledge.ts | 18 + .../generateFieldMap/skylark/patcher/index.ts | 4 +- .../skylark/prompt/knowledge.ts | 38 + .../generateTypeAndFieldMap/GPT/index.ts | 4 +- .../GPT/patcher/index.ts | 71 +- .../GPT/prompt/index.ts | 2 +- .../GPT/prompt/knowledges.ts | 24 +- .../getChartSpec/VChart/chartPipeline.ts | 34 +- .../getChartSpec/VChart/transformers.ts | 240 +- .../taskNodes/getChartSpec/VChart/utils.ts | 79 + packages/vmind/src/common/typings/index.ts | 4 +- 16 files changed, 4593 insertions(+), 122 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index a42eac31..d8f8dd2b 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -150,7 +150,7 @@ importers: '@typescript-eslint/parser': 5.30.0 '@visactor/calculator': workspace:1.2.13 '@visactor/chart-advisor': workspace:1.2.13 - '@visactor/vchart': ^1.10.4 + '@visactor/vchart': ^1.11.4 '@visactor/vchart-theme': ^1.11.2 '@visactor/vdataset': ~0.17.4 '@visactor/vrender-core': ^0.17.23 @@ -161,6 +161,7 @@ importers: axios: ^1.4.0 bayesian-changepoint: ~1.0.1 canvas: ^2.11.2 + chroma-js: ^3.1.1 dayjs: ~1.11.10 density-clustering: ~1.3.0 dotenv: ~16.3.1 @@ -188,7 +189,7 @@ importers: '@stdlib/stats-base-dists-t-quantile': 0.2.1 '@visactor/calculator': link:../calculator '@visactor/chart-advisor': link:../chart-advisor - '@visactor/vchart-theme': 1.11.5_@visactor+vchart@1.10.4 + '@visactor/vchart-theme': 1.11.5_@visactor+vchart@1.12.4 '@visactor/vdataset': 0.17.5 '@visactor/vutils': 0.17.5 alasql: 4.3.3 @@ -219,10 +220,11 @@ importers: '@types/react-dom': 18.3.0 '@typescript-eslint/eslint-plugin': 5.30.0_cow5zg7tx6c3eisi5a4ud5kwia '@typescript-eslint/parser': 5.30.0_vwud3sodsb5zxmzckoj7rdwdbq - '@visactor/vchart': 1.10.4 + '@visactor/vchart': 1.12.4 '@visactor/vrender-core': 0.17.29 '@vitejs/plugin-react': 3.1.0_vite@3.2.6 canvas: 2.11.2 + chroma-js: 3.1.1 dotenv: 16.3.2 eslint: 8.18.0 eslint-config-prettier: 8.5.0_eslint@8.18.0 @@ -5008,31 +5010,32 @@ packages: resolution: {integrity: sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==} dev: true - /@visactor/vchart-theme/1.11.5_@visactor+vchart@1.10.4: + /@visactor/vchart-theme/1.11.5_@visactor+vchart@1.12.4: resolution: {integrity: sha512-scHPV7Gxm7v51fpR+Mv2B9T2vJtOZqwjAaY4CLHu0UE1JZ1REMLC0t/EhxJIfTZAc4ErsTANJ8gYdpi76mCoTw==} peerDependencies: '@visactor/vchart': '>=1.10.4' dependencies: - '@visactor/vchart': 1.10.4 + '@visactor/vchart': 1.12.4 dev: false - /@visactor/vchart/1.10.4: - resolution: {integrity: sha512-vKX+y0H/fvWcOsKqxgFThkGQ220cKDod4vuFURo4osR9frnRqjXtrXi5RT6gYpjveDDVMKHy9wD2T898d9IIug==} + /@visactor/vchart/1.12.4: + resolution: {integrity: sha512-uRw2R/CIzW6LbbnjiSFiIuN4Sno+ZBAMt9dMqqWPh46sp3ZW4MkdVW75dYfWJdekuCZ9x4c5n2aqRbfJPCBI0g==} dependencies: - '@visactor/vdataset': 0.18.8 - '@visactor/vgrammar-core': 0.12.8 - '@visactor/vgrammar-hierarchy': 0.12.8 - '@visactor/vgrammar-projection': 0.12.8 - '@visactor/vgrammar-sankey': 0.12.8 - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vgrammar-wordcloud': 0.12.8 - '@visactor/vgrammar-wordcloud-shape': 0.12.8 - '@visactor/vrender-components': 0.18.8 - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vscale': 0.18.8 - '@visactor/vutils': 0.18.8 - '@visactor/vutils-extension': 1.10.4 + '@visactor/vdataset': 0.18.15 + '@visactor/vgrammar-core': 0.14.5 + '@visactor/vgrammar-hierarchy': 0.14.5 + '@visactor/vgrammar-projection': 0.14.5 + '@visactor/vgrammar-sankey': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vgrammar-venn': 0.14.5 + '@visactor/vgrammar-wordcloud': 0.14.5 + '@visactor/vgrammar-wordcloud-shape': 0.14.5 + '@visactor/vrender-components': 0.20.3 + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vscale': 0.18.15 + '@visactor/vutils': 0.18.15 + '@visactor/vutils-extension': 1.12.4 /@visactor/vdataset/0.17.5: resolution: {integrity: sha512-zVBdLWHWrhldGc8JDjSYF9lvpFT4ZEFQDB0b6yvfSiHzHKHiSco+rWmUFvA7r4ObT6j2QWF1vZAV9To8Ml4vHw==} @@ -5056,13 +5059,13 @@ packages: topojson-client: 3.1.0 dev: false - /@visactor/vdataset/0.18.8: - resolution: {integrity: sha512-vYZtZR0DLaYamZvLWJuelwOPUVY+6c+czN5erdQSAVz+7OEn1b9YgxdFJTzJ/loQIxkJNxF/aYLrqrqBzHaNIA==} + /@visactor/vdataset/0.18.15: + resolution: {integrity: sha512-LWaaunGetH8ThWjz6CE7NMG/LcFS41lTl76UCNJM1msfQxyIsbtoHim7t+Hdvud76oRZM0rxX/V4iB/+JpMRfw==} dependencies: '@turf/flatten': 6.5.0 '@turf/helpers': 6.5.0 '@turf/rewind': 6.5.0 - '@visactor/vutils': 0.18.8 + '@visactor/vutils': 0.18.15 d3-dsv: 2.0.0 d3-geo: 1.12.1 d3-hexbin: 0.2.2 @@ -5077,81 +5080,91 @@ packages: simplify-geojson: 1.0.5 topojson-client: 3.1.0 - /@visactor/vgrammar-coordinate/0.12.8: - resolution: {integrity: sha512-e4/Qc6xVkzDFbPShQ9T7kNa16vELfrE6iwzP5sFHiHOxp3U5nRajYffPK4SSAS6/my9sGplhKK4HC2gpasd0EQ==} + /@visactor/vgrammar-coordinate/0.14.5: + resolution: {integrity: sha512-nfGPvBCXbtsY2yupZSP76+kXnOalP0/Wv6WWiCWW8gWXluiC8pB1HjmmrI82CUxEtJPELF5hBABaaCRw35WZlg==} dependencies: - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vutils': 0.18.8 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vutils': 0.18.15 - /@visactor/vgrammar-core/0.12.8: - resolution: {integrity: sha512-RcvLxYcYK9xJC/vs2IndsVdmtOevqOJ3NWsinO4AGfed/AzWxXKxVTQ30XhylBtOSF3nz8fLdg8vRP8pZYjtng==} + /@visactor/vgrammar-core/0.14.5: + resolution: {integrity: sha512-cVisa48ycJnRxGptBorVVfLEi62MqpBnchUz7qUffkFobTd5t0ACKUttZMSfZksWFnhIuJRNMwqZs4mR5Ne4DQ==} dependencies: - '@visactor/vdataset': 0.18.8 - '@visactor/vgrammar-coordinate': 0.12.8 - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vrender-components': 0.18.8 - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vscale': 0.18.8 - '@visactor/vutils': 0.18.8 + '@visactor/vdataset': 0.18.15 + '@visactor/vgrammar-coordinate': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vrender-components': 0.20.3 + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vscale': 0.18.15 + '@visactor/vutils': 0.18.15 - /@visactor/vgrammar-hierarchy/0.12.8: - resolution: {integrity: sha512-NNYrYRAqa0iGlMdpZPK1yexdmTE2HCmPzz52CotoopLVzzrJ1L7LqO4fTUwT21nZAMsQ2iPHh4AX1bwdQ2IRCQ==} + /@visactor/vgrammar-hierarchy/0.14.5: + resolution: {integrity: sha512-0bqIR802PXeSucOBWSUOGeaWbe47ldqBf+NohycIVa/Wu+cCSgbk9GAaxh0/vTgHQiT384kjfuV+yUt/wlaG3A==} dependencies: - '@visactor/vgrammar-core': 0.12.8 - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vutils': 0.18.8 + '@visactor/vgrammar-core': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vutils': 0.18.15 - /@visactor/vgrammar-projection/0.12.8: - resolution: {integrity: sha512-sDrsMoTticsopUQ4Rtn0uywY6JdCEgF1fresAGOTmLCGPy1AYXNIKXQIi2DYxmNsUb+XUWJNOaml+lHX5YeBbQ==} + /@visactor/vgrammar-projection/0.14.5: + resolution: {integrity: sha512-O84R4HH0j2Et/No2LGMF3xu0+p0hyKMhifn4P2kCW4ts+vjw4xKYOfq/SbZuUUKSnBaCt2Kkait+E8hmSv9y4w==} dependencies: - '@visactor/vgrammar-core': 0.12.8 - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vutils': 0.18.8 + '@visactor/vgrammar-core': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vutils': 0.18.15 d3-geo: 1.12.1 - /@visactor/vgrammar-sankey/0.12.8: - resolution: {integrity: sha512-+Cf8pb8HKcRk/06t+DNbf+EPjPmRdTLI0LefY+QzViWu7+tzEJCH7B79Y4o1pXF4ECWKCx3poiMp8+3I1FWrig==} + /@visactor/vgrammar-sankey/0.14.5: + resolution: {integrity: sha512-ie8cG4xSG4O4sMNm3lKBoRsmIF/scGJ//j+fQ0otmRYDeG7hZJmnXzRPxhynJocE5UuAHyykXbGf7mJuexGXog==} dependencies: - '@visactor/vgrammar-core': 0.12.8 - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vutils': 0.18.8 + '@visactor/vgrammar-core': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vutils': 0.18.15 - /@visactor/vgrammar-util/0.12.8: - resolution: {integrity: sha512-To1YW+oL/zhiIPzRcgjt1TrrSOyKT/kaDh1Agn8w1NpRcac2LZ930QTm9QPAeMHMZC9HjuT8UnJC9GeHbhVlNw==} + /@visactor/vgrammar-util/0.14.5: + resolution: {integrity: sha512-IjsDcdjaypiA8kUZR6a0L8yXdFhhvIabCezVfuoE6l39uOCPUPEiilzot7LNBG37Q4NWkxEPu359Ngh5XHYRpw==} dependencies: - '@visactor/vutils': 0.18.8 + '@visactor/vrender-core': 0.20.3 + '@visactor/vutils': 0.18.15 - /@visactor/vgrammar-wordcloud-shape/0.12.8: - resolution: {integrity: sha512-GaLLEHz/Ru+AyctmD3HcqGbxE+mQsncEY8VQH6F+jf3n0sFZpphWptbSL2Ld2RCf3oo7XERMEPki954XHfuL0A==} + /@visactor/vgrammar-venn/0.14.5: + resolution: {integrity: sha512-Omqk9uXZKos8x8mj4v3lm5aPIMqsJM+LnDGW4Oii2/0loWr9xcGu4TgBqpt4dn78thSEvbQQvssqAwVNrnqM/g==} dependencies: - '@visactor/vgrammar-core': 0.12.8 - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vscale': 0.18.8 - '@visactor/vutils': 0.18.8 + '@visactor/vgrammar-core': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vutils': 0.18.15 - /@visactor/vgrammar-wordcloud/0.12.8: - resolution: {integrity: sha512-WhZCaUc1C0xW5caMHuFsuRPI5Q+4JPkoQdkgNfw1HkJ0oPZUuqnpj++j4h2w+B87yFZ9pAbvyp2VeKZh6Avj2w==} + /@visactor/vgrammar-wordcloud-shape/0.14.5: + resolution: {integrity: sha512-zXrttFEnPVvBv/RnsefM+396Cd21MrlRwiFs+mWM2GcqWs20qf72BceMORNnj1B+2u6kL1ouGqj2XyE2hQowPg==} dependencies: - '@visactor/vgrammar-core': 0.12.8 - '@visactor/vgrammar-util': 0.12.8 - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vutils': 0.18.8 + '@visactor/vgrammar-core': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vscale': 0.18.15 + '@visactor/vutils': 0.18.15 - /@visactor/vrender-components/0.18.8: - resolution: {integrity: sha512-/P7MSXIYFzg++zw/jg+Fqogknp17BJWhaoVrCBIZgF45O+qGkCbxZGa2kMR4dGs05lqLvb6/Zle8D3g3Pr2/qg==} + /@visactor/vgrammar-wordcloud/0.14.5: + resolution: {integrity: sha512-uCCO8liBwTAuBE4wcWKLngpHiRy+JQM1W1Yq7sEF38t+hcPp/zheOkFiPcWoTpLiR+w/rjBDRjq1Z8CGuU2Qqg==} dependencies: - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vscale': 0.17.5 - '@visactor/vutils': 0.18.8 + '@visactor/vgrammar-core': 0.14.5 + '@visactor/vgrammar-util': 0.14.5 + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vutils': 0.18.15 + + /@visactor/vrender-components/0.20.3: + resolution: {integrity: sha512-Lvn2wC9A9QugForwXL8qjqNRTqg8x+gH8Smc4n/g+GJ3GoWEyRfdFofzVhF4lbY0RJBrs+DYmrFrBBvpxS/wkg==} + dependencies: + '@visactor/vrender-core': 0.20.3 + '@visactor/vrender-kits': 0.20.3 + '@visactor/vscale': 0.18.15 + '@visactor/vutils': 0.18.15 /@visactor/vrender-core/0.17.29: resolution: {integrity: sha512-1l2lLfhl33xK8QIotvE/t+k9IqwqyZq66okI+CZvKstRnnC/Ct6VZ5S7Wh7lNyN98zX25bLhA4vsL9GQpu6BMw==} @@ -5160,39 +5173,30 @@ packages: color-convert: 2.0.1 dev: true - /@visactor/vrender-core/0.18.8: - resolution: {integrity: sha512-b3NgF89qsX1DvYAAR/YUIWqZVjjyXt5HF5+xfNh1VGhti6l8CLCa70Hxkg2Hnw3RBVP+o9t2T54U+Mwjg7RBuQ==} + /@visactor/vrender-core/0.20.3: + resolution: {integrity: sha512-R59SlmUZax18tme2A2UZlc70IzX0/vsmNTjkB1e5ugMRHtHon5zbu5Cq87PZCXEm18nl+CKIWUMemyeS2CT7ew==} dependencies: - '@visactor/vutils': 0.18.8 + '@visactor/vutils': 0.18.15 color-convert: 2.0.1 - /@visactor/vrender-kits/0.18.8: - resolution: {integrity: sha512-cJppsZOuFB2BPFKn5Lq3uXDZ0SBQErMblwXdeL70f25MDgv1+FZm+TDoHjo9452rA7nHGSM4GaKoN47gb7HhIg==} + /@visactor/vrender-kits/0.20.3: + resolution: {integrity: sha512-rjdWgDJL1Ulfmnddgjsd23+rp+bjW++7dipmpGPt/zasoaaM72HTaRUznPD9pmBVxnCFCAzCXcRWKxkAxrx6Og==} dependencies: '@resvg/resvg-js': 2.4.1 - '@visactor/vrender-core': 0.18.8 - '@visactor/vutils': 0.18.8 + '@visactor/vrender-core': 0.20.3 + '@visactor/vutils': 0.18.15 roughjs: 4.5.2 - /@visactor/vscale/0.17.5: - resolution: {integrity: sha512-2dkS1IlAJ/IdTp8JElbctOOv6lkHKBKPDm8KvwBo0NuGWQeYAebSeyN3QCdwKbj76gMlCub4zc+xWrS5YiA2zA==} + /@visactor/vscale/0.18.15: + resolution: {integrity: sha512-09dDWc6muJbOMxzp4odCsyLjqAF6u3BOx9kAJJ0tEpKE1AuHL4BTejNe697mJAnXqAo2ynAA+dn+cgWYiW1WQg==} dependencies: - '@visactor/vutils': 0.17.5 + '@visactor/vutils': 0.18.15 - /@visactor/vscale/0.18.8: - resolution: {integrity: sha512-MNeLZ9qKJKZpIGhPTdBl9wPbD5NXIbv7ZuDYSxOXZ3I0gUkeHfgUXBGOpP083r0yjf24/B5o2V9GiJ8w01wSpg==} + /@visactor/vutils-extension/1.12.4: + resolution: {integrity: sha512-LeYlJyzdfuE2z010sLgwKkcj6l2b5ngyYS0VHI5HXsFUo+4mwYKCedBVjD382zY/3ufSOBFBf9F+hNLgeBg6aQ==} dependencies: - '@visactor/vutils': 0.18.8 - - /@visactor/vutils-extension/1.10.4: - resolution: {integrity: sha512-1qoZ+oEz+Ms3/iQrx8AudWi58IFPqjxcRE72Vam5xZN5jxrIdbQftN3BAvmQrcjteDMeSS1HgE0j/mUnxxDR8A==} - dependencies: - '@visactor/vdataset': 0.18.8 - '@visactor/vrender-components': 0.18.8 - '@visactor/vrender-core': 0.18.8 - '@visactor/vrender-kits': 0.18.8 - '@visactor/vscale': 0.18.8 - '@visactor/vutils': 0.18.8 + '@visactor/vdataset': 0.18.15 + '@visactor/vutils': 0.18.15 /@visactor/vutils/0.17.5: resolution: {integrity: sha512-HFN6Pk1Wc1RK842g02MeKOlvdri5L7/nqxMVTqxIvi0XMhHXpmoqN4+/9H+h8LmJpVohyrI/MT85TRBV/rManw==} @@ -5200,6 +5204,14 @@ packages: '@turf/helpers': 6.5.0 '@turf/invariant': 6.5.0 eventemitter3: 4.0.7 + dev: false + + /@visactor/vutils/0.18.15: + resolution: {integrity: sha512-gTw8n14SU4avmqZ6VwpHwqoDfOCq044M2QA43rViNaHBnOQ/ePOPRZHl0heSfGQoMIJSZUD7SowLnn5NJjVXYw==} + dependencies: + '@turf/helpers': 6.5.0 + '@turf/invariant': 6.5.0 + eventemitter3: 4.0.7 /@visactor/vutils/0.18.8: resolution: {integrity: sha512-9+YODg9msVyObDbamt94lsEF/idV8gyW3lf31DhuKsLKbuB/ajvSg6jNKD/FTMoXpmCNwfZgZ0F6wXLwI5aIpw==} @@ -5207,6 +5219,7 @@ packages: '@turf/helpers': 6.5.0 '@turf/invariant': 6.5.0 eventemitter3: 4.0.7 + dev: true /@vitejs/plugin-react/3.1.0_vite@3.2.6: resolution: {integrity: sha512-AfgcRL8ZBhAlc3BFdigClmTUMISmmzHn7sB2h9U1odvc5U/MjWXsAaz18b/WoppUTDBzxOJwo2VdClfUcItu9g==} @@ -6324,6 +6337,10 @@ packages: engines: {node: '>=10'} dev: true + /chroma-js/3.1.1: + resolution: {integrity: sha512-CGr6w73Gi86142RWqZ1RjED/CyduYw2vMTikQZUvr2jGIihnZlMo/Kzm9rYHWDP2pJc6eebwc8CkX0iteBon+A==} + dev: true + /ci-info/2.0.0: resolution: {integrity: sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==} dev: true diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 5d327eac..5cbf1e7e 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -5782,6 +5782,3961 @@ export const singleColumnBarCombinationChartData1 = { input: '请使用组合图展示不同类别的权重随着时间的变化，用四个柱图。' }; +export const dynamicScatterPlotData = { + csv: `年度收益率,净现金流,市值,公司,年份 +50,28,726,公司0,1950 +9,63,301,公司1,1950 +41,8,795,公司2,1950 +93,92,545,公司3,1950 +11,93,798,公司4,1950 +50,11,652,公司5,1950 +35,96,688,公司6,1950 +94,72,795,公司7,1950 +4,4,149,公司8,1950 +4,93,919,公司9,1950 +71,7,976,公司10,1950 +75,7,349,公司11,1950 +93,89,810,公司12,1950 +28,73,602,公司13,1950 +11,73,497,公司14,1950 +87,73,509,公司15,1950 +58,53,256,公司16,1950 +56,69,144,公司17,1950 +49,45,237,公司18,1950 +46,74,387,公司19,1950 +92,42,742,公司20,1950 +45,64,933,公司21,1950 +68,42,284,公司22,1950 +9,46,325,公司23,1950 +70,6,194,公司24,1950 +25,84,552,公司25,1950 +82,87,716,公司26,1950 +5,64,958,公司27,1950 +15,86,593,公司28,1950 +44,11,219,公司29,1950 +46,75,72,公司30,1950 +76,56,530,公司31,1950 +14,8,483,公司32,1950 +82,30,639,公司33,1950 +58,23,876,公司34,1950 +0,28,748,公司35,1950 +55,25,845,公司36,1950 +35,54,185,公司37,1950 +60,1,931,公司38,1950 +64,30,300,公司39,1950 +83,29,510,公司40,1950 +5,77,915,公司41,1950 +23,61,821,公司42,1950 +16,42,22,公司43,1950 +50,73,800,公司44,1950 +93,71,829,公司45,1950 +31,0,293,公司46,1950 +17,75,89,公司47,1950 +8,25,319,公司48,1950 +81,37,851,公司49,1950 +51,31,723,公司0,1951 +10,65,287,公司1,1951 +42,7,813,公司2,1951 +92,90,525,公司3,1951 +12,96,761,公司4,1951 +50,10,681,公司5,1951 +37,96,738,公司6,1951 +97,73,752,公司7,1951 +5,7,189,公司8,1951 +7,93,908,公司9,1951 +72,9,975,公司10,1951 +77,7,309,公司11,1951 +94,89,777,公司12,1951 +31,76,568,公司13,1951 +11,75,523,公司14,1951 +86,73,563,公司15,1951 +58,52,219,公司16,1951 +59,70,176,公司17,1951 +51,48,192,公司18,1951 +49,76,418,公司19,1951 +92,45,747,公司20,1951 +48,67,924,公司21,1951 +71,41,321,公司22,1951 +11,46,304,公司23,1951 +73,8,158,公司24,1951 +25,86,595,公司25,1951 +82,87,673,公司26,1951 +5,67,998,公司27,1951 +18,89,622,公司28,1951 +46,13,247,公司29,1951 +45,74,121,公司30,1951 +79,58,525,公司31,1951 +14,12,439,公司32,1951 +84,30,617,公司33,1951 +61,23,854,公司34,1951 +0,30,765,公司35,1951 +58,25,834,公司36,1951 +37,53,229,公司37,1951 +61,0,955,公司38,1951 +63,30,346,公司39,1951 +84,29,492,公司40,1951 +9,77,916,公司41,1951 +25,64,874,公司42,1951 +18,43,18,公司43,1951 +51,75,755,公司44,1951 +94,71,866,公司45,1951 +32,0,293,公司46,1951 +18,76,111,公司47,1951 +7,25,307,公司48,1951 +83,36,834,公司49,1951 +50,31,710,公司0,1952 +13,65,271,公司1,1952 +41,10,792,公司2,1952 +91,92,552,公司3,1952 +10,94,762,公司4,1952 +50,13,732,公司5,1952 +36,97,782,公司6,1952 +98,72,753,公司7,1952 +5,6,177,公司8,1952 +6,96,924,公司9,1952 +72,10,951,公司10,1952 +80,5,305,公司11,1952 +95,88,827,公司12,1952 +34,77,600,公司13,1952 +11,75,568,公司14,1952 +85,76,562,公司15,1952 +59,55,214,公司16,1952 +61,70,173,公司17,1952 +53,51,156,公司18,1952 +50,79,455,公司19,1952 +95,48,773,公司20,1952 +49,68,927,公司21,1952 +71,43,324,公司22,1952 +10,45,327,公司23,1952 +75,7,166,公司24,1952 +25,86,618,公司25,1952 +81,88,675,公司26,1952 +8,69,1028,公司27,1952 +19,89,636,公司28,1952 +48,15,242,公司29,1952 +44,76,136,公司30,1952 +81,61,531,公司31,1952 +15,14,407,公司32,1952 +87,30,672,公司33,1952 +63,22,881,公司34,1952 +0,30,790,公司35,1952 +59,28,839,公司36,1952 +36,56,257,公司37,1952 +63,0,954,公司38,1952 +66,30,376,公司39,1952 +85,30,496,公司40,1952 +9,77,892,公司41,1952 +26,66,900,公司42,1952 +17,45,21,公司43,1952 +53,77,789,公司44,1952 +96,72,835,公司45,1952 +31,-1,284,公司46,1952 +18,78,103,公司47,1952 +7,25,289,公司48,1952 +86,38,873,公司49,1952 +50,30,672,公司0,1953 +16,64,277,公司1,1953 +44,11,835,公司2,1953 +92,91,561,公司3,1953 +12,94,724,公司4,1953 +50,14,739,公司5,1953 +38,100,767,公司6,1953 +100,72,802,公司7,1953 +5,5,143,公司8,1953 +8,98,894,公司9,1953 +72,10,937,公司10,1953 +79,4,305,公司11,1953 +98,89,873,公司12,1953 +35,79,628,公司13,1953 +12,76,611,公司14,1953 +84,78,545,公司15,1953 +58,57,199,公司16,1953 +63,73,185,公司17,1953 +52,52,117,公司18,1953 +49,78,434,公司19,1953 +94,51,765,公司20,1953 +50,67,899,公司21,1953 +71,45,343,公司22,1953 +13,48,318,公司23,1953 +75,7,185,公司24,1953 +27,89,574,公司25,1953 +84,90,688,公司26,1953 +7,69,991,公司27,1953 +20,90,606,公司28,1953 +51,15,251,公司29,1953 +47,75,154,公司30,1953 +83,63,555,公司31,1953 +14,18,455,公司32,1953 +89,33,709,公司33,1953 +64,22,874,公司34,1953 +-1,33,753,公司35,1953 +61,27,837,公司36,1953 +35,59,280,公司37,1953 +63,3,975,公司38,1953 +67,31,341,公司39,1953 +87,31,471,公司40,1953 +9,76,936,公司41,1953 +28,65,940,公司42,1953 +18,46,59,公司43,1953 +55,76,757,公司44,1953 +96,72,827,公司45,1953 +34,-2,287,公司46,1953 +22,78,107,公司47,1953 +9,25,258,公司48,1953 +87,39,877,公司49,1953 +50,29,645,公司0,1954 +16,64,312,公司1,1954 +47,11,790,公司2,1954 +94,90,553,公司3,1954 +13,93,692,公司4,1954 +48,13,714,公司5,1954 +38,99,725,公司6,1954 +103,72,842,公司7,1954 +6,4,136,公司8,1954 +10,100,894,公司9,1954 +75,9,925,公司10,1954 +79,6,346,公司11,1954 +100,89,878,公司12,1954 +36,80,590,公司13,1954 +14,78,643,公司14,1954 +83,77,575,公司15,1954 +61,59,224,公司16,1954 +62,76,234,公司17,1954 +52,52,161,公司18,1954 +49,79,441,公司19,1954 +96,51,800,公司20,1954 +52,67,931,公司21,1954 +74,48,337,公司22,1954 +14,51,349,公司23,1954 +78,9,238,公司24,1954 +30,91,554,公司25,1954 +83,92,658,公司26,1954 +8,72,967,公司27,1954 +22,89,607,公司28,1954 +52,14,217,公司29,1954 +50,78,156,公司30,1954 +85,63,604,公司31,1954 +16,21,473,公司32,1954 +92,34,753,公司33,1954 +66,22,881,公司34,1954 +1,33,725,公司35,1954 +61,30,797,公司36,1954 +35,63,333,公司37,1954 +63,3,1016,公司38,1954 +69,30,337,公司39,1954 +91,34,445,公司40,1954 +9,78,960,公司41,1954 +27,66,949,公司42,1954 +18,46,113,公司43,1954 +56,79,731,公司44,1954 +98,72,855,公司45,1954 +33,-3,259,公司46,1954 +21,79,109,公司47,1954 +12,26,285,公司48,1954 +91,40,845,公司49,1954 +52,28,667,公司0,1955 +18,65,324,公司1,1955 +50,10,781,公司2,1955 +95,90,581,公司3,1955 +15,95,669,公司4,1955 +46,13,733,公司5,1955 +39,98,759,公司6,1955 +102,74,801,公司7,1955 +8,3,108,公司8,1955 +12,101,882,公司9,1955 +78,7,966,公司10,1955 +80,9,395,公司11,1955 +103,89,847,公司12,1955 +38,81,628,公司13,1955 +14,79,610,公司14,1955 +86,79,625,公司15,1955 +61,60,209,公司16,1955 +62,78,281,公司17,1955 +54,54,186,公司18,1955 +51,82,478,公司19,1955 +95,52,842,公司20,1955 +54,69,886,公司21,1955 +74,51,361,公司22,1955 +17,54,362,公司23,1955 +81,8,212,公司24,1955 +31,94,546,公司25,1955 +82,95,707,公司26,1955 +9,72,929,公司27,1955 +24,90,610,公司28,1955 +54,13,244,公司29,1955 +51,80,208,公司30,1955 +86,67,646,公司31,1955 +20,22,428,公司32,1955 +95,37,758,公司33,1955 +65,25,887,公司34,1955 +4,35,717,公司35,1955 +63,30,819,公司36,1955 +35,65,322,公司37,1955 +65,5,1015,公司38,1955 +71,30,336,公司39,1955 +93,34,472,公司40,1955 +9,81,1001,公司41,1955 +28,67,922,公司42,1955 +18,47,125,公司43,1955 +59,79,778,公司44,1955 +101,75,879,公司45,1955 +32,-1,311,公司46,1955 +23,81,151,公司47,1955 +12,29,285,公司48,1955 +92,42,858,公司49,1955 +50,30,696,公司0,1956 +20,68,327,公司1,1956 +51,8,793,公司2,1956 +98,90,554,公司3,1956 +16,95,689,公司4,1956 +47,15,745,公司5,1956 +39,100,797,公司6,1956 +100,77,765,公司7,1956 +6,4,152,公司8,1956 +14,102,860,公司9,1956 +81,10,1000,公司10,1956 +83,7,412,公司11,1956 +102,91,822,公司12,1956 +37,84,599,公司13,1956 +13,81,584,公司14,1956 +86,81,663,公司15,1956 +63,60,169,公司16,1956 +61,78,309,公司17,1956 +56,56,215,公司18,1956 +52,81,533,公司19,1956 +98,55,882,公司20,1956 +53,69,897,公司21,1956 +73,53,321,公司22,1956 +18,55,399,公司23,1956 +84,7,208,公司24,1956 +30,97,555,公司25,1956 +82,94,687,公司26,1956 +8,74,980,公司27,1956 +24,89,603,公司28,1956 +57,14,255,公司29,1956 +53,81,197,公司30,1956 +88,69,697,公司31,1956 +19,21,454,公司32,1956 +95,38,805,公司33,1956 +67,25,939,公司34,1956 +6,37,714,公司35,1956 +62,32,839,公司36,1956 +34,67,334,公司37,1956 +68,4,983,公司38,1956 +73,29,336,公司39,1956 +94,37,502,公司40,1956 +9,83,969,公司41,1956 +31,69,974,公司42,1956 +19,50,127,公司43,1956 +62,78,743,公司44,1956 +105,75,875,公司45,1956 +36,2,267,公司46,1956 +27,81,116,公司47,1956 +15,29,267,公司48,1956 +92,45,860,公司49,1956 +51,29,675,公司0,1957 +20,66,318,公司1,1957 +50,10,808,公司2,1957 +99,91,593,公司3,1957 +16,97,653,公司4,1957 +46,14,785,公司5,1957 +40,101,838,公司6,1957 +99,77,764,公司7,1957 +5,6,117,公司8,1957 +13,104,852,公司9,1957 +81,12,983,公司10,1957 +82,5,430,公司11,1957 +104,93,835,公司12,1957 +37,82,571,公司13,1957 +14,79,542,公司14,1957 +85,81,692,公司15,1957 +65,61,135,公司16,1957 +60,79,274,公司17,1957 +58,58,219,公司18,1957 +54,83,537,公司19,1957 +97,54,863,公司20,1957 +54,70,885,公司21,1957 +72,55,362,公司22,1957 +18,58,403,公司23,1957 +86,7,166,公司24,1957 +33,96,525,公司25,1957 +84,96,687,公司26,1957 +9,77,963,公司27,1957 +25,91,579,公司28,1957 +57,13,230,公司29,1957 +56,84,185,公司30,1957 +91,71,731,公司31,1957 +20,22,487,公司32,1957 +94,41,834,公司33,1957 +66,27,903,公司34,1957 +8,37,755,公司35,1957 +65,34,861,公司36,1957 +36,66,298,公司37,1957 +71,5,1004,公司38,1957 +75,31,345,公司39,1957 +96,39,512,公司40,1957 +12,87,948,公司41,1957 +34,69,987,公司42,1957 +18,53,150,公司43,1957 +63,77,715,公司44,1957 +109,78,857,公司45,1957 +39,5,237,公司46,1957 +29,84,161,公司47,1957 +19,31,260,公司48,1957 +93,47,857,公司49,1957 +54,30,690,公司0,1958 +23,67,301,公司1,1958 +50,10,816,公司2,1958 +101,91,642,公司3,1958 +18,97,641,公司4,1958 +47,12,750,公司5,1958 +42,102,842,公司6,1958 +98,77,800,公司7,1958 +7,7,117,公司8,1958 +15,105,890,公司9,1958 +84,12,950,公司10,1958 +84,7,457,公司11,1958 +102,92,829,公司12,1958 +39,81,549,公司13,1958 +16,79,578,公司14,1958 +84,83,690,公司15,1958 +63,60,96,公司16,1958 +62,80,237,公司17,1958 +56,59,244,公司18,1958 +54,83,533,公司19,1958 +100,56,859,公司20,1958 +55,72,846,公司21,1958 +72,57,346,公司22,1958 +20,57,430,公司23,1958 +87,9,212,公司24,1958 +33,95,533,公司25,1958 +87,95,658,公司26,1958 +11,80,947,公司27,1958 +27,93,621,公司28,1958 +56,12,195,公司29,1958 +56,84,198,公司30,1958 +92,72,719,公司31,1958 +20,23,466,公司32,1958 +93,43,790,公司33,1958 +68,28,941,公司34,1958 +8,38,718,公司35,1958 +67,37,862,公司36,1958 +38,67,255,公司37,1958 +74,5,967,公司38,1958 +78,32,396,公司39,1958 +96,38,486,公司40,1958 +12,89,981,公司41,1958 +34,70,943,公司42,1958 +20,56,109,公司43,1958 +67,79,745,公司44,1958 +112,82,816,公司45,1958 +43,4,244,公司46,1958 +31,85,136,公司47,1958 +22,32,222,公司48,1958 +96,50,854,公司49,1958 +57,31,709,公司0,1959 +23,69,306,公司1,1959 +51,10,853,公司2,1959 +101,90,683,公司3,1959 +21,97,603,公司4,1959 +49,14,754,公司5,1959 +45,101,860,公司6,1959 +98,78,797,公司7,1959 +10,6,144,公司8,1959 +18,104,918,公司9,1959 +87,15,955,公司10,1959 +84,9,509,公司11,1959 +105,95,803,公司12,1959 +42,82,601,公司13,1959 +19,80,553,公司14,1959 +83,82,649,公司15,1959 +66,63,144,公司16,1959 +64,82,206,公司17,1959 +56,60,200,公司18,1959 +54,86,546,公司19,1959 +101,55,881,公司20,1959 +56,71,839,公司21,1959 +71,57,382,公司22,1959 +22,56,439,公司23,1959 +88,11,242,公司24,1959 +36,95,527,公司25,1959 +86,96,641,公司26,1959 +14,83,979,公司27,1959 +30,92,663,公司28,1959 +56,12,161,公司29,1959 +57,87,234,公司30,1959 +95,75,730,公司31,1959 +22,23,435,公司32,1959 +93,42,754,公司33,1959 +69,29,945,公司34,1959 +7,37,753,公司35,1959 +71,36,853,公司36,1959 +39,66,245,公司37,1959 +75,4,959,公司38,1959 +82,36,425,公司39,1959 +96,41,469,公司40,1959 +12,91,1023,公司41,1959 +35,69,945,公司42,1959 +23,57,140,公司43,1959 +68,78,743,公司44,1959 +115,83,808,公司45,1959 +42,3,203,公司46,1959 +30,84,185,公司47,1959 +25,31,182,公司48,1959 +97,53,852,公司49,1959 +55,30,681,公司0,1960 +22,68,277,公司1,1960 +53,12,823,公司2,1960 +103,90,704,公司3,1960 +19,96,567,公司4,1960 +52,16,775,公司5,1960 +44,101,902,公司6,1960 +100,81,818,公司7,1960 +10,8,159,公司8,1960 +21,105,908,公司9,1960 +90,18,937,公司10,1960 +84,11,498,公司11,1960 +105,94,845,公司12,1960 +45,82,579,公司13,1960 +21,83,592,公司14,1960 +83,81,642,公司15,1960 +68,64,158,公司16,1960 +67,85,248,公司17,1960 +57,59,251,公司18,1960 +53,89,571,公司19,1960 +102,58,885,公司20,1960 +56,70,878,公司21,1960 +73,60,401,公司22,1960 +22,56,423,公司23,1960 +90,10,278,公司24,1960 +39,96,570,公司25,1960 +89,96,598,公司26,1960 +14,86,1003,公司27,1960 +31,95,683,公司28,1960 +59,12,126,公司29,1960 +57,86,192,公司30,1960 +96,77,783,公司31,1960 +22,26,455,公司32,1960 +93,43,787,公司33,1960 +71,31,998,公司34,1960 +8,39,751,公司35,1960 +74,40,867,公司36,1960 +40,66,200,公司37,1960 +75,6,960,公司38,1960 +81,39,391,公司39,1960 +96,41,455,公司40,1960 +12,94,1056,公司41,1960 +38,69,961,公司42,1960 +22,61,119,公司43,1960 +68,81,753,公司44,1960 +115,82,766,公司45,1960 +41,5,205,公司46,1960 +32,83,220,公司47,1960 +28,33,187,公司48,1960 +99,57,893,公司49,1960 +58,33,713,公司0,1961 +23,71,248,公司1,1961 +54,12,787,公司2,1961 +103,92,720,公司3,1961 +19,98,621,公司4,1961 +54,15,826,公司5,1961 +47,103,915,公司6,1961 +102,80,787,公司7,1961 +11,7,173,公司8,1961 +21,107,876,公司9,1961 +90,21,901,公司10,1961 +85,10,492,公司11,1961 +108,95,887,公司12,1961 +47,84,634,公司13,1961 +23,86,633,公司14,1961 +86,80,607,公司15,1961 +70,63,133,公司16,1961 +69,88,266,公司17,1961 +56,62,244,公司18,1961 +54,92,623,公司19,1961 +103,61,878,公司20,1961 +58,71,890,公司21,1961 +74,62,372,公司22,1961 +25,58,396,公司23,1961 +93,13,284,公司24,1961 +40,98,605,公司25,1961 +92,98,647,公司26,1961 +13,88,1003,公司27,1961 +32,95,718,公司28,1961 +59,16,152,公司29,1961 +59,86,215,公司30,1961 +97,80,820,公司31,1961 +21,29,474,公司32,1961 +94,42,755,公司33,1961 +71,33,958,公司34,1961 +8,38,734,公司35,1961 +77,43,878,公司36,1961 +44,67,235,公司37,1961 +75,7,977,公司38,1961 +83,38,393,公司39,1961 +95,41,484,公司40,1961 +13,95,1061,公司41,1961 +41,71,941,公司42,1961 +23,60,119,公司43,1961 +71,84,784,公司44,1961 +117,85,792,公司45,1961 +45,9,235,公司46,1961 +34,84,194,公司47,1961 +30,36,163,公司48,1961 +101,59,921,公司49,1961 +60,32,744,公司0,1962 +21,72,219,公司1,1962 +56,12,801,公司2,1962 +102,90,775,公司3,1962 +22,97,585,公司4,1962 +55,18,838,公司5,1962 +47,106,959,公司6,1962 +104,82,775,公司7,1962 +11,8,148,公司8,1962 +23,107,929,公司9,1962 +89,19,950,公司10,1962 +88,12,496,公司11,1962 +110,96,915,公司12,1962 +47,84,607,公司13,1962 +22,86,639,公司14,1962 +88,79,655,公司15,1962 +73,63,119,公司16,1962 +70,89,246,公司17,1962 +55,64,227,公司18,1962 +57,94,649,公司19,1962 +102,62,921,公司20,1962 +60,72,853,公司21,1962 +74,61,371,公司22,1962 +26,59,412,公司23,1962 +95,16,273,公司24,1962 +41,99,582,公司25,1962 +91,99,668,公司26,1962 +15,89,1047,公司27,1962 +34,94,732,公司28,1962 +59,15,177,公司29,1962 +58,86,203,公司30,1962 +99,82,870,公司31,1962 +21,31,525,公司32,1962 +93,41,725,公司33,1962 +75,34,1005,公司34,1962 +10,37,785,公司35,1962 +77,44,909,公司36,1962 +45,66,195,公司37,1962 +76,10,953,公司38,1962 +86,39,368,公司39,1962 +97,42,441,公司40,1962 +15,95,1089,公司41,1962 +41,71,922,公司42,1962 +24,61,90,公司43,1962 +74,84,758,公司44,1962 +116,85,806,公司45,1962 +46,11,260,公司46,1962 +33,85,174,公司47,1962 +29,39,206,公司48,1962 +101,59,923,公司49,1962 +59,33,766,公司0,1963 +22,75,268,公司1,1963 +56,10,768,公司2,1963 +101,90,740,公司3,1963 +25,96,637,公司4,1963 +54,19,860,公司5,1963 +50,106,978,公司6,1963 +104,81,807,公司7,1963 +10,9,188,公司8,1963 +24,108,941,公司9,1963 +90,20,958,公司10,1963 +88,11,483,公司11,1963 +113,98,872,公司12,1963 +47,87,591,公司13,1963 +21,88,691,公司14,1963 +90,78,681,公司15,1963 +72,63,78,公司16,1963 +69,91,282,公司17,1963 +54,66,197,公司18,1963 +60,97,660,公司19,1963 +101,63,889,公司20,1963 +61,71,867,公司21,1963 +75,62,351,公司22,1963 +25,62,445,公司23,1963 +97,17,250,公司24,1963 +44,99,634,公司25,1963 +91,100,719,公司26,1963 +16,91,1014,公司27,1963 +37,93,756,公司28,1963 +61,17,158,公司29,1963 +59,87,210,公司30,1963 +102,82,833,公司31,1963 +20,30,510,公司32,1963 +96,43,774,公司33,1963 +75,37,1033,公司34,1963 +9,37,793,公司35,1963 +78,44,865,公司36,1963 +44,65,238,公司37,1963 +78,10,985,公司38,1963 +88,39,415,公司39,1963 +97,42,479,公司40,1963 +14,98,1142,公司41,1963 +41,73,967,公司42,1963 +23,60,140,公司43,1963 +75,85,717,公司44,1963 +119,88,834,公司45,1963 +46,10,239,公司46,1963 +34,89,178,公司47,1963 +31,38,186,公司48,1963 +104,61,974,公司49,1963 +59,34,765,公司0,1964 +22,73,238,公司1,1964 +55,11,754,公司2,1964 +103,90,716,公司3,1964 +28,98,684,公司4,1964 +54,20,828,公司5,1964 +53,105,941,公司6,1964 +103,83,797,公司7,1964 +12,9,204,公司8,1964 +22,109,938,公司9,1964 +89,20,949,公司10,1964 +86,13,520,公司11,1964 +112,99,873,公司12,1964 +48,88,605,公司13,1964 +24,89,674,公司14,1964 +90,81,649,公司15,1964 +73,63,106,公司16,1964 +71,93,242,公司17,1964 +54,68,181,公司18,1964 +61,98,667,公司19,1964 +99,65,857,公司20,1964 +64,70,878,公司21,1964 +75,61,329,公司22,1964 +24,65,405,公司23,1964 +99,17,256,公司24,1964 +45,101,680,公司25,1964 +94,101,708,公司26,1964 +16,90,1026,公司27,1964 +40,92,759,公司28,1964 +61,17,146,公司29,1964 +63,88,238,公司30,1964 +103,81,837,公司31,1964 +22,31,474,公司32,1964 +99,46,828,公司33,1964 +76,38,1057,公司34,1964 +10,38,794,公司35,1964 +77,43,847,公司36,1964 +46,67,263,公司37,1964 +80,10,958,公司38,1964 +90,38,380,公司39,1964 +101,45,531,公司40,1964 +18,99,1154,公司41,1964 +45,73,946,公司42,1964 +26,61,144,公司43,1964 +76,85,721,公司44,1964 +119,87,814,公司45,1964 +45,10,256,公司46,1964 +36,88,223,公司47,1964 +31,42,214,公司48,1964 +106,61,987,公司49,1964 +60,34,805,公司0,1965 +21,72,264,公司1,1965 +57,9,771,公司2,1965 +103,93,755,公司3,1965 +26,99,647,公司4,1965 +57,23,857,公司5,1965 +56,107,930,公司6,1965 +106,83,767,公司7,1965 +13,12,195,公司8,1965 +23,111,937,公司9,1965 +91,20,963,公司10,1965 +87,12,561,公司11,1965 +115,102,839,公司12,1965 +51,88,563,公司13,1965 +26,91,675,公司14,1965 +89,83,657,公司15,1965 +76,66,62,公司16,1965 +74,93,274,公司17,1965 +53,70,145,公司18,1965 +60,97,653,公司19,1965 +101,67,838,公司20,1965 +66,70,888,公司21,1965 +78,61,364,公司22,1965 +27,66,439,公司23,1965 +99,16,212,公司24,1965 +45,101,718,公司25,1965 +93,103,749,公司26,1965 +18,89,1070,公司27,1965 +40,91,748,公司28,1965 +62,18,132,公司29,1965 +65,88,261,公司30,1965 +105,83,840,公司31,1965 +22,35,501,公司32,1965 +101,49,828,公司33,1965 +76,39,1050,公司34,1965 +9,37,844,公司35,1965 +77,45,871,公司36,1965 +45,67,267,公司37,1965 +79,11,955,公司38,1965 +93,39,379,公司39,1965 +101,47,508,公司40,1965 +21,100,1160,公司41,1965 +46,76,949,公司42,1965 +29,64,115,公司43,1965 +80,87,767,公司44,1965 +121,86,841,公司45,1965 +44,13,302,公司46,1965 +39,89,264,公司47,1965 +32,43,179,公司48,1965 +106,61,1019,公司49,1965 +59,32,805,公司0,1966 +23,74,274,公司1,1966 +60,8,742,公司2,1966 +102,94,749,公司3,1966 +24,100,640,公司4,1966 +55,22,846,公司5,1966 +56,108,937,公司6,1966 +109,86,769,公司7,1966 +14,12,185,公司8,1966 +26,110,961,公司9,1966 +94,22,985,公司10,1966 +86,12,540,公司11,1966 +117,102,887,公司12,1966 +50,91,613,公司13,1966 +25,93,693,公司14,1966 +89,84,678,公司15,1966 +77,66,62,公司16,1966 +77,95,254,公司17,1966 +54,71,172,公司18,1966 +61,96,677,公司19,1966 +102,67,846,公司20,1966 +69,73,940,公司21,1966 +78,64,403,公司22,1966 +30,69,455,公司23,1966 +98,15,221,公司24,1966 +48,104,704,公司25,1966 +92,102,784,公司26,1966 +21,92,1041,公司27,1966 +43,92,704,公司28,1966 +61,19,100,公司29,1966 +67,91,244,公司30,1966 +106,85,857,公司31,1966 +22,35,545,公司32,1966 +100,52,803,公司33,1966 +79,39,1090,公司34,1966 +9,39,847,公司35,1966 +77,46,865,公司36,1966 +45,69,283,公司37,1966 +80,14,921,公司38,1966 +93,39,341,公司39,1966 +102,49,549,公司40,1966 +22,100,1156,公司41,1966 +48,75,935,公司42,1966 +31,67,136,公司43,1966 +80,88,817,公司44,1966 +122,85,807,公司45,1966 +45,14,259,公司46,1966 +41,90,306,公司47,1966 +32,46,173,公司48,1966 +107,62,1068,公司49,1966 +58,32,775,公司0,1967 +21,76,280,公司1,1967 +59,10,700,公司2,1967 +101,95,788,公司3,1967 +23,100,600,公司4,1967 +55,24,809,公司5,1967 +59,108,912,公司6,1967 +112,86,790,公司7,1967 +17,11,162,公司8,1967 +27,113,940,公司9,1967 +93,21,987,公司10,1967 +88,12,554,公司11,1967 +120,105,894,公司12,1967 +51,91,662,公司13,1967 +26,94,678,公司14,1967 +88,85,694,公司15,1967 +79,65,80,公司16,1967 +80,95,290,公司17,1967 +55,72,181,公司18,1967 +64,95,678,公司19,1967 +101,69,869,公司20,1967 +68,72,993,公司21,1967 +81,67,433,公司22,1967 +31,72,488,公司23,1967 +99,17,211,公司24,1967 +47,104,740,公司25,1967 +93,101,836,公司26,1967 +23,95,1011,公司27,1967 +43,95,688,公司28,1967 +61,19,129,公司29,1967 +68,92,223,公司30,1967 +106,88,837,公司31,1967 +24,36,543,公司32,1967 +100,52,844,公司33,1967 +79,40,1092,公司34,1967 +8,40,862,公司35,1967 +76,47,842,公司36,1967 +48,72,303,公司37,1967 +82,14,965,公司38,1967 +92,40,346,公司39,1967 +101,48,540,公司40,1967 +23,101,1186,公司41,1967 +51,76,921,公司42,1967 +31,67,132,公司43,1967 +79,89,779,公司44,1967 +125,89,808,公司45,1967 +45,14,216,公司46,1967 +40,90,278,公司47,1967 +32,46,203,公司48,1967 +111,64,1115,公司49,1967 +56,31,732,公司0,1968 +20,76,299,公司1,1968 +60,12,722,公司2,1968 +101,95,802,公司3,1968 +25,103,626,公司4,1968 +53,26,786,公司5,1968 +59,108,874,公司6,1968 +112,89,755,公司7,1968 +19,14,202,公司8,1968 +25,114,938,公司9,1968 +95,21,947,公司10,1968 +91,11,558,公司11,1968 +122,108,854,公司12,1968 +52,92,656,公司13,1968 +29,96,711,公司14,1968 +87,86,746,公司15,1968 +81,68,90,公司16,1968 +82,94,319,公司17,1968 +56,72,174,公司18,1968 +64,95,710,公司19,1968 +101,69,836,公司20,1968 +70,75,965,公司21,1968 +80,69,402,公司22,1968 +30,74,485,公司23,1968 +100,17,231,公司24,1968 +50,107,795,公司25,1968 +96,100,861,公司26,1968 +22,96,1058,公司27,1968 +44,94,649,公司28,1968 +60,22,167,公司29,1968 +67,93,260,公司30,1968 +105,90,874,公司31,1968 +28,35,577,公司32,1968 +101,54,880,公司33,1968 +81,43,1145,公司34,1968 +7,42,858,公司35,1968 +77,49,893,公司36,1968 +50,73,346,公司37,1968 +85,14,944,公司38,1968 +92,42,350,公司39,1968 +103,51,526,公司40,1968 +26,104,1162,公司41,1968 +51,75,971,公司42,1968 +31,70,119,公司43,1968 +80,92,741,公司44,1968 +127,92,780,公司45,1968 +48,15,246,公司46,1968 +42,90,321,公司47,1968 +34,46,238,公司48,1968 +115,64,1147,公司49,1968 +58,31,770,公司0,1969 +19,77,284,公司1,1969 +61,10,712,公司2,1969 +100,97,799,公司3,1969 +24,101,651,公司4,1969 +52,26,840,公司5,1969 +60,106,866,公司6,1969 +114,90,771,公司7,1969 +19,13,229,公司8,1969 +25,117,922,公司9,1969 +94,20,985,公司10,1969 +92,14,546,公司11,1969 +124,110,843,公司12,1969 +55,95,615,公司13,1969 +29,99,727,公司14,1969 +86,87,710,公司15,1969 +83,68,139,公司16,1969 +81,94,337,公司17,1969 +59,74,206,公司18,1969 +63,95,685,公司19,1969 +104,69,878,公司20,1969 +73,75,956,公司21,1969 +83,72,409,公司22,1969 +29,73,447,公司23,1969 +101,17,284,公司24,1969 +50,108,821,公司25,1969 +96,101,892,公司26,1969 +24,97,1096,公司27,1969 +44,97,684,公司28,1969 +63,23,184,公司29,1969 +69,95,295,公司30,1969 +105,90,882,公司31,1969 +30,39,568,公司32,1969 +101,55,866,公司33,1969 +83,43,1179,公司34,1969 +8,46,813,公司35,1969 +80,52,872,公司36,1969 +53,72,313,公司37,1969 +86,17,969,公司38,1969 +91,42,317,公司39,1969 +103,53,526,公司40,1969 +26,104,1200,公司41,1969 +53,75,1001,公司42,1969 +32,72,122,公司43,1969 +82,91,741,公司44,1969 +131,92,816,公司45,1969 +48,16,250,公司46,1969 +43,93,357,公司47,1969 +37,45,206,公司48,1969 +114,65,1177,公司49,1969 +57,30,761,公司0,1970 +20,79,314,公司1,1970 +61,10,726,公司2,1970 +99,98,776,公司3,1970 +25,103,699,公司4,1970 +53,25,815,公司5,1970 +60,106,848,公司6,1970 +113,92,806,公司7,1970 +22,12,241,公司8,1970 +24,117,974,公司9,1970 +92,20,1010,公司10,1970 +95,13,599,公司11,1970 +124,110,840,公司12,1970 +58,98,617,公司13,1970 +31,98,781,公司14,1970 +88,85,763,公司15,1970 +81,68,179,公司16,1970 +82,96,316,公司17,1970 +60,73,206,公司18,1970 +64,95,685,公司19,1970 +103,72,889,公司20,1970 +72,77,966,公司21,1970 +84,72,401,公司22,1970 +31,73,495,公司23,1970 +101,18,329,公司24,1970 +51,108,802,公司25,1970 +98,102,892,公司26,1970 +27,97,1063,公司27,1970 +44,99,694,公司28,1970 +66,23,205,公司29,1970 +68,96,347,公司30,1970 +105,90,865,公司31,1970 +30,38,591,公司32,1970 +101,57,897,公司33,1970 +83,42,1218,公司34,1970 +8,45,827,公司35,1970 +81,54,859,公司36,1970 +55,73,347,公司37,1970 +85,17,953,公司38,1970 +93,44,324,公司39,1970 +106,52,542,公司40,1970 +27,106,1201,公司41,1970 +56,75,1011,公司42,1970 +35,72,159,公司43,1970 +86,90,769,公司44,1970 +132,94,861,公司45,1970 +49,17,257,公司46,1970 +45,94,386,公司47,1970 +39,49,180,公司48,1970 +116,65,1160,公司49,1970 +59,30,717,公司0,1971 +22,82,326,公司1,1971 +63,11,733,公司2,1971 +98,100,768,公司3,1971 +26,101,718,公司4,1971 +56,26,776,公司5,1971 +60,106,823,公司6,1971 +113,93,778,公司7,1971 +21,11,209,公司8,1971 +26,119,1000,公司9,1971 +91,21,967,公司10,1971 +94,11,638,公司11,1971 +126,110,842,公司12,1971 +60,99,633,公司13,1971 +31,97,750,公司14,1971 +87,86,793,公司15,1971 +80,71,146,公司16,1971 +83,95,323,公司17,1971 +59,73,215,公司18,1971 +63,96,671,公司19,1971 +105,75,905,公司20,1971 +72,79,964,公司21,1971 +85,73,385,公司22,1971 +30,75,537,公司23,1971 +100,17,317,公司24,1971 +50,108,818,公司25,1971 +97,104,872,公司26,1971 +28,97,1107,公司27,1971 +45,98,741,公司28,1971 +69,22,162,公司29,1971 +68,98,363,公司30,1971 +107,92,878,公司31,1971 +33,38,640,公司32,1971 +100,60,896,公司33,1971 +82,45,1189,公司34,1971 +8,47,863,公司35,1971 +84,56,880,公司36,1971 +56,75,395,公司37,1971 +86,19,1004,公司38,1971 +94,46,307,公司39,1971 +106,53,595,公司40,1971 +30,106,1206,公司41,1971 +58,76,981,公司42,1971 +39,74,184,公司43,1971 +85,93,728,公司44,1971 +134,96,895,公司45,1971 +51,20,226,公司46,1971 +48,97,406,公司47,1971 +40,51,174,公司48,1971 +118,64,1153,公司49,1971 +58,33,714,公司0,1972 +23,81,340,公司1,1972 +61,12,786,公司2,1972 +100,101,816,公司3,1972 +26,103,697,公司4,1972 +56,25,775,公司5,1972 +61,106,801,公司6,1972 +115,95,806,公司7,1972 +24,14,250,公司8,1972 +27,118,1019,公司9,1972 +92,21,959,公司10,1972 +95,11,647,公司11,1972 +126,112,803,公司12,1972 +60,101,648,公司13,1972 +33,100,750,公司14,1972 +88,85,838,公司15,1972 +81,72,158,公司16,1972 +86,97,299,公司17,1972 +61,72,216,公司18,1972 +65,96,698,公司19,1972 +104,77,955,公司20,1972 +74,81,965,公司21,1972 +88,76,434,公司22,1972 +31,77,578,公司23,1972 +103,16,334,公司24,1972 +49,109,793,公司25,1972 +100,106,874,公司26,1972 +27,97,1064,公司27,1972 +45,100,793,公司28,1972 +71,25,186,公司29,1972 +67,101,351,公司30,1972 +110,92,924,公司31,1972 +35,41,640,公司32,1972 +101,59,878,公司33,1972 +82,46,1160,公司34,1972 +10,47,822,公司35,1972 +87,57,843,公司36,1972 +56,74,430,公司37,1972 +85,22,990,公司38,1972 +95,45,344,公司39,1972 +107,54,581,公司40,1972 +31,106,1260,公司41,1972 +60,75,974,公司42,1972 +43,77,185,公司43,1972 +86,93,768,公司44,1972 +134,98,910,公司45,1972 +51,22,234,公司46,1972 +51,99,432,公司47,1972 +39,51,193,公司48,1972 +118,63,1158,公司49,1972 +56,35,728,公司0,1973 +25,83,387,公司1,1973 +64,10,818,公司2,1973 +100,102,834,公司3,1973 +28,103,658,公司4,1973 +55,27,755,公司5,1973 +64,109,824,公司6,1973 +113,98,793,公司7,1973 +26,15,268,公司8,1973 +28,117,1034,公司9,1973 +91,24,980,公司10,1973 +95,12,661,公司11,1973 +128,113,761,公司12,1973 +61,104,615,公司13,1973 +34,100,721,公司14,1973 +90,84,823,公司15,1973 +83,71,139,公司16,1973 +88,99,294,公司17,1973 +61,72,196,公司18,1973 +66,94,682,公司19,1973 +104,78,961,公司20,1973 +76,83,928,公司21,1973 +88,77,426,公司22,1973 +32,76,621,公司23,1973 +106,17,319,公司24,1973 +48,108,825,公司25,1973 +101,107,860,公司26,1973 +30,96,1084,公司27,1973 +47,102,764,公司28,1973 +70,27,147,公司29,1973 +69,100,392,公司30,1973 +110,94,893,公司31,1973 +38,42,665,公司32,1973 +100,60,877,公司33,1973 +82,45,1160,公司34,1973 +9,48,831,公司35,1973 +88,58,833,公司36,1973 +57,75,399,公司37,1973 +87,24,1019,公司38,1973 +98,47,366,公司39,1973 +106,57,597,公司40,1973 +34,109,1253,公司41,1973 +60,76,938,公司42,1973 +44,79,149,公司43,1973 +87,96,753,公司44,1973 +134,98,929,公司45,1973 +50,23,257,公司46,1973 +52,102,452,公司47,1973 +38,54,191,公司48,1973 +119,64,1135,公司49,1973 +58,38,727,公司0,1974 +26,83,405,公司1,1974 +62,11,774,公司2,1974 +101,105,874,公司3,1974 +30,102,700,公司4,1974 +56,30,723,公司5,1974 +64,110,843,公司6,1974 +112,97,756,公司7,1974 +24,18,282,公司8,1974 +26,119,1031,公司9,1974 +91,23,940,公司10,1974 +97,13,659,公司11,1974 +127,116,766,公司12,1974 +62,105,623,公司13,1974 +36,102,735,公司14,1974 +92,86,787,公司15,1974 +82,74,108,公司16,1974 +91,100,311,公司17,1974 +64,72,244,公司18,1974 +64,96,700,公司19,1974 +103,80,929,公司20,1974 +78,86,883,公司21,1974 +87,78,434,公司22,1974 +33,78,584,公司23,1974 +106,17,282,公司24,1974 +48,110,877,公司25,1974 +102,109,852,公司26,1974 +29,95,1115,公司27,1974 +47,101,745,公司28,1974 +73,30,159,公司29,1974 +71,99,440,公司30,1974 +111,93,877,公司31,1974 +39,45,709,公司32,1974 +102,60,872,公司33,1974 +84,45,1175,公司34,1974 +12,49,831,公司35,1974 +89,61,793,公司36,1974 +60,78,386,公司37,1974 +87,27,1016,公司38,1974 +101,49,352,公司39,1974 +109,58,620,公司40,1974 +34,112,1233,公司41,1974 +60,75,959,公司42,1974 +44,83,105,公司43,1974 +90,97,733,公司44,1974 +134,102,885,公司45,1974 +54,25,300,公司46,1974 +55,104,462,公司47,1974 +39,57,180,公司48,1974 +118,66,1155,公司49,1974 +57,38,687,公司0,1975 +24,82,455,公司1,1975 +62,12,826,公司2,1975 +103,107,838,公司3,1975 +32,101,701,公司4,1975 +58,32,757,公司5,1975 +65,111,885,公司6,1975 +111,95,753,公司7,1975 +23,19,262,公司8,1975 +25,122,1082,公司9,1975 +94,21,914,公司10,1975 +99,13,686,公司11,1975 +130,119,771,公司12,1975 +61,103,660,公司13,1975 +36,103,745,公司14,1975 +94,89,802,公司15,1975 +83,77,130,公司16,1975 +91,102,324,公司17,1975 +66,72,224,公司18,1975 +66,98,735,公司19,1975 +106,79,916,公司20,1975 +77,85,863,公司21,1975 +88,78,408,公司22,1975 +35,81,601,公司23,1975 +107,18,326,公司24,1975 +50,112,910,公司25,1975 +102,108,891,公司26,1975 +29,96,1100,公司27,1975 +49,104,741,公司28,1975 +73,31,205,公司29,1975 +72,101,477,公司30,1975 +114,95,900,公司31,1975 +43,48,759,公司32,1975 +104,63,870,公司33,1975 +85,44,1134,公司34,1975 +13,52,827,公司35,1975 +91,62,755,公司36,1975 +59,78,431,公司37,1975 +90,30,977,公司38,1975 +100,50,386,公司39,1975 +111,61,654,公司40,1975 +36,112,1192,公司41,1975 +61,76,925,公司42,1975 +47,83,143,公司43,1975 +92,99,708,公司44,1975 +137,103,889,公司45,1975 +54,28,354,公司46,1975 +56,107,462,公司47,1975 +39,59,194,公司48,1975 +122,66,1180,公司49,1975 +59,40,661,公司0,1976 +24,80,471,公司1,1976 +64,11,782,公司2,1976 +106,107,860,公司3,1976 +34,100,749,公司4,1976 +61,33,740,公司5,1976 +65,111,931,公司6,1976 +110,94,796,公司7,1976 +25,18,243,公司8,1976 +23,125,1110,公司9,1976 +93,22,901,公司10,1976 +99,15,690,公司11,1976 +130,122,755,公司12,1976 +60,105,650,公司13,1976 +35,101,728,公司14,1976 +97,90,824,公司15,1976 +84,80,105,公司16,1976 +90,104,283,公司17,1976 +66,71,278,公司18,1976 +69,98,779,公司19,1976 +108,79,958,公司20,1976 +76,85,833,公司21,1976 +87,81,455,公司22,1976 +36,80,559,公司23,1976 +106,21,306,公司24,1976 +49,115,940,公司25,1976 +105,109,912,公司26,1976 +30,97,1134,公司27,1976 +49,104,748,公司28,1976 +73,34,200,公司29,1976 +72,103,496,公司30,1976 +114,97,890,公司31,1976 +44,50,729,公司32,1976 +103,65,902,公司33,1976 +84,47,1143,公司34,1976 +14,55,811,公司35,1976 +93,64,768,公司36,1976 +59,79,460,公司37,1976 +89,33,972,公司38,1976 +100,50,352,公司39,1976 +113,64,624,公司40,1976 +38,112,1176,公司41,1976 +63,76,893,公司42,1976 +47,82,109,公司43,1976 +92,101,688,公司44,1976 +139,104,926,公司45,1976 +54,30,405,公司46,1976 +56,107,465,公司47,1976 +39,58,166,公司48,1976 +124,68,1218,公司49,1976 +62,41,681,公司0,1977 +23,82,482,公司1,1977 +64,12,812,公司2,1977 +107,109,846,公司3,1977 +36,101,800,公司4,1977 +63,34,786,公司5,1977 +65,111,922,公司6,1977 +109,97,773,公司7,1977 +28,20,225,公司8,1977 +25,128,1113,公司9,1977 +96,25,914,公司10,1977 +98,18,662,公司11,1977 +132,123,785,公司12,1977 +60,106,661,公司13,1977 +35,103,751,公司14,1977 +98,91,852,公司15,1977 +85,83,100,公司16,1977 +92,103,338,公司17,1977 +66,70,248,公司18,1977 +71,101,762,公司19,1977 +110,77,983,公司20,1977 +77,88,858,公司21,1977 +88,84,475,公司22,1977 +38,80,567,公司23,1977 +106,21,356,公司24,1977 +48,116,928,公司25,1977 +105,110,894,公司26,1977 +29,96,1140,公司27,1977 +51,104,731,公司28,1977 +74,35,216,公司29,1977 +72,106,470,公司30,1977 +115,97,938,公司31,1977 +48,51,753,公司32,1977 +105,66,925,公司33,1977 +86,48,1126,公司34,1977 +13,57,803,公司35,1977 +94,67,818,公司36,1977 +59,82,483,公司37,1977 +91,35,960,公司38,1977 +100,50,385,公司39,1977 +114,68,669,公司40,1977 +39,114,1204,公司41,1977 +66,76,898,公司42,1977 +49,85,98,公司43,1977 +92,102,643,公司44,1977 +140,105,915,公司45,1977 +57,30,387,公司46,1977 +57,110,442,公司47,1977 +41,58,134,公司48,1977 +127,70,1242,公司49,1977 +60,44,645,公司0,1978 +24,82,509,公司1,1978 +62,14,815,公司2,1978 +108,112,895,公司3,1978 +36,103,783,公司4,1978 +65,36,811,公司5,1978 +65,113,887,公司6,1978 +107,98,787,公司7,1978 +26,19,239,公司8,1978 +23,130,1070,公司9,1978 +96,27,940,公司10,1978 +99,17,632,公司11,1978 +132,126,797,公司12,1978 +60,108,627,公司13,1978 +37,104,718,公司14,1978 +101,91,869,公司15,1978 +87,83,134,公司16,1978 +92,104,317,公司17,1978 +68,72,277,公司18,1978 +69,104,797,公司19,1978 +110,78,988,公司20,1978 +77,91,888,公司21,1978 +87,87,482,公司22,1978 +38,82,524,公司23,1978 +108,21,326,公司24,1978 +47,118,946,公司25,1978 +105,109,890,公司26,1978 +32,99,1140,公司27,1978 +53,104,747,公司28,1978 +73,38,226,公司29,1978 +74,108,460,公司30,1978 +116,100,926,公司31,1978 +48,53,709,公司32,1978 +106,69,927,公司33,1978 +88,50,1113,公司34,1978 +15,60,817,公司35,1978 +97,69,812,公司36,1978 +60,81,450,公司37,1978 +94,36,980,公司38,1978 +104,52,429,公司39,1978 +113,67,682,公司40,1978 +39,117,1180,公司41,1978 +66,77,881,公司42,1978 +52,86,135,公司43,1978 +92,104,684,公司44,1978 +141,109,968,公司45,1978 +58,29,373,公司46,1978 +57,113,466,公司47,1978 +40,59,140,公司48,1978 +128,73,1291,公司49,1978 +63,43,642,公司0,1979 +25,84,542,公司1,1979 +65,13,787,公司2,1979 +108,115,936,公司3,1979 +38,101,835,公司4,1979 +66,36,836,公司5,1979 +66,111,924,公司6,1979 +109,101,829,公司7,1979 +29,18,258,公司8,1979 +22,133,1088,公司9,1979 +98,30,986,公司10,1979 +99,18,677,公司11,1979 +132,127,759,公司12,1979 +59,111,607,公司13,1979 +36,104,740,公司14,1979 +101,90,845,公司15,1979 +86,82,150,公司16,1979 +95,103,315,公司17,1979 +70,72,312,公司18,1979 +71,103,818,公司19,1979 +110,77,985,公司20,1979 +76,89,913,公司21,1979 +90,90,458,公司22,1979 +37,84,484,公司23,1979 +107,24,295,公司24,1979 +47,118,991,公司25,1979 +104,112,886,公司26,1979 +35,99,1181,公司27,1979 +53,105,706,公司28,1979 +74,39,248,公司29,1979 +76,109,512,公司30,1979 +117,100,979,公司31,1979 +48,55,715,公司32,1979 +107,68,925,公司33,1979 +89,54,1149,公司34,1979 +15,60,790,公司35,1979 +100,70,809,公司36,1979 +63,81,500,公司37,1979 +96,36,1010,公司38,1979 +106,52,470,公司39,1979 +114,67,698,公司40,1979 +39,117,1225,公司41,1979 +70,79,907,公司42,1979 +54,89,144,公司43,1979 +91,106,662,公司44,1979 +144,113,980,公司45,1979 +60,28,344,公司46,1979 +61,117,451,公司47,1979 +40,62,159,公司48,1979 +129,77,1289,公司49,1979 +64,44,615,公司0,1980 +27,85,514,公司1,1980 +68,11,746,公司2,1980 +109,117,944,公司3,1980 +36,99,825,公司4,1980 +67,39,847,公司5,1980 +64,109,921,公司6,1980 +111,103,818,公司7,1980 +29,19,243,公司8,1980 +23,132,1058,公司9,1980 +97,32,1041,公司10,1980 +100,19,665,公司11,1980 +132,125,729,公司12,1980 +58,113,634,公司13,1980 +37,104,776,公司14,1980 +102,92,828,公司15,1980 +85,85,184,公司16,1980 +95,105,335,公司17,1980 +70,72,275,公司18,1980 +72,103,776,公司19,1980 +110,77,998,公司20,1980 +75,91,887,公司21,1980 +92,90,509,公司22,1980 +37,84,448,公司23,1980 +109,25,304,公司24,1980 +47,120,1030,公司25,1980 +104,114,845,公司26,1980 +34,101,1144,公司27,1980 +53,105,686,公司28,1980 +76,40,225,公司29,1980 +77,109,507,公司30,1980 +118,99,955,公司31,1980 +50,55,765,公司32,1980 +109,69,921,公司33,1980 +88,56,1112,公司34,1980 +16,60,771,公司35,1980 +103,70,845,公司36,1980 +66,84,543,公司37,1980 +98,35,1052,公司38,1980 +107,52,440,公司39,1980 +115,69,676,公司40,1980 +42,118,1194,公司41,1980 +73,79,879,公司42,1980 +54,89,106,公司43,1980 +94,110,678,公司44,1980 +147,117,976,公司45,1980 +64,28,330,公司46,1980 +60,118,473,公司47,1980 +44,65,141,公司48,1980 +133,81,1287,公司49,1980 +63,46,590,公司0,1981 +29,83,555,公司1,1981 +70,11,796,公司2,1981 +112,115,971,公司3,1981 +37,100,821,公司4,1981 +67,39,875,公司5,1981 +64,109,927,公司6,1981 +113,106,841,公司7,1981 +30,18,283,公司8,1981 +26,135,1083,公司9,1981 +97,33,1067,公司10,1981 +98,22,656,公司11,1981 +134,124,690,公司12,1981 +61,115,647,公司13,1981 +37,103,767,公司14,1981 +101,91,849,公司15,1981 +87,83,188,公司16,1981 +97,104,383,公司17,1981 +71,74,265,公司18,1981 +71,102,761,公司19,1981 +110,77,1012,公司20,1981 +76,91,877,公司21,1981 +91,92,491,公司22,1981 +36,85,485,公司23,1981 +111,26,287,公司24,1981 +48,119,1045,公司25,1981 +107,117,865,公司26,1981 +33,101,1102,公司27,1981 +53,107,671,公司28,1981 +76,43,231,公司29,1981 +77,110,530,公司30,1981 +121,101,975,公司31,1981 +53,55,747,公司32,1981 +112,73,945,公司33,1981 +90,57,1077,公司34,1981 +17,60,741,公司35,1981 +106,69,897,公司36,1981 +66,84,597,公司37,1981 +100,38,1066,公司38,1981 +108,52,494,公司39,1981 +118,69,702,公司40,1981 +44,118,1150,公司41,1981 +74,79,856,公司42,1981 +56,92,141,公司43,1981 +95,109,725,公司44,1981 +147,121,991,公司45,1981 +66,31,347,公司46,1981 +60,119,501,公司47,1981 +46,66,165,公司48,1981 +134,84,1320,公司49,1981 +65,47,588,公司0,1982 +30,86,514,公司1,1982 +68,11,785,公司2,1982 +113,115,987,公司3,1982 +37,99,815,公司4,1982 +67,39,858,公司5,1982 +65,110,964,公司6,1982 +115,106,846,公司7,1982 +29,20,268,公司8,1982 +26,135,1105,公司9,1982 +98,31,1108,公司10,1982 +97,25,705,公司11,1982 +132,127,723,公司12,1982 +62,115,623,公司13,1982 +38,104,723,公司14,1982 +99,93,828,公司15,1982 +89,83,225,公司16,1982 +100,106,416,公司17,1982 +70,76,318,公司18,1982 +74,104,776,公司19,1982 +109,80,1048,公司20,1982 +79,94,878,公司21,1982 +90,95,474,公司22,1982 +38,87,472,公司23,1982 +110,25,244,公司24,1982 +51,121,1023,公司25,1982 +106,118,855,公司26,1982 +34,102,1070,公司27,1982 +56,109,647,公司28,1982 +78,45,238,公司29,1982 +78,109,544,公司30,1982 +120,102,994,公司31,1982 +54,56,764,公司32,1982 +116,73,915,公司33,1982 +90,56,1072,公司34,1982 +17,62,787,公司35,1982 +109,72,930,公司36,1982 +66,86,627,公司37,1982 +99,39,1035,公司38,1982 +111,55,465,公司39,1982 +119,70,698,公司40,1982 +44,118,1115,公司41,1982 +76,82,892,公司42,1982 +58,95,177,公司43,1982 +95,109,755,公司44,1982 +149,122,949,公司45,1982 +67,34,400,公司46,1982 +61,119,462,公司47,1982 +46,68,124,公司48,1982 +136,86,1371,公司49,1982 +65,46,585,公司0,1983 +32,86,517,公司1,1983 +71,10,748,公司2,1983 +113,115,1030,公司3,1983 +40,102,839,公司4,1983 +65,40,819,公司5,1983 +67,112,983,公司6,1983 +118,109,888,公司7,1983 +30,19,247,公司8,1983 +26,134,1134,公司9,1983 +97,34,1119,公司10,1983 +96,23,694,公司11,1983 +132,125,686,公司12,1983 +62,115,649,公司13,1983 +41,106,680,公司14,1983 +98,94,836,公司15,1983 +89,85,238,公司16,1983 +99,106,439,公司17,1983 +69,78,314,公司18,1983 +74,107,774,公司19,1983 +109,80,1036,公司20,1983 +81,93,883,公司21,1983 +90,94,448,公司22,1983 +38,89,446,公司23,1983 +113,24,235,公司24,1983 +52,124,1014,公司25,1983 +107,117,833,公司26,1983 +36,103,1088,公司27,1983 +59,110,652,公司28,1983 +79,46,251,公司29,1983 +78,112,585,公司30,1983 +121,103,1040,公司31,1983 +54,58,731,公司32,1983 +116,76,944,公司33,1983 +93,59,1036,公司34,1983 +20,65,752,公司35,1983 +112,73,979,公司36,1983 +66,87,621,公司37,1983 +101,41,1007,公司38,1983 +112,58,520,公司39,1983 +119,72,715,公司40,1983 +48,119,1086,公司41,1983 +79,85,904,公司42,1983 +60,96,145,公司43,1983 +96,108,772,公司44,1983 +149,122,970,公司45,1983 +69,37,431,公司46,1983 +61,123,421,公司47,1983 +45,68,110,公司48,1983 +137,86,1347,公司49,1983 +64,45,627,公司0,1984 +33,85,541,公司1,1984 +74,13,733,公司2,1984 +114,115,1042,公司3,1984 +39,103,835,公司4,1984 +68,39,779,公司5,1984 +68,111,1026,公司6,1984 +121,108,902,公司7,1984 +30,21,214,公司8,1984 +27,137,1137,公司9,1984 +100,35,1099,公司10,1984 +96,22,659,公司11,1984 +131,126,659,公司12,1984 +65,115,703,公司13,1984 +41,109,733,公司14,1984 +98,96,818,公司15,1984 +88,86,233,公司16,1984 +100,108,400,公司17,1984 +71,78,301,公司18,1984 +76,106,768,公司19,1984 +109,80,1009,公司20,1984 +81,94,844,公司21,1984 +93,96,459,公司22,1984 +41,91,412,公司23,1984 +116,27,240,公司24,1984 +53,126,1053,公司25,1984 +109,120,873,公司26,1984 +36,106,1076,公司27,1984 +62,111,686,公司28,1984 +81,47,249,公司29,1984 +77,112,585,公司30,1984 +120,106,1003,公司31,1984 +57,58,687,公司32,1984 +119,75,965,公司33,1984 +95,62,1066,公司34,1984 +22,66,735,公司35,1984 +113,74,1026,公司36,1984 +69,89,595,公司37,1984 +101,41,999,公司38,1984 +112,61,510,公司39,1984 +121,74,732,公司40,1984 +49,118,1066,公司41,1984 +78,87,859,公司42,1984 +61,100,143,公司43,1984 +98,108,825,公司44,1984 +153,123,1016,公司45,1984 +72,40,439,公司46,1984 +61,126,398,公司47,1984 +49,68,144,公司48,1984 +136,87,1326,公司49,1984 +63,47,672,公司0,1985 +31,87,570,公司1,1985 +76,12,759,公司2,1985 +116,118,1049,公司3,1985 +39,104,807,公司4,1985 +67,39,790,公司5,1985 +69,109,1033,公司6,1985 +122,108,945,公司7,1985 +31,21,210,公司8,1985 +26,136,1172,公司9,1985 +100,33,1070,公司10,1985 +96,21,686,公司11,1985 +133,127,654,公司12,1985 +64,117,754,公司13,1985 +41,109,698,公司14,1985 +99,97,849,公司15,1985 +91,88,252,公司16,1985 +101,110,381,公司17,1985 +70,79,352,公司18,1985 +76,108,756,公司19,1985 +108,80,965,公司20,1985 +81,94,805,公司21,1985 +92,99,485,公司22,1985 +43,92,396,公司23,1985 +115,30,294,公司24,1985 +54,125,1077,公司25,1985 +108,122,893,公司26,1985 +36,106,1070,公司27,1985 +63,111,684,公司28,1985 +80,46,288,公司29,1985 +80,115,551,公司30,1985 +121,105,1007,公司31,1985 +56,58,701,公司32,1985 +121,75,979,公司33,1985 +96,61,1116,公司34,1985 +22,67,770,公司35,1985 +114,78,1018,公司36,1985 +68,90,622,公司37,1985 +102,41,1014,公司38,1985 +115,64,479,公司39,1985 +125,73,709,公司40,1985 +50,117,1078,公司41,1985 +78,89,899,公司42,1985 +63,101,143,公司43,1985 +100,108,799,公司44,1985 +157,125,1046,公司45,1985 +72,41,445,公司46,1985 +60,126,433,公司47,1985 +49,72,188,公司48,1985 +136,89,1374,公司49,1985 +62,46,664,公司0,1986 +32,90,534,公司1,1986 +75,11,772,公司2,1986 +118,120,1047,公司3,1986 +41,104,774,公司4,1986 +67,41,826,公司5,1986 +70,110,1075,公司6,1986 +123,106,914,公司7,1986 +31,23,251,公司8,1986 +25,135,1129,公司9,1986 +100,32,1093,公司10,1986 +99,20,708,公司11,1986 +135,126,708,公司12,1986 +65,118,777,公司13,1986 +41,109,736,公司14,1986 +102,98,866,公司15,1986 +92,89,230,公司16,1986 +102,109,358,公司17,1986 +73,79,382,公司18,1986 +77,108,734,公司19,1986 +107,79,993,公司20,1986 +83,96,805,公司21,1986 +94,99,512,公司22,1986 +46,93,407,公司23,1986 +116,29,255,公司24,1986 +57,128,1106,公司25,1986 +107,122,872,公司26,1986 +38,109,1110,公司27,1986 +62,113,713,公司28,1986 +82,47,292,公司29,1986 +81,116,521,公司30,1986 +124,106,990,公司31,1986 +58,60,729,公司32,1986 +123,76,967,公司33,1986 +97,62,1109,公司34,1986 +26,66,765,公司35,1986 +113,80,1005,公司36,1986 +71,93,594,公司37,1986 +105,43,1012,公司38,1986 +114,67,485,公司39,1986 +124,75,751,公司40,1986 +49,118,1072,公司41,1986 +82,90,952,公司42,1986 +63,101,110,公司43,1986 +101,109,787,公司44,1986 +161,125,1045,公司45,1986 +72,41,426,公司46,1986 +63,126,451,公司47,1986 +51,71,172,公司48,1986 +138,90,1413,公司49,1986 +64,47,683,公司0,1987 +34,91,576,公司1,1987 +74,14,755,公司2,1987 +121,118,1063,公司3,1987 +43,103,793,公司4,1987 +65,43,875,公司5,1987 +70,108,1076,公司6,1987 +124,108,966,公司7,1987 +33,25,225,公司8,1987 +25,138,1095,公司9,1987 +101,32,1092,公司10,1987 +102,20,703,公司11,1987 +134,127,720,公司12,1987 +67,121,812,公司13,1987 +42,112,756,公司14,1987 +101,100,855,公司15,1987 +95,89,223,公司16,1987 +104,109,406,公司17,1987 +72,79,392,公司18,1987 +80,107,781,公司19,1987 +110,79,1000,公司20,1987 +86,99,779,公司21,1987 +94,99,564,公司22,1987 +49,93,397,公司23,1987 +116,32,236,公司24,1987 +60,128,1080,公司25,1987 +109,121,882,公司26,1987 +39,108,1084,公司27,1987 +65,116,728,公司28,1987 +81,48,299,公司29,1987 +83,119,536,公司30,1987 +125,108,991,公司31,1987 +61,61,745,公司32,1987 +122,77,1018,公司33,1987 +100,63,1101,公司34,1987 +26,65,723,公司35,1987 +113,81,970,公司36,1987 +72,95,597,公司37,1987 +105,44,996,公司38,1987 +116,70,450,公司39,1987 +128,75,778,公司40,1987 +51,118,1089,公司41,1987 +84,89,949,公司42,1987 +62,104,144,公司43,1987 +102,109,787,公司44,1987 +161,129,1023,公司45,1987 +76,43,402,公司46,1987 +65,130,479,公司47,1987 +51,72,138,公司48,1987 +138,92,1384,公司49,1987 +65,46,708,公司0,1988 +34,91,552,公司1,1988 +75,16,750,公司2,1988 +120,116,1093,公司3,1988 +45,101,789,公司4,1988 +68,46,832,公司5,1988 +70,111,1077,公司6,1988 +122,109,956,公司7,1988 +35,27,252,公司8,1988 +28,141,1108,公司9,1988 +104,32,1083,公司10,1988 +102,19,731,公司11,1988 +134,129,702,公司12,1988 +69,123,858,公司13,1988 +41,111,749,公司14,1988 +99,100,876,公司15,1988 +98,89,253,公司16,1988 +104,109,456,公司17,1988 +74,80,411,公司18,1988 +81,110,778,公司19,1988 +112,82,1044,公司20,1988 +86,98,819,公司21,1988 +96,101,527,公司22,1988 +49,93,354,公司23,1988 +116,32,211,公司24,1988 +60,127,1058,公司25,1988 +109,123,839,公司26,1988 +40,112,1095,公司27,1988 +66,118,698,公司28,1988 +81,49,287,公司29,1988 +84,121,589,公司30,1988 +129,111,961,公司31,1988 +62,61,749,公司32,1988 +125,76,1050,公司33,1988 +102,65,1134,公司34,1988 +28,65,745,公司35,1988 +112,84,1003,公司36,1988 +71,96,560,公司37,1988 +106,44,1030,公司38,1988 +118,72,429,公司39,1988 +131,79,756,公司40,1988 +54,120,1137,公司41,1988 +85,88,933,公司42,1988 +66,105,161,公司43,1988 +104,111,821,公司44,1988 +163,132,1066,公司45,1988 +80,43,376,公司46,1988 +68,129,448,公司47,1988 +51,73,191,公司48,1988 +137,96,1346,公司49,1988 +68,49,725,公司0,1989 +37,92,568,公司1,1989 +76,18,802,公司2,1989 +121,115,1099,公司3,1989 +47,101,810,公司4,1989 +68,45,862,公司5,1989 +69,112,1131,公司6,1989 +123,108,958,公司7,1989 +34,29,289,公司8,1989 +27,144,1129,公司9,1989 +106,33,1055,公司10,1989 +102,20,712,公司11,1989 +133,127,673,公司12,1989 +69,126,821,公司13,1989 +41,113,797,公司14,1989 +98,100,835,公司15,1989 +97,90,295,公司16,1989 +107,110,452,公司17,1989 +74,82,385,公司18,1989 +84,109,769,公司19,1989 +115,85,1060,公司20,1989 +86,99,806,公司21,1989 +96,101,512,公司22,1989 +49,95,343,公司23,1989 +118,33,238,公司24,1989 +60,130,1034,公司25,1989 +109,122,888,公司26,1989 +42,113,1091,公司27,1989 +69,118,733,公司28,1989 +82,52,314,公司29,1989 +88,121,557,公司30,1989 +129,114,1003,公司31,1989 +64,64,801,公司32,1989 +126,76,1078,公司33,1989 +106,65,1137,公司34,1989 +27,68,786,公司35,1989 +115,84,993,公司36,1989 +73,96,526,公司37,1989 +106,45,1020,公司38,1989 +121,74,404,公司39,1989 +132,79,795,公司40,1989 +56,122,1101,公司41,1989 +88,90,979,公司42,1989 +68,107,136,公司43,1989 +107,114,828,公司44,1989 +167,132,1062,公司45,1989 +79,44,409,公司46,1989 +71,130,417,公司47,1989 +51,74,242,公司48,1989 +137,95,1367,公司49,1989 +67,47,777,公司0,1990 +39,94,612,公司1,1990 +78,19,830,公司2,1990 +122,116,1110,公司3,1990 +48,102,865,公司4,1990 +67,45,825,公司5,1990 +70,115,1170,公司6,1990 +124,109,972,公司7,1990 +33,31,321,公司8,1990 +30,146,1132,公司9,1990 +107,33,1023,公司10,1990 +101,18,758,公司11,1990 +133,130,710,公司12,1990 +71,126,820,公司13,1990 +43,114,828,公司14,1990 +98,103,869,公司15,1990 +100,93,295,公司16,1990 +108,109,476,公司17,1990 +73,82,344,公司18,1990 +84,110,815,公司19,1990 +118,86,1065,公司20,1990 +89,102,811,公司21,1990 +97,101,473,公司22,1990 +51,94,392,公司23,1990 +121,34,244,公司24,1990 +60,132,1050,公司25,1990 +110,124,915,公司26,1990 +45,113,1117,公司27,1990 +71,119,735,公司28,1990 +81,52,303,公司29,1990 +90,124,547,公司30,1990 +129,114,964,公司31,1990 +66,66,840,公司32,1990 +127,79,1124,公司33,1990 +108,66,1108,公司34,1990 +27,68,747,公司35,1990 +114,85,1047,公司36,1990 +72,97,578,公司37,1990 +106,47,1059,公司38,1990 +120,77,429,公司39,1990 +132,78,790,公司40,1990 +59,124,1061,公司41,1990 +89,91,1033,公司42,1990 +71,108,172,公司43,1990 +106,114,836,公司44,1990 +169,136,1030,公司45,1990 +79,43,407,公司46,1990 +74,133,463,公司47,1990 +55,77,293,公司48,1990 +136,97,1360,公司49,1990 +70,47,768,公司0,1991 +42,95,594,公司1,1991 +79,19,872,公司2,1991 +124,117,1161,公司3,1991 +49,102,822,公司4,1991 +69,48,810,公司5,1991 +69,115,1220,公司6,1991 +122,112,954,公司7,1991 +34,32,319,公司8,1991 +31,145,1112,公司9,1991 +106,35,980,公司10,1991 +100,16,783,公司11,1991 +135,128,748,公司12,1991 +73,128,805,公司13,1991 +44,114,849,公司14,1991 +100,103,863,公司15,1991 +103,91,253,公司16,1991 +106,110,460,公司17,1991 +73,81,331,公司18,1991 +86,109,771,公司19,1991 +119,85,1095,公司20,1991 +92,104,866,公司21,1991 +96,104,495,公司22,1991 +50,94,398,公司23,1991 +123,33,252,公司24,1991 +61,132,1015,公司25,1991 +109,123,917,公司26,1991 +48,112,1096,公司27,1991 +73,121,700,公司28,1991 +82,54,311,公司29,1991 +90,127,534,公司30,1991 +130,114,995,公司31,1991 +69,69,819,公司32,1991 +130,80,1114,公司33,1991 +107,67,1117,公司34,1991 +30,69,716,公司35,1991 +116,86,1098,公司36,1991 +75,100,615,公司37,1991 +108,46,1103,公司38,1991 +122,81,407,公司39,1991 +132,80,764,公司40,1991 +61,124,1096,公司41,1991 +92,91,1032,公司42,1991 +73,109,178,公司43,1991 +106,118,880,公司44,1991 +169,135,1021,公司45,1991 +78,43,375,公司46,1991 +78,136,474,公司47,1991 +56,81,333,公司48,1991 +138,98,1342,公司49,1991 +72,48,781,公司0,1992 +41,97,640,公司1,1992 +80,19,849,公司2,1992 +123,118,1145,公司3,1992 +50,100,847,公司4,1992 +68,49,783,公司5,1992 +72,114,1220,公司6,1992 +125,112,967,公司7,1992 +33,31,336,公司8,1992 +34,143,1079,公司9,1992 +105,36,1007,公司10,1992 +102,17,783,公司11,1992 +134,129,732,公司12,1992 +71,131,825,公司13,1992 +45,116,867,公司14,1992 +99,106,891,公司15,1992 +104,92,303,公司16,1992 +109,112,459,公司17,1992 +73,79,313,公司18,1992 +87,109,745,公司19,1992 +120,87,1110,公司20,1992 +91,107,884,公司21,1992 +99,103,487,公司22,1992 +53,97,379,公司23,1992 +123,33,265,公司24,1992 +60,131,1004,公司25,1992 +111,124,926,公司26,1992 +51,113,1058,公司27,1992 +74,123,724,公司28,1992 +83,53,323,公司29,1992 +93,127,542,公司30,1992 +133,115,1001,公司31,1992 +68,68,776,公司32,1992 +132,80,1113,公司33,1992 +106,69,1126,公司34,1992 +33,71,746,公司35,1992 +116,89,1078,公司36,1992 +78,101,651,公司37,1992 +107,46,1063,公司38,1992 +121,82,372,公司39,1992 +133,81,760,公司40,1992 +60,125,1115,公司41,1992 +94,93,1045,公司42,1992 +72,110,226,公司43,1992 +109,117,849,公司44,1992 +169,138,1047,公司45,1992 +77,43,430,公司46,1992 +78,136,464,公司47,1992 +55,83,336,公司48,1992 +141,98,1375,公司49,1992 +75,49,799,公司0,1993 +42,96,661,公司1,1993 +83,19,813,公司2,1993 +125,120,1199,公司3,1993 +50,101,857,公司4,1993 +71,50,769,公司5,1993 +74,112,1178,公司6,1993 +124,112,1004,公司7,1993 +33,31,364,公司8,1993 +34,144,1063,公司9,1993 +106,39,1026,公司10,1993 +100,16,785,公司11,1993 +132,130,691,公司12,1993 +71,129,874,公司13,1993 +47,117,908,公司14,1993 +101,105,898,公司15,1993 +104,93,334,公司16,1993 +112,112,463,公司17,1993 +75,82,301,公司18,1993 +87,109,767,公司19,1993 +121,89,1136,公司20,1993 +93,105,860,公司21,1993 +102,103,479,公司22,1993 +55,100,418,公司23,1993 +124,36,308,公司24,1993 +61,131,1009,公司25,1993 +114,126,907,公司26,1993 +54,114,1089,公司27,1993 +77,126,690,公司28,1993 +85,55,347,公司29,1993 +96,127,511,公司30,1993 +134,116,1019,公司31,1993 +71,69,762,公司32,1993 +134,80,1128,公司33,1993 +107,70,1142,公司34,1993 +36,74,720,公司35,1993 +115,92,1132,公司36,1993 +78,104,632,公司37,1993 +108,49,1096,公司38,1993 +124,84,338,公司39,1993 +135,82,767,公司40,1993 +61,128,1170,公司41,1993 +93,93,1043,公司42,1993 +76,109,207,公司43,1993 +112,119,873,公司44,1993 +169,141,1049,公司45,1993 +78,43,484,公司46,1993 +82,138,513,公司47,1993 +58,86,316,公司48,1993 +142,100,1373,公司49,1993 +73,47,836,公司0,1994 +43,99,673,公司1,1994 +84,20,804,公司2,1994 +126,119,1235,公司3,1994 +52,101,835,公司4,1994 +70,50,814,公司5,1994 +74,112,1224,公司6,1994 +125,113,1033,公司7,1994 +35,34,395,公司8,1994 +34,143,1066,公司9,1994 +105,39,1025,公司10,1994 +101,16,766,公司11,1994 +132,131,714,公司12,1994 +71,131,899,公司13,1994 +47,119,944,公司14,1994 +103,106,877,公司15,1994 +105,95,319,公司16,1994 +113,115,476,公司17,1994 +77,84,270,公司18,1994 +90,111,747,公司19,1994 +124,89,1107,公司20,1994 +92,105,858,公司21,1994 +105,104,491,公司22,1994 +57,102,460,公司23,1994 +126,37,326,公司24,1994 +60,133,1001,公司25,1994 +113,128,887,公司26,1994 +55,115,1046,公司27,1994 +78,126,668,公司28,1994 +86,54,372,公司29,1994 +99,126,510,公司30,1994 +135,119,994,公司31,1994 +73,68,743,公司32,1994 +136,83,1177,公司33,1994 +107,72,1152,公司34,1994 +38,77,765,公司35,1994 +115,95,1143,公司36,1994 +77,107,671,公司37,1994 +107,50,1128,公司38,1994 +128,86,375,公司39,1994 +135,86,770,公司40,1994 +61,127,1133,公司41,1994 +95,96,1056,公司42,1994 +76,109,218,公司43,1994 +114,121,899,公司44,1994 +168,144,1074,公司45,1994 +78,43,464,公司46,1994 +82,141,545,公司47,1994 +59,86,322,公司48,1994 +144,100,1357,公司49,1994 +74,47,805,公司0,1995 +42,102,700,公司1,1995 +85,22,803,公司2,1995 +124,119,1231,公司3,1995 +53,103,821,公司4,1995 +71,50,858,公司5,1995 +75,112,1200,公司6,1995 +127,114,1013,公司7,1995 +36,34,375,公司8,1995 +36,141,1103,公司9,1995 +106,39,1073,公司10,1995 +102,15,760,公司11,1995 +131,132,749,公司12,1995 +71,133,946,公司13,1995 +47,119,992,公司14,1995 +106,106,841,公司15,1995 +108,96,365,公司16,1995 +113,115,442,公司17,1995 +80,85,287,公司18,1995 +91,113,749,公司19,1995 +125,90,1128,公司20,1995 +91,106,860,公司21,1995 +106,106,467,公司22,1995 +59,102,472,公司23,1995 +129,39,313,公司24,1995 +61,132,961,公司25,1995 +115,130,862,公司26,1995 +57,115,1068,公司27,1995 +79,125,624,公司28,1995 +87,54,418,公司29,1995 +98,129,505,公司30,1995 +138,118,961,公司31,1995 +76,67,719,公司32,1995 +138,84,1163,公司33,1995 +106,71,1149,公司34,1995 +38,78,814,公司35,1995 +115,98,1147,公司36,1995 +78,108,697,公司37,1995 +109,53,1117,公司38,1995 +131,87,347,公司39,1995 +136,89,780,公司40,1995 +61,127,1151,公司41,1995 +96,97,1012,公司42,1995 +79,111,272,公司43,1995 +115,124,909,公司44,1995 +171,146,1122,公司45,1995 +82,44,448,公司46,1995 +84,141,521,公司47,1995 +60,90,313,公司48,1995 +146,102,1372,公司49,1995 +75,47,818,公司0,1996 +45,105,754,公司1,1996 +86,23,793,公司2,1996 +124,121,1245,公司3,1996 +52,105,847,公司4,1996 +70,49,885,公司5,1996 +75,114,1199,公司6,1996 +128,117,1037,公司7,1996 +39,34,359,公司8,1996 +36,144,1061,公司9,1996 +105,41,1054,公司10,1996 +102,15,799,公司11,1996 +132,135,739,公司12,1996 +70,132,917,公司13,1996 +49,121,979,公司14,1996 +106,107,866,公司15,1996 +111,95,345,公司16,1996 +112,116,423,公司17,1996 +83,85,247,公司18,1996 +91,113,800,公司19,1996 +125,91,1130,公司20,1996 +94,106,908,公司21,1996 +108,105,480,公司22,1996 +59,102,492,公司23,1996 +131,41,335,公司24,1996 +60,131,950,公司25,1996 +116,129,874,公司26,1996 +60,116,1118,公司27,1996 +80,127,627,公司28,1996 +91,53,431,公司29,1996 +99,132,491,公司30,1996 +137,121,955,公司31,1996 +79,71,708,公司32,1996 +139,84,1183,公司33,1996 +109,71,1144,公司34,1996 +40,79,823,公司35,1996 +116,99,1142,公司36,1996 +77,111,657,公司37,1996 +113,52,1073,公司38,1996 +135,86,375,公司39,1996 +138,91,777,公司40,1996 +62,128,1127,公司41,1996 +96,100,1024,公司42,1996 +78,114,268,公司43,1996 +114,124,880,公司44,1996 +170,148,1103,公司45,1996 +85,46,436,公司46,1996 +87,144,553,公司47,1996 +59,89,331,公司48,1996 +147,102,1376,公司49,1996 +76,48,804,公司0,1997 +44,106,733,公司1,1997 +88,22,763,公司2,1997 +127,124,1220,公司3,1997 +53,104,850,公司4,1997 +69,52,840,公司5,1997 +77,114,1164,公司6,1997 +127,118,1018,公司7,1997 +41,32,315,公司8,1997 +35,143,1070,公司9,1997 +107,42,1076,公司10,1997 +103,16,851,公司11,1997 +134,137,719,公司12,1997 +71,134,916,公司13,1997 +50,124,984,公司14,1997 +109,106,848,公司15,1997 +113,93,345,公司16,1997 +115,119,476,公司17,1997 +83,85,220,公司18,1997 +93,112,855,公司19,1997 +125,90,1142,公司20,1997 +94,109,883,公司21,1997 +108,108,501,公司22,1997 +62,101,470,公司23,1997 +130,44,337,公司24,1997 +60,134,924,公司25,1997 +115,132,830,公司26,1997 +61,115,1097,公司27,1997 +81,128,629,公司28,1997 +92,56,477,公司29,1997 +102,131,488,公司30,1997 +138,120,973,公司31,1997 +81,71,679,公司32,1997 +138,87,1165,公司33,1997 +110,73,1195,公司34,1997 +39,82,818,公司35,1997 +115,102,1197,公司36,1997 +77,113,660,公司37,1997 +114,56,1048,公司38,1997 +139,87,335,公司39,1997 +141,90,761,公司40,1997 +61,127,1093,公司41,1997 +96,102,1004,公司42,1997 +77,117,302,公司43,1997 +113,127,919,公司44,1997 +171,147,1095,公司45,1997 +87,47,439,公司46,1997 +88,146,573,公司47,1997 +63,89,301,公司48,1997 +147,106,1395,公司49,1997 +79,49,790,公司0,1998 +44,106,738,公司1,1998 +86,25,768,公司2,1998 +126,123,1267,公司3,1998 +53,103,810,公司4,1998 +69,53,804,公司5,1998 +78,116,1180,公司6,1998 +127,118,984,公司7,1998 +40,32,285,公司8,1998 +36,144,1077,公司9,1998 +108,45,1040,公司10,1998 +106,17,897,公司11,1998 +136,139,716,公司12,1998 +73,134,920,公司13,1998 +50,125,989,公司14,1998 +108,107,806,公司15,1998 +114,94,386,公司16,1998 +114,119,495,公司17,1998 +86,85,246,公司18,1998 +92,113,834,公司19,1998 +127,92,1138,公司20,1998 +96,108,900,公司21,1998 +110,108,464,公司22,1998 +64,101,474,公司23,1998 +130,44,310,公司24,1998 +62,137,935,公司25,1998 +115,135,874,公司26,1998 +64,118,1125,公司27,1998 +81,129,599,公司28,1998 +93,56,529,公司29,1998 +105,133,532,公司30,1998 +139,119,928,公司31,1998 +82,73,732,公司32,1998 +137,90,1205,公司33,1998 +112,76,1168,公司34,1998 +40,83,845,公司35,1998 +117,102,1180,公司36,1998 +80,113,661,公司37,1998 +113,58,1050,公司38,1998 +142,88,325,公司39,1998 +141,91,802,公司40,1998 +62,128,1126,公司41,1998 +98,104,972,公司42,1998 +77,117,324,公司43,1998 +113,127,952,公司44,1998 +172,148,1113,公司45,1998 +87,48,396,公司46,1998 +87,148,574,公司47,1998 +62,90,328,公司48,1998 +149,110,1423,公司49,1998 +79,50,748,公司0,1999 +45,106,759,公司1,1999 +88,27,763,公司2,1999 +124,122,1296,公司3,1999 +52,102,805,公司4,1999 +68,52,765,公司5,1999 +78,118,1215,公司6,1999 +128,119,980,公司7,1999 +40,31,244,公司8,1999 +39,143,1061,公司9,1999 +110,46,1060,公司10,1999 +106,20,926,公司11,1999 +137,141,691,公司12,1999 +75,133,935,公司13,1999 +50,126,954,公司14,1999 +107,108,793,公司15,1999 +117,95,376,公司16,1999 +114,122,451,公司17,1999 +89,87,245,公司18,1999 +94,116,827,公司19,1999 +126,94,1161,公司20,1999 +97,110,869,公司21,1999 +110,109,429,公司22,1999 +67,104,474,公司23,1999 +132,45,275,公司24,1999 +61,139,939,公司25,1999 +114,138,928,公司26,1999 +67,120,1106,公司27,1999 +80,130,627,公司28,1999 +93,55,486,公司29,1999 +109,134,526,公司30,1999 +139,121,927,公司31,1999 +81,75,710,公司32,1999 +140,91,1173,公司33,1999 +114,75,1123,公司34,1999 +43,86,874,公司35,1999 +119,105,1213,公司36,1999 +82,115,709,公司37,1999 +115,57,1023,公司38,1999 +143,92,281,公司39,1999 +144,91,796,公司40,1999 +61,128,1136,公司41,1999 +101,106,1026,公司42,1999 +78,116,375,公司43,1999 +115,130,990,公司44,1999 +172,151,1161,公司45,1999 +89,49,414,公司46,1999 +88,147,550,公司47,1999 +64,90,352,公司48,1999 +152,112,1464,公司49,1999 +79,50,741,公司0,2000 +44,105,809,公司1,2000 +89,27,812,公司2,2000 +127,120,1329,公司3,2000 +54,102,831,公司4,2000 +70,55,775,公司5,2000 +80,118,1222,公司6,2000 +130,121,995,公司7,2000 +39,33,259,公司8,2000 +39,143,1114,公司9,2000 +112,46,1016,公司10,2000 +108,23,956,公司11,2000 +139,141,710,公司12,2000 +76,136,955,公司13,2000 +49,129,1008,公司14,2000 +109,108,750,公司15,2000 +117,98,358,公司16,2000 +114,123,423,公司17,2000 +90,87,215,公司18,2000 +94,119,875,公司19,2000 +126,96,1205,公司20,2000 +100,113,923,公司21,2000 +111,109,421,公司22,2000 +68,106,446,公司23,2000 +131,45,237,公司24,2000 +62,141,924,公司25,2000 +117,138,905,公司26,2000 +68,121,1088,公司27,2000 +80,130,612,公司28,2000 +93,54,522,公司29,2000 +109,135,498,公司30,2000 +142,124,897,公司31,2000 +83,75,670,公司32,2000 +141,91,1224,公司33,2000 +115,78,1142,公司34,2000 +47,88,864,公司35,2000 +120,107,1249,公司36,2000 +82,117,704,公司37,2000 +116,57,1074,公司38,2000 +144,95,314,公司39,2000 +143,90,751,公司40,2000 +61,130,1155,公司41,2000 +102,109,1065,公司42,2000 +79,115,418,公司43,2000 +116,131,952,公司44,2000 +172,151,1200,公司45,2000 +92,49,440,公司46,2000 +88,151,581,公司47,2000 +68,94,393,公司48,2000 +154,114,1470,公司49,2000 +79,52,733,公司0,2001 +44,105,834,公司1,2001 +89,29,866,公司2,2001 +128,119,1374,公司3,2001 +53,104,835,公司4,2001 +69,57,784,公司5,2001 +79,119,1209,公司6,2001 +131,120,1010,公司7,2001 +40,32,297,公司8,2001 +37,143,1155,公司9,2001 +112,48,1062,公司10,2001 +109,22,1002,公司11,2001 +141,142,751,公司12,2001 +77,136,960,公司13,2001 +51,128,983,公司14,2001 +112,110,734,公司15,2001 +117,100,403,公司16,2001 +115,122,387,公司17,2001 +91,89,208,公司18,2001 +95,122,871,公司19,2001 +128,97,1162,公司20,2001 +101,114,901,公司21,2001 +110,108,422,公司22,2001 +70,109,475,公司23,2001 +133,44,217,公司24,2001 +64,140,946,公司25,2001 +119,139,883,公司26,2001 +71,124,1071,公司27,2001 +79,133,580,公司28,2001 +95,54,479,公司29,2001 +109,139,527,公司30,2001 +142,124,929,公司31,2001 +85,77,658,公司32,2001 +143,93,1228,公司33,2001 +116,80,1178,公司34,2001 +50,91,881,公司35,2001 +121,110,1280,公司36,2001 +81,120,691,公司37,2001 +120,60,1036,公司38,2001 +146,97,325,公司39,2001 +142,90,790,公司40,2001 +62,130,1166,公司41,2001 +105,110,1097,公司42,2001 +81,116,386,公司43,2001 +117,133,1007,公司44,2001 +171,152,1182,公司45,2001 +92,49,487,公司46,2001 +88,151,605,公司47,2001 +68,98,426,公司48,2001 +158,116,1437,公司49,2001 +80,53,775,公司0,2002 +43,106,802,公司1,2002 +87,31,899,公司2,2002 +127,121,1406,公司3,2002 +54,102,813,公司4,2002 +72,55,741,公司5,2002 +80,119,1210,公司6,2002 +130,119,1006,公司7,2002 +38,32,345,公司8,2002 +35,145,1142,公司9,2002 +111,51,1042,公司10,2002 +112,24,1014,公司11,2002 +141,143,710,公司12,2002 +78,136,963,公司13,2002 +50,131,1003,公司14,2002 +114,109,767,公司15,2002 +118,99,366,公司16,2002 +116,121,409,公司17,2002 +89,90,175,公司18,2002 +94,125,894,公司19,2002 +131,100,1208,公司20,2002 +102,113,882,公司21,2002 +112,111,448,公司22,2002 +69,111,495,公司23,2002 +132,44,219,公司24,2002 +64,141,973,公司25,2002 +119,139,879,公司26,2002 +73,126,1068,公司27,2002 +82,136,542,公司28,2002 +97,53,513,公司29,2002 +110,139,512,公司30,2002 +144,126,952,公司31,2002 +87,76,699,公司32,2002 +144,92,1271,公司33,2002 +119,83,1211,公司34,2002 +52,92,849,公司35,2002 +124,109,1262,公司36,2002 +81,121,677,公司37,2002 +119,63,1031,公司38,2002 +149,100,372,公司39,2002 +143,93,805,公司40,2002 +64,131,1181,公司41,2002 +105,110,1056,公司42,2002 +84,115,420,公司43,2002 +117,133,1020,公司44,2002 +174,152,1156,公司45,2002 +94,52,445,公司46,2002 +91,155,562,公司47,2002 +70,100,429,公司48,2002 +159,118,1398,公司49,2002 +80,56,747,公司0,2003 +43,108,855,公司1,2003 +90,34,917,公司2,2003 +130,120,1413,公司3,2003 +53,101,806,公司4,2003 +72,54,742,公司5,2003 +79,118,1256,公司6,2003 +128,118,1029,公司7,2003 +38,33,388,公司8,2003 +36,146,1155,公司9,2003 +110,52,999,公司10,2003 +111,26,974,公司11,2003 +143,143,675,公司12,2003 +77,139,970,公司13,2003 +51,131,1050,公司14,2003 +117,110,790,公司15,2003 +118,101,381,公司16,2003 +119,120,439,公司17,2003 +91,90,199,公司18,2003 +96,128,928,公司19,2003 +134,101,1210,公司20,2003 +103,114,936,公司21,2003 +111,110,473,公司22,2003 +70,111,501,公司23,2003 +132,47,192,公司24,2003 +63,141,991,公司25,2003 +122,138,839,公司26,2003 +76,127,1119,公司27,2003 +84,136,529,公司28,2003 +97,52,488,公司29,2003 +112,141,559,公司30,2003 +147,128,982,公司31,2003 +89,79,671,公司32,2003 +147,92,1244,公司33,2003 +122,84,1178,公司34,2003 +52,92,855,公司35,2003 +123,113,1273,公司36,2003 +80,121,666,公司37,2003 +118,64,1008,公司38,2003 +150,101,413,公司39,2003 +144,97,837,公司40,2003 +66,131,1184,公司41,2003 +109,113,1042,公司42,2003 +84,117,465,公司43,2003 +116,133,1038,公司44,2003 +175,153,1139,公司45,2003 +98,55,484,公司46,2003 +95,158,551,公司47,2003 +74,102,431,公司48,2003 +160,120,1408,公司49,2003 +82,56,780,公司0,2004 +46,109,885,公司1,2004 +88,35,916,公司2,2004 +129,123,1443,公司3,2004 +52,102,827,公司4,2004 +71,54,728,公司5,2004 +80,119,1226,公司6,2004 +130,121,1028,公司7,2004 +37,33,416,公司8,2004 +36,145,1210,公司9,2004 +110,53,1047,公司10,2004 +110,27,929,公司11,2004 +143,142,654,公司12,2004 +78,138,1011,公司13,2004 +50,130,1053,公司14,2004 +117,110,833,公司15,2004 +118,102,418,公司16,2004 +119,122,460,公司17,2004 +91,91,228,公司18,2004 +98,131,890,公司19,2004 +137,103,1223,公司20,2004 +105,117,907,公司21,2004 +113,109,523,公司22,2004 +70,112,522,公司23,2004 +132,47,243,公司24,2004 +66,144,971,公司25,2004 +123,140,846,公司26,2004 +79,128,1152,公司27,2004 +87,137,583,公司28,2004 +97,52,473,公司29,2004 +112,142,572,公司30,2004 +147,131,1006,公司31,2004 +88,81,674,公司32,2004 +150,92,1250,公司33,2004 +125,85,1183,公司34,2004 +53,94,821,公司35,2004 +124,113,1233,公司36,2004 +81,123,643,公司37,2004 +119,64,1033,公司38,2004 +150,100,436,公司39,2004 +144,100,831,公司40,2004 +67,134,1226,公司41,2004 +108,113,1058,公司42,2004 +86,118,452,公司43,2004 +118,137,997,公司44,2004 +175,155,1168,公司45,2004 +100,57,536,公司46,2004 +96,161,532,公司47,2004 +76,103,441,公司48,2004 +163,123,1373,公司49,2004 +83,58,809,公司0,2005 +47,108,860,公司1,2005 +89,36,934,公司2,2005 +127,123,1466,公司3,2005 +52,104,843,公司4,2005 +70,52,700,公司5,2005 +80,119,1241,公司6,2005 +132,123,1059,公司7,2005 +40,33,405,公司8,2005 +36,146,1204,公司9,2005 +112,52,1036,公司10,2005 +109,26,921,公司11,2005 +145,142,674,公司12,2005 +79,139,992,公司13,2005 +52,132,1077,公司14,2005 +116,113,843,公司15,2005 +118,103,424,公司16,2005 +117,122,432,公司17,2005 +92,92,255,公司18,2005 +99,131,902,公司19,2005 +139,105,1186,公司20,2005 +105,116,935,公司21,2005 +115,108,563,公司22,2005 +72,114,538,公司23,2005 +135,46,269,公司24,2005 +69,144,976,公司25,2005 +124,141,882,公司26,2005 +80,128,1201,公司27,2005 +90,136,596,公司28,2005 +100,53,495,公司29,2005 +114,141,617,公司30,2005 +148,130,991,公司31,2005 +91,83,704,公司32,2005 +153,92,1217,公司33,2005 +125,89,1173,公司34,2005 +56,96,872,公司35,2005 +124,113,1213,公司36,2005 +82,122,664,公司37,2005 +121,67,1000,公司38,2005 +149,104,434,公司39,2005 +144,99,840,公司40,2005 +70,137,1201,公司41,2005 +111,114,1022,公司42,2005 +88,121,409,公司43,2005 +119,141,993,公司44,2005 +177,156,1126,公司45,2005 +104,60,504,公司46,2005 +95,160,499,公司47,2005 +75,102,449,公司48,2005 +162,126,1371,公司49,2005 +86,61,845,公司0,2006 +48,110,907,公司1,2006 +88,36,907,公司2,2006 +128,122,1447,公司3,2006 +54,103,848,公司4,2006 +73,51,718,公司5,2006 +82,119,1264,公司6,2006 +133,122,1070,公司7,2006 +40,35,410,公司8,2006 +35,147,1247,公司9,2006 +112,53,1025,公司10,2006 +109,27,923,公司11,2006 +148,144,667,公司12,2006 +79,140,971,公司13,2006 +51,131,1065,公司14,2006 +118,115,852,公司15,2006 +120,101,472,公司16,2006 +118,122,446,公司17,2006 +95,92,227,公司18,2006 +100,133,942,公司19,2006 +142,104,1212,公司20,2006 +106,117,938,公司21,2006 +117,111,540,公司22,2006 +72,117,552,公司23,2006 +137,48,243,公司24,2006 +69,144,965,公司25,2006 +126,140,905,公司26,2006 +81,130,1239,公司27,2006 +89,138,571,公司28,2006 +103,52,505,公司29,2006 +113,144,671,公司30,2006 +147,133,1025,公司31,2006 +90,83,701,公司32,2006 +154,94,1182,公司33,2006 +127,92,1209,公司34,2006 +55,95,912,公司35,2006 +124,114,1184,公司36,2006 +82,124,650,公司37,2006 +121,70,1004,公司38,2006 +149,106,461,公司39,2006 +143,102,859,公司40,2006 +71,139,1205,公司41,2006 +111,115,1000,公司42,2006 +88,120,376,公司43,2006 +123,140,1016,公司44,2006 +176,156,1088,公司45,2006 +104,61,500,公司46,2006 +98,162,544,公司47,2006 +79,105,485,公司48,2006 +165,126,1343,公司49,2006 +86,60,891,公司0,2007 +50,111,878,公司1,2007 +88,39,883,公司2,2007 +128,124,1469,公司3,2007 +54,104,828,公司4,2007 +73,50,768,公司5,2007 +83,120,1288,公司6,2007 +136,123,1034,公司7,2007 +43,35,425,公司8,2007 +35,148,1258,公司9,2007 +113,54,1047,公司10,2007 +108,26,948,公司11,2007 +149,144,694,公司12,2007 +77,142,1023,公司13,2007 +52,134,1049,公司14,2007 +117,116,859,公司15,2007 +122,104,473,公司16,2007 +120,124,407,公司17,2007 +96,91,187,公司18,2007 +103,135,951,公司19,2007 +141,106,1189,公司20,2007 +105,119,990,公司21,2007 +116,110,582,公司22,2007 +74,115,556,公司23,2007 +138,50,279,公司24,2007 +68,145,953,公司25,2007 +127,142,876,公司26,2007 +83,129,1236,公司27,2007 +91,137,597,公司28,2007 +106,55,471,公司29,2007 +113,143,713,公司30,2007 +149,132,997,公司31,2007 +93,84,668,公司32,2007 +153,96,1174,公司33,2007 +126,95,1180,公司34,2007 +57,97,872,公司35,2007 +126,115,1191,公司36,2007 +86,125,652,公司37,2007 +124,72,992,公司38,2007 +149,108,501,公司39,2007 +144,105,887,公司40,2007 +71,140,1246,公司41,2007 +114,118,1005,公司42,2007 +90,123,335,公司43,2007 +123,142,1015,公司44,2007 +179,157,1114,公司45,2007 +103,65,476,公司46,2007 +99,163,503,公司47,2007 +80,106,508,公司48,2007 +165,127,1300,公司49,2007 +88,60,915,公司0,2008 +51,114,933,公司1,2008 +91,39,914,公司2,2008 +127,126,1443,公司3,2008 +56,103,858,公司4,2008 +72,49,729,公司5,2008 +83,123,1302,公司6,2008 +137,123,1036,公司7,2008 +44,34,424,公司8,2008 +36,150,1261,公司9,2008 +113,52,1092,公司10,2008 +107,25,918,公司11,2008 +150,143,700,公司12,2008 +77,143,1007,公司13,2008 +51,133,1099,公司14,2008 +116,117,815,公司15,2008 +120,103,496,公司16,2008 +122,124,396,公司17,2008 +95,93,181,公司18,2008 +104,138,922,公司19,2008 +140,106,1233,公司20,2008 +105,120,975,公司21,2008 +115,110,635,公司22,2008 +76,115,552,公司23,2008 +140,49,236,公司24,2008 +68,145,986,公司25,2008 +130,143,840,公司26,2008 +85,130,1199,公司27,2008 +90,139,575,公司28,2008 +106,55,464,公司29,2008 +115,142,683,公司30,2008 +151,135,971,公司31,2008 +95,83,627,公司32,2008 +152,96,1202,公司33,2008 +128,97,1147,公司34,2008 +57,99,883,公司35,2008 +126,118,1237,公司36,2008 +89,124,687,公司37,2008 +125,73,950,公司38,2008 +152,110,555,公司39,2008 +148,105,939,公司40,2008 +71,140,1252,公司41,2008 +117,121,1051,公司42,2008 +94,122,328,公司43,2008 +124,144,995,公司44,2008 +179,160,1164,公司45,2008 +106,68,458,公司46,2008 +100,166,495,公司47,2008 +82,108,475,公司48,2008 +168,130,1316,公司49,2008 +89,61,893,公司0,2009 +53,114,913,公司1,2009 +94,40,914,公司2,2009 +128,125,1463,公司3,2009 +58,101,847,公司4,2009 +74,50,753,公司5,2009 +82,123,1338,公司6,2009 +138,125,994,公司7,2009 +46,36,408,公司8,2009 +37,151,1249,公司9,2009 +113,51,1085,公司10,2009 +106,25,901,公司11,2009 +148,143,731,公司12,2009 +76,146,1013,公司13,2009 +54,131,1078,公司14,2009 +115,116,785,公司15,2009 +122,104,541,公司16,2009 +123,122,372,公司17,2009 +98,92,225,公司18,2009 +105,140,945,公司19,2009 +139,109,1236,公司20,2009 +104,123,977,公司21,2009 +118,111,683,公司22,2009 +78,114,536,公司23,2009 +143,52,228,公司24,2009 +69,147,1003,公司25,2009 +129,146,873,公司26,2009 +85,133,1245,公司27,2009 +90,140,611,公司28,2009 +106,57,467,公司29,2009 +115,141,681,公司30,2009 +154,138,1024,公司31,2009 +95,83,587,公司32,2009 +152,99,1161,公司33,2009 +129,97,1115,公司34,2009 +58,99,932,公司35,2009 +129,118,1223,公司36,2009 +89,123,653,公司37,2009 +128,77,907,公司38,2009 +152,112,572,公司39,2009 +149,108,985,公司40,2009 +70,143,1284,公司41,2009 +120,121,1104,公司42,2009 +96,125,339,公司43,2009 +125,145,966,公司44,2009 +178,162,1122,公司45,2009 +106,70,463,公司46,2009 +104,168,515,公司47,2009 +82,108,486,公司48,2009 +171,130,1308,公司49,2009 +90,59,890,公司0,2010 +53,116,920,公司1,2010 +93,39,902,公司2,2010 +127,124,1505,公司3,2010 +58,102,880,公司4,2010 +73,51,710,公司5,2010 +85,122,1309,公司6,2010 +137,124,979,公司7,2010 +47,37,390,公司8,2010 +38,149,1271,公司9,2010 +115,53,1053,公司10,2010 +106,24,916,公司11,2010 +150,142,744,公司12,2010 +79,146,992,公司13,2010 +57,131,1054,公司14,2010 +114,115,780,公司15,2010 +125,104,564,公司16,2010 +126,121,389,公司17,2010 +101,91,182,公司18,2010 +107,143,978,公司19,2010 +142,109,1236,公司20,2010 +104,123,1021,公司21,2010 +118,110,690,公司22,2010 +78,116,534,公司23,2010 +145,55,252,公司24,2010 +72,149,1036,公司25,2010 +131,149,926,公司26,2010 +88,132,1257,公司27,2010 +90,142,577,公司28,2010 +108,57,423,公司29,2010 +114,143,658,公司30,2010 +155,141,1005,公司31,2010 +98,82,558,公司32,2010 +152,100,1150,公司33,2010 +131,98,1075,公司34,2010 +59,101,986,公司35,2010 +130,119,1226,公司36,2010 +89,126,668,公司37,2010 +128,80,938,公司38,2010 +153,115,592,公司39,2010 +151,109,959,公司40,2010 +71,147,1298,公司41,2010 +123,121,1096,公司42,2010 +95,125,392,公司43,2010 +125,144,997,公司44,2010 +180,166,1139,公司45,2010 +107,74,429,公司46,2010 +103,169,570,公司47,2010 +85,110,507,公司48,2010 +174,133,1310,公司49,2010 +93,60,850,公司0,2011 +54,117,928,公司1,2011 +93,41,863,公司2,2011 +128,125,1531,公司3,2011 +58,103,887,公司4,2011 +72,53,692,公司5,2011 +87,124,1333,公司6,2011 +139,123,966,公司7,2011 +49,39,360,公司8,2011 +41,148,1305,公司9,2011 +114,53,1080,公司10,2011 +108,25,876,公司11,2011 +150,141,719,公司12,2011 +81,147,977,公司13,2011 +58,132,1009,公司14,2011 +116,115,745,公司15,2011 +124,105,578,公司16,2011 +125,120,432,公司17,2011 +103,92,182,公司18,2011 +108,145,1032,公司19,2011 +143,108,1197,公司20,2011 +107,125,1059,公司21,2011 +117,111,681,公司22,2011 +79,118,517,公司23,2011 +144,57,297,公司24,2011 +71,150,1022,公司25,2011 +130,152,931,公司26,2011 +89,131,1251,公司27,2011 +91,141,552,公司28,2011 +109,56,444,公司29,2011 +114,143,668,公司30,2011 +154,142,1023,公司31,2011 +98,85,579,公司32,2011 +155,99,1148,公司33,2011 +134,98,1049,公司34,2011 +62,101,1016,公司35,2011 +133,119,1269,公司36,2011 +88,126,719,公司37,2011 +128,79,906,公司38,2011 +157,115,574,公司39,2011 +151,112,997,公司40,2011 +75,146,1278,公司41,2011 +125,124,1096,公司42,2011 +96,127,351,公司43,2011 +129,147,971,公司44,2011 +181,166,1161,公司45,2011 +108,73,463,公司46,2011 +103,170,527,公司47,2011 +87,112,515,公司48,2011 +173,135,1280,公司49,2011 +96,58,872,公司0,2012 +52,117,959,公司1,2012 +92,41,862,公司2,2012 +129,124,1505,公司3,2012 +58,102,843,公司4,2012 +70,55,747,公司5,2012 +87,123,1295,公司6,2012 +140,123,935,公司7,2012 +50,40,360,公司8,2012 +43,150,1274,公司9,2012 +114,54,1053,公司10,2012 +109,28,918,公司11,2012 +150,142,710,公司12,2012 +79,148,949,公司13,2012 +59,133,990,公司14,2012 +116,115,798,公司15,2012 +126,107,567,公司16,2012 +126,120,402,公司17,2012 +105,92,149,公司18,2012 +107,147,1002,公司19,2012 +145,109,1192,公司20,2012 +107,128,1046,公司21,2012 +119,113,733,公司22,2012 +81,119,492,公司23,2012 +147,56,309,公司24,2012 +73,152,1025,公司25,2012 +129,153,892,公司26,2012 +91,131,1236,公司27,2012 +92,143,558,公司28,2012 +112,59,478,公司29,2012 +115,145,720,公司30,2012 +154,142,984,公司31,2012 +99,88,574,公司32,2012 +155,101,1178,公司33,2012 +137,100,1086,公司34,2012 +63,102,973,公司35,2012 +135,121,1240,公司36,2012 +88,128,690,公司37,2012 +127,81,925,公司38,2012 +158,118,599,公司39,2012 +153,115,986,公司40,2012 +77,148,1314,公司41,2012 +126,127,1102,公司42,2012 +97,128,343,公司43,2012 +128,150,947,公司44,2012 +183,169,1188,公司45,2012 +109,75,430,公司46,2012 +105,171,573,公司47,2012 +86,115,568,公司48,2012 +174,136,1259,公司49,2012 +95,60,919,公司0,2013 +52,118,963,公司1,2013 +91,43,864,公司2,2013 +130,123,1544,公司3,2013 +60,100,852,公司4,2013 +73,56,775,公司5,2013 +90,125,1323,公司6,2013 +138,122,945,公司7,2013 +53,42,356,公司8,2013 +43,152,1294,公司9,2013 +115,55,1047,公司10,2013 +109,26,968,公司11,2013 +152,141,676,公司12,2013 +80,150,920,公司13,2013 +58,132,1029,公司14,2013 +116,114,823,公司15,2013 +127,106,537,公司16,2013 +125,122,427,公司17,2013 +104,94,141,公司18,2013 +110,150,1019,公司19,2013 +144,110,1228,公司20,2013 +110,127,1098,公司21,2013 +118,114,784,公司22,2013 +82,121,534,公司23,2013 +148,59,278,公司24,2013 +72,151,1040,公司25,2013 +132,154,897,公司26,2013 +90,133,1279,公司27,2013 +95,142,564,公司28,2013 +112,61,445,公司29,2013 +114,148,696,公司30,2013 +155,143,966,公司31,2013 +100,88,621,公司32,2013 +157,101,1174,公司33,2013 +139,101,1139,公司34,2013 +63,102,931,公司35,2013 +134,125,1213,公司36,2013 +88,129,713,公司37,2013 +129,84,916,公司38,2013 +161,119,557,公司39,2013 +152,118,966,公司40,2013 +79,147,1289,公司41,2013 +127,128,1118,公司42,2013 +100,128,337,公司43,2013 +131,151,904,公司44,2013 +185,171,1217,公司45,2013 +110,75,406,公司46,2013 +105,174,573,公司47,2013 +87,115,621,公司48,2013 +177,138,1295,公司49,2013 +97,62,885,公司0,2014 +54,120,946,公司1,2014 +89,42,866,公司2,2014 +129,126,1527,公司3,2014 +61,101,855,公司4,2014 +74,57,762,公司5,2014 +89,125,1333,公司6,2014 +138,121,906,公司7,2014 +55,42,325,公司8,2014 +44,153,1277,公司9,2014 +114,55,1049,公司10,2014 +112,26,998,公司11,2014 +153,144,698,公司12,2014 +82,148,959,公司13,2014 +59,132,1011,公司14,2014 +115,115,876,公司15,2014 +127,107,540,公司16,2014 +124,125,390,公司17,2014 +104,97,151,公司18,2014 +113,153,1020,公司19,2014 +147,113,1280,公司20,2014 +109,130,1150,公司21,2014 +119,116,802,公司22,2014 +84,123,563,公司23,2014 +148,60,287,公司24,2014 +72,150,1019,公司25,2014 +132,155,911,公司26,2014 +93,133,1291,公司27,2014 +98,145,589,公司28,2014 +112,63,401,公司29,2014 +116,149,714,公司30,2014 +157,142,991,公司31,2014 +99,87,646,公司32,2014 +158,104,1157,公司33,2014 +142,104,1185,公司34,2014 +66,101,944,公司35,2014 +136,124,1177,公司36,2014 +90,129,726,公司37,2014 +129,83,957,公司38,2014 +162,120,600,公司39,2014 +152,120,923,公司40,2014 +78,147,1339,公司41,2014 +131,129,1107,公司42,2014 +100,130,337,公司43,2014 +135,152,878,公司44,2014 +186,171,1254,公司45,2014 +113,77,399,公司46,2014 +106,174,592,公司47,2014 +89,118,584,公司48,2014 +180,138,1297,公司49,2014 +99,63,893,公司0,2015 +54,119,975,公司1,2015 +87,43,840,公司2,2015 +132,124,1574,公司3,2015 +61,101,882,公司4,2015 +73,55,746,公司5,2015 +88,124,1318,公司6,2015 +139,123,894,公司7,2015 +54,45,340,公司8,2015 +45,153,1293,公司9,2015 +116,56,1019,公司10,2015 +115,27,990,公司11,2015 +156,145,709,公司12,2015 +81,147,931,公司13,2015 +61,133,990,公司14,2015 +118,116,894,公司15,2015 +129,106,562,公司16,2015 +126,125,395,公司17,2015 +104,96,166,公司18,2015 +112,152,989,公司19,2015 +149,115,1251,公司20,2015 +109,131,1193,公司21,2015 +119,117,758,公司22,2015 +87,122,593,公司23,2015 +150,62,267,公司24,2015 +72,151,1015,公司25,2015 +132,155,931,公司26,2015 +92,133,1287,公司27,2015 +101,147,583,公司28,2015 +114,62,364,公司29,2015 +117,152,716,公司30,2015 +157,141,1022,公司31,2015 +100,90,669,公司32,2015 +158,106,1131,公司33,2015 +143,103,1154,公司34,2015 +69,103,972,公司35,2015 +136,124,1203,公司36,2015 +92,131,775,公司37,2015 +130,83,951,公司38,2015 +164,122,603,公司39,2015 +156,123,967,公司40,2015 +81,149,1319,公司41,2015 +135,131,1160,公司42,2015 +100,130,302,公司43,2015 +135,155,845,公司44,2015 +188,170,1281,公司45,2015 +116,78,436,公司46,2015 +106,173,603,公司47,2015 +89,120,545,公司48,2015 +181,139,1257,公司49,2015 +98,62,927,公司0,2016 +57,117,1012,公司1,2016 +90,42,866,公司2,2016 +132,124,1602,公司3,2016 +61,103,841,公司4,2016 +75,54,780,公司5,2016 +87,125,1321,公司6,2016 +139,124,862,公司7,2016 +57,47,355,公司8,2016 +47,155,1299,公司9,2016 +119,58,983,公司10,2016 +116,27,988,公司11,2016 +157,145,745,公司12,2016 +80,149,936,公司13,2016 +62,132,1017,公司14,2016 +119,119,892,公司15,2016 +131,109,585,公司16,2016 +125,127,408,公司17,2016 +106,98,135,公司18,2016 +112,153,945,公司19,2016 +150,115,1218,公司20,2016 +108,131,1235,公司21,2016 +121,117,727,公司22,2016 +89,124,645,公司23,2016 +152,63,276,公司24,2016 +73,154,1023,公司25,2016 +132,155,937,公司26,2016 +94,135,1279,公司27,2016 +104,148,599,公司28,2016 +117,65,396,公司29,2016 +119,154,759,公司30,2016 +157,142,993,公司31,2016 +99,89,667,公司32,2016 +161,109,1166,公司33,2016 +142,104,1128,公司34,2016 +73,102,984,公司35,2016 +135,125,1227,公司36,2016 +93,133,771,公司37,2016 +133,82,997,公司38,2016 +163,123,601,公司39,2016 +158,126,1002,公司40,2016 +81,150,1295,公司41,2016 +137,131,1200,公司42,2016 +103,133,305,公司43,2016 +138,159,809,公司44,2016 +189,169,1308,公司45,2016 +118,81,434,公司46,2016 +106,172,595,公司47,2016 +89,122,588,公司48,2016 +183,142,1255,公司49,2016 +101,61,952,公司0,2017 +57,117,987,公司1,2017 +90,42,837,公司2,2017 +134,126,1580,公司3,2017 +61,101,816,公司4,2017 +77,55,823,公司5,2017 +86,127,1350,公司6,2017 +141,123,866,公司7,2017 +59,46,408,公司8,2017 +50,158,1338,公司9,2017 +119,60,1026,公司10,2017 +115,28,1016,公司11,2017 +157,145,797,公司12,2017 +81,148,923,公司13,2017 +61,135,1017,公司14,2017 +120,119,864,公司15,2017 +133,111,629,公司16,2017 +124,129,443,公司17,2017 +107,100,165,公司18,2017 +112,154,949,公司19,2017 +151,116,1254,公司20,2017 +111,133,1248,公司21,2017 +121,117,742,公司22,2017 +89,125,663,公司23,2017 +150,66,298,公司24,2017 +75,154,1051,公司25,2017 +135,154,960,公司26,2017 +93,138,1258,公司27,2017 +104,150,642,公司28,2017 +119,64,352,公司29,2017 +119,154,775,公司30,2017 +157,143,1023,公司31,2017 +100,90,641,公司32,2017 +162,111,1217,公司33,2017 +141,104,1115,公司34,2017 +76,102,968,公司35,2017 +138,124,1245,公司36,2017 +96,135,758,公司37,2017 +136,83,1028,公司38,2017 +162,122,577,公司39,2017 +160,127,1018,公司40,2017 +83,153,1305,公司41,2017 +140,135,1238,公司42,2017 +102,137,303,公司43,2017 +141,162,789,公司44,2017 +188,173,1340,公司45,2017 +121,84,408,公司46,2017 +107,174,584,公司47,2017 +91,123,579,公司48,2017 +185,145,1240,公司49,2017 +103,63,970,公司0,2018 +55,117,1038,公司1,2018 +88,42,875,公司2,2018 +132,126,1631,公司3,2018 +62,104,791,公司4,2018 +79,56,809,公司5,2018 +85,127,1366,公司6,2018 +140,126,880,公司7,2018 +60,48,381,公司8,2018 +51,157,1298,公司9,2018 +120,59,1075,公司10,2018 +114,28,1016,公司11,2018 +159,143,793,公司12,2018 +84,146,882,公司13,2018 +64,138,1033,公司14,2018 +123,121,851,公司15,2018 +135,112,614,公司16,2018 +126,128,429,公司17,2018 +106,101,138,公司18,2018 +115,157,924,公司19,2018 +151,118,1295,公司20,2018 +114,135,1238,公司21,2018 +120,120,763,公司22,2018 +92,128,717,公司23,2018 +151,67,282,公司24,2018 +74,154,1044,公司25,2018 +138,155,928,公司26,2018 +95,139,1270,公司27,2018 +107,153,672,公司28,2018 +119,66,344,公司29,2018 +121,157,734,公司30,2018 +157,143,990,公司31,2018 +101,89,614,公司32,2018 +163,114,1239,公司33,2018 +143,105,1166,公司34,2018 +75,102,1015,公司35,2018 +137,123,1277,公司36,2018 +98,135,811,公司37,2018 +135,86,991,公司38,2018 +164,122,592,公司39,2018 +159,128,1043,公司40,2018 +82,156,1272,公司41,2018 +139,136,1200,公司42,2018 +102,139,309,公司43,2018 +144,161,829,公司44,2018 +187,176,1386,公司45,2018 +121,85,384,公司46,2018 +107,177,628,公司47,2018 +94,124,581,公司48,2018 +186,147,1236,公司49,2018 +105,64,931,公司0,2019 +55,117,1068,公司1,2019 +87,43,912,公司2,2019 +132,125,1637,公司3,2019 +64,106,766,公司4,2019 +82,55,857,公司5,2019 +84,126,1411,公司6,2019 +142,126,852,公司7,2019 +62,49,405,公司8,2019 +50,157,1267,公司9,2019 +123,61,1056,公司10,2019 +114,27,1028,公司11,2019 +158,142,804,公司12,2019 +84,149,924,公司13,2019 +65,140,1007,公司14,2019 +125,122,863,公司15,2019 +137,113,665,公司16,2019 +127,129,424,公司17,2019 +108,101,113,公司18,2019 +115,159,937,公司19,2019 +150,120,1291,公司20,2019 +115,137,1235,公司21,2019 +119,119,770,公司22,2019 +92,128,740,公司23,2019 +150,66,270,公司24,2019 +75,156,1096,公司25,2019 +140,156,920,公司26,2019 +97,138,1256,公司27,2019 +109,154,689,公司28,2019 +120,67,389,公司29,2019 +122,160,764,公司30,2019 +160,142,1034,公司31,2019 +103,90,627,公司32,2019 +163,116,1253,公司33,2019 +143,104,1142,公司34,2019 +78,105,1020,公司35,2019 +141,122,1271,公司36,2019 +99,135,859,公司37,2019 +138,87,952,公司38,2019 +164,121,555,公司39,2019 +162,130,1081,公司40,2019 +83,159,1229,公司41,2019 +138,137,1211,公司42,2019 +105,138,355,公司43,2019 +147,160,788,公司44,2019 +187,175,1438,公司45,2019 +123,85,366,公司46,2019 +110,181,646,公司47,2019 +94,123,589,公司48,2019 +186,146,1262,公司49,2019 +107,65,910,公司0,2020 +57,120,1094,公司1,2020 +86,45,914,公司2,2020 +133,123,1653,公司3,2020 +66,106,818,公司4,2020 +81,57,857,公司5,2020 +86,126,1413,公司6,2020 +142,129,829,公司7,2020 +63,51,445,公司8,2020 +49,157,1278,公司9,2020 +124,61,1099,公司10,2020 +115,27,999,公司11,2020 +161,141,836,公司12,2020 +83,147,892,公司13,2020 +65,139,974,公司14,2020 +126,121,889,公司15,2020 +140,115,630,公司16,2020 +130,132,458,公司17,2020 +107,104,105,公司18,2020 +117,159,949,公司19,2020 +150,122,1319,公司20,2020 +118,136,1284,公司21,2020 +121,122,809,公司22,2020 +93,129,746,公司23,2020 +152,66,267,公司24,2020 +74,156,1065,公司25,2020 +139,156,928,公司26,2020 +100,141,1286,公司27,2020 +110,154,715,公司28,2020 +119,67,383,公司29,2020 +122,159,816,公司30,2020 +161,143,1082,公司31,2020 +105,92,661,公司32,2020 +163,116,1222,公司33,2020 +147,107,1106,公司34,2020 +81,107,992,公司35,2020 +144,123,1247,公司36,2020 +98,138,861,公司37,2020 +139,90,932,公司38,2020 +164,124,513,公司39,2020 +165,130,1105,公司40,2020 +83,159,1192,公司41,2020 +139,140,1178,公司42,2020 +105,138,384,公司43,2020 +148,161,793,公司44,2020 +190,179,1464,公司45,2020 +124,86,381,公司46,2020 +111,182,629,公司47,2020 +98,122,575,公司48,2020 +188,145,1235,公司49,2020 +106,66,941,公司0,2021 +58,121,1117,公司1,2021 +87,46,883,公司2,2021 +133,125,1618,公司3,2021 +69,108,852,公司4,2021 +82,59,890,公司5,2021 +84,128,1421,公司6,2021 +141,130,788,公司7,2021 +63,51,440,公司8,2021 +49,159,1306,公司9,2021 +123,63,1064,公司10,2021 +117,27,1030,公司11,2021 +160,144,842,公司12,2021 +82,146,879,公司13,2021 +67,138,986,公司14,2021 +127,121,914,公司15,2021 +140,117,587,公司16,2021 +130,132,418,公司17,2021 +105,104,132,公司18,2021 +120,160,934,公司19,2021 +153,121,1283,公司20,2021 +119,139,1250,公司21,2021 +120,124,846,公司22,2021 +93,131,758,公司23,2021 +151,66,259,公司24,2021 +76,159,1057,公司25,2021 +140,159,965,公司26,2021 +103,140,1263,公司27,2021 +110,156,759,公司28,2021 +118,68,346,公司29,2021 +123,162,834,公司30,2021 +164,142,1129,公司31,2021 +104,93,696,公司32,2021 +162,116,1257,公司33,2021 +150,109,1124,公司34,2021 +81,106,971,公司35,2021 +143,126,1235,公司36,2021 +101,139,877,公司37,2021 +139,90,930,公司38,2021 +165,123,500,公司39,2021 +167,130,1124,公司40,2021 +83,162,1226,公司41,2021 +138,140,1164,公司42,2021 +108,138,429,公司43,2021 +147,161,774,公司44,2021 +192,180,1510,公司45,2021 +125,89,381,公司46,2021 +113,185,633,公司47,2021 +102,122,626,公司48,2021 +188,147,1222,公司49,2021 +107,67,989,公司0,2022 +56,120,1120,公司1,2022 +87,48,865,公司2,2022 +133,124,1611,公司3,2022 +70,106,881,公司4,2022 +81,60,894,公司5,2022 +83,131,1399,公司6,2022 +141,130,778,公司7,2022 +64,51,411,公司8,2022 +48,161,1339,公司9,2022 +125,62,1054,公司10,2022 +118,30,1056,公司11,2022 +158,146,842,公司12,2022 +80,149,901,公司13,2022 +65,139,988,公司14,2022 +130,121,958,公司15,2022 +140,120,580,公司16,2022 +133,134,436,公司17,2022 +107,103,125,公司18,2022 +118,162,898,公司19,2022 +152,122,1284,公司20,2022 +119,141,1211,公司21,2022 +121,123,830,公司22,2022 +95,131,730,公司23,2022 +151,67,244,公司24,2022 +75,162,1085,公司25,2022 +142,158,998,公司26,2022 +103,141,1290,公司27,2022 +112,157,771,公司28,2022 +120,69,327,公司29,2022 +123,163,846,公司30,2022 +167,143,1183,公司31,2022 +105,96,664,公司32,2022 +164,116,1281,公司33,2022 +152,112,1124,公司34,2022 +83,105,955,公司35,2022 +143,130,1287,公司36,2022 +105,142,869,公司37,2022 +139,89,902,公司38,2022 +167,126,496,公司39,2022 +166,129,1112,公司40,2022 +86,165,1245,公司41,2022 +137,140,1197,公司42,2022 +112,138,464,公司43,2022 +150,164,731,公司44,2022 +194,182,1557,公司45,2022 +125,93,378,公司46,2022 +114,187,649,公司47,2022 +105,122,593,公司48,2022 +192,148,1244,公司49,2022 +106,69,1036,公司0,2023 +59,119,1118,公司1,2023 +89,49,822,公司2,2023 +133,125,1594,公司3,2023 +72,106,860,公司4,2023 +81,60,858,公司5,2023 +82,130,1395,公司6,2023 +140,132,765,公司7,2023 +66,51,387,公司8,2023 +50,163,1392,公司9,2023 +125,60,1086,公司10,2023 +118,29,1052,公司11,2023 +159,146,816,公司12,2023 +83,147,945,公司13,2023 +66,137,1036,公司14,2023 +132,120,954,公司15,2023 +141,122,607,公司16,2023 +132,136,397,公司17,2023 +107,105,109,公司18,2023 +117,165,899,公司19,2023 +151,122,1322,公司20,2023 +120,143,1222,公司21,2023 +121,126,833,公司22,2023 +95,134,746,公司23,2023 +152,70,287,公司24,2023 +75,165,1106,公司25,2023 +141,160,1019,公司26,2023 +105,140,1274,公司27,2023 +115,159,763,公司28,2023 +121,68,363,公司29,2023 +124,162,836,公司30,2023 +166,144,1230,公司31,2023 +105,96,715,公司32,2023 +167,117,1329,公司33,2023 +154,115,1172,公司34,2023 +82,105,962,公司35,2023 +144,131,1274,公司36,2023 +107,144,869,公司37,2023 +140,89,937,公司38,2023 +167,129,542,公司39,2023 +169,132,1136,公司40,2023 +88,165,1232,公司41,2023 +141,142,1217,公司42,2023 +114,141,507,公司43,2023 +154,168,740,公司44,2023 +193,185,1528,公司45,2023 +128,94,341,公司46,2023 +114,191,652,公司47,2023 +108,123,647,公司48,2023 +192,147,1227,公司49,2023`, + input: '使用动态散点图帮我展示各公司年度收益率、净现金流和市值逐年的变化。' +}; + +export const dynamicRoseData = { + csv: `country,continent,GDP,year +美国,美洲,239270,1973 +印度,亚洲,22960,1973 +韩国,亚洲,7870,1973 +土耳其,亚洲,17240,1973 +印度尼西亚,亚洲,10980,1973 +沙特阿拉伯,亚洲,23760,1973 +泰国,亚洲,4130,1973 +菲律宾,亚洲,5660,1973 +马来西亚,亚洲,2780,1973 +英国,欧洲,114750,1973 +意大利,欧洲,107570,1973 +西班牙,欧洲,55990,1973 +荷兰,欧洲,36410,1973 +瑞士,欧洲,33920,1973 +芬兰,欧洲,9930,1973 +瑞典,欧洲,23020,1973 +比利时,欧洲,22280,1973 +挪威,欧洲,14540,1973 +美国,美洲,278390,1978 +印度,亚洲,29220,1978 +韩国,亚洲,13120,1978 +土耳其,亚洲,22620,1978 +印度尼西亚,亚洲,15390,1978 +沙特阿拉伯,亚洲,30080,1978 +泰国,亚洲,6000,1978 +菲律宾,亚洲,7480,1978 +马来西亚,亚洲,3900,1978 +英国,欧洲,121140,1978 +意大利,欧洲,126030,1978 +西班牙,欧洲,64080,1978 +荷兰,欧洲,41420,1978 +瑞士,欧洲,32360,1978 +芬兰,欧洲,10800,1978 +瑞典,欧洲,24650,1978 +比利时,欧洲,24970,1978 +挪威,欧洲,18160,1978 +美国,美洲,336520,1983 +印度,亚洲,34790,1983 +韩国,亚洲,18480,1983 +土耳其,亚洲,25000,1983 +印度尼西亚,亚洲,20870,1983 +沙特阿拉伯,亚洲,24130,1983 +泰国,亚洲,7840,1983 +菲律宾,亚洲,9070,1983 +马来西亚,亚洲,5510,1983 +英国,欧洲,129860.00000000001,1983 +意大利,欧洲,141500,1983 +西班牙,欧洲,67430,1983 +荷兰,欧洲,42830,1983 +瑞士,欧洲,35000,1983 +芬兰,欧洲,13130,1983 +瑞典,欧洲,26980,1983 +比利时,欧洲,26860,1983 +挪威,欧洲,20980,1983 +美国,美洲,427650,1988 +印度,亚洲,45400,1988 +韩国,亚洲,30960,1988 +土耳其,亚洲,33340,1988 +印度尼西亚,亚洲,26890,1988 +沙特阿拉伯,亚洲,25640,1988 +泰国,亚洲,11350,1988 +菲律宾,亚洲,8970,1988 +马来西亚,亚洲,6880,1988 +英国,欧洲,158980,1988 +意大利,欧洲,166060,1988 +西班牙,欧洲,80430,1988 +荷兰,欧洲,49070,1988 +瑞士,欧洲,39950,1988 +芬兰,欧洲,15720,1988 +瑞典,欧洲,31270,1988 +比利时,欧洲,30520,1988 +挪威,欧洲,24800,1988 +美国,美洲,488020,1993 +印度,亚洲,56680,1993 +韩国,亚洲,45790,1993 +土耳其,亚洲,41600,1993 +印度尼西亚,亚洲,37570,1993 +沙特阿拉伯,亚洲,34670,1993 +泰国,亚洲,17990,1993 +菲律宾,亚洲,10000,1993 +马来西亚,亚洲,10720,1993 +英国,欧洲,167180,1993 +意大利,欧洲,177750,1993 +西班牙,欧洲,89630,1993 +荷兰,欧洲,56320,1993 +瑞士,欧洲,42740,1993 +芬兰,欧洲,15040,1993 +瑞典,欧洲,30950,1993 +比利时,欧洲,33350,1993 +挪威,欧洲,28050,1993 +美国,美洲,521740,1998 +印度,亚洲,77270,1998 +韩国,亚洲,59600,1998 +土耳其,亚洲,50560,1998 +印度尼西亚,亚洲,42880,1998 +沙特阿拉伯,亚洲,37310,1998 +泰国,亚洲,19930,1998 +菲律宾,亚洲,12090,1998 +马来西亚,亚洲,14070,1998 +英国,欧洲,196370,1998 +意大利,欧洲,196130,1998 +西班牙,欧洲,104800,1998 +荷兰,欧洲,67570,1998 +瑞士,欧洲,46090,1998 +芬兰,欧洲,18940,1998 +瑞典,欧洲,36510,1998 +比利时,欧洲,37790,1998 +挪威,欧洲,34830,1998 +美国,美洲,545920,2003 +印度,亚洲,102500,2003 +韩国,亚洲,84420,2003 +土耳其,亚洲,55060,2003 +印度尼西亚,亚洲,51460,2003 +沙特阿拉伯,亚洲,40500,2003 +泰国,亚洲,25620,2003 +菲律宾,亚洲,14620,2003 +马来西亚,亚洲,18210,2003 +英国,欧洲,228640,2003 +意大利,欧洲,211730,2003 +西班牙,欧洲,126720,2003 +荷兰,欧洲,75950,2003 +瑞士,欧洲,49450,2003 +芬兰,欧洲,22260,2003 +瑞典,欧洲,42290,2003 +比利时,欧洲,42160,2003 +挪威,欧洲,38320,2003 +美国,美洲,578410,2008 +印度,亚洲,143180,2008 +韩国,亚洲,106280,2008 +土耳其,亚洲,74660,2008 +印度尼西亚,亚洲,67940,2008 +沙特阿拉伯,亚洲,51340,2008 +泰国,亚洲,31950,2008 +菲律宾,亚洲,19140,2008 +马来西亚,亚洲,24100,2008 +英国,欧洲,253560,2008 +意大利,欧洲,221500,2008 +西班牙,欧洲,147390,2008 +荷兰,欧洲,86710,2008 +瑞士,欧洲,57960,2008 +芬兰,欧洲,26270,2008 +瑞典,欧洲,48920,2008 +比利时,欧洲,47720,2008 +挪威,欧洲,43330,2008 +美国,美洲,589420,2013 +印度,亚洲,197840,2013 +韩国,亚洲,125320,2013 +土耳其,亚洲,97510,2013 +印度尼西亚,亚洲,89730,2013 +沙特阿拉伯,亚洲,62900,2013 +泰国,亚洲,37880,2013 +菲律宾,亚洲,24700,2013 +马来西亚,亚洲,29650,2013 +英国,欧洲,260510,2013 +意大利,欧洲,204670,2013 +西班牙,欧洲,134780,2013 +荷兰,欧洲,84970,2013 +瑞士,欧洲,61070,2013 +芬兰,欧洲,24970,2013 +瑞典,欧洲,51470,2013 +比利时,欧洲,49500,2013 +挪威,欧洲,44930,2013 +美国,美洲,617030,2018 +印度,亚洲,282220,2018 +韩国,亚洲,144970,2018 +土耳其,亚洲,124050,2018 +印度尼西亚,亚洲,114690,2018 +沙特阿拉伯,亚洲,70160,2018 +泰国,亚洲,44230,2018 +菲律宾,亚洲,34030,2018 +马来西亚,亚洲,38210,2018 +英国,欧洲,287930,2018 +意大利,欧洲,214100,2018 +西班牙,欧洲,153950,2018 +荷兰,欧洲,94810,2018 +瑞士,欧洲,67460,2018 +芬兰,欧洲,26920,2018 +瑞典,欧洲,58930,2018 +比利时,欧洲,53840,2018 +挪威,欧洲,48930,2018 +`, + input: '使用动态玫瑰图帮我展示各国GDP排名变化' +}; +export const dynamicRoseData1 = { + csv: `country,GDP,year +USA,37,2000 +Russia,32,2000 +China,28,2000 +Australia,16,2000 +Germany,13,2000 +France,13,2000 +Italy,13,2000 +Netherlands,12,2000 +Cuba,11,2000 +U.K.,11,2000 +USA,36,2004 +China,32,2004 +Russia,28,2004 +Australia,17,2004 +Japan,16,2004 +Germany,13,2004 +France,11,2004 +Italy,10,2004 +South Korea,9,2004 +U.K.,9,2004 +China,48,2008 +USA,36,2008 +Russia,24,2008 +U.K.,19,2008 +Germany,16,2008 +Australia,14,2008 +South Korea,13,2008 +Japan,9,2008 +Italy,8,2008 +France,7,2008 +USA,46,2012 +China,39,2012 +U.K.,29,2012 +Russia,19,2012 +South Korea,13,2012 +Germany,11,2012 +France,11,2012 +Australia,8,2012 +Italy,8,2012 +Hungary,8,2012 +USA,46,2016 +U.K.,27,2016 +China,26,2016 +Russia,19,2016 +Germany,17,2016 +Japan,12,2016 +France,10,2016 +South Korea,9,2016 +Italy,8,2016 +Australia,8,2016 +USA,39,2020 +China,38,2020 +Japan,27,2020 +U.K.,22,2020 +Russian Olympic Committee,20,2020 +Australia,17,2020 +Netherlands,10,2020 +France,10,2020 +Germany,10,2020 +Italy,10,2020`, + input: '使用动态玫瑰图帮我展示各国GDP排名变化' +}; + export const mockUserTextInput0 = { text: `快手消失了。快手上市后，市值一度超过2000亿美元，现在只剩200多亿美元。去年快手的营收破了千亿，公司也赚钱了，但市场不买账了。 滴滴也不见了。滴滴去年营收接近2000亿元，并首次实现年度盈利，不过滴滴从美股退市后，一直没在港股上市，所以没有市值参考。`, diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index c093aa0b..e744de0b 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -53,7 +53,10 @@ import { singleColumnLineCombinationChartData, singleColumnLineCombinationChartData1, singleColumnBarCombinationChartData1, - singleColumnBarCombinationChartData + singleColumnBarCombinationChartData, + dynamicScatterPlotData, + dynamicRoseData, + dynamicRoseData1 } from '../../constants/mockData'; import VMind, { ArcoTheme, builtinThemeMap, BuiltinThemeType } from '../../../../../src/index'; import { Model } from '../../../../../src/index'; @@ -114,7 +117,10 @@ const demoDataList: { [key: string]: any } = { SingleColumnLineCommon: singleColumnLineCombinationChartData, SingleColumnLineCommon1: singleColumnLineCombinationChartData1, SingleColumnBarCommon: singleColumnBarCombinationChartData, - SingleColumnBarCommon1: singleColumnBarCombinationChartData1 + SingleColumnBarCommon1: singleColumnBarCombinationChartData1, + dynamicScatterPlotData: dynamicScatterPlotData, + dynamicRoseData: dynamicRoseData, + dynamicRoseData1: dynamicRoseData1 }; const globalVariables = (import.meta as any).env; diff --git a/packages/vmind/package.json b/packages/vmind/package.json index c86a6a5a..db531caa 100644 --- a/packages/vmind/package.json +++ b/packages/vmind/package.json @@ -89,6 +89,7 @@ "@types/euclidean-distance": "~1.0.3" }, "dependencies": { + "chroma-js": "^3.1.1", "@visactor/chart-advisor": "workspace:1.2.13", "@visactor/vdataset": "~0.17.4", "@visactor/vutils": "~0.17.4", diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index f40c6bca..d666f061 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -34,6 +34,12 @@ export const CARTESIAN_CHART_LIST = [ ChartType.BasicHeatMap ]; +export const DYNAMIC_CHART_LIST = [ + ChartType.DynamicBarChart, + ChartType.DynamicScatterPlotChart, + ChartType.DynamicRoseChart +]; + export const DEFAULT_MAP_OPTION: BasemapOption = { regionProjectType: null, regionCoordinate: MapRegionCoordinate.GEO, diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts index 38fb8e62..aced9307 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts @@ -122,5 +122,23 @@ export const chartKnowledgeBase: ChartKnowledgeBase = { [ChartType.SingleColumnCombinationChart]: { knowledge: ['Single column combination charts can be combined with a variety of different basic chart types'], constraints: [`subChartType cannot be empty, it is an array of values in ${COMBINATION_BASIC_CHART_LIST}.`] + }, + [ChartType.DynamicScatterPlotChart]: { + knowledge: [ + 'Dynamic Scatter Plot Chart can highlight changes in the correlation between variables as time progresses.', + 'It can display the distribution of data points and trends, along with movement or growth of specific categories over time.' + ], + constraints: [ + 'Use a dynamic scatter plot chart when you want to visualize the relationship between two or three quantitative variables and observe how that relationship evolves over time.' + ] + }, + [ChartType.DynamicRoseChart]: { + knowledge: [ + 'Dynamic Rose Chart is used to display cyclical or seasonal data over time, with values represented by the length of radial bars.', + 'Dynamic Rose Chart highlights changes in categorical data or periodic trends across multiple categories over time.' + ], + constraints: [ + 'Use Dynamic Rose Chart if you want to show cyclical data and observe changes in multiple categories over time.' + ] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/patcher/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/patcher/index.ts index 4eb8763d..61616e32 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/patcher/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/patcher/index.ts @@ -15,6 +15,7 @@ import type { Cell } from '../../../../types'; import { patchBasicHeatMapChart, patchCartesianXField, + patchDynamicScatterPlotChart, patchLinearProgressChart, patchNeedColor, patchNeedSize, @@ -355,5 +356,6 @@ export const patchPipelines: Transformer= 2) ) { return { ...context @@ -421,7 +422,7 @@ export const patchNeedColor: Transformer< if (colorField) { cellNew.color = colorField.fieldName; } else { - cellNew.color = remainedFields[0].fieldName; + cellNew.color = remainedFields[0] ? remainedFields[0].fieldName : null; } } } @@ -452,7 +453,7 @@ export const patchNeedSize: Transformer< if (sizeField) { cellNew.size = sizeField.fieldName; } else { - cellNew.size = remainedFields[0].fieldName; + cellNew.size = remainedFields[0] ? remainedFields[0].fieldName : null; } } } @@ -607,3 +608,67 @@ export const patchSingleColumnCombinationChart: Transformer< } return {}; }; + +export const patchDynamicScatterPlotChart: Transformer< + GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput, + Partial +> = (context: GenerateChartAndFieldMapContext & GenerateChartAndFieldMapOutput) => { + const { chartType, cells, fieldInfo } = context; + if (isCombinationChartType(chartType)) { + return {}; + } + const cellNew = { ...getCell(cells) }; + let chartTypeNew = chartType; + + if (chartType === ChartType.DynamicScatterPlotChart.toUpperCase()) { + if (cellNew.y && isArray(cellNew.y)) { + cellNew.y = cellNew.y[0]; + } + if (cellNew.color && isArray(cellNew.color)) { + cellNew.color = cellNew.color[0]; + } + if (!cellNew.time || cellNew.time === '' || cellNew.time.length === 0) { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + + //Dynamic scatter plot chart does not have a time field, choose a discrete field as time. + const timeField = getFieldByDataType(remainedFields, [DataType.DATE]); + if (timeField) { + cellNew.time = timeField.fieldName; + } else { + const stringField = getFieldByDataType(remainedFields, [DataType.STRING]); + if (stringField) { + cellNew.time = stringField.fieldName; + } else { + //no available field, set chart type to scatter plot chart + chartTypeNew = ChartType.ScatterPlot.toUpperCase(); + return { + //...context, + cells: [cellNew], + chartType: chartTypeNew + }; + } + } + } + if (!cellNew.x || cellNew.x === cellNew.time) { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const xField = getFieldByDataType(remainedFields, [DataType.INT, DataType.FLOAT]); + if (xField) { + cellNew.x = xField.fieldName; + } + } + + if (!cellNew.size || cellNew.size === cellNew.time) { + const remainedFields = getRemainedFields(cellNew, fieldInfo); + const sizeField = getFieldByDataType(remainedFields, [DataType.INT, DataType.FLOAT]); + if (sizeField) { + cellNew.size = sizeField.fieldName; + } + } + } + + return { + //...context, + cells: [cellNew], + chartType: chartTypeNew + }; +}; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts index 3203f364..e9d46b9a 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/index.ts @@ -19,7 +19,7 @@ const patchUserInput = (userInput: string) => { const HALF_WIDTH_SYMBOLS = [',', '.']; const BANNED_WORD_LIST = ['动态']; - const ALLOWED_WORD_LIST = ['动态条形图', '动态柱状图', '动态柱图']; + const ALLOWED_WORD_LIST = ['动态条形图', '动态柱状图', '动态柱图', '动态散点图', '动态玫瑰图', '动态饼图']; const PLACEHOLDER = '_USER_INPUT_PLACE_HOLDER'; const tempStr1 = ALLOWED_WORD_LIST.reduce((prev, cur, index) => { return prev.split(cur).join(PLACEHOLDER + '_' + index); diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts index e45c3432..87d866e0 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts @@ -6,7 +6,8 @@ import { CARTESIAN_CHART_LIST, NEED_COLOR_FIELD_CHART_LIST, NEED_SIZE_FIELD_CHART_LIST, - NEED_COLOR_AND_SIZE_CHART_LIST + NEED_COLOR_AND_SIZE_CHART_LIST, + DYNAMIC_CHART_LIST } from '../../../../constants'; const getColorKnowledge = (chartTypeList: ChartType[]) => { @@ -92,8 +93,9 @@ export const visualChannelInfoMap = { }, time: (chartTypeList: ChartType[]) => { return { - singleFieldInfo: - "This is usually a date field and can be used only in Dynamic Bar Chart. Can't be empty in Dynamic Bar Chart." + singleFieldInfo: `This is usually a date field and cannot be empty in all dynamic chart types. For example, ${DYNAMIC_CHART_LIST.join( + ',' + )}.` }; }, source: (chartTypeList: ChartType[]) => { @@ -146,7 +148,7 @@ export const chartKnowledgeDict: ChartKnowledge = { }, [ChartType.RoseChart]: { index: 7, - visualChannels: ['color', 'radius', 'angle'], + visualChannels: ['color', 'radius'], examples: [] }, [ChartType.RadarChart]: { @@ -271,6 +273,20 @@ export const chartKnowledgeDict: ChartKnowledge = { visualChannels: [], examples: [], knowledge: ['Single column combination charts can be combined with a variety of different basic chart types'] + }, + [ChartType.DynamicScatterPlotChart]: { + index: 28, + visualChannels: ['x', 'y', 'color', 'size', 'time'], + examples: [], + knowledge: [ + 'The five channels that need to be mapped in the dynamic scatter plot are: x, y, color, size, and time; the x, y, and size channels require numeric data fields; the time field must be mapped.' + ] + }, + [ChartType.DynamicRoseChart]: { + index: 29, + visualChannels: ['color', 'radius', 'time'], + examples: [], + knowledge: ['The three channels that need to be mapped in the dynamic rose chart are: color, radius, and time;'] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts index 0e41920c..c8a31a62 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts @@ -80,7 +80,15 @@ import { commonSingleColumnSeries, commonSingleColumnLegend, commonSingleColumnAxes, - commonSingleColumnLayout + commonSingleColumnLayout, + dynamicScatterPlotAxes, + dynamicScatterPlotSeries, + dynamicScatterPlotAnimation, + colorDynamicScatterPlot, + dynamicScatterPlotTooltip, + dynamicRoseAnimation, + dynamicRoseField, + dynamicRoseDisplayConf } from './transformers'; const pipelineBar = [ @@ -237,6 +245,26 @@ const pipelineSingleColumnCombinationChart = [ commonSingleColumnLayout, theme ]; +const pipelineDynamicScatterPlotChart = [ + chartType, + sequenceData, + colorDynamicScatterPlot, + dynamicScatterPlotAnimation, + dynamicScatterPlotAxes, + dynamicScatterPlotSeries, + customMark, + dynamicScatterPlotTooltip, + theme +]; +const pipelineDynamicRoseChart = [ + chartType, + sequenceData, + dynamicRoseAnimation, + dynamicRoseField, + dynamicRoseDisplayConf, + customMark, + theme +]; const pipelineMap: { [chartType: string]: any } = { [ChartType.BarChart.toUpperCase()]: pipelineBar, @@ -263,7 +291,9 @@ const pipelineMap: { [chartType: string]: any } = { [ChartType.Gauge.toUpperCase()]: pipelineGauge, [ChartType.BasicHeatMap.toUpperCase()]: pipelineBasicHeatMap, [ChartType.VennChart.toUpperCase()]: pipelineVenn, - [ChartType.SingleColumnCombinationChart.toUpperCase()]: pipelineSingleColumnCombinationChart + [ChartType.SingleColumnCombinationChart.toUpperCase()]: pipelineSingleColumnCombinationChart, + [ChartType.DynamicScatterPlotChart.toUpperCase()]: pipelineDynamicScatterPlotChart, + [ChartType.DynamicRoseChart.toUpperCase()]: pipelineDynamicRoseChart }; export const beforePipe: Transformer = ( diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index d25a98d6..6c662cab 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -26,6 +26,7 @@ import { builtinThemeMap } from '../../../../../common/builtinTheme'; import { FOLD_NAME, FOLD_VALUE, COLOR_FIELD } from '@visactor/chart-advisor'; import { CARTESIAN_CHART_LIST } from '../../../constants'; import { getCell } from '../../utils'; +import { fillColorForData, sortArray, SortOrder } from './utils'; const CARTESIAN_CHART_LIST_UPPER = CARTESIAN_CHART_LIST.map(chartType => chartType.toUpperCase() as ChartType); @@ -56,7 +57,9 @@ const chartTypeMap: { [chartName: string]: string } = { [ChartType.Gauge.toUpperCase()]: 'gauge', [ChartType.BasicHeatMap.toUpperCase()]: 'common', [ChartType.VennChart.toUpperCase()]: 'venn', - [ChartType.SingleColumnCombinationChart.toUpperCase()]: 'common' + [ChartType.SingleColumnCombinationChart.toUpperCase()]: 'common', + [ChartType.DynamicScatterPlotChart.toUpperCase()]: 'common', + [ChartType.DynamicRoseChart.toUpperCase()]: 'rose' }; export const chartType: Transformer = (context: Context) => { @@ -112,10 +115,19 @@ export const wordCloudData: Transformer = (context: export const sequenceData: Transformer = ( context: Context & { totalTime: number } ) => { - const { dataset, cells, totalTime, spec } = context; + const { dataset, cells, totalTime, spec, chartType } = context; const cell = getCell(cells); const timeField = cell.time as string; - const latestData = isValidDataset(dataset) ? dataset : []; + let latestData = isValidDataset(dataset) ? dataset : []; + if (chartType.toUpperCase() === ChartType.DynamicRoseChart.toUpperCase()) { + latestData = fillColorForData( + sortArray(latestData, [ + { field: timeField, order: SortOrder.ASC }, + { field: cell.radius, order: SortOrder.DESC } + ]), + cell.color as string + ); + } //add the time field into spec, although it has no use for chart rendering. //it can be used in getCellFromSpec. @@ -2069,3 +2081,225 @@ export const commonSingleColumnLayout: Transformer } return { spec }; }; + +export const colorDynamicScatterPlot: Transformer = (context: Context) => { + const { colors, spec, cells, fieldInfo } = context; + const cell = getCell(cells); + const colorThemes = COLOR_THEMES.default; + if (colors && colors.length > 0) { + spec.color = colors; + } else { + spec.color = { + type: 'ordinal', + domain: fieldInfo.filter(fieldInfo => { + return fieldInfo.fieldName === cell.color; + })?.[0].domain, + range: colorThemes + }; + } + + return { spec }; +}; + +export const dynamicScatterPlotAxes: Transformer = (context: Context) => { + const { spec, cells, fieldInfo } = context; + const cell = getCell(cells); + const xFieldInfo = fieldInfo.filter(fieldInfo => { + return fieldInfo.fieldName === cell.x; + })?.[0]; + const yFieldInfo = fieldInfo.filter(fieldInfo => { + return fieldInfo.fieldName === cell.y; + })?.[0]; + + spec.axes = [ + { + orient: 'left', + nice: true, + zero: false, + type: 'linear', + range: { min: yFieldInfo.domain[0], max: yFieldInfo.domain[1] }, + tick: { + tickCount: 10 + }, + grid: { + visible: true, + style: { + lineDash: [0] + } + } + }, + { + orient: 'bottom', + nice: true, + label: { visible: true }, + type: 'linear', + range: { min: xFieldInfo.domain[0], max: xFieldInfo.domain[1] }, + tick: { + tickCount: 10 + }, + grid: { + visible: true, + style: { + lineDash: [0] + } + } + } + ]; + return { spec }; +}; + +export const dynamicScatterPlotSeries: Transformer = (context: Context) => { + const { spec, cells } = context; + const cell = getCell(cells); + spec.series = [ + { + type: 'scatter', + // 通过数据中的 index 进行数据匹配 + dataKey: cell.color, + // 声明点半径大小 + sizeField: cell.size, + size: { + type: 'linear', + range: [5, 30] + }, + seriesField: cell.color, + point: { + style: {} + }, + xField: cell.x, + yField: cell.y, + animationAppear: { + duration: 300, + easing: 'linear' + }, + animationUpdate: { + duration: 300, + easing: 'linear' + } + } + ]; + return { spec }; +}; + +export const dynamicScatterPlotAnimation: Transformer = (context: Context) => { + const { spec } = context; + const duration = spec.animationUpdate.bar[0].duration; + spec.animationUpdate = { + point: [ + { + type: 'update', + options: { + excludeChannels: ['x', 'y'] + }, + duration: duration, + easing: 'cubicInOut' + }, + { + channel: ['x', 'y'], + options: { + excludeChannels: ['width'] + }, + duration: duration, + easing: 'cubicInOut' + } + ], + axis: { + duration: duration, + easing: 'cubicInOut' + } + }; + return { spec }; +}; + +export const dynamicScatterPlotTooltip: Transformer = (context: Context) => { + const { spec, cells } = context; + const cell = getCell(cells); + spec.tooltip = { + mark: { + title: { + key: cell.color, + value: (datum: any) => datum[cell.color as string] + }, + content: [ + { + key: cell.x, + value: (datum: any) => datum[cell.x] + }, + { + key: cell.y, + value: (datum: any) => datum[cell.y as string] + }, + { + key: cell.size, + value: (datum: any) => datum[cell.size] + } + ] + } + }; + return { spec }; +}; + +export const dynamicRoseAnimation: Transformer = (context: Context) => { + const { spec } = context; + const duration = 1000; + const exchangeDuration = 600; + spec.animationUpdate = { + rose: [ + { + type: 'update', + options: { excludeChannels: ['startAngle', 'endAngle'] }, + easing: 'linear', + duration + }, + { + channel: ['startAngle', 'endAngle'], + easing: 'circInOut', + duration: exchangeDuration + } + ] + }; + spec.animationEnter = { + easing: 'linear', + duration: exchangeDuration + }; + spec.animationExit = { + easing: 'linear', + duration: exchangeDuration + }; + return { spec }; +}; +export const dynamicRoseField: Transformer = (context: Context) => { + const { spec, cells } = context; + const cell = getCell(cells); + spec.categoryField = cell.color; + spec.valueField = cell.radius; + spec.seriesField = cell.color; + return { spec }; +}; +export const dynamicRoseDisplayConf: Transformer = (context: Context) => { + const { spec } = context; + spec.outerRadius = 1; + spec.innerRadius = 0.2; + spec.rose = { + style: { + fill: (datum: any) => datum.fill + } + }; + spec.axes = [ + { + orient: 'angle', + label: { + visible: false + } + }, + { + orient: 'radius' + } + ]; + spec.label = { + visible: true, + position: 'outside' + }; + spec.startAngle = -90; + return { spec }; +}; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/utils.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/utils.ts index e69de29b..0f534200 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/utils.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/utils.ts @@ -0,0 +1,79 @@ +import chroma from 'chroma-js'; +import type { VMindDataset } from '../../../../../common/typings'; +import { COLOR_THEMES } from './constants'; + +export enum InterpolationType { + Linear = 'linear', + NonLinear = 'nonlinear' +} + +export type InterpolationColor = { + color: string; + position?: string; +}; + +export type Interpolation = { + type: InterpolationType; + colors: InterpolationColor[]; + num: number; +}; + +export const getColorInterpolation = (interpolation: Interpolation) => { + const colorList = interpolation.colors.map(interpolationColor => interpolationColor.color); + if (interpolation.type === InterpolationType.Linear) { + return chroma.scale(colorList).colors(interpolation.num); + } + const positionList = interpolation.colors.map(interpolationColor => interpolationColor.position); + return chroma.scale(colorList).domain(positionList).colors(interpolation.num); +}; + +export const fillColorForData = (datsets: VMindDataset, categoryField: string) => { + const categoryList = new Array( + ...new Set( + datsets.map(dataset => { + return dataset[categoryField]; + }) + ) + ); + const colorList = getColorInterpolation({ + type: InterpolationType.Linear, + colors: COLOR_THEMES.default.map(color => { + return { color: color } as InterpolationColor; + }), + num: categoryList.length + }); + const category2colorMap = categoryList.reduce((acc, key, index) => { + acc[key as string] = colorList[index]; + return acc; + }, {} as { [key: string]: any }); + return datsets.map(dataset => { + dataset.fill = category2colorMap[dataset[categoryField]]; + return dataset; + }); +}; + +export enum SortOrder { + ASC = 'asc', + DESC = 'desc' +} + +export interface SortField { + field: string; + order: SortOrder; +} + +export const sortArray = (array: T[], sortFields: SortField[]): T[] => { + return array.sort((a, b) => { + for (const { field, order } of sortFields) { + const aValue = a[field as keyof T]; + const bValue = b[field as keyof T]; + + if (aValue < bValue) { + return order === 'asc' ? -1 : 1; + } else if (aValue > bValue) { + return order === 'asc' ? 1 : -1; + } + } + return 0; + }); +}; diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index d2fb2b52..c5f5bfd1 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -84,7 +84,9 @@ export enum ChartType { Gauge = 'Gauge Chart', BasicHeatMap = 'Basic Heat Map', VennChart = 'Venn Chart', - SingleColumnCombinationChart = 'Single Column Combination Chart' + SingleColumnCombinationChart = 'Single Column Combination Chart', + DynamicScatterPlotChart = 'Dynamic Scatter Plot Chart', + DynamicRoseChart = 'Dynamic Rose Chart' } export enum CombinationChartType { From 962aac7135dd9821f1ba96f9ffe8c6ddca9de001 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 13 Sep 2024 17:31:25 +0800 Subject: [PATCH 020/128] feat: finish v2 framework, add data extraction taskand prompt --- packages/vmind/src/atom/base.ts | 188 ++++++++++++++++++ .../vmind/src/atom/chartGenerator/index.ts | 48 +++++ packages/vmind/src/atom/dataClean/index.ts | 72 +++++++ .../vmind/src/atom/dataExtraction/index.ts | 87 ++++++++ .../vmind/src/atom/dataExtraction/prompt.ts | 153 ++++++++++++++ packages/vmind/src/atom/dataQuery/index.ts | 80 ++++++++ packages/vmind/src/atom/dataQuery/prompt.ts | 98 +++++++++ packages/vmind/src/atom/dataQuery/utils.ts | 23 +++ packages/vmind/src/atom/index.ts | 5 + packages/vmind/src/atom/type.ts | 18 ++ packages/vmind/src/core/llm.ts | 96 +++++++++ packages/vmind/src/index.ts | 6 +- packages/vmind/src/schedule/index.ts | 137 +++++++++++++ packages/vmind/src/types/atom.ts | 116 +++++++++++ packages/vmind/src/types/index.ts | 3 + packages/vmind/src/types/llm.ts | 59 ++++++ packages/vmind/src/types/schedule.ts | 28 +++ packages/vmind/src/utils/json.ts | 30 +++ packages/vmind/src/utils/llm.ts | 0 packages/vmind/src/utils/text.ts | 9 + 20 files changed, 1255 insertions(+), 1 deletion(-) create mode 100644 packages/vmind/src/atom/base.ts create mode 100644 packages/vmind/src/atom/chartGenerator/index.ts create mode 100644 packages/vmind/src/atom/dataClean/index.ts create mode 100644 packages/vmind/src/atom/dataExtraction/index.ts create mode 100644 packages/vmind/src/atom/dataExtraction/prompt.ts create mode 100644 packages/vmind/src/atom/dataQuery/index.ts create mode 100644 packages/vmind/src/atom/dataQuery/prompt.ts create mode 100644 packages/vmind/src/atom/dataQuery/utils.ts create mode 100644 packages/vmind/src/atom/index.ts create mode 100644 packages/vmind/src/atom/type.ts create mode 100644 packages/vmind/src/core/llm.ts create mode 100644 packages/vmind/src/schedule/index.ts create mode 100644 packages/vmind/src/types/atom.ts create mode 100644 packages/vmind/src/types/index.ts create mode 100644 packages/vmind/src/types/llm.ts create mode 100644 packages/vmind/src/types/schedule.ts create mode 100644 packages/vmind/src/utils/json.ts create mode 100644 packages/vmind/src/utils/llm.ts create mode 100644 packages/vmind/src/utils/text.ts diff --git a/packages/vmind/src/atom/base.ts b/packages/vmind/src/atom/base.ts new file mode 100644 index 00000000..836a190b --- /dev/null +++ b/packages/vmind/src/atom/base.ts @@ -0,0 +1,188 @@ +/** + * Base Class of Atom Module + */ + +import { merge } from '@visactor/vutils'; +import type { BaseContext } from '../types/atom'; +import { AtomName } from '../types/atom'; +import type { LLMMessage, LLMResponse } from '../types/llm'; +import type { BaseOptions } from './type'; + +export class BaseAtom { + /** name */ + name: AtomName = AtomName.BASE; + /** current context to self-update */ + protected context: Ctx; + /** llm response and user's query */ + protected responses: LLMMessage[]; + /** base Options */ + options: O; + /** is based on LLM */ + isLLMAtom: boolean; + /** historys of context update */ + history: { + /** map of history context */ + map: Map; + /** id of history Step */ + idList: number[]; + /** current id */ + id: number; + }; + + constructor(context: Ctx, options: O) { + this.options = merge({}, this.buildDefaultOptions(), options); + this.responses = []; + this.history = { + map: new Map(), + idList: [], + id: null + }; + this.setNewContext(this.buildDefaultContext(context)); + if (!this.options.llm && this.isLLMAtom) { + console.error(`Does\'t support LLM Mange in ${this.name} Atom which need LLM`); + } + } + + protected setNewContext(context: Ctx) { + this.context = context; + const newHistoryId = (this.history.id || 0) + 1; + this.history.map.set(newHistoryId, context); + this.history.idList.push(newHistoryId); + this.history.id = newHistoryId; + } + + undo(id?: string) { + /** todo */ + } + + redo(id?: string) { + /** todo */ + } + + buildDefaultContext(context: Ctx) { + return this.context; + } + + buildDefaultOptions(): O { + return {} as O; + } + + updateContext(context: Ctx) { + if (context) { + this.context = merge({}, this.context, context); + } + return this.context; + } + + updateOptions(options: O) { + this.options = merge({}, this.options, options); + } + + reset(context?: Ctx) { + this.updateContext(context); + this.responses = []; + this.history.map.clear(); + this.history.idList = []; + this.history.id = null; + } + + getContext() { + return this.context; + } + + /** check should run or not when context in schdule changed */ + shouldRunByContextUpdate(context: Ctx) { + return false; + } + + /** + * run atom function to update context + * @param userInput + * @param userInput.context new context to update + * @param userInput.query user's query to adjust context + * @returns new context after execute atom function + */ + async run(userInput?: { context?: Ctx; query?: string }) { + const { context, query } = userInput || {}; + this.updateContext(context); + if (this.isLLMAtom && query) { + return await this.runWithChat(query); + } + if (this.isLLMAtom) { + const messages = this.getLLMMessages(); + const data = await this.options.llm.run(this.name, messages); + const resJson = this.options.llm.parseJson(data); + if (resJson.err) { + return this.context; + } + this.recordLLMResponse(data); + this._runWithOutLLM(); + this.setNewContext(this.parseLLMContent(resJson)); + } else { + this._runWithOutLLM(); + } + return this.context; + } + + /** + * after run function, user can adjust context result by multi-turn dialogue + * @param query user's new query + * @returns new context after execute + */ + protected async runWithChat(query: string) { + const messages = this.getLLMMessages(query); + const data = await this.options.llm.run(this.name, messages); + const resJson = this.options.llm.parseJson(data); + if (!resJson.err) { + this.recordLLMResponse(data, query); + this.setNewContext(this.parseLLMContent(resJson)); + } + return this.context; + } + + protected _runWithOutLLM(): Ctx { + return this.context; + } + + protected getHistoryLLMMessages(query?: string): LLMMessage[] { + return query + ? [ + ...this.responses, + { + role: 'user' as const, + content: query + } + ] + : this.responses; + } + + protected getLLMMessages(query?: string): LLMMessage[] { + return []; + } + + protected parseLLMContent(resJson: any) { + return { ...this.context }; + } + + /** record LLM response and user's query to multi-turn dialog */ + protected recordLLMResponse(data: LLMResponse, query?: string) { + const newResponse = data.choices[0].message; + const assistantMsg: LLMMessage = { + role: 'assistant', + content: newResponse + }; + if (!query) { + // record with out uesr's query, it's new to record + this.responses = [assistantMsg]; + } else { + // record a round of conversation + this.responses.push( + { + role: 'user', + content: newResponse + }, + assistantMsg + ); + } + } +} diff --git a/packages/vmind/src/atom/chartGenerator/index.ts b/packages/vmind/src/atom/chartGenerator/index.ts new file mode 100644 index 00000000..c802c702 --- /dev/null +++ b/packages/vmind/src/atom/chartGenerator/index.ts @@ -0,0 +1,48 @@ +import type { ChartGeneratorCtx } from '../../types/atom'; +import { AtomName } from '../../types/atom'; +import type { BaseOptions } from '../type'; +import { BaseAtom } from '../base'; +import { merge } from '@visactor/vutils'; +import type { LLMMessage, LLMResponse } from '../../types/llm'; + +export class ChartGeneratorAtom extends BaseAtom { + name = AtomName.DATA_QUERY; + + isLLMAtom: true; + + constructor(context: ChartGeneratorCtx, option: BaseOptions) { + super(context, option); + } + + buildDefaultContext(context: ChartGeneratorCtx): ChartGeneratorCtx { + return merge( + {}, + { + dataTable: [], + fieldInfo: [], + dataTableSummary: '' + }, + context + ); + } + + getLLMMessages(): LLMMessage[] { + /** todo */ + return []; + } + + parseLLMContent(data: LLMResponse) { + const resJson = this.options.llm.parseJson(data); + if (resJson.err) { + return this.context; + } + const { dataTable, fieldInfo, cells, chartType } = resJson; + return { + ...this.context, + chartType, + cells, + fieldInfo: this.context.fieldInfo ?? fieldInfo, + dataTable + } as ChartGeneratorCtx; + } +} diff --git a/packages/vmind/src/atom/dataClean/index.ts b/packages/vmind/src/atom/dataClean/index.ts new file mode 100644 index 00000000..73be1e5b --- /dev/null +++ b/packages/vmind/src/atom/dataClean/index.ts @@ -0,0 +1,72 @@ +import type { DataCleanCtx } from '../../types/atom'; +import { AtomName, type DataExtractionCtx } from '../../types/atom'; +import type { DataCleanOptions } from '../type'; +import { BaseAtom } from '../base'; +import { merge, pick } from '@visactor/vutils'; + +export class DataCleanAtom extends BaseAtom { + name = AtomName.DATA_CLEAN; + + constructor(context: DataExtractionCtx, option: DataCleanOptions) { + super(context, option); + } + + buildDefaultContext(context: DataExtractionCtx): DataExtractionCtx { + return merge( + {}, + { + dataTable: [], + fieldInfo: [] + }, + context + ); + } + + buildDefaultOptions(): DataCleanOptions { + return { + filterSameValueColumn: true, + needNumericalFields: true + }; + } + + shouldRunByContextUpdate(context: DataCleanCtx): boolean { + return context.dataTable !== this.context.dataTable; + } + + _runWithOutLLM(): DataCleanCtx { + const { filterSameValueColumn, needNumericalFields } = this.options; + // @attention the value of context maybe not new after some clean task + const { fieldInfo = [], dataTable = [] } = this.context || {}; + let newContext = { ...this.context }; + if (filterSameValueColumn && dataTable.length > 1 && fieldInfo.length) { + const cleanFieldKey: string[] = []; + fieldInfo.forEach(info => { + if (info.fieldType === 'numerical') { + return; + } + let shouldFilter = true; + const prev = dataTable[0][info.fieldName]; + for (let i = 1; i < dataTable.length; i++) { + if (dataTable[i][info.fieldName] !== prev) { + shouldFilter = false; + break; + } + } + shouldFilter && cleanFieldKey.push(info.fieldName); + }); + if (cleanFieldKey.length) { + newContext.fieldInfo = fieldInfo.filter(info => !cleanFieldKey.includes(info.fieldName)); + const fieldNameList = newContext.fieldInfo.map(info => info.fieldName); + newContext.dataTable = dataTable.map(dataItem => pick(dataItem, fieldNameList)); + } + } + if (needNumericalFields && newContext.fieldInfo.findIndex(info => info?.fieldType === 'numerical') === -1) { + newContext = { + dataTable: [], + fieldInfo: [] + }; + } + this.setNewContext(newContext); + return this.context; + } +} diff --git a/packages/vmind/src/atom/dataExtraction/index.ts b/packages/vmind/src/atom/dataExtraction/index.ts new file mode 100644 index 00000000..fe8c1f22 --- /dev/null +++ b/packages/vmind/src/atom/dataExtraction/index.ts @@ -0,0 +1,87 @@ +import { AtomName, type DataExtractionCtx } from '../../types/atom'; +import type { BaseOptions } from '../type'; +import { BaseAtom } from '../base'; +import { merge, pick } from '@visactor/vutils'; +import type { LLMMessage } from '../../types/llm'; +import { getBasePrompt, getFieldInfoPrompt } from './prompt'; +import { getLanguageOfText } from '../../utils/text'; + +export class DataExtractionAtom extends BaseAtom { + name = AtomName.DATA_EXTRACT; + + isLLMAtom = true; + + constructor(context: DataExtractionCtx, option: BaseOptions) { + super(context, option); + } + + buildDefaultContext(context: DataExtractionCtx): DataExtractionCtx { + return merge( + {}, + { + dataTable: [], + fieldInfo: [] + }, + context + ); + } + + shouldRunByContextUpdate(context: DataExtractionCtx): boolean { + return context.text !== this.context.text || context.fieldInfo !== this.context.fieldInfo; + } + + getLLMMessages(query?: string): LLMMessage[] { + const { fieldInfo, text } = this.context; + const { showThoughts } = this.options; + const addtionContent = this.getHistoryLLMMessages(query); + const language = getLanguageOfText(text); + if (!fieldInfo || !fieldInfo?.length) { + return [ + { + role: 'system', + content: getBasePrompt(language, showThoughts) + }, + { + role: 'user', + content: `Extracted text is bellow: ${text}` + }, + ...addtionContent + ]; + } + const fieldInfoContent = fieldInfo.map(info => + pick(info, ['fieldName', 'dataExample', 'fieldType', 'description']) + ); + const fieldInfoString = JSON.stringify(fieldInfoContent); + const userContent = `User's fieldInfo is bellow: +\`\`\` TypeScript +${fieldInfoString} +\`\`\` +Extracted text is bellow: +text: ${text} +`; + return [ + { + role: 'system', + content: getFieldInfoPrompt(language, showThoughts) + }, + { + role: 'user', + content: userContent + }, + ...addtionContent + ]; + } + + parseLLMContent(resJson: any) { + const { dataTable, fieldInfo, isDataExtraction } = resJson; + if (!isDataExtraction) { + console.error("It's not a data extraction task"); + return this.context; + } + return { + ...this.context, + fieldInfo: fieldInfo ?? this.context?.fieldInfo ?? [], + dataTable + } as DataExtractionCtx; + } +} diff --git a/packages/vmind/src/atom/dataExtraction/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt.ts new file mode 100644 index 00000000..5ebe5dfe --- /dev/null +++ b/packages/vmind/src/atom/dataExtraction/prompt.ts @@ -0,0 +1,153 @@ +/* eslint-disable max-len */ +const dataTableExplanation = `# Data Table Explanation +1. ALWAYS generate flatten data table rather than unflatten data table + +# Flatten Data Table Example +\`\`\` +dataTable: [{ date: "Monday", class: "class No.1", score: 20 },{ date: "Monday", class: "class No.2", score: 30 },{ date: "Tuesday", class: "class No.1", score: 25 },{ date: "Tuesday", class: "class No.2", score: 28 }] +\`\`\` + +# Unflatten Data Table Example +\`\`\` +dataTable: [{date: "Monday", class No.1: 20, class No.2: 30},{date: "Tuesday", class No.1: 25, class No.2: 28}] +\`\`\``; +const baseExamples = `# Examples1: + +text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 + +Response: +\`\`\` +{"fieldInfo:":[{"fieldName":"公司","description":"公司名称","fieldType":"string",},{"fieldName":"月份","description":"具体月份","fieldType":"string",},{"fieldName":"利润调整","description":"是否经过利润调整","fieldType":"string",},{"fieldName":"利润额","description":"利润总额","fieldType":"numerical",}],"dataTable":[{"公司":"阿里","月份":"5月","利润调整":"调整前","利润额":100000000000,},{"公司":"阿里","月份":"5月","利润调整":"调整后","利润额":10000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整前","利润额":80000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整后","利润额":12000000000,},]} + +# Examples2: + +text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. + +Response: +\`\`\` +{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string",},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"numerical"}],"dataTable":[{"name":"John Smith","ranking":0.9,},{"name":"Jane Doe","ranking":0.75}]} +\`\`\``; + +const getFieldTypeExplanation = (language: 'chinese' | 'english') => { + return `field type explanation is below: 'ratio' means ratio value or percent value, such as ${ + language === 'english' ? 'YoY or MoM' : '同比、环比、增长率、占比等' + }, 'count' means count data`; +}; +export const getBasePrompt = ( + language: 'chinese' | 'english', + showThoughs: boolean = true +) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, dates, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. + +# Field Information Explanation +1. ALWAYS generate a field information, which represents the specific information of each column field in the data table. +2. ALWAYS generate a field description +3. ALWAYS generate a field type, chosen from 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count';${getFieldTypeExplanation( + language +)} +4. ALWASY display time fields using the same time format.Different date fields can have different formats. +${dataTableExplanation} + +You should think step-by-step as follow: +# Steps +0. using language answer: ${language} +1. Determine whether the current task is related to data extraction. +2. If not, return isDataExtraction is false in json mode; If yes, continue follow Steps +3. Read the entire text and find the most important and most frequently occurring field with a numerical field type. +4. Read all text again and generate field information associated with the numerical field.The newly generated fields are all simple. +5. Finally, read all text and extract all corresponding data table based on the field information. +6. Adjust the data to ensure consistency within the same field especially time field. +7. Convert percentage values to actual values. +8. Assume the data is incomplete, then reconsider and execute the task again. + +Response in the following format: +\`\`\` +{ +isDataExtraction: boolean; // current task is data extraction or not +${showThoughs ? 'thoughts: string, // your thought process' : ''} +fieldInfo: { +fieldName: string; //name of the field. +description?: string; //description of the field. +fieldType?: 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count'; // type of field +}[], +dataTable: Record\[]; // Extracted data set, key of dataTable is fieldName in fieldInfo +} +\`\`\` +${baseExamples} +---------------------------------- + +You only need to return the JSON in your response directly to the user. +Finish your tasks in one-step. + +# Constraints: +1. Strictly define the type of return format, use JSON format to reply, do not include any extra content. +2. The numbers in the dataset do not carry any units. +3. Keep the ratio value unchanged, such as '95%' --> '95' +4. Only use numbers that appear in the text. +5. If you do not know the value of an field, return null for the field's value. +6. There is exactly one field of numerical type in the data table.`; + +export const getFieldInfoPrompt = ( + language: 'chinese' | 'english', + showThoughs: boolean = true +) => `You are an expert extraction algorithm and are highly sensitive to comparative data, trend data, and similar information.Only extract relevant information from the text. Your goal is to extract structured information from the user's input that matches the form described below. When extracting information please make sure it matches the type information exactly. +The definition of the field information is as follows. +\`\`\` +fieldInfo: { +fieldName: string; //name of the field. +description?: string; //description of the field. +fieldType?: 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count'; // type of field;${getFieldTypeExplanation( + language +)} +dataExample?: (string | number)[] // data example of this field +}[] +\`\`\` + +You should think step-by-step as follow: +# Steps +0. using language answer: ${language} +1. Determine whether the current task is related to data extraction. +2. If not, return isDataExtraction is false in json mode; If yes, continue follow Steps +3. Read all text and extract all corresponding data table based on the field information. +4. Adjust the data to ensure consistency within the same field especially time field. +5. Convert percentage values to actual values. +6. Assume the data is incomplete, then reconsider and execute the task again. + +# Respones +Response in the following format: +\`\`\` +{ +isDataExtraction: boolean; // current task is data extraction or not +${showThoughs ? 'thoughts: string, // your thought process' : ''} +dataTable: Record\[]; // Extracted data set, key of dataTable is fieldName in user's fieldInfo +} +\`\`\` + +# Examples1: +text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 +\`\`\` +{"fieldInfo:":[{"fieldName":"公司","description":"公司名称","fieldType":"string",},{"fieldName":"月份","description":"具体月份","fieldType":"string",},{"fieldName":"利润调整","description":"是否经过利润调整","fieldType":"string",},{"fieldName":"利润额","description":"利润总额","fieldType":"numerical",}]} +\`\`\` +Response: +\`\`\` +{"dataTable":[{"公司":"阿里","月份":"5月","利润调整":"调整前","利润额":100000000000,},{"公司":"阿里","月份":"5月","利润调整":"调整后","利润额":10000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整前","利润额":80000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整后","利润额":12000000000}]} +\`\`\` +# Examples2: + +text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. +\`\`\` +{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string","dataExample":["Roy","Stepen Curry","张三","李四"]},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"numerical","dataExample": [0.1, 0.8]]}}]} +\`\`\` +Response: +\`\`\` +{"dataTable":[{"name":"John Smith","ranking":0.9,},{"name":"Jane Doe","ranking":0.75}]} +---------------------------------- + +You only need to return the JSON in your response directly to the user. +Finish your tasks in one-step. +# Constraints: +1. Strictly define the type of return format, use JSON format to reply, do not include any extra content. +2. The numbers in the dataset do not carry any units. +3. Only use numbers that appear in the text. +4. If you do not know the value of an field, return null for the field's value. +5. Do not add any attributes that do not appear in the user's fieldInfo. +`; diff --git a/packages/vmind/src/atom/dataQuery/index.ts b/packages/vmind/src/atom/dataQuery/index.ts new file mode 100644 index 00000000..9c6f398c --- /dev/null +++ b/packages/vmind/src/atom/dataQuery/index.ts @@ -0,0 +1,80 @@ +import type { DataQueryCtx } from '../../types/atom'; +import { AtomName } from '../../types/atom'; +import type { BaseOptions, DataQueryOptions } from '../type'; +import { BaseAtom } from '../base'; +import { merge, pick } from '@visactor/vutils'; +import type { LLMMessage, LLMResponse } from '../../types/llm'; +import { getQueryDatasetPrompt } from './prompt'; +import { parseSQLResponse } from './utils'; + +export class DataQueryAtom extends BaseAtom { + name = AtomName.DATA_QUERY; + + isLLMAtom: true; + + constructor(context: DataQueryCtx, option: BaseOptions) { + super(context, option); + } + + buildDefaultContext(context: DataQueryCtx): DataQueryCtx { + return merge( + {}, + { + dataTable: [], + fieldInfo: [], + llmFieldInfo: [], + dataTableSummary: '' + }, + context + ); + } + + buildDefaultOptions(): DataQueryOptions { + return { + useSQL: true + }; + } + + getLLMMessages(query?: string): LLMMessage[] { + const { fieldInfo } = this.context; + const { showThoughts } = this.options; + const addtionContent = this.getHistoryLLMMessages(query); + if (this.options.useSQL) { + const fieldInfoContent = fieldInfo.map(info => pick(info, ['fieldName', 'fieldType', 'role'])); + return [ + { + role: 'system', + content: getQueryDatasetPrompt(showThoughts) + }, + { + role: 'user', + content: `User's Command: ${query}\nColumn Information: ${JSON.stringify(fieldInfoContent)}` + }, + ...addtionContent + ]; + } + /** @todo */ + return []; + } + + parseLLMContent(data: LLMResponse) { + const resJson = this.options.llm.parseJson(data); + if (resJson.err) { + return this.context; + } + const { sql, fieldInfo: responseFiledInfo } = resJson; + if (!sql || !responseFiledInfo) { + //try to parse the response with another format + const content = data.choices[0].message.content; + return { + ...this.context, + ...parseSQLResponse(content) + }; + } + return { ...this.context, sql, llmFieldInfo: responseFiledInfo }; + } + + protected _runWithOutLLM(): DataQueryCtx { + return this.context; + } +} diff --git a/packages/vmind/src/atom/dataQuery/prompt.ts b/packages/vmind/src/atom/dataQuery/prompt.ts new file mode 100644 index 00000000..25820ccb --- /dev/null +++ b/packages/vmind/src/atom/dataQuery/prompt.ts @@ -0,0 +1,98 @@ +/* eslint-disable max-len */ +export const VMIND_DATA_SOURCE = 'VMind_data_source'; + +export const getQueryDatasetPrompt = ( + showThoughts: boolean +) => `You are an expert in data analysis. Here is a raw dataset named ${VMIND_DATA_SOURCE}. User will tell you his command and column information of ${VMIND_DATA_SOURCE}. Your task is to generate a sql and fieldInfo according to Instruction. Response one JSON object only. + +# Instruction +- Supported sql keywords: ["SELECT", "FROM", "WHERE", "GROUP BY", "HAVING", "ORDER BY", "LIMIT", "DISTINCT"]. Supported aggregation methods: ["MAX()", "MIN()", "SUM()", "COUNT()", "AVG()"]. +- Generate a sql query like this: "SELECT \`columnA\`, SUM(\`columnB\`) as \`sum_b\` FROM ${VMIND_DATA_SOURCE} WHERE \`columnA\` = value1 GROUP BY \`columnA\` HAVING \`sum_b\`>0 ORDER BY \`sum_b\` LIMIT 10". +- Don't use unsupported keywords such as WITHIN, FIELD, RANK() OVER, OVER. Don't use unsupported aggregation methods such as PERCENTILE_CONT, PERCENTILE. Don't use unsupported operators. We will execute your sql using alasql. Unsupported keywords, methods and operators will cause system crash. If current keywords and methods can't meet your needs, just simply select the column without any process. +- Don't use aliases in HAVING. +- Make your sql as simple as possible. + +You need to follow the steps below. + +# Steps +1. Extract the part related to the data from the user's instruction. Ignore other parts that is not related to the data. +2. Select useful dimension and measure columns from ${VMIND_DATA_SOURCE}. Don't miss some important columns such as dimensions related to date or time. You can only use columns in Column Information and do not assume non-existent columns. If the existing columns can't meet user's command, just select the most related columns in Column Information. +3. Use the original dimension columns without any process. Aggregate the measure columns using aggregation methods no matter what chart type the user has specified. Don't use unsupported methods. If current keywords and methods can't meet your needs, just simply select the column without any process. +4. Group the data using dimension columns. +5. You can also use WHERE, HAVING, ORDER BY, LIMIT in your sql if necessary. Use the supported operators to finish the WHERE and HAVING. You can only use binary expression such as columnA = value1, sum_b > 0. You can only use dimension values appearing in the domain of dimension columns in your expression. + +Let's think step by step. + +User will parse the content of your response with JSON.parse() directly without further process. Response one JSON object without any additional words. Your JSON object must contain sql and fieldInfo. + +Response in the following format: +\`\`\` +{ + ${showThoughts ? 'thoughts: string //your thoughts' : ''} + sql: string; //your sql. Note that it's a string in a JSON object so it must be in one line without any \\n. + fieldInfo: { + fieldName: string; //name of the field. + description?: string; //description of the field. If it is an aggregated field, please describe how it is generated in detail. + }[]; //array of the information about the fields in your sql. Describing its aggregation method and other information of the fields. +} +\`\`\` + +#Examples: + +User's Command: Show me the change of the GDP rankings of each country. +Column Information: [{"fieldName":"country","fieldType":"string","role":"dimension"},{"fieldName":"continent","fieldType":"string","role":"dimension"},{"fieldName":"GDP","fieldType":"float","role":"measure"},{"fieldName":"year","fieldType":"int","role":"measure"}] + +Response: +\`\`\` +{ + ${showThoughts ? '"thoughts": string //your thoughts' : ''} + "sql": "SELECT \`country\`, \`year\`, SUM(\`GDP\`) AS \`total_GDP\` FROM ${VMIND_DATA_SOURCE} GROUP BY \`country\`, \`year\` ORDER BY \`year\`, \`total_GDP\` DESC", + "fieldInfo": [ + { + "fieldName": "country", + "description": "The name of the country." + }, + { + "fieldName": "year", + "description": "The year of the GDP data." + }, + { + "fieldName": "total_GDP", + "description": "An aggregated field representing the total GDP of each country in each year. It is generated by summing up the GDP values for each country in each year." + } + ] +} +\`\`\` +---------------------------------- + +User's Command: 请使用[柱状图]展示[2022年GDP排名前五的中国城市及其2022年的GDP]. +Column Information: [{"fieldName":"城市","fieldType":"string","role":"dimension"},{"fieldName":"2022年GDP（亿元）","fieldType":"int","role":"measure"}] + +Response: +\`\`\` +{ + ${showThoughts ? '"thoughts": string //your thoughts' : ''} + "sql": "SELECT 城市, SUM(\`2022年GDP（亿元）\`) as \`sum_2022_GDP\` FROM ${VMIND_DATA_SOURCE} ORDER BY \`sum_2022_GDP\` DESC LIMIT 5", + "fieldInfo": [ + { + "fieldName": "城市", + "description": "The name of the city." + }, + { + "fieldName": "sum_2022_GDP", + "description": "The GDP value of the city in 2022." + } + ] +} +\`\`\` +---------------------------------- + +You only need to return the JSON in your response directly to the user. +Finish your tasks in one-step. + +# Constraints: +1. Write your sql statement in one line without any \\n. Your sql must be executable by alasql. +2. Please don't change or translate the field names in your sql statement. Don't miss the GROUP BY in your sql. +3. Wrap all the columns with \`\` in your sql. +4. Response the JSON object directly without any other contents. Make sure it can be directly parsed by JSON.parse() in JavaScript. +`; diff --git a/packages/vmind/src/atom/dataQuery/utils.ts b/packages/vmind/src/atom/dataQuery/utils.ts new file mode 100644 index 00000000..34e9e477 --- /dev/null +++ b/packages/vmind/src/atom/dataQuery/utils.ts @@ -0,0 +1,23 @@ +import JSON5 from 'json5'; +import { isArray } from '@visactor/vutils'; + +export const parseSQLResponse = (response: string) => { + const sql = response.match(/sql:\n?```(.*?)```/s)[1]; + const fieldInfoStr = response.match(/fieldInfo:\n?```(.*?)```/s)[1]; + let fieldInfo = []; + try { + const tempFieldInfo = JSON5.parse(fieldInfoStr); + if (isArray(tempFieldInfo)) { + fieldInfo = tempFieldInfo; + } else { + fieldInfo = tempFieldInfo.fieldInfo; + } + } catch (e) { + //fieldInfoStr is not a json string; try to wrap it with [] + fieldInfo = JSON5.parse(`[${fieldInfoStr}]`); + } + return { + sql, + llmFieldInfo: fieldInfo + }; +}; diff --git a/packages/vmind/src/atom/index.ts b/packages/vmind/src/atom/index.ts new file mode 100644 index 00000000..959c4b17 --- /dev/null +++ b/packages/vmind/src/atom/index.ts @@ -0,0 +1,5 @@ +export { DataCleanAtom } from './dataClean'; +export { DataExtractionAtom } from './dataExtraction'; +export { DataQueryAtom } from './dataQuery'; +export { ChartGeneratorAtom } from './chartGenerator'; +export { BaseAtom } from './base'; diff --git a/packages/vmind/src/atom/type.ts b/packages/vmind/src/atom/type.ts new file mode 100644 index 00000000..9232a34b --- /dev/null +++ b/packages/vmind/src/atom/type.ts @@ -0,0 +1,18 @@ +import type { LLMManage } from '../core/llm'; + +export interface BaseOptions { + /** llm manage instance */ + llm?: LLMManage; + /** show llm thoughs or not */ + showThoughts?: boolean; +} + +export interface DataCleanOptions extends BaseOptions { + needNumericalFields?: boolean; + filterSameValueColumn?: boolean; +} + +export interface DataQueryOptions extends BaseOptions { + /** use SQL to execute data query or not */ + useSQL?: boolean; +} diff --git a/packages/vmind/src/core/llm.ts b/packages/vmind/src/core/llm.ts new file mode 100644 index 00000000..5bf4c04f --- /dev/null +++ b/packages/vmind/src/core/llm.ts @@ -0,0 +1,96 @@ +import { merge } from '@visactor/vutils'; +import axios from 'axios'; +import type { BaseContext } from '../types/atom'; +import type { AtomName } from '../types/atom'; +import type { LLMResponse } from '../types/llm'; +import { Model, type ILLMOptions, type LLMMessage } from '../types/llm'; +import { matchJSONStr, parseLLMJson } from '../utils/json'; + +/** LLM Manager Class */ +export class LLMManage { + options: ILLMOptions; + + /** history chatId to support multi-turn conversation */ + historys: Record; + + constructor(options: ILLMOptions) { + this.options = merge({}, this.getDefaultOptions(), options); + this.historys = {}; + } + + getDefaultOptions(): ILLMOptions { + return { + url: 'https://api.openai.com/v1/chat/completions', + headers: { 'Content-Type': 'application/json' }, + method: 'POST', + model: Model.DOUBAO_PRO, + maxTokens: 1024, + temperature: 0 + }; + } + + updateOptions(options: ILLMOptions) { + this.options = merge({}, this.options, options); + } + + async run(name: AtomName, messages: LLMMessage[]) { + const { url, headers, method, maxTokens, temperature, model } = this.options; + if (!this.historys[name]) { + this.historys[name] = []; + } + try { + const res: LLMResponse = await axios(url, { + method, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + headers: headers as any, + data: { + model, + messages, + max_tokens: maxTokens, + temperature, + stream: false + // Only models after gpt-3.5-turbo-1106 support this parameter. + // response_format: + // { "type": "json_object" } + } + }).then(response => response.data); + + const { logId, id } = res; + this.historys[name].push({ + logId, + id + }); + if (res.error) { + console.error(res.error); + return {}; + } + return res; + } catch (err: any) { + console.error(err); + return err.response.data; + } + } + + parseJson(res: LLMResponse) { + const { choices, error } = res; + if (error) { + return { + error: true, + message: error + }; + } + try { + const content = choices[0].message.content; + const jsonStr = matchJSONStr(content); + + const resJson = parseLLMJson(jsonStr, '```'); + return resJson; + } catch (err: any) { + console.error(err); + return { + error: true, + message: err.message + }; + } + } +} diff --git a/packages/vmind/src/index.ts b/packages/vmind/src/index.ts index 70fe889b..6440cc8c 100644 --- a/packages/vmind/src/index.ts +++ b/packages/vmind/src/index.ts @@ -1,6 +1,10 @@ import VMind from './core/VMind'; -export { Model, ChartType, InputType } from './common/typings'; +export { ChartType, InputType } from './common/typings'; +export { Model } from './types/llm'; export * from './common/colorScheme'; export * from './common/builtinTheme'; export default VMind; +export { Schedule } from './schedule'; +export { LLMManage } from './core/llm'; +export * from './types'; diff --git a/packages/vmind/src/schedule/index.ts b/packages/vmind/src/schedule/index.ts new file mode 100644 index 00000000..b11a912d --- /dev/null +++ b/packages/vmind/src/schedule/index.ts @@ -0,0 +1,137 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { merge } from '@visactor/vutils'; +import type { BaseContext } from '../types/atom'; +import { AtomName } from '../types/atom'; +import { BaseAtom } from '../atom/base'; +import { DataQueryAtom, DataExtractionAtom, ChartGeneratorAtom } from '../atom'; +import type { CombineAll, MapAtomTypes, TaskMapping } from '../types/schedule'; +import { DataCleanAtom } from '../atom/dataClean'; +import type { BaseOptions, DataCleanOptions } from '../atom/type'; + +export interface ScheduleOptions { + [AtomName.BASE]?: BaseOptions; + [AtomName.DATA_EXTRACT]?: BaseOptions; + [AtomName.DATA_CLEAN]?: DataCleanOptions; + [AtomName.DATA_QUERY]?: BaseOptions; + [AtomName.CHART_GENERATE]?: BaseOptions; +} + +export class Schedule { + atomInstaces: ( + | DataExtractionAtom + | DataCleanAtom + | DataQueryAtom + | ChartGeneratorAtom + | BaseAtom + )[]; + + atomList: AtomName[]; + + private context: any; + + options: ScheduleOptions; + + /** current query */ + protected query: string; + + /** @todo */ + historySteps: any; + + constructor(atomList: T, options: ScheduleOptions, context?: CombineAll>) { + this.atomList = atomList; + this.options = options; + this.query = ''; + this.atomInstaces = atomList.map(atomName => this.atomFactory(atomName)); + this.setNewTask(context); + } + + initContext() { + this.context = {} as T; + this.atomInstaces.forEach(atom => { + this.context = atom.buildDefaultContext(this.context); + }); + } + + getAtomOptions(atomName: AtomName) { + return merge({}, this.options[AtomName.BASE], this.options[atomName]); + } + + atomFactory(atomName: AtomName) { + const options = this.getAtomOptions(atomName); + switch (atomName) { + case AtomName.DATA_EXTRACT: + return new DataExtractionAtom(this.context, options); + case AtomName.DATA_CLEAN: + return new DataCleanAtom(this.context, options); + case AtomName.DATA_QUERY: + return new DataQueryAtom(this.context, options); + case AtomName.CHART_GENERATE: + return new ChartGeneratorAtom(this.context, options); + default: + return new BaseAtom(this.context, options); + } + } + + /** regonize user intention and parse as sub tasks to atom instances */ + private parseSubTasks(query?: string): TaskMapping { + /** @todo */ + let taskMapping: TaskMapping = {}; + this.atomList.forEach(name => { + taskMapping = { + ...taskMapping, + [name]: { + shouldRun: true, + query + } + }; + }); + return taskMapping; + } + + async run(query?: string): Promise>>> { + this.query = query; + const subTasks = this.parseSubTasks(query); + for (const atom of this.atomInstaces) { + const { shouldRun, query: taskQuery } = subTasks?.[atom.name] || {}; + if (shouldRun || atom.shouldRunByContextUpdate(this.context)) { + this.context = await atom.run({ context: this.context, query: taskQuery }); + } + } + return this.context; + } + + /** init new task ready to run new query */ + setNewTask(context: any) { + this.initContext(); + this.updateContext(context); + this.atomInstaces.forEach(atom => { + atom.reset(this.context); + }); + } + + updateOptions(options: ScheduleOptions) { + this.options = merge({}, this.options, options); + this.atomInstaces.forEach(atom => atom.updateOptions(this.getAtomOptions(atom.name))); + } + + updateContext(context: any, isReplace = false) { + this.context = isReplace ? context : merge({}, this.context, context); + } + + /** + * get schedule context or specific context of atom + * @param atomName name of atom(optional) + * @returns context + */ + getContext(atomName?: AtomName): CombineAll> { + if (atomName) { + const atomInstaces = this.atomInstaces.find(atom => atom.name === atomName); + if (!atomInstaces) { + console.error(`Doesn\'t exist ${atomName}`); + return null; + } + return atomInstaces.getContext() as any; + } + return this.context; + } +} diff --git a/packages/vmind/src/types/atom.ts b/packages/vmind/src/types/atom.ts new file mode 100644 index 00000000..d39284e0 --- /dev/null +++ b/packages/vmind/src/types/atom.ts @@ -0,0 +1,116 @@ +/** Atom Function Types */ + +import type { AutoChartCell } from '@visactor/chart-advisor/src/type'; +import type { ChartType } from 'src/common/typings'; + +/** Base DataCell */ +export type DataCell = string | number; + +/** Base Data Item */ +export type DataItem = Record; + +/** Data Table */ +export type DataTable = DataItem[]; + +/** Base LLM Context */ +export interface BaseContext { + /** response logId in chat */ + logId?: string; + /** response id in chat */ + id?: string; + /** user query */ + query?: string; + /** llm response content */ + response?: string; +} + +export enum AtomName { + BASE = 'base', + DATA_EXTRACT = 'dataExtract', + DATA_CLEAN = 'dataClean', + DATA_QUERY = 'dataQuery', + CHART_GENERATE = 'chartGenerate' +} + +export enum DataType { + DATE = 'date', + TIME = 'time', + STRING = 'string', + REGION = 'region', + NUMERICAL = 'numerical', + RATIO = 'RATIO', + COUNT = 'count' +} + +export enum ROLE { + DIMENSION = 'dimension', + MEASURE = 'measure' +} + +export enum LOCATION { + DIMENSION = 'dimension', + MEASURE = 'measure' +} + +/** field information Of Data Table */ +export interface FieldInfo { + /** field ID */ + fieldId: string; + /** name of field */ + fieldName: string; + /** description of field */ + description?: string; + /** field type, eg: time / category / numerical */ + fieldType: DataType; + /** field role */ + role: ROLE; + /** field location */ + location: LOCATION; + /** example of field value */ + dataExample?: DataCell[]; +} + +/** Context of Data Extraction Atom */ +export interface DataExtractionCtx extends BaseContext { + /** text object of data extraction */ + text: string; + /** current summary of text */ + textSummary?: string; + /** extra fieldsInfo of dataTable */ + fieldInfo?: FieldInfo[]; + /** Data Table values */ + dataTable?: DataTable; +} + +export interface DataCleanCtx extends BaseContext { + /** extra fieldsInfo of dataTable */ + fieldInfo?: FieldInfo[]; + /** Data Table values */ + dataTable?: DataTable; +} + +/** Context of Data Query Atom */ +export interface DataQueryCtx extends BaseContext { + /** current summary of dataTable */ + dataTableSummary?: string; + /** extra fieldsInfo of dataTable */ + fieldInfo?: FieldInfo[]; + /** fieldsInfo of sql query result */ + llmFieldInfo?: FieldInfo[]; + /** Data Table values */ + dataTabel: DataTable; + /** sql */ + sql?: string; +} + +/** Context of Chart Generator Atom */ +export interface ChartGeneratorCtx extends BaseContext { + /** extra fieldsInfo of dataTable */ + fieldInfo?: FieldInfo[]; + /** Data Table values */ + dataTabel: DataTable; + /** chart type generator result */ + chartType?: ChartType; + /** field mapping result */ + cells: AutoChartCell; +} diff --git a/packages/vmind/src/types/index.ts b/packages/vmind/src/types/index.ts new file mode 100644 index 00000000..9be62dc3 --- /dev/null +++ b/packages/vmind/src/types/index.ts @@ -0,0 +1,3 @@ +export * from './llm'; +export * from './atom'; +export * from './schedule'; diff --git a/packages/vmind/src/types/llm.ts b/packages/vmind/src/types/llm.ts new file mode 100644 index 00000000..345074cb --- /dev/null +++ b/packages/vmind/src/types/llm.ts @@ -0,0 +1,59 @@ +import type { BaseContext } from './atom'; + +/** LLM Model Type */ +export enum ModelType { + GPT3_5 = 'gpt3.5', + GPT4 = 'gpt4', + doubao = 'doubao', + CHART_ADVISOR = 'chart-advisor' +} + +/** Specific LLM MODEL */ +export enum Model { + GPT3_5 = 'gpt-3.5-turbo', + GPT3_5_1106 = 'gpt-3.5-turbo-1106', + GPT4 = 'gpt-4', + GPT_4_0613 = 'gpt-4-0613', + GPT_4o = 'gpt-4o-2024-05-13', + DOUBAO_LITE = 'doubao-lite-32K', + DOUBAO_PRO = 'doubao-pro-128k', + SKYLARK2 = 'skylark2-pro-4k', + SKYLARK2_v1_2 = 'skylark2-pro-4k-v1.2', + CHART_ADVISOR = 'chart-advisor' +} + +/** LLM Options */ +export interface ILLMOptions { + /** URL of your LLM service. For gpt, default is openAI API. */ + url?: string; + /** llm request header, which has higher priority */ + headers?: HeadersInit; + /** post or get */ + method?: 'POST' | 'GET'; + /** LLM Model */ + model?: Model | string; + /** Max token in LLM Chart */ + maxTokens?: number; + /** Temperature of LLM */ + temperature?: number; + /** show llm thoughs or not */ + showThoughts?: boolean; +} + +/** LLM Messages api */ +export interface LLMMessage { + /** prompt role, system or user query */ + role: 'system' | 'user' | 'assistant'; + content: string; +} + +/** LLM Response API */ +export interface LLMResponse extends BaseContext { + choices: { + index: number; + message: any; + }[]; + usage: any; + error: string; + [key: string]: any; +} diff --git a/packages/vmind/src/types/schedule.ts b/packages/vmind/src/types/schedule.ts new file mode 100644 index 00000000..1d07a05c --- /dev/null +++ b/packages/vmind/src/types/schedule.ts @@ -0,0 +1,28 @@ +import type { AtomName, BaseContext, DataExtractionCtx, DataCleanCtx, DataQueryCtx, ChartGeneratorCtx } from './atom'; + +export interface Tasks { + /** current atom task should run or not */ + shouldRun: boolean; + /** current user's query in atom task */ + query: string; +} + +export type TaskMapping = Partial>; + +/** Map of atom - context */ +export type AtomTypeMap = { + [AtomName.BASE]: BaseContext; + [AtomName.DATA_EXTRACT]: DataExtractionCtx; + [AtomName.DATA_CLEAN]: DataCleanCtx; + [AtomName.DATA_QUERY]: DataQueryCtx; + [AtomName.CHART_GENERATE]: ChartGeneratorCtx; +}; + +export type MapAtomTypes = { + [K in keyof T]: T[K] extends keyof AtomTypeMap ? AtomTypeMap[T[K]] : never; +}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type CombineAll = T extends [infer First, ...infer Rest] + ? First & CombineAll + : Record; diff --git a/packages/vmind/src/utils/json.ts b/packages/vmind/src/utils/json.ts new file mode 100644 index 00000000..50c8a649 --- /dev/null +++ b/packages/vmind/src/utils/json.ts @@ -0,0 +1,30 @@ +import JSON5 from 'json5'; + +export const matchJSONStr = (str: string) => { + const first = str.indexOf('{'); + const last = str.lastIndexOf('}'); + const result = str.substring(first, last + 1); + return result && result.length > 0 ? result : str; +}; + +export const parseLLMJson = (JsonStr: string, prefix?: string) => { + const parseNoPrefixStr = (str: string) => { + try { + return JSON5.parse(str); + } catch (err) { + return { + error: true + }; + } + }; + if (prefix) { + const splitArr = JsonStr.split(prefix); + const splittedStr = splitArr[splitArr.length - 2]; + const res = parseNoPrefixStr(splittedStr); + if (!res.error) { + return res; + } + } + const res2 = parseNoPrefixStr(JsonStr); + return res2; +}; diff --git a/packages/vmind/src/utils/llm.ts b/packages/vmind/src/utils/llm.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/vmind/src/utils/text.ts b/packages/vmind/src/utils/text.ts new file mode 100644 index 00000000..58a78c43 --- /dev/null +++ b/packages/vmind/src/utils/text.ts @@ -0,0 +1,9 @@ +export const getLanguageOfText = (text: string) => { + const chineseRegex = /[\u4e00-\u9fa5]/; + + const chineseMatch = text.match(chineseRegex); + if (chineseMatch) { + return 'chinese'; + } + return 'english'; +}; From 8d998e5805a173d1b57a1d5dd9e6c35bb2d73335 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 13 Sep 2024 17:33:47 +0800 Subject: [PATCH 021/128] feat: add new data extraction page to run test --- .../browser/src/constants/capcutData.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 packages/vmind/__tests__/browser/src/constants/capcutData.ts diff --git a/packages/vmind/__tests__/browser/src/constants/capcutData.ts b/packages/vmind/__tests__/browser/src/constants/capcutData.ts new file mode 100644 index 00000000..39808c3c --- /dev/null +++ b/packages/vmind/__tests__/browser/src/constants/capcutData.ts @@ -0,0 +1,64 @@ +/* eslint-disable max-len */ +export const capcutMockData: any[] = [ + { + text: '地图带你看懂法国大选。 2024 年法国议会选举结果揭示了法国社会的撕裂，没有任何党派能获得 577 个议题中的绝对多数。左翼联盟、新人民战线执政党中间派联盟和极右翼国民联盟分别占据 182 席、 168 席、 143 席。除了这三家以外，其余党派席位最多的也只有右翼共和党的 48 席，因此左翼联盟、执政党和极右翼算是形成了三分天下的格局。今天我们便结合地图和数据，聊一聊法国大选。本期视频的所有分析均为个人观点，仅供参考。在开始之前，我先快速的放一下各党派在几个主要政治议题上的立场，有需要的朋友可以截图保存一下。', + fieldInfo: [ + { + fieldName: '党派', + fieldType: 'string' + }, + { + fieldName: '席位数', + fieldType: 'numerical' + } + ] + }, + { + text: '现在我们进入正题，首先需要说明法国的选区制度，全国共有 577 个选区，每个选区包括 10- 12 万的人口密集的地方选区就小，人口稀少的区域选区域就大。因此地图上各个政党所占的面积并不能与其票数划等号，而是要看具体的选区分布在人口密集的大城市，譬如巴黎、里昂、马赛，虽然看起来面积小，实际上选区数量很多，占的比重很大。', + fieldInfo: [] + }, + { + text: '尽管如此，马克龙强调市场自由化，支持创新和创业，对欧盟一体化和国际合作的开放还是得到了许多大城市选民的支持。大城市仍然是马克龙的重要支持来源。除了大城市外，我们刚才提到过法国东北地区工业的衰落，其实在法国的西北地区又是另一番景象。之前那张关于中小工业城市分布的途中，我们可以清晰的看到西北地区的工业对于当代法国的重要性，当然工业只是一个缩影。我们再来看失业率地图，黄色代表 2022 年失业率最低的 20 个省，橙色是 21- 40 低的省份，可以看出西北部地区的失业率明显低于东北部。再看移民分布地图，西北部地区因为大城市少，而且离地中海更远，接受的移民也比较少。综上，西北地区的经济较为稳定，而且不受移民带来的社会问题困扰，日子过得比较舒服，这些选民对于未来持乐观态度，也成为了马克龙的另一主要票仓。除了以上三大势力外，浅蓝色代表的右翼共和党也获得了 48 个席位。他们的支持者主要位于经济同样比较富足，但是政治观点更为保守的地区。这里我就不展开讲总结。本期视频我们从法国官方的报告与数据出发，从经济与人口地理的角度分析了法国的大选结果。左翼联盟新人民战线以大城市为根基拿下最多的 182 席，马克龙的执政党中间派联盟则凭借大城市和西北地区的支持者取得了 168 席。乐旁的极右翼国民联盟则主要扎根于东北与东南地区，以 143 起居于第三这样的三分割据局面使得法国议会缺乏绝对多数，并且三方势力相差不大。可以预见在未来法案的通过上会面临极大的阻碍。举例来说，左翼和极右翼甚至存在联手撤回延迟退休法案的理论可能。虽然实际操作起来也面临很多困难，双方都不太愿意和对方合作，但即便是理论，可能也已经能表明未来的不确定性。', + fieldInfo: [] + }, + { + text: '住手，你们住手，不要再砸了，你们不要再砸了。万万没想到，瑞幸和库迪的九块九大战，快把星巴克给卷死了。前段时间，星巴克公布了第二季度财报，营业收入 85.6 亿美元，同比下降了2%，净利润大跌15%，股票市值一天内蒸发了 1, 150 亿人民币。另一方面，星巴克的咖啡也在悄悄降价。如果你手机上有星巴克的APP，几乎每天都会收到 5 张以上的优惠券，比如满60.10、满75.15、任意新冰乐 7 折等等，部分单品的团购价优惠下来低至 9 元。终于， 9.9 的风还是卷到了星巴克。在过去很长一段时间里，星巴克是小资生活的代表，一杯咖啡动辄几十块钱，也只有电视剧里那些白领们和云淡风轻的走进去，熟练地点一杯拿铁，找个位置坐下，悠闲地打开电脑喝咖啡。岁月静好人间，值得有人点一杯星巴克，朋友圈能发十几条动态，有人为了抢星巴克限量版的猫爪杯，能通宵排队，甚至大打出手。', + fieldInfo: [] + }, + { + text: '星巴克在中国国内的定位一直都是高端咖啡品牌，但是很多人不知道的是，它在国外的定位其实是平民咖啡，在美国一杯星巴克大杯美式咖啡大概是 2.95 美元，而根据美国劳工部的统计，美国平均月工资是 6, 228 美元，最低时薪是 7.25 美元，这是啥意思呢？ 1002.65 美元的星巴克还不到美国人平均月收入的 2, 000 份之一，也就是平常坐一趟地铁的价格吧。而在中国市场，星巴克的饮品价格普遍要超过 30 元一杯。如果按照美国的对应消费力，它的主力消费人群应该是月薪至少6万元的人。', + fieldInfo: [] + }, + { + text: '2019 年2月，星巴克就发售了一款粉爪杯，那它长这样？售价 199 元，但是在网上最高炒到了 1, 800 元。有人寒冬腊月在星巴克门口通宵排队，就是为了买到这么一个杯子。还有人因为排队顺序大打出手，最后喜提免费食宿。靠着这些营销方法，在很长一段时间里，普通人对于星巴克是仰望的，觉得去星巴克消费是很有品的，再往前推十年，你甚至可以看到有人去星巴克点一杯咖啡就可以发十几条朋友圈的各种角度各种场景，还要配文低调有实力，天天喝都喝腻了好像呢？甚至有人专门发帖认真的提问，第一次去星巴克主要注意什么？怎么装的像老手呢？我一开始还以为是来搞笑和反讽的，没想到点开帖子此还真的是教大家怎么去星巴克抓老手的，包括但不限于怎么下载APP，问店员这周用的是啥肚子萃取时间是多少？张度和烘焙度怎么样？要不要加糖和加奶？这唬得我一愣一愣的，但是时过境迁，如今星巴克已经支棱不起来了，一边是疯狂降价买三送一搞促销，一边是继续下沉到四五线城市。', + fieldInfo: [] + }, + { + text: '星巴克近期发布的 2025 中国战略愿景当中，中国总部直言不讳地表示，星巴克看中的不仅仅是全国 300 多个 d 级市场，也包括近 3, 000 个县域市场。星巴克的愿景也体现在它的选址变化上，它在中国的门店已经突破了 7, 000 家，但是这开店位置却让人越来越看不懂了。', + fieldInfo: [] + }, + { + text: '一家卖咖啡的店。星巴克没落的第二个原因是当代年轻人更偏向实用消费主义。坦白讲，大部分人喝咖啡其实就是为了遮住那点咖啡因，好让自己在一天的工作中保持清醒。你跟不懂咖啡的人聊什么豆子产地、风味，他只会回你一句，冰美式和中药有什么区别啊？如果你面前有三杯咖啡，第一杯是星巴克的 30 元美式，后面两杯是瑞幸库里的 9 块 9 咖啡也让你买单，大部分人都会选择后面两个，当然也有人会吹星巴克的豆子有多么多么的好，所以它买的这么贵也是值得的。', + fieldInfo: [] + }, + { + text: '咱聊一下最新的重磅数据，反正挺复杂的，国内6月 M2 同比增长6.2%，预期6.8%。 M1 同比下滑5%，预期下滑5.4%。 M M 一剪刀差走扩至 11.2% 再创新高。6月人民币存款增加 2.46 万亿，其中居民存款增加 2.14 万亿，增量几乎全都是老百姓存的。与 M1 的下滑相对，上半年人民币存款总共增加了 11.46 万亿，其中居民存款增加 9.27 万亿。大头也是老百姓，但增速逐月放缓。', + fieldInfo: [] + }, + { + text: '7月 11 日晚上，由于美国公布的 CPI 数据超预期回落，让市场对美联储降息预期大幅升温。从这场所的美联储观察工具看，虽然7月 30 一日的美联储议息会议，市场预期不降息的概率仍然是达到93%，但9月 18 日的美联储议息会议，市场预期降息一次的概率是达到90%，这比起上个月概率是上升很多。不过我还是得强调一下，这个美联储观察工具将来只能反映市场当前的预期态度，不能拿来预测美联储货币政策，因为这个概率是会不断随着最新经济数据变化而变化。比如要是下个月美国 CPI 出现较大反弹，那9月降息的概率就会大幅下降。而这次市场预期美联储9月降息的概率大幅上升，主要有两个原因，一、美联储鲍威尔在7月 10 日的国会听证会上整体态度偏戈。鲍威尔称，劳动力市场降温意味着持续高通胀的潜在源头已经减弱。他还表示，就业市场的进一步疲软可能是不必要的，也是不受欢迎的。鲍威尔说，通胀方面的工作还没有完成，我们还有更多工作要做，但与此同时，我们需要注意劳动力市场现况，我们已经观察到劳动力市场出现相当明显的疲软，有着新美联储通讯社之称的知名记者尼奇默尔斯认为，鲍威尔本周其实已暗示美联储的利率政策即将开始改变方向。', + fieldInfo: [] + }, + { + text: '第二个原因是7月 11 日晚上 8 点半美国劳工部公布的通胀数据，6月 CPI 是同比上涨3%，市场预期值3.1%，前值3.3%。这次美联储 CPI 回落，更关键是 CPI 环比是负增长0.1%，这是美国 2020 年5月以来 CPI 环比首次出现下降，而且美国 2020 年5月还是因为疫情导致的 CPI 骤降，是比较特殊时期，所以美国 CPI 环比负增长确实不太常见。但仔细看美国 VI 月 CPI 的具体构成，感觉猫腻还是不少的。', + fieldInfo: [] + }, + { + text: '美国 VI 月 CPI 下降的主要贡献是汽油价格下跌。美国 VI 月汽油价格下跌了3.8%，抑制了当月通胀，抵消了食品和住房价格 0.2% 的上涨。比较诡异的是，美国原油期货价格6月是明明出现大幅上涨，这是因为原油期货价格传导到汽油价格有一些迟滞效应，但那样的话，下个月公布的 CPI 数据，汽油价格可能就得反弹了。要是下个月公布的 CPI 数据，汽油价格还继续下降，那就实在说不过去了。美国刨除能源和食品价格的核心通胀率6月是3.3%，但整体降幅还是低于CPI。', + fieldInfo: [] + }, + { + text: '美国通胀目前最顽固的就是服务业通胀，美国 VI 月服务业通胀仍然是同比上涨5%。美国毕竟是服务业为主的国家，服务业通胀还高居5%，美国要说自己已经控制住通胀，完全就是忽悠人。不过虽然美国大选临近，美国现在经济数据基本是为选型服务，比如已经假的不能再假的美国非农就业数据，美国 VI 月非农就业人口增加20.6万人，高于市场预期的 19 万人。然而美国同时把5月数据从 27.2 万人大幅下修至 21.8 万人，4月从 16.5 万人修正至10.8万人，修正后两个月合计较修正前减少 11.1 万人。', + fieldInfo: [] + } +]; From 9ed394bc3218386dc80101074a13268f99770693 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 13 Sep 2024 17:37:40 +0800 Subject: [PATCH 022/128] feat: add new data extraction page to run test --- .../__tests__/browser/src/pages/Home.tsx | 4 +- .../src/pages/NewDataExtraction/DataInput.tsx | 198 ++++++++++++++++++ .../src/pages/NewDataExtraction/DataTable.tsx | 59 ++++++ .../src/pages/NewDataExtraction/index.tsx | 40 ++++ .../__tests__/browser/src/pages/constants.tsx | 11 +- 5 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx create mode 100644 packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataTable.tsx create mode 100644 packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx diff --git a/packages/vmind/__tests__/browser/src/pages/Home.tsx b/packages/vmind/__tests__/browser/src/pages/Home.tsx index 4bf6ecc2..8c3b154f 100644 --- a/packages/vmind/__tests__/browser/src/pages/Home.tsx +++ b/packages/vmind/__tests__/browser/src/pages/Home.tsx @@ -7,9 +7,7 @@ const MenuItem = Menu.Item; const LOCAL_STORAGE_MENU_KEY = 'VMind_playground_menu_key'; export const Home: React.FC = props => { - const [selectedPage, setSelectedPage] = React.useState( - localStorage.getItem(LOCAL_STORAGE_MENU_KEY) ?? PLAYGROUND_PAGES.CHART_GENERATION - ); + const [selectedPage, setSelectedPage] = React.useState(PLAYGROUND_PAGES.CHART_GENERATION); const [collapsed, setCollapsed] = React.useState(true); return ( diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx new file mode 100644 index 00000000..cbfff0c8 --- /dev/null +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx @@ -0,0 +1,198 @@ +/* eslint-disable no-console */ +import React, { useState, useEffect } from 'react'; +import '../DataExtraction/index.scss'; +import { Avatar, Input, Divider, Button, Select, Checkbox, Modal } from '@arco-design/web-react'; +import type { FieldInfo } from '../../../../../src/index'; +import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; +import { capcutMockData } from '../../constants/capcutData'; + +const TextArea = Input.TextArea; +const Option = Select.Option; + +type IPropsType = { + onOk: (extractCtx: any, dataCleanCtx: any) => void; + setLoading: (loading: boolean) => void; +}; + +const globalVariables = (import.meta as any).env; +const ModelConfigMap: any = { + [Model.SKYLARK2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, + [Model.SKYLARK2_v1_2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, + [Model.GPT3_5]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT4]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT_4_0613]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } +}; +const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions'; + +export function DataInput(props: IPropsType) { + const defaultIndex = 0; + const [text, setText] = useState(capcutMockData[defaultIndex].text); + const [userInput, setUserInput] = useState(capcutMockData[defaultIndex].input); + + const [model, setModel] = useState(Model.GPT_4o); + const [useFieldInfo, setUseFieldInfo] = useState(true); + const [showThoughts, setShowThoughts] = useState(false); + const [visible, setVisible] = React.useState(false); + const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); + const [apiKey, setApiKey] = React.useState(ModelConfigMap[model]?.key); + const [fieldInfo, setFieldInfo] = useState(capcutMockData[defaultIndex].fieldInfo || []); + + const llm = React.useRef( + new LLMManage({ + url, + headers: { + 'api-key': apiKey, + Authorization: `Bearer ${apiKey}` + }, + model + }) + ); + const schedule = React.useRef>( + new Schedule( + [AtomName.DATA_EXTRACT, AtomName.DATA_CLEAN], + { base: { llm: llm.current, showThoughts } }, + { text, fieldInfo: useFieldInfo ? fieldInfo : [] } + ) + ); + useEffect(() => { + llm.current.updateOptions({ + url, + headers: { + 'api-key': apiKey, + Authorization: `Bearer ${apiKey}` + }, + model + }); + }, [url, model, apiKey]); + useEffect(() => { + schedule.current.updateOptions({ base: { showThoughts } }); + }, [showThoughts]); + const handleQuery = React.useCallback(async () => { + props.setLoading(true); + await schedule.current.run(userInput); + props.onOk(schedule.current.getContext(AtomName.DATA_EXTRACT), schedule.current.getContext(AtomName.DATA_CLEAN)); + }, [props, userInput]); + + return ( +
+
+
+ +
+
+
+
+

+ + 0 + + Select Demo Data (optional) +

+ +
+ +
+

+ + 1 + + Input your data in text format +

+ +
+ + +
+ ); +}; diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx new file mode 100644 index 00000000..605493e4 --- /dev/null +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx @@ -0,0 +1,40 @@ +import React, { useState } from 'react'; +import { Layout } from '@arco-design/web-react'; +import { DataInput } from './DataInput'; +import { DataTableComp } from './DataTable'; +import type { DataTable } from '../../../../../src'; +import type { FieldInfo } from '../../../../../src'; +const Sider = Layout.Sider; +const Content = Layout.Content; + +export function NewDataExtractionPage() { + const [dataset, setDataset] = useState([]); + const [finalDataset, setFianlDataset] = useState([]); + const [fieldInfo, setFieldInfo] = useState([]); + const [loading, setLoading] = useState(false); + + const handleOk = React.useCallback(async (dataExtractCtx: any, dataCleanCtx: any) => { + setDataset(dataExtractCtx.dataTable); + setFieldInfo(dataExtractCtx.fieldInfo); + setFianlDataset(dataCleanCtx.dataTable); + setLoading(false); + // eslint-disable-next-line no-console + console.info(dataExtractCtx, dataCleanCtx); + }, []); + + return ( + + + + + + + + + ); +} diff --git a/packages/vmind/__tests__/browser/src/pages/constants.tsx b/packages/vmind/__tests__/browser/src/pages/constants.tsx index 38ca2e08..ea812cd3 100644 --- a/packages/vmind/__tests__/browser/src/pages/constants.tsx +++ b/packages/vmind/__tests__/browser/src/pages/constants.tsx @@ -3,6 +3,7 @@ import { ChartGenerationPage } from './ChartGeneration/ChartGeneration'; import React from 'react'; import { InsightPage } from './Insight/Insight'; import { DataExtractionPage } from './DataExtraction/DataExtraction'; +import { NewDataExtractionPage } from './NewDataExtraction'; type MenuInfo = { menuItem: string; pageName: string; @@ -14,7 +15,9 @@ type MenuInfo = { export enum PLAYGROUND_PAGES { CHART_GENERATION = 'chart_generation', SMART_INSIGHT = 'smart-insight', - DATA_EXTRACTION = 'data-extraction' + DATA_EXTRACTION = 'data-extraction', + NEW_DATA_EXTRACTION = 'new-data-extraction', + DATA_EXTRACTIONI_TASK = 'data-extraction-task' } export const PLAYGROUND_MENU_INFO: { @@ -37,6 +40,12 @@ export const PLAYGROUND_MENU_INFO: { pageName: 'Data Extraction', component: , icon: + }, + [PLAYGROUND_PAGES.NEW_DATA_EXTRACTION]: { + menuItem: 'New Data Extraction', + pageName: 'New Data Extraction', + component: , + icon: } }; export const CollapseCSS = { From 80cf1923c36e3a8d646fd943d0e3edd6cd169df0 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Sat, 14 Sep 2024 11:09:26 +0800 Subject: [PATCH 023/128] feat: optimize prompt --- packages/vmind/src/atom/base.ts | 2 +- .../vmind/src/atom/dataExtraction/index.ts | 5 ++-- .../vmind/src/atom/dataExtraction/prompt.ts | 23 ++++++++----------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/vmind/src/atom/base.ts b/packages/vmind/src/atom/base.ts index 836a190b..5683f403 100644 --- a/packages/vmind/src/atom/base.ts +++ b/packages/vmind/src/atom/base.ts @@ -153,7 +153,7 @@ export class BaseAtom { content: query } ] - : this.responses; + : []; } protected getLLMMessages(query?: string): LLMMessage[] { diff --git a/packages/vmind/src/atom/dataExtraction/index.ts b/packages/vmind/src/atom/dataExtraction/index.ts index fe8c1f22..1b0208f0 100644 --- a/packages/vmind/src/atom/dataExtraction/index.ts +++ b/packages/vmind/src/atom/dataExtraction/index.ts @@ -43,7 +43,7 @@ export class DataExtractionAtom extends BaseAtom }, { role: 'user', - content: `Extracted text is bellow: ${text}` + content: `${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：'}:${text}` }, ...addtionContent ]; @@ -56,8 +56,7 @@ export class DataExtractionAtom extends BaseAtom \`\`\` TypeScript ${fieldInfoString} \`\`\` -Extracted text is bellow: -text: ${text} +${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：'}:${text} `; return [ { diff --git a/packages/vmind/src/atom/dataExtraction/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt.ts index 5ebe5dfe..7e38643d 100644 --- a/packages/vmind/src/atom/dataExtraction/prompt.ts +++ b/packages/vmind/src/atom/dataExtraction/prompt.ts @@ -25,13 +25,13 @@ text: John Smith was very tall, ranking in the 90th percentile for his age group Response: \`\`\` -{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string",},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"numerical"}],"dataTable":[{"name":"John Smith","ranking":0.9,},{"name":"Jane Doe","ranking":0.75}]} +{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string",},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"ratio"}],"dataTable":[{"name":"John Smith","ranking":90,},{"name":"Jane Doe","ranking":75}]} \`\`\``; const getFieldTypeExplanation = (language: 'chinese' | 'english') => { - return `field type explanation is below: 'ratio' means ratio value or percent value, such as ${ + return `field type explanation is below: 'ratio' means ratio value or percentage(%), such as ${ language === 'english' ? 'YoY or MoM' : '同比、环比、增长率、占比等' - }, 'count' means count data`; + }.The forms of ratio data are usually Percentage (%) such as 60%.'count' means count data`; }; export const getBasePrompt = ( language: 'chinese' | 'english', @@ -52,12 +52,11 @@ You should think step-by-step as follow: 0. using language answer: ${language} 1. Determine whether the current task is related to data extraction. 2. If not, return isDataExtraction is false in json mode; If yes, continue follow Steps -3. Read the entire text and find the most important and most frequently occurring field with a numerical field type. -4. Read all text again and generate field information associated with the numerical field.The newly generated fields are all simple. +3. Read the entire text and fields with numerical or ratio or count field type first. +4. Read all text again and generate field information associated with the fields found in Step3.The newly generated fields are all simple. 5. Finally, read all text and extract all corresponding data table based on the field information. 6. Adjust the data to ensure consistency within the same field especially time field. -7. Convert percentage values to actual values. -8. Assume the data is incomplete, then reconsider and execute the task again. +7. Assume the data is incomplete, then reconsider and execute the task again. Response in the following format: \`\`\` @@ -83,8 +82,7 @@ Finish your tasks in one-step. 2. The numbers in the dataset do not carry any units. 3. Keep the ratio value unchanged, such as '95%' --> '95' 4. Only use numbers that appear in the text. -5. If you do not know the value of an field, return null for the field's value. -6. There is exactly one field of numerical type in the data table.`; +5. If you do not know the value of an field, return null for the field's value.`; export const getFieldInfoPrompt = ( language: 'chinese' | 'english', @@ -109,8 +107,7 @@ You should think step-by-step as follow: 2. If not, return isDataExtraction is false in json mode; If yes, continue follow Steps 3. Read all text and extract all corresponding data table based on the field information. 4. Adjust the data to ensure consistency within the same field especially time field. -5. Convert percentage values to actual values. -6. Assume the data is incomplete, then reconsider and execute the task again. +5. Assume the data is incomplete, then reconsider and execute the task again. # Respones Response in the following format: @@ -135,11 +132,11 @@ Response: text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. \`\`\` -{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string","dataExample":["Roy","Stepen Curry","张三","李四"]},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"numerical","dataExample": [0.1, 0.8]]}}]} +{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string","dataExample":["Roy","Stepen Curry","张三","李四"]},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"ratio","dataExample": [10, 80]]}}]} \`\`\` Response: \`\`\` -{"dataTable":[{"name":"John Smith","ranking":0.9,},{"name":"Jane Doe","ranking":0.75}]} +{"dataTable":[{"name":"John Smith","ranking":90,},{"name":"Jane Doe","ranking":75}]} ---------------------------------- You only need to return the JSON in your response directly to the user. From 6fbeb114e02c433b3c4cd8c71702234b49ab3358 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Sat, 14 Sep 2024 11:09:58 +0800 Subject: [PATCH 024/128] feat: add experiment page to run case study --- .../vmind/__tests__/experiment/index.html | 14 + .../vmind/__tests__/experiment/src/App.tsx | 11 + .../vmind/__tests__/experiment/src/Layout.tsx | 15 + .../experiment/src/data/capcutData.ts | 320 ++++++++++++++++++ .../vmind/__tests__/experiment/src/index.scss | 4 + .../vmind/__tests__/experiment/src/index.tsx | 11 + .../src/pages/DataExtraction/caseStudy.tsx | 121 +++++++ .../src/pages/DataExtraction/test.tsx | 195 +++++++++++ .../__tests__/experiment/src/pages/Home.tsx | 63 ++++ .../experiment/src/pages/constants.tsx | 48 +++ .../__tests__/experiment/src/pages/page.scss | 77 +++++ .../vmind/__tests__/experiment/tsconfig.json | 112 ++++++ .../__tests__/experiment/tsconfig.node.json | 10 + .../vmind/__tests__/experiment/vite.config.ts | 61 ++++ 14 files changed, 1062 insertions(+) create mode 100644 packages/vmind/__tests__/experiment/index.html create mode 100644 packages/vmind/__tests__/experiment/src/App.tsx create mode 100644 packages/vmind/__tests__/experiment/src/Layout.tsx create mode 100644 packages/vmind/__tests__/experiment/src/data/capcutData.ts create mode 100644 packages/vmind/__tests__/experiment/src/index.scss create mode 100644 packages/vmind/__tests__/experiment/src/index.tsx create mode 100644 packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx create mode 100644 packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx create mode 100644 packages/vmind/__tests__/experiment/src/pages/Home.tsx create mode 100644 packages/vmind/__tests__/experiment/src/pages/constants.tsx create mode 100644 packages/vmind/__tests__/experiment/src/pages/page.scss create mode 100644 packages/vmind/__tests__/experiment/tsconfig.json create mode 100644 packages/vmind/__tests__/experiment/tsconfig.node.json create mode 100644 packages/vmind/__tests__/experiment/vite.config.ts diff --git a/packages/vmind/__tests__/experiment/index.html b/packages/vmind/__tests__/experiment/index.html new file mode 100644 index 00000000..7be3c11d --- /dev/null +++ b/packages/vmind/__tests__/experiment/index.html @@ -0,0 +1,14 @@ + + + + + + + VMind + + + +
+ + + diff --git a/packages/vmind/__tests__/experiment/src/App.tsx b/packages/vmind/__tests__/experiment/src/App.tsx new file mode 100644 index 00000000..6edf6cc5 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/App.tsx @@ -0,0 +1,11 @@ +import * as React from 'react'; +import { Home } from './pages/Home'; +import { LayoutWrap } from './Layout'; + +export default function App() { + return ( + + + + ); +} diff --git a/packages/vmind/__tests__/experiment/src/Layout.tsx b/packages/vmind/__tests__/experiment/src/Layout.tsx new file mode 100644 index 00000000..7f4bc5af --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/Layout.tsx @@ -0,0 +1,15 @@ +import * as React from 'react'; +import { PageHeader, Button, Layout } from '@arco-design/web-react'; + +export function LayoutWrap(props: any) { + return ( + + + {props.children} + + ); +} diff --git a/packages/vmind/__tests__/experiment/src/data/capcutData.ts b/packages/vmind/__tests__/experiment/src/data/capcutData.ts new file mode 100644 index 00000000..776e8d54 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/data/capcutData.ts @@ -0,0 +1,320 @@ +/* eslint-disable max-len */ +export const capcutMockData: any[] = [ + { + text: '地图带你看懂法国大选。 2024 年法国议会选举结果揭示了法国社会的撕裂，没有任何党派能获得 577 个议题中的绝对多数。左翼联盟、新人民战线执政党中间派联盟和极右翼国民联盟分别占据 182 席、 168 席、 143 席。除了这三家以外，其余党派席位最多的也只有右翼共和党的 48 席，因此左翼联盟、执政党和极右翼算是形成了三分天下的格局。今天我们便结合地图和数据，聊一聊法国大选。本期视频的所有分析均为个人观点，仅供参考。在开始之前，我先快速的放一下各党派在几个主要政治议题上的立场，有需要的朋友可以截图保存一下。', + fieldInfo: [ + { + fieldName: '党派', + fieldType: 'string' + }, + { + fieldName: '席位数', + fieldType: 'numerical' + } + ] + }, + { + text: '现在我们进入正题，首先需要说明法国的选区制度，全国共有 577 个选区，每个选区包括 10- 12 万的人口密集的地方选区就小，人口稀少的区域选区域就大。因此地图上各个政党所占的面积并不能与其票数划等号，而是要看具体的选区分布在人口密集的大城市，譬如巴黎、里昂、马赛，虽然看起来面积小，实际上选区数量很多，占的比重很大。', + fieldInfo: [] + }, + { + text: '尽管如此，马克龙强调市场自由化，支持创新和创业，对欧盟一体化和国际合作的开放还是得到了许多大城市选民的支持。大城市仍然是马克龙的重要支持来源。除了大城市外，我们刚才提到过法国东北地区工业的衰落，其实在法国的西北地区又是另一番景象。之前那张关于中小工业城市分布的途中，我们可以清晰的看到西北地区的工业对于当代法国的重要性，当然工业只是一个缩影。我们再来看失业率地图，黄色代表 2022 年失业率最低的 20 个省，橙色是 21- 40 低的省份，可以看出西北部地区的失业率明显低于东北部。再看移民分布地图，西北部地区因为大城市少，而且离地中海更远，接受的移民也比较少。综上，西北地区的经济较为稳定，而且不受移民带来的社会问题困扰，日子过得比较舒服，这些选民对于未来持乐观态度，也成为了马克龙的另一主要票仓。除了以上三大势力外，浅蓝色代表的右翼共和党也获得了 48 个席位。他们的支持者主要位于经济同样比较富足，但是政治观点更为保守的地区。这里我就不展开讲总结。本期视频我们从法国官方的报告与数据出发，从经济与人口地理的角度分析了法国的大选结果。左翼联盟新人民战线以大城市为根基拿下最多的 182 席，马克龙的执政党中间派联盟则凭借大城市和西北地区的支持者取得了 168 席。乐旁的极右翼国民联盟则主要扎根于东北与东南地区，以 143 起居于第三这样的三分割据局面使得法国议会缺乏绝对多数，并且三方势力相差不大。可以预见在未来法案的通过上会面临极大的阻碍。举例来说，左翼和极右翼甚至存在联手撤回延迟退休法案的理论可能。虽然实际操作起来也面临很多困难，双方都不太愿意和对方合作，但即便是理论，可能也已经能表明未来的不确定性。', + fieldInfo: [] + }, + { + text: '住手，你们住手，不要再砸了，你们不要再砸了。万万没想到，瑞幸和库迪的九块九大战，快把星巴克给卷死了。前段时间，星巴克公布了第二季度财报，营业收入 85.6 亿美元，同比下降了2%，净利润大跌15%，股票市值一天内蒸发了 1, 150 亿人民币。另一方面，星巴克的咖啡也在悄悄降价。如果你手机上有星巴克的APP，几乎每天都会收到 5 张以上的优惠券，比如满60.10、满75.15、任意新冰乐 7 折等等，部分单品的团购价优惠下来低至 9 元。终于， 9.9 的风还是卷到了星巴克。在过去很长一段时间里，星巴克是小资生活的代表，一杯咖啡动辄几十块钱，也只有电视剧里那些白领们和云淡风轻的走进去，熟练地点一杯拿铁，找个位置坐下，悠闲地打开电脑喝咖啡。岁月静好人间，值得有人点一杯星巴克，朋友圈能发十几条动态，有人为了抢星巴克限量版的猫爪杯，能通宵排队，甚至大打出手。', + fieldInfo: [] + }, + { + text: '星巴克在中国国内的定位一直都是高端咖啡品牌，但是很多人不知道的是，它在国外的定位其实是平民咖啡，在美国一杯星巴克大杯美式咖啡大概是 2.95 美元，而根据美国劳工部的统计，美国平均月工资是 6, 228 美元，最低时薪是 7.25 美元，这是啥意思呢？ 1002.65 美元的星巴克还不到美国人平均月收入的 2, 000 份之一，也就是平常坐一趟地铁的价格吧。而在中国市场，星巴克的饮品价格普遍要超过 30 元一杯。如果按照美国的对应消费力，它的主力消费人群应该是月薪至少6万元的人。', + fieldInfo: [] + }, + { + text: '2019 年2月，星巴克就发售了一款粉爪杯，那它长这样？售价 199 元，但是在网上最高炒到了 1, 800 元。有人寒冬腊月在星巴克门口通宵排队，就是为了买到这么一个杯子。还有人因为排队顺序大打出手，最后喜提免费食宿。靠着这些营销方法，在很长一段时间里，普通人对于星巴克是仰望的，觉得去星巴克消费是很有品的，再往前推十年，你甚至可以看到有人去星巴克点一杯咖啡就可以发十几条朋友圈的各种角度各种场景，还要配文低调有实力，天天喝都喝腻了好像呢？甚至有人专门发帖认真的提问，第一次去星巴克主要注意什么？怎么装的像老手呢？我一开始还以为是来搞笑和反讽的，没想到点开帖子此还真的是教大家怎么去星巴克抓老手的，包括但不限于怎么下载APP，问店员这周用的是啥肚子萃取时间是多少？张度和烘焙度怎么样？要不要加糖和加奶？这唬得我一愣一愣的，但是时过境迁，如今星巴克已经支棱不起来了，一边是疯狂降价买三送一搞促销，一边是继续下沉到四五线城市。', + fieldInfo: [] + }, + { + text: '星巴克近期发布的 2025 中国战略愿景当中，中国总部直言不讳地表示，星巴克看中的不仅仅是全国 300 多个 d 级市场，也包括近 3, 000 个县域市场。星巴克的愿景也体现在它的选址变化上，它在中国的门店已经突破了 7, 000 家，但是这开店位置却让人越来越看不懂了。', + fieldInfo: [] + }, + { + text: '一家卖咖啡的店。星巴克没落的第二个原因是当代年轻人更偏向实用消费主义。坦白讲，大部分人喝咖啡其实就是为了遮住那点咖啡因，好让自己在一天的工作中保持清醒。你跟不懂咖啡的人聊什么豆子产地、风味，他只会回你一句，冰美式和中药有什么区别啊？如果你面前有三杯咖啡，第一杯是星巴克的 30 元美式，后面两杯是瑞幸库里的 9 块 9 咖啡也让你买单，大部分人都会选择后面两个，当然也有人会吹星巴克的豆子有多么多么的好，所以它买的这么贵也是值得的。', + fieldInfo: [] + }, + { + text: '咱聊一下最新的重磅数据，反正挺复杂的，国内6月 M2 同比增长6.2%，预期6.8%。 M1 同比下滑5%，预期下滑5.4%。 M M 一剪刀差走扩至 11.2% 再创新高。6月人民币存款增加 2.46 万亿，其中居民存款增加 2.14 万亿，增量几乎全都是老百姓存的。与 M1 的下滑相对，上半年人民币存款总共增加了 11.46 万亿，其中居民存款增加 9.27 万亿。大头也是老百姓，但增速逐月放缓。', + fieldInfo: [] + }, + { + text: '7月 11 日晚上，由于美国公布的 CPI 数据超预期回落，让市场对美联储降息预期大幅升温。从这场所的美联储观察工具看，虽然7月 30 一日的美联储议息会议，市场预期不降息的概率仍然是达到93%，但9月 18 日的美联储议息会议，市场预期降息一次的概率是达到90%，这比起上个月概率是上升很多。不过我还是得强调一下，这个美联储观察工具将来只能反映市场当前的预期态度，不能拿来预测美联储货币政策，因为这个概率是会不断随着最新经济数据变化而变化。比如要是下个月美国 CPI 出现较大反弹，那9月降息的概率就会大幅下降。而这次市场预期美联储9月降息的概率大幅上升，主要有两个原因，一、美联储鲍威尔在7月 10 日的国会听证会上整体态度偏戈。鲍威尔称，劳动力市场降温意味着持续高通胀的潜在源头已经减弱。他还表示，就业市场的进一步疲软可能是不必要的，也是不受欢迎的。鲍威尔说，通胀方面的工作还没有完成，我们还有更多工作要做，但与此同时，我们需要注意劳动力市场现况，我们已经观察到劳动力市场出现相当明显的疲软，有着新美联储通讯社之称的知名记者尼奇默尔斯认为，鲍威尔本周其实已暗示美联储的利率政策即将开始改变方向。', + fieldInfo: [] + }, + { + text: '第二个原因是7月 11 日晚上 8 点半美国劳工部公布的通胀数据，6月 CPI 是同比上涨3%，市场预期值3.1%，前值3.3%。这次美联储 CPI 回落，更关键是 CPI 环比是负增长0.1%，这是美国 2020 年5月以来 CPI 环比首次出现下降，而且美国 2020 年5月还是因为疫情导致的 CPI 骤降，是比较特殊时期，所以美国 CPI 环比负增长确实不太常见。但仔细看美国 VI 月 CPI 的具体构成，感觉猫腻还是不少的。', + fieldInfo: [] + }, + { + text: '美国 VI 月 CPI 下降的主要贡献是汽油价格下跌。美国 VI 月汽油价格下跌了3.8%，抑制了当月通胀，抵消了食品和住房价格 0.2% 的上涨。比较诡异的是，美国原油期货价格6月是明明出现大幅上涨，这是因为原油期货价格传导到汽油价格有一些迟滞效应，但那样的话，下个月公布的 CPI 数据，汽油价格可能就得反弹了。要是下个月公布的 CPI 数据，汽油价格还继续下降，那就实在说不过去了。美国刨除能源和食品价格的核心通胀率6月是3.3%，但整体降幅还是低于CPI。', + fieldInfo: [] + }, + { + text: '美国通胀目前最顽固的就是服务业通胀，美国 VI 月服务业通胀仍然是同比上涨5%。美国毕竟是服务业为主的国家，服务业通胀还高居5%，美国要说自己已经控制住通胀，完全就是忽悠人。不过虽然美国大选临近，美国现在经济数据基本是为选型服务，比如已经假的不能再假的美国非农就业数据，美国 VI 月非农就业人口增加20.6万人，高于市场预期的 19 万人。然而美国同时把5月数据从 27.2 万人大幅下修至 21.8 万人，4月从 16.5 万人修正至10.8万人，修正后两个月合计较修正前减少 11.1 万人。', + fieldInfo: [] + }, + { + text: '我注意到虽然这次市场大幅提高了美联储9月降息概率，但美股和日股反而不涨反跌，纳斯达克在7月 11 日是下跌了2%，日股在7月 12 日也跟随下跌了2.45%。这是市场反身性效应的某种预演，股市炒的是预期，之前美股和日股是基于美联储降息预期，已经提前涨了一年多了，那么当美联储真正降息之后，市场是可能出现反身性效应，也就是所谓利好落地势利空的说法。当然股市走势千变万化，这也只是其中一种可能性。历史的参考例子，比如 2004 年美联储降息， 2006 年停止加息， 2007 年开始降息，但股市是一直涨到 2007 年底，随后自带危机爆发，股市开始大跌。我之前也梳理过，从 1980 年以来，美联储每次加息超过 5% 的幅度，首次加息后的 2- 4 年内都会爆发金融危机，这次美联储是 2022 年开始加息，所以按照历史路径， 2024 年到 2026 年是有可能爆发世界金融危机，这个结合当前国际局势和地缘形势，还是有挺大的可能性。', + fieldInfo: [] + }, + { + text: '以前的小学空荡荡的，老年人养起了鸡鸭鹅狗，彻底荒废了整个黑龙江省， 10 年时间荒废了近六成，小学加起来有 1, 900 余所，整个东北十年荒废了 6, 800 余所小学，少了一半。', + fieldInfo: [] + }, + { + text: '而小学缺孩子这个趋势早已经蔓延至全国各地了。东部的兹西县，有的小学一个班只有一个学生。华南的徐文县，去年某小学开学，一年级也只有一个学生。中部人口大省河南，据测算， 2023- 2027 年小学学龄人口预计下降 200 多万人，缩水超两成，出现了基数的坍塌。从全国来看，基于学龄人口的预测显示，全国超 1, 400 个县域中，近九成县域小学学龄人口预计下滑，小学鹤岗化以谁也没想到的方式在扩散，从东北开始到大江南北，下一步可能是上海最新的总和生育率只有 0.6 了，该来的总是要来，从民政局冷冷清清到妇产科缺孩子，再到幼儿园关停潮，现在轮到了小学关停潮，这个传播链条还在扩散。', + fieldInfo: [] + }, + { + text: '实际上，如果鹤岗化只是局限于教育领域，那还好说，但不是实际情况复杂的多楼市，像鹤岗那样房子白菜价的城市越来越多了，据不完全统计，至少有 10 个省 24 个城市陷入几万元买房的讨论。不是每平方米单价几万，而是一套房总价几万。网传广东惠州6万，广西南宁5万，山东东营4万，江苏南京3万，黑龙江大靶1万，这个传播势头在这轮楼市调整的加持下，现在已经来到了北京的外围，在京津两市的交界处，抹楼盘从 160 万元降到了 39 万，而且打了骨折还卖不出去。', + fieldInfo: [] + }, + { + text: '地方财力，之前鹤岗是全国第一个财政重整的地级市，甚至传出来停招公务员，现在过紧日子的城市也越来越多了。秦岭深处某县人口只有3万，编制人员却有 2, 194 名，一年的行政管理支出 1, 800 万，排在支出的首位。乌蒙山区某县一般公共预算收入 7 个亿，但工资预算总支出 26.3 亿，其中在职人员 20 亿，离退休人员 1.7 亿，零聘人员 4.6 亿。注意一个细节，在职人员数量 1.5 万，临聘人员数量 2.8 万。', + fieldInfo: [] + }, + { + text: '中国房价一度被视为坚不可摧的资产堡垒，更一度有京沪永远涨的口号。然而，自 2021 年以来，包括一线城市在内，房价持续低迷，深圳全市二手房均价距离 2021 年初的最高点跌幅已接近40%，而且还没有停下来的意思。各热点城市二手房每成交一套就要多出好几套，新增的房源和房价表现几乎完全正相关的是飞天茅台的价格， 53 度。', + fieldInfo: [] + }, + { + text: '飞天茅台在 2021 年巅峰时期，一瓶售价超过 3, 500 元，如今已经快跌破 2, 000 元，是巧合吗？过去一线城市房价和飞天茅台价格可以说是最硬的人民币计价资产了，甚至比现金还要优质。一线房产和飞天茅台在相当长的一段时间内有两个相同属性，一他们可以长期增值。二他们易于套现。然而现在情况出现了前所未有的变化，房价和飞天茅台两者双双在 2012 一年见顶。', + fieldInfo: [] + }, + { + text: '这不是巧合，其他很多数据也都在 2021 年见顶，比如另一个在 2021 年见顶并开始走下神坛的保时捷。保时捷销量的恶化还在加速，过去一年多，中国市场上保时捷的落地价可以说是惨不忍睹。近期，保时捷只卖 44 万的话题也引发了热议，其华南区域一家终端门店称， Macan 正在进行优惠促销，最高优惠 16 万，该车优惠后最低售价为 44.8 万。另外，在山东、湖北、江西、福建、浙江、江苏等多省份，该车均出现了 50 万元以下的裸车价，而报价达到 103.8 万的泰肯，现在 70 多万就可以拿下。目前，保时捷几乎所有的车型都可以打 7- 8 折。', + fieldInfo: [] + }, + { + text: '不只是保时捷，包括奔驰、宝马、奥迪在内的豪华汽车品牌今年以来都在大幅降价，但仍然止不住销量断崖式下降。 2024 年一季度，保时捷中国卖出 16, 340 辆，同比大幅下降24%。保时捷在今年 5 月份仅卖出 4, 633 辆，同比去年5月下滑高达40.61%。这说明保时捷在中国的销量正在加速减少。', + fieldInfo: [] + }, + { + text: '从两年前开始，特斯拉的 3 和 y 中国售价就是世界最低的， 7.98 万可以买到原本指导价 13.18 万的油电混动的卡罗拉。做一个对比，美国的油电混动卡罗拉的起售价是 2.35 万美元，有时甚至还要加价。按照美元人民币汇率计算，这款车的中国售价居然只有美国的一半水平，尽管配置存在差异，但不影响价格差异巨大的这个结论。除了卡罗拉外，汉兰达和凯美瑞也都大幅降价，即使是两年前，我们也很难想象只要 14.98 万人民币就可以买到最新款的混动版凯美瑞。', + fieldInfo: [] + }, + { + text: '中国物价的下降不仅仅体现在商品上，服务价格也是类似的趋势。举个例子，十多年前我常驻北京，当时经常在晚上十一二点从首都机场打车到西直门这样一段单程大约需要 110- 120 元，而现在滴滴大概只要 60- 70 元。如果是现在的出租车，价格和十多年前还是一样的。', + fieldInfo: [] + }, + { + text: '以工行、农行、建行发布的数据来看，其 2023 年个人住房贷款不良率分别由 2022 年的0.39%、0.51%、 0.37% 增长至了0.44%、 0.55% 和0.42%，基本都实现了两位数的增幅。大家别觉得这些小数字没啥大不了的，要知道这三家银行每一家的个人住房贷款余额都超过了5万亿，而且按揭贷款往年基本上都是银行稳赚不赔的买卖。供建农三家银行之所以每年能够包揽中国最赚钱企业的前三名，按揭贷款所带来的收益贡献巨大。现在这个优质资产的不良率正在以每年两位数的增幅增加。', + fieldInfo: [] + }, + { + text: '你是银行，你慌不慌？而另一项作为佐证的数据则是法拍房，大家知道现在法拍房的数据有多夸张吗？根据瀚海研究院发布的数据显示， 2022 年全国共挂牌法拍房 98 万套，去年这个数字变成了 141 万套，增长了43.9%。而今年光是一季度的挂牌数量就已飙升至 60.44 万套通，同比上涨192%。这种局势下，银行要是再不改变断供处置策略，那今年的法拍数量估计有望达到 200 万。', + fieldInfo: [] + }, + { + text: '事实上，对银行来说，现在的行情即使他收了房也难以处置。我们以北京为例， 2023 年北京挂牌法拍房 8, 153 套，最终成交仅 2, 771 套，处置率为33%，这还是房价波动相对较小的北京，换到其他已经跌穿首付的地区，处置率恐怕只会更低。而在法拍流程里，流拍和拍品二次上拍都会在此前的价格上更进一步降低，这也导致了银行回款难度的进一步提高。虽然按照现在的规则，这部分差价是由贷款人承担的，但对方既然已经到了选择断供的地步，可想而知最终也执行不了多少。', + fieldInfo: [] + }, + { + text: '另一项推动银行改变策略的原因，则是今年4月 30 号的一场会议，这场会议确定了房地产行业未来一年的发展方向，统筹消化存量房产和优化增量住房，用大家都熟悉的话来说就是去库存。根据国家统计局的官方数据显示，截至 2024 年5月，我国未出售商品房为 7.46 亿平米，远超 5.9 亿平米的正常库存水平。而整个上半年，根据 CRS 的统计，全国 222 个城市总计出台了 341 项宽松政策，但带来的效果均不理想，无论是销售面积还是投资金额，仍然在持续走低。', + fieldInfo: [] + }, + { + text: '大家好，我拍拍一名做过财经记者，大学老师和滴滴司届 up 主。 2023 年，我国汽车出口量达到了 491 万辆，超越日本成为世界第一汽车出口国。要知道，日本在这个位置坐了 8 年之久，而中国仅在过去三年时间里接连赶超韩国、德国、日本。中国汽车出口， 2021 年 202 万辆， 2022 年 311 万辆， 2023 年 491 万辆。', + fieldInfo: [] + }, + { + text: '中国汽车转眼间为何变得这么受欢迎呢？又到底是哪些国家在购买中国汽车？中国卖给老外汽车又是些什么品牌和价位车型？本期视频就为大家打开中国汽车出口全球第一背后的真实数据。首先，中国出口的 491 万辆汽车都是些什么车呢？根据乘联会统计，中国乘用车出口量前十车型分别为名爵ZS、特斯拉 model y、奇瑞瑞虎7、特斯拉 model 3、名爵 4 EV、奇瑞虎5X、欧盟达名爵5、缤越元plus。除了特斯拉的 model y 和 model 3，其他车型国内指导价基本都在 10 万元左右，比如排名第一的名爵ZS，指导价 8- 9万元，最便宜的名爵 5 和缤越低到6万元就能拿下，可见中国汽车出海主打的还是一个性价比。', + fieldInfo: [] + }, + { + text: '如果按燃油车、新能源车的分类来看， 2013 年中国出口燃油车 371 万辆，出口新能源汽车 120 万辆，新能源车占到出口总量的25%，虽然这个占比目前只有 1/ 4，但去年新能源出口增速是77.6%，势头不可谓不猛。', + fieldInfo: [] + }, + { + text: '那中国汽车出口都卖到了哪些国家呢？ 2013 年中国汽车出口量前十的国家分别是，俄罗斯90.9万辆，墨西哥 41.5 万辆，比利时 21.7 万辆，澳大利亚 21.4 万辆，英国 21.4 万辆，沙特阿拉伯 21.3 万辆，菲律宾 17.2 万辆，泰国 16.9 万辆，阿联酋 15.9 万辆，西班牙 13.9 万辆。按地区来看的话，欧洲市场占中国汽车对外出口的38%，远超其他任何单一大洲，可见中国汽车正在得到全世界更多人的认可。', + fieldInfo: [] + }, + { + text: '当然了，中国汽车出口世界第一，又不得不提议俄罗斯和墨西哥这两个国家可以说去年是把中国车买爆了。 203 年，中国对俄罗斯的汽车出口量从上一年度的 16 万辆暴增到了90.9万辆，增加了468%。在俄罗斯的新车市场中，第一名是俄罗斯品牌拉达，第二至第七名则全都是中国品牌，比如第二名是奇瑞金车，市场占有率11.2%。第三名是哈弗，新车，市场占有率10.6%。俄罗斯卖最好的新能源车也是来自中国的极客。目前中国汽车已经占据俄罗斯新车市场的51%，可以说是拿下了半壁江山。而对于中国而言，仅俄罗斯一个国家 203 年就贡献了中国汽车出口增量的42%，甚至有俄罗斯本土汽车经销商预测， 2024 年中国汽车可能占据俄罗斯新车份额的80%。', + fieldInfo: [] + }, + { + text: '有人说俄罗斯满爆中国汽车是因为欧美的贸易封锁，这也步无道理。 2013 年在俄罗斯的新车市场中，欧洲的市场份额从 18% 降到了4%，韩国从 16% 降到了6%，日本从 12% 降到了5%，和欧日韩都是对俄罗斯实行了限制出口国家，其中就包括了部分汽车，可以说中国汽车吃下的正是欧日韩在俄罗斯丢掉市场。', + fieldInfo: [] + }, + { + text: '当然了，除了俄罗斯之外，其他国家也在买中国汽车，比如墨西哥。 2013 年，墨西哥所有销售汽车中有 25% 来自中国，而在 6 年前这个数字为0。澳大利亚也在不断买中国汽车，最受澳大利亚欢迎的中国汽车品牌是名爵，去年卖了 5.8 万辆。在新能源车市场，比亚迪则占据了澳大利亚的新能源汽车 14% 份额，位于第二名。', + fieldInfo: [] + }, + { + text: '当然，这里也不得不提一下第一名，那就是特斯拉市场份额高达53%，在东南亚市场，中国车企业销量在 2013 年同样实现小幅上升，最典型的就是泰国，在泰国的新能源车市场，中国品牌占据了 80% 的份额，比如比亚迪的原 plus 就是泰国的新能源车爆款，那到底是什么原因让中国汽车爆卖呢？基本还可以总结为三方面原因，首先是全球疫情爆发，由于中国汽车的供应链完善，疫情期间仍能维持稳定生产，而日韩这些过去的出口大户受疫情影响，芯片、钢材、橡胶等关键原材料短缺，不仅汽车产能下降，而且成本升高，这就让中国汽车更具性价比。而随着中国国内新能源汽车市场越来越卷出海，成为不少中国车企的选择，比如比亚迪 2023 年进入全球 58 个国家和地区，出口汽车 24 万辆，是上一年度的 3.34 倍。在泰国新能源车市场，比亚迪单独占到了 40% 的市场份额，是名副其实的泰国新能源汽车销冠。而且中国新能源汽车并非只是具备成本优势，汽车与 AI 互联网融合的智能化更是中国车企的拿手好戏。从豪华配置到智能大屏，从外观设计到内饰比拼，这让中国新能源汽车的溢价能力明显变高。2019 年中国新能源汽车平均出口价格每量只有 5, 000 美元， 2022 年涨到了 2.2 万美元。比如比亚迪汉在欧洲发布时价格接近 50 万人民币，是国内售价的两倍多。在泰国、以色列、新西兰等多个国家，比亚迪也已经是新能源汽车的销售冠军。不过，中国汽车征服海外虽然是一部励志爽门，但其实有不少挑战。', + fieldInfo: [] + }, + { + text: '其实中国汽车出海不禁让人想起曾经的中国摩托车出海。 2000 年前后，中国摩托车进军越南，一度占据了 80% 的越南市场份额，但不到三四年时间，却被日本摩托车打得片甲不留。如今日本摩托车在越南占据 95% 的份额，而中国摩托车百分之一都不到。曾经也有大量中国摩托车车企在越南建厂，但却形成了恶性竞争的关系，疯狂打价格战，导致服务和质量越来越差，越南的中国摩托车车企仿佛是飘在越南的。', + fieldInfo: [] + }, + { + text: '每个中国制造品牌的背后都有一批优秀的零部件供应商，像汇川技术、恒力液压、先导智能、顺域光学、军胜电子这样的零部件企业也是中国制造的骄傲，只不过知名度无法媲美消费者直接接触的终端品牌。按照官方的口径，中国规模以上也就是年销售收入 2, 000 万以上的制造企业有 44.5 万家。至于中国一共有多少家制造业企业存在各种口径，从 300 多万家到近千万家不等。万德资讯给我的数据是，中国大约有 622 万家存续的制造业企业。海之在线是一家总部在上海，聚焦中间品贸易的数字化平台，连接着 70 万家工厂，他们给我的数据是，中国规模以下的中小微工厂大致有 400 万家。这期节目标题中的 400 万家沉默工厂处处记载于此。海志在线的创始人、 CEO 佘莹对我说，从平台看， 40% 的工厂规模不到 50 人，近 90% 的工厂不到 500 人，大部分工厂的年产值在数百万元到数千万元。如果和大企业比，你可以说他们就是一个个的小做法。如果走进去可能会看到老旧的机器上油漆斑驳，可以看到生产计划就用记号笔写在车间墙上挂着的白板上，甚至会发现用破洞的木板随意围搭起来的厕所，待客的茶水里则混杂着浓浓的机油味。但他们就是中国制造业毛细血管层面的供应链小节点，勤勤恳恳的维护设备、搞生产，他们最在意的是生存，是接到订单以及在满足客户之后能够完整的收到货款。', + fieldInfo: [] + }, + { + text: '有报道称，一些地方政府面临收入短缺，要求企业缴纳可追溯到 1990 年代的税单。这种紧缩政策在房地产市场寻找几步的时刻，会损害上信心和经济。高盛认为，中国中央政府可以通过加大对西方政府的财政支持来切断机房政府无紧缩所出现的负面溢出效应，那如同美国监管机构在次贷危机期间通过成为最后贷款人来切断金融危机的传播一样。关于出口和机产之间的分化，可以同中国的金属生产中得到证实啊。铝和其他废且金属的产量相比疫情之前上升了 20% 以上，而钢铁的产量下降了 5% 到10%。在房产方面，开发商越来越依赖银行融资， 5 月份对开发商的银行贷款同比增长了19%。而随着房地产销售的下滑，房贷和存款跟预付款的比例同比下降了 30% 到40%。高盛银行股票团队预计从 2024 年到 2026 年，房地产贷款将增加 4.5 万亿人民币，以完全期待收缩的房地产债券和设防的影子银行贷款。', + fieldInfo: [] + }, + { + text: '中国经济分化并不止于出口和房地产基础设施的固定资产投资。细分项显示，建立燃气和水的生产投资已同疫情前水平翻倍，远远超过了整体的基建的投资增长。因为中国政府首先优先考虑农源供用安全和脱碳。在零售销售当中， 5 月份代线销售商品同比增长13%，而餐饮销售仅增长5%，线下商品的销售保持在去年同期的水平。', + fieldInfo: [] + }, + { + text: '7月中旬将举行两场重要的政策会议，前者将专注于评估当前的经济状况，并为今年剩余时间制定周期性的政策安排。后者将专注于至少未来五年解决经济结构当中的重大改革议程。鉴于一季度的实际， g GPT 同比增长5.3%，而且去年基数较低，那么 G2 季度增长可能高于5%，政府的全年增长目标仍然在轨道上。', + fieldInfo: [] + }, + { + text: '高盛认为，政策执行者不会在 7 月份的首场会议上释放任何重大的刺激措施，宏观政策的证件可能大于维持当前立场和执行现有的政策。另一方面，鉴于 5 月份宣布的最后一批措施不及预期，可能存在引入更多期产宽松政策的可能性。在货币政策方面，资本外流的担忧和银行利润率的下降限制了央行降息的能力。高盛预计三季度将降息 25 个基减，以适应大量政府债券发行，并预计季四季度9月首次降息后会再降息 10 个期减。在财政政策方面，高盛预计政府债券的发行将在下半年显著增加，以完成年初发行缓慢的全年配额。除非增长急剧放缓，否则基建的投资不会加速太多。对于政府 3 月份公布的预算计划，高盛虽然预测中国的增强型的弹盛赤字从去年的 gdp 11.2% 会适度扩大到今年的11.9%，但由于今年出口强计可能会存在财政扩张不及预期的风险。信贷政策方面，正如央行行长6月 19 号陆家嘴论坛上所说，由于金融套利的虚假贷款和监管机构随后对金融系统中这种资金空转的打击，信贷增长与 GDP 增长之间的联系已经减弱。预计摄容总量的增长将同去年的 9.5% 放缓到今年的9%。在住房政策方面，4月的政治局会议表明，决策者希望严防房地产市场的尾部风险。由于地产价格和活动的持续下行，以及机房国企通过央行的贷款计划购买空池公寓的速度缓慢，高盛预计进一步削减房贷利率以刺激需求，同时为去库存提供更多的资金和效率的支持。在外汇政策方面，鉴于美元持续强势和资本外流的压力，高盛认为央行将在短期内保持美元对人民币汇率的稳定，三个月的高盛预测是7.3，因为外汇市决策者可以迎来抵消关税对出口负面影响的工具。 2018- 19 年的经验表明，如果特朗普赢得美国大选，而且正如他最近几个月所宣称的，会对中国实施正大的关税，那美元兑人民币可能会显著贬值。', + fieldInfo: [] + }, + { + text: 'Because Japan is one of the most import dependent countries in the world, importing over 90% of its energy and over 60% of its food, a weak yen means inflation has returned to Japan for the first time in decades. ', + fieldInfo: [] + }, + { + text: ' For context, in the post war years, Japan experienced many decades of rapid economic growth in a period dubbed the Japanese Miracle. From 1955 to 1990, Japanese growth averaged 6.8% per year, and GDP multiplied eight times, with growth falling below 3% only once during the 1974 oil shock.', + fieldInfo: [] + }, + { + text: `Anyway, this anxiety subsided in the 90s when Japan experienced an enormous financial crisis after a rapid appreciation in Japanese stock and real estate prices during the 80s. In 1990, the bubble burst and continue to burst for a while. In the decade after 1990, residential house prices fell by more than 50%, commercial property prices fell by something like 85%. And Japan's main stock index, the Nike 2,2,5 fell by about 75%. Japan's economy never really recovered and growth and inflation both remained close to zero until very recently.`, + fieldInfo: [] + }, + { + text: `From 2016 until late last year. The Bank of Japan even began what's called yield curve control, which essentially involved buying up enough debt to guarantee that government borrowing costs wouldn't go above a certain level. Japan's ultra loose monetary policy came under pressure in 2022 when inflation started rising across the world. Usually, central banks raise interest rates, but the bank of Japan decided not to, both because inflation was relatively low in Japan, but also because thanks to its enormous debt burden, even a slight raise in interest rates would translate to a massive increase in debt servicing costs, especially for the Japanese government. Unfortunately, things have become more difficult as other central banks have raised rates, making their currencies relatively more attractive and sparking a decline in the yen. In the last year, the yen has fallen from about 130 to the dollar to a 34 year low of 160 to the dollar on Monday. Now the speed and severity of this decline presents a difficult dilemma for the bank of Japan because thanks to Japan's reliance on imports, it sparked significant inflation in essential items like food and energy. But they still don't want to raise rates for the reasons we've just mentioned earlier. This is why on Monday evening, instead of raising rates, the bank of Japan used billions of dollars worth of Japan's foreign exchange reserves to buy up the yen on the international market, artificially inflating its value. While this seems to have worked in the short term, as of Tuesday morning, the yen is now trading at nearer 155 to the dollar. It's both expensive and fundamentally unsustainable. Even if the bank of Japan has some of the largest foreign exchange reserves in the world. All in all, assuming the bank of Japan won't engage in significant rate hikes, this means that the yen is very much dependent on what goes on in the rest of the world. If inflation comes down and other central banks start cutting rates, then this will reduce some of the pressure on the yen. But if inflation turns out to be stickier than we'd like, which seems to be the case, then the divergence between the bank of Japan and other central banks will persist, which means more downward pressure on the yen. If this happens, then the bank of Japan won't be able to stave off the yen's decline with exchange reserves forever. And eventually, they'll have to choose a horn of their uncomfortable dilemma, either to just accept the yen's decline and all the inflation related political turmoil that comes with it all, raise rates and just hope that the world's most debt burdened economy can somehow deal with it.`, + fieldInfo: [] + }, + { + text: `Blanket is all over the news. Linkage is now more valuable than Zomato. Linkage reported over 2,300 for rupees and revenue. And then now speak about Zomato, they speak about blinking bigger than tomato right now with the valuation of roughly $13 million and a market share of 46%, it has disrupted India's 23,000 crore quick commerce industry, a company that Zomato acquired in 2022 to enter into quick commerce, but now has become more valuable, the Zomato's own food delivery business. In fact, the company 3 x its revenue from 800 crores to 23 crores and is expected to break even in the first quarter of Fi 2025.`, + fieldInfo: [] + }, + { + text: `By the way, this is going to be detailed, so feel free to pause the video wherever you feel confused. First of all, let's take a realistic average order value of 600 rupees, which is very close to what most people usually order on blink it. If you look at the revenue side, which is the money that bind it earns here for themselves from each order is divided into three sources. The first one is warehousing services and marketplace commissions. This is basically the amount that suppliers are paying to blanket for showing and selling their products. And see on every order of 600 rupees, 11 to 13% is coming from suppliers, which is roughly 72 rupees.`, + fieldInfo: [] + }, + { + text: `Now the second part of the income of this order is the ads that come companies show on blanket. This is the price that brands pay to show their products above other products as you scroll the app. For example, brew might pay to show its coffee first when someone searches for a keyword like coffee, it's roughly 2.3 to 3.5%. In our case, let's take 3.5% and it will come down to 21 rupees.`, + fieldInfo: [] + }, + { + text: `The next is customer fees, which includes your delivery fees, handling fees for packaging and delivering the food to your doorstep, and even additional fees like fees they charge you for having a small cut. This percentage comes at around three portion of the average order value and is roughly 18 rupees in our case. By the way, there are also other levers like membership plans or free delivery plans that these plan platforms try to sell you often like Zepto does it with their offering of zeptopass. But if we don't over complicate and dive much deeper into this, we see that in a nutshell, on an average order of 600 rupees, blanket owns roughly 110 rupees. This 110 rupees is known as a take rate, the share that blanket keeps for itself from an order.`, + fieldInfo: [] + }, + { + text: `Now let's come down to the cost side. Even here you have four elements. The first one is the biggest one, which is the last mile delivery cost, which is the last step when the riders deliver the orders to you and cost about roughly 7% to blanket. And in our case, it would come down to 42 rupees. The next one is dark store mid mile and warehousing cost. This entire combination of cost comes down to about 6.5%. And in our case, it would be 39 rupees. The other variable cost, which includes packaging, washes, support, communication and payment charges are roughly 2%, which comes down to 12 rupees. And now the fourth and the last one is customer acquisition cost, which is the discount, the incentives and the offers they try to give you to make tempting these for you. This comes at about 0.2 to 0.3% at about 1.8 rupees. So if you subtract these two amounts, blink it on roughly 15 rupees from entire transaction. This 15 rupees is not the net profit, by the way, but the contributing profit. Now what is that? See, contributing profit is the profit that the company is earning by serving each order. And company considers only the variable expenses in this case, which is the expense that we have already discussed. And it does not mean net profit because there are so many fixed expenses that are not considered like expensive salary of tech folks, rent, insurance, depreciation, and all similar big sums of money.`, + fieldInfo: [] + }, + { + text: `The answer to this is the dark stores, the 2,500 to 4,000 square feet big stores that are located in 1.5 to 3 kilometer radius near your homes to ensure super quick delivery. And by the way, these stores are super big. For example, if your nearest kirana shop has about 1,500 SKUs, these stores can have 4 x a number of excuse. In fact, a highly efficient dark store can do a better gross merchandise value per square foot than a highly organized supermarket like Dmart. So while a dark store or blanket can do a GMV of 90,000 rupees per square fit, Mart can only do a GMV of 47,000 rupees per square fit. But how does this work? See, these stores are like supermarkets, but have no Hawkins, which means that only rider can go and collect stuff from there. These stores have a lot of inventory that comes from Mother Warehouse store, which is located in the outskirts. So to give you context, for every 40 dark stores located in the city, there is always a mother warehouse, which is located at the outskirts or city. And that is more than 10 times big as a dark store, which is about 20,000 to 1 lax, 75,000 square feet big. It is super huge. And companies don't set dark stores everywhere, by the way. They are smartly set based on multiple parameters like average household income of the area, peak time traffic of the area, infrastructure structure of the area, and also the population density. Also, there's usually about a staff of 25 to 30 people who are working in three shifts in these dark stores who would take care of the packaging. And as we've discussed earlier as well in the video, that the operating cost for a dark store comes at about rupees 22 for each order. And if you want to understand this calculation better, we have put it here. So you can pause the video and look at this table.`, + fieldInfo: [] + }, + { + text: `Now blanket has done a really solid job here as they have, right now, the highest number of dark stores with 451 stores in 27 cities compared to 450 stores of Insta margin, 25 cities, 30 of Zepto intensities and 350 stores of Big Basket in 35 cities. By the way, this is not something they built in a day because it was the first grocery app in the country, which started in 2014 as growers. So their team and their execution is way more experienced if you compare them with other competitors, and now they're just building on this and increasing their penetration throughout the country. By the way, they're mostly penetrated in north and east India, and 90% of the GMV comes from top paid cities. But as they enter south and other cities, this can be a huge opportunity for them as they already have an experience DNA running in the organization.`, + fieldInfo: [] + }, + { + text: `Now coming down to the second insight, which is cracking high average order value. But why are we talking about AOVs? Average order value plays a big role in quick commerce because bigger the average order value, bigger is the contribution margin for the company, which means that delivering just a set of bananas or apples is less profitable for blanket. Then delivering a set of bananas, apples, onions, tomatoes and a packet of bread together. And blanket has the highest average order value if you compare it with all the comparators. And just look at the stark difference by yourself. For big Basket, the AOV is about 400 to 500 rupees. For Zepto and Instamar, it is around 450+. And for blanket Au UV is about staggering 6,35 rupees. This is something that is definitely giving them an edge in pulling one of the most important levers in the ecosystem. In fact, in the last quarter, this number was 523 rupees at the start of Q1 fi 23. It just shows the speed at which they are growing really fast, and they have done this really well through their amazing SKU strategy.`, + fieldInfo: [] + }, + { + text: `Not talking about the third insight, new customer acquisition. See, blinkits market share has not been the highest forever. It was 32%2082. And in the same period, Instama's market share has fallen from 52%, while Zeptos has increased from 15 to 28%. And we had to talk about the elephant in the room, the Zomato Effect. See, Zomato is the biggest food delivery app that has more than hundred million active users every month on its app and a market share of more than 56%. When it comes to food delivery, these users are more than three ties what blanket has at the moment. So even getting 5% of Zomato's monthly active users as new customers could bring more than 33% rise to their current Mau base.`, + fieldInfo: [] + }, + { + text: `Want to show you something cute little condo in Toronto's Harbor Front neighborhood, bustling part of the city. It goes on the market in the summer of 2022. The sellers put it up for $480,000. It didn't sell early. 2023, it's back on the market for 460 k. No luck later that year, posted again at 4,50 k. Still nothing. And a few months ago, the sellers tried again. At $430,000, no takers. This condo was sitting on the market for more than 400 days without a sale. Right now in Canada's biggest cities, there's a ton of condos like this struggling to sell. A condo sales in Toronto, for example, haven't been this low since the financial crisis in 2009.`, + fieldInfo: [] + }, + { + text: `Let's compare what a condo investment in Toronto looked like in 2016 to today. So according to the Toronto Real Estate Board, the average price of a one bedroom condo back then in 2016 was about three hundred thousand dollars and with 2016 interest rates about 2.7%, that worked out to a mortgage payment of about eleven hundred dollars a month. Rent for the average one bedroom apartment at that time was about sixteen hundred and sixty dollars. Not bad from an investor point of view, even with, you know, maintenance fees, property taxes, that condo is essentially paying for itself while you build equity.`, + fieldInfo: [] + }, + { + text: `Now what if you were to buy a one bedroom condo today in Toronto? That could cost you about 550 grand. So prices have very clearly gone up, but so have interest rates. The carrying costs on a property are far higher than they ever used to be. Well, not ever, but in recent memory. At current rates, 6.8%, maybe now you're paying more than $3,000 a month for your mortgage payment alone. Then you've got, you know, property taxes, maintenance fees and the average rent per unit like that, about 24 hundred dollars a month you're now paying out of.`, + fieldInfo: [] + }, + { + text: `Now the renewals come in and you're losing a lot of money, but you've actually come to realize in the last 12 months the price of that condo has probably dropped about $40,000. So you're starting to enter into a state of actual fear.`, + fieldInfo: [] + }, + { + text: `According to stats Cam, the average size of a new condo in Toronto has shrunk significantly over time. From 1981 to 1990, new condos were on average about 1,000 square feet. From 2016 to 2020, they were around 650 square feet or about 40% smaller. And if you ask anyone who lives in the downtown core right now, 650 square feet actually feels pretty big.`, + fieldInfo: [] + }, + { + text: `Right now, for example, in Toronto and Vancouver, three quarters of larger condos, the ones, you know, maybe more suitable for a family built decades ago, those are being lived in by the people who own them. But for those much smaller condos built after 2016 with investors in mind, only about half are being lived in by owners. Remember that cute little studio that we showed you at the start that took 400 some days to sell. It was 330 square feet. Here's a two bedroom, 700 square foot unit in that same building. It's sold earlier this month on the first try in just 13 days. If we.`, + fieldInfo: [] + }, + { + text: `Back then in 2020,2021, even into the beginning of 2022, we saw these historic low interest rates and people were taking out loans then and taking out bigger loans, right? They go and buy a house that they wouldn't have been able to afford. 3% or 4%, but they could afford at 1.25%. They called.`, + fieldInfo: [] + }, + { + text: `No. So let's say there's a young couple in Ontario that bought an average priced house in 2019,$631,990. They put pretty standard down payments, down 20 percents, which leaves you with a mortgage size of about $500,000. Amortize the loan over 25 years, they take a five year fixed rate of 2.9%, which was also kind of standard at the time. That means they've been paying about, and I'm rounding to the nearest dollar here, 2,367 bucks for the past few years. But now that mortgage is up for renewal. Welcome to 2024. And the couple signs on to another fixed rate mortgage, but they have to do it at the bank's current rate, which is around 6%. That means their new monthly payment is, again, I'm rounding here, $3,075 a month or an extra 700 bucks a month. Right.`, + fieldInfo: [] + }, + { + text: `Think of it this way. This circle is your $2,200 a month mortgage payment at a relatively low interest rate pending on the overall size of your mortgage, that might mean $900 of that payment is just servicing interest with the other 13 actually paying your loan back as the interest rate rises. Maybe now instead of 900 dollars a month, it's 20. One hundred dollars a month in interest. But your contract says your total payments stay the same, so you're still paying 22 dollars a month, meaning only 100 of those 22 hundred dollars are actually paying down your loan at that point. I have news for you. You're very close to being underwater, stuck in a perpetual state of paying down a loan where you're only barely paying down that loan. Welcome to lifelong debt, the time it will take to pay off his mortgage nearly doubled from.`, + fieldInfo: [] + }, + { + text: `Collectively, we're just starting to approach the edge of this cliff. RBC estimates about $186 billion worth of mortgages are up for renewal in 2024. Next year, it'll be 350,15 billion. That's a ton of mortgage debt that was taken on at historically low interest rates by people who may or may not have been able to afford getting into the housing market otherwise.`, + fieldInfo: [] + }, + { + text: `What would you do if one day your neighbor, who you've known for years, you see him building a bomb shelter? Would you think that guy's crazy or would you wonder what does he know that I don't? Turns out banks right now are building a bombshelter. Just this quarter, Canada's big six banks have set $4.3 billion aside to cover bad loans. That's almost double what they set aside in the first quarter of last year and more than 11 times what they set aside in the first quarter before that.`, + fieldInfo: [] + }, + { + text: `This is important. About 65% of them, 70% of them have not noticed any radical increase in their payment because their mortgage company sold a product where the payment did not go up when prime went up. Their payment did not go up when prime went up. That's the important part to understand, because it means you might have people ultimately paying more without even realizing it.`, + fieldInfo: [] + }, + { + text: `Now, luckily, most economists do expect rates to keep dropping. According to that same RBC report, it still might not be enough to, quote, save this cohort to get them down to a more manageable monthly increase like 20%. They argue the bank of Canada would have to lower its prime rate way down to around point two five % by July 2026, something they admit is an unreasonable expectation.`, + fieldInfo: [] + }, + { + text: `The old rule of thumb in my day was you spend about a third of your income on housing. Now you'll hear anecdotal evidence, particularly in places like Vancouver, Toronto, where you'll hear 70 to 75% of income is being spent on housing. So in those cases, then if they have to reset at a higher interest rate, it's the house that's gonna go down first. So.`, + fieldInfo: [] + }, + { + text: `In order to get the most accurate reading on the US housing market inventory situation, we need to consider both supply and demand. Supply in this case is the level of inventory for sale and demand is the current case of sales volume. If we start with sales volume, we can see the existing home market has a sales pace of roughly 3.7 million homes per year, while the new home market has a sales pace of 619,000 homes per year. Right off the bat, we can see how the existing home market is almost six times larger than the new construction market in terms of sales volume. Both of these markets, existing and new, have seen a huge reduction in sales volume or buyer demand since interest rates have increased, as affordability is out of reach for most people. If we take this annual sales pace and divided by 12, we can get an idea of the average sales pace per month. Of course, there are seasonal patterns, but if we normalize for those effects, we can see that the current monthly sales pace in the existing home market is roughly 309,000, and the new construction market is selling about 51,000 homes per month. At the peak of the housing boom during the pandemic, the existing hallmark had a sales pace of 500,000 units per month and the new construction market was selling nearly 90,000 units per month. Now that we have an idea of the current level of demand, we have to look at supply or the level of inventory for sale in the existing home market. There are currently 1.1 million homes for sale, one of the lowest on record. If we just look at inventory alone. In the new construction market, there are 479,000 units for sale, one of the highest levels on record, only surpassed by the housing bubble of 2007.`, + fieldInfo: [] + }, + { + text: `You can clearly see how there is nuance to the situation of US housing inventory, the best measure of US housing inventory takes into account both supply and demand and is called the month supply. In other words, how many months of inventory are available for the current pace of sales value. If we take the current level of inventory for both the existing hall market and the new hall market, and we divide it by the current monthly pace of sales. We can see how many months it would take to sell all the inventory at the current pace of sales in the existing home market, that month's supply figure or the inventory level divided by the sales volume, 3.7. In the new construction market, it is 9.3.`, + fieldInfo: [] + }, + { + text: `Generally speaking, the 5.5 to 6.0 level is considered a balanced market where there is no significant upward or downward pressure on prices. When the month's supply level is below 5.5, that means there is very little inventory available for today's market conditions and prices generally rise. On the flip side, a high level of month's supply means there's too much inventory and prices must fall. The existing home market has a month supply of 3.7, which is very low, but it's been increasing since the absolute historic level of one point six in January 2022. Never in the history of the US housing market has the month's supply of existing homes been that low. This created a very unhealthy situation in the existing home market. The inventory situation in the existing hall market is still very tight, but it's now at the highest level since before for the pandemic. The new construction market is a completely different ballgame with a month supply at 9.3, super high and way above the balanced 6.0 level. There is way more detail to the situation in the new construction market that we'll uncover in just a moment. If we look at the total US housing market situation, both existing and new construction, the aggregate month supply figure has increased to 4.5. So yes, the total inventory situation is still very tight on a national level, but the level is rising and at the highest point since 2,015. The monthly numbers are volatile, so let's look at a yearly average to make things more clear. This shows the month's supply for the total US housing market by year. 2024 is a partial year, and the current level as of May 2024 is also noted in the chart. As of May 2024, the total month supply situation is 4.5, which it hasn't been since 2015. You can also see the four years where the total month supply was below 4.0, which is extremely tight. And you can also see the 2021 situation at 2.6, which was crazy unhealthy. One major point is that this current 4.5 level is the national average. But as we know, real estate is very regional. So this means that there are some markets that are near a month supply of 6 and feeling downward price pressure, while some markets are still down at 3 and seeing prices rise with multiple offers. The new home market with a month supply of 9.3 is way above the balance level of six point zero, and there are price cuts and discounts in that market.`, + fieldInfo: [] + }, + { + text: `In June, the National Association of homebuilders reported that 29% of builders cut home prices with an average price reduction of 6%. And 61% of builders use sales incentives like mortgage rate ByteDance to boost sales. But we need to talk about the new hall market and the 9.3 month supply level in a bit more detail, because there is more than what meets BI in the new construction market. Currently, there are 479,000 units for sale in the new construction market and a current pace of sales of about 50 thousand per month. In the new home market, a home could be listed for sale when it's completed, under construction or not yet started. If we look at the percentage of inventory that is completed, we can see that of those 479,000 new home units for sale, Only 20% are completed, which means 80% are either under construction or not yet started.`, + fieldInfo: [] + }, + { + text: `The level of completed new home inventory for sale fell below 10% during the most acute part of the US housing shortage. Normally, in between 20% and 30% of new construction inventory is completed. During really bad downturns like 2008, builders were sitting on almost 50% completed inventory, which is what led to dramatic price cuts and big layoffs of construction crews. The level of completed inventory is now back into the normal range at 20%, and it's rising, which means that if we move towards 30%, price cuts will get more intense and construction crews will be at high risk of job losses. If we take the current 50,000 new home sales pace and divide that by the amount of completed new home inventory for sale. We can see that there's currently about 1.9 months of completed inventory for sale, which is getting past the average level and into the range consistent with recessions and job losses for construction crews. If the home builders have a lot of completed new home inventory for sale, they aren't going to apply for new building permits and build even more homes. And that's exactly what we're starting to see.`, + fieldInfo: [] + }, + { + text: `Homelessness has been rare since the reemergence of homelessness in the mid 1980s. Usually it's been people in their 20s and 30s and 40s, but now we're approaching close to 30% of the adult homeless population are people 55 and over.`, + fieldInfo: [] + }, + { + text: `Exports from China are rising fast and US officials are nervous about this gap. We're not gonna let China flood our market. Sound familiar? There has to be a level playing field for American companies competing in China. In the early 2 Chinese factories like this one pumped out clothes and flags and cars. The sticker prices were cheap. But there was another price to be paid, American jobs. Every single employee of this plant, we'll be out of work by January. 15 years and almost 6 million jobs later, economists are debating what price another wave of imports could exact from American workers. But two key differences between then and now may have a new effect on the US economy. Look at this line. Before 2001, American manufacturing employed over 17 million people, making toys, furniture, paper goods and much more. Then word began to circulate that China was joining the World Trade Organization. And here's what it did. Joining the WTO meant China faced fewer tariffs and restrictions from its trading partners, and the result was dubbed the China.`, + fieldInfo: [] + }, + { + text: `2.5 million Americans lost their jobs from 2000 to 2007. They're represented by the blue on this map. Look at this dark blue area here. That's Silicon Valley, where companies like Apple, HP and Cisco used to manufacture goods. After 2001, they moved most of their production to China, causing a 50+ percent drop in manufacturing jobs in the county, this other dark blue county, Cedar Rapids, Iowa, lost 46% of their manufacturing jobs, primarily in furniture and machinery. But this lighter blue region here was largely spared. It's called Auto Alley. The main reason it succeeded where, say, Silicon Valley's manufacturing failed, is due to investment from competitors like Japan.`, + fieldInfo: [] + } +]; diff --git a/packages/vmind/__tests__/experiment/src/index.scss b/packages/vmind/__tests__/experiment/src/index.scss new file mode 100644 index 00000000..7334a2fc --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/index.scss @@ -0,0 +1,4 @@ +#root { + display: flex; + height: 100vh; +} diff --git a/packages/vmind/__tests__/experiment/src/index.tsx b/packages/vmind/__tests__/experiment/src/index.tsx new file mode 100644 index 00000000..53ddbc53 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/index.tsx @@ -0,0 +1,11 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import App from './App'; +import '@arco-design/web-react/dist/css/arco.min.css'; +import './index.scss'; + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + +); diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx new file mode 100644 index 00000000..2914bfc1 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx @@ -0,0 +1,121 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable no-console */ +import React from 'react'; +import type { FieldInfo } from '../../../../../src/index'; +import { capcutMockData } from '../../data/capcutData'; +import type { TableColumnProps } from '@arco-design/web-react'; +import { Avatar, Button, Card, Checkbox, Divider, Message, Select, Table, Tooltip } from '@arco-design/web-react'; +// import { result } from '../../results/dataExtraction/result'; +// import { result } from '../../results/dataExtraction/result2'; +import { result } from '../../results/dataExtraction/result3'; +import '../page.scss'; +import { IconInfoCircle } from '@arco-design/web-react/icon'; + +const llmList = result.map((v, index) => ({ + index, + llm: v.llm +})); +const datasetMap: Record = { + capcut: capcutMockData +}; +const datasetList = result[0].result.map((v, index) => ({ + index, + name: v.dataset, + dataset: datasetMap[v.dataset] +})); + +export function DataExtractionResult() { + const [datasetIndex, setDatasetIndex] = React.useState(0); + const currentDataset = datasetList[datasetIndex]; + const renderTable = React.useCallback((context: any) => { + const { dataTable, fieldInfo } = context; + const columns: TableColumnProps[] = fieldInfo.map((info: FieldInfo) => ({ + title: ( +
+ +
{`${info.fieldType[0]}__`}
+ + {info.fieldName} + + + +
+ ), + dataIndex: info.fieldName + })); + return ( + + ); + }, []); + + const width = `${(0.8 / llmList.length) * 100}%`; + return ( +
+
+

Please select dataset to anlysis:

+ +
+ +
Comparision Result:
+
+
+
USER INPUT
+ {llmList.map(v => ( +
+ {v.llm} +
+ ))} +
+
+
+
+ {currentDataset.dataset.map((v: any, index: number) => ( + + + {index + 1} + + {v.text} + + ))} +
+
+ {llmList.map(v => { + const llmResult = result[v.index].result[datasetIndex].defaultResult; + return ( +
+
+ {llmResult.map((dataResult, index) => { + return ( + console.log('Context is :', dataResult.context)} + > + {renderTable(dataResult.context)} + + ); + })} +
+
+ ); + })} +
+
+
+ ); +} diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx new file mode 100644 index 00000000..7a5519d4 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx @@ -0,0 +1,195 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable no-console */ +import React from 'react'; +import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; +import { capcutMockData } from '../../data/capcutData'; +import { Button, Checkbox, Divider, Message } from '@arco-design/web-react'; + +const globalVariables = (import.meta as any).env; +const ModelConfigMap: any = { + [Model.DOUBAO_PRO]: { url: globalVariables.VITE_DOUBAO_URL, key: globalVariables.VITE_DOUBAO_KEY }, + [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } +}; +const datasets = [ + { + name: 'capcut', + data: capcutMockData + } +]; + +function getCurrentFormattedTime() { + const now = new Date(); + + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); // 月份从0开始，所以需要加1 + const day = String(now.getDate()).padStart(2, '0'); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + + return `${year}-${month}-${day} ${hours}:${minutes}`; +} + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export function DataExtractionTask() { + const [selectedDataset, setSelectedDataset] = React.useState>( + datasets.reduce((prev, cur) => ({ ...prev, [cur.name]: true }), {}) + ); + const [selectedLLM, setSelectedLLm] = React.useState>( + Object.keys(ModelConfigMap).reduce((prev, cur) => ({ ...prev, [cur]: true }), {}) + ); + const [useDefault, setUseDefault] = React.useState(true); + const [useFieldInfo, setUseFieldInfo] = React.useState(false); + const [messageApi, contextHolder] = Message.useMessage(); + const handleRun = React.useCallback(async () => { + (messageApi as any).info('Run Data Extraction Task!'); + console.info('---------Run Data Extraction Task!---------'); + + const llmResult: any = []; + const llmKeys = Object.keys(ModelConfigMap); + for (let index = 0; index < llmKeys.length; index++) { + const model = llmKeys[index]; + if (!selectedLLM[model]) { + continue; + } + const sleepTime = model === Model.DOUBAO_PRO ? 8000 : 2000; + const apiKey = ModelConfigMap[model]?.key; + const llm = new LLMManage({ + url: ModelConfigMap[model]?.url, + headers: { + 'api-key': apiKey, + Authorization: `Bearer ${apiKey}` + }, + model + }); + const schedule = new Schedule([AtomName.DATA_EXTRACT], { base: { llm, showThoughts: false } }); + (messageApi as any).info(`Begin ${model}!`); + console.info(`---------Begin ${model}---------`); + + const datasetResult: any[] = []; + for (let datasetIndex = 0; datasetIndex < datasets.length; datasetIndex++) { + const dataset = datasets[datasetIndex]; + if (!selectedDataset[dataset.name]) { + continue; + } + (messageApi as any).info(`Begin ${dataset.name} dataset!`); + console.info(`---------Begin ${dataset.name} dataset---------`); + + const defaultResult: any[] = []; + const fieldInfoResult: any[] = []; + for (let dataIndex = 0; dataIndex < dataset.data.length; dataIndex++) { + const data = dataset.data[dataIndex]; + if (useDefault) { + schedule.setNewTask({ + text: data.text + }); + const result = await schedule.run(); + defaultResult.push({ + context: result + }); + await sleep(sleepTime); + } + if (useFieldInfo) { + schedule.setNewTask({ + text: data.text, + fieldInfo: data.fieldInfo + }); + const result = await schedule.run(); + fieldInfoResult.push({ + context: result + }); + await sleep(sleepTime); + } + console.info(`---------Finish dataset_${dataIndex}---------`); + } + datasetResult.push({ + dataset: dataset.name, + defaultResult, + fieldInfoResult + }); + } + + llmResult.push({ + llm: model, + result: datasetResult + }); + } + (messageApi as any).info(`Finish ALL!`); + console.info(`---------Finish ALL---------`); + console.info(llmResult); + // 将 JSON 对象转换为字符串 + const jsonString = JSON.stringify(llmResult, null, 2); + + // 创建一个 Blob 对象 + const blob = new Blob([jsonString], { type: 'application/json' }); + + // 创建一个 URL 对象 + const url = URL.createObjectURL(blob); + + // 创建一个 a 标签并设置相关属性 + const a = document.createElement('a'); + a.href = url; + a.download = `data_${getCurrentFormattedTime()}.json`; + + // 将 a 标签添加到 DOM 并触发点击事件 + document.body.appendChild(a); + a.click(); + + // 移除 a 标签 + document.body.removeChild(a); + + // 释放 URL 对象 + URL.revokeObjectURL(url); + }, [messageApi, selectedDataset, selectedLLM, useDefault, useFieldInfo]); + + return ( +
+ {contextHolder} +
+

DataSet to Test

+ {datasets.map(dataset => { + return ( + setSelectedDataset(prev => ({ ...prev, [dataset.name]: v }))} + > + {dataset.name} + + ); + })} +
+
+

LLM Model To Select

+ {Object.keys(ModelConfigMap).map(modelName => { + return ( + setSelectedLLm(prev => ({ ...prev, [modelName]: v }))} + > + {modelName} + + ); + })} +
+
+

Config of Data Extraction

+ setUseFieldInfo(v)}> + With FieldInfo + + setUseDefault(v)}> + Without FieldInfo + +
+ +
+ +
+
+ ); +} diff --git a/packages/vmind/__tests__/experiment/src/pages/Home.tsx b/packages/vmind/__tests__/experiment/src/pages/Home.tsx new file mode 100644 index 00000000..19e2ff30 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/pages/Home.tsx @@ -0,0 +1,63 @@ +import React from 'react'; +import { Layout, Menu } from '@arco-design/web-react'; +import { CollapseCSS, PLAYGROUND_MENU_INFO, PLAYGROUND_PAGES } from './constants'; +const Sider = Layout.Sider; +const Content = Layout.Content; +const MenuItem = Menu.Item; +const SubMenu = Menu.SubMenu; + +const LOCAL_STORAGE_MENU_KEY = 'VMind_playground_menu_key'; +export const Home: React.FC = props => { + const [selectedPage, setSelectedPage] = React.useState( + localStorage.getItem(LOCAL_STORAGE_MENU_KEY) ?? `${PLAYGROUND_PAGES.DATA_EXTRACTION}_0` + ); + const [collapsed, setCollapsed] = React.useState(false); + const component = React.useMemo(() => { + const [menu, index] = selectedPage.split('_'); + return PLAYGROUND_MENU_INFO[menu]?.subItems?.[Number(index)]?.component; + }, [selectedPage]); + + return ( + + { + type === 'clickTrigger' && setCollapsed(v); + }} + breakpoint="xl" + collapsible + > + { + setSelectedPage(key); + localStorage.setItem(LOCAL_STORAGE_MENU_KEY, key); + }} + > + {Object.keys(PLAYGROUND_MENU_INFO).map(pageName => { + const item = PLAYGROUND_MENU_INFO[pageName]; + return ( + + {item.icon} + {item.menuItem} + + } + > + {item.subItems.map(v => { + return {v.name}; + })} + + ); + })} + + + {component} + + ); +}; diff --git a/packages/vmind/__tests__/experiment/src/pages/constants.tsx b/packages/vmind/__tests__/experiment/src/pages/constants.tsx new file mode 100644 index 00000000..8639d506 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/pages/constants.tsx @@ -0,0 +1,48 @@ +import { IconBgColors, IconBulb, IconLanguage } from '@arco-design/web-react/icon'; +import React from 'react'; +import { DataExtractionTask } from './DataExtraction/test'; +import { DataExtractionResult } from './DataExtraction/caseStudy'; +type MenuInfo = { + menuItem: string; + subItems: { + key: string; + name: string; + component: any; + }[]; + icon: any; + slogan?: string; +}; + +export enum PLAYGROUND_PAGES { + CHART_GENERATION = 'chart_generation', + SMART_INSIGHT = 'smart-insight', + DATA_EXTRACTION = 'data-extraction' +} + +export const PLAYGROUND_MENU_INFO: { + [key: string]: MenuInfo; +} = { + [PLAYGROUND_PAGES.DATA_EXTRACTION]: { + menuItem: 'Data Extraction', + icon: , + subItems: [ + { + key: '0', + name: 'Run Case', + component: + }, + { + key: '1', + name: 'Case Study', + component: + } + ] + } +}; +export const CollapseCSS = { + width: '100vw', + height: '100vh', + border: '1px solid var(--color-border)', + background: 'var(--color-fill-2)', + overflow: 'hidden' +}; diff --git a/packages/vmind/__tests__/experiment/src/pages/page.scss b/packages/vmind/__tests__/experiment/src/pages/page.scss new file mode 100644 index 00000000..47a351c1 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/pages/page.scss @@ -0,0 +1,77 @@ +.case-container { + display: flex; + padding: 16px; + flex-direction: column; + height: 100%; + overflow: hidden; + + .sub-title { + font-size: 16px; + margin-bottom: 4px; + } + .compare { + flex: 1; + width: 100%; + display: flex; + flex-direction: column; + overflow: auto; + + .title { + display: flex; + height: 24px; + width: 100%; + border-bottom: 1px #efefef solid; + text-align: center; + } + + .result { + display: flex; + flex: 1; + overflow: auto; + padding-right: 8px; + } + .one-result { + overflow: visible; + + .result-container { + padding: 4px 12px; + border-right: 1px #efefef solid; + } + .one-card { + height: 400px; + margin: 8px 0; + overflow: auto; + } + .column-title { + display: flex; + align-items: center; + span { + overflow: hidden; + text-overflow: ellipsis; + } + + svg { + flex-shrink: 0; + } + + &:hover { + cursor: pointer; + } + } + } + } +} + +.dataset-selector { + display: flex; + align-items: center; + flex-shrink: 0; + + p { + margin-right: 12px; + } + + .arco-select { + width: 200px; + } +} \ No newline at end of file diff --git a/packages/vmind/__tests__/experiment/tsconfig.json b/packages/vmind/__tests__/experiment/tsconfig.json new file mode 100644 index 00000000..07e477c8 --- /dev/null +++ b/packages/vmind/__tests__/experiment/tsconfig.json @@ -0,0 +1,112 @@ +{ + "compilerOptions": { + "declaration": true, //Generates corresponding d.ts files. + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Projects */ + // "incremental": true, /* Enable incremental compilation */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "ES2018" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, + "lib": [ + "dom", + "es2021" + ] /* Specify a set of bundled library declaration files that describe the target runtime environment. */, + "jsx": "react" /* Specify what JSX code is generated. */, + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ + // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + + /* Modules */ + "module": "ESNext" /* Specify what module code is generated. */, + // "rootDir": "./", /* Specify the root folder within your source files. */ + "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, + "baseUrl": "./" /* Specify the base directory to resolve non-relative module names. */, + "paths": { + /* Specify a set of entries that re-map imports to additional lookup locations. */ "@src/*": ["./src/*"], + "@packages/*": ["./src/packages/*"], + "@containers/*": ["./src/containers/*"] + }, + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + "resolveJsonModule": true /* Enable importing .json files */, + // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, + "esModuleInterop": false /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, + + /* Type Checking */ + "strict": true /* Enable all strict type-checking options. */, + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ + // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ + // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": ["./src"], + "exclude": ["node_modules", "assets"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/packages/vmind/__tests__/experiment/tsconfig.node.json b/packages/vmind/__tests__/experiment/tsconfig.node.json new file mode 100644 index 00000000..42872c59 --- /dev/null +++ b/packages/vmind/__tests__/experiment/tsconfig.node.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/packages/vmind/__tests__/experiment/vite.config.ts b/packages/vmind/__tests__/experiment/vite.config.ts new file mode 100644 index 00000000..1c3dd54e --- /dev/null +++ b/packages/vmind/__tests__/experiment/vite.config.ts @@ -0,0 +1,61 @@ +import { defineConfig, loadEnv } from 'vite'; +import react from '@vitejs/plugin-react'; +import path from 'path'; +import libCss from 'vite-plugin-libcss'; +import dynamicImportVars from '@rollup/plugin-dynamic-import-vars'; + +//import css from 'vite-plugin-css'; +export default defineConfig(({ mode }) => { + const env = loadEnv(mode, process.cwd(), ''); + const proxyConfig = env.VITE_PROXY_CONFIG ? JSON.parse(env.VITE_PROXY_CONFIG) : undefined; + + return { + // root: path.join(__dirname, 'src/site'), + envDir: process.cwd(), + plugins: [ + react(), + libCss(), + //css(), + dynamicImportVars({ + //这里配置插件在那个文件夹内生效 这里是在router文件夹内生效 + include: ['src'], + //这里是哪些文件夹内不生效 + exclude: [], + //插件在遇到错误时会退出构建过程。如果您将此选项设置为 true，它将引发警告，并且保持代码不变。 + warnOnError: false + }) as any + ], + define: { + __DEV__: true + }, + server: { + host: '0.0.0.0', + port: 3300, + headers: { + 'Cross-Origin-Embedder-Policy': 'require-corp', + 'Cross-Origin-Opener-Policy': 'same-origin' + }, + ...proxyConfig + }, + resolve: { + alias: { + //src: path.resolve(__dirname, '../../src'), + '@visactor/calculator': path.resolve(__dirname, '../../../calculator/src/index.ts'), + '@visactor/chart-advisor': path.resolve(__dirname, '../../../chart-advisor/src/index.ts') + // ...localConf.resolve?.alias + } + }, + build: { + cssCodeSplit: true + }, + css: { + preprocessorOptions: { + less: { + math: 'always', + relativeUrls: true, + javascriptEnabled: true + } + } + } + }; +}); From 92f0efc1e1e7ea88ab6d7a0ed278f4e39d8a7970 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Sat, 14 Sep 2024 16:55:30 +0800 Subject: [PATCH 025/128] feat: optimize prompt for date field --- .../vmind/src/atom/dataExtraction/prompt.ts | 9 +++++---- packages/vmind/src/core/llm.ts | 17 ++++++++++++----- packages/vmind/src/types/llm.ts | 8 +++++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/vmind/src/atom/dataExtraction/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt.ts index 7e38643d..17796893 100644 --- a/packages/vmind/src/atom/dataExtraction/prompt.ts +++ b/packages/vmind/src/atom/dataExtraction/prompt.ts @@ -29,7 +29,7 @@ Response: \`\`\``; const getFieldTypeExplanation = (language: 'chinese' | 'english') => { - return `field type explanation is below: 'ratio' means ratio value or percentage(%), such as ${ + return `field type explanation is below: Date data refers to data that can be specified down to the year, quarter, month, week, or day.'ratio' means ratio value or percentage(%), such as ${ language === 'english' ? 'YoY or MoM' : '同比、环比、增长率、占比等' }.The forms of ratio data are usually Percentage (%) such as 60%.'count' means count data`; }; @@ -44,7 +44,6 @@ export const getBasePrompt = ( 3. ALWAYS generate a field type, chosen from 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count';${getFieldTypeExplanation( language )} -4. ALWASY display time fields using the same time format.Different date fields can have different formats. ${dataTableExplanation} You should think step-by-step as follow: @@ -55,7 +54,7 @@ You should think step-by-step as follow: 3. Read the entire text and fields with numerical or ratio or count field type first. 4. Read all text again and generate field information associated with the fields found in Step3.The newly generated fields are all simple. 5. Finally, read all text and extract all corresponding data table based on the field information. -6. Adjust the data to ensure consistency within the same field especially time field. +6. When a date field contains data with multiple date granularities, convert the fieldType to string. 7. Assume the data is incomplete, then reconsider and execute the task again. Response in the following format: @@ -67,6 +66,7 @@ fieldInfo: { fieldName: string; //name of the field. description?: string; //description of the field. fieldType?: 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count'; // type of field +dateGranularity?: 'year' | 'quarter' | 'month' | 'week' | 'day'; // generate when fieldType is 'date', represent the date granularity of date time }[], dataTable: Record\[]; // Extracted data set, key of dataTable is fieldName in fieldInfo } @@ -82,7 +82,8 @@ Finish your tasks in one-step. 2. The numbers in the dataset do not carry any units. 3. Keep the ratio value unchanged, such as '95%' --> '95' 4. Only use numbers that appear in the text. -5. If you do not know the value of an field, return null for the field's value.`; +5. If you do not know the value of an field, return null for the field's value. +6. If it is a date field, standardize the data format according to the date granularity into forms such as the following: yyyy-mm-dd | mm-dd | mm | yyyy-mm | yyyy-qq.`; export const getFieldInfoPrompt = ( language: 'chinese' | 'english', diff --git a/packages/vmind/src/core/llm.ts b/packages/vmind/src/core/llm.ts index 5bf4c04f..816cc10d 100644 --- a/packages/vmind/src/core/llm.ts +++ b/packages/vmind/src/core/llm.ts @@ -25,7 +25,8 @@ export class LLMManage { method: 'POST', model: Model.DOUBAO_PRO, maxTokens: 1024, - temperature: 0 + temperature: 0, + frequency_penalty: 0 }; } @@ -34,7 +35,7 @@ export class LLMManage { } async run(name: AtomName, messages: LLMMessage[]) { - const { url, headers, method, maxTokens, temperature, model } = this.options; + const { url, headers, method, maxTokens, temperature, model, frequency_penalty } = this.options; if (!this.historys[name]) { this.historys[name] = []; } @@ -48,7 +49,9 @@ export class LLMManage { messages, max_tokens: maxTokens, temperature, - stream: false + stream: false, + frequency_penalty + // seedSwitch: false // Only models after gpt-3.5-turbo-1106 support this parameter. // response_format: // { "type": "json_object" } @@ -62,12 +65,16 @@ export class LLMManage { }); if (res.error) { console.error(res.error); - return {}; + return { + error: res.error + }; } return res; } catch (err: any) { console.error(err); - return err.response.data; + return { + error: err + }; } } diff --git a/packages/vmind/src/types/llm.ts b/packages/vmind/src/types/llm.ts index 345074cb..a6e24214 100644 --- a/packages/vmind/src/types/llm.ts +++ b/packages/vmind/src/types/llm.ts @@ -38,6 +38,8 @@ export interface ILLMOptions { temperature?: number; /** show llm thoughs or not */ showThoughts?: boolean; + /** repetition penalty */ + frequency_penalty?: number; } /** LLM Messages api */ @@ -49,11 +51,11 @@ export interface LLMMessage { /** LLM Response API */ export interface LLMResponse extends BaseContext { - choices: { + choices?: { index: number; message: any; }[]; - usage: any; - error: string; + usage?: any; + error?: string; [key: string]: any; } From 59a62e5b3f1b11217decc4f50768aca90628c4d5 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Sat, 14 Sep 2024 16:56:29 +0800 Subject: [PATCH 026/128] feat: distinguish query and generate --- .../src/pages/NewDataExtraction/DataInput.tsx | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx index cbfff0c8..dabc4868 100644 --- a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx @@ -31,7 +31,7 @@ export function DataInput(props: IPropsType) { const [userInput, setUserInput] = useState(capcutMockData[defaultIndex].input); const [model, setModel] = useState(Model.GPT_4o); - const [useFieldInfo, setUseFieldInfo] = useState(true); + const [useFieldInfo, setUseFieldInfo] = useState(false); const [showThoughts, setShowThoughts] = useState(false); const [visible, setVisible] = React.useState(false); const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); @@ -45,7 +45,8 @@ export function DataInput(props: IPropsType) { 'api-key': apiKey, Authorization: `Bearer ${apiKey}` }, - model + model, + maxTokens: 2048 }) ); const schedule = React.useRef>( @@ -154,9 +155,20 @@ export function DataInput(props: IPropsType) { - +
+ + +
@@ -170,9 +182,6 @@ export function DataInput(props: IPropsType) {
- Date: Wed, 18 Sep 2024 11:36:22 +0800 Subject: [PATCH 027/128] feat: Added chart type - time series chart. --- .../browser/src/constants/mockData.ts | 114 ++++++++++++++ .../src/pages/ChartGeneration/DataInput.tsx | 6 +- .../skylark/prompt/knowledge.ts | 10 ++ .../skylark/prompt/knowledge.ts | 18 +++ .../GPT/prompt/knowledges.ts | 8 +- .../getChartSpec/VChart/chartPipeline.ts | 9 +- .../getChartSpec/VChart/transformers.ts | 149 +++++++++++++++++- packages/vmind/src/common/typings/index.ts | 3 +- 8 files changed, 308 insertions(+), 9 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 5cbf1e7e..0f89fa9e 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -9737,6 +9737,120 @@ Italy,10,2020`, input: '使用动态玫瑰图帮我展示各国GDP排名变化' }; +export const sequenceData = { + csv: `playName,time,eventType +Deandre Ayton,-2209017943000,start +Deandre Ayton,-2209017476000,end +Deandre Ayton,-2209016998000,start +Deandre Ayton,-2209016057000,end +Deandre Ayton,-2209015551000,start +Deandre Ayton,-2209015127000,end +Deandre Ayton,-2209015116000,start +Deandre Ayton,-2209015063000,end +Devin Booker,-2209017943000,start +Devin Booker,-2209017293000,end +Devin Booker,-2209016892000,start +Devin Booker,-2209015783000,end +Devin Booker,-2209015380000,start +Devin Booker,-2209015063000,end +Kyle Lowry,-2209017943000,start +Kyle Lowry,-2209017476000,end +Kyle Lowry,-2209017223000,start +Kyle Lowry,-2209016762000,end +Kyle Lowry,-2209016614000,start +Kyle Lowry,-2209016057000,end +Kyle Lowry,-2209015884000,start +Kyle Lowry,-2209015063000,end +Jae Crowder,-2209017943000,start +Jae Crowder,-2209017293000,end +Jae Crowder,-2209016998000,start +Jae Crowder,-2209016139000,end +Jae Crowder,-2209015651000,start +Jae Crowder,-2209015063000,end +Aron Baynes,-2209017943000,start +Aron Baynes,-2209017476000,end +Aron Baynes,-2209016503000,start +Aron Baynes,-2209016166000,end +Pascal Siakam,-2209016892000,start +Pascal Siakam,-2209015788000,end +Pascal Siakam,-2209015528000,start +Pascal Siakam,-2209015063000,end +Pascal Siakam,-2209017943000,start +Pascal Siakam,-2209017223000,end +Mikal Bridges,-2209017943000,start +Mikal Bridges,-2209017650000,end +Mikal Bridges,-2209016998000,start +Mikal Bridges,-2209016762000,end +Mikal Bridges,-2209016503000,start +Mikal Bridges,-2209016003000,end +Mikal Bridges,-2209015783000,start +Mikal Bridges,-2209015063000,end +Chris Paul,-2209017943000,start +Chris Paul,-2209017476000,end +Chris Paul,-2209016998000,start +Chris Paul,-2209016057000,end +Chris Paul,-2209015551000,start +Chris Paul,-2209015063000,end +OG Anunoby,-2209017943000,start +OG Anunoby,-2209017476000,end +OG Anunoby,-2209017223000,start +OG Anunoby,-2209016614000,end +OG Anunoby,-2209016503000,start +OG Anunoby,-2209016021000,end +OG Anunoby,-2209015788000,start +OG Anunoby,-2209015063000,end +Fred VanVleet,-2209016892000,start +Fred VanVleet,-2209015884000,end +Fred VanVleet,-2209015651000,start +Fred VanVleet,-2209015063000,end +Fred VanVleet,-2209017943000,start +Fred VanVleet,-2209017223000,end +Cameron Johnson,-2209017650000,start +Cameron Johnson,-2209016998000,end +Cameron Johnson,-2209016762000,start +Cameron Johnson,-2209016503000,end +Cameron Johnson,-2209016139000,start +Cameron Johnson,-2209015651000,end +Yuta Watanabe,-2209017476000,start +Yuta Watanabe,-2209016892000,end +Yuta Watanabe,-2209016021000,start +Yuta Watanabe,-2209015651000,end +Dario Saric,-2209017476000,start +Dario Saric,-2209016998000,end +Dario Saric,-2209016057000,start +Dario Saric,-2209015551000,end +Dario Saric,-2209015127000,start +Dario Saric,-2209015116000,end +Chris Boucher,-2209017476000,start +Chris Boucher,-2209016998000,end +Chris Boucher,-2209015962000,start +Chris Boucher,-2209015528000,end +Norman Powell,-2209017476000,start +Norman Powell,-2209017119000,end +Norman Powell,-2209016762000,start +Norman Powell,-2209016547000,end +Norman Powell,-2209016057000,start +Norman Powell,-2209015063000,end +Cameron Payne,-2209017476000,start +Cameron Payne,-2209016998000,end +Cameron Payne,-2209016057000,start +Cameron Payne,-2209015551000,end +Langston Galloway,-2209017293000,start +Langston Galloway,-2209016998000,end +Jevon Carter,-2209017293000,start +Jevon Carter,-2209016892000,end +Jevon Carter,-2209016003000,start +Jevon Carter,-2209015380000,end +Malachi Flynn,-2209017119000,start +Malachi Flynn,-2209016892000,end +Alex Len,-2209016998000,start +Alex Len,-2209016503000,end +Alex Len,-2209016166000,start +Alex Len,-2209015962000,end +DeAndre' Bembry,-2209016547000,start +DeAndre' Bembry,-2209015783000,end`, + input: '帮我展示各运动员在比赛中的行动记录。' +}; export const mockUserTextInput0 = { text: `快手消失了。快手上市后，市值一度超过2000亿美元，现在只剩200多亿美元。去年快手的营收破了千亿，公司也赚钱了，但市场不买账了。 滴滴也不见了。滴滴去年营收接近2000亿元，并首次实现年度盈利，不过滴滴从美股退市后，一直没在港股上市，所以没有市值参考。`, diff --git a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx index e744de0b..7a4caab4 100644 --- a/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/ChartGeneration/DataInput.tsx @@ -56,7 +56,8 @@ import { singleColumnBarCombinationChartData, dynamicScatterPlotData, dynamicRoseData, - dynamicRoseData1 + dynamicRoseData1, + sequenceData } from '../../constants/mockData'; import VMind, { ArcoTheme, builtinThemeMap, BuiltinThemeType } from '../../../../../src/index'; import { Model } from '../../../../../src/index'; @@ -120,7 +121,8 @@ const demoDataList: { [key: string]: any } = { SingleColumnBarCommon1: singleColumnBarCombinationChartData1, dynamicScatterPlotData: dynamicScatterPlotData, dynamicRoseData: dynamicRoseData, - dynamicRoseData1: dynamicRoseData1 + dynamicRoseData1: dynamicRoseData1, + sequenceData: sequenceData }; const globalVariables = (import.meta as any).env; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts index aced9307..6ec85a11 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts @@ -140,5 +140,15 @@ export const chartKnowledgeBase: ChartKnowledgeBase = { constraints: [ 'Use Dynamic Rose Chart if you want to show cyclical data and observe changes in multiple categories over time.' ] + }, + [ChartType.SequenceChart]: { + knowledge: [ + 'Sequence Chart visualizes events in chronological order along a time axis.', + 'Sequence Chart is ideal for showing the progression of time-based events.' + ], + constraints: [ + 'Use Sequence Chart when the data contains a sequence of events that are ordered by time.', + 'Sequence Chart requires a continuous time field in the data for accurate rendering.' + ] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts index bac963ab..c7dfabe6 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts @@ -406,5 +406,23 @@ export const ChartFieldInfo: ChannelInfo = { 'All visual channels must reflect the structure of the data presented.', 'Dynamic Rose Chart is useful for visualizing cyclical patterns, where radius represents the magnitude, color distinguishes categories, and time drives the dynamic updates.' ] + }, + [ChartType.SequenceChart.toUpperCase()]: { + visualChannels: { + x: "x channel of sequence chart. Represents the initiator of the timeline or event, showing which entity or individual is involved in each sequence. Can't be empty.", + y: "y channel of sequence chart. Represents the timeline or time series, displaying the chronological order of events. This is essential for showing when events occur. Often a numeric value. Can't be empty.", + color: + 'color channel of sequence chart. Differentiates the types or categories of events within the timeline. It is useful for distinguishing between different types of events in the sequence. For example: start or end' + }, + responseDescription: { + x: 'field assigned to x channel, representing the initiator of the timeline or event', + y: 'field assigned to y channel, typically used for time progression', + color: 'field assigned to color channel, representing event types' + }, + knowledge: [ + 'All visual channels must be aligned with the structure of the data being visualized.', + 'Sequence charts are ideal for displaying event sequences over time, where the y-axis represents time progression, the x-axis shows the initiator of the timeline, and color helps distinguish between event types.', + 'This chart is useful for visualizing workflows, event timelines, or sequences where time, initiators, and event types need to be clearly represented.' + ] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts index 87d866e0..6aee2b9c 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts @@ -258,7 +258,7 @@ export const chartKnowledgeDict: ChartKnowledge = { index: 23, visualChannels: ['y', 'x', 'size'], examples: [], - knowledge: [] + knowledge: ['The three channels that need to be mapped in the Basic Heat Map are: x, y, size;'] }, [ChartType.VennChart]: { index: 24, @@ -287,6 +287,12 @@ export const chartKnowledgeDict: ChartKnowledge = { visualChannels: ['color', 'radius', 'time'], examples: [], knowledge: ['The three channels that need to be mapped in the dynamic rose chart are: color, radius, and time;'] + }, + [ChartType.SequenceChart]: { + index: 30, + visualChannels: ['x', 'y', 'color'], + examples: [], + knowledge: ['The three channels that need to be mapped in the sequence chart are: x, y, and color;'] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts index c8a31a62..cd6db20e 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/chartPipeline.ts @@ -88,7 +88,10 @@ import { dynamicScatterPlotTooltip, dynamicRoseAnimation, dynamicRoseField, - dynamicRoseDisplayConf + dynamicRoseDisplayConf, + sequenceChartData, + sequenceChartSeries, + sequenceChartAxes } from './transformers'; const pipelineBar = [ @@ -265,6 +268,7 @@ const pipelineDynamicRoseChart = [ customMark, theme ]; +const pipelineSequenceChart = [chartType, sequenceChartData, sequenceChartSeries, sequenceChartAxes, theme]; const pipelineMap: { [chartType: string]: any } = { [ChartType.BarChart.toUpperCase()]: pipelineBar, @@ -293,7 +297,8 @@ const pipelineMap: { [chartType: string]: any } = { [ChartType.VennChart.toUpperCase()]: pipelineVenn, [ChartType.SingleColumnCombinationChart.toUpperCase()]: pipelineSingleColumnCombinationChart, [ChartType.DynamicScatterPlotChart.toUpperCase()]: pipelineDynamicScatterPlotChart, - [ChartType.DynamicRoseChart.toUpperCase()]: pipelineDynamicRoseChart + [ChartType.DynamicRoseChart.toUpperCase()]: pipelineDynamicRoseChart, + [ChartType.SequenceChart.toUpperCase()]: pipelineSequenceChart }; export const beforePipe: Transformer = ( diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index 6c662cab..cc148b0f 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -20,8 +20,8 @@ import { import { getFieldByDataType } from '../../../../../common/utils/utils'; import { array, isArray } from '@visactor/vutils'; import { isValidDataset } from '../../../../../common/dataProcess'; -import type { VMindDataset } from '../../../../../common/typings'; -import { CombinationBasicChartType, ChartType, DataType } from '../../../../../common/typings'; +import type { DataItem, VMindDataset } from '../../../../../common/typings'; +import { ChartType, CombinationBasicChartType, DataType } from '../../../../../common/typings'; import { builtinThemeMap } from '../../../../../common/builtinTheme'; import { FOLD_NAME, FOLD_VALUE, COLOR_FIELD } from '@visactor/chart-advisor'; import { CARTESIAN_CHART_LIST } from '../../../constants'; @@ -59,7 +59,8 @@ const chartTypeMap: { [chartName: string]: string } = { [ChartType.VennChart.toUpperCase()]: 'venn', [ChartType.SingleColumnCombinationChart.toUpperCase()]: 'common', [ChartType.DynamicScatterPlotChart.toUpperCase()]: 'common', - [ChartType.DynamicRoseChart.toUpperCase()]: 'rose' + [ChartType.DynamicRoseChart.toUpperCase()]: 'rose', + [ChartType.SequenceChart.toUpperCase()]: 'sequence' }; export const chartType: Transformer = (context: Context) => { @@ -2303,3 +2304,145 @@ export const dynamicRoseDisplayConf: Transformer = spec.startAngle = -90; return { spec }; }; + +export const sequenceChartData: Transformer = (context: Context) => { + const { cells, dataset, spec } = context; + const cell = getCell(cells); + const yField = cell.y as string; + const xField = cell.x; + const colorField = cell.color as string; + const dataMap: { [key: string]: DataItem[] } = {}; + dataset.forEach(data => { + dataMap[data[xField]] ? dataMap[data[xField]].push(data) : (dataMap[data[xField]] = [data]); + }); + const dataDot: { [key: string]: DataItem[] | string }[] = []; + const dataLink: { [key: string]: string }[] = []; + Object.keys(dataMap).forEach(key => { + dataDot.push({ + [cell.x]: key, + dots: dataMap[key] + }); + sortArray(dataMap[key], [{ field: yField, order: SortOrder.ASC }]).forEach((data, index, array) => { + if (index % 2 !== 0) { + dataLink.push({ + from: `${data[xField]}_${Math.floor(index / 2).toString()}_${array[index - 1][colorField]}_node`, + to: `${data[xField]}_${Math.floor(index / 2).toString()}_${data[colorField]}_node` + }); + } + data.node_name = `${data[xField]}_${Math.floor(index / 2).toString()}_${data[colorField]}_node`; + return data; + }); + }); + spec.data = [ + { id: 'dataDotSeries', values: dataDot }, + { id: 'dataLinkSeries', values: dataLink } + ]; + return { spec }; +}; + +export const sequenceChartSeries: Transformer = (context: Context) => { + const { cells, spec } = context; + const cell = getCell(cells); + + spec.series = [ + { + type: 'link', + dataId: 'dataLinkSeries', + dotSeriesIndex: 1, + fromField: 'from', + toField: 'to', + arrow: { + style: { + visible: false + } + } + }, + { + type: 'dot', + dataId: 'dataDotSeries', + xField: cell.y as string, + yField: cell.x, + dotTypeField: cell.color as string, + titleField: cell.x, + highLightSeriesGroup: '', + height: 500, + clipHeight: 800, + title: { + style: { + fill: 'rgba(46, 47, 50)' + } + }, + subTitle: { + style: { + fill: 'rgba(46, 47, 50)', + dy: 7 + } + }, + grid: { + style: { + visible: false + } + }, + symbol: { + style: { + visible: false + } + }, + tooltip: { + mark: { + title: { + key: 'event 信息', + value: 'event 信息' + }, + content: [ + { + hasShape: true, + shapeType: 'square', + key: (datum: any) => datum[cell.x] + }, + { + hasShape: false, + key: 'event_time_stamp', + value: (datum: any) => datum[cell.color as string] + } + ] + } + } + } + ]; + return { spec }; +}; +export const sequenceChartAxes: Transformer = (context: Context) => { + const { cells, spec, fieldInfo } = context; + const cell = getCell(cells); + + spec.axes = [ + { + orient: 'top', + type: 'time', + range: { + min: fieldInfo.filter(fieldInfo => { + return fieldInfo.fieldName === cell.y; + })[0].domain[0], + max: fieldInfo.filter(fieldInfo => { + return fieldInfo.fieldName === cell.y; + })[0].domain[1] + }, + layers: [ + { + tickStep: 28800, + timeFormat: '%Y%m%d' + }, + { + tickStep: 28800, + timeFormat: '%H:%M' + } + ] + } + ]; + spec.appendPadding = { + left: 80, + right: 80 + }; + return { spec }; +}; diff --git a/packages/vmind/src/common/typings/index.ts b/packages/vmind/src/common/typings/index.ts index c5f5bfd1..0700a3d5 100644 --- a/packages/vmind/src/common/typings/index.ts +++ b/packages/vmind/src/common/typings/index.ts @@ -86,7 +86,8 @@ export enum ChartType { VennChart = 'Venn Chart', SingleColumnCombinationChart = 'Single Column Combination Chart', DynamicScatterPlotChart = 'Dynamic Scatter Plot Chart', - DynamicRoseChart = 'Dynamic Rose Chart' + DynamicRoseChart = 'Dynamic Rose Chart', + SequenceChart = 'Sequence Chart' } export enum CombinationChartType { From 40855e74a5070dffb0f85bec54486dadc10f1425 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 20 Sep 2024 10:06:32 +0800 Subject: [PATCH 028/128] feat: optimize prompt and atom/schedule process --- packages/vmind/src/atom/dataClean/index.ts | 10 ++- .../vmind/src/atom/dataExtraction/index.ts | 17 ++-- .../vmind/src/atom/dataExtraction/prompt.ts | 87 ++++++++++++------- packages/vmind/src/atom/type.ts | 4 + packages/vmind/src/schedule/index.ts | 4 +- packages/vmind/src/utils/field.ts | 18 ++++ packages/vmind/src/utils/json.ts | 6 +- 7 files changed, 104 insertions(+), 42 deletions(-) create mode 100644 packages/vmind/src/utils/field.ts diff --git a/packages/vmind/src/atom/dataClean/index.ts b/packages/vmind/src/atom/dataClean/index.ts index 73be1e5b..65bfaa8d 100644 --- a/packages/vmind/src/atom/dataClean/index.ts +++ b/packages/vmind/src/atom/dataClean/index.ts @@ -1,8 +1,9 @@ import type { DataCleanCtx } from '../../types/atom'; -import { AtomName, type DataExtractionCtx } from '../../types/atom'; +import { AtomName, ROLE, type DataExtractionCtx } from '../../types/atom'; import type { DataCleanOptions } from '../type'; import { BaseAtom } from '../base'; import { merge, pick } from '@visactor/vutils'; +import { getRoleByFieldType } from '../../utils/field'; export class DataCleanAtom extends BaseAtom { name = AtomName.DATA_CLEAN; @@ -41,7 +42,7 @@ export class DataCleanAtom extends BaseAtom { if (filterSameValueColumn && dataTable.length > 1 && fieldInfo.length) { const cleanFieldKey: string[] = []; fieldInfo.forEach(info => { - if (info.fieldType === 'numerical') { + if ((info.role ?? getRoleByFieldType(info.fieldType)) === ROLE.MEASURE) { return; } let shouldFilter = true; @@ -60,7 +61,10 @@ export class DataCleanAtom extends BaseAtom { newContext.dataTable = dataTable.map(dataItem => pick(dataItem, fieldNameList)); } } - if (needNumericalFields && newContext.fieldInfo.findIndex(info => info?.fieldType === 'numerical') === -1) { + if ( + needNumericalFields && + newContext.fieldInfo.findIndex(info => (info.role ?? getRoleByFieldType(info.fieldType)) === ROLE.MEASURE) === -1 + ) { newContext = { dataTable: [], fieldInfo: [] diff --git a/packages/vmind/src/atom/dataExtraction/index.ts b/packages/vmind/src/atom/dataExtraction/index.ts index 1b0208f0..8b5c94f7 100644 --- a/packages/vmind/src/atom/dataExtraction/index.ts +++ b/packages/vmind/src/atom/dataExtraction/index.ts @@ -1,12 +1,13 @@ import { AtomName, type DataExtractionCtx } from '../../types/atom'; -import type { BaseOptions } from '../type'; +import type { BaseOptions, DataExtractionOptions } from '../type'; import { BaseAtom } from '../base'; import { merge, pick } from '@visactor/vutils'; import type { LLMMessage } from '../../types/llm'; import { getBasePrompt, getFieldInfoPrompt } from './prompt'; import { getLanguageOfText } from '../../utils/text'; +import { formatFieldInfo } from '../../utils/field'; -export class DataExtractionAtom extends BaseAtom { +export class DataExtractionAtom extends BaseAtom { name = AtomName.DATA_EXTRACT; isLLMAtom = true; @@ -32,7 +33,7 @@ export class DataExtractionAtom extends BaseAtom getLLMMessages(query?: string): LLMMessage[] { const { fieldInfo, text } = this.context; - const { showThoughts } = this.options; + const { showThoughts, reGenerateFieldInfo } = this.options; const addtionContent = this.getHistoryLLMMessages(query); const language = getLanguageOfText(text); if (!fieldInfo || !fieldInfo?.length) { @@ -43,7 +44,7 @@ export class DataExtractionAtom extends BaseAtom }, { role: 'user', - content: `${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：'}:${text}` + content: `${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：'}${text}` }, ...addtionContent ]; @@ -56,12 +57,12 @@ export class DataExtractionAtom extends BaseAtom \`\`\` TypeScript ${fieldInfoString} \`\`\` -${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：'}:${text} +${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：'}${text} `; return [ { role: 'system', - content: getFieldInfoPrompt(language, showThoughts) + content: getFieldInfoPrompt(language, showThoughts, reGenerateFieldInfo) }, { role: 'user', @@ -79,7 +80,9 @@ ${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：' } return { ...this.context, - fieldInfo: fieldInfo ?? this.context?.fieldInfo ?? [], + fieldInfo: formatFieldInfo( + (this.options?.reGenerateFieldInfo ? fieldInfo : null) ?? this.context?.fieldInfo ?? [] + ), dataTable } as DataExtractionCtx; } diff --git a/packages/vmind/src/atom/dataExtraction/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt.ts index 17796893..1120137b 100644 --- a/packages/vmind/src/atom/dataExtraction/prompt.ts +++ b/packages/vmind/src/atom/dataExtraction/prompt.ts @@ -1,32 +1,47 @@ /* eslint-disable max-len */ const dataTableExplanation = `# Data Table Explanation 1. ALWAYS generate flatten data table rather than unflatten data table - -# Flatten Data Table Example +## Flatten Data Table Example \`\`\` dataTable: [{ date: "Monday", class: "class No.1", score: 20 },{ date: "Monday", class: "class No.2", score: 30 },{ date: "Tuesday", class: "class No.1", score: 25 },{ date: "Tuesday", class: "class No.2", score: 28 }] \`\`\` - -# Unflatten Data Table Example +## Unflatten Data Table Example \`\`\` dataTable: [{date: "Monday", class No.1: 20, class No.2: 30},{date: "Tuesday", class No.1: 25, class No.2: 28}] \`\`\``; -const baseExamples = `# Examples1: +const baseExamples = `# Examples1 text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 Response: \`\`\` {"fieldInfo:":[{"fieldName":"公司","description":"公司名称","fieldType":"string",},{"fieldName":"月份","description":"具体月份","fieldType":"string",},{"fieldName":"利润调整","description":"是否经过利润调整","fieldType":"string",},{"fieldName":"利润额","description":"利润总额","fieldType":"numerical",}],"dataTable":[{"公司":"阿里","月份":"5月","利润调整":"调整前","利润额":100000000000,},{"公司":"阿里","月份":"5月","利润调整":"调整后","利润额":10000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整前","利润额":80000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整后","利润额":12000000000,},]} - -# Examples2: - +\`\`\` +# Examples2 text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. Response: \`\`\` {"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string",},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"ratio"}],"dataTable":[{"name":"John Smith","ranking":90,},{"name":"Jane Doe","ranking":75}]} -\`\`\``; +\`\`\` +# Examples3 +text: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%。 + +Response: +\`\`\` +{"fieldInfo:":[{"fieldName":"年份","description":"数据对应时间","fieldType":"date",dateGranularity:"year"},{"fieldName":"入睡困难占比","description":"年轻人入睡困呐占总人数的比例","fieldType":"ratio"}],"dataTable":[{"年份":"2024","占比":[0.6,0.7],},{"年份":"2022","占比":0.3}]} +\`\`\` +`; + +const getCommonInfomation = (language: 'chinese' | 'english') => + `# Common Information +${ + language === 'chinese' + ? `1. 今年是${new Date().getFullYear()}年 +2. 8.5折和85折含义相同，都代表85%的折扣` + : `1. This year is ${new Date().getFullYear()}` +} +`; const getFieldTypeExplanation = (language: 'chinese' | 'english') => { return `field type explanation is below: Date data refers to data that can be specified down to the year, quarter, month, week, or day.'ratio' means ratio value or percentage(%), such as ${ @@ -36,8 +51,7 @@ const getFieldTypeExplanation = (language: 'chinese' | 'english') => { export const getBasePrompt = ( language: 'chinese' | 'english', showThoughs: boolean = true -) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, dates, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. - +) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, dates data, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. # Field Information Explanation 1. ALWAYS generate a field information, which represents the specific information of each column field in the data table. 2. ALWAYS generate a field description @@ -45,17 +59,19 @@ export const getBasePrompt = ( language )} ${dataTableExplanation} - -You should think step-by-step as follow: +${getCommonInfomation(language)} # Steps +You should think step-by-step as follow: + 0. using language answer: ${language} 1. Determine whether the current task is related to data extraction. 2. If not, return isDataExtraction is false in json mode; If yes, continue follow Steps 3. Read the entire text and fields with numerical or ratio or count field type first. 4. Read all text again and generate field information associated with the fields found in Step3.The newly generated fields are all simple. -5. Finally, read all text and extract all corresponding data table based on the field information. +5. Read all text and extract all corresponding data table based on the field information. The data corresponding to a field should always be concise, and a field should express only one meaning. 6. When a date field contains data with multiple date granularities, convert the fieldType to string. -7. Assume the data is incomplete, then reconsider and execute the task again. +7. Only use numbers that appear in the text, Do not perform any calculations or numerical conversion such as currency conversion calculation. +8. Assume the data is incomplete, then reconsider and execute the task again. Response in the following format: \`\`\` @@ -68,27 +84,26 @@ description?: string; //description of the field. fieldType?: 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count'; // type of field dateGranularity?: 'year' | 'quarter' | 'month' | 'week' | 'day'; // generate when fieldType is 'date', represent the date granularity of date time }[], -dataTable: Record\[]; // Extracted data set, key of dataTable is fieldName in fieldInfo +dataTable: Record[]; // Extracted data set, key of dataTable is fieldName in fieldInfo; The type is number[] if and only if current data is range data. } \`\`\` ${baseExamples} ----------------------------------- - -You only need to return the JSON in your response directly to the user. -Finish your tasks in one-step. +--- +You only need to return the JSON in your response directly to the user.Finish your tasks in one-step. # Constraints: 1. Strictly define the type of return format, use JSON format to reply, do not include any extra content. 2. The numbers in the dataset do not carry any units. -3. Keep the ratio value unchanged, such as '95%' --> '95' -4. Only use numbers that appear in the text. -5. If you do not know the value of an field, return null for the field's value. -6. If it is a date field, standardize the data format according to the date granularity into forms such as the following: yyyy-mm-dd | mm-dd | mm | yyyy-mm | yyyy-qq.`; +3. Only extract value in ratio type, such as '95%' --> '95'; 'reduce 30%' --> '-30' +4. If you do not know the value of an field, return null for the field's value. +5. If it is a date field, standardize the data format according to the date granularity into forms such as the following: yyyy-mm-dd | mm-dd | mm | yyyy-mm | yyyy-qq. +6. The change in values should be reflected in the positive or negative nature of the data, not in the field names.`; export const getFieldInfoPrompt = ( language: 'chinese' | 'english', - showThoughs: boolean = true -) => `You are an expert extraction algorithm and are highly sensitive to comparative data, trend data, and similar information.Only extract relevant information from the text. Your goal is to extract structured information from the user's input that matches the form described below. When extracting information please make sure it matches the type information exactly. + showThoughs: boolean = true, + reGenerateFieldInfo: boolean = false +) => `You are an expert extraction algorithm and are highly sensitive to comparative data, trend data, date data and similar information.Only extract relevant information from the text. Your goal is to extract structured information from the user's input that matches the form described below. When extracting information please make sure it matches the type information exactly. The definition of the field information is as follows. \`\`\` fieldInfo: { @@ -101,6 +116,7 @@ dataExample?: (string | number)[] // data example of this field }[] \`\`\` +${getCommonInfomation(language)} You should think step-by-step as follow: # Steps 0. using language answer: ${language} @@ -108,7 +124,8 @@ You should think step-by-step as follow: 2. If not, return isDataExtraction is false in json mode; If yes, continue follow Steps 3. Read all text and extract all corresponding data table based on the field information. 4. Adjust the data to ensure consistency within the same field especially time field. -5. Assume the data is incomplete, then reconsider and execute the task again. +5. Only use numbers that appear in the text, Do not perform any calculations or numerical conversion such as currency conversion calculation. +6. Assume the data is incomplete, then reconsider and execute the task again. # Respones Response in the following format: @@ -116,6 +133,16 @@ Response in the following format: { isDataExtraction: boolean; // current task is data extraction or not ${showThoughs ? 'thoughts: string, // your thought process' : ''} +${ + reGenerateFieldInfo + ? `fieldInfo: { + fieldName: string; //name of the field. + description?: string; //description of the field. + fieldType?: 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count'; // type of field + dateGranularity?: 'year' | 'quarter' | 'month' | 'week' | 'day'; // generate when fieldType is 'date', represent the date granularity of date time + }[]` + : '' +} dataTable: Record\[]; // Extracted data set, key of dataTable is fieldName in user's fieldInfo } \`\`\` @@ -146,6 +173,8 @@ Finish your tasks in one-step. 1. Strictly define the type of return format, use JSON format to reply, do not include any extra content. 2. The numbers in the dataset do not carry any units. 3. Only use numbers that appear in the text. -4. If you do not know the value of an field, return null for the field's value. -5. Do not add any attributes that do not appear in the user's fieldInfo. +4. Only extract value in ratio type, such as '95%' --> '95'; 'reduce 30%' --> '-30' +5. If you do not know the value of an field, return null for the field's value. +6. If it is a date field, standardize the data format according to the date granularity into forms such as the following: yyyy-mm-dd | mm-dd | mm | yyyy-mm | yyyy-qq. +7. The change in values should be reflected in the positive or negative nature of the data, not in the field names. `; diff --git a/packages/vmind/src/atom/type.ts b/packages/vmind/src/atom/type.ts index 9232a34b..a61e90fb 100644 --- a/packages/vmind/src/atom/type.ts +++ b/packages/vmind/src/atom/type.ts @@ -7,6 +7,10 @@ export interface BaseOptions { showThoughts?: boolean; } +export interface DataExtractionOptions extends BaseOptions { + reGenerateFieldInfo?: boolean; +} + export interface DataCleanOptions extends BaseOptions { needNumericalFields?: boolean; filterSameValueColumn?: boolean; diff --git a/packages/vmind/src/schedule/index.ts b/packages/vmind/src/schedule/index.ts index b11a912d..536303a3 100644 --- a/packages/vmind/src/schedule/index.ts +++ b/packages/vmind/src/schedule/index.ts @@ -6,11 +6,11 @@ import { BaseAtom } from '../atom/base'; import { DataQueryAtom, DataExtractionAtom, ChartGeneratorAtom } from '../atom'; import type { CombineAll, MapAtomTypes, TaskMapping } from '../types/schedule'; import { DataCleanAtom } from '../atom/dataClean'; -import type { BaseOptions, DataCleanOptions } from '../atom/type'; +import type { BaseOptions, DataCleanOptions, DataExtractionOptions } from '../atom/type'; export interface ScheduleOptions { [AtomName.BASE]?: BaseOptions; - [AtomName.DATA_EXTRACT]?: BaseOptions; + [AtomName.DATA_EXTRACT]?: DataExtractionOptions; [AtomName.DATA_CLEAN]?: DataCleanOptions; [AtomName.DATA_QUERY]?: BaseOptions; [AtomName.CHART_GENERATE]?: BaseOptions; diff --git a/packages/vmind/src/utils/field.ts b/packages/vmind/src/utils/field.ts new file mode 100644 index 00000000..11dfac8d --- /dev/null +++ b/packages/vmind/src/utils/field.ts @@ -0,0 +1,18 @@ +import type { FieldInfo } from '../types'; +import { DataType, ROLE } from '../types'; + +export const getRoleByFieldType = (type: DataType) => { + if ([DataType.DATE, DataType.TIME, DataType.STRING, DataType.RATIO].includes(type)) { + return ROLE.DIMENSION; + } + return ROLE.MEASURE; +}; + +/** Format FieldInfo, add role or location attribtuion, it will change fieldInfo directly */ +export const formatFieldInfo = (fieldInfo: FieldInfo[]) => { + fieldInfo.forEach(info => { + info.role = getRoleByFieldType(info.fieldType); + info.location = getRoleByFieldType(info.fieldType) as any; + }); + return fieldInfo; +}; diff --git a/packages/vmind/src/utils/json.ts b/packages/vmind/src/utils/json.ts index 50c8a649..b6b52b34 100644 --- a/packages/vmind/src/utils/json.ts +++ b/packages/vmind/src/utils/json.ts @@ -1,10 +1,14 @@ import JSON5 from 'json5'; +export const replaceAll = (originStr: string, replaceStr: string, newStr: string) => { + return originStr.split(replaceStr).join(newStr); +}; + export const matchJSONStr = (str: string) => { const first = str.indexOf('{'); const last = str.lastIndexOf('}'); const result = str.substring(first, last + 1); - return result && result.length > 0 ? result : str; + return result && result.length > 0 ? replaceAll(result, '\n', ' ') : str; }; export const parseLLMJson = (JsonStr: string, prefix?: string) => { From 1f155e72ac763814561b0425f815a0204af13c72 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 20 Sep 2024 10:29:22 +0800 Subject: [PATCH 029/128] feat: optimize prompt and fix data type error --- packages/vmind/src/atom/dataExtraction/prompt.ts | 10 +++++----- packages/vmind/src/types/atom.ts | 2 +- packages/vmind/src/types/llm.ts | 2 +- packages/vmind/src/utils/field.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/vmind/src/atom/dataExtraction/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt.ts index 1120137b..57ab1267 100644 --- a/packages/vmind/src/atom/dataExtraction/prompt.ts +++ b/packages/vmind/src/atom/dataExtraction/prompt.ts @@ -11,21 +11,21 @@ dataTable: [{date: "Monday", class No.1: 20, class No.2: 30},{date: "Tuesday", c \`\`\``; const baseExamples = `# Examples1 -text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 +提取文本如下：:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 Response: \`\`\` {"fieldInfo:":[{"fieldName":"公司","description":"公司名称","fieldType":"string",},{"fieldName":"月份","description":"具体月份","fieldType":"string",},{"fieldName":"利润调整","description":"是否经过利润调整","fieldType":"string",},{"fieldName":"利润额","description":"利润总额","fieldType":"numerical",}],"dataTable":[{"公司":"阿里","月份":"5月","利润调整":"调整前","利润额":100000000000,},{"公司":"阿里","月份":"5月","利润调整":"调整后","利润额":10000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整前","利润额":80000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整后","利润额":12000000000,},]} \`\`\` # Examples2 -text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. +Extracted text is bellow:: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. Response: \`\`\` {"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string",},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"ratio"}],"dataTable":[{"name":"John Smith","ranking":90,},{"name":"Jane Doe","ranking":75}]} \`\`\` # Examples3 -text: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%。 +提取文本如下：: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%。 Response: \`\`\` @@ -148,7 +148,7 @@ dataTable: Record\[]; // Extracted data set, key of dataT \`\`\` # Examples1: -text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 +提取文本如下：:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 \`\`\` {"fieldInfo:":[{"fieldName":"公司","description":"公司名称","fieldType":"string",},{"fieldName":"月份","description":"具体月份","fieldType":"string",},{"fieldName":"利润调整","description":"是否经过利润调整","fieldType":"string",},{"fieldName":"利润额","description":"利润总额","fieldType":"numerical",}]} \`\`\` @@ -158,7 +158,7 @@ Response: \`\`\` # Examples2: -text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. +Extracted text is bellow: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. \`\`\` {"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string","dataExample":["Roy","Stepen Curry","张三","李四"]},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"ratio","dataExample": [10, 80]]}}]} \`\`\` diff --git a/packages/vmind/src/types/atom.ts b/packages/vmind/src/types/atom.ts index d39284e0..3d50c5ff 100644 --- a/packages/vmind/src/types/atom.ts +++ b/packages/vmind/src/types/atom.ts @@ -38,7 +38,7 @@ export enum DataType { STRING = 'string', REGION = 'region', NUMERICAL = 'numerical', - RATIO = 'RATIO', + RATIO = 'ratio', COUNT = 'count' } diff --git a/packages/vmind/src/types/llm.ts b/packages/vmind/src/types/llm.ts index a6e24214..d39b882e 100644 --- a/packages/vmind/src/types/llm.ts +++ b/packages/vmind/src/types/llm.ts @@ -16,7 +16,7 @@ export enum Model { GPT_4_0613 = 'gpt-4-0613', GPT_4o = 'gpt-4o-2024-05-13', DOUBAO_LITE = 'doubao-lite-32K', - DOUBAO_PRO = 'doubao-pro-128k', + DOUBAO_PRO = 'doubao-pro-32k', SKYLARK2 = 'skylark2-pro-4k', SKYLARK2_v1_2 = 'skylark2-pro-4k-v1.2', CHART_ADVISOR = 'chart-advisor' diff --git a/packages/vmind/src/utils/field.ts b/packages/vmind/src/utils/field.ts index 11dfac8d..3265ed35 100644 --- a/packages/vmind/src/utils/field.ts +++ b/packages/vmind/src/utils/field.ts @@ -2,7 +2,7 @@ import type { FieldInfo } from '../types'; import { DataType, ROLE } from '../types'; export const getRoleByFieldType = (type: DataType) => { - if ([DataType.DATE, DataType.TIME, DataType.STRING, DataType.RATIO].includes(type)) { + if ([DataType.DATE, DataType.TIME, DataType.STRING, DataType.REGION].includes(type)) { return ROLE.DIMENSION; } return ROLE.MEASURE; From 03bdfbb57384f7a7c1b14242b177d9026542f8e2 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 20 Sep 2024 11:19:11 +0800 Subject: [PATCH 030/128] feat: update data extraction test page --- .../__tests__/browser/src/pages/Home.tsx | 2 +- .../src/pages/NewDataExtraction/DataInput.tsx | 16 +++++-- .../src/pages/NewDataExtraction/DataTable.tsx | 48 +++++++++---------- .../src/pages/NewDataExtraction/index.tsx | 10 +++- 4 files changed, 44 insertions(+), 32 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/pages/Home.tsx b/packages/vmind/__tests__/browser/src/pages/Home.tsx index 8c3b154f..9127629c 100644 --- a/packages/vmind/__tests__/browser/src/pages/Home.tsx +++ b/packages/vmind/__tests__/browser/src/pages/Home.tsx @@ -7,7 +7,7 @@ const MenuItem = Menu.Item; const LOCAL_STORAGE_MENU_KEY = 'VMind_playground_menu_key'; export const Home: React.FC = props => { - const [selectedPage, setSelectedPage] = React.useState(PLAYGROUND_PAGES.CHART_GENERATION); + const [selectedPage, setSelectedPage] = React.useState(PLAYGROUND_PAGES.NEW_DATA_EXTRACTION); const [collapsed, setCollapsed] = React.useState(true); return ( diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx index dabc4868..925d098b 100644 --- a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx @@ -1,13 +1,14 @@ /* eslint-disable no-console */ import React, { useState, useEffect } from 'react'; import '../DataExtraction/index.scss'; -import { Avatar, Input, Divider, Button, Select, Checkbox, Modal } from '@arco-design/web-react'; +import { Avatar, Input, Divider, Button, Select, Checkbox, Modal, Radio } from '@arco-design/web-react'; import type { FieldInfo } from '../../../../../src/index'; import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; import { capcutMockData } from '../../constants/capcutData'; const TextArea = Input.TextArea; const Option = Select.Option; +const RadioGroup = Radio.Group; type IPropsType = { onOk: (extractCtx: any, dataCleanCtx: any) => void; @@ -52,7 +53,7 @@ export function DataInput(props: IPropsType) { const schedule = React.useRef>( new Schedule( [AtomName.DATA_EXTRACT, AtomName.DATA_CLEAN], - { base: { llm: llm.current, showThoughts } }, + { base: { llm: llm.current, showThoughts }, dataExtract: { reGenerateFieldInfo: !useFieldInfo } }, { text, fieldInfo: useFieldInfo ? fieldInfo : [] } ) ); @@ -67,8 +68,8 @@ export function DataInput(props: IPropsType) { }); }, [url, model, apiKey]); useEffect(() => { - schedule.current.updateOptions({ base: { showThoughts } }); - }, [showThoughts]); + schedule.current.updateOptions({ base: { showThoughts }, dataExtract: { reGenerateFieldInfo: !useFieldInfo } }); + }, [showThoughts, useFieldInfo]); const handleQuery = React.useCallback(async () => { props.setLoading(true); await schedule.current.run(userInput); @@ -171,6 +172,12 @@ export function DataInput(props: IPropsType) { +
+ setModel(v)}> + GPT-4-0613 + Doubao-pro + +
setUseFieldInfo(v)}> Use FieldInfo @@ -181,7 +188,6 @@ export function DataInput(props: IPropsType) { Show Thoughts
- { +const SimpleTable = ({ data, fieldInfo }: { data: DataTable; fieldInfo: FieldInfo[] }) => { if (data.length === 0) { return
No data available
; } - const columns = Object.keys(data[0]); - + const columns: TableColumnProps[] = fieldInfo.map((info: FieldInfo) => { + return { + title: info.fieldName, + dataIndex: info.fieldName + }; + }); return ( -
- - - {columns.map(column => ( - - ))} - - - - {data.map((row, rowIndex) => ( - - {columns.map((column, index) => ( - - ))} - - ))} - -
{column}
{row[column]}
+ ); }; @@ -37,18 +34,19 @@ interface Props { dataset: DataTable; finalDataset: DataTable; fieldInfo: FieldInfo[]; + finalFieldInfo: FieldInfo[]; loading: boolean; } -export const DataTableComp = ({ dataset, finalDataset, fieldInfo, loading }: Props) => { +export const DataTableComp = ({ dataset, finalDataset, fieldInfo, finalFieldInfo, loading }: Props) => { return (

DataClean Result:

- +

DataExtration Result:

- +

fieldInfo:

diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx index 605493e4..289b8755 100644 --- a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx @@ -11,12 +11,14 @@ export function NewDataExtractionPage() { const [dataset, setDataset] = useState([]); const [finalDataset, setFianlDataset] = useState([]); const [fieldInfo, setFieldInfo] = useState([]); + const [finalFieldInfo, setFinalFieldInfo] = useState([]); const [loading, setLoading] = useState(false); const handleOk = React.useCallback(async (dataExtractCtx: any, dataCleanCtx: any) => { setDataset(dataExtractCtx.dataTable); setFieldInfo(dataExtractCtx.fieldInfo); setFianlDataset(dataCleanCtx.dataTable); + setFinalFieldInfo(dataCleanCtx.fieldInfo); setLoading(false); // eslint-disable-next-line no-console console.info(dataExtractCtx, dataCleanCtx); @@ -33,7 +35,13 @@ export function NewDataExtractionPage() { - + ); From 19ab14db445f28a1727fba2473ffaaad529d5514 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 20 Sep 2024 11:19:41 +0800 Subject: [PATCH 031/128] feat: isDaatExtraction may undefined in doubao --- packages/vmind/src/atom/dataExtraction/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vmind/src/atom/dataExtraction/index.ts b/packages/vmind/src/atom/dataExtraction/index.ts index 8b5c94f7..1de55f83 100644 --- a/packages/vmind/src/atom/dataExtraction/index.ts +++ b/packages/vmind/src/atom/dataExtraction/index.ts @@ -74,7 +74,7 @@ ${language === 'english' ? 'Extracted text is bellow:' : '提取文本如下：' parseLLMContent(resJson: any) { const { dataTable, fieldInfo, isDataExtraction } = resJson; - if (!isDataExtraction) { + if (isDataExtraction === false) { console.error("It's not a data extraction task"); return this.context; } From 902adfd036dd3e6905d25d7aa9277f92a9a06d85 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 20 Sep 2024 11:20:39 +0800 Subject: [PATCH 032/128] feat: finish data extraction case study page and vertify function --- .../src/pages/DataExtraction/caseStudy.tsx | 287 ++++++++++++++++-- .../src/pages/DataExtraction/test.tsx | 55 ++-- .../src/pages/DataExtraction/type.tsx | 32 ++ .../src/pages/DataExtraction/verify.tsx | 184 +++++++++++ .../__tests__/experiment/src/pages/page.scss | 2 +- 5 files changed, 507 insertions(+), 53 deletions(-) create mode 100644 packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx create mode 100644 packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx index 2914bfc1..46da5325 100644 --- a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx @@ -3,20 +3,28 @@ import React from 'react'; import type { FieldInfo } from '../../../../../src/index'; import { capcutMockData } from '../../data/capcutData'; +import { dataExtractionCommonDataset, commonAnswer } from '../../data/dataExtractionData'; import type { TableColumnProps } from '@arco-design/web-react'; import { Avatar, Button, Card, Checkbox, Divider, Message, Select, Table, Tooltip } from '@arco-design/web-react'; -// import { result } from '../../results/dataExtraction/result'; -// import { result } from '../../results/dataExtraction/result2'; -import { result } from '../../results/dataExtraction/result3'; +import { result as capcutResult } from '../../results/dataExtraction/result3'; +import { result as caseResult } from '../../results/dataExtraction/commonResult'; import '../page.scss'; import { IconInfoCircle } from '@arco-design/web-react/icon'; +import { getLanguageOfText } from '../../../../../src/utils/text'; +import { mergeResult, updateScoreInDataExtraction } from './verify'; +import type { DataExtractionDataSetResult } from './type'; + +const result = [...mergeResult(capcutResult as any, caseResult as any), commonAnswer]; +updateScoreInDataExtraction(result as any, commonAnswer); +console.info(result); const llmList = result.map((v, index) => ({ index, llm: v.llm })); const datasetMap: Record = { - capcut: capcutMockData + capcut: capcutMockData, + common: dataExtractionCommonDataset }; const datasetList = result[0].result.map((v, index) => ({ index, @@ -24,25 +32,56 @@ const datasetList = result[0].result.map((v, index) => ({ dataset: datasetMap[v.dataset] })); +interface Options { + index: number; + llm: string; + resType: 'defaultResult' | 'fieldInfoResult'; +} +const targetScore = 0.7; export function DataExtractionResult() { const [datasetIndex, setDatasetIndex] = React.useState(0); const currentDataset = datasetList[datasetIndex]; + const currentDatasetName = currentDataset.name; + const [language, setLanguage] = React.useState<'all' | 'zh' | 'en'>('all'); + const [leftOptions, setLeftOptions] = React.useState({ + index: datasetIndex, + llm: llmList[0].llm, + resType: 'defaultResult' + }); + const [rightOptions, setRightOptions] = React.useState({ + index: 1, + llm: llmList?.[1]?.llm, + resType: 'defaultResult' + }); + const leftResult = React.useMemo( + () => result[leftOptions.index].result.find(v => v.dataset === currentDatasetName)?.[leftOptions.resType], + [leftOptions.index, leftOptions.resType, currentDatasetName] + ); + const rightResult = React.useMemo( + () => + rightOptions?.llm + ? result[rightOptions.index].result.find(v => v.dataset === currentDatasetName)?.[rightOptions.resType] + : null, + [rightOptions?.llm, rightOptions.index, rightOptions.resType, currentDatasetName] + ); const renderTable = React.useCallback((context: any) => { const { dataTable, fieldInfo } = context; - const columns: TableColumnProps[] = fieldInfo.map((info: FieldInfo) => ({ - title: ( -
- -
{`${info.fieldType[0]}__`}
- - {info.fieldName} - - - -
- ), - dataIndex: info.fieldName - })); + const columns: TableColumnProps[] = fieldInfo.map((info: FieldInfo) => { + return { + title: ( +
+ +
{`${info.fieldType[0]}__`}
+ + {info.fieldName} + + + +
+ ), + dataIndex: info.fieldName + }; + }); return (
); }, []); - const width = `${(0.8 / llmList.length) * 100}%`; + const changeOptions = React.useCallback((v: any, type: 'llm' | 'resType' | 'language', index: number) => { + const newOptions = + type === 'llm' + ? { + index: v, + llm: llmList?.[v]?.llm + } + : { + [type]: v + }; + if (index === 0) { + setLeftOptions(prev => ({ + ...prev, + ...newOptions + })); + } else { + setRightOptions(prev => ({ + ...prev, + ...newOptions + })); + } + }, []); + + const answerResult = React.useMemo(() => { + const answer = result.find(v => v.llm === 'answer'); + return (answer?.result || []).find(v => v.dataset === currentDataset.name); + }, [currentDataset.name]); + + const getCardDisplayStyle = React.useCallback( + (data: DataExtractionDataSetResult) => { + const { context, score } = data; + const isEn = context?.isEnglish ?? getLanguageOfText(context.text) === 'english'; + const displayStyle = language === 'all' || isEn === (language === 'en') ? {} : { display: 'none ' }; + const borderColor = + answerResult && score !== undefined && score < targetScore + ? { + borderColor: 'red' + } + : {}; + return { + ...displayStyle, + ...borderColor + }; + }, + [answerResult, language] + ); + + const renderScore = React.useCallback( + (res: DataExtractionDataSetResult[] | undefined | null) => { + if (answerResult && res) { + const validRes = res.filter(v => !!v.score && v.score !== 0); + const allCount = validRes.length; + const reachedCount = validRes.filter(v => v.score! >= targetScore).length; + let score = 0; + let fieldScore = 0; + let dataScore = 0; + validRes.forEach(v => { + score += v.score || 0; + fieldScore += v.fieldScore || 0; + dataScore += v.dataScore || 0; + }); + score /= allCount; + fieldScore /= allCount; + dataScore /= allCount; + const columns: TableColumnProps[] = [ + { + title: 'All', + dataIndex: 'allCount' + }, + { + title: 'Success', + dataIndex: 'reachedCount' + }, + { + title: 'Rate', + dataIndex: 'rate' + }, + { + title: 'Score', + dataIndex: 'score' + }, + { + title: 'FieldScore', + dataIndex: 'fieldScore' + }, + { + title: 'DataScore', + dataIndex: 'dataScore' + } + ]; + return ( +
+ ); + } + return

No answer to get score

; + }, + [answerResult] + ); + + const renderCaseScore = React.useCallback( + (data: DataExtractionDataSetResult) => { + if (answerResult && data?.score !== undefined) { + return ( +
+ Score: {data.score.toFixed(2)} + FieldScore: {data.fieldScore?.toFixed(2)} + DataScore: {data.dataScore?.toFixed(2)} +
+ ); + } + return null; + }, + [answerResult] + ); + const count = rightResult ? 2 : 1; + const width = `${(0.8 / count) * 100}%`; return (
@@ -74,18 +247,68 @@ export function DataExtractionResult() {
Comparision Result:
-
USER INPUT
- {llmList.map(v => ( -
- {v.llm} -
- ))} +
+ USER INPUT + +
+ {[leftOptions, rightOptions].map((v, index) => { + if (count === 1 && index === 1) { + return null; + } + return ( +
+
+ + +
+
{renderScore(index === 0 ? leftResult : rightResult)}
+
+ ); + })}
{currentDataset.dataset.map((v: any, index: number) => ( - + {index + 1} @@ -94,18 +317,22 @@ export function DataExtractionResult() { ))}
- {llmList.map(v => { - const llmResult = result[v.index].result[datasetIndex].defaultResult; + {[leftResult, rightResult].map((llmResult, index) => { + if (!llmResult) { + return null; + } return ( -
+
{llmResult.map((dataResult, index) => { return ( console.log('Context is :', dataResult.context)} > + {renderCaseScore(dataResult)} {renderTable(dataResult.context)} ); diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx index 7a5519d4..439c8df2 100644 --- a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx @@ -3,7 +3,9 @@ import React from 'react'; import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; import { capcutMockData } from '../../data/capcutData'; -import { Button, Checkbox, Divider, Message } from '@arco-design/web-react'; +import { Button, Checkbox, Divider, Message, Select } from '@arco-design/web-react'; +import { dataExtractionCommonDataset } from '../../data/dataExtractionData'; +import { pick } from '@visactor/vutils'; const globalVariables = (import.meta as any).env; const ModelConfigMap: any = { @@ -14,6 +16,10 @@ const datasets = [ { name: 'capcut', data: capcutMockData + }, + { + name: 'common', + data: dataExtractionCommonDataset } ]; @@ -34,9 +40,7 @@ function sleep(ms: number) { } export function DataExtractionTask() { - const [selectedDataset, setSelectedDataset] = React.useState>( - datasets.reduce((prev, cur) => ({ ...prev, [cur.name]: true }), {}) - ); + const [selectedDataset, setSelectedDataset] = React.useState(['common']); const [selectedLLM, setSelectedLLm] = React.useState>( Object.keys(ModelConfigMap).reduce((prev, cur) => ({ ...prev, [cur]: true }), {}) ); @@ -54,7 +58,7 @@ export function DataExtractionTask() { if (!selectedLLM[model]) { continue; } - const sleepTime = model === Model.DOUBAO_PRO ? 8000 : 2000; + const sleepTime = model === Model.DOUBAO_PRO ? 8000 : 4000; const apiKey = ModelConfigMap[model]?.key; const llm = new LLMManage({ url: ModelConfigMap[model]?.url, @@ -64,14 +68,19 @@ export function DataExtractionTask() { }, model }); - const schedule = new Schedule([AtomName.DATA_EXTRACT], { base: { llm, showThoughts: false } }); + const schedule = new Schedule([AtomName.DATA_EXTRACT], { + base: { llm, showThoughts: false }, + dataExtract: { + reGenerateFieldInfo: true + } + }); (messageApi as any).info(`Begin ${model}!`); console.info(`---------Begin ${model}---------`); const datasetResult: any[] = []; for (let datasetIndex = 0; datasetIndex < datasets.length; datasetIndex++) { const dataset = datasets[datasetIndex]; - if (!selectedDataset[dataset.name]) { + if (!selectedDataset.includes(dataset.name)) { continue; } (messageApi as any).info(`Begin ${dataset.name} dataset!`); @@ -94,7 +103,7 @@ export function DataExtractionTask() { if (useFieldInfo) { schedule.setNewTask({ text: data.text, - fieldInfo: data.fieldInfo + fieldInfo: data.fieldInfo.map((v: any) => pick(v, ['fieldName'])) }); const result = await schedule.run(); fieldInfoResult.push({ @@ -147,19 +156,21 @@ export function DataExtractionTask() { return (
{contextHolder} -
-

DataSet to Test

- {datasets.map(dataset => { - return ( - setSelectedDataset(prev => ({ ...prev, [dataset.name]: v }))} - > - {dataset.name} - - ); - })} +
+

Please select dataset to run:

+

LLM Model To Select

@@ -178,7 +189,7 @@ export function DataExtractionTask() {

Config of Data Extraction

setUseFieldInfo(v)}> - With FieldInfo + With FieldName Only setUseDefault(v)}> Without FieldInfo diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx new file mode 100644 index 00000000..bb1c407b --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx @@ -0,0 +1,32 @@ +import type { DataTable, FieldInfo } from '../../../../../src'; + +export interface ScoreDetail { + originIndex: number; + matchedIndex: number; + nameScore: number; + typeScore: number; + dataScore: number; +} + +export interface DataExtractionDataSetResult { + context: { + isEnglish?: boolean; + dataTable: DataTable; + fieldInfo: FieldInfo[]; + text: string; + }; + score?: number; + fieldScore?: number; + dataScore?: number; + scoreDetail?: ScoreDetail[]; +} +export interface DataExtractionCase { + llm: string; + result: { + dataset: string; + defaultResult: DataExtractionDataSetResult[]; + fieldInfoResult: DataExtractionDataSetResult[]; + }[]; +} + +export type DataExtractionResult = DataExtractionCase[]; diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx new file mode 100644 index 00000000..bdf8d5ab --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx @@ -0,0 +1,184 @@ +/* eslint-disable no-console */ +import stringSimilarity from 'string-similarity'; +import { DataType } from '../../../../../src'; +import type { DataCell, DataExtractionCtx, FieldInfo, DataTable } from '../../../../../src'; +import { getRoleByFieldType } from '../../../../../src/utils/field'; +import { ROLE } from '../../../../../src/common/typings'; +import type { DataExtractionCase, DataExtractionResult, ScoreDetail } from './type'; + +function cosineSimilarity(text1: string, text2: string): number { + return stringSimilarity.compareTwoStrings(text1, text2); +} + +function getFieldTypeScore(typeA: DataType, typeB: DataType) { + if (typeA === typeB) { + return 1; + } + const roleA = getRoleByFieldType(typeA); + const roleB = getRoleByFieldType(typeB); + if (roleA !== roleB) { + return -1; + } + if (roleA === ROLE.MEASURE && [typeA, typeB].includes(DataType.RATIO)) { + return -0.5; + } + return 0; +} + +const getFieldScore = (nameScore: number, typeScore: number) => nameScore * 0.75 + typeScore * 0.25; + +const getFinalScore = (fieldScore: number, dataScore: number) => fieldScore * 0.4 + dataScore * 0.6; + +const getDataScore = (currentDataList: DataCell[], answerDataList: DataCell[], role: ROLE) => { + let dataScore = 0; + const minLength = Math.min(currentDataList.length, answerDataList.length); + const seletedList = new Array(minLength).fill(false); + for (let i = 0; i < minLength; i++) { + let maxScore = -1; + let currentSelectedIndex = -1; + const valueCompareFunction = getValueCompareFunction(role); + for (let j = 0; j < minLength; j++) { + if (!seletedList[j]) { + const currentScore = valueCompareFunction(currentDataList[i], answerDataList[j]); + if (maxScore < currentScore) { + currentSelectedIndex = j; + maxScore = currentScore; + } + } + } + dataScore += maxScore; + seletedList[currentSelectedIndex] = true; + } + if (answerDataList.length > currentDataList.length) { + dataScore -= (answerDataList.length - currentDataList.length) * 0.5; + dataScore /= answerDataList.length; + } else { + dataScore /= currentDataList.length; + } + return dataScore; +}; + +/** fieldInfoA is answer */ +function getScoreOfDataset( + fieldInfoA: FieldInfo[], + fieldInfoB: FieldInfo[], + dataTableA: DataTable, + dataTableB: DataTable +) { + const seletedList = fieldInfoB.map(v => false); + const scoreList: ScoreDetail[] = []; + fieldInfoA.forEach((fieldInfo, originIndex) => { + let finalScore = 0; + let nameScore = 0; + let index = -1; + let typeScore = -1; + let dataScore = -1; + const dataListA = dataTableA?.map(v => v[fieldInfo.fieldName]) || []; + for (let i = 0; i < fieldInfoB.length; i++) { + if (!seletedList[i]) { + const dataListB = dataTableB?.map(v => v[fieldInfoB[i].fieldName]) || []; + const cosValue = cosineSimilarity(fieldInfo.fieldName, fieldInfoB[i].fieldName); + const currentTypeScore = getFieldTypeScore(fieldInfo.fieldType, fieldInfoB[i].fieldType); + const role = fieldInfo.role ?? getRoleByFieldType(fieldInfo.fieldType); + const currentDataScore = getDataScore(dataListB, dataListA, role); + const currentScore = getFinalScore(getFieldScore(cosValue, currentTypeScore), currentDataScore); + if (currentScore > finalScore && currentTypeScore !== -1 && currentDataScore > 0) { + index = i; + finalScore = currentScore; + nameScore = cosValue; + typeScore = currentTypeScore; + dataScore = currentDataScore; + } + } + } + scoreList.push({ + originIndex, + matchedIndex: index, + nameScore, + typeScore, + dataScore + }); + seletedList[index] = true; + }); + return scoreList; +} + +function getValueCompareFunction(role: ROLE) { + if (role === ROLE.MEASURE) { + return (a: DataCell, b: DataCell) => { + if (+a === +b || a === b) { + return 1; + } + return 0; + }; + } + return (a: DataCell, b: DataCell) => { + return cosineSimilarity(`${a}`, `${b}`); + }; +} + +export const getScoreOfDataExtraction = (resultCtx: DataExtractionCtx, answerCtx: DataExtractionCtx) => { + const { fieldInfo = [], dataTable } = resultCtx; + const { fieldInfo: answerInfo = [], dataTable: answerTable } = answerCtx; + const scoreList = getScoreOfDataset(answerInfo || [], fieldInfo || [], answerTable!, dataTable!).filter( + v => v.matchedIndex !== -1 + ); + let fieldScore = 0; + let dataScore = 0; + scoreList.forEach(v => { + fieldScore += getFieldScore(v.nameScore, v.typeScore); + dataScore += v.dataScore; + }); + dataScore /= scoreList.length; + if (answerInfo?.length > fieldInfo?.length) { + fieldScore -= (answerInfo?.length - fieldInfo?.length) * 0.5; + fieldScore /= answerInfo?.length; + } else { + fieldScore /= fieldInfo.length; + } + return { + score: getFinalScore(fieldScore, dataScore), + fieldScore, + dataScore, + scoreDetail: scoreList + }; +}; + +export const updateScoreInDataExtraction = (result: DataExtractionResult, answer: DataExtractionCase) => { + result.forEach(llmResult => { + llmResult.result.forEach(caseResult => { + const answerCase = answer.result.find(v => v.dataset === caseResult.dataset); + if (answerCase) { + if (caseResult.defaultResult.length) { + caseResult.defaultResult = caseResult.defaultResult.map((v, index) => ({ + ...v, + ...getScoreOfDataExtraction(v.context, answerCase.defaultResult[index].context) + })); + } + if (caseResult.fieldInfoResult.length) { + caseResult.fieldInfoResult = caseResult.fieldInfoResult.map((v, index) => ({ + ...v, + ...getScoreOfDataExtraction(v.context, answerCase.defaultResult[index].context) + })); + } + } + }); + }); + return result; +}; + +export const mergeResult = (result1: DataExtractionResult, result2: DataExtractionResult) => { + const modelResult: Record = {}; + [...result1, ...result2].forEach(v => { + const { llm, result } = v; + if (!modelResult[llm]) { + modelResult[llm] = result; + } else { + modelResult[llm].push(...result); + } + }); + return Object.keys(modelResult).map(llm => ({ + llm, + result: modelResult[llm] + })); +}; diff --git a/packages/vmind/__tests__/experiment/src/pages/page.scss b/packages/vmind/__tests__/experiment/src/pages/page.scss index 47a351c1..90e94076 100644 --- a/packages/vmind/__tests__/experiment/src/pages/page.scss +++ b/packages/vmind/__tests__/experiment/src/pages/page.scss @@ -18,7 +18,7 @@ .title { display: flex; - height: 24px; + min-height:30px; width: 100%; border-bottom: 1px #efefef solid; text-align: center; From a8a9a7ad8a0d7c6e77c1eef165bb841b59c25041 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 20 Sep 2024 11:55:41 +0800 Subject: [PATCH 033/128] fix: return error in try catch not return err --- packages/vmind/src/atom/base.ts | 4 ++-- packages/vmind/src/atom/chartGenerator/index.ts | 2 +- packages/vmind/src/atom/dataQuery/index.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/vmind/src/atom/base.ts b/packages/vmind/src/atom/base.ts index 5683f403..76a80129 100644 --- a/packages/vmind/src/atom/base.ts +++ b/packages/vmind/src/atom/base.ts @@ -112,7 +112,7 @@ export class BaseAtom { const messages = this.getLLMMessages(); const data = await this.options.llm.run(this.name, messages); const resJson = this.options.llm.parseJson(data); - if (resJson.err) { + if (resJson.error) { return this.context; } this.recordLLMResponse(data); @@ -133,7 +133,7 @@ export class BaseAtom { const messages = this.getLLMMessages(query); const data = await this.options.llm.run(this.name, messages); const resJson = this.options.llm.parseJson(data); - if (!resJson.err) { + if (!resJson.error) { this.recordLLMResponse(data, query); this.setNewContext(this.parseLLMContent(resJson)); } diff --git a/packages/vmind/src/atom/chartGenerator/index.ts b/packages/vmind/src/atom/chartGenerator/index.ts index c802c702..f84bf47b 100644 --- a/packages/vmind/src/atom/chartGenerator/index.ts +++ b/packages/vmind/src/atom/chartGenerator/index.ts @@ -33,7 +33,7 @@ export class ChartGeneratorAtom extends BaseAtom parseLLMContent(data: LLMResponse) { const resJson = this.options.llm.parseJson(data); - if (resJson.err) { + if (resJson.error) { return this.context; } const { dataTable, fieldInfo, cells, chartType } = resJson; diff --git a/packages/vmind/src/atom/dataQuery/index.ts b/packages/vmind/src/atom/dataQuery/index.ts index 9c6f398c..d47e0bf9 100644 --- a/packages/vmind/src/atom/dataQuery/index.ts +++ b/packages/vmind/src/atom/dataQuery/index.ts @@ -59,7 +59,7 @@ export class DataQueryAtom extends BaseAtom { parseLLMContent(data: LLMResponse) { const resJson = this.options.llm.parseJson(data); - if (resJson.err) { + if (resJson.error) { return this.context; } const { sql, fieldInfo: responseFiledInfo } = resJson; From d92cd68b933b72f66baeac94075e1e3020ae7c3b Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Fri, 20 Sep 2024 15:23:59 +0800 Subject: [PATCH 034/128] feat: optimize prompt --- .../vmind/src/atom/dataExtraction/prompt.ts | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/vmind/src/atom/dataExtraction/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt.ts index 57ab1267..b7539695 100644 --- a/packages/vmind/src/atom/dataExtraction/prompt.ts +++ b/packages/vmind/src/atom/dataExtraction/prompt.ts @@ -1,6 +1,7 @@ /* eslint-disable max-len */ const dataTableExplanation = `# Data Table Explanation -1. ALWAYS generate flatten data table rather than unflatten data table +1. The value type of a 'numerical', 'ratio', or 'count' field MUST be 'number' or 'number[]'. +2. ALWAYS generate flatten data table rather than unflatten data table ## Flatten Data Table Example \`\`\` dataTable: [{ date: "Monday", class: "class No.1", score: 20 },{ date: "Monday", class: "class No.2", score: 30 },{ date: "Tuesday", class: "class No.1", score: 25 },{ date: "Tuesday", class: "class No.2", score: 28 }] @@ -11,21 +12,21 @@ dataTable: [{date: "Monday", class No.1: 20, class No.2: 30},{date: "Tuesday", c \`\`\``; const baseExamples = `# Examples1 -提取文本如下：:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 +text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 Response: \`\`\` {"fieldInfo:":[{"fieldName":"公司","description":"公司名称","fieldType":"string",},{"fieldName":"月份","description":"具体月份","fieldType":"string",},{"fieldName":"利润调整","description":"是否经过利润调整","fieldType":"string",},{"fieldName":"利润额","description":"利润总额","fieldType":"numerical",}],"dataTable":[{"公司":"阿里","月份":"5月","利润调整":"调整前","利润额":100000000000,},{"公司":"阿里","月份":"5月","利润调整":"调整后","利润额":10000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整前","利润额":80000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整后","利润额":12000000000,},]} \`\`\` # Examples2 -Extracted text is bellow:: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. +text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. Response: \`\`\` {"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string",},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"ratio"}],"dataTable":[{"name":"John Smith","ranking":90,},{"name":"Jane Doe","ranking":75}]} \`\`\` # Examples3 -提取文本如下：: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%。 +text: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%。 Response: \`\`\` @@ -51,11 +52,11 @@ const getFieldTypeExplanation = (language: 'chinese' | 'english') => { export const getBasePrompt = ( language: 'chinese' | 'english', showThoughs: boolean = true -) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, dates data, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. +) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, date, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. # Field Information Explanation -1. ALWAYS generate a field information, which represents the specific information of each column field in the data table. -2. ALWAYS generate a field description -3. ALWAYS generate a field type, chosen from 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count';${getFieldTypeExplanation( +1. ALWAYS generate field information, which represents the specific information of each column field in the data table. +2. ALWAYS generate field description +3. ALWAYS generate field type, chosen from 'date' | 'time' | 'string' | 'region' | 'numerical' | 'ratio' ｜ 'count';${getFieldTypeExplanation( language )} ${dataTableExplanation} @@ -69,9 +70,10 @@ You should think step-by-step as follow: 3. Read the entire text and fields with numerical or ratio or count field type first. 4. Read all text again and generate field information associated with the fields found in Step3.The newly generated fields are all simple. 5. Read all text and extract all corresponding data table based on the field information. The data corresponding to a field should always be concise, and a field should express only one meaning. -6. When a date field contains data with multiple date granularities, convert the fieldType to string. -7. Only use numbers that appear in the text, Do not perform any calculations or numerical conversion such as currency conversion calculation. -8. Assume the data is incomplete, then reconsider and execute the task again. +6. Format date data according to the date granularity such as the following: yyyy-mm-dd | mm-dd | mm | yyyy-mm | yyyy-qq. +7. When a date field contains data with multiple date granularities, convert the fieldType to string. +8. Only use numbers that appear in the text, Do not perform any calculations or numerical conversion such as currency conversion calculation. +9. Assume the data is incomplete, then reconsider and execute the task again. Response in the following format: \`\`\` @@ -95,9 +97,8 @@ You only need to return the JSON in your response directly to the user.Finish yo 1. Strictly define the type of return format, use JSON format to reply, do not include any extra content. 2. The numbers in the dataset do not carry any units. 3. Only extract value in ratio type, such as '95%' --> '95'; 'reduce 30%' --> '-30' -4. If you do not know the value of an field, return null for the field's value. -5. If it is a date field, standardize the data format according to the date granularity into forms such as the following: yyyy-mm-dd | mm-dd | mm | yyyy-mm | yyyy-qq. -6. The change in values should be reflected in the positive or negative nature of the data, not in the field names.`; +4. If you do not know the value of a field, return null for the field's value. +5. The change in values should be reflected in the positive or negative nature of the data, not in the field names.`; export const getFieldInfoPrompt = ( language: 'chinese' | 'english', @@ -117,14 +118,14 @@ dataExample?: (string | number)[] // data example of this field \`\`\` ${getCommonInfomation(language)} -You should think step-by-step as follow: +You should think step-by-step as follows: # Steps 0. using language answer: ${language} 1. Determine whether the current task is related to data extraction. 2. If not, return isDataExtraction is false in json mode; If yes, continue follow Steps 3. Read all text and extract all corresponding data table based on the field information. 4. Adjust the data to ensure consistency within the same field especially time field. -5. Only use numbers that appear in the text, Do not perform any calculations or numerical conversion such as currency conversion calculation. +5. Only use numbers that appear in the text, Do not perform any calculations or numerical conversion, such as currency conversion calculation. 6. Assume the data is incomplete, then reconsider and execute the task again. # Respones @@ -148,7 +149,7 @@ dataTable: Record\[]; // Extracted data set, key of dataT \`\`\` # Examples1: -提取文本如下：:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 +text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 \`\`\` {"fieldInfo:":[{"fieldName":"公司","description":"公司名称","fieldType":"string",},{"fieldName":"月份","description":"具体月份","fieldType":"string",},{"fieldName":"利润调整","description":"是否经过利润调整","fieldType":"string",},{"fieldName":"利润额","description":"利润总额","fieldType":"numerical",}]} \`\`\` @@ -158,13 +159,13 @@ Response: \`\`\` # Examples2: -Extracted text is bellow: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. +text: John Smith was very tall, ranked in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. \`\`\` -{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string","dataExample":["Roy","Stepen Curry","张三","李四"]},{"fieldName":"ranking","description":"The ranking of height in age group","fieldType":"ratio","dataExample": [10, 80]]}}]} +{"fieldInfo:":[{"fieldName":"name","description":"The name of a person","fieldType":"string","dataExample":["Roy","Stepen Curry","张三","李四"]},{"fieldName":"rank","description":"The rank of height in age group","fieldType":"ratio","dataExample": [10, 80]]}}]} \`\`\` Response: \`\`\` -{"dataTable":[{"name":"John Smith","ranking":90,},{"name":"Jane Doe","ranking":75}]} +{"dataTable":[{"name":"John Smith","rank":90,},{"name":"Jane Doe","rank":75}]} ---------------------------------- You only need to return the JSON in your response directly to the user. @@ -174,7 +175,7 @@ Finish your tasks in one-step. 2. The numbers in the dataset do not carry any units. 3. Only use numbers that appear in the text. 4. Only extract value in ratio type, such as '95%' --> '95'; 'reduce 30%' --> '-30' -5. If you do not know the value of an field, return null for the field's value. +5. If you do not know the value of a field, return null for the field's value. 6. If it is a date field, standardize the data format according to the date granularity into forms such as the following: yyyy-mm-dd | mm-dd | mm | yyyy-mm | yyyy-qq. 7. The change in values should be reflected in the positive or negative nature of the data, not in the field names. `; From b716f33619f2c4f70fd3b2c62ba83b92c98c52e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E4=BB=95=E4=BC=98?= <2279038930@qq.com> Date: Mon, 23 Sep 2024 19:56:43 +0800 Subject: [PATCH 035/128] fix: Optimize field mapping for sequence chart --- .../browser/src/constants/mockData.ts | 14 +++--- .../applications/chartGeneration/constants.ts | 17 ++++--- .../skylark/prompt/knowledge.ts | 16 +++---- .../skylark/prompt/knowledge.ts | 19 ++++---- .../skylark/prompt/template.ts | 2 +- .../GPT/prompt/knowledges.ts | 23 ++++++--- .../getChartSpec/VChart/transformers.ts | 47 ++++++++++--------- .../src/applications/chartGeneration/types.ts | 1 + 8 files changed, 77 insertions(+), 62 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/mockData.ts b/packages/vmind/__tests__/browser/src/constants/mockData.ts index 0f89fa9e..3eced4b4 100644 --- a/packages/vmind/__tests__/browser/src/constants/mockData.ts +++ b/packages/vmind/__tests__/browser/src/constants/mockData.ts @@ -185,7 +185,7 @@ export const mockUserInput6 = { 比利时,欧洲,53840,2018 挪威,欧洲,48930,2018 `, - input: '帮我展示各国GDP排名变化' + input: '使用动态条形图帮我展示各国GDP排名变化' }; /* @@ -4766,7 +4766,7 @@ export const bubbleCirclePackingData = { 房地产业,73821.3 其他,279918.4 `, - input: '请使用气泡图帮我绘制' + input: '请使用气泡圈图帮我绘制' }; export const mapChartData = { @@ -5076,7 +5076,7 @@ analytics,cluster,HierarchicalCluster,,6714 analytics,cluster,MergeEdge,,743 analytics,optimization,AspectRatioBanker,,7074 flex,FlareVis,,,4116`, - input: '请使用矩形树图渲染数据' + input: '请使用矩形树图渲染数据的层次结构' }; export const gaugeChartData = { @@ -5398,7 +5398,7 @@ export const singleColumnLineCombinationChartData = { 2022-09-01,0.259653267575217,2.040817156148029,2.19857288799284,2.229208371156883 2022-09-02,1.398428414171018,0.071469482611002,0.9048807067534731,0.0022491420541680004 2022-09-03,1.7166677805176591,1.903668070163285,1.866568462888393,1.8648831840830011`, - input: '请使用四个折线的组合图展示不同类别的权重随着时间的变化' + input: '请使用四个独立的折线的组合图展示不同类别的权重随着时间的变化' }; export const singleColumnLineCombinationChartData1 = { @@ -5594,7 +5594,7 @@ east,1027,654,654,830 west,1027,159,2100,532 north,1027,28,1679,498 `, - input: '帮我使用四个柱图的组合展示不同区域各商品销售额' + input: '帮我使用四个不同柱图的组合展示四个区域上不同商品销售额' }; export const singleColumnBarCombinationChartData1 = { @@ -5779,7 +5779,7 @@ export const singleColumnBarCombinationChartData1 = { 2022-09-01,0.259653267575217,2.040817156148029,2.19857288799284,2.229208371156883 2022-09-02,1.398428414171018,0.071469482611002,0.9048807067534731,0.0022491420541680004 2022-09-03,1.7166677805176591,1.903668070163285,1.866568462888393,1.8648831840830011`, - input: '请使用组合图展示不同类别的权重随着时间的变化，用四个柱图。' + input: '请使用四个独立柱图的组合图展示前十天不同类别的权重随着时间的变化。' }; export const dynamicScatterPlotData = { @@ -9849,7 +9849,7 @@ Alex Len,-2209016166000,start Alex Len,-2209015962000,end DeAndre' Bembry,-2209016547000,start DeAndre' Bembry,-2209015783000,end`, - input: '帮我展示各运动员在比赛中的行动记录。' + input: '使用时序图帮我展示各运动员在比赛中的行动记录。' }; export const mockUserTextInput0 = { text: `快手消失了。快手上市后，市值一度超过2000亿美元，现在只剩200多亿美元。去年快手的营收破了千亿，公司也赚钱了，但市场不买账了。 diff --git a/packages/vmind/src/applications/chartGeneration/constants.ts b/packages/vmind/src/applications/chartGeneration/constants.ts index d666f061..c1a29d46 100644 --- a/packages/vmind/src/applications/chartGeneration/constants.ts +++ b/packages/vmind/src/applications/chartGeneration/constants.ts @@ -5,9 +5,14 @@ export const SUPPORTED_CHART_LIST = Object.values(ChartType); export const COMBINATION_BASIC_CHART_LIST = Object.values(CombinationBasicChartType); export const COMBINATION_CHART_LIST = Object.values(CombinationChartType); -export const NEED_COLOR_FIELD_CHART_LIST = [ChartType.PieChart, ChartType.RoseChart, ChartType.LinearProgress]; +export const NEED_COLOR_FIELD_CHART_LIST = [ + ChartType.PieChart, + ChartType.RoseChart, + ChartType.LinearProgress, + ChartType.CircularProgress +]; -export const NEED_SIZE_FIELD_CHART_LIST = [ChartType.ScatterPlot, ChartType.BasicHeatMap, ChartType.LiquidChart]; +export const NEED_SIZE_FIELD_CHART_LIST = [ChartType.ScatterPlot, ChartType.BasicHeatMap]; export const NEED_COLOR_AND_SIZE_CHART_LIST = [ ChartType.WordCloud, @@ -16,8 +21,7 @@ export const NEED_COLOR_AND_SIZE_CHART_LIST = [ ChartType.VennChart, ChartType.Gauge, ChartType.SunburstChart, - ChartType.TreemapChart, - ChartType.CircularProgress + ChartType.TreemapChart ]; export const CARTESIAN_CHART_LIST = [ @@ -34,10 +38,11 @@ export const CARTESIAN_CHART_LIST = [ ChartType.BasicHeatMap ]; -export const DYNAMIC_CHART_LIST = [ +export const TIME_SERIES_CHART_LIST = [ ChartType.DynamicBarChart, ChartType.DynamicScatterPlotChart, - ChartType.DynamicRoseChart + ChartType.DynamicRoseChart, + ChartType.SequenceChart ]; export const DEFAULT_MAP_OPTION: BasemapOption = { diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts index 6ec85a11..c82fed1b 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateChartType/skylark/prompt/knowledge.ts @@ -92,20 +92,18 @@ export const chartKnowledgeBase: ChartKnowledgeBase = { }, [ChartType.SunburstChart]: { knowledge: [ - 'Sunburst Charts are excellent for visualizing hierarchical data, allowing users to see relationships between categories and subcategories at varying levels of detail.', - 'The colors field for sunburst chart and treemap chart must be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + 'Sunburst Charts are excellent for visualizing hierarchical data, allowing users to see relationships between categories and subcategories at varying levels of detail.' ] }, [ChartType.TreemapChart]: { knowledge: [ - 'Treemap Charts are effective for displaying large amounts of hierarchical data in a compact space, where areas represent the size of each category.', - 'The colors field for sunburst chart and treemap chart must be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + 'Treemap Charts are effective for displaying large amounts of hierarchical data in a compact space, where areas represent the size of each category.' ] }, [ChartType.Gauge]: { knowledge: [ - 'Gauge Charts are useful for displaying performance metrics against a target, providing a quick visual summary at a glance.', - 'The gauge chart must contain two fields: size and color.' + 'Gauge Charts are useful for displaying performance metrics against a target.', + 'If you want to display a dashboard, use a Gauge Chart.' ] }, [ChartType.BasicHeatMap]: { @@ -115,8 +113,7 @@ export const chartKnowledgeBase: ChartKnowledgeBase = { }, [ChartType.VennChart]: { knowledge: [ - 'Venn Charts are useful for displaying the relationships between different groups, emphasizing similarities and differences visually.', - 'The color field of the Venn diagram requires an array of length 2. The field with subscript 0 maps to the sets, and the field with subscript 1 maps to the name.' + 'Venn Charts are useful for displaying the relationships between different groups, emphasizing similarities and differences visually.' ] }, [ChartType.SingleColumnCombinationChart]: { @@ -134,8 +131,7 @@ export const chartKnowledgeBase: ChartKnowledgeBase = { }, [ChartType.DynamicRoseChart]: { knowledge: [ - 'Dynamic Rose Chart is used to display cyclical or seasonal data over time, with values represented by the length of radial bars.', - 'Dynamic Rose Chart highlights changes in categorical data or periodic trends across multiple categories over time.' + 'Dynamic Rose Chart is used to display cyclical or seasonal data over time, with values represented by the length of radial bars.' ], constraints: [ 'Use Dynamic Rose Chart if you want to show cyclical data and observe changes in multiple categories over time.' diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts index c7dfabe6..13a5df1c 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/knowledge.ts @@ -312,17 +312,16 @@ export const ChartFieldInfo: ChannelInfo = { size: 'field assigned to size channel' }, knowledge: [ - 'The color field for treemap chart must be an array. The order of the elements in the array needs to be sorted from large to small according to the coverage described by the data field.' + 'The color field must be an array sorted from large to small according to the coverage described by the data field.' ] }, [ChartType.Gauge.toUpperCase()]: { visualChannels: { - color: - 'color channel of gauge chart. Used to represent the current value against a range. Must be a string field.', + color: 'color channel of gauge chart. Must be a string field.', size: "size channel of gauge chart. Represents a numeric value indicating the current state. Can't be empty." }, responseDescription: { - color: 'field assigned to color channel', + color: 'field assigned to color channel. Often used to distinguish themes.', size: 'field assigned to size channel' }, knowledge: [ @@ -409,19 +408,19 @@ export const ChartFieldInfo: ChannelInfo = { }, [ChartType.SequenceChart.toUpperCase()]: { visualChannels: { - x: "x channel of sequence chart. Represents the initiator of the timeline or event, showing which entity or individual is involved in each sequence. Can't be empty.", - y: "y channel of sequence chart. Represents the timeline or time series, displaying the chronological order of events. This is essential for showing when events occur. Often a numeric value. Can't be empty.", + group: + "group channel of sequence chart. Represents the initiator of the timeline or event, showing which entity or individual is involved in each sequence. Can't be empty.", + time: "time channel of sequence chart. Represents the timeline or time series, displaying the chronological order of events. This is essential for showing when events occur. Often a numeric value. Can't be empty.", color: 'color channel of sequence chart. Differentiates the types or categories of events within the timeline. It is useful for distinguishing between different types of events in the sequence. For example: start or end' }, responseDescription: { - x: 'field assigned to x channel, representing the initiator of the timeline or event', - y: 'field assigned to y channel, typically used for time progression', + group: 'field assigned to group channel, representing the initiator of the timeline or event', + time: 'field assigned to time channel, typically used for time progression', color: 'field assigned to color channel, representing event types' }, knowledge: [ - 'All visual channels must be aligned with the structure of the data being visualized.', - 'Sequence charts are ideal for displaying event sequences over time, where the y-axis represents time progression, the x-axis shows the initiator of the timeline, and color helps distinguish between event types.', + 'Sequence charts are ideal for displaying event sequences over time, where the time-axis represents time progression, the group-axis shows the initiator of the timeline, and color helps distinguish between event types.', 'This chart is useful for visualizing workflows, event timelines, or sequences where time, initiators, and event types need to be clearly represented.' ] } diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts index 8325c590..f3fb48ec 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateFieldMap/skylark/prompt/template.ts @@ -18,7 +18,7 @@ Your task is: 1. Filter out useful fields related to user's command. 2. Assign the useful fields to the available visual channels according to field name and type. 3. If the chart type is a combination chart, the above two steps need to be repeated for each sub-chart generation task of the combination chart. The number of mapping relationships of visual channels in the response needs to be consistent with the number of sub-charts generated. -4. The response is an array in YAML format without any additional descriptions. +4. The outermost structure of the response must be an array in YAML format without any additional descriptions. The YAML array must be generated strictly according to the corresponding format at the end of the prompt, and special attention should be paid to the length of the array and the meaning of the elements in the array. Available visual channels: ${availableChannels} diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts index 6aee2b9c..b2beef34 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/generateTypeAndFieldMap/GPT/prompt/knowledges.ts @@ -7,7 +7,7 @@ import { NEED_COLOR_FIELD_CHART_LIST, NEED_SIZE_FIELD_CHART_LIST, NEED_COLOR_AND_SIZE_CHART_LIST, - DYNAMIC_CHART_LIST + TIME_SERIES_CHART_LIST } from '../../../../constants'; const getColorKnowledge = (chartTypeList: ChartType[]) => { @@ -93,7 +93,7 @@ export const visualChannelInfoMap = { }, time: (chartTypeList: ChartType[]) => { return { - singleFieldInfo: `This is usually a date field and cannot be empty in all dynamic chart types. For example, ${DYNAMIC_CHART_LIST.join( + singleFieldInfo: `This is usually a date field and cannot be empty in all dynamic chart types. For example, ${TIME_SERIES_CHART_LIST.join( ',' )}.` }; @@ -115,6 +115,12 @@ export const visualChannelInfoMap = { singleFieldInfo: "the field mapped to the value channel. Only used in Sankey Chart. Can't be empty in Sankey Chart." }; + }, + group: (chartTypeList: ChartType[]) => { + return { + singleFieldInfo: + "the field mapped to the group channel. Only used in Sequence Chart. Can't be empty in Sequence Chart." + }; } }; export const chartKnowledgeDict: ChartKnowledge = { @@ -194,7 +200,7 @@ export const chartKnowledgeDict: ChartKnowledge = { }, [ChartType.LiquidChart]: { index: 14, - visualChannels: ['x', 'y'], + visualChannels: ['value'], examples: [], knowledge: [ 'Liquid chart is used to display a single value, with the value range typically from 0 to 1. The value usually represents progress, completion, or percentage, and is associated with only one field' @@ -210,7 +216,7 @@ export const chartKnowledgeDict: ChartKnowledge = { }, [ChartType.CircularProgress]: { index: 16, - visualChannels: ['x', 'y'], + visualChannels: ['color', 'value'], examples: [], knowledge: [ 'Circular progress chart is also used to display progress data, presented in a circular form, with the values on the numerical axis typically ranging from 0 to 1.' @@ -279,7 +285,7 @@ export const chartKnowledgeDict: ChartKnowledge = { visualChannels: ['x', 'y', 'color', 'size', 'time'], examples: [], knowledge: [ - 'The five channels that need to be mapped in the dynamic scatter plot are: x, y, color, size, and time; the x, y, and size channels require numeric data fields; the time field must be mapped.' + 'The five channels that need to be mapped in the dynamic scatter plot are: x, y, color, size, and time; the x, y, and size channels require numeric data fields; the color channel is needed to distinguish different categories of scatter points; the time field must be mapped.' ] }, [ChartType.DynamicRoseChart]: { @@ -290,9 +296,12 @@ export const chartKnowledgeDict: ChartKnowledge = { }, [ChartType.SequenceChart]: { index: 30, - visualChannels: ['x', 'y', 'color'], + visualChannels: ['group', 'time', 'color'], examples: [], - knowledge: ['The three channels that need to be mapped in the sequence chart are: x, y, and color;'] + knowledge: [ + 'The three channels that need to be mapped in the sequence chart are: group, time, and color;', + 'The sequence chart is used to display information such as time nodes with a sequence.' + ] } }; diff --git a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts index cc148b0f..41caf32a 100644 --- a/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts +++ b/packages/vmind/src/applications/chartGeneration/taskNodes/getChartSpec/VChart/transformers.ts @@ -1304,7 +1304,7 @@ export const liquidField: Transformer = (context: C const { cells, dataset, spec } = context; const cell = getCell(cells); - spec.valueField = cell.value; + spec.valueField = (cell.value ?? cell.y ?? cell.size) as string; spec.indicatorSmartInvert = true; return { spec }; @@ -1387,7 +1387,7 @@ export const circularProgressField: Transformer = ( const cell = getCell(cells); spec.categoryField = cell.color; - spec.valueField = cell.value; + spec.valueField = (cell.value ?? cell.size) as string; spec.seriesField = cell.color; spec.radius = 0.8; @@ -1414,7 +1414,7 @@ export const indicator: Transformer = (context: Con if (!firstEntry) { return { spec }; } - const valueField = (cell.value ?? cell.y) as string; + const valueField = (cell.value ?? cell.y ?? cell.size) as string; const value = firstEntry[valueField]; const cat = firstEntry[cell.radius ?? cell.x]; @@ -2308,29 +2308,34 @@ export const dynamicRoseDisplayConf: Transformer = export const sequenceChartData: Transformer = (context: Context) => { const { cells, dataset, spec } = context; const cell = getCell(cells); - const yField = cell.y as string; - const xField = cell.x; + const timeField = cell.time as string; + const groupField = cell.group; const colorField = cell.color as string; const dataMap: { [key: string]: DataItem[] } = {}; dataset.forEach(data => { - dataMap[data[xField]] ? dataMap[data[xField]].push(data) : (dataMap[data[xField]] = [data]); + dataMap[data[groupField]] + ? dataMap[data[groupField]].push({ ...data }) + : (dataMap[data[groupField]] = [{ ...data }]); }); const dataDot: { [key: string]: DataItem[] | string }[] = []; - const dataLink: { [key: string]: string }[] = []; + const dataLink: { [key: string]: string | number }[] = []; Object.keys(dataMap).forEach(key => { + const dotList = sortArray(dataMap[key], [{ field: timeField, order: SortOrder.ASC }]).map((data, index, array) => { + const newData = { ...data }; + newData.node_name = `${data[groupField]}_${Math.floor(index / 2).toString()}_${data[colorField]}_node`; + return newData; + }); dataDot.push({ - [cell.x]: key, - dots: dataMap[key] + [cell.group]: key, + dots: dotList }); - sortArray(dataMap[key], [{ field: yField, order: SortOrder.ASC }]).forEach((data, index, array) => { + dotList.forEach((dot, index, array) => { if (index % 2 !== 0) { dataLink.push({ - from: `${data[xField]}_${Math.floor(index / 2).toString()}_${array[index - 1][colorField]}_node`, - to: `${data[xField]}_${Math.floor(index / 2).toString()}_${data[colorField]}_node` + from: array[index - 1].node_name, + to: dot.node_name }); } - data.node_name = `${data[xField]}_${Math.floor(index / 2).toString()}_${data[colorField]}_node`; - return data; }); }); spec.data = [ @@ -2360,10 +2365,10 @@ export const sequenceChartSeries: Transformer = (co { type: 'dot', dataId: 'dataDotSeries', - xField: cell.y as string, - yField: cell.x, + xField: cell.time as string, + yField: cell.group, dotTypeField: cell.color as string, - titleField: cell.x, + titleField: cell.group, highLightSeriesGroup: '', height: 500, clipHeight: 800, @@ -2398,12 +2403,12 @@ export const sequenceChartSeries: Transformer = (co { hasShape: true, shapeType: 'square', - key: (datum: any) => datum[cell.x] + key: (datum: any) => datum[cell.group] }, { hasShape: false, key: 'event_time_stamp', - value: (datum: any) => datum[cell.color as string] + value: (datum: any) => datum[cell.group as string] } ] } @@ -2422,10 +2427,10 @@ export const sequenceChartAxes: Transformer = (cont type: 'time', range: { min: fieldInfo.filter(fieldInfo => { - return fieldInfo.fieldName === cell.y; + return fieldInfo.fieldName === cell.time; })[0].domain[0], max: fieldInfo.filter(fieldInfo => { - return fieldInfo.fieldName === cell.y; + return fieldInfo.fieldName === cell.time; })[0].domain[1] }, layers: [ diff --git a/packages/vmind/src/applications/chartGeneration/types.ts b/packages/vmind/src/applications/chartGeneration/types.ts index 5103fed2..8cfa6572 100644 --- a/packages/vmind/src/applications/chartGeneration/types.ts +++ b/packages/vmind/src/applications/chartGeneration/types.ts @@ -11,4 +11,5 @@ export type Cell = { target?: string; value?: string; category?: string; + group?: string; }; From f4b9a45597842f95cd13584fcacb0672301351f6 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Tue, 24 Sep 2024 14:12:00 +0800 Subject: [PATCH 036/128] feat: add time cost in case study, fix vertify error --- .../browser/src/constants/capcutData.ts | 341 +- .../src/pages/NewDataExtraction/DataInput.tsx | 20 +- .../src/pages/NewDataExtraction/DataTable.tsx | 8 +- .../src/pages/NewDataExtraction/index.tsx | 5 +- .../data/{capcutData.ts => capcutDataCn.ts} | 235 +- .../experiment/src/data/capcutDataEn.ts | 149 + .../experiment/src/data/dataExtractionData.ts | 4864 +++++++++++++++++ .../src/pages/DataExtraction/caseStudy.tsx | 37 +- .../src/pages/DataExtraction/test.tsx | 23 +- .../src/pages/DataExtraction/type.tsx | 1 + .../src/pages/DataExtraction/verify.tsx | 25 +- 11 files changed, 5459 insertions(+), 249 deletions(-) rename packages/vmind/__tests__/experiment/src/data/{capcutData.ts => capcutDataCn.ts} (51%) create mode 100644 packages/vmind/__tests__/experiment/src/data/capcutDataEn.ts create mode 100644 packages/vmind/__tests__/experiment/src/data/dataExtractionData.ts diff --git a/packages/vmind/__tests__/browser/src/constants/capcutData.ts b/packages/vmind/__tests__/browser/src/constants/capcutData.ts index 39808c3c..a6c436f0 100644 --- a/packages/vmind/__tests__/browser/src/constants/capcutData.ts +++ b/packages/vmind/__tests__/browser/src/constants/capcutData.ts @@ -2,63 +2,360 @@ export const capcutMockData: any[] = [ { text: '地图带你看懂法国大选。 2024 年法国议会选举结果揭示了法国社会的撕裂，没有任何党派能获得 577 个议题中的绝对多数。左翼联盟、新人民战线执政党中间派联盟和极右翼国民联盟分别占据 182 席、 168 席、 143 席。除了这三家以外，其余党派席位最多的也只有右翼共和党的 48 席，因此左翼联盟、执政党和极右翼算是形成了三分天下的格局。今天我们便结合地图和数据，聊一聊法国大选。本期视频的所有分析均为个人观点，仅供参考。在开始之前，我先快速的放一下各党派在几个主要政治议题上的立场，有需要的朋友可以截图保存一下。', - fieldInfo: [ - { - fieldName: '党派', - fieldType: 'string' - }, - { - fieldName: '席位数', - fieldType: 'numerical' - } - ] + fieldInfo: ['党派', '席位数'] }, { text: '现在我们进入正题，首先需要说明法国的选区制度，全国共有 577 个选区，每个选区包括 10- 12 万的人口密集的地方选区就小，人口稀少的区域选区域就大。因此地图上各个政党所占的面积并不能与其票数划等号，而是要看具体的选区分布在人口密集的大城市，譬如巴黎、里昂、马赛，虽然看起来面积小，实际上选区数量很多，占的比重很大。', - fieldInfo: [] + fieldInfo: ['选区数量', '选区人口范围', '城市'] }, { text: '尽管如此，马克龙强调市场自由化，支持创新和创业，对欧盟一体化和国际合作的开放还是得到了许多大城市选民的支持。大城市仍然是马克龙的重要支持来源。除了大城市外，我们刚才提到过法国东北地区工业的衰落，其实在法国的西北地区又是另一番景象。之前那张关于中小工业城市分布的途中，我们可以清晰的看到西北地区的工业对于当代法国的重要性，当然工业只是一个缩影。我们再来看失业率地图，黄色代表 2022 年失业率最低的 20 个省，橙色是 21- 40 低的省份，可以看出西北部地区的失业率明显低于东北部。再看移民分布地图，西北部地区因为大城市少，而且离地中海更远，接受的移民也比较少。综上，西北地区的经济较为稳定，而且不受移民带来的社会问题困扰，日子过得比较舒服，这些选民对于未来持乐观态度，也成为了马克龙的另一主要票仓。除了以上三大势力外，浅蓝色代表的右翼共和党也获得了 48 个席位。他们的支持者主要位于经济同样比较富足，但是政治观点更为保守的地区。这里我就不展开讲总结。本期视频我们从法国官方的报告与数据出发，从经济与人口地理的角度分析了法国的大选结果。左翼联盟新人民战线以大城市为根基拿下最多的 182 席，马克龙的执政党中间派联盟则凭借大城市和西北地区的支持者取得了 168 席。乐旁的极右翼国民联盟则主要扎根于东北与东南地区，以 143 起居于第三这样的三分割据局面使得法国议会缺乏绝对多数，并且三方势力相差不大。可以预见在未来法案的通过上会面临极大的阻碍。举例来说，左翼和极右翼甚至存在联手撤回延迟退休法案的理论可能。虽然实际操作起来也面临很多困难，双方都不太愿意和对方合作，但即便是理论，可能也已经能表明未来的不确定性。', - fieldInfo: [] + fieldInfo: ['政党', '席位数', '地区'] }, { text: '住手，你们住手，不要再砸了，你们不要再砸了。万万没想到，瑞幸和库迪的九块九大战，快把星巴克给卷死了。前段时间，星巴克公布了第二季度财报，营业收入 85.6 亿美元，同比下降了2%，净利润大跌15%，股票市值一天内蒸发了 1, 150 亿人民币。另一方面，星巴克的咖啡也在悄悄降价。如果你手机上有星巴克的APP，几乎每天都会收到 5 张以上的优惠券，比如满60.10、满75.15、任意新冰乐 7 折等等，部分单品的团购价优惠下来低至 9 元。终于， 9.9 的风还是卷到了星巴克。在过去很长一段时间里，星巴克是小资生活的代表，一杯咖啡动辄几十块钱，也只有电视剧里那些白领们和云淡风轻的走进去，熟练地点一杯拿铁，找个位置坐下，悠闲地打开电脑喝咖啡。岁月静好人间，值得有人点一杯星巴克，朋友圈能发十几条动态，有人为了抢星巴克限量版的猫爪杯，能通宵排队，甚至大打出手。', - fieldInfo: [] + fieldInfo: ['营业收入', '营业收入同比', '净利润变化'] }, { text: '星巴克在中国国内的定位一直都是高端咖啡品牌，但是很多人不知道的是，它在国外的定位其实是平民咖啡，在美国一杯星巴克大杯美式咖啡大概是 2.95 美元，而根据美国劳工部的统计，美国平均月工资是 6, 228 美元，最低时薪是 7.25 美元，这是啥意思呢？ 1002.65 美元的星巴克还不到美国人平均月收入的 2, 000 份之一，也就是平常坐一趟地铁的价格吧。而在中国市场，星巴克的饮品价格普遍要超过 30 元一杯。如果按照美国的对应消费力，它的主力消费人群应该是月薪至少6万元的人。', - fieldInfo: [] + fieldInfo: ['咖啡价格', '平均工资', '最低时薪'] }, { text: '2019 年2月，星巴克就发售了一款粉爪杯，那它长这样？售价 199 元，但是在网上最高炒到了 1, 800 元。有人寒冬腊月在星巴克门口通宵排队，就是为了买到这么一个杯子。还有人因为排队顺序大打出手，最后喜提免费食宿。靠着这些营销方法，在很长一段时间里，普通人对于星巴克是仰望的，觉得去星巴克消费是很有品的，再往前推十年，你甚至可以看到有人去星巴克点一杯咖啡就可以发十几条朋友圈的各种角度各种场景，还要配文低调有实力，天天喝都喝腻了好像呢？甚至有人专门发帖认真的提问，第一次去星巴克主要注意什么？怎么装的像老手呢？我一开始还以为是来搞笑和反讽的，没想到点开帖子此还真的是教大家怎么去星巴克抓老手的，包括但不限于怎么下载APP，问店员这周用的是啥肚子萃取时间是多少？张度和烘焙度怎么样？要不要加糖和加奶？这唬得我一愣一愣的，但是时过境迁，如今星巴克已经支棱不起来了，一边是疯狂降价买三送一搞促销，一边是继续下沉到四五线城市。', - fieldInfo: [] + fieldInfo: ['商品', '售价', '最高价'] }, { text: '星巴克近期发布的 2025 中国战略愿景当中，中国总部直言不讳地表示，星巴克看中的不仅仅是全国 300 多个 d 级市场，也包括近 3, 000 个县域市场。星巴克的愿景也体现在它的选址变化上，它在中国的门店已经突破了 7, 000 家，但是这开店位置却让人越来越看不懂了。', - fieldInfo: [] + fieldInfo: ['市场级别', '市场数量'] }, { text: '一家卖咖啡的店。星巴克没落的第二个原因是当代年轻人更偏向实用消费主义。坦白讲，大部分人喝咖啡其实就是为了遮住那点咖啡因，好让自己在一天的工作中保持清醒。你跟不懂咖啡的人聊什么豆子产地、风味，他只会回你一句，冰美式和中药有什么区别啊？如果你面前有三杯咖啡，第一杯是星巴克的 30 元美式，后面两杯是瑞幸库里的 9 块 9 咖啡也让你买单，大部分人都会选择后面两个，当然也有人会吹星巴克的豆子有多么多么的好，所以它买的这么贵也是值得的。', - fieldInfo: [] + fieldInfo: ['品牌', '价格'] }, { text: '咱聊一下最新的重磅数据，反正挺复杂的，国内6月 M2 同比增长6.2%，预期6.8%。 M1 同比下滑5%，预期下滑5.4%。 M M 一剪刀差走扩至 11.2% 再创新高。6月人民币存款增加 2.46 万亿，其中居民存款增加 2.14 万亿，增量几乎全都是老百姓存的。与 M1 的下滑相对，上半年人民币存款总共增加了 11.46 万亿，其中居民存款增加 9.27 万亿。大头也是老百姓，但增速逐月放缓。', - fieldInfo: [] + fieldInfo: ['指标', '同比增长', '预期增长'] }, { text: '7月 11 日晚上，由于美国公布的 CPI 数据超预期回落，让市场对美联储降息预期大幅升温。从这场所的美联储观察工具看，虽然7月 30 一日的美联储议息会议，市场预期不降息的概率仍然是达到93%，但9月 18 日的美联储议息会议，市场预期降息一次的概率是达到90%，这比起上个月概率是上升很多。不过我还是得强调一下，这个美联储观察工具将来只能反映市场当前的预期态度，不能拿来预测美联储货币政策，因为这个概率是会不断随着最新经济数据变化而变化。比如要是下个月美国 CPI 出现较大反弹，那9月降息的概率就会大幅下降。而这次市场预期美联储9月降息的概率大幅上升，主要有两个原因，一、美联储鲍威尔在7月 10 日的国会听证会上整体态度偏戈。鲍威尔称，劳动力市场降温意味着持续高通胀的潜在源头已经减弱。他还表示，就业市场的进一步疲软可能是不必要的，也是不受欢迎的。鲍威尔说，通胀方面的工作还没有完成，我们还有更多工作要做，但与此同时，我们需要注意劳动力市场现况，我们已经观察到劳动力市场出现相当明显的疲软，有着新美联储通讯社之称的知名记者尼奇默尔斯认为，鲍威尔本周其实已暗示美联储的利率政策即将开始改变方向。', - fieldInfo: [] + fieldInfo: ['时间', '预计降息概率'] }, { text: '第二个原因是7月 11 日晚上 8 点半美国劳工部公布的通胀数据，6月 CPI 是同比上涨3%，市场预期值3.1%，前值3.3%。这次美联储 CPI 回落，更关键是 CPI 环比是负增长0.1%，这是美国 2020 年5月以来 CPI 环比首次出现下降，而且美国 2020 年5月还是因为疫情导致的 CPI 骤降，是比较特殊时期，所以美国 CPI 环比负增长确实不太常见。但仔细看美国 VI 月 CPI 的具体构成，感觉猫腻还是不少的。', - fieldInfo: [] + fieldInfo: ['经济指标', '同比', '环比', '市场预期值', '前值'] }, { text: '美国 VI 月 CPI 下降的主要贡献是汽油价格下跌。美国 VI 月汽油价格下跌了3.8%，抑制了当月通胀，抵消了食品和住房价格 0.2% 的上涨。比较诡异的是，美国原油期货价格6月是明明出现大幅上涨，这是因为原油期货价格传导到汽油价格有一些迟滞效应，但那样的话，下个月公布的 CPI 数据，汽油价格可能就得反弹了。要是下个月公布的 CPI 数据，汽油价格还继续下降，那就实在说不过去了。美国刨除能源和食品价格的核心通胀率6月是3.3%，但整体降幅还是低于CPI。', - fieldInfo: [] + fieldInfo: ['价格类别', '价格变动比例'] }, { text: '美国通胀目前最顽固的就是服务业通胀，美国 VI 月服务业通胀仍然是同比上涨5%。美国毕竟是服务业为主的国家，服务业通胀还高居5%，美国要说自己已经控制住通胀，完全就是忽悠人。不过虽然美国大选临近，美国现在经济数据基本是为选型服务，比如已经假的不能再假的美国非农就业数据，美国 VI 月非农就业人口增加20.6万人，高于市场预期的 19 万人。然而美国同时把5月数据从 27.2 万人大幅下修至 21.8 万人，4月从 16.5 万人修正至10.8万人，修正后两个月合计较修正前减少 11.1 万人。', - fieldInfo: [] + fieldInfo: ['月份', '修正前就业人口增加', '修正后就业人口增加'] + }, + { + text: '我注意到虽然这次市场大幅提高了美联储9月降息概率，但美股和日股反而不涨反跌，纳斯达克在7月 11 日是下跌了2%，日股在7月 12 日也跟随下跌了2.45%。这是市场反身性效应的某种预演，股市炒的是预期，之前美股和日股是基于美联储降息预期，已经提前涨了一年多了，那么当美联储真正降息之后，市场是可能出现反身性效应，也就是所谓利好落地势利空的说法。当然股市走势千变万化，这也只是其中一种可能性。历史的参考例子，比如 2004 年美联储降息， 2006 年停止加息， 2007 年开始降息，但股市是一直涨到 2007 年底，随后自带危机爆发，股市开始大跌。我之前也梳理过，从 1980 年以来，美联储每次加息超过 5% 的幅度，首次加息后的 2- 4 年内都会爆发金融危机，这次美联储是 2022 年开始加息，所以按照历史路径， 2024 年到 2026 年是有可能爆发世界金融危机，这个结合当前国际局势和地缘形势，还是有挺大的可能性。', + fieldInfo: ['股市', '时间', '涨跌幅度'] + }, + { + text: '以前的小学空荡荡的，老年人养起了鸡鸭鹅狗，彻底荒废了整个黑龙江省， 10 年时间荒废了近六成，小学加起来有 1, 900 余所，整个东北十年荒废了 6, 800 余所小学，少了一半。', + fieldInfo: ['地区', '荒废比例', '荒废数量'] + }, + { + text: '而小学缺孩子这个趋势早已经蔓延至全国各地了。东部的兹西县，有的小学一个班只有一个学生。华南的徐文县，去年某小学开学，一年级也只有一个学生。中部人口大省河南，据测算， 2023- 2027 年小学学龄人口预计下降 200 多万人，缩水超两成，出现了基数的坍塌。从全国来看，基于学龄人口的预测显示，全国超 1, 400 个县域中，近九成县域小学学龄人口预计下滑，小学鹤岗化以谁也没想到的方式在扩散，从东北开始到大江南北，下一步可能是上海最新的总和生育率只有 0.6 了，该来的总是要来，从民政局冷冷清清到妇产科缺孩子，再到幼儿园关停潮，现在轮到了小学关停潮，这个传播链条还在扩散。', + fieldInfo: ['地区', '时间', '小学学龄人口变化'] + }, + { + text: '实际上，如果鹤岗化只是局限于教育领域，那还好说，但不是实际情况复杂的多楼市，像鹤岗那样房子白菜价的城市越来越多了，据不完全统计，至少有 10 个省 24 个城市陷入几万元买房的讨论。不是每平方米单价几万，而是一套房总价几万。网传广东惠州6万，广西南宁5万，山东东营4万，江苏南京3万，黑龙江大靶1万，这个传播势头在这轮楼市调整的加持下，现在已经来到了北京的外围，在京津两市的交界处，抹楼盘从 160 万元降到了 39 万，而且打了骨折还卖不出去。', + fieldInfo: ['城市', '房价'] + }, + { + text: '地方财力，之前鹤岗是全国第一个财政重整的地级市，甚至传出来停招公务员，现在过紧日子的城市也越来越多了。秦岭深处某县人口只有3万，编制人员却有 2, 194 名，一年的行政管理支出 1, 800 万，排在支出的首位。乌蒙山区某县一般公共预算收入 7 个亿，但工资预算总支出 26.3 亿，其中在职人员 20 亿，离退休人员 1.7 亿，零聘人员 4.6 亿。注意一个细节，在职人员数量 1.5 万，临聘人员数量 2.8 万。', + fieldInfo: ['地区', '人员数量', '人员类别', '公共预算收入', '工资预算支出'] + }, + { + text: '中国房价一度被视为坚不可摧的资产堡垒，更一度有京沪永远涨的口号。然而，自 2021 年以来，包括一线城市在内，房价持续低迷，深圳全市二手房均价距离 2021 年初的最高点跌幅已接近40%，而且还没有停下来的意思。各热点城市二手房每成交一套就要多出好几套，新增的房源和房价表现几乎完全正相关的是飞天茅台的价格， 53 度。', + fieldInfo: ['城市', '年份', '房价变化'] + }, + { + text: '飞天茅台在 2021 年巅峰时期，一瓶售价超过 3, 500 元，如今已经快跌破 2, 000 元，是巧合吗？过去一线城市房价和飞天茅台价格可以说是最硬的人民币计价资产了，甚至比现金还要优质。一线房产和飞天茅台在相当长的一段时间内有两个相同属性，一他们可以长期增值。二他们易于套现。然而现在情况出现了前所未有的变化，房价和飞天茅台两者双双在 2012 一年见顶。', + fieldInfo: ['年份', '价格'] + }, + { + text: '这不是巧合，其他很多数据也都在 2021 年见顶，比如另一个在 2021 年见顶并开始走下神坛的保时捷。保时捷销量的恶化还在加速，过去一年多，中国市场上保时捷的落地价可以说是惨不忍睹。近期，保时捷只卖 44 万的话题也引发了热议，其华南区域一家终端门店称， Macan 正在进行优惠促销，最高优惠 16 万，该车优惠后最低售价为 44.8 万。另外，在山东、湖北、江西、福建、浙江、江苏等多省份，该车均出现了 50 万元以下的裸车价，而报价达到 103.8 万的泰肯，现在 70 多万就可以拿下。目前，保时捷几乎所有的车型都可以打 7- 8 折。', + fieldInfo: ['车型', '落地价', '优惠金额', '报价'] + }, + { + text: '不只是保时捷，包括奔驰、宝马、奥迪在内的豪华汽车品牌今年以来都在大幅降价，但仍然止不住销量断崖式下降。 2024 年一季度，保时捷中国卖出 16, 340 辆，同比大幅下降24%。保时捷在今年 5 月份仅卖出 4, 633 辆，同比去年5月下滑高达40.61%。这说明保时捷在中国的销量正在加速减少。', + fieldInfo: ['时间范围', '销量', '同比'] + }, + { + text: '从两年前开始，特斯拉的 3 和 y 中国售价就是世界最低的， 7.98 万可以买到原本指导价 13.18 万的油电混动的卡罗拉。做一个对比，美国的油电混动卡罗拉的起售价是 2.35 万美元，有时甚至还要加价。按照美元人民币汇率计算，这款车的中国售价居然只有美国的一半水平，尽管配置存在差异，但不影响价格差异巨大的这个结论。除了卡罗拉外，汉兰达和凯美瑞也都大幅降价，即使是两年前，我们也很难想象只要 14.98 万人民币就可以买到最新款的混动版凯美瑞。', + fieldInfo: ['车型', '售价', '指导价', '美元起售价'] + }, + { + text: '中国物价的下降不仅仅体现在商品上，服务价格也是类似的趋势。举个例子，十多年前我常驻北京，当时经常在晚上十一二点从首都机场打车到西直门这样一段单程大约需要 110- 120 元，而现在滴滴大概只要 60- 70 元。如果是现在的出租车，价格和十多年前还是一样的。', + fieldInfo: ['交通方式', '时间', '价格区间'] + }, + { + text: '以工行、农行、建行发布的数据来看，其 2023 年个人住房贷款不良率分别由 2022 年的0.39%、0.51%、 0.37% 增长至了0.44%、 0.55% 和0.42%，基本都实现了两位数的增幅。大家别觉得这些小数字没啥大不了的，要知道这三家银行每一家的个人住房贷款余额都超过了5万亿，而且按揭贷款往年基本上都是银行稳赚不赔的买卖。供建农三家银行之所以每年能够包揽中国最赚钱企业的前三名，按揭贷款所带来的收益贡献巨大。现在这个优质资产的不良率正在以每年两位数的增幅增加。', + fieldInfo: ['银行', '年份', '不良率'] + }, + { + text: '你是银行，你慌不慌？而另一项作为佐证的数据则是法拍房，大家知道现在法拍房的数据有多夸张吗？根据瀚海研究院发布的数据显示， 2022 年全国共挂牌法拍房 98 万套，去年这个数字变成了 141 万套，增长了43.9%。而今年光是一季度的挂牌数量就已飙升至 60.44 万套通，同比上涨192%。这种局势下，银行要是再不改变断供处置策略，那今年的法拍数量估计有望达到 200 万。', + fieldInfo: ['对应时间', '法拍房数量', '同比'] + }, + { + text: '事实上，对银行来说，现在的行情即使他收了房也难以处置。我们以北京为例， 2023 年北京挂牌法拍房 8, 153 套，最终成交仅 2, 771 套，处置率为33%，这还是房价波动相对较小的北京，换到其他已经跌穿首付的地区，处置率恐怕只会更低。而在法拍流程里，流拍和拍品二次上拍都会在此前的价格上更进一步降低，这也导致了银行回款难度的进一步提高。虽然按照现在的规则，这部分差价是由贷款人承担的，但对方既然已经到了选择断供的地步，可想而知最终也执行不了多少。', + fieldInfo: ['地区', '法拍房数量', '成交量', '处置率'] + }, + { + text: '另一项推动银行改变策略的原因，则是今年4月 30 号的一场会议，这场会议确定了房地产行业未来一年的发展方向，统筹消化存量房产和优化增量住房，用大家都熟悉的话来说就是去库存。根据国家统计局的官方数据显示，截至 2024 年5月，我国未出售商品房为 7.46 亿平米，远超 5.9 亿平米的正常库存水平。而整个上半年，根据 CRS 的统计，全国 222 个城市总计出台了 341 项宽松政策，但带来的效果均不理想，无论是销售面积还是投资金额，仍然在持续走低。', + fieldInfo: ['时间', '商品房面积', '面积类别'] + }, + { + text: '大家好，我拍拍一名做过财经记者，大学老师和滴滴司届 up 主。 2023 年，我国汽车出口量达到了 491 万辆，超越日本成为世界第一汽车出口国。要知道，日本在这个位置坐了 8 年之久，而中国仅在过去三年时间里接连赶超韩国、德国、日本。中国汽车出口， 2021 年 202 万辆， 2022 年 311 万辆， 2023 年 491 万辆。', + fieldInfo: ['年份', '国家', '出口量'] + }, + { + text: '中国汽车转眼间为何变得这么受欢迎呢？又到底是哪些国家在购买中国汽车？中国卖给老外汽车又是些什么品牌和价位车型？本期视频就为大家打开中国汽车出口全球第一背后的真实数据。首先，中国出口的 491 万辆汽车都是些什么车呢？根据乘联会统计，中国乘用车出口量前十车型分别为名爵ZS、特斯拉 model y、奇瑞瑞虎7、特斯拉 model 3、名爵 4 EV、奇瑞虎5X、欧盟达名爵5、缤越元plus。除了特斯拉的 model y 和 model 3，其他车型国内指导价基本都在 10 万元左右，比如排名第一的名爵ZS，指导价 8- 9万元，最便宜的名爵 5 和缤越低到6万元就能拿下，可见中国汽车出海主打的还是一个性价比。', + fieldInfo: ['车型', '指导价'] + }, + { + text: '如果按燃油车、新能源车的分类来看， 2013 年中国出口燃油车 371 万辆，出口新能源汽车 120 万辆，新能源车占到出口总量的25%，虽然这个占比目前只有 1/ 4，但去年新能源出口增速是77.6%，势头不可谓不猛。', + fieldInfo: ['车辆类型', '年份', '出口量', '出口占比', '出口增速'] + }, + { + text: '那中国汽车出口都卖到了哪些国家呢？ 2013 年中国汽车出口量前十的国家分别是，俄罗斯90.9万辆，墨西哥 41.5 万辆，比利时 21.7 万辆，澳大利亚 21.4 万辆，英国 21.4 万辆，沙特阿拉伯 21.3 万辆，菲律宾 17.2 万辆，泰国 16.9 万辆，阿联酋 15.9 万辆，西班牙 13.9 万辆。按地区来看的话，欧洲市场占中国汽车对外出口的38%，远超其他任何单一大洲，可见中国汽车正在得到全世界更多人的认可。', + fieldInfo: ['出口国家', '出口量'] + }, + { + text: '当然了，中国汽车出口世界第一，又不得不提议俄罗斯和墨西哥这两个国家可以说去年是把中国车买爆了。 203 年，中国对俄罗斯的汽车出口量从上一年度的 16 万辆暴增到了90.9万辆，增加了468%。在俄罗斯的新车市场中，第一名是俄罗斯品牌拉达，第二至第七名则全都是中国品牌，比如第二名是奇瑞金车，市场占有率11.2%。第三名是哈弗，新车，市场占有率10.6%。俄罗斯卖最好的新能源车也是来自中国的极客。目前中国汽车已经占据俄罗斯新车市场的51%，可以说是拿下了半壁江山。而对于中国而言，仅俄罗斯一个国家 203 年就贡献了中国汽车出口增量的42%，甚至有俄罗斯本土汽车经销商预测， 2024 年中国汽车可能占据俄罗斯新车份额的80%。', + fieldInfo: ['年份', '出口量', '占有率'] + }, + { + text: '有人说俄罗斯满爆中国汽车是因为欧美的贸易封锁，这也步无道理。 2013 年在俄罗斯的新车市场中，欧洲的市场份额从 18% 降到了4%，韩国从 16% 降到了6%，日本从 12% 降到了5%，和欧日韩都是对俄罗斯实行了限制出口国家，其中就包括了部分汽车，可以说中国汽车吃下的正是欧日韩在俄罗斯丢掉市场。', + fieldInfo: ['国家', '新市场份额', '旧市场份额'] + }, + { + text: '当然了，除了俄罗斯之外，其他国家也在买中国汽车，比如墨西哥。 2013 年，墨西哥所有销售汽车中有 25% 来自中国，而在 6 年前这个数字为0。澳大利亚也在不断买中国汽车，最受澳大利亚欢迎的中国汽车品牌是名爵，去年卖了 5.8 万辆。在新能源车市场，比亚迪则占据了澳大利亚的新能源汽车 14% 份额，位于第二名。', + fieldInfo: ['国家', '年份', '中国汽车占比'] + }, + { + text: '当然，这里也不得不提一下第一名，那就是特斯拉市场份额高达53%，在东南亚市场，中国车企业销量在 2013 年同样实现小幅上升，最典型的就是泰国，在泰国的新能源车市场，中国品牌占据了 80% 的份额，比如比亚迪的原 plus 就是泰国的新能源车爆款，那到底是什么原因让中国汽车爆卖呢？基本还可以总结为三方面原因，首先是全球疫情爆发，由于中国汽车的供应链完善，疫情期间仍能维持稳定生产，而日韩这些过去的出口大户受疫情影响，芯片、钢材、橡胶等关键原材料短缺，不仅汽车产能下降，而且成本升高，这就让中国汽车更具性价比。而随着中国国内新能源汽车市场越来越卷出海，成为不少中国车企的选择，比如比亚迪 2023 年进入全球 58 个国家和地区，出口汽车 24 万辆，是上一年度的 3.34 倍。在泰国新能源车市场，比亚迪单独占到了 40% 的市场份额，是名副其实的泰国新能源汽车销冠。而且中国新能源汽车并非只是具备成本优势，汽车与 AI 互联网融合的智能化更是中国车企的拿手好戏。从豪华配置到智能大屏，从外观设计到内饰比拼，这让中国新能源汽车的溢价能力明显变高。2019 年中国新能源汽车平均出口价格每量只有 5, 000 美元， 2022 年涨到了 2.2 万美元。比如比亚迪汉在欧洲发布时价格接近 50 万人民币，是国内售价的两倍多。在泰国、以色列、新西兰等多个国家，比亚迪也已经是新能源汽车的销售冠军。不过，中国汽车征服海外虽然是一部励志爽门，但其实有不少挑战。', + fieldInfo: ['品牌', '市场份额'] + }, + { + text: '其实中国汽车出海不禁让人想起曾经的中国摩托车出海。 2000 年前后，中国摩托车进军越南，一度占据了 80% 的越南市场份额，但不到三四年时间，却被日本摩托车打得片甲不留。如今日本摩托车在越南占据 95% 的份额，而中国摩托车百分之一都不到。曾经也有大量中国摩托车车企在越南建厂，但却形成了恶性竞争的关系，疯狂打价格战，导致服务和质量越来越差，越南的中国摩托车车企仿佛是飘在越南的。', + fieldInfo: ['时间', '摩托车品牌', '市场份额'] + }, + { + text: '每个中国制造品牌的背后都有一批优秀的零部件供应商，像汇川技术、恒力液压、先导智能、顺域光学、军胜电子这样的零部件企业也是中国制造的骄傲，只不过知名度无法媲美消费者直接接触的终端品牌。按照官方的口径，中国规模以上也就是年销售收入 2, 000 万以上的制造企业有 44.5 万家。至于中国一共有多少家制造业企业存在各种口径，从 300 多万家到近千万家不等。万德资讯给我的数据是，中国大约有 622 万家存续的制造业企业。海之在线是一家总部在上海，聚焦中间品贸易的数字化平台，连接着 70 万家工厂，他们给我的数据是，中国规模以下的中小微工厂大致有 400 万家。这期节目标题中的 400 万家沉默工厂处处记载于此。海志在线的创始人、 CEO 佘莹对我说，从平台看， 40% 的工厂规模不到 50 人，近 90% 的工厂不到 500 人，大部分工厂的年产值在数百万元到数千万元。如果和大企业比，你可以说他们就是一个个的小做法。如果走进去可能会看到老旧的机器上油漆斑驳，可以看到生产计划就用记号笔写在车间墙上挂着的白板上，甚至会发现用破洞的木板随意围搭起来的厕所，待客的茶水里则混杂着浓浓的机油味。但他们就是中国制造业毛细血管层面的供应链小节点，勤勤恳恳的维护设备、搞生产，他们最在意的是生存，是接到订单以及在满足客户之后能够完整的收到货款。', + fieldInfo: ['企业类型', '企业规模', '企业数量'] + }, + { + text: '有报道称，一些地方政府面临收入短缺，要求企业缴纳可追溯到 1990 年代的税单。这种紧缩政策在房地产市场寻找几步的时刻，会损害上信心和经济。高盛认为，中国中央政府可以通过加大对西方政府的财政支持来切断机房政府无紧缩所出现的负面溢出效应，那如同美国监管机构在次贷危机期间通过成为最后贷款人来切断金融危机的传播一样。关于出口和机产之间的分化，可以同中国的金属生产中得到证实啊。铝和其他废且金属的产量相比疫情之前上升了 20% 以上，而钢铁的产量下降了 5% 到10%。在房产方面，开发商越来越依赖银行融资， 5 月份对开发商的银行贷款同比增长了19%。而随着房地产销售的下滑，房贷和存款跟预付款的比例同比下降了 30% 到40%。高盛银行股票团队预计从 2024 年到 2026 年，房地产贷款将增加 4.5 万亿人民币，以完全期待收缩的房地产债券和设防的影子银行贷款。', + fieldInfo: ['时间', '金属种类', '产量变化'] + }, + { + text: '中国经济分化并不止于出口和房地产基础设施的固定资产投资。细分项显示，建立燃气和水的生产投资已同疫情前水平翻倍，远远超过了整体的基建的投资增长。因为中国政府首先优先考虑农源供用安全和脱碳。在零售销售当中， 5 月份代线销售商品同比增长13%，而餐饮销售仅增长5%，线下商品的销售保持在去年同期的水平。', + fieldInfo: ['经济领域', '同比增长'] + }, + { + text: '7月中旬将举行两场重要的政策会议，前者将专注于评估当前的经济状况，并为今年剩余时间制定周期性的政策安排。后者将专注于至少未来五年解决经济结构当中的重大改革议程。鉴于一季度的实际， g GPT 同比增长5.3%，而且去年基数较低，那么 G2 季度增长可能高于5%，政府的全年增长目标仍然在轨道上。', + fieldInfo: ['季度', '同比'] + }, + { + text: '高盛认为，政策执行者不会在 7 月份的首场会议上释放任何重大的刺激措施，宏观政策的证件可能大于维持当前立场和执行现有的政策。另一方面，鉴于 5 月份宣布的最后一批措施不及预期，可能存在引入更多期产宽松政策的可能性。在货币政策方面，资本外流的担忧和银行利润率的下降限制了央行降息的能力。高盛预计三季度将降息 25 个基减，以适应大量政府债券发行，并预计季四季度9月首次降息后会再降息 10 个期减。在财政政策方面，高盛预计政府债券的发行将在下半年显著增加，以完成年初发行缓慢的全年配额。除非增长急剧放缓，否则基建的投资不会加速太多。对于政府 3 月份公布的预算计划，高盛虽然预测中国的增强型的弹盛赤字从去年的 gdp 11.2% 会适度扩大到今年的11.9%，但由于今年出口强计可能会存在财政扩张不及预期的风险。信贷政策方面，正如央行行长6月 19 号陆家嘴论坛上所说，由于金融套利的虚假贷款和监管机构随后对金融系统中这种资金空转的打击，信贷增长与 GDP 增长之间的联系已经减弱。预计摄容总量的增长将同去年的 9.5% 放缓到今年的9%。在住房政策方面，4月的政治局会议表明，决策者希望严防房地产市场的尾部风险。由于地产价格和活动的持续下行，以及机房国企通过央行的贷款计划购买空池公寓的速度缓慢，高盛预计进一步削减房贷利率以刺激需求，同时为去库存提供更多的资金和效率的支持。在外汇政策方面，鉴于美元持续强势和资本外流的压力，高盛认为央行将在短期内保持美元对人民币汇率的稳定，三个月的高盛预测是7.3，因为外汇市决策者可以迎来抵消关税对出口负面影响的工具。 2018- 19 年的经验表明，如果特朗普赢得美国大选，而且正如他最近几个月所宣称的，会对中国实施正大的关税，那美元兑人民币可能会显著贬值。', + fieldInfo: ['降息时间', '降息基点', '原因'] + }, + { + text: 'Because Japan is one of the most import dependent countries in the world, importing over 90% of its energy and over 60% of its food, a weak yen means inflation has returned to Japan for the first time in decades. ', + fieldInfo: ['import ratio', 'import type'] + }, + { + text: ' For context, in the post war years, Japan experienced many decades of rapid economic growth in a period dubbed the Japanese Miracle. From 1955 to 1990, Japanese growth averaged 6.8% per year, and GDP multiplied eight times, with growth falling below 3% only once during the 1974 oil shock.', + fieldInfo: ['year', 'gdp growth rate'] + }, + { + text: `Anyway, this anxiety subsided in the 90s when Japan experienced an enormous financial crisis after a rapid appreciation in Japanese stock and real estate prices during the 80s. In 1990, the bubble burst and continue to burst for a while. In the decade after 1990, residential house prices fell by more than 50%, commercial property prices fell by something like 85%. And Japan's main stock index, the Nike 2,2,5 fell by about 75%. Japan's economy never really recovered and growth and inflation both remained close to zero until very recently.`, + fieldInfo: ['rate value', 'rate type'] + }, + { + text: `From 2016 until late last year. The Bank of Japan even began what's called yield curve control, which essentially involved buying up enough debt to guarantee that government borrowing costs wouldn't go above a certain level. Japan's ultra loose monetary policy came under pressure in 2022 when inflation started rising across the world. Usually, central banks raise interest rates, but the bank of Japan decided not to, both because inflation was relatively low in Japan, but also because thanks to its enormous debt burden, even a slight raise in interest rates would translate to a massive increase in debt servicing costs, especially for the Japanese government. Unfortunately, things have become more difficult as other central banks have raised rates, making their currencies relatively more attractive and sparking a decline in the yen. In the last year, the yen has fallen from about 130 to the dollar to a 34 year low of 160 to the dollar on Monday. Now the speed and severity of this decline presents a difficult dilemma for the bank of Japan because thanks to Japan's reliance on imports, it sparked significant inflation in essential items like food and energy. But they still don't want to raise rates for the reasons we've just mentioned earlier. This is why on Monday evening, instead of raising rates, the bank of Japan used billions of dollars worth of Japan's foreign exchange reserves to buy up the yen on the international market, artificially inflating its value. While this seems to have worked in the short term, as of Tuesday morning, the yen is now trading at nearer 155 to the dollar. It's both expensive and fundamentally unsustainable. Even if the bank of Japan has some of the largest foreign exchange reserves in the world. All in all, assuming the bank of Japan won't engage in significant rate hikes, this means that the yen is very much dependent on what goes on in the rest of the world. If inflation comes down and other central banks start cutting rates, then this will reduce some of the pressure on the yen. But if inflation turns out to be stickier than we'd like, which seems to be the case, then the divergence between the bank of Japan and other central banks will persist, which means more downward pressure on the yen. If this happens, then the bank of Japan won't be able to stave off the yen's decline with exchange reserves forever. And eventually, they'll have to choose a horn of their uncomfortable dilemma, either to just accept the yen's decline and all the inflation related political turmoil that comes with it all, raise rates and just hope that the world's most debt burdened economy can somehow deal with it.`, + fieldInfo: ['date', 'yen to dollar'] + }, + { + text: `Blanket is all over the news. Linkage is now more valuable than Zomato. Linkage reported over 2,300 for rupees and revenue. And then now speak about Zomato, they speak about blinking bigger than tomato right now with the valuation of roughly $13 million and a market share of 46%, it has disrupted India's 23,000 crore quick commerce industry, a company that Zomato acquired in 2022 to enter into quick commerce, but now has become more valuable, the Zomato's own food delivery business. In fact, the company 3 x its revenue from 800 crores to 23 crores and is expected to break even in the first quarter of Fi 2025.`, + fieldInfo: ['company', 'marker share', 'valuation'] + }, + { + text: `By the way, this is going to be detailed, so feel free to pause the video wherever you feel confused. First of all, let's take a realistic average order value of 600 rupees, which is very close to what most people usually order on blink it. If you look at the revenue side, which is the money that bind it earns here for themselves from each order is divided into three sources. The first one is warehousing services and marketplace commissions. This is basically the amount that suppliers are paying to blanket for showing and selling their products. And see on every order of 600 rupees, 11 to 13% is coming from suppliers, which is roughly 72 rupees.`, + fieldInfo: ['revenue source', 'revenue value', 'revenue ratio'] + }, + { + text: `Now the second part of the income of this order is the ads that come companies show on blanket. This is the price that brands pay to show their products above other products as you scroll the app. For example, brew might pay to show its coffee first when someone searches for a keyword like coffee, it's roughly 2.3 to 3.5%. In our case, let's take 3.5% and it will come down to 21 rupees.`, + fieldInfo: ['keyword', 'ad rate', 'cost'] + }, + { + text: `The next is customer fees, which includes your delivery fees, handling fees for packaging and delivering the food to your doorstep, and even additional fees like fees they charge you for having a small cut. This percentage comes at around three portion of the average order value and is roughly 18 rupees in our case. By the way, there are also other levers like membership plans or free delivery plans that these plan platforms try to sell you often like Zepto does it with their offering of zeptopass. But if we don't over complicate and dive much deeper into this, we see that in a nutshell, on an average order of 600 rupees, blanket owns roughly 110 rupees. This 110 rupees is known as a take rate, the share that blanket keeps for itself from an order.`, + fieldInfo: ['take rate', 'customer fees ratio'] + }, + { + text: `Now let's come down to the cost side. Even here you have four elements. The first one is the biggest one, which is the last mile delivery cost, which is the last step when the riders deliver the orders to you and cost about roughly 7% to blanket. And in our case, it would come down to 42 rupees. The next one is dark store mid mile and warehousing cost. This entire combination of cost comes down to about 6.5%. And in our case, it would be 39 rupees. The other variable cost, which includes packaging, washes, support, communication and payment charges are roughly 2%, which comes down to 12 rupees. And now the fourth and the last one is customer acquisition cost, which is the discount, the incentives and the offers they try to give you to make tempting these for you. This comes at about 0.2 to 0.3% at about 1.8 rupees. So if you subtract these two amounts, blink it on roughly 15 rupees from entire transaction. This 15 rupees is not the net profit, by the way, but the contributing profit. Now what is that? See, contributing profit is the profit that the company is earning by serving each order. And company considers only the variable expenses in this case, which is the expense that we have already discussed. And it does not mean net profit because there are so many fixed expenses that are not considered like expensive salary of tech folks, rent, insurance, depreciation, and all similar big sums of money.`, + fieldInfo: ['cost element', 'cost ratio', 'cost value'] + }, + { + text: `The answer to this is the dark stores, the 2,500 to 4,000 square feet big stores that are located in 1.5 to 3 kilometer radius near your homes to ensure super quick delivery. And by the way, these stores are super big. For example, if your nearest kirana shop has about 1,500 SKUs, these stores can have 4 x a number of excuse. In fact, a highly efficient dark store can do a better gross merchandise value per square foot than a highly organized supermarket like Dmart. So while a dark store or blanket can do a GMV of 90,000 rupees per square fit, Mart can only do a GMV of 47,000 rupees per square fit. But how does this work? See, these stores are like supermarkets, but have no Hawkins, which means that only rider can go and collect stuff from there. These stores have a lot of inventory that comes from Mother Warehouse store, which is located in the outskirts. So to give you context, for every 40 dark stores located in the city, there is always a mother warehouse, which is located at the outskirts or city. And that is more than 10 times big as a dark store, which is about 20,000 to 1 lax, 75,000 square feet big. It is super huge. And companies don't set dark stores everywhere, by the way. They are smartly set based on multiple parameters like average household income of the area, peak time traffic of the area, infrastructure structure of the area, and also the population density. Also, there's usually about a staff of 25 to 30 people who are working in three shifts in these dark stores who would take care of the packaging. And as we've discussed earlier as well in the video, that the operating cost for a dark store comes at about rupees 22 for each order. And if you want to understand this calculation better, we have put it here. So you can pause the video and look at this table.`, + fieldInfo: ['store type', 'store size', 'SKU', 'GMV', 'staff count'] + }, + { + text: `Now blanket has done a really solid job here as they have, right now, the highest number of dark stores with 451 stores in 27 cities compared to 450 stores of Insta margin, 25 cities, 30 of Zepto intensities and 350 stores of Big Basket in 35 cities. By the way, this is not something they built in a day because it was the first grocery app in the country, which started in 2014 as growers. So their team and their execution is way more experienced if you compare them with other competitors, and now they're just building on this and increasing their penetration throughout the country. By the way, they're mostly penetrated in north and east India, and 90% of the GMV comes from top paid cities. But as they enter south and other cities, this can be a huge opportunity for them as they already have an experience DNA running in the organization.`, + fieldInfo: ['company', 'stores number', 'cities number'] + }, + { + text: `Now coming down to the second insight, which is cracking high average order value. But why are we talking about AOVs? Average order value plays a big role in quick commerce because bigger the average order value, bigger is the contribution margin for the company, which means that delivering just a set of bananas or apples is less profitable for blanket. Then delivering a set of bananas, apples, onions, tomatoes and a packet of bread together. And blanket has the highest average order value if you compare it with all the comparators. And just look at the stark difference by yourself. For big Basket, the AOV is about 400 to 500 rupees. For Zepto and Instamar, it is around 450+. And for blanket Au UV is about staggering 6,35 rupees. This is something that is definitely giving them an edge in pulling one of the most important levers in the ecosystem. In fact, in the last quarter, this number was 523 rupees at the start of Q1 fi 23. It just shows the speed at which they are growing really fast, and they have done this really well through their amazing SKU strategy.`, + fieldInfo: ['company', 'AOV'] + }, + { + text: `Not talking about the third insight, new customer acquisition. See, blinkits market share has not been the highest forever. It was 32%2082. And in the same period, Instama's market share has fallen from 52%, while Zeptos has increased from 15 to 28%. And we had to talk about the elephant in the room, the Zomato Effect. See, Zomato is the biggest food delivery app that has more than hundred million active users every month on its app and a market share of more than 56%. When it comes to food delivery, these users are more than three ties what blanket has at the moment. So even getting 5% of Zomato's monthly active users as new customers could bring more than 33% rise to their current Mau base.`, + fieldInfo: ['company', 'marker share'] + }, + { + text: `Want to show you something cute little condo in Toronto's Harbor Front neighborhood, bustling part of the city. It goes on the market in the summer of 2022. The sellers put it up for $480,000. It didn't sell early. 2023, it's back on the market for 460 k. No luck later that year, posted again at 4,50 k. Still nothing. And a few months ago, the sellers tried again. At $430,000, no takers. This condo was sitting on the market for more than 400 days without a sale. Right now in Canada's biggest cities, there's a ton of condos like this struggling to sell. A condo sales in Toronto, for example, haven't been this low since the financial crisis in 2009.`, + fieldInfo: ['year', 'price'] + }, + { + text: `Let's compare what a condo investment in Toronto looked like in 2016 to today. So according to the Toronto Real Estate Board, the average price of a one bedroom condo back then in 2016 was about three hundred thousand dollars and with 2016 interest rates about 2.7%, that worked out to a mortgage payment of about eleven hundred dollars a month. Rent for the average one bedroom apartment at that time was about sixteen hundred and sixty dollars. Not bad from an investor point of view, even with, you know, maintenance fees, property taxes, that condo is essentially paying for itself while you build equity.`, + fieldInfo: ['average price', 'inerset rate', 'mortgage_payment', 'rent'] + }, + { + text: `Now what if you were to buy a one bedroom condo today in Toronto? That could cost you about 550 grand. So prices have very clearly gone up, but so have interest rates. The carrying costs on a property are far higher than they ever used to be. Well, not ever, but in recent memory. At current rates, 6.8%, maybe now you're paying more than $3,000 a month for your mortgage payment alone. Then you've got, you know, property taxes, maintenance fees and the average rent per unit like that, about 24 hundred dollars a month you're now paying out of.`, + fieldInfo: ['price', 'inerset rate', 'mortgage_payment', 'average rent'] + }, + { + text: `Now the renewals come in and you're losing a lot of money, but you've actually come to realize in the last 12 months the price of that condo has probably dropped about $40,000. So you're starting to enter into a state of actual fear.`, + fieldInfo: ['time period', 'price change'] + }, + { + text: `According to stats Cam, the average size of a new condo in Toronto has shrunk significantly over time. From 1981 to 1990, new condos were on average about 1,000 square feet. From 2016 to 2020, they were around 650 square feet or about 40% smaller. And if you ask anyone who lives in the downtown core right now, 650 square feet actually feels pretty big.`, + fieldInfo: ['time period', 'average size'] + }, + { + text: `Right now, for example, in Toronto and Vancouver, three quarters of larger condos, the ones, you know, maybe more suitable for a family built decades ago, those are being lived in by the people who own them. But for those much smaller condos built after 2016 with investors in mind, only about half are being lived in by owners. Remember that cute little studio that we showed you at the start that took 400 some days to sell. It was 330 square feet. Here's a two bedroom, 700 square foot unit in that same building. It's sold earlier this month on the first try in just 13 days. If we.`, + fieldInfo: ['days to cell', 'unit size', ''] + }, + { + text: `Back then in 2020,2021, even into the beginning of 2022, we saw these historic low interest rates and people were taking out loans then and taking out bigger loans, right? They go and buy a house that they wouldn't have been able to afford. 3% or 4%, but they could afford at 1.25%. They called.`, + fieldInfo: ['year', 'interest rate'] + }, + { + text: `No. So let's say there's a young couple in Ontario that bought an average priced house in 2019,$631,990. They put pretty standard down payments, down 20 percents, which leaves you with a mortgage size of about $500,000. Amortize the loan over 25 years, they take a five year fixed rate of 2.9%, which was also kind of standard at the time. That means they've been paying about, and I'm rounding to the nearest dollar here, 2,367 bucks for the past few years. But now that mortgage is up for renewal. Welcome to 2024. And the couple signs on to another fixed rate mortgage, but they have to do it at the bank's current rate, which is around 6%. That means their new monthly payment is, again, I'm rounding here, $3,075 a month or an extra 700 bucks a month. Right.`, + fieldInfo: ['year', 'fixed rate', 'monthly payment'] + }, + { + text: `Think of it this way. This circle is your $2,200 a month mortgage payment at a relatively low interest rate pending on the overall size of your mortgage, that might mean $900 of that payment is just servicing interest with the other 13 actually paying your loan back as the interest rate rises. Maybe now instead of 900 dollars a month, it's 20. One hundred dollars a month in interest. But your contract says your total payments stay the same, so you're still paying 22 dollars a month, meaning only 100 of those 22 hundred dollars are actually paying down your loan at that point. I have news for you. You're very close to being underwater, stuck in a perpetual state of paying down a loan where you're only barely paying down that loan. Welcome to lifelong debt, the time it will take to pay off his mortgage nearly doubled from.`, + fieldInfo: ['monthly payment', 'interset payment', 'principal payment'] + }, + { + text: `Collectively, we're just starting to approach the edge of this cliff. RBC estimates about $186 billion worth of mortgages are up for renewal in 2024. Next year, it'll be 350,15 billion. That's a ton of mortgage debt that was taken on at historically low interest rates by people who may or may not have been able to afford getting into the housing market otherwise.`, + fieldInfo: ['year', 'mortgage_debt'] + }, + { + text: `What would you do if one day your neighbor, who you've known for years, you see him building a bomb shelter? Would you think that guy's crazy or would you wonder what does he know that I don't? Turns out banks right now are building a bombshelter. Just this quarter, Canada's big six banks have set $4.3 billion aside to cover bad loans. That's almost double what they set aside in the first quarter of last year and more than 11 times what they set aside in the first quarter before that.`, + fieldInfo: ['quater', 'year', 'amount_set_aside'] + }, + { + text: `This is important. About 65% of them, 70% of them have not noticed any radical increase in their payment because their mortgage company sold a product where the payment did not go up when prime went up. Their payment did not go up when prime went up. That's the important part to understand, because it means you might have people ultimately paying more without even realizing it.`, + fieldInfo: ['percentage'] + }, + { + text: `Now, luckily, most economists do expect rates to keep dropping. According to that same RBC report, it still might not be enough to, quote, save this cohort to get them down to a more manageable monthly increase like 20%. They argue the bank of Canada would have to lower its prime rate way down to around point two five % by July 2026, something they admit is an unreasonable expectation.`, + fieldInfo: ['date', 'rate change', 'monthly_increase'] + }, + { + text: `The old rule of thumb in my day was you spend about a third of your income on housing. Now you'll hear anecdotal evidence, particularly in places like Vancouver, Toronto, where you'll hear 70 to 75% of income is being spent on housing. So in those cases, then if they have to reset at a higher interest rate, it's the house that's gonna go down first. So.`, + fieldInfo: ['location', 'income_spent_on_housing'] + }, + { + text: `In order to get the most accurate reading on the US housing market inventory situation, we need to consider both supply and demand. Supply in this case is the level of inventory for sale and demand is the current case of sales volume. If we start with sales volume, we can see the existing home market has a sales pace of roughly 3.7 million homes per year, while the new home market has a sales pace of 619,000 homes per year. Right off the bat, we can see how the existing home market is almost six times larger than the new construction market in terms of sales volume. Both of these markets, existing and new, have seen a huge reduction in sales volume or buyer demand since interest rates have increased, as affordability is out of reach for most people. If we take this annual sales pace and divided by 12, we can get an idea of the average sales pace per month. Of course, there are seasonal patterns, but if we normalize for those effects, we can see that the current monthly sales pace in the existing home market is roughly 309,000, and the new construction market is selling about 51,000 homes per month. At the peak of the housing boom during the pandemic, the existing hallmark had a sales pace of 500,000 units per month and the new construction market was selling nearly 90,000 units per month. Now that we have an idea of the current level of demand, we have to look at supply or the level of inventory for sale in the existing home market. There are currently 1.1 million homes for sale, one of the lowest on record. If we just look at inventory alone. In the new construction market, there are 479,000 units for sale, one of the highest levels on record, only surpassed by the housing bubble of 2007.`, + fieldInfo: ['market type', 'sale pace annual', 'sale pace monthly', 'sales_pace_peak_monthly', 'inventory'] + }, + { + text: `You can clearly see how there is nuance to the situation of US housing inventory, the best measure of US housing inventory takes into account both supply and demand and is called the month supply. In other words, how many months of inventory are available for the current pace of sales value. If we take the current level of inventory for both the existing hall market and the new hall market, and we divide it by the current monthly pace of sales. We can see how many months it would take to sell all the inventory at the current pace of sales in the existing home market, that month's supply figure or the inventory level divided by the sales volume, 3.7. In the new construction market, it is 9.3.`, + fieldInfo: ['market_type', 'month supply'] + }, + { + text: `Generally speaking, the 5.5 to 6.0 level is considered a balanced market where there is no significant upward or downward pressure on prices. When the month's supply level is below 5.5, that means there is very little inventory available for today's market conditions and prices generally rise. On the flip side, a high level of month's supply means there's too much inventory and prices must fall. The existing home market has a month supply of 3.7, which is very low, but it's been increasing since the absolute historic level of one point six in January 2022. Never in the history of the US housing market has the month's supply of existing homes been that low. This created a very unhealthy situation in the existing home market. The inventory situation in the existing hall market is still very tight, but it's now at the highest level since before for the pandemic. The new construction market is a completely different ballgame with a month supply at 9.3, super high and way above the balanced 6.0 level. There is way more detail to the situation in the new construction market that we'll uncover in just a moment. If we look at the total US housing market situation, both existing and new construction, the aggregate month supply figure has increased to 4.5. So yes, the total inventory situation is still very tight on a national level, but the level is rising and at the highest point since 2,015. The monthly numbers are volatile, so let's look at a yearly average to make things more clear. This shows the month's supply for the total US housing market by year. 2024 is a partial year, and the current level as of May 2024 is also noted in the chart. As of May 2024, the total month supply situation is 4.5, which it hasn't been since 2015. You can also see the four years where the total month supply was below 4.0, which is extremely tight. And you can also see the 2021 situation at 2.6, which was crazy unhealthy. One major point is that this current 4.5 level is the national average. But as we know, real estate is very regional. So this means that there are some markets that are near a month supply of 6 and feeling downward price pressure, while some markets are still down at 3 and seeing prices rise with multiple offers. The new home market with a month supply of 9.3 is way above the balance level of six point zero, and there are price cuts and discounts in that market.`, + fieldInfo: ['date', 'markey type', 'month supply'] + }, + { + text: `In June, the National Association of homebuilders reported that 29% of builders cut home prices with an average price reduction of 6%. And 61% of builders use sales incentives like mortgage rate ByteDance to boost sales. But we need to talk about the new hall market and the 9.3 month supply level in a bit more detail, because there is more than what meets BI in the new construction market. Currently, there are 479,000 units for sale in the new construction market and a current pace of sales of about 50 thousand per month. In the new home market, a home could be listed for sale when it's completed, under construction or not yet started. If we look at the percentage of inventory that is completed, we can see that of those 479,000 new home units for sale, Only 20% are completed, which means 80% are either under construction or not yet started.`, + fieldInfo: [ + 'builders cut prices percentage', + 'average price reduction', + 'builders use incentives percentage', + 'units for sale', + 'completed inventory percentage' + ] + }, + { + text: `The level of completed new home inventory for sale fell below 10% during the most acute part of the US housing shortage. Normally, in between 20% and 30% of new construction inventory is completed. During really bad downturns like 2008, builders were sitting on almost 50% completed inventory, which is what led to dramatic price cuts and big layoffs of construction crews. The level of completed inventory is now back into the normal range at 20%, and it's rising, which means that if we move towards 30%, price cuts will get more intense and construction crews will be at high risk of job losses. If we take the current 50,000 new home sales pace and divide that by the amount of completed new home inventory for sale. We can see that there's currently about 1.9 months of completed inventory for sale, which is getting past the average level and into the range consistent with recessions and job losses for construction crews. If the home builders have a lot of completed new home inventory for sale, they aren't going to apply for new building permits and build even more homes. And that's exactly what we're starting to see.`, + fieldInfo: ['period', 'completed_inventory_ratio'] + }, + { + text: `Homelessness has been rare since the reemergence of homelessness in the mid 1980s. Usually it's been people in their 20s and 30s and 40s, but now we're approaching close to 30% of the adult homeless population are people 55 and over.`, + fieldInfo: ['age group', 'percentage'] + }, + { + text: `Exports from China are rising fast and US officials are nervous about this gap. We're not gonna let China flood our market. Sound familiar? There has to be a level playing field for American companies competing in China. In the early 2 Chinese factories like this one pumped out clothes and flags and cars. The sticker prices were cheap. But there was another price to be paid, American jobs. Every single employee of this plant, we'll be out of work by January. 15 years and almost 6 million jobs later, economists are debating what price another wave of imports could exact from American workers. But two key differences between then and now may have a new effect on the US economy. Look at this line. Before 2001, American manufacturing employed over 17 million people, making toys, furniture, paper goods and much more. Then word began to circulate that China was joining the World Trade Organization. And here's what it did. Joining the WTO meant China faced fewer tariffs and restrictions from its trading partners, and the result was dubbed the China.`, + fieldInfo: ['year', 'jobs number'] + }, + { + text: `2.5 million Americans lost their jobs from 2000 to 2007. They're represented by the blue on this map. Look at this dark blue area here. That's Silicon Valley, where companies like Apple, HP and Cisco used to manufacture goods. After 2001, they moved most of their production to China, causing a 50+ percent drop in manufacturing jobs in the county, this other dark blue county, Cedar Rapids, Iowa, lost 46% of their manufacturing jobs, primarily in furniture and machinery. But this lighter blue region here was largely spared. It's called Auto Alley. The main reason it succeeded where, say, Silicon Valley's manufacturing failed, is due to investment from competitors like Japan.`, + fieldInfo: ['region', 'job_loss_percentage'] + }, + { + isEnglish: true, + text: 'Snap’s latest AR glasses aren’t its first hardware product. The company has a history of experimenting with hardware and has launched smart glasses under the brand name Spectacles. However, the company has struggled to make money from selling hardware. It first debuted a pair of smart glasses in 2016; the $130 device made it easy for users to capture short, first-person videos that they could post on Snapchat. However, the product failed, and the company had to write down $40 million due to losses from unsold Spectacles. Snap once again tried to sell new Spectacles smart glasses in 2019—this time a more premium version for $380—but that too didn’t take off. In 2021, it unveiled new glasses with advanced AR capabilities, but instead of selling them to consumers, the company sold the device to developers, hinting that the glasses were more of a prototype than an actual consumer product. Beyond glasses, Snap also tried to sell a flying drone for $230, but that device was abruptly discontinued after just a few months', + dataTable: [ + { + year: 2016, + product: 'Spectacles', + price: 130 + }, + { + year: 2019, + product: 'Spectacles (premium version)', + price: 380 + }, + { + year: 2021, + product: 'new glasses with AR', + price: null + }, + { + year: null, + product: 'flying drone', + price: 230 + } + ], + fieldInfo: [ + { + fieldName: 'year', + description: 'The year when the product was launched', + fieldType: 'date', + dateGranularity: 'year' + }, + { + fieldName: 'product', + description: 'the name of product', + fieldType: 'string' + }, + { + fieldName: 'price', + description: 'The price of the product', + fieldType: 'numerical' + } + ] } ]; diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx index 925d098b..464012a9 100644 --- a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataInput.tsx @@ -11,7 +11,7 @@ const Option = Select.Option; const RadioGroup = Radio.Group; type IPropsType = { - onOk: (extractCtx: any, dataCleanCtx: any) => void; + onOk: (extractCtx: any, dataCleanCtx: any, timeCost: number) => void; setLoading: (loading: boolean) => void; }; @@ -53,7 +53,7 @@ export function DataInput(props: IPropsType) { const schedule = React.useRef>( new Schedule( [AtomName.DATA_EXTRACT, AtomName.DATA_CLEAN], - { base: { llm: llm.current, showThoughts }, dataExtract: { reGenerateFieldInfo: !useFieldInfo } }, + { base: { llm: llm.current, showThoughts }, dataExtract: { reGenerateFieldInfo: true } }, { text, fieldInfo: useFieldInfo ? fieldInfo : [] } ) ); @@ -68,12 +68,19 @@ export function DataInput(props: IPropsType) { }); }, [url, model, apiKey]); useEffect(() => { - schedule.current.updateOptions({ base: { showThoughts }, dataExtract: { reGenerateFieldInfo: !useFieldInfo } }); + schedule.current.updateOptions({ base: { showThoughts } }); }, [showThoughts, useFieldInfo]); const handleQuery = React.useCallback(async () => { props.setLoading(true); + const time1: any = new Date(); await schedule.current.run(userInput); - props.onOk(schedule.current.getContext(AtomName.DATA_EXTRACT), schedule.current.getContext(AtomName.DATA_CLEAN)); + const time2: any = new Date(); + const diff = (time2 - time1) / 1000; + props.onOk( + schedule.current.getContext(AtomName.DATA_EXTRACT), + schedule.current.getContext(AtomName.DATA_CLEAN), + diff + ); }, [props, userInput]); return ( @@ -103,7 +110,7 @@ export function DataInput(props: IPropsType) { onChange={index => { const dataObj = capcutMockData[index]; setText(dataObj.text); - setFieldInfo(dataObj.fieldInfo || []); + setFieldInfo(dataObj.fieldInfo.map((v: any) => ({ fieldName: v })) || []); setUserInput(dataObj.input); schedule.current.setNewTask({ text: dataObj.text, @@ -113,7 +120,7 @@ export function DataInput(props: IPropsType) { > {capcutMockData.map((data, index) => ( ))} @@ -176,6 +183,7 @@ export function DataInput(props: IPropsType) { setModel(v)}> GPT-4-0613 Doubao-pro + Doubao-pro-32k
diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataTable.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataTable.tsx index 23ba38e5..56cc60a7 100644 --- a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataTable.tsx +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/DataTable.tsx @@ -2,6 +2,7 @@ import React from 'react'; import type { DataTable, FieldInfo } from '../../../../../src/index'; import type { TableColumnProps } from '@arco-design/web-react'; import { Card, Input, Spin, Table } from '@arco-design/web-react'; +import { isArray } from '@visactor/vutils'; const TextArea = Input.TextArea; @@ -13,7 +14,10 @@ const SimpleTable = ({ data, fieldInfo }: { data: DataTable; fieldInfo: FieldInf const columns: TableColumnProps[] = fieldInfo.map((info: FieldInfo) => { return { title: info.fieldName, - dataIndex: info.fieldName + dataIndex: info.fieldName, + render: (col: any) => { + return isArray(col) ? col.join('-') : col; + } }; }); return ( @@ -24,7 +28,7 @@ const SimpleTable = ({ data, fieldInfo }: { data: DataTable; fieldInfo: FieldInf hideOnSinglePage: true }} scroll={{ - x: columns.length * 100 + x: true }} /> ); diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx index 289b8755..22fb6443 100644 --- a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx @@ -13,13 +13,15 @@ export function NewDataExtractionPage() { const [fieldInfo, setFieldInfo] = useState([]); const [finalFieldInfo, setFinalFieldInfo] = useState([]); const [loading, setLoading] = useState(false); + const [timeCost, setTimeCost] = useState(0); - const handleOk = React.useCallback(async (dataExtractCtx: any, dataCleanCtx: any) => { + const handleOk = React.useCallback(async (dataExtractCtx: any, dataCleanCtx: any, timeCost: number) => { setDataset(dataExtractCtx.dataTable); setFieldInfo(dataExtractCtx.fieldInfo); setFianlDataset(dataCleanCtx.dataTable); setFinalFieldInfo(dataCleanCtx.fieldInfo); setLoading(false); + setTimeCost(timeCost); // eslint-disable-next-line no-console console.info(dataExtractCtx, dataCleanCtx); }, []); @@ -35,6 +37,7 @@ export function NewDataExtractionPage() { +

{`Time Cost: ${timeCost.toFixed(1)}s`}

[], cureV) => { + return [ + ...prev, + ...cureV.data.map(v => ({ + ...v, + domain: cureV.domain + })) + ]; +}, []); + +export const commonAnswer: DataExtractionCase = { + llm: 'answer', + result: [ + { + dataset: 'common', + defaultResult: dataExtractionCommonDataset.map(v => ({ + context: v + })) as any, + fieldInfoResult: [] + } + ] +}; diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx index 46da5325..5a5caba8 100644 --- a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx @@ -2,17 +2,21 @@ /* eslint-disable no-console */ import React from 'react'; import type { FieldInfo } from '../../../../../src/index'; -import { capcutMockData } from '../../data/capcutData'; +import { capcutCnData } from '../../data/capcutDataCn'; +import { capcutEnData } from '../../data/capcutDataEn'; import { dataExtractionCommonDataset, commonAnswer } from '../../data/dataExtractionData'; import type { TableColumnProps } from '@arco-design/web-react'; import { Avatar, Button, Card, Checkbox, Divider, Message, Select, Table, Tooltip } from '@arco-design/web-react'; -import { result as capcutResult } from '../../results/dataExtraction/result3'; +// import { result as capcutResult } from '../../results/dataExtraction/result4'; +import { result as capcutResult } from '../../results/dataExtraction/result7'; import { result as caseResult } from '../../results/dataExtraction/commonResult'; import '../page.scss'; import { IconInfoCircle } from '@arco-design/web-react/icon'; import { getLanguageOfText } from '../../../../../src/utils/text'; import { mergeResult, updateScoreInDataExtraction } from './verify'; import type { DataExtractionDataSetResult } from './type'; +import { isArray } from '@visactor/vutils'; +import { sum } from '@visactor/vchart/esm/util'; const result = [...mergeResult(capcutResult as any, caseResult as any), commonAnswer]; updateScoreInDataExtraction(result as any, commonAnswer); @@ -23,7 +27,8 @@ const llmList = result.map((v, index) => ({ llm: v.llm })); const datasetMap: Record = { - capcut: capcutMockData, + capcut_cn: capcutCnData, + capcut_en: capcutEnData, common: dataExtractionCommonDataset }; const datasetList = result[0].result.map((v, index) => ({ @@ -37,7 +42,7 @@ interface Options { llm: string; resType: 'defaultResult' | 'fieldInfoResult'; } -const targetScore = 0.7; +const targetScore = 0.75; export function DataExtractionResult() { const [datasetIndex, setDatasetIndex] = React.useState(0); const currentDataset = datasetList[datasetIndex]; @@ -70,16 +75,17 @@ export function DataExtractionResult() { return { title: (
- -
{`${info.fieldType[0]}__`}
- + {info.fieldType &&
{`${info.fieldType[0]}__`}
} {info.fieldName} - +
), - dataIndex: info.fieldName + dataIndex: info.fieldName, + render: (col: any) => { + return isArray(col) ? col.join('-') : col; + } }; }); return ( @@ -143,10 +149,18 @@ export function DataExtractionResult() { [answerResult, language] ); + const renderTimeCost = React.useCallback((res: DataExtractionDataSetResult[] | undefined | null) => { + if (res) { + const validTimeCost = res.map(v => Number(v.timeCost)).filter(v => !isNaN(v) && v > 1); + const timeCost = (sum(validTimeCost) / validTimeCost.length).toFixed(1); + return validTimeCost.length ? {`Time Cost: ${timeCost}s`} : null; + } + }, []); + const renderScore = React.useCallback( (res: DataExtractionDataSetResult[] | undefined | null) => { if (answerResult && res) { - const validRes = res.filter(v => !!v.score && v.score !== 0); + const validRes = res.filter(v => !!v.score || v.score === 0); const allCount = validRes.length; const reachedCount = validRes.filter(v => v.score! >= targetScore).length; let score = 0; @@ -271,7 +285,7 @@ export function DataExtractionResult() { return null; } return ( -
+
+ {/*
{JSON.stringify(props.spec, null, 4)}
*/} +
+ ) : null} + + +
+ ); +} diff --git a/packages/vmind/__tests__/browser/src/pages/NewChartGeneration/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/NewChartGeneration/DataInput.tsx new file mode 100644 index 00000000..e84ba86d --- /dev/null +++ b/packages/vmind/__tests__/browser/src/pages/NewChartGeneration/DataInput.tsx @@ -0,0 +1,261 @@ +/* eslint-disable no-console */ +import React, { useState, useCallback, useMemo } from 'react'; +import './index.scss'; +import { Avatar, Input, Divider, Button, Message, Select, Radio, Modal } from '@arco-design/web-react'; +import { chartGenerationMockData } from '../../constants/chartGeneratorData'; +import type { DataTable, FieldInfo } from '../../../../../src/index'; +import VMind from '../../../../../src/index'; +import { Model, AtomName, Schedule, LLMManage } from '../../../../../src/index'; +import type { SimpleFieldInfo } from '../../../../../src/common/typings'; +import { DataType } from '../../../../../src/common/typings'; + +const TextArea = Input.TextArea; +const Option = Select.Option; +const RadioGroup = Radio.Group; + +type IPropsType = { + onSpecGenerate: (spec: any, command: string, costTime: number) => void; + onSpecListGenerate?: any; +}; + +const globalVariables = (import.meta as any).env; +const ModelConfigMap: any = { + [Model.DOUBAO_PRO]: { url: globalVariables.VITE_DOUBAO_URL, key: globalVariables.VITE_DOUBAO_KEY }, + [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } +}; +const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions'; + +const specTemplateTest = false; + +function dataTypeTransfer(dataType: string): DataType { + switch (dataType) { + case 'date': + case 'time': + return DataType.DATE; + case 'count': + return DataType.INT; + case 'numerical': + case 'ratio': + return DataType.FLOAT; + default: + return DataType.STRING; + } +} + +function transferFieldInfoInSimpleFieldInfo(fieldInfo: FieldInfo[]): SimpleFieldInfo[] { + return fieldInfo.map(item => ({ + fieldName: item.fieldName, + description: item.description, + type: dataTypeTransfer(item.fieldType), + role: item.role + })); +} + +export function DataInput(props: IPropsType) { + const [dataTableIndex, setdataTableIndex] = useState(0); + const [describe, setDescribe] = useState(''); + const [dataTable, setDataTable] = useState( + chartGenerationMockData[0].result[dataTableIndex].context.dataTable as any + ); + const [fieldInfo, setFieldInfo] = useState( + chartGenerationMockData[0].result[dataTableIndex].context.fieldInfo as any + ); + const [model, setModel] = useState(Model.GPT_4o); + const [visible, setVisible] = React.useState(false); + const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); + const [apiKey, setApiKey] = React.useState(ModelConfigMap[model]?.key); + + const [loading, setLoading] = useState(false); + + const llm = React.useRef( + new LLMManage({ + url, + headers: { + 'api-key': apiKey, + Authorization: `Bearer ${apiKey}` + }, + model, + maxTokens: 2048 + }) + ); + const schedule = React.useRef>( + new Schedule([AtomName.CHART_COMMAND], { base: { llm: llm.current } }) + ); + + const vmind: VMind = useMemo(() => { + if (!url || !apiKey) { + Message.error('Please set your LLM URL and API Key!!!'); + return null as unknown as VMind; + } + return new VMind({ + url, + model, + headers: { + //must has Authorization: `Bearer ${openAIKey}` if use openai api + Authorization: `Bearer ${apiKey}`, + 'api-key': apiKey + } + }); + }, [apiKey, model, url]); + + const askGPT = useCallback(async () => { + const finaldataTable = specTemplateTest && model !== Model.CHART_ADVISOR ? undefined : dataTable; + + const startTime = new Date().getTime(); + + setLoading(true); + let finalDescribe = describe; + if (!finalDescribe) { + schedule.current.setNewTask({ + ...chartGenerationMockData[0].result[dataTableIndex].context + }); + finalDescribe = (await schedule.current.run()).command; + } + const chartGenerationRes = await vmind.generateChart( + finalDescribe, + transferFieldInfoInSimpleFieldInfo(fieldInfo), + finaldataTable, + { + theme: 'light' + } + ); + const endTime = new Date().getTime(); + const { spec } = chartGenerationRes; + + const finalSpec = specTemplateTest ? vmind.fillSpecWithData(spec, dataTable) : spec; + + const costTime = endTime - startTime; + props.onSpecGenerate(finalSpec, finalDescribe, costTime); + + setLoading(false); + }, [model, dataTable, describe, vmind, fieldInfo, props, dataTableIndex]); + + return ( +
+
+
+ +
+
+
+

+ + 0 + + Select Demo Data (optional) +

+ +
+
+

+ + 1 + + What would you like to visualize? +

+ + + {showFieldInfo ? ( + <> +

fieldInfo:

+ + + ) : null}
diff --git a/packages/vmind/__tests__/browser/src/pages/Text2Chart/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/Text2Chart/DataInput.tsx new file mode 100644 index 00000000..0d51a33b --- /dev/null +++ b/packages/vmind/__tests__/browser/src/pages/Text2Chart/DataInput.tsx @@ -0,0 +1,231 @@ +/* eslint-disable no-console */ +import React, { useState, useEffect } from 'react'; +import '../DataExtraction/index.scss'; +import { Avatar, Input, Divider, Button, Select, Checkbox, Modal, Radio } from '@arco-design/web-react'; +import type { FieldInfo } from '../../../../../src/index'; +import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; +import { capcutMockData } from '../../constants/capcutData'; + +const TextArea = Input.TextArea; +const Option = Select.Option; +const RadioGroup = Radio.Group; + +type IPropsType = { + onOk: (extractCtx: any, dataCleanCtx: any, chartCtx: any, timeCost: number) => void; + setLoading: (loading: boolean) => void; +}; + +const globalVariables = (import.meta as any).env; +const ModelConfigMap: any = { + [Model.SKYLARK2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, + [Model.SKYLARK2_v1_2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, + [Model.GPT3_5]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT4]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT_4_0613]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } +}; +const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions'; + +export function DataInput(props: IPropsType) { + const defaultIndex = 0; + const [text, setText] = useState(capcutMockData[defaultIndex].text); + const [userInput, setUserInput] = useState(capcutMockData[defaultIndex].input); + + const [model, setModel] = useState(Model.GPT_4o); + const [useFieldInfo, setUseFieldInfo] = useState(false); + const [showThoughts, setShowThoughts] = useState(false); + const [visible, setVisible] = React.useState(false); + const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); + const [apiKey, setApiKey] = React.useState(ModelConfigMap[model]?.key); + const [fieldInfo, setFieldInfo] = useState(capcutMockData[defaultIndex].fieldInfo || []); + + const llm = React.useRef( + new LLMManage({ + url, + headers: { + 'api-key': apiKey, + Authorization: `Bearer ${apiKey}` + }, + model, + maxTokens: 2048 + }) + ); + const schedule = React.useRef< + Schedule< + [AtomName.DATA_EXTRACT, AtomName.DATA_CLEAN, AtomName.CHART_COMMAND, AtomName.DATA_QUERY, AtomName.CHART_GENERATE] + > + >( + new Schedule( + [ + AtomName.DATA_EXTRACT, + AtomName.DATA_CLEAN, + AtomName.CHART_COMMAND, + AtomName.DATA_QUERY, + AtomName.CHART_GENERATE + ], + { base: { llm: llm.current, showThoughts }, dataExtract: { reGenerateFieldInfo: true } } + ) + ); + useEffect(() => { + llm.current.updateOptions({ + url, + headers: { + 'api-key': apiKey, + Authorization: `Bearer ${apiKey}` + }, + model + }); + }, [url, model, apiKey]); + useEffect(() => { + schedule.current.updateOptions({ base: { showThoughts } }); + }, [showThoughts, useFieldInfo]); + const handleQuery = React.useCallback(async () => { + props.setLoading(true); + const time1: any = new Date(); + await schedule.current.run(userInput); + const time2: any = new Date(); + const diff = (time2 - time1) / 1000; + props.onOk( + schedule.current.getContext(AtomName.DATA_EXTRACT), + schedule.current.getContext(AtomName.DATA_CLEAN), + schedule.current.getContext(AtomName.CHART_GENERATE), + diff + ); + }, [props, userInput]); + + return ( +
+
+
+ +
+
+
+
+

+ + 0 + + Select Demo Data (optional) +

+ +
+ +
+

+ + 1 + + Input your data in text format +

+ - - ) : null} +
diff --git a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx index 22fb6443..e12a60a4 100644 --- a/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx +++ b/packages/vmind/__tests__/browser/src/pages/NewDataExtraction/index.tsx @@ -1,50 +1,48 @@ import React, { useState } from 'react'; import { Layout } from '@arco-design/web-react'; import { DataInput } from './DataInput'; +import type { TableProps } from './DataTable'; import { DataTableComp } from './DataTable'; -import type { DataTable } from '../../../../../src'; -import type { FieldInfo } from '../../../../../src'; const Sider = Layout.Sider; const Content = Layout.Content; export function NewDataExtractionPage() { - const [dataset, setDataset] = useState([]); - const [finalDataset, setFianlDataset] = useState([]); - const [fieldInfo, setFieldInfo] = useState([]); - const [finalFieldInfo, setFinalFieldInfo] = useState([]); + const [dataExtractionRes, setDataExtractionRes] = useState([]); + const [dataCleanRes, setDataCleanRes] = useState([]); const [loading, setLoading] = useState(false); const [timeCost, setTimeCost] = useState(0); + const [type, setType] = useState<'multiple' | 'normal'>('multiple'); - const handleOk = React.useCallback(async (dataExtractCtx: any, dataCleanCtx: any, timeCost: number) => { - setDataset(dataExtractCtx.dataTable); - setFieldInfo(dataExtractCtx.fieldInfo); - setFianlDataset(dataCleanCtx.dataTable); - setFinalFieldInfo(dataCleanCtx.fieldInfo); - setLoading(false); - setTimeCost(timeCost); - // eslint-disable-next-line no-console - console.info(dataExtractCtx, dataCleanCtx); - }, []); + const handleOk = React.useCallback( + async (dataExtractCtx: any, dataCleanCtx: any, timeCost: number) => { + setLoading(false); + setTimeCost(timeCost); + // eslint-disable-next-line no-console + console.info(dataExtractCtx, dataCleanCtx); + if (type === 'multiple') { + setDataExtractionRes(dataExtractCtx.datasets); + setDataCleanRes(dataCleanCtx.datasets); + } else { + setDataExtractionRes([dataExtractCtx]); + setDataCleanRes([dataCleanCtx]); + } + }, + [type] + ); return ( - + - +

{`Time Cost: ${timeCost.toFixed(1)}s`}

- + ); diff --git a/packages/vmind/__tests__/browser/src/pages/Text2Chart/ChartPreview.tsx b/packages/vmind/__tests__/browser/src/pages/Text2Chart/ChartPreview.tsx new file mode 100644 index 00000000..cad3ff12 --- /dev/null +++ b/packages/vmind/__tests__/browser/src/pages/Text2Chart/ChartPreview.tsx @@ -0,0 +1,114 @@ +import React, { useState, useEffect, useRef } from 'react'; +import './index.scss'; +import { Input, Card, Modal, Spin } from '@arco-design/web-react'; +import VChart, { registerLiquidChart } from '@visactor/vchart'; +import { isNil } from '@visactor/vutils'; +import type { DataTable, FieldInfo } from '../../../../../src/index'; +import { SimpleTable } from '../NewDataExtraction/DataTable'; + +type IPropsType = { + commands: string[]; + specList: any[]; + style?: any; + dataExtractionResult: { + dataTable: DataTable; + fieldInfo: FieldInfo[]; + textRange: string[]; + }[]; + dataCleanResult: { + dataTable: DataTable; + fieldInfo: FieldInfo[]; + textRange: string[]; + }[]; + costTime: number; + loading: boolean; + text: string; + showTable: boolean; +}; + +registerLiquidChart(); + +export function ChartPreview(props: IPropsType) { + const { specList, commands, loading, dataCleanResult, text, showTable = false } = props; + const [textRange, setTextRange] = React.useState<[number, number]>([0, -1]); + useEffect(() => { + //defaultTicker.mode = 'raf'; + specList.forEach((spec: any, index: number) => { + if (!spec) { + return; + } + (document.getElementById(`chart-${index}`) as any).innerHTML = ''; + const cs = new VChart(spec, { + dom: `chart-${index}`, + mode: 'desktop-browser', + dpr: 2, + disableDirtyBounds: true + }); + cs.renderAsync(); + }); + }, [specList]); + + return ( +
+ +
+ + {text.split('').map((v, textIndex) => { + return ( + = textRange[0] && textIndex <= textRange[1] ? '#ffa500' : 'rgba(0,0,0,0.7)' + }} + > + {v} + + ); + })} + +
+ {props.specList.map((spec: any, index: number) => { + const currentTextRange = dataCleanResult?.[index]?.textRange; + return ( + +

Command: {commands[index]}

+
{ + if (textRange) { + setTextRange([ + text.indexOf(currentTextRange[0]), + text.indexOf(currentTextRange[1]) + currentTextRange[1].length - 1 + ]); + } + }} + onMouseLeave={() => { + setTextRange([0, -1]); + }} + > + {spec && ( +
+ )} + {(showTable || !spec) && ( +
+ +
+ )} +
+ + ); + })} +
+
+ +
+ ); +} diff --git a/packages/vmind/__tests__/browser/src/pages/Text2Chart/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/Text2Chart/DataInput.tsx index 0d51a33b..108d24cd 100644 --- a/packages/vmind/__tests__/browser/src/pages/Text2Chart/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/Text2Chart/DataInput.tsx @@ -1,24 +1,24 @@ /* eslint-disable no-console */ import React, { useState, useEffect } from 'react'; import '../DataExtraction/index.scss'; -import { Avatar, Input, Divider, Button, Select, Checkbox, Modal, Radio } from '@arco-design/web-react'; -import type { FieldInfo } from '../../../../../src/index'; +import { Avatar, Input, Divider, Button, Select, Modal, Radio } from '@arco-design/web-react'; import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; -import { capcutMockData } from '../../constants/capcutData'; +import { capcutMockV2Data as capcutMockData } from '../../constants/capcutData'; const TextArea = Input.TextArea; const Option = Select.Option; const RadioGroup = Radio.Group; type IPropsType = { - onOk: (extractCtx: any, dataCleanCtx: any, chartCtx: any, timeCost: number) => void; + type: 'normal' | 'multiple'; + setType: (value: 'normal' | 'multiple') => void; + onOk: (text: string, extractCtx: any, dataCleanCtx: any, chartCtx: any[], timeCost: number) => void; setLoading: (loading: boolean) => void; }; const globalVariables = (import.meta as any).env; const ModelConfigMap: any = { - [Model.SKYLARK2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, - [Model.SKYLARK2_v1_2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, + [Model.DOUBAO_PRO]: { url: globalVariables.VITE_DOUBAO_URL, key: globalVariables.VITE_DOUBAO_KEY }, [Model.GPT3_5]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, [Model.GPT4]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, [Model.GPT_4_0613]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, @@ -31,13 +31,16 @@ export function DataInput(props: IPropsType) { const [text, setText] = useState(capcutMockData[defaultIndex].text); const [userInput, setUserInput] = useState(capcutMockData[defaultIndex].input); - const [model, setModel] = useState(Model.GPT_4o); - const [useFieldInfo, setUseFieldInfo] = useState(false); - const [showThoughts, setShowThoughts] = useState(false); + const [model, setModel] = useState(Model.DOUBAO_PRO); const [visible, setVisible] = React.useState(false); const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); const [apiKey, setApiKey] = React.useState(ModelConfigMap[model]?.key); - const [fieldInfo, setFieldInfo] = useState(capcutMockData[defaultIndex].fieldInfo || []); + const llmModel = React.useMemo(() => { + if (model.endsWith('-advisor')) { + return model.split('-&&-')[0]; + } + return model; + }, [model]); const llm = React.useRef( new LLMManage({ @@ -50,21 +53,27 @@ export function DataInput(props: IPropsType) { maxTokens: 2048 }) ); - const schedule = React.useRef< - Schedule< - [AtomName.DATA_EXTRACT, AtomName.DATA_CLEAN, AtomName.CHART_COMMAND, AtomName.DATA_QUERY, AtomName.CHART_GENERATE] - > - >( - new Schedule( - [ - AtomName.DATA_EXTRACT, - AtomName.DATA_CLEAN, - AtomName.CHART_COMMAND, - AtomName.DATA_QUERY, - AtomName.CHART_GENERATE - ], - { base: { llm: llm.current, showThoughts }, dataExtract: { reGenerateFieldInfo: true } } - ) + const getScheduleByType = React.useCallback( + (value: string) => { + if (value === 'normal') { + return new Schedule( + [AtomName.DATA_EXTRACT, AtomName.DATA_CLEAN, AtomName.CHART_COMMAND], + { base: { llm: llm.current, showThoughts: false }, dataExtract: { reGenerateFieldInfo: true } }, + { text } + ); + } + return new Schedule([AtomName.DATA_EXTRACT, AtomName.MULTIPLE_DATA_CLEAN, AtomName.MULTIPLE_CHART_COMMAND], { + base: { llm: llm.current, showThoughts: false }, + dataExtract: { isCapcut: true } + }); + }, + [text] + ); + const schedule = React.useRef(getScheduleByType(props.type)); + const chartSchedule = React.useRef( + new Schedule([AtomName.CHART_GENERATE], { + base: { llm: llm.current, showThoughts: false } + }) ); useEffect(() => { llm.current.updateOptions({ @@ -73,25 +82,50 @@ export function DataInput(props: IPropsType) { 'api-key': apiKey, Authorization: `Bearer ${apiKey}` }, - model + model: llmModel }); - }, [url, model, apiKey]); - useEffect(() => { - schedule.current.updateOptions({ base: { showThoughts } }); - }, [showThoughts, useFieldInfo]); + }, [url, llmModel, apiKey]); const handleQuery = React.useCallback(async () => { props.setLoading(true); const time1: any = new Date(); await schedule.current.run(userInput); + const chartResult = []; + if (props.type === 'multiple') { + const { datasets, commands } = schedule.current.getContext() as any; + for (let i = 0; i < datasets.length; i++) { + const dataset = datasets[i]; + const { dataTable, fieldInfo, textRange } = dataset; + chartSchedule.current.setNewTask({ + dataTable, + fieldInfo, + command: commands[i] + }); + chartResult.push({ + context: await chartSchedule.current.run(), + textRange + }); + } + } else { + const { dataTable, fieldInfo } = schedule.current.getContext(); + chartSchedule.current.setNewTask({ + text, + dataTable, + fieldInfo + }); + chartResult.push({ + context: await chartSchedule.current.run() + }); + } const time2: any = new Date(); const diff = (time2 - time1) / 1000; props.onOk( + text, schedule.current.getContext(AtomName.DATA_EXTRACT), - schedule.current.getContext(AtomName.DATA_CLEAN), - schedule.current.getContext(AtomName.CHART_GENERATE), + schedule.current.getContext(), + chartResult, diff ); - }, [props, userInput]); + }, [chartSchedule, props, text, userInput]); return (
@@ -106,12 +140,32 @@ export function DataInput(props: IPropsType) {
-
+

0 + Select DataExtraction Type +

+ +
+ +
+

+ + 1 + Select Demo Data (optional)

-
+

- 1 + 2 Input your data in text format

@@ -157,10 +209,10 @@ export function DataInput(props: IPropsType) { />
-
+ {/*

- 2 + 3 Query to adjust?

@@ -170,33 +222,44 @@ export function DataInput(props: IPropsType) { onChange={v => setUserInput(v)} style={{ minHeight: 80, background: 'transparent', border: '1px solid #eee' }} /> -
+
*/}
-
- +
- setModel(v)}> - GPT-4-0613 + { + setModel(v); + // const preModel = v.split('-&&-')[0]; + chartSchedule.current = v.endsWith('-advisor') + ? new Schedule([AtomName.CHART_GENERATE], { + chartGenerate: { useChartAdvisor: true } + }) + : new Schedule([AtomName.CHART_GENERATE], { + base: { llm: llm.current, showThoughts: false } + }); + }} + > + GPT-4o Doubao-pro - Doubao-pro-32k + GPT-4o + chart-advisor + Doubao-pro + chart-advisor
-
+ {/*
setUseFieldInfo(v)}> Use FieldInfo @@ -205,7 +268,7 @@ export function DataInput(props: IPropsType) { setShowThoughts(v)}> Show Thoughts -
+
*/} (''); - const [command, setCommand] = useState(''); + const [commands, setCommands] = useState([]); const [specList, setSpecList] = useState([]); - const [dataset, setDataset] = useState([]); - const [finalDataset, setFianlDataset] = useState([]); - const [fieldInfo, setFieldInfo] = useState([]); - const [finalFieldInfo, setFinalFieldInfo] = useState([]); + const [type, setType] = useState<'multiple' | 'normal'>('multiple'); + const [dataExtractionRes, setDataExtractionRes] = useState([]); + const [dataCleanRes, setDataCleanRes] = useState([]); const [loading, setLoading] = useState(false); + const [text, setText] = useState(''); + const [showTabe, setShowTable] = React.useState(false); const [costTime, setCostTime] = useState(0); const handleOk = React.useCallback( - async (dataExtractCtx: any, dataCleanCtx: any, chartCtx: any, timeCost: number) => { - setDataset(dataExtractCtx.dataTable); - setFieldInfo(dataExtractCtx.fieldInfo); - setFianlDataset(dataCleanCtx.dataTable); - setFinalFieldInfo(dataCleanCtx.fieldInfo); + async (text: string, dataExtractCtx: any, dataCleanCtx: any, chartCtx: any, timeCost: number) => { + setText(text); + if (type === 'multiple') { + setDataExtractionRes(dataExtractCtx.datasets); + setDataCleanRes(dataCleanCtx.datasets); + } else { + setDataExtractionRes([dataExtractCtx]); + setDataCleanRes([dataCleanCtx]); + } setLoading(false); setCostTime(timeCost); - setSpec(chartCtx.spec); - setCommand(chartCtx.command); + setSpecList(chartCtx.map((v: any) => v.context.spec)); + setCommands(chartCtx.map((v: any) => v.context.command)); // eslint-disable-next-line no-console console.info(dataExtractCtx, dataCleanCtx, chartCtx); }, - [] + [type] ); return ( - + - +

{`Time Cost: ${costTime.toFixed(1)}s`}

- diff --git a/packages/vmind/__tests__/browser/src/pages/Text2Chart/index.scss b/packages/vmind/__tests__/browser/src/pages/Text2Chart/index.scss index b67ce3a9..5fa615aa 100644 --- a/packages/vmind/__tests__/browser/src/pages/Text2Chart/index.scss +++ b/packages/vmind/__tests__/browser/src/pages/Text2Chart/index.scss @@ -32,3 +32,8 @@ .arco-modal { width: unset; } + +.single-table { + min-width: 300px; + flex: 1; +} \ No newline at end of file diff --git a/packages/vmind/__tests__/browser/src/pages/constants.tsx b/packages/vmind/__tests__/browser/src/pages/constants.tsx index 6e342d96..c1727322 100644 --- a/packages/vmind/__tests__/browser/src/pages/constants.tsx +++ b/packages/vmind/__tests__/browser/src/pages/constants.tsx @@ -66,7 +66,7 @@ export const PLAYGROUND_MENU_INFO: { }; export const CollapseCSS = { width: '100vw', - height: '100vh', border: '1px solid var(--color-border)', + overflow: 'auto', background: 'var(--color-fill-2)' }; diff --git a/packages/vmind/__tests__/experiment/src/data/capcutV2Data.ts b/packages/vmind/__tests__/experiment/src/data/capcutV2Data.ts new file mode 100644 index 00000000..c0d86457 --- /dev/null +++ b/packages/vmind/__tests__/experiment/src/data/capcutV2Data.ts @@ -0,0 +1,311 @@ +/* eslint-disable max-len */ +export const capcutMockV2Data: any[] = [ + { + text: `富士康为什么突然被查？这对全国120万的富士康员工 有何影响？最近， 富士康在广东、江苏等地被税务稽查。与此同时呢， 自然资源部门 对富士康在河南、湖北等省市的用地 进行现场勘查。别小看这短短的一句话啊， 释放了两个信息， 一、是税务部门调查有没有偷税漏税。二是自然资源部门 调查土地的使用情况。要知道， 当年全国各地为了吸引富士康 使出浑身解数， 不仅帮富士康招人， 还有各种减免税， 甚至有些地方啊， 还给富士康 不能多说了啊， 按理说， 咱们给了富士康这么多优惠政策， 富士康应该感激才是啊！ +可谁能想到， 老郭这是吃尽红利 翻脸不认人呐！他居然说这是在给大陆赏饭吃！富士康为啥敢如此傲慢呢？ +先来看一组数据， 富士康位居财富世界500强第20位， 在中国企业中 仅次于国家电网、中石油、中石化 以及中国建筑集团。 +就连咱们熟知的宇宙第一大行 工商银行啊， 都排在富士康后面。 +富士康不仅财大气粗， 还跟全国各地啊深度绑定。 +目前 富士康在全国的工厂达到40多个， 总员工数超过了100万。 +所以很多人说 不能碰富士康啊， 万一富士康跑了该咋办？ +富士康真的是大到不能碰了吗？ +并不是。过去40年 我们之所以给富士康各种政策优惠， 不是因为富士康有多强， 而是因为富士康背后的苹果公司呀， 太过强大。 +没有苹果的订单， 富士康啊 连一毛钱都不值。 +过去我们扶持富士康， 那是希望打入苹果供应 链，希望苹果的零部件啊， 可以在中国生产， 在中国组装， 最后出口到全世界， 构建起一条啊 完整的电子信息产业链。 +可谁能想到， 从2019年开始， 苹果供应链开始大规模的去中国化， 中国大陆的供应占比 从巅峰时期的26%下降到如今的2.5%。 +按照苹果的如意算盘， 要想完成去中国化， 第一步呀， 就是先在越南和印度 建设一两座富士康工厂。 +第二步呢， 就是把中国供应商一点点踢出去， 换成海外的供应商。 +第三步，逐渐增加越南、 印度这些海外供应商的订单。 +第四步，也是最后的一步， 就是把中国大陆的富士康啊， 全部迁出去， 彻底完成去中国化。 +如今的苹果已经走完第二步了， 马上就要进行第三步了。 +可让他万万没想到的是， 华为啊，居然在今年9月起死回生了。 +华为的崛起 彻底打乱了苹果去中国化的节奏。 +还记得今年国庆黄金周吗？ +华为上市仅仅一个月啊， 就把苹果在中国的市场份额 干到只剩17.4%。 +吓得哭客呀， 赶紧来中国亲自下场带货！ +顾客为啥荒凉？ +如今的中国是苹果全球第一大市场， 也是苹果全球最大的食物来源。 +吃着中国人的饭， 砸着中国人的碗， 只想在中国赚钱， 不想为中国做贡献。 +不仅不做贡献啊， 还一门心思向外转移产链， 这样的企业以及背后的代工厂， 你说凭啥还要给他特殊优待呢？ +现在你明白 国家为啥要坚定支持华为了吗？ +支持华为，就是在支持中国产业链， 只有华为链才能对抗苹果链。 +华为链呀， 就是中国链。关注4G实务分析，收。 +` + }, + { + text: `俄罗斯这个国家 真的是个非常神奇的存在啊， 说到政治军事， 没有人敢小瞧他， 他拥有全世界最大的国土面积 和最多的核弹头。 +但是同时呢， 一提到经济啊， 那似乎就没有俄罗斯什么事了。 +你看这俄罗斯哈， 在短短的半个世纪里面 就经历了什么政治动荡啊、 恶性通胀、 寡头黑帮、 高速发展、 战争、政府违约、 经济危机、 激进的改革等等等等。 +所以今天呢， 小林就来给大家一起梳理一下啊 这个错综复杂而又独一无二的 俄罗斯的经济。 +它到底是怎么回事？ +我先声明一下啊， 一个国家的经济 确实是太错综复杂了， 所以呢，我只能挑我觉得比较重要 又有意思的地方来跟大家分享一下， 但是我觉得 重点应该基本上也都是重点。 +大家如果有什么自己的观点 或者想法， 也欢迎在评论区留言讨论。 +但是哈，咱话说好了， 留言区我们好好说话， 别动不动就打起来。 +还有就是最后啊， 我会有一个对俄罗斯经济的 一个简要 橄榄， 你要是也不感兴趣，都发生了什么， 就想看那个最经典的橄榄， 你可以直接划到最后， 但是我还是非常建议哈， 把前面这些东西都看一看， 毕竟能像小林这种 把这些复杂的东西又讲的清楚直白 又很有意思的人， 在这个世界上也不多了啊！ +你也可以点个赞表示一下。 +说到俄罗斯经济啊， 咱必须得从前苏联说起。 +所以这块呢， 我就一丢丢带到历史， 帮大家刷新一下记忆。 +沙俄爆发民主革命， 迫使沙皇退位。 +之后，以 列宁为首的社会民主工党 推翻了临时政府。 +经过了5年的内战， 1922年苏联诞生。 +两年之后， 1924年这个人上台， 成为了前苏联的最高领导人， 他就是约瑟夫维萨里亚的诺维奇 斯大林。从此啊， 苏联就真正的开始 全面推行计划经济。 +计划经济哈， 就是说所有的经济资源 从分配、生产 甚至到一部分的消费 都是按照计划走。 +就比如说哈， 这点物资急缺了， 上面只要一指命令，哗， 所有的物资就都调过来。 +再比如说哈， 我们现在要开始发展重工业， 需要大家一起来建工厂， 然后上面一指命令， 这工人全就都过去， 他也不需要时间 等市场经济进行自我调节。 +其实这种模式哈， 在你方向非常明确 或者 要进行整体的大规模改革的情况下 是非常高效的。 +而当时的苏联呢， 就处于工业化和现代化 都严重不足的情况， 他就非常适合计划经济。 +于是1928年之后， 斯大林的前三个五年计划， 那可谓是效果拔群， 前苏联就迅速摇身一变， 从一个落后的农业国家 变成了一个工业大国。 +你看啊， 这使前苏联的人均GDP从20年到40年 翻了三倍多。 +而你想啊， 那时候1929年的美国 正在经历着大萧条， 整个西方哈都跟着遭殃。 +所以说啊， 这20年真的是在斯大林的带领下， 把前苏联的经济带上了一个新高度。 +虽然说哈，也有各种什么政治迫害、 大清洗，但是呢， 光从经济的角度上来说， 还是效果罢群的。 +这边呢，尝到了计划经济的甜头。 +另一边呢， 二战啊，苏联的国际地位又 得到了极大的提升， 可谓形势是一片大好， 又经历了一段时期的高速增长。 +不过慢慢的啊， 从60年代开始， 这个前苏联的经济结构 也越来越复杂， 经济发展也逐渐稳定， 那计划就不是那么好计划的了。 +你说 要没有市场这个无形的手在那调控 他，再厉害的政府 也不可能说 哪哪的经济 都给你计划的那么井井有条， 更何况啊高度集权的这种计划经济。 +他还有一个问题， 就他一般情况下都是领导指哪打哪， 这就导致了前苏联政府的贪腐呀 也日益严重， 企业的创新就会严重受损。 +前苏联就陷入了停滞时期。 +` + }, + { + text: `除非是一场巨大的危机， 不然的话 你根本没有办法去解释 此刻的股神巴菲特， 他到底在等待些什么。 +美国银行， 这是一家在西方世界， 甚至说全球金融行业里 都非常有声望、 位高权重的超级巨头。 +巴菲特则是从2011年开始 对这家公司进行了大手笔的投资。 +13年时间过去， 他一路买入， 从不卖出， 甚至说一口气把自己1/10的仓位 都压在了美国银行的身上。 +几乎所有人都觉得 美国的标志性金融大佬 买入美国标志性的金融巨头 这笔买卖天作之合。 +但是94岁的巴菲特 却把所有人都打了一个措手不及， 此时此刻， 他在狂卖美银。 +根据美国SEC官方披露啊， 光是在9月的4号、5号 还有6号这三天， 他就卖出了1870 是4万股的美国银行。 +而如果要把时间拉得更长一些的话， 那么从今年的7月份开始， 巴菲特就已经从美银的身上 套现了将近70亿的美元。 +什么概念？ +500亿的人民币。 +这笔钱对于巴菲特巴老爷子来说 也许不算太多， 但你要知道， 上个月 他不计成本的抛售苹果公司， 就已经把全球金融市场吓死 魂飞魄散。 +当时啊，他在毫无征兆 没有任何预告的情况下， 直接把苹果的仓位砍掉了50%。 +说真的，在我们金融行业当中啊， 甭管你干了多久， 也没有人见过这样的场。 +甚至就连美国的华尔街 现在也纷纷开始了议论， 说抛售苹果 还有美英这样的常见股票， 根本就不是巴菲特的风格。 +看到这你肯定会以为啊， 这是他年事已高退出江湖的 前兆。就像是曾经的地主一样， 每到过年就要颗粒归仓， 清算一波自己的家底。 +但你知道吗？ +在过去的2023年， 巴菲特依靠投资定数了6900个亿， 而且还在年初正式的宣布了接班人， 规划之清晰， 战略之稳定， 看不见任何散场的。 +而同时呢， 全世界的金融机构、 各大投行的交易员 又眼睁睁的啊 看着他手里的现金累的跟山一样高。 +截止到目前 已经有差不多3000亿的。 +哪怕你再不关心经济， 也应该思考了， 巴菲特现在到底在等待什么？ +上一次 他手里持有如此高比例的现金， 还得回忆到2005年。 +而在之后的没多久， 美国就爆发了次贷危机， 他顺势抄了一波大底。 +夕阳之所以耀眼， 是因为牺牲了整个白天 所有的光芒。 +而94岁的巴菲特， 此刻完全有可能啊， 正在屏息等待， 准备挥出他光荣山崖里最后一棒！ +` + }, + { + text: `这个菲律宾怎么回事啊？ +双倍差战，五渣， 回回碰瓷， 次次灰头土脸， 这有完没完啊， 这人啊，那么好了， 这货到底是个什么样的国家？ +这么做图什么？ +谁在背后给他撑腰？ +今天我们从经济的角度 来给大家讲讲菲律宾， 三个词来概括一下菲律宾， 那就是美国舔狗、 寡头经济和菲佣帝国 对美国爱的深沉。 +一度甘当美国殖民地， 不想独立， 甚至想加入美国成为人家一个州， 奈何美上百个门阀家族轮流坐庄， 把持政权， 断经济， 把曾经的亚洲第二富国祸祸 成了亚洲最穷的国家之一。 +而另外支棱起菲律宾经济的 不是什么高科技、工业、 金融业， 而是遍布全球的雇佣和劳工。 +总之横看竖看就是有点挫呀， 人是怎么变成今天这个样子呢？ +简单来说呢， 就是先天条件一般， 还总想走捷径 哈哈哈。这个国家，从地图上来看啊， 挺有意思的， 由7000多个岛来组成 七零八落， 很难大规模连成片的发展， 于是很长时间呢， 人家都没有建成统一的国家， 而是各个岛上的家族部落啊， 各管各的。 +直到16世纪啊，迈之伦转悠到这了， 带着西班牙殖民者 征服了这，菲律宾呢， 才正式建了国。 +接着人又相继被美国殖民 日本占领， 再被美国收回， 而那些岛上的大家族啊， 也成了殖民代理人。 +那这个过程啊， 美国的影响是最大的。 +人，一边在这搞建设、 搞教育，让菲律宾 成了亚洲最早 走上工业化道路的国家之一。 +一边呢， 又残酷的镇压菲律宾的民族主义， 扶持清美势力。 +那结果 就是，菲律宾的经济是发展起来了， 可是完全依赖美国。 +而看似有了国家政权， 其实硬骨头几乎都被打没了。 +那情况有多夸张啊？ +1929年美国经济大萧条， 没精力管菲律宾了， 想甩手啊， 让人独立得了， 这菲律宾人反而不干了， 说怎么的， 霍霍够我了， 翻脸不认人了是吧？ +想分手啊， 没那么容易。 +于是两边各种谈条件 扯皮，直到1934年才谈妥了。 +美国接着扶持菲律宾， 10年之后再让菲律宾独立。 +后来呢， 人民意义上确实也是独立了， 但是加入美国的心啊， 他一直没死。 +上世纪60年代， 就有菲律宾的国会议员呢， 就发起入美运动， 要让菲律宾加入美国， 甚至呢，还说可以靠人多优势啊， 选菲律宾人当美国总统。 +而这类运动啊， 断断续续到现在都还有， 那场面 大概就是美国，我叫你一声爸爸， 你敢应吗？ +可美国呢？ +大家看到了啊， 看到了是他主动啊， 跟我没关系啊。 +其实1946年独立之后的菲律宾啊， 经济情况一度非常好， 工业基础 是完全继承美国殖民时期的遗产， 经济贸易呢 也几乎完全适合美国的单边贸易。 +而50年代之后， 朝鲜战争和越南战争先后爆发， 菲律宾凭借地理优势啊， 拿下了大量的美军订单， 这生意红火呀， 这也让菲律宾的经济坐上了火箭。 +1960年，菲律宾GDP啊，是排全球第20。 +>而放眼亚洲， 除了日本几乎没有对手， 而人均GDP 甚至一度都是我们的3倍之多。 +人家首都马尼拉 人送外号是亚洲小纽约， 甚至一度有机构预测说， 50年之后， 菲律宾将会成为世界第十大强国。 +换句话来说， 人能跟法国、 意大利这种老牌资本主义国家 坐在一张桌子上吃饭， 好家伙呀！ +啊！那现在我们听起来 感觉跟科幻故事一样， 因为不用50年， 你看看现在 人已经凉透了。 +全亚洲最穷的国家之一， 泰国、马来西亚都能甩他几条街。 +为什么会变成这样呢？ +就是因为菲律宾大家族门阀遍地， 菲律宾从被殖民到老大 不乐意的独立， 这就相当于什么呀， 殖民的走了， 世家大族们呢， 继续接力统治。 +比如在1987年的菲律宾国会， 总共200个众议员， 其中有130个是来自于大家族的， 另外还有39个呢， 跟大家族还是沾亲带故的。 +这些家族门阀呀， 各种资源在手， 那还不利用特许经营权、 垄断权、 免税权等等这种各种特权大捞特捞， 奉旨发财呀。 +而一马当先呢， 就是菲律宾现任总统 小马克斯的老爸 老马克斯， 这也是菲律宾的前总统。 +这位老马的形式风格 就是从政界到商界 都塞满了自己人。 +在菲律宾的任何生意 都要先给老马家族10%的好， 而所以人送外号10%先生。 +最终 这个老马因为做的实在太过分的， 吃相太难堪了， 把其他大家族都得罪光了， 不得不带着上百亿美元资产 逃到了美国。 +而留下的菲律宾呢， 已经成了 外债高达265亿美元的穷光蛋国家。 +为什么会这样？ +因为相比贪腐家族门法，垄断经济 最要命的问题是让菲律宾走上了趣 工业化的， 把美国留的家底都给败光了。 +毕竟啊，你想啊， 只要垄断必需品 他就能躺着赚， 干嘛还要搞什么苦哈哈的 什么制造业高科技啊。 +所以你就会发现， 菲律宾的工业化进程 在上世纪60年代之后就卡死不动了。 +同样是和美国经济关系密切的日韩， 累死累活先从做衣服、做鞋子干起， 积累资本， 然后产业升级 发展高精尖的时候， 菲律宾反而从亚洲第二大工业国， 慢慢 先是被新马泰这样的邻居给超越， 接着又倒退成了农业国。 +然后呢，又随着全球贸易自由化， 叠加了拉美危机、 亚洲金融危机 等等几波全球经济动荡的冲击， 菲律宾那是全面溃败啊。 +低端制造业丧失竞争力， 高端制造完全没升级。 +看来看去， 最后 人家只能搞搞轻松好赚钱的服务业。 +2015年的菲律宾的农业和工业 占GDP的份额 分别从1980年代超过20%和40， 掉到了只有10%和33%。 +而由于去工业化太早了， 就算是人家搞服务业， 也搞不出什么高附加值的服务业， 基本上只能搞搞旅游啊、 休闲等等低端服务。 +这就是今天我们看到菲律宾的样子。 +2022年菲律宾的GDP总量是4000亿美元出头， 什么概念呢？ +还没咱们中国的一个零头高， 而人均收入是在1200-1800人民币左右， 这是亚洲最穷的国家之一， 同时呢，贫富差距特别巨大。 +如果说你站在今天的菲律宾首都 马尼拉的市中心里面， 就会觉得，哎， 跟咱们北上广好像没什么两样， 高楼林立， 车水马龙， 房价也得四五万一平往上走， 而一街之隔的是一座座垃圾山 以及上百万人居住的贫民窟。 +这贫民窟啊， 几千块钱 就可以买走一个女孩做老婆。 +大家把从垃圾堆里面 捡来的吃的东西 洗一洗，重新加热 就成了。菲律宾的特产 叫派派。这叫什么呀？ +富者田连阡陌， 贫者无立锥之地。 +而因为工业、农业拉垮人， 服务业又没有办法提供大量的就业， 大量普通菲律宾人呢， 想混口饭吃就只能出国打工。 +这又诞生了菲律宾的另一个奇观。 +因为早年美国人的介入管理啊， 菲律宾人的英语都不错啊， 服务意识也很强， 直接就把菲佣 干成了菲律宾的支柱产业之一。 +光去年菲律宾的海外劳工 就赚了372亿美元的外汇， 这占到了人家经济总量的8.5%， 这个比例什么概念？ +去年中国汽车超越了房地产， 成了中国经济第一支柱， 占的比例也就差不多10%啊， 这就是目前菲律宾的经济现状。 +所以你就能够理解， 为什么每当全球经济环境下，行 人不会去努力奋斗扭转国运， 而是习惯性靠走捷径来解决问题。 +比如频频碰瓷东方大国， 试图向自己精神上的父亲那投名状。 +那为什么 人连像样的大船都造不出来， 每次碰瓷还都是小破渔船出卖。 +用今天的话说， 这纯纯的又菜又爱闹。 +可用2000年前的韩非的总结 则是，国小而不储卑， 利少而不畏强， 无礼而武大林， 贪鄙而拙骄。` + }, + { + text: `娃哈哈又一次站上风口浪尖了。 +这次是接班人发生变化了。 +刚刚去世没多久， 也就是娃哈哈创始人钟老的女儿 宋富丽 突然公开提交了一份辞职信。 +信的内容。大概我们总结一下啊， 就是我爸去世以后， 娃哈哈部分股东就质疑， 由我来接班掌管娃哈哈是不合理的， 这活我干不下去了， 所以 我要辞去副董事长和总经理的职。 +这事一出， 问题很多的小明就表示说，哎呦， 有那味了， 先帝驾崩， 老臣和新王 有些能相辅相成， 有些却是有你陪我呀。 +哈哈哈，假人呐。 +不光是这样， 娃哈哈 远远没有女神副业这么简单呢。 +你得知道这公司的真正老板是谁？ +股东都是谁啊？ +娃哈哈 它并不是一家纯粹的民营企业， 它是一家国企改制而来的 混合所有制企业。 +简单来讲， 那就是既是有国有股东 也有民营股。 +怎么回事呢？ +这公司 不是宗老个人直接创立的企业， 当年前身 是 浙江杭州上城区的一家校办企业 的经销部， 算是地方国资企业吧。 +宗老当时是那公司里面的员工啊。 +在1987年，42岁的宗庆后呢， 是以职工的身份 承包了这家校办企业， 获得了完全独立的经营权。 +后来呢，给这企业再取名为娃哈哈。 +接着，大家都知道了， 做大做强， 再创辉煌， 一步一步 把这家曾经经营困难的校办企业 办成了全国V百强。 +对娃哈哈有再造之功的宗老啊， 他的身份既不是娃哈哈的老板， 其当年手上也没有一丁点的股份， 毕竟啊，只是承包嘛， 说白了这公司又不是你的。 +而这个转变呢， 就是在2000年， 在那一年， 娃哈哈完成了改制， 也就是这家国资企业决定啊， 让出一部分股东给员工， 大家伙，哥几个量力掏钱买股份， 为的就是什么， 大家都当股东， 那可不就更能激发干劲了吗？ +宗庆后自然是首当其冲。 +而之后 哇哈哈的股权就一分为三了， 国资占一部分， 中金后自己占一部分， 然后员工又占一部分。 +那现在最新的股东结构是什么呀？ +杭州国资委占股百分之46， 宗亲后占股29.4%， 员工占股24.6%。 +这看上去好像这终老股份不多 还不到30%， 可是员工们跟他几乎是一条心啊啊， 几十年风雨同舟， 那是一致行动人， 这两边加一块 占股有54%。 +所以说 当年宗庆后虽然不是大股东， 但却始终是哇哈的实际控制， 看上去一切是不是都挺完美的呀？ +啊，但实际上也有隐患， 创一代的强人治理跟人格魅力 在去世之后往往会留下真空。 +说白了，对于那老员工来说， 当年我认你， 宗庆后，我跟你， 我服你，但是你走了， 你家孩子那就另错了。 +一方面呢， 是啊，宗老去世之后， 这宗复利 还没有正式继承 宗老在娃哈哈的股份， 他的身份也一直是副董事长、总经理。 +接管娃哈哈 暂时从法律上来讲 还没有依据， 而且哪怕他后面继承了， 还得获得员工持股会的认可 以及国资股东的批准， 但这不重要啊。 +另一方面， 东富力进入到这， 哇哈哈，仿佛感觉跟这家公司啊， 他有代沟啊， 而且在处理跟老员工的关系上呢， 似乎也一直是他的短板。 +你看这小 宗什么学历啊， 海外留学归来， 性格跟作风呢， 相对来讲更加直接。 +而宗老是什么？ +为人更加传统， 更加讲究人情。 +我举几个例子啊， 比如宗老曾经就表态说， 咱们娃哈哈呀， 不缺钱，没有上市的必要。 +而宗富利呢， 却公开唱反调 说娃哈哈要上市很正常， 只有跟资本市场结合才能走的更远。 +再比如啊， 宗富利对内是铁腕管理啊， 如果说你没有完成任务， 可以直接开除， 丝毫不讲情面， 我管你是不是老员工， 而宗老呢， 讲究的是用人质、 聚人心，甚至呢， 还会再偷偷的把这些老员工啊 再请回来。 +但是你要是从商业 或者说管理层面单独来讲， 你很难就说 这小宗做的就一定有问题， 老宗就做的一定没毛病， 而这问题的核心其实是什么呀？ +娃哈哈，如果说是你宗馥利创立的， 那就是你宗馥利做的对。 +而更重要的一点， 那就是在经营的业绩层。 +这小宗早年呢， 做了一系列的尝试， 效果只能说是一般般。 +21年底，这小宗宗富力 升任娃哈哈的副董事长兼总经理， 同时宣布推出了20多款新品。 +可结果呢？ +整个22年呢， 娃哈哈的总营收是512亿， 比21年还下跌了7个亿。 +这是什么呀？ +一顿操作猛于虎， 一看还在原地杵啊！ +你不能说综合力不努力， 抬头看看全球这几年大环境， 你说他真的是有点生不逢时吧？ +虽然不算准确， 但也有点影响， 但只不过出了问题 就一定需要有人去负责。 +老宗在的时候， 也许可以给小宗不断试错的机会， 而那些开朝老臣们呢， 就算有点想法，也不方便明着撕破脸， 而现在 谁还能站在小宗身后来给他善后？ +>当然了，无论如何啊， 都希望娃哈哈这个民族品牌 能够好好的， 毕竟啊，那是包括我在内 几代人童年的回忆啊！` + }, + { + text: `你敢信吗？ +特朗普给国货来了一波实力带货。 +上个月啊， 特朗普跟马斯克 可以说是俩全球顶流的男人 连线直播， 结果有眼见的网友就发现说，哎， 特朗普手机下面垫的这个充电宝 好像是中国品牌安克的， 你看看，工作归工作， 这个MADE in China才是生活。 +就这么个小小的细节啊， 让安克的股价在被发现的一瞬间 就跳涨了3%。 +安克的客服都说了， 好多人都来咨询这个充电宝， 安克也非常机智的 在这个产品描述里 加上了一个懂王同款。 +>不得不说啊， 特朗普这个带货实力确实不一般。 +安克这个品牌啊， 可以说是一个隐藏的巨头， 别看它就是卖卖充电宝啊、 充电线这些小配件， 但它 在中国全球化品牌50强的榜单里边 能排到第16， 仅次于亚迪， 绝对属于那种低调的行业隐形冠军。 +安克是2011年 由一个前谷歌工程师杨萌成立的， 那时候 深圳华强北正好有一波出海潮， 国内市场太卷， 加上阳萌又有海外背景， 所以他第一步就另辟蹊径， 选择跳过国内市场， 直接冲击海外。 +想利用中国的供应链去打海外市场， 这个思路 我们现在可能感觉习以为常了， 但放在当时 还是挺大胆的。 +而这招专攻海外， 也直接让安克抱上了两条大腿， 这第一条大腿就是苹果。 +2011年左右， 正好是智能手机加速普及， 但这个官方的配件吧， 大家都懂， 太贵了。它大部分平替的配件呢， 倒是便宜， 但质量就一般般。 +哎，安克他就瞄准了这个市场， 专门给苹果做高质量的 配件。 +他很聪明的一点哈， 就是他会盯着iPhone去宣传， 紧跟着苹果蹭流量。 +iPhone在哪打广告， 或者有人在哪搜iPhone相关的关键词， 哎，安克就把广告打在哪， 你前脚刚看完iPhone， 后脚安克就弹出来了。 +感兴趣iPhone手机， 我这有配件， 还物美价廉。 +当然哈，安克这产品确实不错， 颜值也很高， 慢慢呢就拿下了一批苹果用户， 而他的第二条大腿就是亚马逊。 +安克在非常早期就入驻了亚马逊， 当时亚马逊的排序机制 主要就是看销量和评价。 +安克呢， 就把广告都导到亚马逊去成交， 积累了非常好的销售数据， 同时用户也很满意， 很快就成为了亚马逊 消费电子里的王牌卖家。 +同类产品里， 它的市场份额可以达到30%。 +而后来到2020年的时候， 又吹来两股风， 让安克直接起飞。 +一个就是 疫情，让美国人也开始习惯网购了， 线上购物的需求大大增加， 这就让非常擅长线上营销的安克销量跟着大涨。 +而另一股风呢？ +哎，还是苹果， 苹果不是从iPhone 12开始 就不送充电器了吗？ +这就让他把一部分业务 拱手又让给了安克， 这也让安克即使这两年 还能实现百分之三四十的增长， 成为了 可以说是配件品牌出海的标杆。 +所以你看最近这几年 品牌出海已经不光是汽车啊、手机啊、 家电这些大品牌的趋势， 类似安克这种细分领域的自主品牌、 新兴品牌 也掀起了国货出海的一股浪潮。 +前两天我去上海，那外滩大会 就看到了非常多新奇的中国 出海好货， 有那种自主设计师设计的鞋呀、 衣服呀、椅子这种日常消费品， 还有很多非常新颖的， 像智能猫砂盆啊， 纸牌屋同款的划船机啊， 超轻钛合金的伊拜克 三防固态硬盘等等， 这些 其实都是比较细分领域的行业冠军， 不管是从技术质量 还是那个设计的精致度， 你看完感觉还真挺震撼。 +到现在啊， 在亚马逊的美、英、日、法等等站点， 中国卖家的比例都已经超过了50%。 +同时呢， 这些出海的品牌也更注重本土化、 多元化和品牌化。 +比如像安克 最开始是靠着亚马逊， 但慢慢也延伸到了阿里express、 ebay等等， 现在也开始有了自己的独立站， 同时在线下拓展渠道。 +根据万里汇2024年上半年的数据啊， 在平台上改走品牌化道路的卖家 同比暴涨了138%， 他们也更注重 在本地区设立主体运营， 打通供应链等等。 +你像这些品牌卖家， 他在出海的过程当中 经常会遇到一个问题， 其实就是金融上的， 比如说跨境支付啊， 外汇的收付啊等等， 所以就会有一些 这样的跨境金融管理工具， 比如蚂蚁旗下的万里汇， 假设你想把货卖到欧美、 东南亚等等等很多国家， 那你可能会需要，哎， 我亚马逊开一个，户 Lazada开一个， TikTok shop开一个， 然后每个平台都得单独认证， 什么美元、欧元、 印度卢比都得单独管理 是吧？当时就头大， 所以呢， 他们就会用万里汇这样一个账户， 链接130多个 涵盖了几乎所有 主流的跨境电商平台， 就可以在像亚马逊、shein、沃尔玛 TikTok等等的这些平台收付款， 还能一键完成平台的认证啊、 合规啊等等， 包括很多品牌独立站， 它也能支持全球200多个国家地区 30多个币种的收款， 还有近百种币种的付款。 +而且哈，这个万里汇账户 它还是一个出海的资金管理中心。 +怎么讲呢， 你像理财贷款， 还有一些外汇的避险工具， 包括应急的资金周转等等， 这些金融功能也有。 +就好像有一个非常懂出海的CFO 给你提供金融支持， 包括资源对接， 这样卖家就可以专注在把货给做好、 卖好， 让这些出海的国货可以走的更稳。` + }, + { + text: `哪个国家 印出了这张100万亿元的钞票？ +我跟你说， 他的通货膨胀可以说是世界闻名， 从2000年开始 通胀就没下过三位数， 而伴随着这个恶性通胀的 其实就是金元对美元的急速贬值。 +98年的时候， 一美元还能兑24津巴布韦元， 到06年就贬值到1美元兑10万金元。 +这津巴布韦政府一看， 对吧？这0也太多了， 干脆咱就抹掉3个0， 定义一个新的津巴布韦元 叫ZWN啊， 他原来那个货币叫ZWD， 这样一美元就兑100金元了， 对吧？你看 多聪明。结果 津巴布韦这边 没止住恶性的通货膨胀， 这金元贬值的速度就越来越快。 +一年后，07年4月的时候 又跌到了一美元兑3万津巴布韦元， 年底的时候 又变成一美元兑200万金元， 之后每个月都在狂泻， 半年之后 就出现了那张5000亿金元的钞票。 +津巴布韦政府一看， 这0又太多了 对吧？老办法 再抹0，这次直接抹掉10个0， 就是除以100亿， 然后这新的货币呢， 就叫做DWR。 +好吧，这个时候啊， 津巴布韦通过膨胀 据估算已经飙升到了 百分之九乘以10的22次方， 那外汇市场可想而知。 +到半年之后， 就09年初的时候， 它又印出来那张举世闻名的 100万亿金元的钞票。 +不过你看着这么多万亿， 最后其实也就值不到0.4美元， 那央行说好 咱再去要12个0 再除以1万亿啊， 这个新的货币就叫ZWL。 +你看，短短三年里， 津巴布韦政府已经砍掉了25个零， 到最后这金元已经完全没有信用了。 +你想你这政府整天这么砍零，然 后推出新货币， 到最后可能这26个字母都不够你用。 +无奈之下， 政府被迫放弃金鱼，哎， 用欧元美元， 这才止住了国内的恶性膨胀。 +不过最近哈，刚缓过来， 1.19年的时候， 津巴布韦又火奋起来， 推出了自己的货币， 通胀又飙到三位数。 +然后今年4月份， 他就又又又推出自己新货币， 叫ZWG津巴布韦金， 这回号称是有贵金属倍数， 你就说这津巴布韦它乱不乱？ +通过这个极端的例子啊， 你其实非常容易看出来， 通货膨胀跟外汇之间的关系。 +物价，它对汇率的影响最底层， 但同时 也可能会非常的缓慢的隐形。 +所以你如果一般看 那种比较主流的货币啊， 就比如说美元 过去50年通过膨胀跟汇率的对比， 只有80年代美国高速通胀的时期， 你才能非常明显的看到富相关。 +从90年代之后 就没有什么特别明显的相关性， 这为什么啊？ +就是因为像美元、欧元、日元， 包括人民币， 他大部分的时间通胀都是在10%以下， 这个区间叫轻微的波动。 +你要不是放到像什么津巴布韦、 委内瑞拉、 土耳其这种 通胀一下就飙到超过50%的 这些国家， 一般看起来啊 可能都不会那么明显。` + }, + { + text: `国产芯片终于迎来重大突破了。 +说两个好消息， 第一， 国产光刻机已经开始量产推广了。 +这是工信部公开的数据， 差不多相当于阿斯麦公司的1460K， 可以生产65纳米级别的芯片。 +虽然制程不是很先进， 但是它的重要意义 在于核心零部件国产化 被西方卡脖子。 +用官方的话术来说， 这是事关综合国力 和国家安全的国之重器。 +到今天，全国上下 其实都已经达成了一项共识， 就是 我们不能再对外资抱有任何幻想。 +芯片产业链国产化势在必行。 +当然，要实现这么宏大的目标， 不仅需要国家支持， 更需要企业参与。 +这里就要讲到第二个好消息了， 2024年9月20日， 由长城汽车牵头 联合开发的车规级MCU芯片紫金M100 已经全部完成并成功点亮。 +光看这条消息， 很多人可能体会不到 这颗芯片的重要意义。 +说几个数据吧， 2023年，中国新能源汽车产量 达到了958.7万辆， 然而 我们的汽车芯片自给率却不到10%。 +这其中像IGBT这类功率芯片 国产率大概有35%， 但像紫金M100这种 难度更大的MCU芯片 自给率却只有10%。 +这就意味着 中国的汽车芯片市场 绝大部分营收和利润 都被外国企业拿走了。 +而且我们的汽车产业发展 也会随时受到外人的限制。 +比如2021年 西方国家的供应链危机 引发了一轮芯片荒， 导致中国汽车减产了198.2万辆。 +所以长城这颗国产自研的紫荆M100 不仅是国内芯片产业链的重大突破， 也是中国汽车产业打破垄断、 实现芯片自由 的关键一步。 +更重要的是， 这颗芯片的国产化程度非常高， 从硅知识产权的获取 到精源制造、封装测试 各个核心环节都在国内完成， 这样就能大幅度降低 被国外卡脖子的风险。 +而且这还只是个开始。 +这次公布的紫荆M100 会首先应用在X55大灯控制器平台， 计划5年上车量不低于250万辆， 今后 还会拓展到空调抬头显示等等模块。 +未来还会推出紫荆M200和M300系列， 主要应用在动力、底盘系统， 甚至 还包括汽车最核心的中央处理器 紫荆S300等。 +到长城汽车完成了这套芯片矩阵， 再加上智能驾驶方面 开启全国体验的长城全场景。NOA， 在智能座舱领域相当亮眼 的coffee OS3的全新升级， 这足以让它在智能化时代的竞争中 立于不败之地。 +事实上，长城在这些技术上的投入 已经获得了非常不错的回报。 +比如说今年9月上旬， 全新魏牌阑珊在6座7座SUV排行榜中 以1741辆的销量拿下周榜。 +还有长城汽车 今年的季度盈利和营收 双双超过预期， 直接带动最近的长城股价大幅上涨。 +这就告诉我们一个道理， 国内车企必须死磕技术， 为消费者带来好产品， 才能获得更好的业绩 和更长远的发展。 +这次推出的芯片和光刻机 可能不是行业里最先进的， 也不能一步到位完全取代西方技术。 +我们需要正视差距， 尊重事实。 +我们也需要更多的企业像长城一样， 勇于踏出自主研发的第一步。` + }, + { + text: `当然了，除了俄罗斯之外，其他国家也在买中国汽车，比如墨西哥。 2013 年，墨西哥所有销售汽车中有 25% 来自中国，而在 6 年前这个数字为0。澳大利亚也在不断买中国汽车，最受澳大利亚欢迎的中国汽车品牌是名爵，去年卖了 5.8 万辆。在新能源车市场，比亚迪则占据了澳大利亚的新能源汽车 14% 份额，位于第二名。当然，这里也不得不提一下第一名，那就是特斯拉市场份额高达53%，在东南亚市场，中国车企业销量在 2013 年同样实现小幅上升，最典型的就是泰国，在泰国的新能源车市场，中国品牌占据了 80% 的份额，比如比亚迪的原 plus 就是泰国的新能源车爆款，那到底是什么原因让中国汽车爆卖呢？基本还可以总结为三方面原因，首先是全球疫情爆发，由于中国汽车的供应链完善，疫情期间仍能维持稳定生产，而日韩这些过去的出口大户受疫情影响，芯片、钢材、橡胶等关键原材料短缺，不仅汽车产能下降，而且成本升高，这就让中国汽车更具性价比。而随着中国国内新能源汽车市场越来越卷出海，成为不少中国车企的选择，比如比亚迪 2023 年进入全球 58 个国家和地区，出口汽车 24 万辆，是上一年度的 3.34 倍。在泰国新能源车市场，比亚迪单独占到了 40% 的市场份额，是名副其实的泰国新能源汽车销冠。而且中国新能源汽车并非只是具备成本优势，汽车与 AI 互联网融合的智能化更是中国车企的拿手好戏。从豪华配置到智能大屏，从外观设计到内饰比拼，这让中国新能源汽车的溢价能力明显变高。2019 年中国新能源汽车平均出口价格每量只有 5, 000 美元， 2022 年涨到了 2.2 万美元。比如比亚迪汉在欧洲发布时价格接近 50 万人民币，是国内售价的两倍多。在泰国、以色列、新西兰等多个国家，比亚迪也已经是新能源汽车的销售冠军。不过，中国汽车征服海外虽然是一部励志爽门，但其实有不少挑战。` + } +]; diff --git a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx index bf344f92..61a7192d 100644 --- a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx @@ -3,20 +3,31 @@ import React from 'react'; import type { FieldInfo } from '../../../../../src/index'; import type { TableColumnProps } from '@arco-design/web-react'; -import { Avatar, Card, Divider, Empty, Select, Table, Tooltip } from '@arco-design/web-react'; -import { result } from '../../results/chartGeneration/result2'; +import { Avatar, Card, Divider, Empty, Select, Table, Tooltip, Tabs, Typography } from '@arco-design/web-react'; +import { result } from '../../results/chartGeneration/capcutPromptResult'; import { IconInfoCircle } from '@arco-design/web-react/icon'; import { isArray } from '@visactor/vutils'; import { uniqBy } from '../../../../../src/utils/common'; -import VChart from '@visactor/vchart'; +import VChart, { registerLiquidChart } from '@visactor/vchart'; +import type { ChartGeneratorCase } from './type'; import '../page.scss'; +import { average } from '@visactor/vchart/esm/util'; +const TabPane = Tabs.TabPane; + +registerLiquidChart(); const llmList: string[] = uniqBy(result, 'model').map((v: any) => v.model); const datasets: string[] = uniqBy(result, 'dataset').map((v: any) => v.dataset); export function ChartGeneratorResult() { const [llm, setLLM] = React.useState(llmList[0]); const [dataset, setDataset] = React.useState(datasets[0]); const [type, setType] = React.useState('default'); + const [resultType, setResultType] = React.useState<'all' | 'chart'>('chart'); + const [highlightInfo, setHighlightInfo] = React.useState({ + index: -1, + textRange: [0, 0] + }); + const [activeTab, setActiveTab] = React.useState('1'); const filterRes = React.useMemo( () => result.filter(v => v.model === llm && v.dataset === dataset && v.type === type), @@ -25,20 +36,120 @@ export function ChartGeneratorResult() { const idPrefix = React.useMemo(() => `${llm}-${dataset}-${type}`, [dataset, llm, type]); - React.useEffect(() => { - filterRes.forEach((res, index: number) => { - if (res.spec) { - (document.getElementById(`${idPrefix}-chart-${index}`) as any).innerHTML = ''; - const cs = new VChart(res.spec as any, { - dom: `${idPrefix}-chart-${index}`, - mode: 'desktop-browser', - autoFit: true, - disableDirtyBounds: true - }); + const renderTimeCost = React.useCallback(() => { + llmList.forEach(model => { + const divId = `${model}-${dataset}-${type}`; + const timeCostDiv = document.getElementById(divId); + if (timeCostDiv) { + timeCostDiv.innerHTML = ''; + const timeCostList: any = []; + const textLengthList: any = []; + result + .filter(v => v.model === model && v.dataset === dataset && v.type === type) + .forEach((v, index) => { + timeCostList.push( + { + index, + type: 'all', + y: v.timeCost + }, + { + index, + type: 'extraction', + y: v.extractionCost + }, + { + index, + type: 'chart', + y: v.generationCost + } + ); + textLengthList.push({ + index, + type: 'textLength', + y: v.context.text.length + }); + }); + const cs = new VChart( + { + type: 'common', + seriesField: 'color', + data: [ + { + id: 'timeCost', + values: timeCostList + }, + { + id: 'textLength', + values: textLengthList + } + ], + series: [ + { + type: 'bar', + id: 'bar', + dataIndex: 1, + label: { visible: true }, + seriesField: 'type', + xField: ['index', 'type'], + yField: 'y' + }, + { + type: 'line', + id: 'line', + dataIndex: 0, + label: { visible: true }, + seriesField: 'type', + xField: 'index', + yField: 'y', + stack: false + } + ], + axes: [ + { orient: 'left', seriesIndex: [0] }, + { orient: 'right', seriesId: ['line'], grid: { visible: false } }, + { orient: 'bottom', label: { visible: true }, type: 'band' } + ], + legends: { + visible: true, + orient: 'bottom' + } + }, + { + dom: divId, + mode: 'desktop-browser', + autoFit: true, + disableDirtyBounds: true + } + ); cs.renderAsync(); } }); - }, [idPrefix, filterRes]); + }, [dataset, type]); + + React.useEffect(() => { + renderTimeCost(); + }, [activeTab]); + + React.useEffect(() => { + filterRes.forEach((res, index: number) => { + const { chartRes: caseChartRes } = res; + const chartData = caseChartRes?.length ? caseChartRes.map(v => v.context.spec) : [res.spec]; + chartData.forEach((spec, subIndex) => { + if (spec) { + (document.getElementById(`${idPrefix}-chart-${index}-${subIndex}`) as any).innerHTML = ''; + const cs = new VChart(spec as any, { + dom: `${idPrefix}-chart-${index}-${subIndex}`, + mode: 'desktop-browser', + autoFit: true, + disableDirtyBounds: true + }); + cs.renderAsync(); + } + }); + }); + renderTimeCost(); + }, [idPrefix, filterRes, renderTimeCost]); const renderTable = React.useCallback((context: any) => { const { dataTable, fieldInfo } = context; @@ -73,6 +184,166 @@ export function ChartGeneratorResult() { ); }, []); + const renderOneCard = React.useCallback( + (caseData: ChartGeneratorCase, id: string, caseIndex: number) => { + const { context: caseCtx, command, chartRes: caseChartRes } = caseData; + const chartData = caseChartRes?.length + ? caseChartRes + : [ + { + context: { + ...caseCtx, + spec: caseData?.spec, + command: caseData?.command + }, + textRange: null + } + ]; + const datasets = caseCtx?.datasets?.length + ? caseCtx.datasets + : [ + { + dataTable: caseCtx.dataTable, + fieldInfo: caseCtx.fieldInfo + } + ]; + const text = caseCtx.text; + return ( + console.log('Context is :', caseData)}> + {chartData.map((chart, index: number) => { + const { context } = chart; + const { command, spec } = context; + const { textRange } = datasets[index]; + return ( +
{ + if (textRange) { + setHighlightInfo({ + index: caseIndex, + textRange: [text.indexOf(textRange[0]), text.indexOf(textRange[1]) + textRange[1].length - 1] + }); + } + }} + onMouseLeave={() => { + setHighlightInfo({ + index: -1, + textRange: [0, 0] + }); + }} + > +

Command: {command}

+
+ {spec && ( +
+ +
+ )} + {!spec || resultType === 'all' ? renderTable(datasets[index]) : null} +
+
+ ); + })} + + ); + }, + [renderTable, resultType] + ); + + const renderTextByIndex = React.useCallback( + (text: string, index: number) => { + const { index: hIndex, textRange } = highlightInfo; + return text.split('').map((v, textIndex) => { + return ( + = textRange[0] && textIndex <= textRange[1] + ? '#ffa500' + : 'rgba(0,0,0,0.7)' + }} + > + {v} + + ); + }); + }, + [highlightInfo] + ); + + const renderCompareResult = React.useMemo(() => { + return ( +
+
+
TEXT CONTENT
+
+ CHART RESULT + +
+
+
+
+
+ {filterRes.map((v, index: number) => ( + + + {index + 1} + + {renderTextByIndex(v.context.text, index)} + + ))} +
+
+
+
+ {filterRes.map((dataResult, index) => { + return renderOneCard(dataResult, `${idPrefix}-chart-${index}`, index); + })} +
+
+
+
+ ); + }, [filterRes, idPrefix, renderOneCard, renderTextByIndex, resultType]); + + const renderSummaryResult = React.useMemo(() => { + return ( +
+ {llmList.map(model => { + const modeLData = result.filter( + v => v.model === model && v.dataset === dataset && v.type === type && v.timeCost >= 1 + ); + return ( + <> +
{`${model}`}
+
{`Avg Time Cost: ${average(modeLData.map(v => v.timeCost)).toFixed(1)}s`}
+
{`Avg Data Extraction Cost: ${average( + modeLData.map(v => v.extractionCost).filter(v => v >= 1) + ).toFixed(1)}s`}
+
{`Avg Chart Generation Time Cost: ${average(modeLData.map(v => v.generationCost)).toFixed( + 1 + )}s`}
+
+ + ); + })} +
+ ); + }, [dataset, type]); return (
@@ -105,62 +376,14 @@ export function ChartGeneratorResult() {
-
Chart Generation Result:
-
-
-
TEXT CONTENT
-
DATA TABLE && COMMAND
-
CHART RESULT
-
-
-
-
- {filterRes.map((v, index: number) => ( - - - {index + 1} - - {v.context.text} - - ))} -
-
-
-
- {filterRes.map((dataResult, index) => { - return ( - console.log('Context is :', dataResult)} - > -

Command:{` ${dataResult.command}`}

- {renderTable(dataResult.context)} - - ); - })} -
-
-
-
- {filterRes.map((dataResult, index) => { - return ( - console.log('Context is :', dataResult)} - > -
- -
- - ); - })} -
-
-
-
+ + + {renderCompareResult} + + + {renderSummaryResult} + +
); } diff --git a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/test.tsx b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/test.tsx index 5601a1d6..68e5b519 100644 --- a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/test.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/test.tsx @@ -1,75 +1,112 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable no-console */ import React from 'react'; -import VMind, { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; +import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; import { Button, Checkbox, Divider, Message, Select } from '@arco-design/web-react'; -import { - getCurrentFormattedTime, - getDataExtractionCaseData, - sleep, - transferFieldInfoInSimpleFieldInfo -} from '../../utils'; +import { getCurrentFormattedTime, getDataExtractionCaseData, sleep } from '../../utils'; import type { DataExtractionDataSetResult } from '../DataExtraction/type'; const globalVariables = (import.meta as any).env; const ModelConfigMap: any = { [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.DOUBAO_PRO_32K]: { url: globalVariables.VITE_VMIND_URL, key: globalVariables.VITE_VMIND_KEY }, [Model.DOUBAO_PRO]: { url: globalVariables.VITE_DOUBAO_URL, key: globalVariables.VITE_DOUBAO_KEY } }; -const datasetList = ['capcut_cn', 'capcut_en', 'common']; +const datasetList = ['capcut_cn', 'capcut_en', 'common', 'capcut_v2']; const dataExtractionResult = getDataExtractionCaseData(); export function ChartGenerationTask() { - const [selectedDataset, setSelectedDataset] = React.useState(datasetList); + const [selectedDataset, setSelectedDataset] = React.useState(['capcut_v2']); const [selectedLLM, setSelectedLLm] = React.useState>( Object.keys(ModelConfigMap).reduce((prev, cur) => ({ ...prev, [cur]: true }), {}) ); const [messageApi, contextHolder] = Message.useMessage(); + const [extractionType, setExtractionType] = React.useState<'multiple' | 'normal'>('multiple'); + const getDataSetResult = React.useCallback( async ( typeResult: DataExtractionDataSetResult[], type: 'default' | 'fieldInfo', - vmind: VMind, - schedule: Schedule<[AtomName.DATA_CLEAN, AtomName.CHART_COMMAND]>, + commandSchedule: Schedule<[AtomName.MULTIPLE_CHART_COMMAND]>, + chartSchedule: Schedule<[AtomName.CHART_GENERATE]>, baseOptions: { dataset: string; model: Model; } ) => { const result: any[] = []; + const sleepTime = baseOptions.model.includes('doubao') ? 10000 : 5000; for (let i = 0; i < typeResult.length; i++) { - const { dataTable, fieldInfo, text } = typeResult[i].context; - schedule.setNewTask({ - dataTable, - fieldInfo, - text: text - }); - const newCtx = await schedule.run(); - const chartGenerationResult = newCtx.command - ? await vmind.generateChart( - newCtx.command, - transferFieldInfoInSimpleFieldInfo(newCtx.fieldInfo!), - newCtx.dataTable, - { - theme: 'light' - } - ) - : { - spec: null - }; + const cleanCtx = typeResult[i].dataClean ?? typeResult[i].context; + const chartRes = []; + const time1: any = new Date(); + let cnt = 0; + if (extractionType === 'multiple') { + commandSchedule.setNewTask({ + datasets: cleanCtx.datasets + }); + const commandCtx = await commandSchedule.run(); + const { commands = [] } = commandCtx; + for (let j = 0; j < commands.length; j++) { + const { dataTable, fieldInfo, textRange } = commandCtx.datasets[j]; + chartSchedule.setNewTask({ + dataTable, + fieldInfo, + command: commands[j] + }); + const chartCtx = await chartSchedule.run(); + chartRes.push({ + context: { + spec: chartCtx.spec, + summary: chartCtx.summary, + command: chartCtx.command, + cell: chartCtx.cell + }, + textRange + }); + cnt++; + await sleep(sleepTime); + } + } else { + const { fieldInfo, dataTable } = cleanCtx; + commandSchedule.setNewTask({ + datasets: [cleanCtx] + }); + const commandCtx = await commandSchedule.run(); + chartSchedule.setNewTask({ + dataTable, + fieldInfo, + command: commandCtx.commands[0] + }); + const chartCtx = await chartSchedule.run(); + chartRes.push({ + context: { + spec: chartCtx.spec, + summary: chartCtx.summary, + command: chartCtx.command, + cell: chartCtx.cell + } + }); + cnt++; + await sleep(sleepTime); + } + const time2: any = new Date(); + const timeCost = (time2 - time1 - cnt * sleepTime) / 1000; + const extractionCost = Number(typeResult[i].timeCost); result.push({ ...baseOptions, type, - context: typeResult[i].context, - command: newCtx.command, - spec: chartGenerationResult.spec + context: cleanCtx, + chartRes, + extractionCost, + generationCost: Number(timeCost.toFixed(1)), + timeCost: Number((timeCost + extractionCost).toFixed(1)) }); - await sleep(baseOptions.model.includes('doubao') ? 10000 : 5000); console.info(`Current index:${i} Result: `, result); } return result; }, - [] + [extractionType] ); const handleRun = React.useCallback(async () => { (messageApi as any).info('Run Chart Generator Task!'); @@ -80,7 +117,7 @@ export function ChartGenerationTask() { for (let llmIndex = 0; llmIndex < llmKeys.length; llmIndex++) { const model = llmKeys[llmIndex]; if (!selectedLLM[model]) { - return; + continue; } const apiKey = ModelConfigMap[model]?.key; const llm = new LLMManage({ @@ -92,17 +129,11 @@ export function ChartGenerationTask() { maxTokens: 2048, model }); - const schedule = new Schedule([AtomName.DATA_CLEAN, AtomName.CHART_COMMAND], { - base: { llm, showThoughts: false } + const commandSchedule = new Schedule([AtomName.MULTIPLE_CHART_COMMAND], { + base: { llm } }); - const vmind: VMind = new VMind({ - model, - cache: false, - url: ModelConfigMap[model]?.url, - headers: { - 'api-key': apiKey, - Authorization: `Bearer ${apiKey}` - } + const chartSchedule = new Schedule([AtomName.CHART_GENERATE], { + base: { llm } }); console.info('Begin Model: ', model); const currentModelDataExtractionRes = dataExtractionResult.find(v => v.llm === model)?.result || []; @@ -113,7 +144,9 @@ export function ChartGenerationTask() { continue; } console.info('Begin Dataset: ', dataset); - result.push(...(await getDataSetResult(defaultResult, 'default', vmind, schedule, { dataset, model }))); + result.push( + ...(await getDataSetResult(defaultResult, 'default', commandSchedule, chartSchedule, { dataset, model })) + ); // result.push(...(await getDataSetResult(fieldInfoResult, 'fieldInfo', vmind, { dataset, model }))); console.info('Current Result: ', result); } @@ -144,7 +177,7 @@ export function ChartGenerationTask() { // 释放 URL 对象 URL.revokeObjectURL(url); - }, [getDataSetResult, messageApi, selectedDataset]); + }, [getDataSetResult, messageApi, selectedDataset, selectedLLM]); return (
diff --git a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/type.tsx b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/type.tsx index 48d24a9f..cd92ba62 100644 --- a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/type.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/type.tsx @@ -7,10 +7,23 @@ export interface ChartGeneratorCase { context: { dataTable: DataTable; fieldInfo: FieldInfo[]; + datasets: any[]; text: string; }; - command: string; - spec: any; + command?: string; + spec?: any; + chartRes?: { + context: { + dataTable: DataTable; + fieldInfo: FieldInfo[]; + spec: any; + command: string; + }; + textRange: [string, string]; + }[]; + timeCost: number; + extractionCost: number; + generationCost: number; } export type ChartGeneratorResult = ChartGeneratorCase[]; diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx index 1f414b25..94fc2bb9 100644 --- a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/caseStudy.tsx @@ -4,6 +4,7 @@ import React from 'react'; import type { FieldInfo } from '../../../../../src/index'; import { capcutCnData } from '../../data/capcutDataCn'; import { capcutEnData } from '../../data/capcutDataEn'; +import { capcutMockV2Data } from '../../data/capcutV2Data'; import { dataExtractionCommonDataset } from '../../data/dataExtractionData'; import type { TableColumnProps } from '@arco-design/web-react'; import { Avatar, Card, Divider, Select, Table, Tooltip } from '@arco-design/web-react'; @@ -15,23 +16,12 @@ import { isArray, isObject } from '@visactor/vutils'; import { sum } from '@visactor/vchart/esm/util'; import { getDataExtractionCaseData } from '../../utils'; -const result = getDataExtractionCaseData(); -console.info(result); - -const llmList = result.map((v, index) => ({ - index, - llm: v.llm -})); const datasetMap: Record = { common: dataExtractionCommonDataset, capcut_cn: capcutCnData, - capcut_en: capcutEnData + capcut_en: capcutEnData, + capcut_v2: capcutMockV2Data }; -const datasetList = result[0].result.map((v, index) => ({ - index, - name: v.dataset, - dataset: datasetMap[v.dataset] -})); interface Options { index: number; @@ -41,10 +31,31 @@ interface Options { const targetScore = 0.75; export function DataExtractionResult() { + const result = React.useMemo(() => getDataExtractionCaseData(), []); + const llmList = React.useMemo( + () => + result.map((v, index) => ({ + index, + llm: v.llm + })), + [result] + ); + const datasetList = React.useMemo( + () => + result[0].result.map((v, index) => ({ + index, + name: v.dataset, + dataset: datasetMap[v.dataset] + })), + [result] + ); + const [datasetIndex, setDatasetIndex] = React.useState(0); const currentDataset = datasetList[datasetIndex]; const currentDatasetName = currentDataset.name; const [language, setLanguage] = React.useState<'all' | 'zh' | 'en'>('all'); + const [contextType, setContextType] = React.useState<'clean' | 'extraction'>('extraction'); + const [showDetail, setShowDetail] = React.useState(true); const [leftOptions, setLeftOptions] = React.useState({ index: datasetIndex, llm: llmList[0].llm, @@ -57,17 +68,17 @@ export function DataExtractionResult() { }); const leftResult = React.useMemo( () => result[leftOptions.index].result.find(v => v.dataset === currentDatasetName)?.[leftOptions.resType], - [leftOptions.index, leftOptions.resType, currentDatasetName] + [result, leftOptions.index, leftOptions.resType, currentDatasetName] ); const rightResult = React.useMemo( () => rightOptions?.llm ? result[rightOptions.index].result.find(v => v.dataset === currentDatasetName)?.[rightOptions.resType] : null, - [rightOptions?.llm, rightOptions.index, rightOptions.resType, currentDatasetName] + [rightOptions?.llm, rightOptions.index, rightOptions.resType, result, currentDatasetName] ); const renderTable = React.useCallback((context: any) => { - const { dataTable, fieldInfo } = context; + const { dataTable = [], fieldInfo = [] } = context || {}; const columns: TableColumnProps[] = fieldInfo.map((info: FieldInfo) => { return { title: ( @@ -102,37 +113,53 @@ export function DataExtractionResult() { ); }, []); - const changeOptions = React.useCallback((v: any, type: 'llm' | 'resType' | 'language', index: number) => { - const newOptions = - type === 'llm' - ? { - index: v, - llm: llmList?.[v]?.llm - } - : { - [type]: v - }; - if (index === 0) { - setLeftOptions(prev => ({ - ...prev, - ...newOptions - })); - } else { - setRightOptions(prev => ({ - ...prev, - ...newOptions - })); - } - }, []); + const renderResult = React.useCallback( + (result: any) => { + const context = contextType === 'clean' ? result?.dataClean ?? result?.context : result?.context; + if (!context?.datasets?.length) { + return renderTable(context); + } + return context.datasets.map((data: DataExtractionDataSetResult, index: number) => { + return renderTable(data); + }); + }, + [contextType, renderTable] + ); + const changeOptions = React.useCallback( + (v: any, type: 'llm' | 'resType' | 'language', index: number) => { + const newOptions = + type === 'llm' + ? { + index: v, + llm: llmList?.[v]?.llm + } + : { + [type]: v + }; + if (index === 0) { + setLeftOptions(prev => ({ + ...prev, + ...newOptions + })); + } else { + setRightOptions(prev => ({ + ...prev, + ...newOptions + })); + } + }, + [llmList] + ); const answerResult = React.useMemo(() => { const answer = result.find(v => v.llm === 'answer'); return (answer?.result || []).find(v => v.dataset === currentDataset.name); - }, [currentDataset.name]); + }, [currentDataset.name, result]); const getCardDisplayStyle = React.useCallback( (data: DataExtractionDataSetResult) => { - const { context, score } = data; + const { context } = data; + const score = contextType === 'clean' ? data?.dataClean?.score ?? data?.score : data?.score; const isEn = context?.isEnglish ?? getLanguageOfText(context.text) === 'english'; const displayStyle = language === 'all' || isEn === (language === 'en') ? {} : { display: 'none ' }; const borderColor = @@ -146,7 +173,7 @@ export function DataExtractionResult() { ...borderColor }; }, - [answerResult, language] + [answerResult, contextType, language] ); const renderTimeCost = React.useCallback((res: DataExtractionDataSetResult[] | undefined | null) => { @@ -160,7 +187,8 @@ export function DataExtractionResult() { const renderScore = React.useCallback( (res: DataExtractionDataSetResult[] | undefined | null) => { if (answerResult && res) { - const validRes = res.filter(v => !!v.score || v.score === 0); + const resByType = contextType === 'clean' ? res.map(v => v.dataClean ?? v) : res; + const validRes = resByType.filter(v => !!v.score || v.score === 0); const allCount = validRes.length; const reachedCount = validRes.filter(v => v.score! >= targetScore).length; let score = 0; @@ -225,23 +253,24 @@ export function DataExtractionResult() { } return

No answer to get score

; }, - [answerResult] + [answerResult, contextType] ); const renderCaseScore = React.useCallback( (data: DataExtractionDataSetResult) => { - if (answerResult && data?.score !== undefined) { + const resByType = contextType === 'clean' ? data?.dataClean ?? data : data; + if (answerResult && resByType?.score !== undefined) { return (
- Score: {data.score.toFixed(2)} - FieldScore: {data.fieldScore?.toFixed(2)} - DataScore: {data.dataScore?.toFixed(2)} + Score: {resByType.score.toFixed(2)} + FieldScore: {resByType.fieldScore?.toFixed(2)} + DataScore: {resByType.dataScore?.toFixed(2)}
); } return null; }, - [answerResult] + [answerResult, contextType] ); const count = rightResult ? 2 : 1; const width = `${(0.8 / count) * 100}%`; @@ -256,6 +285,11 @@ export function DataExtractionResult() { ))} +

Please select result Type to show:

+
Comparision Result:
@@ -348,7 +382,7 @@ export function DataExtractionResult() { onClick={() => console.log('Context is :', dataResult.context)} > {renderCaseScore(dataResult)} - {renderTable(dataResult.context)} + {renderResult(dataResult)} ); })} diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx index 258848a8..0ca0515e 100644 --- a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/test.tsx @@ -4,15 +4,18 @@ import React from 'react'; import { AtomName, LLMManage, Model, Schedule } from '../../../../../src/index'; import { capcutCnData } from '../../data/capcutDataCn'; import { capcutEnData } from '../../data/capcutDataEn'; +import { capcutMockV2Data } from '../../data/capcutV2Data'; import { Button, Checkbox, Divider, Message, Select } from '@arco-design/web-react'; import { dataExtractionCommonDataset } from '../../data/dataExtractionData'; import { pick } from '@visactor/vutils'; import { getCurrentFormattedTime, sleep } from '../../utils'; +const Option = Select.Option; + const globalVariables = (import.meta as any).env; const ModelConfigMap: any = { [Model.DOUBAO_PRO]: { url: globalVariables.VITE_DOUBAO_URL, key: globalVariables.VITE_DOUBAO_KEY }, - [Model.DOUBAO_PRO_32K]: { url: globalVariables.VITE_DOUBAO_URL, key: globalVariables.VITE_DOUBAO_KEY }, + [Model.DOUBAO_PRO_32K]: { url: globalVariables.VITE_VMIND_URL, key: globalVariables.VITE_VMIND_KEY }, [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } }; const datasets = [ @@ -24,6 +27,10 @@ const datasets = [ name: 'capcut_en', data: capcutEnData }, + { + name: 'capcut_v2', + data: capcutMockV2Data + }, { name: 'common', data: dataExtractionCommonDataset @@ -38,6 +45,8 @@ export function DataExtractionTask() { const [useDefault, setUseDefault] = React.useState(true); const [useFieldInfo, setUseFieldInfo] = React.useState(false); const [messageApi, contextHolder] = Message.useMessage(); + const [type, setType] = React.useState<'multiple' | 'normal'>('multiple'); + const handleRun = React.useCallback(async () => { (messageApi as any).info('Run Data Extraction Task!'); console.info('---------Run Data Extraction Task!---------'); @@ -60,12 +69,17 @@ export function DataExtractionTask() { maxTokens: 2048, model }); - const schedule = new Schedule([AtomName.DATA_EXTRACT], { - base: { llm, showThoughts: false }, - dataExtract: { - reGenerateFieldInfo: true - } - }); + const schedule = + type === 'normal' + ? new Schedule([AtomName.DATA_EXTRACT, AtomName.DATA_CLEAN], { + base: { llm, showThoughts: false }, + dataExtract: { reGenerateFieldInfo: true } + }) + : new Schedule([AtomName.DATA_EXTRACT, AtomName.MULTIPLE_DATA_CLEAN], { + base: { llm, showThoughts: false }, + dataExtract: { isCapcut: true } + }); + (messageApi as any).info(`Begin ${model}!`); console.info(`---------Begin ${model}---------`); @@ -87,10 +101,12 @@ export function DataExtractionTask() { text: data.text }); const time1: any = new Date(); - const result = await schedule.run(); + const res = await schedule.run(); + const dataExtractionRes = schedule.getContext(AtomName.DATA_EXTRACT); const time2: any = new Date(); defaultResult.push({ - context: result, + context: dataExtractionRes, + dataClean: res, timeCost: ((time2 - time1) / 1000).toFixed(1) }); await sleep(sleepTime); @@ -101,10 +117,12 @@ export function DataExtractionTask() { fieldInfo: data.fieldInfo.map((v: any) => pick(v, ['fieldName'])) }); const time1: any = new Date(); - const result = await schedule.run(); + const res = await schedule.run(); + const dataExtractionRes = schedule.getContext(AtomName.DATA_EXTRACT); const time2: any = new Date(); fieldInfoResult.push({ - context: result, + context: dataExtractionRes, + dataClean: res, timeCost: ((time2 - time1) / 1000).toFixed(1) }); await sleep(sleepTime); @@ -151,7 +169,7 @@ export function DataExtractionTask() { // 释放 URL 对象 URL.revokeObjectURL(url); - }, [messageApi, selectedDataset, selectedLLM, useDefault, useFieldInfo]); + }, [messageApi, selectedDataset, selectedLLM, type, useDefault, useFieldInfo]); return (
@@ -172,6 +190,19 @@ export function DataExtractionTask() { ))}
+
+

Select DataExtraction Type

+ +

LLM Model To Select

{Object.keys(ModelConfigMap).map(modelName => { diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx index 056c1658..66f06846 100644 --- a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/type.tsx @@ -14,6 +14,17 @@ export interface DataExtractionDataSetResult { dataTable: DataTable; fieldInfo: FieldInfo[]; text: string; + datasets?: any[]; + }; + dataClean?: { + dataTable: DataTable; + fieldInfo: FieldInfo[]; + text: string; + datasets?: any[]; + score?: number; + fieldScore?: number; + dataScore?: number; + scoreDetail?: ScoreDetail[]; }; timeCost?: string; score?: number; diff --git a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx index 8e9291dc..5ecd52f0 100644 --- a/packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/DataExtraction/verify.tsx @@ -31,13 +31,18 @@ const getFieldScore = (nameScore: number, typeScore: number) => nameScore * 0.75 const getFinalScore = (fieldScore: number, dataScore: number) => fieldScore * 0.3 + dataScore * 0.7; -const getDataScore = (currentDataList: DataCell[], answerDataList: DataCell[], role: ROLE) => { +const getDataScore = ( + currentDataList: DataCell[], + answerDataList: DataCell[], + fieldInfo: FieldInfo, + isStrict = true +) => { let dataScore = 0; const seletedList = new Array(answerDataList.length).fill(false); for (let i = 0; i < currentDataList.length; i++) { let maxScore = 0; let currentSelectedIndex = -1; - const valueCompareFunction = getValueCompareFunction(role); + const valueCompareFunction = getValueCompareFunction(fieldInfo); for (let j = 0; j < answerDataList.length; j++) { if (!seletedList[j]) { const currentScore = valueCompareFunction(currentDataList[i], answerDataList[j]); @@ -52,7 +57,7 @@ const getDataScore = (currentDataList: DataCell[], answerDataList: DataCell[], r seletedList[currentSelectedIndex] = true; } } - if (answerDataList.length > currentDataList.length) { + if (answerDataList.length > currentDataList.length && isStrict) { dataScore /= answerDataList.length; } else { dataScore /= seletedList.filter(v => !!v).length; @@ -65,7 +70,8 @@ function getScoreOfDataset( fieldInfoA: FieldInfo[], fieldInfoB: FieldInfo[], dataTableA: DataTable, - dataTableB: DataTable + dataTableB: DataTable, + isStrict = true ) { const seletedList = fieldInfoB.map(v => false); const scoreList: ScoreDetail[] = []; @@ -81,8 +87,7 @@ function getScoreOfDataset( const dataListB = dataTableB?.map(v => v[fieldInfoB[i].fieldName]) || []; const cosValue = cosineSimilarity(fieldInfo.fieldName, fieldInfoB[i].fieldName); const currentTypeScore = getFieldTypeScore(fieldInfo.type, fieldInfoB[i].type); - const role = fieldInfo.role ?? getRoleByFieldType(fieldInfo.type); - const currentDataScore = getDataScore(dataListB, dataListA, role); + const currentDataScore = getDataScore(dataListB, dataListA, fieldInfo, isStrict); const currentScore = getFinalScore(getFieldScore(cosValue, currentTypeScore), currentDataScore); if (currentScore > finalScore && currentTypeScore !== -1 && currentDataScore > 0) { index = i; @@ -105,13 +110,18 @@ function getScoreOfDataset( return scoreList; } -function getValueCompareFunction(role: ROLE) { +function getValueCompareFunction(fieldInfo: FieldInfo) { + const { role, type, ratioGranularity } = fieldInfo; if (role === ROLE.MEASURE) { return (a: any, b: any) => { if (+a === +b || a === b) { return 1; } - return isNumber(a) && isNumber(b) && Math.abs(a) === Math.abs(b) ? 0.2 : 0; + if (type === DataType.RATIO) { + const rate = ratioGranularity === '‰' ? 1000 : 100; + return (isNumber(a) && isNumber(b) && a * rate === b) || b * rate === a ? 1 : 0; + } + return isNumber(a) && isNumber(b) && Math.abs(a) === Math.abs(b) ? 0.5 : 0; }; } return (a: DataCell, b: DataCell) => { @@ -119,12 +129,18 @@ function getValueCompareFunction(role: ROLE) { }; } -export const getScoreOfDataExtraction = (resultCtx: DataExtractionCtx, answerCtx: DataExtractionCtx) => { +export const getScoreOfDataExtraction = ( + resultCtx: DataExtractionCtx, + answerCtx: DataExtractionCtx, + isStrict = true +) => { const { fieldInfo = [], dataTable } = resultCtx; const { fieldInfo: answerInfo = [], dataTable: answerTable } = answerCtx; - const scoreList = getScoreOfDataset(answerInfo || [], fieldInfo || [], answerTable!, dataTable!).filter( - v => v.matchedIndex !== -1 - ); + const scoreList = ( + isStrict + ? getScoreOfDataset(answerInfo || [], fieldInfo || [], answerTable!, dataTable!) + : getScoreOfDataset(fieldInfo || [], answerInfo || [], dataTable!, answerTable!, false) + ).filter(v => v.matchedIndex !== -1); if (!scoreList.length) { return { score: 0, @@ -140,7 +156,7 @@ export const getScoreOfDataExtraction = (resultCtx: DataExtractionCtx, answerCtx dataScore += v.dataScore; }); dataScore /= scoreList.length; - if (answerInfo?.length > fieldInfo?.length) { + if (answerInfo?.length > fieldInfo?.length && isStrict) { fieldScore /= answerInfo?.length; } else { fieldScore /= scoreList.length; @@ -153,6 +169,26 @@ export const getScoreOfDataExtraction = (resultCtx: DataExtractionCtx, answerCtx }; }; +const getValidCellNumber = (ctx: any) => { + const { dataTable = [], fieldInfo = [] } = ctx; + return dataTable.length * fieldInfo.length; +}; + +const getCtx = (ctx: any) => { + if (ctx.datasets?.length) { + let res = ctx.datasets[0]; + let preCount = getValidCellNumber(res); + for (let i = 1; i < ctx.datasets.length; i++) { + const curCount = getValidCellNumber(ctx.datasets[i]); + if (curCount > preCount) { + preCount = curCount; + res = ctx.datasets[i]; + } + } + return res; + } + return ctx; +}; export const updateScoreInDataExtraction = (result: DataExtractionResult, answer: DataExtractionCase) => { result.forEach(llmResult => { llmResult.result.forEach(caseResult => { @@ -161,13 +197,23 @@ export const updateScoreInDataExtraction = (result: DataExtractionResult, answer if (caseResult.defaultResult.length) { caseResult.defaultResult = caseResult.defaultResult.map((v, index) => ({ ...v, - ...getScoreOfDataExtraction(v.context, answerCase.defaultResult[index].context) + ...getScoreOfDataExtraction(getCtx(v.context), getCtx(answerCase.defaultResult[index].context)), + dataClean: v?.dataClean + ? { + ...v.dataClean, + ...getScoreOfDataExtraction( + getCtx(v.dataClean), + getCtx(answerCase.defaultResult[index].context), + false + ) + } + : undefined })); } if (caseResult.fieldInfoResult.length) { caseResult.fieldInfoResult = caseResult.fieldInfoResult.map((v, index) => ({ ...v, - ...getScoreOfDataExtraction(v.context, answerCase.defaultResult[index].context) + ...getScoreOfDataExtraction(getCtx(v.context), getCtx(answerCase.defaultResult[index].context)) })); } } diff --git a/packages/vmind/__tests__/experiment/src/pages/page.scss b/packages/vmind/__tests__/experiment/src/pages/page.scss index 024a3cef..f4801bad 100644 --- a/packages/vmind/__tests__/experiment/src/pages/page.scss +++ b/packages/vmind/__tests__/experiment/src/pages/page.scss @@ -74,4 +74,9 @@ margin: 0 12px; } } + + .arco-tabs-content { + overflow: auto; + height: 100%; + } } diff --git a/packages/vmind/__tests__/experiment/src/utils.ts b/packages/vmind/__tests__/experiment/src/utils.ts index bbea94d6..bb3b48ab 100644 --- a/packages/vmind/__tests__/experiment/src/utils.ts +++ b/packages/vmind/__tests__/experiment/src/utils.ts @@ -1,12 +1,14 @@ -import { result as capcutResult } from './results/dataExtraction/result7'; -import { result as caseResult } from './results/dataExtraction/commonResult'; +// import { result as capcutResult } from './results/dataExtraction/result7'; +import { result as capcutResult, caseResult, capcutV2Result } from './results/dataExtraction/version1'; +// import { result as caseResult } from './results/dataExtraction/commonResult'; import { result as doubaoResult } from './results/dataExtraction/doubao1'; import { commonAnswer } from './data/dataExtractionData'; import { mergeResult, updateScoreInDataExtraction } from './pages/DataExtraction/verify'; -import type { FieldInfo } from '../../../src'; +import { AtomName, Schedule, type FieldInfo } from '../../../src'; import { DataType } from '../../../src/common/typings'; import type { SimpleFieldInfo } from '../../../src/common/typings'; import type { DataExtractionResult } from './pages/DataExtraction/type'; +import { DataCleanAtom, MultipleDataCleanAtom } from '../../../src/atom'; export function getCurrentFormattedTime() { const now = new Date(); @@ -24,9 +26,49 @@ export function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } -export function getDataExtractionCaseData(): DataExtractionResult { +function revisedTestData(data: DataExtractionResult, reGenerateDataClean = false) { + if (!reGenerateDataClean) { + return data as any; + } + const dataClean = new MultipleDataCleanAtom({} as any, {}); + // for (let i = 0; i < data.length; i++) { + // for (let j = 0; j < data[i].result.length; j++) { + // const caseResult = data[i].result[j]; + // for (let l = 0; l < caseResult.defaultResult.length; l++) { + // const defaultRes = caseResult.defaultResult[l]; + // schedule.setNewTask(defaultRes.context); + // caseResult.defaultResult[l].dataClean = (await schedule.run()) as any; + // } + // } + // } + return data.map(llmResult => { + return { + llm: llmResult.llm, + result: llmResult.result.map(caseResult => { + return { + dataset: caseResult.dataset, + fieldInfoResult: caseResult.fieldInfoResult, + defaultResult: caseResult.defaultResult.map(v => { + dataClean.reset(v.context as any); + return { + ...v, + dataClean: dataClean._runWithOutLLM() + }; + }) + }; + }) + }; + }) as any; +} +export function getDataExtractionCaseData(reGenerateDataClean = false): DataExtractionResult { // const result = [...doubaoResult, commonAnswer]; - const result = [...mergeResult(capcutResult as any, caseResult as any), commonAnswer]; + const result = [ + ...mergeResult( + revisedTestData(capcutResult as any, reGenerateDataClean), + revisedTestData(capcutV2Result as any, reGenerateDataClean) + ), + commonAnswer + ]; updateScoreInDataExtraction(result as any, commonAnswer); return result as any; } diff --git a/packages/vmind/package.json b/packages/vmind/package.json index 42f69804..4db994d0 100644 --- a/packages/vmind/package.json +++ b/packages/vmind/package.json @@ -89,7 +89,8 @@ "@types/density-clustering": "~1.3.3", "@types/euclidean-distance": "~1.0.3", "string-similarity": "~4.0.4", - "@types/string-similarity": "~4.0.2" + "@types/string-similarity": "~4.0.2", + "express": "~4.21.1" }, "dependencies": { "@visactor/chart-advisor": "workspace:1.2.13", diff --git a/packages/vmind/src/atom/base.ts b/packages/vmind/src/atom/base.ts index a1cfe6d6..e7203549 100644 --- a/packages/vmind/src/atom/base.ts +++ b/packages/vmind/src/atom/base.ts @@ -81,7 +81,7 @@ export class BaseAtom { } reset(context?: Ctx) { - this.updateContext(context); + this.context = this.buildDefaultContext(context); this.responses = []; this.history.map.clear(); this.history.idList = []; @@ -110,27 +110,38 @@ export class BaseAtom { */ async run(userInput?: { context?: Ctx; query?: string }) { const { context, query } = userInput || {}; + this.context.error = null; this.updateContext(context); this.originContext = this.context; - if (this.isLLMAtom && query) { - return await this.runWithChat(query); - } - if (this.isLLMAtom) { - const messages = this.getLLMMessages(); - const data = await this.options.llm.run(this.name, messages); - const resJson = this.options.llm.parseJson(data); - if (resJson.error) { - return this.context; + try { + this.runBeforeLLM(); + if (this.isLLMAtom && query) { + return await this.runWithChat(query); } - this.recordLLMResponse(data); - this.setNewContext(this.parseLLMContent(resJson, data)); - this._runWithOutLLM(); - } else { - this._runWithOutLLM(); + if (this.isLLMAtom) { + const messages = this.getLLMMessages(); + const data = await this.options.llm.run(this.name, messages); + const resJson = this.options.llm.parseJson(data); + if (resJson.error) { + this.updateContext({ error: resJson.error } as any); + return this.context; + } + this.recordLLMResponse(data); + this.setNewContext(this.parseLLMContent(resJson, data)); + this._runWithOutLLM(); + } else { + this._runWithOutLLM(); + } + } catch (error) { + this.context.error = error as string; } return this.context; } + protected runBeforeLLM() { + return this.context; + } + /** * after run function, user can adjust context result by multi-turn dialogue * @param query user's new query diff --git a/packages/vmind/src/atom/chartCommand/index.ts b/packages/vmind/src/atom/chartCommand/index.ts index 6736c1dc..2a86684f 100644 --- a/packages/vmind/src/atom/chartCommand/index.ts +++ b/packages/vmind/src/atom/chartCommand/index.ts @@ -38,8 +38,8 @@ export class ChartCommandAtom extends BaseAtom ({ - fieldName: info.fieldName, - type: info.role || getRoleByFieldType(info.type), - dataLength: dataTable?.filter(v => isValidData(v[info.fieldName]))?.length || undefined - })), - dataTable: this.options?.useDataTable ? JSON.stringify(dataTable) : undefined + userInput: [ + { + text, + summary, + fieldInfo: fieldInfo.map(info => ({ + fieldName: info.fieldName, + type: info.role || getRoleByFieldType(info.type), + dataLength: dataTable?.filter(v => isValidData(v[info.fieldName]))?.length || undefined + })), + dataTable: this.options?.useDataTable ? JSON.stringify(dataTable) : undefined + } + ] }) }, ...addtionContent @@ -63,8 +68,9 @@ export class ChartCommandAtom extends BaseAtom { + name = AtomName.MULTIPLE_CHART_COMMAND; + + isLLMAtom = true; + + ruleList: boolean[]; + + constructor(context: MultipleChartCommandsCtx, option: BaseOptions) { + super(context, option); + } + + buildDefaultOptions(): ChartCommandOptions { + return { + useDataTable: false, + filterByRule: true + }; + } + + buildDefaultContext(context: MultipleChartCommandsCtx): MultipleChartCommandsCtx { + return merge( + {}, + { + commands: [] + }, + context + ); + } + + shouldRunByContextUpdate(context: MultipleChartCommandsCtx): boolean { + return context.datasets !== this.context.datasets; + } + + protected runBeforeLLM(): MultipleChartCommandsCtx { + this.ruleList = this.context.datasets.map(dataset => { + return this.options?.filterByRule ? dataset?.dataTable?.length < 2 : false; + }); + return this.context; + } + + protected getLLMMessages(query?: string): LLMMessage[] { + const { datasets } = this.context; + const language = getLanguageOfText(datasets?.[0]?.text || datasets?.[0]?.summary); + const addtionContent = this.getHistoryLLMMessages(query); + return [ + { + role: 'system', + content: getChartCommandPrompt(language) + }, + { + role: 'user', + content: JSON.stringify({ + userInput: datasets + .filter((datasets, i) => !this.ruleList[i]) + .map(dataset => { + const { text, summary, fieldInfo, dataTable } = dataset; + return { + text, + summary, + fieldInfo: fieldInfo.map(info => ({ + fieldName: info.fieldName, + type: info.role || getRoleByFieldType(info.type), + dataLength: dataTable?.filter(v => isValidData(v[info.fieldName]))?.length || undefined + })), + dataTable: this.options?.useDataTable ? JSON.stringify(dataTable) : undefined + }; + }) + }) + }, + ...addtionContent + ]; + } + + protected parseLLMContent(resJson: any): MultipleChartCommandsCtx { + const { commands = [] } = resJson; + if (!isArray(commands) || commands.length === 0) { + console.error("Can't generate chart command in this case"); + return { + ...this.context, + commands: this.context.datasets.map(v => '') + }; + } + const res = []; + let commandIndex = 0; + for (let i = 0; i < this.context.datasets.length; i++) { + const isRule = this.ruleList[i]; + if (isRule) { + res.push(''); + } else { + res.push(commands[commandIndex++]); + } + } + return { + ...this.context, + commands: res + }; + } +} diff --git a/packages/vmind/src/atom/chartCommand/prompt.ts b/packages/vmind/src/atom/chartCommand/prompt.ts index 803e05e4..c5d1218d 100644 --- a/packages/vmind/src/atom/chartCommand/prompt.ts +++ b/packages/vmind/src/atom/chartCommand/prompt.ts @@ -1,6 +1,6 @@ /* eslint-disable max-len */ export const getChartCommandPrompt = (language: 'chinese' | 'english') => { - return `You are an expert in data visualization and data analysis.Your task is generate a visual description based on the text content and the field information of the existing data table. + return `As an expert in data visualization and analysis, your task is to generate a visual description based on the requirements. # User Input 1. The data table and field information are extracted from the existing text. 2. The data in the data table comes from the text, so some fields may be incomplete. @@ -8,39 +8,46 @@ export const getChartCommandPrompt = (language: 'chinese' | 'english') => { User Input is Bellow: \`\`\`typescript { +userInput: { fieldInfo: { fieldName: string; // name of this field, type?: 'measure' | 'dimension', dataLength?: number, // The number of valid data contained in the current field. }[], text?: string; +summary?: string; // summary of text and data table dataTable?: Record[]; // Specific data } +}[] \`\`\` # Response \`\`\` { - command: string | false; // visual description or false + commands: (string | false)[]; // visual description or false } \`\`\` +# Requirments +1. For each element in the user input array, generate a visual description. +2. Visual descriptions need to correspond one-to-one with user inputs. +3. The visual description information needs to be concise and clear. # Steps You should carefully think and execute the following steps: -0. Answer language MUST: ${language} -1. Find the MOST IMPORTANT measure field based on the user's input. -2. Ignore those fields where data is clearly missing or insufficient. +1. Traverse user's input, executing steps 2 to 8 for each element. +2. Find the MOST IMPORTANT measure field based on the user's input. +3. Ignore those fields where data is clearly missing or insufficient. 4. Try to use the fewest fields to display the most important information. 5. Generate a precise and concise visual description based on your rich experience. 6. The description needs to include the field name. 7. The final description does not need to use all fields, as some fields may be incomplete or unimportant. 8. If unable to generate, return command is false. -9. MUST return in JSON mode. +9. MUST return in JSON mode with ${language} language. # Examples1 ## User Input: \`\`\` -{"text":'今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 ',"fieldInfo:":[{"fieldName":"公司","type":"dimension",},{"fieldName":"月份",type":"dimension",},{"fieldName":"利润调整","type":"dimension",},{"fieldName":"利润额","type":"measure",}]} +{"userInput":[{"fieldInfo":[{"fieldName":"公司","type":"dimension","dataLength":2},{"fieldName":"调整前利润额","type":"measure","dataLength":2},{"fieldName":"调整后利润额","type":"measure","dataLength":2}],"summary":"各大厂商的财报数据","dataTable":[{"公司":"阿里","调整前利润额":1000,"调整后利润额":100},{"公司":"字节跳动","调整前利润额":800,"调整后利润额":120}]},{"summary":"入睡困难分布","fieldInfo":[{"fieldName":"性别","type":"dimension","dataLength":2},{"fieldName":"占比","type":"measure","dataLength":2}],"dataTable":[{"性别":"男","占比":60},{"性别":"女","占比":40}]}]} \`\`\` ## Response \`\`\` -{"command": '对比查看不同公司调整后的利润表现'} +{"commands": ["对比查看不同公司利润表现", "查看入睡困难的性别分布"]} \`\`\``; }; diff --git a/packages/vmind/src/atom/chartGenerator/advisor/index.ts b/packages/vmind/src/atom/chartGenerator/advisor/index.ts new file mode 100644 index 00000000..056c2fe4 --- /dev/null +++ b/packages/vmind/src/atom/chartGenerator/advisor/index.ts @@ -0,0 +1,97 @@ +import type { ChartType } from '@visactor/chart-advisor'; +import { chartAdvisor } from '@visactor/chart-advisor'; + +import type { GenerateChartCellContext } from '../type'; +import { isValidDataTable } from '../../../utils/dataTable'; +import { chartTypeMap, getCell, typeMap, VMindChartTypeMap } from '../utils'; +import type { VizSchema } from '../../type'; +import { ChartType as VMindChartType } from '../../../types'; +import type { DataTable } from '../../../types'; +/** + * call @visactor/chart-advisor to get the list of advised charts + * sorted by scores of each chart type + * @param schema + * @param dataset + * @returns + */ +const getAdvisedChartList = (schema: Partial, dataset: DataTable) => { + const dimensionList: any = schema.fields + .filter(d => d.role === 'dimension') + .map(d => ({ + uniqueId: d.id, + type: typeMap(d.type) + })); + const measureList: any = schema.fields + .filter(d => d.role === 'measure') + .map(d => ({ + uniqueId: d.id, + type: typeMap(d.type) + })); + const aliasMap = Object.fromEntries(schema.fields.map(d => [d.id, d.alias])); + const advisorResult = chartAdvisor({ originDataset: dataset as any, dimensionList, measureList, aliasMap }); + return advisorResult; +}; + +const getAdvisedListTransformer = (context: GenerateChartCellContext) => { + const { vizSchema, dataTable, chartTypeList } = context; + const chartSource = 'chartAdvisor'; + + if (!isValidDataTable(dataTable)) { + return { + advisedList: [], + chartSource, + usage: { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0 + } + }; + } + // call rule-based method to get recommended chart type and fieldMap(cell) + const { scores } = getAdvisedChartList(vizSchema, dataTable); + const availableChartTypeList = chartTypeList.reduce( + (res, chartType) => [...res, ...(VMindChartTypeMap?.[chartType] ?? [])], + [] + ); + const advisedList = scores + .filter((d: any) => availableChartTypeList.includes(d.chartType) && d.score - 0 >= 0.00000001) + .map((result: any) => ({ + chartType: chartTypeMap(result.chartType as unknown as ChartType).toUpperCase(), + cell: getCell(result.cell), + dataset: result.dataset, + score: result.score + })); + + return { + advisedList, + chartSource, + usage: { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0 + } + }; +}; + +export const getCellContextByAdvisor = (context: GenerateChartCellContext) => { + const advisorResult = getAdvisedListTransformer(context); + const { advisedList, chartSource, usage } = advisorResult; + // call rule-based method to get recommended chart type and fieldMap(cell) + if (advisedList.length === 0) { + return { + chartType: VMindChartType.BarChart.toUpperCase() as VMindChartType, + cell: {}, + dataset: undefined, + chartSource, + usage + }; + } + const result = advisedList[0]; + return { + chartType: result.chartType as VMindChartType, + cell: getCell(result.cell), + dataset: result.dataset, + chartSource, + usage + }; +}; diff --git a/packages/vmind/src/atom/chartGenerator/index.ts b/packages/vmind/src/atom/chartGenerator/index.ts index c841e962..da369e6e 100644 --- a/packages/vmind/src/atom/chartGenerator/index.ts +++ b/packages/vmind/src/atom/chartGenerator/index.ts @@ -1,25 +1,28 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { ChartGeneratorCtx } from '../../types/atom'; import { AtomName } from '../../types/atom'; -import type { BaseOptions } from '../type'; +import type { ChartGeneratorOptions } from '../type'; import { BaseAtom } from '../base'; import { merge } from '@visactor/vutils'; import type { LLMMessage } from '../../types/llm'; import { DEFAULT_MAP_OPTION, SUPPORTED_CHART_LIST } from './const'; import { getPrompt, revisedUserInput } from './prompt'; import { getContextAfterRevised } from './llmResultRevise'; -import { getVizSchema } from './vizSchema'; -import { checkChartTypeAndCell } from './utils'; +import { checkChartTypeAndCell, getVizSchema } from './utils'; import { getChartSpecWithContext } from './spec'; +import { getRuleLLMContent } from './spec/rule'; +import { getCellContextByAdvisor } from './advisor'; -export class ChartGeneratorAtom extends BaseAtom { +export class ChartGeneratorAtom extends BaseAtom { name = AtomName.CHART_GENERATE; isLLMAtom = true; + useRule = false; + useChartAdvisor: boolean; - constructor(context: ChartGeneratorCtx, option: BaseOptions) { + constructor(context: ChartGeneratorCtx, option: ChartGeneratorOptions) { super(context, option); } @@ -35,6 +38,12 @@ export class ChartGeneratorAtom extends BaseAtom context ); } + + buildDefaultOptions(): ChartGeneratorOptions { + return { + useChartAdvisor: false + }; + } updateContext(context: ChartGeneratorCtx) { this.context = super.updateContext(context); this.context.vizSchema = getVizSchema(this.context); @@ -73,15 +82,50 @@ export class ChartGeneratorAtom extends BaseAtom return newContext; } + protected runBeforeLLM(): ChartGeneratorCtx { + const { dataTable } = this.context; + this.useRule = false; + if (this.options.useChartAdvisor) { + this.isLLMAtom = false; + } + if (dataTable.length > 1) { + return this.context; + } + this.isLLMAtom = false; + this.useRule = true; + const ruleResJson = getRuleLLMContent(this.context); + if (ruleResJson) { + this.updateContext(this.parseLLMContent(ruleResJson)); + } else { + this.updateContext({ + cell: null + } as any); + } + return this.context; + } + protected _runWithOutLLM(): ChartGeneratorCtx { - if (this.useChartAdvisor) { + this.isLLMAtom = true; + if (this.useRule && !this.context.cell) { + /** use table */ + this.context.spec = null; + return this.context; + } + if (!this.useRule && (this.useChartAdvisor || this.options.useChartAdvisor)) { // @todo + const { cell, dataset, chartType } = getCellContextByAdvisor(this.context); + this.context = { + ...this.context, + cell, + dataTable: dataset, + chartType + }; } const newContext = { ...this.context, ...getChartSpecWithContext(this.context) }; this.updateContext(newContext); - return getChartSpecWithContext(this.context); + return newContext; } } diff --git a/packages/vmind/src/atom/chartGenerator/llmResultRevise.ts b/packages/vmind/src/atom/chartGenerator/llmResultRevise.ts index 20716026..3e33e316 100644 --- a/packages/vmind/src/atom/chartGenerator/llmResultRevise.ts +++ b/packages/vmind/src/atom/chartGenerator/llmResultRevise.ts @@ -54,8 +54,7 @@ export const patchChartType = (context: GenerateChartCellContext) => { if (!chartTypeList.includes(chartTypeNew)) { console.error('Unsupported Chart Type. Please Change User Input'); return { - error: true, - message: 'Unsupported Chart Type. Please Change User Input' + error: 'Unsupported Chart Type. Please Change User Input' }; } @@ -253,7 +252,7 @@ export const patchPieChart = (context: GenerateChartCellContext) => { if (colorField) { cellNew.color = colorField.fieldName; } else { - cellNew.color = remainedFields[0].fieldName; + cellNew.color = remainedFields?.[0].fieldName; } } if (!cellNew.angle) { @@ -262,7 +261,7 @@ export const patchPieChart = (context: GenerateChartCellContext) => { if (angleField) { cellNew.angle = angleField.fieldName; } else { - cellNew.angle = remainedFields[0].fieldName; + cellNew.angle = remainedFields?.[0].fieldName; } } } @@ -290,7 +289,7 @@ export const patchWordCloud = (context: GenerateChartCellContext) => { if (sizeField) { cellNew.size = sizeField.fieldName; } else { - cellNew.size = remainedFields[0].fieldName; + cellNew.size = remainedFields?.[0].fieldName; } } } @@ -303,7 +302,7 @@ export const patchWordCloud = (context: GenerateChartCellContext) => { if (colorField) { cellNew.color = colorField.fieldName; } else { - cellNew.color = remainedFields[0].fieldName; + cellNew.color = remainedFields?.[0].fieldName; } } } @@ -368,7 +367,7 @@ export const patchCartesianXField = (context: GenerateChartCellContext) => { if (xField) { cellNew.x = xField.fieldName; } else { - cellNew.x = remainedFields[0].fieldName; + cellNew.x = remainedFields?.[0].fieldName; } } } @@ -393,7 +392,7 @@ export const patchNeedColor = (context: GenerateChartCellContext) => { if (colorField) { cellNew.color = colorField.fieldName; } else { - cellNew.color = remainedFields[0].fieldName; + cellNew.color = remainedFields?.[0].fieldName; } } } @@ -418,7 +417,7 @@ export const patchNeedSize = (context: GenerateChartCellContext) => { if (sizeField) { cellNew.size = sizeField.fieldName; } else { - cellNew.size = remainedFields[0].fieldName; + cellNew.size = remainedFields?.[0].fieldName; } } } @@ -445,8 +444,7 @@ export const patchRangeColumnChart = (context: GenerateChartCellContext) => { 'The y-axis of the range column chart requires two numeric fields, but the result of data aggregation does not have two numeric fields'; console.error(message); return { - error: true, - message + error: message }; } } @@ -468,7 +466,7 @@ export const patchLinearProgressChart = (context: GenerateChartCellContext) => { if (xField) { cellNew.x = xField.fieldName; } else { - cellNew.x = remainedFields[0].fieldName; + cellNew.x = remainedFields?.[0].fieldName; } } @@ -481,7 +479,7 @@ export const patchLinearProgressChart = (context: GenerateChartCellContext) => { if (yField) { cellNew.y = yField.fieldName; } else { - cellNew.y = remainedFields[0].fieldName; + cellNew.y = remainedFields?.[0].fieldName; } } } @@ -505,7 +503,7 @@ export const patchBasicHeatMapChart = (context: GenerateChartCellContext) => { cellNew.x = colorField[0]; cellNew.y = colorField[1]; } else { - cellNew.x = remainedFields[0].fieldName; + cellNew.x = remainedFields?.[0].fieldName; cellNew.y = remainedFields[1].fieldName; } } diff --git a/packages/vmind/src/atom/chartGenerator/prompt/index.ts b/packages/vmind/src/atom/chartGenerator/prompt/index.ts index ba0faee4..dd2cc420 100644 --- a/packages/vmind/src/atom/chartGenerator/prompt/index.ts +++ b/packages/vmind/src/atom/chartGenerator/prompt/index.ts @@ -54,7 +54,8 @@ Here are some examples: ${examples} `; -export const getPrompt = (chartTypeList: ChartType[], showThoughts: boolean = true) => { +export const getPrompt = (propsChartList: ChartType[], showThoughts: boolean = true) => { + const chartTypeList = propsChartList.filter(v => !!chartKnowledgeDict[v]); const sortedChartTypeList = chartTypeList.sort((a, b) => chartKnowledgeDict[a].index - chartKnowledgeDict[b].index); const chartKnowledge = sortedChartTypeList.reduce( diff --git a/packages/vmind/src/atom/chartGenerator/spec/index.ts b/packages/vmind/src/atom/chartGenerator/spec/index.ts index 68917563..13e91aca 100644 --- a/packages/vmind/src/atom/chartGenerator/spec/index.ts +++ b/packages/vmind/src/atom/chartGenerator/spec/index.ts @@ -102,6 +102,7 @@ const pipelineLine = [ //animationCartisianLine, theme ]; +const pipeAreaLine = [...pipelineLine]; const pipelinePie = [ chartType, data, @@ -230,6 +231,7 @@ const pipelineVenn = [chartType, registerChart, vennData, color, vennField, lege const pipelineMap: { [chartType: string]: any } = { [ChartType.BarChart.toUpperCase()]: pipelineBar, [ChartType.LineChart.toUpperCase()]: pipelineLine, + [ChartType.AreaChart.toUpperCase()]: pipeAreaLine, [ChartType.PieChart.toUpperCase()]: pipelinePie, [ChartType.WordCloud.toUpperCase()]: pipelineWordCloud, [ChartType.ScatterPlot.toUpperCase()]: pipelineScatterPlot, diff --git a/packages/vmind/src/atom/chartGenerator/spec/rule.ts b/packages/vmind/src/atom/chartGenerator/spec/rule.ts new file mode 100644 index 00000000..05a214b1 --- /dev/null +++ b/packages/vmind/src/atom/chartGenerator/spec/rule.ts @@ -0,0 +1,19 @@ +import { ROLE, DataType, ChartType } from '../../../types'; +import type { Cell, ChartGeneratorCtx } from '../../../types'; + +export const getRuleLLMContent = (context: ChartGeneratorCtx) => { + const { fieldInfo } = context; + const measureFields = fieldInfo.filter(field => field.role === ROLE.MEASURE); + let chartType = null; + const cell: Cell = {}; + if (measureFields.length === 1 && measureFields[0].type === DataType.RATIO) { + // waterwave spec + chartType = ChartType.LiquidChart; + cell.value = measureFields[0].fieldName; + return { + CHART_TYPE: chartType, + FIELD_MAP: cell + }; + } + return null; +}; diff --git a/packages/vmind/src/atom/chartGenerator/spec/transformers.ts b/packages/vmind/src/atom/chartGenerator/spec/transformers.ts index 00d36fcb..f79762b3 100644 --- a/packages/vmind/src/atom/chartGenerator/spec/transformers.ts +++ b/packages/vmind/src/atom/chartGenerator/spec/transformers.ts @@ -21,7 +21,7 @@ import type { GenerateChartCellContext } from '../type'; import { getFieldByDataType } from '../../../utils/field'; import { isValidDataTable } from '../../../utils/dataTable'; import { DataType, ChartType, ROLE } from '../../../types'; -import type { DataCell, DataTable } from '../../../types'; +import type { DataCell, DataTable, FieldInfo } from '../../../types'; import { builtinThemeMap } from '../const'; const chartTypeMap: { [chartName: string]: string } = { @@ -373,9 +373,10 @@ export const colorLine = (context: GenerateChartCellContext) => { export const seriesField = (context: GenerateChartCellContext) => { const { spec, fieldInfo, dataTable, cell } = context; const cellNew: any = { ...cell }; - const { seriesField, xField } = spec; + const { seriesField, xField: propsXField } = spec; const colorField = isArray(seriesField) ? seriesField[0] : seriesField; const colorFieldInfo = fieldInfo.find(v => v.fieldName === colorField); + const xField = isArray(propsXField) ? propsXField : [propsXField]; if (colorField && colorFieldInfo?.role === ROLE.DIMENSION && xField) { const xMap = new Map(); dataTable.forEach(row => { @@ -398,7 +399,7 @@ export const seriesField = (context: GenerateChartCellContext) => { } if (!isValidColor) { spec.seriesField = undefined; - spec.xField = spec.xField?.filter((field: string) => field !== colorField); + spec.xField = xField.filter((field: string) => field !== colorField); cellNew.color = undefined; } } @@ -1401,7 +1402,7 @@ export const circularProgressStyle = (context: GenerateChartCellContext) => { }; export const indicator = (context: GenerateChartCellContext) => { - const { spec, cell } = context; + const { spec, cell, fieldInfo } = context; const firstEntry = spec.data.values[0]; if (!firstEntry) { return { spec }; diff --git a/packages/vmind/src/atom/chartGenerator/utils.ts b/packages/vmind/src/atom/chartGenerator/utils.ts index d04e9206..3ae2c5ae 100644 --- a/packages/vmind/src/atom/chartGenerator/utils.ts +++ b/packages/vmind/src/atom/chartGenerator/utils.ts @@ -3,6 +3,28 @@ import type { DataTypeName } from '@visactor/chart-advisor'; import { ChartType } from '@visactor/chart-advisor'; import type { Cell, FieldInfo } from '../../types'; import { ChartType as VMindChartType } from '../../types'; +import type { ChartGeneratorCtx } from '../../types'; + +/** + * Generate a vizSchema from fieldInfo + * @param fieldInfo SimpleFieldInfo[] - An array of field information, each element contains the field name, description, type, and role, etc. + * @returns Partial - Returns a partial VizSchema object, containing the transformed field information. + */ +export const getVizSchema = (context: ChartGeneratorCtx) => { + const { fieldInfo } = context; + return { + fields: fieldInfo.map(d => ({ + id: d.fieldName, + alias: d.fieldName, + description: d.description, + visible: true, + type: d.type, + role: d.role, + location: d.role + })) + }; +}; + export const typeMap = (type: string): DataTypeName => { if (['string'].includes(type)) { return 'string'; @@ -14,7 +36,7 @@ export const typeMap = (type: string): DataTypeName => { return 'string'; }; -export const VMindChartTypeMap = { +export const VMindChartTypeMap: Record = { [VMindChartType.BarChart]: [ ChartType.COLUMN, ChartType.COLUMN_PERCENT, @@ -24,6 +46,7 @@ export const VMindChartTypeMap = { ChartType.BAR_PARALLEL ], [VMindChartType.LineChart]: [ChartType.LINE, ChartType.AREA, ChartType.AREA_PERCENT], + [VMindChartType.AreaChart]: [ChartType.AREA, ChartType.AREA_PERCENT], [VMindChartType.PieChart]: [ChartType.PIE, ChartType.ANNULAR], [VMindChartType.RoseChart]: [ChartType.ROSE], [VMindChartType.ScatterPlot]: [ChartType.SCATTER], diff --git a/packages/vmind/src/atom/chartGenerator/vizSchema.ts b/packages/vmind/src/atom/chartGenerator/vizSchema.ts deleted file mode 100644 index 7de0c777..00000000 --- a/packages/vmind/src/atom/chartGenerator/vizSchema.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { ChartGeneratorCtx } from '../../types'; - -/** - * Generate a vizSchema from fieldInfo - * @param fieldInfo SimpleFieldInfo[] - An array of field information, each element contains the field name, description, type, and role, etc. - * @returns Partial - Returns a partial VizSchema object, containing the transformed field information. - */ -export const getVizSchema = (context: ChartGeneratorCtx) => { - const { fieldInfo } = context; - return { - fields: fieldInfo.map(d => ({ - id: d.fieldName, - alias: d.fieldName, - description: d.description, - visible: true, - type: d.type, - role: d.role, - location: d.role - })) - }; -}; diff --git a/packages/vmind/src/atom/dataClean/dataClean.ts b/packages/vmind/src/atom/dataClean/dataClean.ts new file mode 100644 index 00000000..33cbf317 --- /dev/null +++ b/packages/vmind/src/atom/dataClean/dataClean.ts @@ -0,0 +1,106 @@ +import type { DataCleanCtx } from '../../types/atom'; +import { AtomName } from '../../types/atom'; +import type { DataCleanOptions } from '../type'; +import { BaseAtom } from '../base'; +import { merge } from '@visactor/vutils'; +import { + getCtxByfilterSameValueColumn, + getCtxByneedNumericalFields, + getCtxBymeasureAutoTransfer, + getCtxByfilterSameDataItem, + getCtxByFilterRowWithNonEmptyValues, + getCtxByRangeValueTranser, + getSplitDataViewOfDataTable, + transferFieldInfo +} from './utils'; + +/** The order of pipeline is meaningful */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const pipelines: { key: string; func: (ctx: DataCleanCtx, ...args: any[]) => DataCleanCtx }[] = [ + /** convert the interval data */ + { + key: 'rangeValueTransfer', + func: getCtxByRangeValueTranser + }, + /** Correct the measurement fields. */ + { + key: 'measureAutoTransfer', + func: getCtxBymeasureAutoTransfer + }, + /** Filter out the Row where all measurements are null. */ + { + key: 'filterRowWithEmptyValues', + func: getCtxByFilterRowWithNonEmptyValues + }, + /** Filter out completely identical rows. */ + { + key: 'filterSameDataItem', + func: getCtxByfilterSameDataItem + }, + /** Remove the fields where all dimension values are the same. */ + { + key: 'filterSameValueColumn', + func: getCtxByfilterSameValueColumn + }, + /** Need at least one valid measure field */ + { + key: 'needNumericalFields', + func: getCtxByneedNumericalFields + } +]; + +export class DataCleanAtom extends BaseAtom { + name = AtomName.DATA_CLEAN; + + constructor(context: DataCleanCtx, option: DataCleanOptions) { + super(context, option); + } + + buildDefaultContext(context: DataCleanCtx): DataCleanCtx { + return merge( + {}, + { + dataTable: [], + fieldInfo: [] + }, + context + ); + } + + buildDefaultOptions(): DataCleanOptions { + return { + filterSameValueColumn: true, + needNumericalFields: true, + measureAutoTransfer: true, + filterSameDataItem: true, + filterRowWithEmptyValues: true, + rangeValueTransfer: 'avg', + hierarchicalClustering: true + }; + } + + updateContext(context: DataCleanCtx): DataCleanCtx { + this.context = transferFieldInfo(super.updateContext(context)); + return this.context; + } + + shouldRunByContextUpdate(context: DataCleanCtx): boolean { + return context.dataTable !== this.context.dataTable; + } + + _runWithOutLLM(): DataCleanCtx { + let newContext = { ...this.context }; + pipelines.forEach(({ key, func }) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const currentOption = (this.options as any)[key]; + if (currentOption !== false) { + newContext = func(newContext, currentOption); + } + }); + this.setNewContext(newContext); + if (this.options.hierarchicalClustering) { + this.setNewContext(getSplitDataViewOfDataTable(newContext, this.options.clusterThreshold)); + } + return this.context; + } +} diff --git a/packages/vmind/src/atom/dataClean/index.ts b/packages/vmind/src/atom/dataClean/index.ts index 516e8ac9..2d673068 100644 --- a/packages/vmind/src/atom/dataClean/index.ts +++ b/packages/vmind/src/atom/dataClean/index.ts @@ -1,106 +1,2 @@ -import type { DataCleanCtx } from '../../types/atom'; -import { AtomName } from '../../types/atom'; -import type { DataCleanOptions } from '../type'; -import { BaseAtom } from '../base'; -import { merge } from '@visactor/vutils'; -import { - getCtxByfilterSameValueColumn, - getCtxByneedNumericalFields, - getCtxBymeasureAutoTransfer, - getCtxByfilterSameDataItem, - getCtxByFilterRowWithNonEmptyValues, - getCtxByRangeValueTranser, - getSplitDataViewOfDataTable, - transferFieldInfo -} from './utils'; - -/** The order of pipeline is meaningful */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const pipelines: { key: string; func: (ctx: DataCleanCtx, ...args: any[]) => DataCleanCtx }[] = [ - /** convert the interval data */ - { - key: 'rangeValueTransfer', - func: getCtxByRangeValueTranser - }, - /** Correct the measurement fields. */ - { - key: 'measureAutoTransfer', - func: getCtxBymeasureAutoTransfer - }, - /** Filter out the Row where all measurements are null. */ - { - key: 'filterRowWithEmptyValues', - func: getCtxByFilterRowWithNonEmptyValues - }, - /** Filter out completely identical rows. */ - { - key: 'filterSameDataItem', - func: getCtxByfilterSameDataItem - }, - /** Remove the fields where all dimension values are the same. */ - { - key: 'filterSameValueColumn', - func: getCtxByfilterSameValueColumn - }, - /** Need at least one valid measure field */ - { - key: 'needNumericalFields', - func: getCtxByneedNumericalFields - } -]; - -export class DataCleanAtom extends BaseAtom { - name = AtomName.DATA_CLEAN; - - constructor(context: DataCleanCtx, option: DataCleanOptions) { - super(context, option); - } - - buildDefaultContext(context: DataCleanCtx): DataCleanCtx { - return merge( - {}, - { - dataTable: [], - fieldInfo: [] - }, - context - ); - } - - buildDefaultOptions(): DataCleanOptions { - return { - filterSameValueColumn: true, - needNumericalFields: true, - measureAutoTransfer: true, - filterSameDataItem: true, - filterRowWithEmptyValues: true, - rangeValueTransfer: 'avg', - hierarchicalClustering: true - }; - } - - updateContext(context: DataCleanCtx): DataCleanCtx { - this.context = transferFieldInfo(super.updateContext(context)); - return this.context; - } - - shouldRunByContextUpdate(context: DataCleanCtx): boolean { - return context.dataTable !== this.context.dataTable; - } - - _runWithOutLLM(): DataCleanCtx { - let newContext = { ...this.context }; - pipelines.forEach(({ key, func }) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const currentOption = (this.options as any)[key]; - if (currentOption !== false) { - newContext = func(newContext, currentOption); - } - }); - this.setNewContext(newContext); - if (this.options.hierarchicalClustering) { - this.setNewContext(getSplitDataViewOfDataTable(newContext, this.options.clusterThreshold)); - } - return this.context; - } -} +export { MultipleDataCleanAtom } from './multiple'; +export { DataCleanAtom } from './dataClean'; diff --git a/packages/vmind/src/atom/dataClean/multiple.ts b/packages/vmind/src/atom/dataClean/multiple.ts new file mode 100644 index 00000000..acbfe8ef --- /dev/null +++ b/packages/vmind/src/atom/dataClean/multiple.ts @@ -0,0 +1,101 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { DatasetFromText, MultipleDataCleanCtx } from '../../types/atom'; +import { AtomName } from '../../types/atom'; +import type { MultipleDataCleanOptions } from '../type'; +import { BaseAtom } from '../base'; +import { isArray, merge } from '@visactor/vutils'; +import { + canMergeClusterResult, + canMergeDataTable, + getSplitDataViewOfDataTable, + mergeClusterDataView, + mergeDataTable +} from './utils'; +import { pipelines } from './dataClean'; + +export class MultipleDataCleanAtom extends BaseAtom { + name = AtomName.MULTIPLE_DATA_CLEAN; + + constructor(context: MultipleDataCleanCtx, option: MultipleDataCleanOptions) { + super(context, option); + } + + buildDefaultContext(context: MultipleDataCleanCtx): MultipleDataCleanCtx { + return merge( + {}, + { + datasets: [] + }, + context + ); + } + + buildDefaultOptions(): MultipleDataCleanOptions { + return { + filterSameValueColumn: true, + needNumericalFields: true, + measureAutoTransfer: true, + filterSameDataItem: true, + filterRowWithEmptyValues: true, + rangeValueTransfer: 'last', + hierarchicalClustering: true, + clusterThreshold: 0.4, + filterRatioInDataset: 0.6 + }; + } + + shouldRunByContextUpdate(context: MultipleDataCleanCtx): boolean { + return context.datasets !== this.context.datasets; + } + + _runWithOutLLM(): MultipleDataCleanCtx { + const { datasets } = this.context; + const { filterRatioInDataset } = this.options; + const result: DatasetFromText[] = []; + datasets.forEach(dataset => { + let newDataset: any = { ...dataset }; + pipelines.forEach(({ key, func }) => { + const currentOption = (this.options as any)[key]; + if (currentOption !== false) { + newDataset = { + ...newDataset, + ...func(newDataset, currentOption) + }; + } + }); + if (this.options.hierarchicalClustering) { + const { clusterResult = [] } = getSplitDataViewOfDataTable(newDataset, this.options.clusterThreshold); + if (false && canMergeClusterResult(clusterResult)) { + /** todo */ + newDataset = { + ...newDataset, + ...mergeClusterDataView(clusterResult) + }; + } else if (clusterResult.length) { + const maxValidCount = clusterResult[0].validCellCount; + newDataset = clusterResult + .filter(dataView => dataView.validCellCount / maxValidCount >= filterRatioInDataset) + .map(dataView => ({ + ...newDataset, + dataTable: dataView.dataTable, + fieldInfo: dataView.fieldInfo + })); + } + } + if (isArray(newDataset) && newDataset.length === 1) { + newDataset = newDataset[0]; + } + if (isArray(newDataset)) { + result.push(...newDataset); + } else if (canMergeDataTable(result[result.length - 1], newDataset)) { + result[result.length - 1] = mergeDataTable(result[result.length - 1], newDataset); + } else { + result.push(newDataset); + } + }); + this.updateContext({ + datasets: result + }); + return this.context; + } +} diff --git a/packages/vmind/src/atom/dataClean/utils.ts b/packages/vmind/src/atom/dataClean/utils.ts index f2cfff6f..a25880a0 100644 --- a/packages/vmind/src/atom/dataClean/utils.ts +++ b/packages/vmind/src/atom/dataClean/utils.ts @@ -1,8 +1,8 @@ import { getRoleByFieldType } from '../../utils/field'; -import type { ClusterDataView } from '../../types/atom'; -import type { DataItem } from '../../types'; +import type { ClusterDataView, DatasetFromText } from '../../types/atom'; +import type { DataItem, DataTable } from '../../types'; import { DataType, ROLE, type DataCleanCtx, type FieldInfo } from '../../types'; -import { isArray, isNumber, pick } from '@visactor/vutils'; +import { isArray, isNumber, isString, pick } from '@visactor/vutils'; import { extractFirstNumberInString } from '../../utils/text'; import { isValidData, uniqBy, average } from '../../utils/common'; import type { RangeValueTransferType } from '../type'; @@ -60,6 +60,15 @@ export const getCtxByfilterSameValueColumn = (context: DataCleanCtx) => { }); return removeFieldInfoInCtx(newContext, cleanFieldKey); } + if (dataTable.length && fieldInfo.length) { + const cleanFieldKey: string[] = []; + fieldInfo.forEach(info => { + if (info.role === ROLE.DIMENSION && !isValidData(dataTable[0][info.fieldName])) { + cleanFieldKey.push(info.fieldName); + } + }); + return removeFieldInfoInCtx(newContext, cleanFieldKey); + } return newContext; }; @@ -87,23 +96,43 @@ export const getCtxByneedNumericalFields = (context: DataCleanCtx) => { export const getCtxBymeasureAutoTransfer = (context: DataCleanCtx, text?: string) => { const { fieldInfo = [], dataTable = [] } = context || {}; const isStringText = text && typeof text === 'string'; - if (dataTable.length > 1 && fieldInfo.length) { + if (dataTable.length >= 1 && fieldInfo.length) { fieldInfo.forEach(info => { if (info.role === ROLE.DIMENSION) { return; } - if (info.type === DataType.RATIO) { - } for (let i = 0; i < dataTable.length; i++) { let value = dataTable[i][info.fieldName]; - if (typeof dataTable[i][info.fieldName] === 'string') { - dataTable[i][info.fieldName] = extractFirstNumberInString(value as string); + if (typeof dataTable[i][info.fieldName] === 'string' && isNaN(Number(value))) { + const extractionValue = `${extractFirstNumberInString(value as string)}`; + const beforeLen = (value as string).length; + const curLen = extractionValue.length; + dataTable[i][info.fieldName] = + extractionValue !== 'null' && (curLen / beforeLen > 0.9 || beforeLen - curLen <= 2) + ? Number(extractionValue) + : null; + } else if (typeof dataTable[i][info.fieldName] === 'string') { + value = Number(value); + dataTable[i][info.fieldName] = value; + } else if (!isNumber(value)) { + value = null; + dataTable[i][info.fieldName] = null; } value = dataTable[i][info.fieldName]; - if (info.type === DataType.RATIO && isStringText && isNumber(value)) { - const ratioValue = value * 100; - if (text.includes(`${ratioValue}%`) && !text.includes(`${value}`)) { - dataTable[i][info.fieldName] = ratioValue; + if (info.type === DataType.RATIO && isNumber(value)) { + if (isStringText) { + // revised wrong ratio value in extraction + const ratioValue = value * 100; + if (text.includes(`${ratioValue}%`) && !text.includes(`${value}`)) { + dataTable[i][info.fieldName] = ratioValue; + value = ratioValue; + } + } + // transfer ratio value to absolue value without unit + if (info.ratioGranularity === '%') { + dataTable[i][info.fieldName] = value / 100; + } else if (info.ratioGranularity === '‰') { + dataTable[i][info.fieldName] = value / 1000; } } } @@ -144,17 +173,22 @@ export const getCtxByFilterRowWithNonEmptyValues = (context: DataCleanCtx) => { }; const transferRangeData = (cell: number[], type: RangeValueTransferType) => { + const validCell = cell.filter(v => isValidData(v)); switch (type) { case 'avg': - return average(cell); + return average(validCell); case 'filter': return null; case 'max': - return Math.max(...cell); + return Math.max(...validCell); case 'min': - return Math.min(...cell); + return Math.min(...validCell); + case 'first': + return validCell[0]; + case 'last': + return validCell[validCell.length - 1]; default: - return cell.join('-'); + return validCell.join('-'); } }; @@ -164,12 +198,13 @@ export const getCtxByRangeValueTranser = (context: DataCleanCtx, type: RangeValu return { ...context, dataTable: dataTable.map(item => { + const newItem = { ...item }; fieldInfo.forEach(info => { - if (isArray(item[info.fieldName])) { - item[info.fieldName] = transferRangeData(item[info.fieldName] as any, type); + if (info.role === ROLE.MEASURE && !isString(item[info.fieldName]) && isArray(item[info.fieldName])) { + newItem[info.fieldName] = transferRangeData(item[info.fieldName] as any, type); } }); - return item; + return newItem; }) }; }; @@ -198,8 +233,25 @@ export const getCtxByValidColumnRatio = (context: DataCleanCtx, ratio = 0.2) => ); }; +export const canMergeDataTable = (ctxA: DatasetFromText, ctxB: DatasetFromText) => { + const { fieldInfo: fieldInfoA = [], summary: summaryA } = ctxA || {}; + const { fieldInfo: fieldInfoB = [], summary: summaryB } = ctxB || {}; + if (fieldInfoA.length !== fieldInfoB.length || !fieldInfoA.length || !summaryA || !summaryB) { + return false; + } + return fieldInfoA.every(item => { + return fieldInfoB.find( + itemB => + itemB.fieldName === item.fieldName && + itemB.type === item.type && + itemB?.unit === item?.unit && + itemB?.ratioGranularity === item?.ratioGranularity + ); + }); +}; + /** get main data view and cluster result */ -export const getSplitDataViewOfDataTable = (context: DataCleanCtx, threshold: number) => { +export const getSplitDataViewOfDataTable = (context: DataCleanCtx, threshold = 0.4) => { const { dataTable = [], fieldInfo } = context || {}; const measureFieldInfo = fieldInfo.filter(info => info.role === ROLE.MEASURE); const dimensionFieldInfo = fieldInfo.filter(info => info.role === ROLE.DIMENSION); @@ -240,7 +292,7 @@ export const getSplitDataViewOfDataTable = (context: DataCleanCtx, threshold: nu validRowLength: newContext.dataTable.length, validCellCount }; - dataViewList.push(dataView); + validCellCount > 0 && dataViewList.push(dataView); }); dataViewList.sort((a, b) => a.validCellCount < b.validCellCount || @@ -251,6 +303,9 @@ export const getSplitDataViewOfDataTable = (context: DataCleanCtx, threshold: nu ? 1 : -1 ); + if (dataViewList.length === 0) { + return context; + } return { ...context, originDataTable: dataTable, @@ -259,3 +314,98 @@ export const getSplitDataViewOfDataTable = (context: DataCleanCtx, threshold: nu clusterResult: dataViewList }; }; + +export const canMergeClusterResult = (clusterResult: ClusterDataView[]) => { + if (!clusterResult.length) { + return false; + } + return clusterResult.every(dataView => { + const { fieldInfo, dataTable } = dataView; + return ( + dataTable.length === 1 && fieldInfo.findIndex(info => [DataType.DATE, DataType.TIME].includes(info.type)) === -1 + ); + }); +}; + +export const mergeClusterDataView = (clusterResult: ClusterDataView[]) => { + const newFieldInfo: FieldInfo[] = []; + const newDataTable: DataTable = [{}]; + clusterResult.forEach(dataView => { + const { fieldInfo, dataTable } = dataView; + const measureFields = fieldInfo.filter(info => info.role === ROLE.MEASURE); + newFieldInfo.push(...measureFields); + measureFields.forEach(field => { + newDataTable[0][field.fieldName] = dataTable[0][field.fieldName]; + }); + }); + return { + fieldInfo: newFieldInfo, + dataTable: newDataTable + }; +}; + +const isSameFields = (a: FieldInfo[], b: FieldInfo[]) => { + if (a.length !== b.length) { + return false; + } + return a.every((info, index) => { + const matchInB = b.find(item => item.fieldName === info.fieldName); + return ( + matchInB && + matchInB.role === info.role && + matchInB.type === info.type && + matchInB.unit === info.unit && + matchInB.dateGranularity === info.dateGranularity + ); + }); +}; + +function longestCommonSubstringAtEdges(a: string, b: string) { + // 检查开头 + let startLen = 0; + while (startLen < a.length && startLen < b.length && a[startLen] === b[startLen]) { + startLen++; + } + + // 检查结尾 + let endLen = 0; + while (endLen < a.length && endLen < b.length && a[a.length - 1 - endLen] === b[b.length - 1 - endLen]) { + endLen++; + } + + // 返回较长的公共子字符串 + if (startLen >= endLen) { + return { + strA: a.substring(startLen, a.length), + strB: b.substring(startLen, b.length), + commonStr: a.substring(0, startLen) + }; + } + return { + strA: a.substring(0, a.length - endLen), + strB: b.substring(0, b.length - endLen), + commonStr: a.substring(endLen, a.length) + }; +} + +export const mergeDataTable = (ctxA: DatasetFromText, ctxB: DatasetFromText) => { + const { dataTable: tableA, summary: summaryA, textRange: rangeA } = ctxA; + const { dataTable: tableB, summary: summaryB, textRange: rangeB } = ctxB; + const { strA, strB, commonStr } = longestCommonSubstringAtEdges(summaryA, summaryB); + const newFieldInfo: FieldInfo = { + fieldName: 'mergedSummary', + role: ROLE.DIMENSION, + type: DataType.STRING + }; + const newDataTable = [ + ...tableA.map(v => ({ ...v, mergedSummary: strA })), + ...tableB.map(v => ({ ...v, mergedSummary: strB })) + ]; + const textRange = rangeA && rangeB ? [rangeA[0], rangeB[1]] : null; + return { + dataTable: newDataTable, + fieldInfo: [newFieldInfo, ...ctxA.fieldInfo], + summary: `${summaryA} and ${summaryB}`, + textRange: textRange as any + }; +}; diff --git a/packages/vmind/src/atom/dataExtraction/index.ts b/packages/vmind/src/atom/dataExtraction/index.ts index ce9779b9..ca50925b 100644 --- a/packages/vmind/src/atom/dataExtraction/index.ts +++ b/packages/vmind/src/atom/dataExtraction/index.ts @@ -3,10 +3,12 @@ import type { BaseOptions, DataExtractionOptions } from '../type'; import { BaseAtom } from '../base'; import { merge, pick } from '@visactor/vutils'; import type { LLMMessage } from '../../types/llm'; -import { getBasePrompt, getFieldInfoPrompt } from './prompt'; +import { getBasePrompt, getFieldInfoPrompt } from './prompt/prompt'; import { getLanguageOfText } from '../../utils/text'; -import { formatFieldInfo } from '../../utils/field'; +import { formatFieldInfo, hasMeasureField } from '../../utils/field'; import { getCtxBymeasureAutoTransfer } from '../dataClean/utils'; +import type { DatasetFromText, FieldInfo } from '../../types'; +import { DataType } from '../../types'; export class DataExtractionAtom extends BaseAtom { name = AtomName.DATA_EXTRACT; @@ -28,24 +30,32 @@ export class DataExtractionAtom extends BaseAtom { + const { fieldName, type, isRatio, isDate, unit, ratioGranularity } = info; + let finalType = type === 'dimension' ? DataType.STRING : DataType.NUMERICAL; + if (isRatio) { + finalType = DataType.RATIO; + } else if (isDate) { + finalType = DataType.DATE; + } + return { + fieldName, + unit, + ratioGranularity, + type: finalType, + role: type + }; + }); + } + + private parseMultipleResult(dataset: DatasetFromText[]) { + return dataset + .map(result => { + return { + ...result, + fieldInfo: formatFieldInfo(this.revisedFieldInfo(result.fieldInfo)) + }; + }) + .filter(result => hasMeasureField(result.fieldInfo)); + } + parseLLMContent(resJson: any) { - const { dataTable, fieldInfo, isDataExtraction } = resJson; - if (isDataExtraction === false) { + const { isCapcut } = this.options; + const { dataTable, fieldInfo, isDataExtraction, dataset } = resJson; + if (isDataExtraction === false || (isCapcut && !dataset)) { console.error("It's not a data extraction task"); return this.context; } + if (isCapcut) { + return { + ...this.context, + datasets: this.parseMultipleResult(dataset) + }; + } + const llmFieldInfo = this.revisedFieldInfo(fieldInfo); return { ...this.context, fieldInfo: formatFieldInfo( - (this.options?.reGenerateFieldInfo ? fieldInfo : null) ?? this.context?.fieldInfo ?? [] + (this.options?.reGenerateFieldInfo ? llmFieldInfo : null) ?? this.context?.fieldInfo ?? [] ), dataTable } as DataExtractionCtx; diff --git a/packages/vmind/src/atom/dataExtraction/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt.ts deleted file mode 100644 index ec3dd360..00000000 --- a/packages/vmind/src/atom/dataExtraction/prompt.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { isDoubaoModel } from '../../utils/llm'; -import { getBasePrompt as doubaoBasePrompt } from './doubaoPrompt'; -import { getBasePrompt as gptBasePrompt } from './gptPrompt'; -export { getFieldInfoPrompt } from './gptPrompt'; - -export const getBasePrompt = (model: string, language: 'chinese' | 'english', showThoughs: boolean = true) => { - const func = isDoubaoModel(model) ? doubaoBasePrompt : gptBasePrompt; - return func(language, showThoughs); -}; diff --git a/packages/vmind/src/atom/dataExtraction/prompt/capcutPrompt.ts b/packages/vmind/src/atom/dataExtraction/prompt/capcutPrompt.ts new file mode 100644 index 00000000..50ad6eee --- /dev/null +++ b/packages/vmind/src/atom/dataExtraction/prompt/capcutPrompt.ts @@ -0,0 +1,156 @@ +/* eslint-disable max-len */ +export { getFieldInfoPrompt } from './gptPrompt'; + +/** @todo @czx enlish should ajust */ +export const getCapCutPrompt = (language: 'chinese' | 'english') => { + return `You are an expert extraction algorithm. Your task is to extract data according to the requirements. +## Response +Response in the following format: +\`\`\` +{ +dataset: { +summary: string; // Summarize this table in the simplest way. +textRange: [string, string]; // They represent the specific first 5 texts and last 5 texts +fieldInfo: { +fieldName: string; +type: 'measure' | 'dimension'; +unit?: string; // unit of measure value +isRatio?: boolean; // means ratio value or percentage(%), such as 同比、环比、增长率、占比等.The forms of ratio data are usually Percentage (%) such as 60%. +ratioGranularity?: '%' | '‰'; // generate when fieldType is 'ratio', represent the ratio granularity of ratio data +}[], +dataTable: Record[]; // Key of dataTable is fieldName in fieldInfo +}[] +} +\`\`\` +### dataset +The text may contain multiple unrelated data tables, forming an array. +### fieldInfo +FieldInfo represents the specific information of each column field in the data table. +1. MUST generate ratioGranularity when 'isRatio' is true. +2. Measures MUST generate unit. +### dataTable +1. Key of dataTable MUST be fieldName in fieldInfo +2. The value type of measure field MUST be 'number'. + +# Requirements +## General Requirements +1. Strictly define the type of return format, use JSON format to reply, do not include any extra content. +2. Ensuring data accuracy, making sure the correspondence between numerical and dimensional data is precise and error-free. +3. Extract as many data tables as possible, ensuring the completeness of metric values. +4. Data extraction should directly extract data from the text without any calculation or inference. +5. For vague data expressions, extract only the numerical values. +6. Only extract dimensions with simple values and simplify the expression of dimensions and their values to speed up the response. +7. FieldNames are very concise, without unit information or data change sign information. +8. Only extract data for ratio data., eg., '95%' becomes '95'; 'reduce 30%' becomes '-30'. +## DataTable Requirements +The number of dimensions and measures in a data table can ONLY BE ONE of the following scenarios: +1. ONE measure, and ONE or TWO dimensions. +2. Two measures and ONE dimension. +3. ONLY measures, NO dimensions. + +# Steps +You should think step-by-step as follow, while ensuring all requirements are met during the process. +1. Check if the task involves data extraction. If not, set isDataExtraction to false in json mode; otherwise, proceed with steps. +2. Read the text and think carefully, generate k(k>=1) data tables as much as possible and generate summary for them. +3. Traverse these k tables, executing steps 4 to 9 for each table. +4. Extract key measure fields and dimension fields based on the summary and DataTable Requirements. +5. Check the current number of dimensions. If it exceeds 2, MUST split and return to Step 3. +6. Check dimensions and measures; if BOTH exceed 1, MUST split the table and return to Step 3. +7. Extract data tables from the text based on fields and requirements. +8. Ensure the correctness of the value and type of each row in dataTable, return null for the value of unknown fields. +9. Recheck all data to ensure that no numerical or ratio data is missing. +10. Generate a concise, clear final answer in ${language} with correct JSON format. +--- +# Examples1 +text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿元，经调整后的利润额为100亿元，而字节跳动V月份的利润额为800亿元，经调整后利润额为120亿元。 + +Response: +\`\`\` +{"dataset": [{"summary": "各大厂商的财报数据", "textRange":["今年6月各","120亿元。"], "fieldInfo":[{"fieldName":"公司","type":"dimension"},{"fieldName":"调整前利润额","type":"measure", "unit": "亿元"},{"fieldName":"调整后利润额","type":"measure", "unit": "亿元"}],"dataTable":[{"公司":"阿里","调整前利润额":1000, "调整后利润额": 100},{"公司":"字节跳动","调整前利润额":800, "调整后利润额": 120}]}]} +\`\`\` + +# Examples2 +text: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%多。在这30%多的年轻人中，其中60%是女性，40%是男性。 + +Response: +\`\`\` +{"dataset":[{"summary":"年轻人入睡困难占比变化","textRange": ["现在有大约", "30%多。"],"fieldInfo":[{"fieldName":"年份","type":"dimension",},{"fieldName":"入睡困难占比","type":"measure","isRatio":true,"ratioGranularity": "%"}],"dataTable":[{"年份":"现在","占比":60},{"年份":"两年前","占比":30}]},{"summary":"入睡困难分布","textRange": ["在这30%", "40%是男性。"],"fieldInfo":[{"fieldName":"性别","type":"dimension"},{"fieldName":"占比","type":"measure","isRatio":true, "ratioGranularity": "%"}],"dataTable":[{"性别":"男","占比":60},{"性别":"女","占比":40}]}]} +\`\`\``; +}; + +export const getCapCutPromptInGpt = (language: 'chinese' | 'english') => { + return `You are an expert extraction algorithm, especially sensitive to data, date, category, data comparison and similar content.Your task is to extract data according to the requirements. +## Response +Response in the following format: +\`\`\` +{ +dataset: { +summary: string; // Summarize this table in the simplest way. +textRange: [string, string]; // They represent the specific first 5 texts and last 5 texts +fieldInfo: { +fieldName: string; +type: 'measure' | 'dimension'; +unit?: string; // unit of measure value +isRatio?: boolean; // means ratio value or percentage(%), such as 同比、环比、增长率、占比等.The forms of ratio data are usually Percentage (%) such as 60%. +ratioGranularity?: '%' | '‰'; // generate when fieldType is 'ratio', represent the ratio granularity of ratio data +isDate?: boolean; //'date' refers to data that can be specified down to the year, quarter, month, week, or day. +}[], +dataTable: Record[]; // Key of dataTable is fieldName in fieldInfo +}[] +} +\`\`\` +### dataset +The text may contain multiple unrelated data tables. Therefore, the dataset is an array. Each element, containing fieldInfo and dataTable, represents the header information and data of a data table. +### fieldInfo +FieldInfo represents the specific information of each column field in the data table. +1. MUST generate ratioGranularity when 'isRatio' is true. +2. Measures MUST generate unit. +### dataTable +The data tables are ultimately used for statistical chart display. +1. Key of dataTable is fieldName in fieldInfo +2. The value type of measure field MUST be 'number', unless it is interval data, in which case use 'number[]'. + +# Requirements +## General Requirements +1. Strictly define the type of return format, use JSON format to reply, do not include any extra content. +2. Ensuring data accuracy, making sure the correspondence between numerical and dimensional data is precise and error-free. +3. Data extraction should directly extract data from the text without any calculation or inference. +4. For vague data expressions, extract only the numerical values; if expressed as a range, treat it as interval data. +5. Only extract dimensions with simple values. +6. FieldNames are very concise, without unit information or data change sign information. +7. Only extract data for ratio data., eg., '95%' becomes '95'; 'reduce 30%' becomes '-30'. +## DataTable Requirements +The number of dimensions and measures in a data table can ONLY BE ONE of the following scenarios: +1. ONE measure, and ONE or TWO dimensions. +2. Two measures and ONE dimension. +3. Multiple measures and ZERO dimension. + +# Steps +You should think step-by-step as follow, while ensuring all requirements are met during the process. +1. Read the entire text to find the key measure fields. +2. Reread the text, split it into k(k>=1) data tables based on measure fields. +3. Traverse these k tables, executing steps 4 to 9 for each table. +4. Extract dimension fields based on the current summary and measure fields and DataTable Requirements. +5. Check the current number of dimensions. If it exceeds 2, split and return to Step 3. +6. Check dimensions and measures; if both exceed 1, split the table and return to Step 3. +7. Extract data tables from the text based on field and requirements. +8. Extract interval/range data in the form of an array. +9. Ensure the correctness of the value and type of each row in dataTable, return null for the value of unknown fields. +10. Generate a concise, clear final answer in ${language} with correct JSON format. +--- +# Examples1 +text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿元，经调整后的利润额为100亿元，而字节跳动V月份的利润额为800亿元，经调整后利润额为120亿元。 + +Response: +\`\`\` +{"dataset": [{"summary": "各大厂商的财报数据", "textRange":["今年6月各","120亿元。"], "fieldInfo":[{"fieldName":"公司","type":"dimension"},{"fieldName":"调整前利润额","type":"measure", "unit": "亿元"},{"fieldName":"调整后利润额","type":"measure", "unit": "亿元"}],"dataTable":[{"公司":"阿里","调整前利润额":1000, "调整后利润额": 100},{"公司":"字节跳动","调整前利润额":800, "调整后利润额": 120}]}]} +\`\`\` + +# Examples2 +text: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%多。在这30%多的年轻人中，其中60%是女性，40%是男性。 + +Response: +\`\`\` +{"dataset":[{"summary":"年轻人入睡困难占比变化","textRange": ["现在有大约", "30%多。"],"fieldInfo":[{"fieldName":"年份","type":"dimension",},{"fieldName":"入睡困难占比","type":"measure","isRatio":true,"ratioGranularity": "%"}],"dataTable":[{"年份":"2024","占比":[60,70]},{"年份":"2022","占比":30}]},{"summary":"入睡困难分布","textRange": ["在这30%", "%是男性。"],"fieldInfo":[{"fieldName":"性别","type":"dimension"},{"fieldName":"占比","type":"measure","isRatio":true, "ratioGranularity": "%"}],"dataTable":[{"性别":"男","占比":60},{"性别":"女","占比":40}]}]} +\`\`\``; +}; diff --git a/packages/vmind/src/atom/dataExtraction/doubaoPrompt.ts b/packages/vmind/src/atom/dataExtraction/prompt/doubaoPrompt.ts similarity index 54% rename from packages/vmind/src/atom/dataExtraction/doubaoPrompt.ts rename to packages/vmind/src/atom/dataExtraction/prompt/doubaoPrompt.ts index 38701a8a..ad664bdf 100644 --- a/packages/vmind/src/atom/dataExtraction/doubaoPrompt.ts +++ b/packages/vmind/src/atom/dataExtraction/prompt/doubaoPrompt.ts @@ -74,7 +74,7 @@ const getFieldTypeExplanation = (language: 'chinese' | 'english') => { }; export const getBasePrompt = ( language: 'chinese' | 'english', - showThoughs: boolean = true + showThoughs: boolean = false ) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, date, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. # Response ${getResponse(showThoughs)} @@ -108,8 +108,96 @@ You should think step-by-step as follow: --- ${baseExamples}`; +// export const getBasePrompt = ( +// language: 'chinese' | 'english', +// showThoughs: boolean = false +// ) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, date, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. +// # Response + +// Response in the following format: +// \`\`\` +// { +// fieldInfo: { +// fieldName: string; +// type: 'measure' | 'dimension'; +// isRatio?: boolean; // means ratio value or percentage(%), such as 同比、环比、增长率、占比等.The forms of ratio data are usually Percentage (%) such as 60%. +// ratioGranularity?: '%' | '‰'; // generate when fieldType is 'ratio', represent the ratio granularity of ratio data +// isDate?: boolean; //'date' refers to data that can be specified down to the year, quarter, month, week, or day. +// dateGranularity?: 'year' | 'quarter' | 'month' | 'week' | 'day'; // generate when fieldType is 'date', represent the date granularity of date time +// }[], +// dataTable: Record[]; + +// } +// \`\`\` + +// ## Field Information +// 1. ALWAYS generate a field information, which represents the specific information of each column field in the data table. +// 2. ALWAYS generate dateGranularity when 'isDate' is true +// 3. ALWAYS generate ratioGranularity when 'isRatio' is true. +// ## Data Table +// 1. Key of dataTable is fieldName in fieldInfo +// 1. The value type of measure field MUST be 'number' or 'number[]', and DO NOT perform any arithmetic operations.. +// 2. ALWAYS generate flatten data table rather than unflatten data table +// ### Flatten Data Table Example +// \`\`\` +// dataTable:[{date:"Monday",class:"class No.1",score:20},{date:"Monday",class:"class No.2",score:30},{date:"Tuesday",class:"class No.1",score:25},{date:"Tuesday",class:"class No.2",score:28}] +// \`\`\` +// ### Unflatten Data Table Example +// \`\`\` +// dataTable:[{date:"Monday",classNo.1:20,classNo.2:30},{date:"Tuesday",classNo.1:25,classNo.2:28}] +// \`\`\` +// # Common Information +// 1. 今年是2024年 +// 2. 8.5折和85折含义相同，都代表85%的折扣 + +// # Constraints: +// 1. Answer language MUST: chinese +// 2. Strictly define the type of return format, use JSON format to reply, do not include any extra content. +// 3. The extracted data strives for simplicity. +// 4. Prefer flatten data table rather than unflatten data table. +// 5. Numerical and Ratio data are unit-free, e.g., '10万' becomes '100000', '1k' becomes '1000'. +// 6. Only extract value in ratio type, eg., '95%' becomes '95'; 'reduce 30%' becomes '-30'. +// 7. Ensure the correctness of the value and type of each row in dataTable, return null for the value of unknown fields. +// 8. The change in values should be reflected in the positive or negative nature of the data, not in the field names, such as '下降了50个点'的提取结果为'-50'. +// 9. The text may contain the contents of multiple data tables, and unrelated data can be set to null to distinguish them. +// # Steps +// You should think step-by-step as follow: +// 0. Answer language MUST: chinese +// 1. Check if the task involves data extraction. If not, set isDataExtraction to false in json mode; otherwise, proceed with steps. +// 2. Read the entire text and generate the MOST IMPORTANT fields with numerical or ratio or count field type first. +// 3. Re-read the text, generate concise and clear fields associated with the fields found in Step2. +// 4. Extract data tables from the text based on field information directly. Each field's data should be concise and convey a single meaning. +// 5. Extract interval/range data in the form of an array. +// 6. Format date data based on granularity, e.g., yyyy-mm-dd, mm-dd, mm, yyyy-mm, or yyyy-qq. +// 7. When a date field has multiple date granularities, change the type of field to string. +// 8. Check the data in the dataTable to ensure the correctness of the type. +// 9. Recheck all data to ensure that no measure data is missing. +// --- +// # Examples1 +// text:今年6月各大厂商发布了过去1个月的财报数据，其中阿里在V月份利润额达到了1000亿，经调整后的利润额为100亿，而字节跳动V月份的利润额为800亿，经调整后利润额为120亿。 + +// Response: +// \`\`\` +// {"fieldInfo":[{"fieldName":"公司","type":"dimension",},{"fieldName":"月份","type":"dimension","isDate":true, "dateGranularity": "month"},{"fieldName":"利润调整","type":"dimension",},{"fieldName":"利润额","type":"measure",}],"dataTable":[{"公司":"阿里","月份":"5月","利润调整":"调整前","利润额":100000000000,},{"公司":"阿里","月份":"5月","利润调整":"调整后","利润额":10000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整前","利润额":80000000000,},{"公司":"字节跳动","月份":"5月","利润调整":"调整后","利润额":12000000000,},]} +// \`\`\` +// # Examples2 +// text: John Smith was very tall, ranking in the 90th percentile for his age group. He knew Jane Doe. who ranking in the 75th percentile for her age group. + +// Response: +// \`\`\` +// {"fieldInfo":[{"fieldName":"name","type":"dimension",},{"fieldName":"ranking","type":"measure", "isRatio": true, "ratioGranularity": "%"}],"dataTable":[{"name":"John Smith","ranking":90,},{"name":"Jane Doe","ranking":75}]} +// \`\`\` +// # Examples3 +// text: 现在有大约60%-70%的年轻人有入睡困难，而在两年前，入睡困难的年轻人占比才只有30%。 + +// Response: +// \`\`\` +// {"fieldInfo":[{"fieldName":"年份","type":"dimension",dateGranularity:"year"},{"fieldName":"入睡困难占比","type":"measure", "isRatio": true,}],"dataTable":[{"年份":"2024","占比":[0.6,0.7],},{"年份":"2022","占比":0.3}]} +// \`\`\` +// `; + export const getFieldInfoPrompt = ( language: 'chinese' | 'english', - showThoughs: boolean = true, + showThoughs: boolean = false, reGenerateFieldInfo: boolean = false ) => gptFieldInfoPrompt(language, showThoughs, reGenerateFieldInfo); diff --git a/packages/vmind/src/atom/dataExtraction/gptPrompt.ts b/packages/vmind/src/atom/dataExtraction/prompt/gptPrompt.ts similarity index 99% rename from packages/vmind/src/atom/dataExtraction/gptPrompt.ts rename to packages/vmind/src/atom/dataExtraction/prompt/gptPrompt.ts index 2d565939..b67b92e6 100644 --- a/packages/vmind/src/atom/dataExtraction/gptPrompt.ts +++ b/packages/vmind/src/atom/dataExtraction/prompt/gptPrompt.ts @@ -51,7 +51,7 @@ const getFieldTypeExplanation = (language: 'chinese' | 'english') => { }; export const getBasePrompt = ( language: 'chinese' | 'english', - showThoughs: boolean = true + showThoughs: boolean = false ) => `You are an expert extraction algorithm.You are an expert extraction algorithm, especially sensitive to data, date, category, data comparison and similar content.Your task is to extract high-quality data tables and field information from the text for further analysis, such as visualization charts, etc. # Field Information Explanation 1. ALWAYS generate a field information, which represents the specific information of each column field in the data table. @@ -103,7 +103,7 @@ You only need to return the JSON in your response directly to the user.Finish yo export const getFieldInfoPrompt = ( language: 'chinese' | 'english', - showThoughs: boolean = true, + showThoughs: boolean = false, reGenerateFieldInfo: boolean = false ) => `You are an expert extraction algorithm and are highly sensitive to comparative data, trend data, date data and similar information.Only extract relevant information from the text. Your goal is to extract structured information from the user's input that matches the form described below. When extracting information please make sure it matches the type information exactly. The definition of the field information is as follows. diff --git a/packages/vmind/src/atom/dataExtraction/prompt/prompt.ts b/packages/vmind/src/atom/dataExtraction/prompt/prompt.ts new file mode 100644 index 00000000..20c21867 --- /dev/null +++ b/packages/vmind/src/atom/dataExtraction/prompt/prompt.ts @@ -0,0 +1,18 @@ +import { isDoubaoModel } from '../../../utils/llm'; +import { getCapCutPrompt, getCapCutPromptInGpt } from './capcutPrompt'; +import { getBasePrompt as doubaoBasePrompt } from './doubaoPrompt'; +import { getBasePrompt as gptBasePrompt } from './gptPrompt'; +export { getFieldInfoPrompt } from './gptPrompt'; + +export const getBasePrompt = ( + model: string, + language: 'chinese' | 'english', + isCapcut = false, + showThoughs: boolean = false +) => { + if (isCapcut) { + return isDoubaoModel(model) ? getCapCutPrompt(language) : getCapCutPromptInGpt(language); + } + const func = isDoubaoModel(model) ? doubaoBasePrompt : gptBasePrompt; + return func(language, showThoughs); +}; diff --git a/packages/vmind/src/atom/index.ts b/packages/vmind/src/atom/index.ts index 28105eab..d0d7f9b0 100644 --- a/packages/vmind/src/atom/index.ts +++ b/packages/vmind/src/atom/index.ts @@ -1,6 +1,8 @@ -export { DataCleanAtom } from './dataClean'; +export { DataCleanAtom, MultipleDataCleanAtom } from './dataClean'; export { DataExtractionAtom } from './dataExtraction'; export { DataQueryAtom } from './dataQuery'; +export { DataInsightAtom } from './dataInsight'; export { ChartCommandAtom } from './chartCommand'; +export { MultipleChartCommandAtom } from './chartCommand/multiple'; export { ChartGeneratorAtom } from './chartGenerator'; export { BaseAtom } from './base'; diff --git a/packages/vmind/src/atom/type.ts b/packages/vmind/src/atom/type.ts index 73547a1e..781d7d51 100644 --- a/packages/vmind/src/atom/type.ts +++ b/packages/vmind/src/atom/type.ts @@ -1,6 +1,6 @@ import type { FieldInfo } from 'src/types'; import type { LLMManage } from '../core/llm'; -import { extend } from 'dayjs'; +import type { AlgorithmType, AlgorithmOptions } from './dataInsight/type'; export interface BaseOptions { /** llm manage instance */ @@ -11,13 +11,15 @@ export interface BaseOptions { export interface DataExtractionOptions extends BaseOptions { reGenerateFieldInfo?: boolean; + isCapcut?: boolean; } export interface ChartCommandOptions extends BaseOptions { useDataTable?: boolean; + filterByRule?: boolean; } -export type RangeValueTransferType = 'string' | 'filter' | 'avg' | 'max' | 'min'; +export type RangeValueTransferType = 'string' | 'filter' | 'avg' | 'max' | 'min' | 'first' | 'last'; export interface DataCleanOptions extends BaseOptions { needNumericalFields?: boolean; @@ -30,11 +32,30 @@ export interface DataCleanOptions extends BaseOptions { clusterThreshold?: number; } +export interface MultipleDataCleanOptions extends DataCleanOptions { + filterRatioInDataset?: number; +} + export interface DataQueryOptions extends BaseOptions { /** use SQL to execute data query or not */ useSQL?: boolean; } +export interface ChartGeneratorOptions extends BaseOptions { + useChartAdvisor?: boolean; +} + +export interface DataInsightOptions extends BaseOptions { + /** max number of insight */ + maxNum?: number; + /** using algorithms list */ + algorithms?: AlgorithmType[]; + /** options of each algorithms */ + algorithmOptions?: AlgorithmOptions; + /** insight limited by chartType, eg. cluster only in scatter chart */ + isLimitedbyChartType?: boolean; +} + export interface SchemaFieldInfo extends Pick { id: string; /** aliasName */ diff --git a/packages/vmind/src/schedule/index.ts b/packages/vmind/src/schedule/index.ts index 1010ffee..b8f70a38 100644 --- a/packages/vmind/src/schedule/index.ts +++ b/packages/vmind/src/schedule/index.ts @@ -3,12 +3,21 @@ import { merge } from '@visactor/vutils'; import type { BaseContext } from '../types/atom'; import { AtomName } from '../types/atom'; import { BaseAtom } from '../atom/base'; -import { DataQueryAtom, DataExtractionAtom, ChartGeneratorAtom, ChartCommandAtom, DataInsightAtom } from '../atom'; +import { + DataQueryAtom, + DataExtractionAtom, + ChartGeneratorAtom, + ChartCommandAtom, + DataInsightAtom, + MultipleDataCleanAtom, + MultipleChartCommandAtom +} from '../atom'; import type { CombineAll, MapAtomTypes, TaskMapping } from '../types/schedule'; -import { DataCleanAtom } from '../atom/dataClean'; +import { DataCleanAtom } from '../atom/dataClean/dataClean'; import type { BaseOptions, ChartCommandOptions, + ChartGeneratorOptions, DataCleanOptions, DataExtractionOptions, DataInsightOptions @@ -18,10 +27,12 @@ export interface ScheduleOptions { [AtomName.BASE]?: BaseOptions; [AtomName.DATA_EXTRACT]?: DataExtractionOptions; [AtomName.DATA_CLEAN]?: DataCleanOptions; + [AtomName.MULTIPLE_DATA_CLEAN]?: DataCleanOptions; [AtomName.DATA_QUERY]?: BaseOptions; [AtomName.DATA_INSIGHT]?: DataInsightOptions; - [AtomName.CHART_GENERATE]?: BaseOptions; + [AtomName.CHART_GENERATE]?: ChartGeneratorOptions; [AtomName.CHART_COMMAND]?: ChartCommandOptions; + [AtomName.MULTIPLE_CHART_COMMAND]?: ChartCommandOptions; } export class Schedule { @@ -45,9 +56,9 @@ export class Schedule { /** @todo */ historySteps: any; - constructor(atomList: T, options: ScheduleOptions, context?: CombineAll>) { + constructor(atomList: T, options?: ScheduleOptions, context?: CombineAll>) { this.atomList = atomList; - this.options = options; + this.options = options || {}; this.query = ''; this.atomInstaces = atomList.map(atomName => this.atomFactory(atomName)); this.setNewTask(context); @@ -57,6 +68,7 @@ export class Schedule { this.context = {} as T; this.atomInstaces.forEach(atom => { this.context = atom.buildDefaultContext(this.context); + atom.reset(); }); } @@ -71,12 +83,16 @@ export class Schedule { return new DataExtractionAtom(this.context, options); case AtomName.DATA_CLEAN: return new DataCleanAtom(this.context, options); + case AtomName.MULTIPLE_DATA_CLEAN: + return new MultipleDataCleanAtom(this.context, options); case AtomName.DATA_QUERY: return new DataQueryAtom(this.context, options); case AtomName.DATA_INSIGHT: return new DataInsightAtom(this.context, options); case AtomName.CHART_COMMAND: return new ChartCommandAtom(this.context, options); + case AtomName.MULTIPLE_CHART_COMMAND: + return new MultipleChartCommandAtom(this.context, options); case AtomName.CHART_GENERATE: return new ChartGeneratorAtom(this.context, options); default: diff --git a/packages/vmind/src/types/atom.ts b/packages/vmind/src/types/atom.ts index 64ec1acf..8d6302df 100644 --- a/packages/vmind/src/types/atom.ts +++ b/packages/vmind/src/types/atom.ts @@ -9,8 +9,10 @@ export enum AtomName { BASE = 'base', DATA_EXTRACT = 'dataExtract', DATA_CLEAN = 'dataClean', + MULTIPLE_DATA_CLEAN = 'multipleDataClean', DATA_QUERY = 'dataQuery', CHART_COMMAND = 'chartCommand', + MULTIPLE_CHART_COMMAND = 'multipleChartCommand', CHART_GENERATE = 'chartGenerate', DATA_INSIGHT = 'dataInsight' } @@ -42,6 +44,20 @@ export interface ClusterDataView { validCellCount: number; } +/** Multiple Dataset */ +export interface DatasetFromText { + /** text object of data extraction */ + text?: string; + /** summary of dataset */ + summary: string; + /** data table of dataset */ + dataTable: DataTable; + /** field info of dataset */ + fieldInfo: FieldInfo[]; + /** data position in text */ + textRange?: [string, string]; +} + /** Context of Data Extraction Atom */ export interface DataExtractionCtx extends BaseContext { /** text object of data extraction */ @@ -53,17 +69,15 @@ export interface DataExtractionCtx extends BaseContext { /** Data Table values */ dataTable?: DataTable; /** multiple results */ - dataset?: { - summary: string; - textPosition: [number, number]; - fieldInfo: FieldInfo[]; - }[]; + datasets?: DatasetFromText[]; } /** Context of Chart Command Atom */ export interface ChartCommandCtx extends BaseContext { /** text object of data extraction */ text: string; + /** summary of data table */ + summary?: string; /** extra fieldsInfo of dataTable */ fieldInfo?: FieldInfo[]; /** Data Table values */ @@ -83,6 +97,18 @@ export interface DataCleanCtx extends BaseContext { originalDataTable?: DataTable; } +export interface MultipleDataCleanCtx extends BaseContext { + /** multiple dataset */ + datasets: DatasetFromText[]; +} + +export interface MultipleChartCommandsCtx extends BaseContext { + /** multiple dataset */ + datasets: DatasetFromText[]; + /** commands */ + commands: string[]; +} + /** Context of Data Query Atom */ export interface DataQueryCtx extends BaseContext { /** current summary of dataTable */ diff --git a/packages/vmind/src/types/base.ts b/packages/vmind/src/types/base.ts index f52b5235..a534db85 100644 --- a/packages/vmind/src/types/base.ts +++ b/packages/vmind/src/types/base.ts @@ -42,6 +42,8 @@ export interface FieldInfo { domain?: (string | number)[]; /** unit of measure field */ unit?: string; + /** */ + ratioGranularity?: '%' | '‰'; /** granularity of date field */ dateGranularity?: 'year' | 'quarter' | 'month' | 'week' | 'day'; } diff --git a/packages/vmind/src/types/llm.ts b/packages/vmind/src/types/llm.ts index ec652f19..28052906 100644 --- a/packages/vmind/src/types/llm.ts +++ b/packages/vmind/src/types/llm.ts @@ -14,10 +14,10 @@ export enum Model { GPT3_5_1106 = 'gpt-3.5-turbo-1106', GPT4 = 'gpt-4', GPT_4_0613 = 'gpt-4-0613', - GPT_4o = 'gpt-4o-2024-05-13', + GPT_4o = 'gpt-4o-2024-08-06', DOUBAO_LITE = 'doubao-lite-32K', DOUBAO_PRO = 'doubao-pro-128k', - DOUBAO_PRO_32K = 'doubao-pro-32k', + DOUBAO_PRO_32K = 'doubao-pro-32k-240828', SKYLARK2 = 'skylark2-pro-4k', SKYLARK2_v1_2 = 'skylark2-pro-4k-v1.2', CHART_ADVISOR = 'chart-advisor' diff --git a/packages/vmind/src/types/schedule.ts b/packages/vmind/src/types/schedule.ts index 05649d88..b3d39644 100644 --- a/packages/vmind/src/types/schedule.ts +++ b/packages/vmind/src/types/schedule.ts @@ -6,7 +6,9 @@ import type { DataQueryCtx, ChartGeneratorCtx, ChartCommandCtx, - DataInsightCtx + DataInsightCtx, + MultipleDataCleanCtx, + MultipleChartCommandsCtx } from './atom'; export interface Tasks { @@ -23,9 +25,11 @@ export type AtomTypeMap = { [AtomName.BASE]: BaseContext; [AtomName.DATA_EXTRACT]: DataExtractionCtx; [AtomName.DATA_CLEAN]: DataCleanCtx; + [AtomName.MULTIPLE_DATA_CLEAN]: MultipleDataCleanCtx; [AtomName.DATA_QUERY]: DataQueryCtx; [AtomName.DATA_INSIGHT]: DataInsightCtx; [AtomName.CHART_COMMAND]: ChartCommandCtx; + [AtomName.MULTIPLE_CHART_COMMAND]: MultipleChartCommandsCtx; [AtomName.CHART_GENERATE]: ChartGeneratorCtx; }; From ae3ba0b424abd3dd376a04cd49fe347625b6c296 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Thu, 24 Oct 2024 09:08:17 +0800 Subject: [PATCH 054/128] feat: add polish insight, update base option --- packages/vmind/src/atom/base.ts | 1 + packages/vmind/src/atom/chartCommand/index.ts | 5 +- .../vmind/src/atom/chartCommand/multiple.ts | 2 +- packages/vmind/src/atom/dataClean/utils.ts | 12 +- .../vmind/src/atom/dataExtraction/index.ts | 2 +- .../algorithms/abnormalTrend/index.ts | 21 +- .../dataInsight/algorithms/drift/index.ts | 12 +- .../algorithms/drift/pageHinkley.ts | 11 +- .../src/atom/dataInsight/algorithms/index.ts | 17 +- .../algorithms/majorityValue/index.ts | 9 +- .../algorithms/overallTrending/index.ts | 34 ++- .../atom/dataInsight/algorithms/revised.ts | 15 +- .../atom/dataInsight/algorithms/statistics.ts | 34 +++ .../atom/dataInsight/algorithms/template.ts | 284 ++++++++++++++++++ .../algorithms/turningPoint/index.ts | 3 +- .../algorithms/volatility/index.ts | 1 + .../vmind/src/atom/dataInsight/dataProcess.ts | 19 +- packages/vmind/src/atom/dataInsight/index.ts | 79 ++++- packages/vmind/src/atom/dataInsight/prompt.ts | 60 ++++ packages/vmind/src/atom/dataInsight/type.ts | 20 +- packages/vmind/src/atom/dataInsight/utils.ts | 28 +- packages/vmind/src/atom/type.ts | 2 + packages/vmind/src/types/atom.ts | 2 + packages/vmind/src/types/base.ts | 2 + 24 files changed, 611 insertions(+), 64 deletions(-) create mode 100644 packages/vmind/src/atom/dataInsight/algorithms/template.ts create mode 100644 packages/vmind/src/atom/dataInsight/prompt.ts diff --git a/packages/vmind/src/atom/base.ts b/packages/vmind/src/atom/base.ts index e7203549..e39bade9 100644 --- a/packages/vmind/src/atom/base.ts +++ b/packages/vmind/src/atom/base.ts @@ -123,6 +123,7 @@ export class BaseAtom { const data = await this.options.llm.run(this.name, messages); const resJson = this.options.llm.parseJson(data); if (resJson.error) { + console.error(resJson.error); this.updateContext({ error: resJson.error } as any); return this.context; } diff --git a/packages/vmind/src/atom/chartCommand/index.ts b/packages/vmind/src/atom/chartCommand/index.ts index 2a86684f..8bbe0f8b 100644 --- a/packages/vmind/src/atom/chartCommand/index.ts +++ b/packages/vmind/src/atom/chartCommand/index.ts @@ -39,7 +39,7 @@ export class ChartCommandAtom extends BaseAtom }; }; -export const transferFieldInfo = (context: DataCleanCtx) => { +export const transferFieldInfo = (context: DataCleanCtx, fieldMapping?: Record) => { (context.fieldInfo || []).forEach(info => { if (!info.role || !info.location) { info.role = getRoleByFieldType(info.type); info.location = info.role as any; } + if (fieldMapping?.[info.fieldName]) { + info.alias = info.alias ?? fieldMapping[info.fieldName]?.alias; + } }); return context; }; @@ -393,13 +396,14 @@ export const mergeDataTable = (ctxA: DatasetFromText, ctxB: DatasetFromText) => const { dataTable: tableB, summary: summaryB, textRange: rangeB } = ctxB; const { strA, strB, commonStr } = longestCommonSubstringAtEdges(summaryA, summaryB); const newFieldInfo: FieldInfo = { - fieldName: 'mergedSummary', + fieldName: commonStr, + description: `${summaryA} and ${summaryB}`, role: ROLE.DIMENSION, type: DataType.STRING }; const newDataTable = [ - ...tableA.map(v => ({ ...v, mergedSummary: strA })), - ...tableB.map(v => ({ ...v, mergedSummary: strB })) + ...tableA.map(v => ({ ...v, [commonStr]: strA })), + ...tableB.map(v => ({ ...v, [commonStr]: strB })) ]; const textRange = rangeA && rangeB ? [rangeA[0], rangeB[1]] : null; return { diff --git a/packages/vmind/src/atom/dataExtraction/index.ts b/packages/vmind/src/atom/dataExtraction/index.ts index ca50925b..40ce1024 100644 --- a/packages/vmind/src/atom/dataExtraction/index.ts +++ b/packages/vmind/src/atom/dataExtraction/index.ts @@ -45,7 +45,7 @@ export class DataExtractionAtom extends BaseAtom d.dataItem[measureId]); const { trend, pValue, zScore } = originalMKTest(seriesDataset, 0.05, false); if (trend !== TrendType.NO_TREND) { + const startValue = seriesDataset[0]; + const endValue = seriesDataset[seriesDataset.length - 1]; seriesTrendInfo.push({ trend, pValue, zScore, measureId, - series + series, + info: { + startValue, + endValue, + change: trend === TrendType.INCREASING ? endValue / startValue - 1 : 1 - endValue / startValue + } }); } }); @@ -47,7 +55,7 @@ const abnormalTrendAlgo = (context: DataInsightExtractContext, options: Abnormal yField.forEach(measureId => { const measureOverallTrend = insights.find( - (i: { type: InsightType; fieldId: DataCell }) => i.type === InsightType.OverallTrend && i.fieldId === measureId + (i: { type: InsightType; fieldId?: DataCell }) => i.type === InsightType.OverallTrend && i?.fieldId === measureId ); if (measureOverallTrend) { const measureSeriesTrend = seriesTrendInfo.filter( @@ -61,7 +69,8 @@ const abnormalTrendAlgo = (context: DataInsightExtractContext, options: Abnormal fieldId: measureId, value: seriesTrend.trend, significant: 1 - seriesTrend.pValue, - seriesName: seriesTrend.series + seriesName: seriesTrend.series, + info: seriesTrend.info } as unknown as Insight) ); result.push(...seriesInsights); @@ -81,7 +90,8 @@ const abnormalTrendAlgo = (context: DataInsightExtractContext, options: Abnormal fieldId: measureId, value: dt.trend, significant: 1 - dt.pValue, - seriesName: dt.series + seriesName: dt.series, + info: dt.info } as unknown as Insight) ); result.push(...decreaseInsights); @@ -96,7 +106,8 @@ const abnormalTrendAlgo = (context: DataInsightExtractContext, options: Abnormal fieldId: measureId, value: it.trend, significant: 1 - it.pValue, - seriesName: it.series + seriesName: it.series, + info: it.info } as unknown as Insight) ); result.push(...increaseInsights); diff --git a/packages/vmind/src/atom/dataInsight/algorithms/drift/index.ts b/packages/vmind/src/atom/dataInsight/algorithms/drift/index.ts index 1bdd4211..586e5084 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/drift/index.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/drift/index.ts @@ -2,6 +2,7 @@ * adwin and pageHinkley all based on super parmaeters, so it's difficult to use without data information */ import { isArray } from '@visactor/vutils'; +import normalize from 'array-normalize'; import type { InsightAlgorithm } from '../../type'; import { InsightType, type DataInsightExtractContext, type Insight } from '../../type'; import { ChartType, type DataItem } from '../../../../types'; @@ -14,7 +15,7 @@ export interface PageHinkleyOptions { export const pageHinkleyFunc = (context: DataInsightExtractContext, options: PageHinkleyOptions) => { const result: Insight[] = []; const { seriesDataMap, cell } = context; - const { delta, lambda } = options; + const { delta, lambda } = options || {}; const { y: celly } = cell; const yField: string[] = isArray(celly) ? celly.flat() : [celly]; @@ -22,14 +23,15 @@ export const pageHinkleyFunc = (context: DataInsightExtractContext, options: Pag const dataset: { index: number; dataItem: DataItem }[] = seriesDataMap[group]; yField.forEach(field => { const pageHinkley = new PageHinkley(delta, lambda); - dataset.forEach(d => { - const isDrift = pageHinkley.setInput(d.dataItem[field] as number); + const normalizedDataset = normalize(dataset.map(v => v.dataItem[field] as number)); + normalizedDataset.forEach((d, index) => { + const isDrift = pageHinkley.setInput(d); if (isDrift) { result.push({ type: InsightType.Outlier, - data: [d], + data: [dataset[index]], fieldId: field, - value: d.dataItem[field], + value: d, significant: 1, seriesName: group } as unknown as Insight); diff --git a/packages/vmind/src/atom/dataInsight/algorithms/drift/pageHinkley.ts b/packages/vmind/src/atom/dataInsight/algorithms/drift/pageHinkley.ts index a7cbd6c1..61292d1a 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/drift/pageHinkley.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/drift/pageHinkley.ts @@ -1,19 +1,22 @@ +/** Page Hinkely algorithm, @todo remove trend influence */ export class PageHinkley { delta: number; lambda: number; alpha: number; sum: number; + minSum: number; xMean: number; num: number; changeDetected: boolean; - constructor(delta = 0.005, lambda = 50, alpha = 1 - 0.0001) { + constructor(delta = 0.006, lambda = 0.8, alpha = 0.95) { this.delta = delta; this.lambda = lambda; this.alpha = alpha; this.sum = 0; this.xMean = 0; this.num = 0; + this.minSum = 0; this.changeDetected = false; } @@ -21,6 +24,7 @@ export class PageHinkley { this.num = 0; this.xMean = 0; this.sum = 0; + this.minSum = 0; } setInput(x: number) { @@ -32,8 +36,11 @@ export class PageHinkley { this.num += 1; this.xMean = (x + this.xMean * (this.num - 1)) / this.num; this.sum = this.sum * this.alpha + (x - this.xMean - this.delta); + if (this.sum < this.minSum) { + this.minSum = this.sum; + } - this.changeDetected = this.sum > this.lambda; + this.changeDetected = this.sum - this.minSum > this.lambda; if (this.changeDetected) { this._resetParams(); } diff --git a/packages/vmind/src/atom/dataInsight/algorithms/index.ts b/packages/vmind/src/atom/dataInsight/algorithms/index.ts index ca86f2f0..f1e31112 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/index.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/index.ts @@ -15,6 +15,7 @@ import type { RevisedInsightParams } from './revised'; import { filterCorrelationInsight, filterInsightByType, mergePointInsight } from './revised'; import { DifferenceAlg } from './outlier/difference'; import { PageHinkleyAlg } from './drift'; +import { generateInsightTemplate } from './template'; const algorithmMapping = { [AlgorithmType.OverallTrending]: { @@ -96,13 +97,19 @@ export const getInsights = (context: DataInsightExtractContext, options: DataIns algorithms.sort((a, b) => algorithmMapping[a].priority - algorithmMapping[b].priority); algorithms.forEach(key => { const algoInfo = algorithmMapping[key].info; - const { chartType: algoSupportedChartType, algorithmFunction, forceChartType } = algoInfo; + const { chartType: algoSupportedChartType, algorithmFunction, forceChartType, name, canRun } = algoInfo; if ( (!forceChartType || forceChartType.includes(chartType)) && - (!isLimitedbyChartType || !algoSupportedChartType || algoSupportedChartType.includes(chartType)) + (!isLimitedbyChartType || !algoSupportedChartType || algoSupportedChartType.includes(chartType)) && + (!canRun || canRun(insightAlgorithmContext)) ) { const res = algorithmFunction(insightAlgorithmContext, options?.algorithmOptions?.[key]); - insights.push(...res); + insights.push( + ...res.map(v => ({ + ...v, + name + })) + ); } }); @@ -125,7 +132,7 @@ export const getInsights = (context: DataInsightExtractContext, options: DataIns const significant2 = b.significant ?? -1; return significant2 - significant1; }); - const finalInsights = maxNum ? revisedInsights.slice(0, maxNum) : revisedInsights; + const finalInsights = generateInsightTemplate(maxNum ? revisedInsights.slice(0, maxNum) : revisedInsights, context); - return { insights: finalInsights }; + return finalInsights; }; diff --git a/packages/vmind/src/atom/dataInsight/algorithms/majorityValue/index.ts b/packages/vmind/src/atom/dataInsight/algorithms/majorityValue/index.ts index 86fdf224..3dd6dfd5 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/majorityValue/index.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/majorityValue/index.ts @@ -1,14 +1,14 @@ import { isArray, isNumber } from '@visactor/vutils'; import type { InsightAlgorithm } from '../../type'; import { InsightType, type DataInsightExtractContext, type Insight } from '../../type'; -import { type DataItem } from '../../../../types'; +import { ChartType, type DataItem } from '../../../../types'; const getMajorityInGroup = ( dataset: { index: number; dataItem: DataItem }[], measureId: string | number, seriesId: string | number, dimensionId: string | number, - threshold = 0.8 + threshold = 0.85 ) => { const result: Insight[] = []; const sum = dataset.reduce((prev, cur) => { @@ -70,12 +70,13 @@ const calcMajorityValue = (context: DataInsightExtractContext, options: Majority const dimensionInsights = getMajorityInGroup(dimensionDataset, yField[0], groupField, xField, threshold); result.push(...dimensionInsights); }); - return []; - // return result; + // return []; + return result; }; export const LineChartMajorityValue: InsightAlgorithm = { name: 'majorityValue', + forceChartType: [ChartType.BarChart, ChartType.AreaChart], insightType: InsightType.MajorityValue, algorithmFunction: calcMajorityValue }; diff --git a/packages/vmind/src/atom/dataInsight/algorithms/overallTrending/index.ts b/packages/vmind/src/atom/dataInsight/algorithms/overallTrending/index.ts index 090d1430..ae3a7352 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/overallTrending/index.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/overallTrending/index.ts @@ -1,12 +1,13 @@ import { isArray, isNumber } from '@visactor/vutils'; -import { originalMKTest, TrendType } from '../statistics'; +import { longestTrendInterval, originalMKTest, TrendType } from '../statistics'; import type { InsightAlgorithm } from '../../type'; import { InsightType, type DataInsightExtractContext, type Insight } from '../../type'; import { ChartType, type DataItem } from '../../../../types'; +import { isPercentChart, isStackChart } from '../../utils'; -const sumDimensionValues = (dataset: DataItem[], measureId: string | number) => { +const sumDimensionValues = (dataset: DataItem[], measureId: string | number, getValue = (v: number) => Math.abs(v)) => { const sum = dataset.reduce((prev, cur) => { - const value = isNumber(cur[measureId] as number) ? Math.abs(cur[measureId] as number) : 0; + const value = isNumber(cur[measureId] as number) ? getValue(cur[measureId] as number) : 0; return prev + value; }, 0); return sum; @@ -18,18 +19,21 @@ export interface OverallTrendingOptions { } const overallTrendingAlgo = (context: DataInsightExtractContext, options: OverallTrendingOptions) => { - const { dimensionDataMap, cell } = context; + const { dimensionDataMap, cell, seriesDataMap } = context; const { alpha = 0.05, calcScope = false } = options || {}; const result: Insight[] = []; const { y: celly } = cell; const yField: string[] = isArray(celly) ? celly.flat() : [celly]; + const onlyOneSeries = Object.keys(seriesDataMap).length === 1; yField.forEach(measureId => { - const overallDataset = Object.keys(dimensionDataMap).map(dimension => { + const dimensionValues = Object.keys(dimensionDataMap); + const overallDataset = dimensionValues.map(dimension => { const dimensionDataset = dimensionDataMap[dimension].map((d: any) => d.dataItem); - return sumDimensionValues(dimensionDataset, measureId); + return sumDimensionValues(dimensionDataset, measureId, onlyOneSeries ? v => v : undefined); }); const { trend, pValue, zScore, slope, intercept } = originalMKTest(overallDataset, alpha, calcScope); if (trend !== TrendType.NO_TREND) { + const { length, start, end } = longestTrendInterval(overallDataset); result.push({ type: InsightType.OverallTrend, fieldId: measureId, @@ -37,7 +41,15 @@ const overallTrendingAlgo = (context: DataInsightExtractContext, options: Overal significant: 1 - pValue, info: { slope, - intercept + intercept, + length, + start, + end, + change: overallDataset[end] / overallDataset[start] - (trend === TrendType.INCREASING ? 1 : 0), + startDimValue: dimensionValues[start], + endDimValue: dimensionValues[end], + startValue: overallDataset[start], + endValue: overallDataset[end] } } as unknown as Insight); } @@ -46,9 +58,15 @@ const overallTrendingAlgo = (context: DataInsightExtractContext, options: Overal return result; }; +const canRun = (context: DataInsightExtractContext) => { + const { seriesDataMap } = context; + return (isStackChart(context.spec) && !isPercentChart(context.spec)) || Object.keys(seriesDataMap).length === 1; +}; + export const OverallTrending: InsightAlgorithm = { name: 'overallTrending', - chartType: [ChartType.DualAxisChart, ChartType.LineChart, ChartType.BarChart, ChartType.AreaChart], + forceChartType: [ChartType.DualAxisChart, ChartType.LineChart, ChartType.BarChart, ChartType.AreaChart], insightType: InsightType.OverallTrend, + canRun, algorithmFunction: overallTrendingAlgo }; diff --git a/packages/vmind/src/atom/dataInsight/algorithms/revised.ts b/packages/vmind/src/atom/dataInsight/algorithms/revised.ts index 43c7a33d..e484bfad 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/revised.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/revised.ts @@ -62,9 +62,9 @@ const getBandInsightByOutliear = (context: DataInsightExtractContext, outliearFi if (band.length > 1) { abnormalBand.push({ type: InsightType.AbnormalBand, + name: InsightType.AbnormalBand, data: band.map(v => v.content.insight.data[0]), - fieldId, - text: [], + seriesName: fieldId, value: null, significant: band.length }); @@ -105,6 +105,12 @@ export const mergePointInsight = ( } outliear[key].push(insight); }); + const majorityValueInsight = filterInsight(insights, InsightType.MajorityValue).filter(insight => { + const { data } = insight; + const seriesName = insight?.seriesName as DataCell; + const key = `${data[0].index}-&&&-${seriesName}`; + return !outliear[key]; + }); const pairOutlier = filterPairInsight(insights, filterOutliearInsight); const { abnormalBand, bandInsightKeys } = getBandInsightByOutliear(context, outliearFieldMapping); @@ -120,7 +126,8 @@ export const mergePointInsight = ( ...insightCtx, [InsightType.Outlier]: outliearInsight, [InsightType.PairOutlier]: pairOutlier, - [InsightType.AbnormalBand]: abnormalBand + [InsightType.AbnormalBand]: abnormalBand, + [InsightType.MajorityValue]: majorityValueInsight }; }; @@ -155,6 +162,6 @@ export const filterCorrelationInsight = (insightCtx: RevisedInsightParams) => { export const filterInsightByType = (insightCtx: RevisedInsightParams, type: InsightType) => { return { ...insightCtx, - [type]: filterInsight(insightCtx.insights, type) + [type]: insightCtx?.[type]?.length ? insightCtx?.[type] : filterInsight(insightCtx.insights, type) }; }; diff --git a/packages/vmind/src/atom/dataInsight/algorithms/statistics.ts b/packages/vmind/src/atom/dataInsight/algorithms/statistics.ts index c0d7e0b9..fd862bee 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/statistics.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/statistics.ts @@ -157,3 +157,37 @@ export function studentTQuantile(x: number, degree: number) { export function coefficientVariation(data: number[]) { return jStat.coeffvar(data); } + +export function longestTrendInterval(data: number[]) { + if (data.length === 0) { + return { length: 0, start: -1, end: -1 }; + } + + let maxLength = 1; + let currentLength = 1; + let start = 0; + let end = 0; + let maxStart = 0; + let maxEnd = 0; + let trend = 0; // 0: no trend, 1: increasing, -1: decreasing + + for (let i = 1; i <= data.length; i++) { + const currentTrend = i === data.length ? null : Math.sign(data[i] - data[i - 1]); + + if (currentTrend === trend) { + currentLength++; + end = i; + } else { + if (currentLength > maxLength) { + maxLength = currentLength; + maxStart = start; + maxEnd = end; + } + trend = currentTrend; + currentLength = currentTrend !== 0 ? 2 : 1; + start = i - 1; + end = i; + } + } + return { length: maxLength, start: maxStart, end: maxEnd }; +} diff --git a/packages/vmind/src/atom/dataInsight/algorithms/template.ts b/packages/vmind/src/atom/dataInsight/algorithms/template.ts new file mode 100644 index 00000000..fc7880aa --- /dev/null +++ b/packages/vmind/src/atom/dataInsight/algorithms/template.ts @@ -0,0 +1,284 @@ +import { isArray } from '@visactor/vutils'; +import { ChartType } from '../../../types'; +import type { DataCell, FieldInfo } from '../../../types'; +import { InsightType, type DataInsightExtractContext, type Insight } from '../type'; +import { DEFAULT_SERIES_NAME } from '../const'; +import { TrendType } from './statistics'; + +const getFieldIdInCell = (cellField: any): string => { + return isArray(cellField) ? cellField[0] : cellField; +}; +const getFieldInfoById = (fieldInfo: FieldInfo[], fieldId: string) => { + return fieldInfo.find(info => info.fieldName === fieldId); +}; +const isEmptySeries = (seriesName: any) => !seriesName || seriesName === DEFAULT_SERIES_NAME; + +const getOutlierTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const { seriesName, data, value, fieldId } = insight; + const { fieldInfo, cell, chartType } = ctx; + const xFieldId = getFieldIdInCell(cell.x); + const seriesField = getFieldIdInCell(cell?.color); + if ([ChartType.ScatterPlot].includes(chartType)) { + return { + content: isEmptySeries(seriesName) ? '(${b}, ${c})上存在异常点' : '${a}在(${b}, ${c})上存在异常点', + variables: { + ...(isEmptySeries(seriesName) + ? {} + : { + a: { + value: seriesName as string, + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + } + }), + b: { + value: data?.[0]?.dataItem?.[fieldId[0]], + fieldName: getFieldInfoById(fieldInfo, fieldId[0])?.alias ?? fieldId[0] + }, + c: { + value: data?.[0]?.dataItem?.[fieldId[1]], + fieldName: getFieldInfoById(fieldInfo, fieldId[1])?.alias ?? fieldId[1] + } + } + }; + } + return { + content: isEmptySeries(seriesName) ? '${b}上有异常值,值为${c}' : '${a}在${b}上有异常值，值为${c}', + variables: { + ...(isEmptySeries(seriesName) + ? {} + : { + a: { + value: seriesName as string, + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + } + }), + b: { + value: data?.[0]?.dataItem?.[xFieldId], + fieldName: getFieldInfoById(fieldInfo, xFieldId)?.alias ?? xFieldId + }, + c: { + value, + fieldName: getFieldInfoById(fieldInfo, fieldId as string)?.alias ?? fieldId + } + } + }; +}; + +const getTurnPointTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const res = getOutlierTemplate(insight, ctx); + return { + content: res.content.replaceAll('异常值', '拐点'), + variables: res.variables + }; +}; + +const getExtremeTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const res = getOutlierTemplate(insight, ctx); + return { + content: res.content.replaceAll('异常值', '极值'), + variables: res.variables + }; +}; + +const getMajorityTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const { seriesName, fieldId, info } = insight; + const { fieldInfo, cell } = ctx; + const { ratio, dimensionName } = info; + const xFieldId = getFieldIdInCell(cell.x); + const seriesField = getFieldIdInCell(cell?.color); + return { + content: '${a}在${b}的占比贡献度显著，占比高达${c}', + variables: { + a: { + value: seriesName as string, + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + }, + b: { + value: dimensionName, + fieldName: getFieldInfoById(fieldInfo, xFieldId)?.alias ?? xFieldId + }, + c: { + value: ratio, + fieldName: getFieldInfoById(fieldInfo, fieldId as string)?.alias ?? fieldId + } + } + }; +}; + +const getAbnormalBandTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const { seriesName, data } = insight; + const { fieldInfo, cell } = ctx; + const xFieldId = getFieldIdInCell(cell.x); + const seriesField = getFieldIdInCell(cell?.color); + return { + content: isEmptySeries(seriesName) ? '${b}-${c}之间存在异常区间' : '${a}在${b}-${c}之间存在异常区间', + variables: { + ...(isEmptySeries(seriesName) + ? {} + : { + a: { + value: seriesName as string, + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + } + }), + b: { + value: data[0].dataItem[xFieldId], + fieldName: getFieldInfoById(fieldInfo, xFieldId)?.alias ?? xFieldId + }, + c: { + value: data[data.length - 1].dataItem[xFieldId], + fieldName: getFieldInfoById(fieldInfo, xFieldId)?.alias ?? xFieldId + } + } + }; +}; + +const getOverallTrendTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const { value, info } = insight; + const { fieldInfo, cell } = ctx; + const { startDimValue, endDimValue, change } = info; + const xFieldId = getFieldIdInCell(cell.x); + return { + content: + '数据整体呈${a}趋势，其中在${b}-${c}连续${a},数据' + + (value === TrendType.INCREASING ? '增长了' : '减少至') + + '${d}', + variables: { + a: { + value: value === TrendType.INCREASING ? '上升' : '下降', + fieldName: null as any, + icon: value === TrendType.INCREASING ? 'ascendTrend' : 'descendTrend' + }, + b: { + value: startDimValue, + fieldName: getFieldInfoById(fieldInfo, xFieldId)?.alias ?? xFieldId + }, + c: { + value: endDimValue, + fieldName: getFieldInfoById(fieldInfo, xFieldId)?.alias ?? xFieldId + }, + d: { + value: (+change * 100).toFixed(1) + '%', + fieldName: null as any + } + } + }; +}; + +const getAbnormalTrendTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const { seriesName, value, info } = insight; + const { fieldInfo, cell } = ctx; + const seriesField = getFieldIdInCell(cell?.color); + return { + content: '${a}趋势异常，呈${b}趋势，整体${b}了${c}', + variables: { + a: { + value: seriesName as string, + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + }, + b: { + value: value === TrendType.INCREASING ? '上升' : '下降', + fieldName: null as any, + icon: value === TrendType.INCREASING ? 'ascendTrend' : 'descendTrend' + }, + c: { + value: (+info.change * 100).toFixed(1) + '%', + fieldName: null as any + } + } + }; +}; + +const getCorrelationTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const { seriesName, value, info, name } = insight; + const { fieldInfo, cell } = ctx; + const { correlationType } = info || {}; + const seriesField = getFieldIdInCell(cell?.color); + if (name === 'spearman') { + return { + content: '${a}和${b}呈${c}相关', + variables: { + a: { + value: (seriesName as DataCell[])[0], + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + }, + b: { + value: (seriesName as DataCell[])[1], + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + }, + c: { + value: correlationType === 'positive' ? '正' : '负', + fieldName: null as any, + icon: value === TrendType.INCREASING ? 'ascendTrend' : 'descendTrend' + } + } + }; + } + return { + content: '${a}在xy上呈线性相关', + variables: { + a: { + value: seriesName as string, + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + } + } + }; +}; + +const getVolatilityTemplate = (insight: Insight, ctx: DataInsightExtractContext) => { + const { seriesName } = insight; + const { fieldInfo, cell } = ctx; + const seriesField = getFieldIdInCell(cell?.color); + return { + content: isEmptySeries(seriesName) ? '数据呈周期性波动' : '${a}呈周期性波动', + variables: isEmptySeries(seriesName) + ? {} + : { + a: { + value: seriesName as string, + fieldName: getFieldInfoById(fieldInfo, seriesField)?.alias ?? seriesField + } + } + }; +}; + +export const generateInsightTemplate = (insights: Insight[], ctx: DataInsightExtractContext) => { + for (let i = 0; i < insights.length; i++) { + const { type } = insights[i]; + switch (type) { + case InsightType.Outlier: + insights[i].textContent = getOutlierTemplate(insights[i], ctx); + break; + case InsightType.TurningPoint: + insights[i].textContent = getTurnPointTemplate(insights[i], ctx); + break; + case InsightType.MajorityValue: + insights[i].textContent = getMajorityTemplate(insights[i], ctx); + break; + case InsightType.AbnormalBand: + insights[i].textContent = getAbnormalBandTemplate(insights[i], ctx); + break; + case InsightType.OverallTrend: + insights[i].textContent = getOverallTrendTemplate(insights[i], ctx); + break; + case InsightType.AbnormalTrend: + insights[i].textContent = getAbnormalTrendTemplate(insights[i], ctx); + break; + case InsightType.Correlation: + insights[i].textContent = getCorrelationTemplate(insights[i], ctx); + break; + case InsightType.Volatility: + insights[i].textContent = getVolatilityTemplate(insights[i], ctx); + break; + case InsightType.ExtremeValue: + insights[i].textContent = getExtremeTemplate(insights[i], ctx); + break; + default: + insights[i].textContent = { + content: `数据含有${insights[i].type}的见解` + }; + break; + } + } + return insights; +}; diff --git a/packages/vmind/src/atom/dataInsight/algorithms/turningPoint/index.ts b/packages/vmind/src/atom/dataInsight/algorithms/turningPoint/index.ts index 2fea5177..d05a81c0 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/turningPoint/index.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/turningPoint/index.ts @@ -66,9 +66,8 @@ export const TurningPoint: InsightAlgorithm = { ChartType.LineChart, ChartType.BarChart, ChartType.AreaChart, - ChartType.RadarChart, ChartType.WaterFallChart ], - insightType: InsightType.ExtremeValue, + insightType: InsightType.TurningPoint, algorithmFunction: turningPointAlgo }; diff --git a/packages/vmind/src/atom/dataInsight/algorithms/volatility/index.ts b/packages/vmind/src/atom/dataInsight/algorithms/volatility/index.ts index 16b18440..f4bc555b 100644 --- a/packages/vmind/src/atom/dataInsight/algorithms/volatility/index.ts +++ b/packages/vmind/src/atom/dataInsight/algorithms/volatility/index.ts @@ -42,6 +42,7 @@ const volatilityAlgo = (context: DataInsightExtractContext, optioins: Volatility return result; }; +/** @todo @czx add fluctuation period */ export const Volatility: InsightAlgorithm = { name: 'volatility', chartType: [ChartType.LineChart, ChartType.DualAxisChart, ChartType.BarChart, ChartType.AreaChart], diff --git a/packages/vmind/src/atom/dataInsight/dataProcess.ts b/packages/vmind/src/atom/dataInsight/dataProcess.ts index e4cdac1b..71b53292 100644 --- a/packages/vmind/src/atom/dataInsight/dataProcess.ts +++ b/packages/vmind/src/atom/dataInsight/dataProcess.ts @@ -1,4 +1,10 @@ -import { getCellFromSpec, getChartTypeFromSpec, getDatasetFromSpec, revisedCell } from './utils'; +import { + getCellFromSpec, + getChartTypeFromSpec, + getDatasetFromSpec, + getFieldMappingFromSpec, + revisedCell +} from './utils'; import type { DataCell } from '../../types'; import { type DataInsightCtx } from '../../types'; import { isArray } from '@visactor/vutils'; @@ -22,13 +28,18 @@ export const extractDataFromContext = (context: DataInsightCtx) => { //no dataset in the input, extract from spec dataset = getDatasetFromSpec(spec); } + const specFieldMapping = getFieldMappingFromSpec(spec); - let fieldInfo = transferFieldInfo({ - fieldInfo: inputFieldInfo - })?.fieldInfo; + let fieldInfo = inputFieldInfo; if (!fieldInfo || fieldInfo?.length === 0) { fieldInfo = getFieldInfoFromDataset(dataset); } + fieldInfo = transferFieldInfo( + { + fieldInfo + }, + specFieldMapping + )?.fieldInfo; dataset = transferMeasureInTable(dataset, fieldInfo); const cell = revisedCell(getCellFromSpec(spec, chartType), dataset); diff --git a/packages/vmind/src/atom/dataInsight/index.ts b/packages/vmind/src/atom/dataInsight/index.ts index 6fa53fcb..27cfda02 100644 --- a/packages/vmind/src/atom/dataInsight/index.ts +++ b/packages/vmind/src/atom/dataInsight/index.ts @@ -6,11 +6,13 @@ import { merge } from '@visactor/vutils'; import { extractDataFromContext } from './dataProcess'; import { AlgorithmType } from './type'; import { getInsights } from './algorithms'; +import type { LLMMessage } from '../../types/llm'; +import { getPolishPrompt } from './prompt'; export class DataInsightAtom extends BaseAtom { - name = AtomName.DATA_CLEAN; + name = AtomName.DATA_INSIGHT; - isLLMAtom: boolean = false; + isLLMAtom: boolean = true; constructor(context: DataInsightCtx, option: DataInsightOptions) { super(context, option); @@ -20,7 +22,8 @@ export class DataInsightAtom extends BaseAtom ({ + type: insight.type, + content: insight.textContent?.content, + variables: insight.textContent?.variables + })) + }) + }, + ...addtionContent + ]; + } + + protected parseLLMContent(resJson: any): DataInsightCtx { + const { results } = resJson; + if (!results) { + console.error('Insights polish error in LLM'); + return { + ...this.context, + error: 'Insights polish error in LLM' + }; + } + const newInsights = this.context.insights.map((insight, index) => ({ + ...insight, + textContent: { + content: results[index] || insight.textContent?.content, + variables: insight.textContent?.variables + } + })); + return { + ...this.context, + insights: newInsights + }; + } + protected runBeforeLLM(): DataInsightCtx { + this.isLLMAtom = true; const dataInfo = extractDataFromContext(this.context); - const { insights } = getInsights(dataInfo, this.options); + const insights = getInsights( + { + ...dataInfo, + spec: this.context.spec + }, + this.options + ); const newContext = { ...this.context, + chartType: dataInfo.chartType, + fieldInfo: dataInfo.fieldInfo, insights }; this.updateContext(newContext); + if (insights.length === 0) { + this.isLLMAtom = false; + } return this.context; } } diff --git a/packages/vmind/src/atom/dataInsight/prompt.ts b/packages/vmind/src/atom/dataInsight/prompt.ts new file mode 100644 index 00000000..e4693dd7 --- /dev/null +++ b/packages/vmind/src/atom/dataInsight/prompt.ts @@ -0,0 +1,60 @@ +/* eslint-disable max-len */ +export const getPolishPrompt = (language: 'chinese' | 'english') => { + return `# Task +You are a visualization and data analysis expert, and you possess excellent language polishing skills. The user extracted some insights from a chart.Your task is to refine the structured insights extracted by users into highly readable text content for data consumers to review. + +# User Import +\`\`\`typescript +{ + insights: { // user's insights + type: InsightType, + content: string; // text template of insight + variables: Record + }[] +} +\`\`\` +## InsightType +InsightType is an enum that represents the type of insight.Specific details are as follows: + +- Outlier: Outlier Point in Chart +- Turning Point: Turning point in the chart, where the data distribution changes. +- Extreme Value: Extreme value in the chart, such as the maximum or minimum value. +- Majority Value: A value with a large contribution in a group. +- Abnormal Band: Only for time series data, indicating an overall anomaly in a specific time interval. +- Overall Trend: Overall trend of the data. +- Abnormal Trend: A particular series has an anomalous trend, differing from the overall trend. +- Correlation: Correlation between two or more variables. +- Volatility: A series in the chart has cyclical fluctuations. + +## Content of Insight +Content is the current insight text template, which includes placeholders like '\\\${a}'. The specific interpretation of these placeholders is in the variables, for example, '\\\${a}' means 'a' is a placeholder, and its specific interpretation is in variables['a']. +## Variables of Insight +Variables contain the specific interpretation of the placeholders, where the key represents the specific placeholder.The specific interpretation is as follows: +\`\`\`typescript +{ + value: string; // the specific value of the placeholder + fieldName: string; // the fieldName of the specific value; Fields are the mapped fields in the chart, such as the x-axis, y-axis, or color. +} +\`\`\` + +# Response and Requirments +Output the polished result for each insight text. +1. Strictly define the type of return format, use JSON format to reply。 +2. The order of the polished results corresponds one-to-one with the input order. +3. KEEP the placeholders UNCHANGED in the polished results. +4. The final result must be answered in ${language}. + +The return format is as follows: +\`\`\` +{results: string[]} +\`\`\` +# Example +## Input +\`\`\` +{"insights":[{"type":"outlier","content":"\${a}在\${b}上有异常值，值为\${c}","variables":{"a":{"value":"西南","fieldName":"区域"},"b":{"field":"2024-07-07","fieldName":"日期"},"c":{"value":1000,"fieldName":"销售额"}}},{"type":"abnormal_band","content":"\${a}在\${b}-\${c}之间存在异常区间","variables":{"a":{"value":"西南","fieldName":"区域"},"b":{"field":"2024-07-07","fieldName":"日期"},"c":{"value":"2024-87-07","fieldName":"日期"}}}]} +\`\`\` +## Response +\`\`\` +{"results": ["\${a}地区的销售额在\${b}存在异常值，值为\${}。", "在\${b}至\${c}期间，\${a}地区的数据表现异常。"]} +\`\`\``; +}; diff --git a/packages/vmind/src/atom/dataInsight/type.ts b/packages/vmind/src/atom/dataInsight/type.ts index 69411000..74fd6f09 100644 --- a/packages/vmind/src/atom/dataInsight/type.ts +++ b/packages/vmind/src/atom/dataInsight/type.ts @@ -57,23 +57,28 @@ export enum InsightType { Volatility = 'volatility' } +export interface InsightTextContent { + value: DataCell; + fieldName: string; + color?: string; + icon?: 'ratio' | 'ascendTrend' | 'descendTrend' | string; +} + export interface Insight { + name: string; // algorithm name type: InsightType; // Insight type data: { // Abnormal data points index: number; // The index of the data point in the dataset dataItem: DataItem; // Data item }[]; - measureId?: string | number; //abnormal measure id - fieldId: string | number; //abnormal field id + fieldId?: string; //abnormal field id seriesName?: DataCell | DataCell[]; //series name - text: { + textContent?: { // The text explanation of the insight, generated by LLM, can be used for chart annotation. content: string; // Text content - color?: string; // Text color - icon?: string; // Icon - type?: 'number' | 'text' | 'percentage'; - }[]; // The text array is composed of text fragments, and all elements in the array are combined to form a complete text. + variables?: Record; + }; // The text array is composed of text fragments, and all elements in the array are combined to form a complete text. value?: number | string; // The specific value of the insight significant: number; // The prominence of insights, used for sorting info?: { [key: string]: any }; //additional information about this insight @@ -124,5 +129,6 @@ export interface DataInsightExtractContext { seriesDataMap: DimValueDataMap; chartType: ChartType; cell: Cell; + spec: any; insights?: Insight[]; } diff --git a/packages/vmind/src/atom/dataInsight/utils.ts b/packages/vmind/src/atom/dataInsight/utils.ts index d5810699..d6dd0739 100644 --- a/packages/vmind/src/atom/dataInsight/utils.ts +++ b/packages/vmind/src/atom/dataInsight/utils.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { isArray, uniqArray } from '@visactor/vutils'; +import { isArray, merge, uniqArray } from '@visactor/vutils'; import type { Cell, DataCell, DataTable } from '../../types'; import { ChartType } from '../../types'; @@ -71,6 +71,19 @@ export const getDatasetFromSpec = (spec: any) => { return spec.data.map((d: any) => d.values).flat(2); }; +export const getFieldMappingFromSpec = (spec: any) => { + if (!spec) { + return {}; + } + let res = {}; + spec.data.forEach((d: any) => { + if (d?.fields) { + res = merge(res, d.fields); + } + }); + return res; +}; + /** * Auto generate cell from a spec template * @param spec @@ -125,9 +138,11 @@ export const getCellFromSpec = (spec: any, chartType?: string) => { if ('common' === type) { //dual-axis chart const series = spec.series ?? []; + const seriesField = uniqArray(series.map((s: any) => s?.seriesField).filter((v: string) => !!v)); return { x: series[0]?.xField, - y: uniqArray([series[0]?.yField, series[1]?.yField].filter(Boolean)) + y: uniqArray([series[0]?.yField, series[1]?.yField].filter(Boolean)), + color: seriesField?.length === 1 ? seriesField[0] : undefined }; } if (type === 'wordCloud') { @@ -169,3 +184,12 @@ export const revisedCell = (cell: Cell, dataset: DataTable) => { } return cell; }; + +export const isStackChart = (spec: any) => { + const { stack, xField = [], seriesField } = spec || {}; + return stack && !(isArray(xField) && xField.length === 2 && seriesField && xField[1] !== seriesField); +}; + +export const isPercentChart = (spec: any) => { + return !!spec?.percent; +}; diff --git a/packages/vmind/src/atom/type.ts b/packages/vmind/src/atom/type.ts index 781d7d51..abe3b9e1 100644 --- a/packages/vmind/src/atom/type.ts +++ b/packages/vmind/src/atom/type.ts @@ -7,6 +7,8 @@ export interface BaseOptions { llm?: LLMManage; /** show llm thoughs or not */ showThoughts?: boolean; + /** answer language */ + language?: 'chinese' | 'english'; } export interface DataExtractionOptions extends BaseOptions { diff --git a/packages/vmind/src/types/atom.ts b/packages/vmind/src/types/atom.ts index 8d6302df..e7aa0655 100644 --- a/packages/vmind/src/types/atom.ts +++ b/packages/vmind/src/types/atom.ts @@ -158,4 +158,6 @@ export interface DataInsightCtx extends BaseContext { insights: Insight[]; /** chartType of vchart */ vChartType?: string; + /** chartType */ + chartType?: ChartType; } diff --git a/packages/vmind/src/types/base.ts b/packages/vmind/src/types/base.ts index a534db85..5b5fdf23 100644 --- a/packages/vmind/src/types/base.ts +++ b/packages/vmind/src/types/base.ts @@ -28,6 +28,8 @@ export enum ROLE { export interface FieldInfo { /** name of field */ fieldName: string; + /** alias of field */ + alias?: string; /** description of field */ description?: string; /** field type, eg: time / category / numerical */ From 22d80a47d6c99b3bfa68cf93a3b142ccfe64548f Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Thu, 24 Oct 2024 09:09:44 +0800 Subject: [PATCH 055/128] feat: update mock data to do case study --- .../vmind/__tests__/browser/src/constants/capcutData.ts | 3 +++ .../experiment/src/pages/ChartGenerator/caseStudy.tsx | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/constants/capcutData.ts b/packages/vmind/__tests__/browser/src/constants/capcutData.ts index 90296254..d531e793 100644 --- a/packages/vmind/__tests__/browser/src/constants/capcutData.ts +++ b/packages/vmind/__tests__/browser/src/constants/capcutData.ts @@ -665,5 +665,8 @@ iPhone在哪打广告， 或者有人在哪搜iPhone相关的关键词， 哎， 这次推出的芯片和光刻机 可能不是行业里最先进的， 也不能一步到位完全取代西方技术。 我们需要正视差距， 尊重事实。 我们也需要更多的企业像长城一样， 勇于踏出自主研发的第一步。` + }, + { + text: `当然了，除了俄罗斯之外，其他国家也在买中国汽车，比如墨西哥。 2013 年，墨西哥所有销售汽车中有 25% 来自中国，而在 6 年前这个数字为0。澳大利亚也在不断买中国汽车，最受澳大利亚欢迎的中国汽车品牌是名爵，去年卖了 5.8 万辆。在新能源车市场，比亚迪则占据了澳大利亚的新能源汽车 14% 份额，位于第二名。当然，这里也不得不提一下第一名，那就是特斯拉市场份额高达53%，在东南亚市场，中国车企业销量在 2013 年同样实现小幅上升，最典型的就是泰国，在泰国的新能源车市场，中国品牌占据了 80% 的份额，比如比亚迪的原 plus 就是泰国的新能源车爆款，那到底是什么原因让中国汽车爆卖呢？基本还可以总结为三方面原因，首先是全球疫情爆发，由于中国汽车的供应链完善，疫情期间仍能维持稳定生产，而日韩这些过去的出口大户受疫情影响，芯片、钢材、橡胶等关键原材料短缺，不仅汽车产能下降，而且成本升高，这就让中国汽车更具性价比。而随着中国国内新能源汽车市场越来越卷出海，成为不少中国车企的选择，比如比亚迪 2023 年进入全球 58 个国家和地区，出口汽车 24 万辆，是上一年度的 3.34 倍。在泰国新能源车市场，比亚迪单独占到了 40% 的市场份额，是名副其实的泰国新能源汽车销冠。而且中国新能源汽车并非只是具备成本优势，汽车与 AI 互联网融合的智能化更是中国车企的拿手好戏。从豪华配置到智能大屏，从外观设计到内饰比拼，这让中国新能源汽车的溢价能力明显变高。2019 年中国新能源汽车平均出口价格每量只有 5, 000 美元， 2022 年涨到了 2.2 万美元。比如比亚迪汉在欧洲发布时价格接近 50 万人民币，是国内售价的两倍多。在泰国、以色列、新西兰等多个国家，比亚迪也已经是新能源汽车的销售冠军。不过，中国汽车征服海外虽然是一部励志爽门，但其实有不少挑战。` } ]; diff --git a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx index 61a7192d..c909a353 100644 --- a/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx +++ b/packages/vmind/__tests__/experiment/src/pages/ChartGenerator/caseStudy.tsx @@ -49,23 +49,23 @@ export function ChartGeneratorResult() { .forEach((v, index) => { timeCostList.push( { - index, + index: index + 1, type: 'all', y: v.timeCost }, { - index, + index: index + 1, type: 'extraction', y: v.extractionCost }, { - index, + index: index + 1, type: 'chart', y: v.generationCost } ); textLengthList.push({ - index, + index: index + 1, type: 'textLength', y: v.context.text.length }); From 6a3ec9bb9cd884fcee1a64bdeb60a0a6148a3885 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Thu, 24 Oct 2024 09:12:14 +0800 Subject: [PATCH 056/128] feat: update test browser of insights --- .../src/pages/Insight/ChartPreview.tsx | 20 +-- .../browser/src/pages/Insight/DataInput.tsx | 115 ++++++++---------- .../browser/src/pages/Insight/Insight.tsx | 2 +- 3 files changed, 64 insertions(+), 73 deletions(-) diff --git a/packages/vmind/__tests__/browser/src/pages/Insight/ChartPreview.tsx b/packages/vmind/__tests__/browser/src/pages/Insight/ChartPreview.tsx index f8a343f2..7cce06f4 100644 --- a/packages/vmind/__tests__/browser/src/pages/Insight/ChartPreview.tsx +++ b/packages/vmind/__tests__/browser/src/pages/Insight/ChartPreview.tsx @@ -1,14 +1,13 @@ -import React, { useState, useCallback, useEffect, useRef } from 'react'; +import React, { useState, useEffect, useRef } from 'react'; import './index.scss'; import { Button, Input, Card, Space, Modal, Spin } from '@arco-design/web-react'; import VChart from '@visactor/vchart'; import { isNil } from '@visactor/vutils'; - -const TextArea = Input.TextArea; +import type { Insight } from '../../../../../src/atom/dataInsight/type'; type IPropsType = { spec: any; - insights: any; + insights: Insight[]; costTime: number; }; @@ -57,10 +56,15 @@ export function ChartPreview(props: IPropsType) {

Total Time: {props.costTime / 1000} s

insights:

- -

spec:

- - {/*
{JSON.stringify(props.spec, null, 4)}
*/} + {props.insights.map((insight, index) => { + const { content, variables } = insight?.textContent || {}; + let text = content ?? ''; + variables && + Object.keys(variables).forEach(key => { + text = text.replaceAll(`\${${key}}`, variables[key].value as string); + }); + return

{text}

; + })}
) : null} diff --git a/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx index 218e3df9..952d7c30 100644 --- a/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/Insight/DataInput.tsx @@ -1,20 +1,8 @@ /* eslint-disable no-console */ import React, { useState, useCallback, useMemo, useEffect } from 'react'; import './index.scss'; -import { - Avatar, - Input, - Divider, - Button, - InputNumber, - Upload, - Message, - Select, - Radio, - Checkbox, - Modal -} from '@arco-design/web-react'; -import VMind, { ArcoTheme, AtomName, LLMManage, Schedule } from '../../../../../src/index'; +import { Avatar, Input, Divider, Button, InputNumber, Select, Radio, Modal } from '@arco-design/web-react'; +import { AtomName, LLMManage, Schedule } from '../../../../../src/index'; import { Model } from '../../../../../src/index'; import { ChangePointChart, @@ -57,8 +45,6 @@ const globalVariables = (import.meta as any).env; const ModelConfigMap: any = { [Model.DOUBAO_PRO]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, [Model.GPT3_5]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, - [Model.GPT4]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, - [Model.GPT_4_0613]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } }; const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions'; @@ -66,13 +52,11 @@ const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions'; export function DataInput(props: IPropsType) { const defaultDataKey = Object.keys(demoDataList)[0]; const [spec, setSpec] = useState(JSON.stringify(demoDataList[defaultDataKey].spec)); - const [fieldInfo, setFieldInfo] = useState(demoDataList[defaultDataKey].fieldInfo); //const [spec, setSpec] = useState(''); //const [time, setTime] = useState(1000); - const [model, setModel] = useState(Model.GPT3_5); - const [cache, setCache] = useState(true); - const [showThoughts, setShowThoughts] = useState(false); + const [model, setModel] = useState(Model.GPT_4o); + const [numLimits, setNumLimits] = useState(8); const [visible, setVisible] = React.useState(false); const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); const [apiKey, setApiKey] = React.useState(ModelConfigMap[model]?.key); @@ -91,7 +75,12 @@ export function DataInput(props: IPropsType) { }) ); const schedule = React.useRef>( - new Schedule([AtomName.DATA_INSIGHT, AtomName.DATA_CLEAN], { base: { llm: llm.current, showThoughts } }) + new Schedule([AtomName.DATA_INSIGHT], { + base: { llm: llm.current }, + dataInsight: { + maxNum: numLimits + } + }) ); useEffect(() => { llm.current.updateOptions({ @@ -152,7 +141,6 @@ export function DataInput(props: IPropsType) { const dataObj = demoDataList[v]; setSpec(JSON.stringify(dataObj.spec)); props.onSpecChange(dataObj.spec); - setFieldInfo(dataObj.fieldInfo); }} > {Object.keys(demoDataList).map(name => ( @@ -163,55 +151,54 @@ export function DataInput(props: IPropsType) {
-
-

- - 2 - - Input your spec -

- -

fieldInfo:

- -

instruction:

- - - {/*
{JSON.stringify(props.spec, null, 4)}
*/} -
- - -
- ); -} diff --git a/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataExtraction.tsx b/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataExtraction.tsx deleted file mode 100644 index 376ccd01..00000000 --- a/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataExtraction.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import React, { useState } from 'react'; -import { Layout } from '@arco-design/web-react'; -import { DataInput } from './DataInput'; -import { ChartPreview } from './ChartPreview'; -import type { SimpleFieldInfo, VMindDataset } from '../../../../../src/common/typings'; -const Sider = Layout.Sider; -const Content = Layout.Content; - -export function DataExtractionPage() { - const [dataset, setDataset] = useState([]); - const [fieldInfo, setFieldInfo] = useState([]); - - const [instructionByLLM, setInstructionByLLM] = useState(''); - - const [costTime, setCostTime] = useState(0); - return ( - - - { - const { dataset, fieldInfo, instruction } = option; - setDataset(dataset); - setInstructionByLLM(instruction); - setFieldInfo(fieldInfo); - }} - /> - - - - - - ); -} diff --git a/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataExtractionPage.tsx b/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataExtractionPage.tsx new file mode 100644 index 00000000..dd9c8d32 --- /dev/null +++ b/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataExtractionPage.tsx @@ -0,0 +1,63 @@ +import React, { useState } from 'react'; +import { Card, Layout, Spin } from '@arco-design/web-react'; +import { DataInput } from './DataInput'; +import type { TableProps } from './DataTable'; +import { SimpleTable } from './DataTable'; +import { ChartPreview } from '../ChartGeneration/ChartPreview'; +const Sider = Layout.Sider; +const Content = Layout.Content; + +export function DataExtractionPage() { + const [dataExtractionRes, setDataExtractionRes] = useState([]); + const [spec, setSpec] = useState(''); + const [specList, setSpecList] = useState([]); + const [loading, setLoading] = useState(false); + const [timeCost, setTimeCost] = useState(0); + const [time, setTime] = useState<{ + totalTime: number; + frameArr: any[]; + }>(); + const handleOk = React.useCallback( + async (dataExtractCtx: any, spec: any, specList: any[], time: any, timeCost: number) => { + setLoading(false); + setTimeCost(timeCost); + // eslint-disable-next-line no-console + console.info(dataExtractCtx, spec); + setDataExtractionRes([dataExtractCtx]); + setSpecList(specList); + setTime(time); + setSpec(spec); + }, + [] + ); + + return ( + + + + + +
+ +
+ +

Data Table Result:

+ +
+ +
+ + + ); +} diff --git a/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataInput.tsx b/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataInput.tsx index 79331f22..3a31ffab 100644 --- a/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataInput.tsx +++ b/packages/vmind/__tests__/browser/src/pages/DataExtraction/DataInput.tsx @@ -1,21 +1,18 @@ /* eslint-disable no-console */ -import React, { useState, useCallback, useMemo } from 'react'; -import './index.scss'; -import { - Avatar, - Input, - Divider, - Button, - InputNumber, - Upload, - Message, - Select, - Radio, - Checkbox, - Modal -} from '@arco-design/web-react'; -import VMind, { ArcoTheme } from '../../../../../src/index'; -import { Model } from '../../../../../src/index'; +import React, { useState, useEffect } from 'react'; +import '../index.scss'; +import { Avatar, Input, Divider, Button, Select, Checkbox, Modal, Radio, Message } from '@arco-design/web-react'; +import VMind, { Model } from '../../../../../src/index'; + +const TextArea = Input.TextArea; +const Option = Select.Option; +const RadioGroup = Radio.Group; + +type IPropsType = { + onOk: (dataExtractCtx: any, spec: any, specList: any[], time: any, timeCost: number) => void; + setLoading: (loading: boolean) => void; +}; + import { mockUserTextInput0, mockUserTextInput1, @@ -29,58 +26,42 @@ import { mockUserTextInput8, mockUserTextInput9 } from '../../constants/mockData'; -import type { VMindDataset } from '../../../../../src/common/typings'; - -const TextArea = Input.TextArea; -const Option = Select.Option; -const RadioGroup = Radio.Group; - -type IPropsType = { - onDatasetGenerate: (payload: any) => void; -}; -const demoTextList: { [key: string]: any } = { - demo0: mockUserTextInput0, - demo: mockUserTextInput1, - demo2: mockUserTextInput2, - demo3: mockUserTextInput3, - demo4: mockUserTextInput4, - demo5: mockUserTextInput5, - demo6: mockUserTextInput6, - demo7: mockUserTextInput7, - demo8: mockUserTextInput8, - demo9: mockUserTextInput9, - demo10: mockUserTextInput10 -}; +const demoTextList: { text: string; input?: string }[] = [ + mockUserTextInput0, + mockUserTextInput1, + mockUserTextInput10, + mockUserTextInput2, + mockUserTextInput3, + mockUserTextInput4, + mockUserTextInput5, + mockUserTextInput6, + mockUserTextInput7, + mockUserTextInput8, + mockUserTextInput9 +]; const globalVariables = (import.meta as any).env; const ModelConfigMap: any = { - [Model.SKYLARK2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, - [Model.SKYLARK2_v1_2]: { url: globalVariables.VITE_SKYLARK_URL, key: globalVariables.VITE_SKYLARK_KEY }, + [Model.DOUBAO_PRO]: { url: globalVariables.VITE_DOUBAO_URL, key: globalVariables.VITE_DOUBAO_KEY }, [Model.GPT3_5]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, - [Model.GPT4]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } + [Model.GPT4]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT_4_0613]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY }, + [Model.GPT_4o]: { url: globalVariables.VITE_GPT_URL, key: globalVariables.VITE_GPT_KEY } }; const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions'; export function DataInput(props: IPropsType) { - const { onDatasetGenerate } = props; - const defaultDataKeyForTextInput = Object.keys(demoTextList)[0]; - const [text, setText] = useState(demoTextList[defaultDataKeyForTextInput].text); - const [userInput, setUserInput] = useState(demoTextList[defaultDataKeyForTextInput].input); - const [dataset, setDataset] = useState([]); - const [instructionByLLM, setInstructionByLLM] = useState(''); + const defaultIndex = 0; + const [text, setText] = useState(demoTextList[defaultIndex].text); + const [userInput, setUserInput] = useState(demoTextList[defaultIndex].input || ''); - //const [spec, setSpec] = useState(''); - //const [time, setTime] = useState(1000); - const [model, setModel] = useState(Model.GPT3_5); - const [cache, setCache] = useState(true); + const [model, setModel] = useState(Model.GPT_4o); const [showThoughts, setShowThoughts] = useState(false); const [visible, setVisible] = React.useState(false); const [url, setUrl] = React.useState(ModelConfigMap[model]?.url ?? OPENAI_API_URL); const [apiKey, setApiKey] = React.useState(ModelConfigMap[model]?.key); - const [loading, setLoading] = useState(false); - - const vmind: VMind = useMemo(() => { + const vmind: VMind = React.useMemo(() => { if (!url || !apiKey) { Message.error('Please set your LLM URL and API Key!!!'); return null as unknown as VMind; @@ -88,23 +69,30 @@ export function DataInput(props: IPropsType) { return new VMind({ url, model, - cache, showThoughts: showThoughts, headers: { - 'api-key': apiKey, - Authorization: `Bearer ${apiKey}` + //must has Authorization: `Bearer ${openAIKey}` if use openai api + Authorization: `Bearer ${apiKey}`, + 'api-key': apiKey } }); - }, [apiKey, cache, model, showThoughts, url]); - - const askGPTForGenerateData = useCallback(async () => { - const { instruction, dataset, fieldInfo } = await vmind.extractDataFromText(text, userInput); - - console.log(instruction, dataset, fieldInfo); - setDataset(dataset); - setInstructionByLLM(instruction); - onDatasetGenerate({ dataset, instruction, fieldInfo }); - }, [vmind, text, userInput, onDatasetGenerate]); + }, [apiKey, model, showThoughts, url]); + const handleQuery = React.useCallback(async () => { + props.setLoading(true); + const time1: any = new Date(); + const { dataTable, fieldInfo, spec, time, chartAdvistorRes } = await vmind.text2Chart(text, userInput, { + theme: 'light' + }); + const time2: any = new Date(); + const diff = (time2 - time1) / 1000; + props.onOk( + { dataTable, fieldInfo }, + spec, + (chartAdvistorRes || [])?.map(v => v.spec), + time, + diff + ); + }, [props, text, userInput, vmind]); return (
@@ -119,50 +107,52 @@ export function DataInput(props: IPropsType) {
-
+

- 0 + 1 Select Demo Data (optional)

-
+

- 1 + 2 Input your data in text format