From a0b7d00f03524a61548dbc58d1b0a2bea0266af3 Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Fri, 15 Nov 2024 11:58:22 -0500 Subject: [PATCH 01/10] Fix: improve image and text paste handling in TipTapEditor - Add check for image content before processing paste events - Prevent default paste behavior only when handling images - Insert images at cursor position instead of document start - Add plain text paste handler to strip formatting - Clean up image paste logic with better type checking and error handling --- gui/src/components/mainInput/TipTapEditor.tsx | 67 ++++++++++++++----- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 4c4e41830e..3ca35ad2e5 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -327,31 +327,45 @@ const TipTapEditor = memo(function TipTapEditor({ props: { handleDOMEvents: { paste(view, event) { - console.log("Pasting image"); const items = event.clipboardData.items; - for (const item of items) { - const file = item.getAsFile(); - file && - modelSupportsImages( + + const hasImageItem = Array.from(items).some( + item => item.type.startsWith('image/') + ); + + // Only log and process if we actually have an image + if (hasImageItem) { + console.log("Pasting image"); + for (const item of items) { + if (!item.type.startsWith('image/')) continue; + + const file = item.getAsFile(); + if (!file) continue; + + if (modelSupportsImages( defaultModel.provider, defaultModel.model, defaultModel.title, defaultModel.capabilities, - ) && - handleImageFile(file).then((resp) => { - if (!resp) { - return; - } - const [img, dataUrl] = resp; - const { schema } = view.state; - const node = schema.nodes.image.create({ - src: dataUrl, + )) { + event.preventDefault(); + + handleImageFile(file).then((resp) => { + if (!resp) return; + + const [img, dataUrl] = resp; + const { schema } = view.state; + const node = schema.nodes.image.create({ + src: dataUrl, + }); + const pos = view.state.selection.from; + const tr = view.state.tr.insert(pos, node); + view.dispatch(tr); }); - const tr = view.state.tr.insert(0, node); - view.dispatch(tr); - }); + } + } } - }, + } }, }, }); @@ -489,6 +503,23 @@ const TipTapEditor = memo(function TipTapEditor({ class: "outline-none -mt-1 mb-1 overflow-hidden", style: `font-size: ${getFontSize()}px;`, }, + handlePaste: (view, event) => { + // If it's not an image paste, handle as plain text + const items = event.clipboardData.items; + const hasImageItem = Array.from(items).some( + item => item.type.startsWith('image/') + ); + + if (!hasImageItem) { + event.preventDefault(); + const text = event.clipboardData.getData('text/plain'); + const { tr } = view.state; + tr.insertText(text); + view.dispatch(tr); + return true; + } + return false; + }, }, content: lastContentRef.current, editable: true, From 5332300476719b6bf386a118dc5bced1fb27b350 Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Fri, 15 Nov 2024 12:02:45 -0500 Subject: [PATCH 02/10] Updated message --- gui/src/components/mainInput/TipTapEditor.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 3ca35ad2e5..270bedeb41 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -295,7 +295,7 @@ const TipTapEditor = memo(function TipTapEditor({ } else { ideMessenger.post("errorPopup", { message: - "Images need to be in jpg or png format and less than 10MB in size.", + "Images need to be in jpg, gif, svg, webp, or png format and less than 10MB in size.", }); } return undefined; From 5b6c83a95bc93e7775bc93750d6efffca764736c Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Sun, 17 Nov 2024 12:10:46 -0500 Subject: [PATCH 03/10] Fixed image positioning and a bug where newlines were being removed when a user starts typing after pasting. --- gui/src/components/mainInput/TipTapEditor.tsx | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 97f2a03fe5..0f53c75e5f 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -61,7 +61,7 @@ import ActiveFileIndicator from "./ActiveFileIndicator"; import { setActiveFilePath } from "@/redux/slices/uiStateSlice"; import TopBar from "./TopBarIndicators"; import { isAiderMode, isPerplexityMode } from "../../util/bareChatMode"; - +import HardBreak from '@tiptap/extension-hard-break'; const InputBoxDiv = styled.div` resize: none; @@ -295,7 +295,7 @@ const TipTapEditor = memo(function TipTapEditor({ } else { ideMessenger.post("errorPopup", { message: - "Images need to be in jpg, gif, svg, webp, or png format and less than 10MB in size.", + "Images need to be in jpg or png format and less than 10MB in size.", }); } return undefined; @@ -358,8 +358,7 @@ const TipTapEditor = memo(function TipTapEditor({ const node = schema.nodes.image.create({ src: dataUrl, }); - const pos = view.state.selection.from; - const tr = view.state.tr.insert(pos, node); + const tr = view.state.tr.insert(0, node); view.dispatch(tr); }); } @@ -465,6 +464,7 @@ const TipTapEditor = memo(function TipTapEditor({ }, }), Text, + HardBreak, Mention.configure({ HTMLAttributes: { class: "mention", @@ -503,18 +503,26 @@ const TipTapEditor = memo(function TipTapEditor({ class: "outline-none -mt-1 mb-1 overflow-hidden", style: `font-size: ${getFontSize()}px;`, }, - handlePaste: (view, event) => { - // If it's not an image paste, handle as plain text + handlePaste(view, event) { const items = event.clipboardData.items; - const hasImageItem = Array.from(items).some( - item => item.type.startsWith('image/') - ); + const hasImageItem = Array.from(items).some(item => item.type.startsWith('image/')); if (!hasImageItem) { event.preventDefault(); const text = event.clipboardData.getData('text/plain'); + const lines = text.split(/\r?\n/); const { tr } = view.state; - tr.insertText(text); + const { schema } = view.state; + let pos = view.state.selection.from; + + lines.forEach((line, index) => { + if (index > 0) { + tr.insert(pos++, schema.nodes.hardBreak.create()); + } + tr.insertText(line, pos); + pos += line.length; + }); + view.dispatch(tr); return true; } From 5ef8491b8e68ba854ae3fd08268064ecf81d86a8 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Sun, 17 Nov 2024 23:04:33 +0530 Subject: [PATCH 04/10] delete selected text --- gui/src/components/mainInput/TipTapEditor.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 0f53c75e5f..68aa2009e1 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -515,6 +515,9 @@ const TipTapEditor = memo(function TipTapEditor({ const { schema } = view.state; let pos = view.state.selection.from; + // Delete the selected text before inserting new text + tr.delete(view.state.selection.from, view.state.selection.to); + lines.forEach((line, index) => { if (index > 0) { tr.insert(pos++, schema.nodes.hardBreak.create()); From 48228ddf76a2fb5f522b6c2a78eacffff200eb1c Mon Sep 17 00:00:00 2001 From: Duke Pan Date: Sun, 17 Nov 2024 16:52:25 -0800 Subject: [PATCH 05/10] Remove console log --- gui/src/components/mainInput/TipTapEditor.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 0f53c75e5f..db9890282b 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -335,7 +335,6 @@ const TipTapEditor = memo(function TipTapEditor({ // Only log and process if we actually have an image if (hasImageItem) { - console.log("Pasting image"); for (const item of items) { if (!item.type.startsWith('image/')) continue; From 50e63bdadad4a83c6624354938caa345e9a9a8f1 Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Sat, 23 Nov 2024 22:37:12 -0500 Subject: [PATCH 06/10] Found a bug when pasting images with CTRL+V --- gui/src/components/mainInput/TipTapEditor.tsx | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 1d582a8767..a021d6d1e0 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -589,9 +589,20 @@ const TipTapEditor = memo(function TipTapEditor({ event.preventDefault(); } else if ((event.metaKey || event.ctrlKey) && event.key === "v") { // Paste - event.preventDefault(); // Prevent default paste behavior - const clipboardText = await navigator.clipboard.readText(); - editor.commands.insertContent(clipboardText); + // Let the paste event handlers deal with it if there's data in the clipboard + const clipboardItems = await navigator.clipboard.read(); + const hasImage = await Promise.all( + clipboardItems.map(async item => { + return item.types.some(type => type.startsWith('image/')); + }) + ).then(results => results.some(Boolean)); + + if (!hasImage) { + // Only handle text if there's no image + event.preventDefault(); + const clipboardText = await navigator.clipboard.readText(); + editor.commands.insertContent(clipboardText); + } } }; From 22aa59b6fcbbc86fcacf52a3ea907ae8e69cf4d6 Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Sun, 24 Nov 2024 06:17:06 -0500 Subject: [PATCH 07/10] Updated useEffect so that the paste handlers handle paste operations --- gui/src/components/mainInput/TipTapEditor.tsx | 74 +++---------------- 1 file changed, 11 insertions(+), 63 deletions(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index a021d6d1e0..f65020f1ab 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -560,81 +560,29 @@ const TipTapEditor = memo(function TipTapEditor({ }, [ideMessenger]); useEffect(() => { - if (isJetBrains()) { - // This is only for VS Code .ipynb files - return; - } - - if (isWebEnvironment()) { - const handleKeyDown = async (event: KeyboardEvent) => { - if (!editor || !editorFocusedRef.current) { - return; - } - if ((event.metaKey || event.ctrlKey) && event.key === "x") { - // Cut - const selectedText = editor.state.doc.textBetween( - editor.state.selection.from, - editor.state.selection.to, - ); - navigator.clipboard.writeText(selectedText); - editor.commands.deleteSelection(); - event.preventDefault(); - } else if ((event.metaKey || event.ctrlKey) && event.key === "c") { - // Copy - const selectedText = editor.state.doc.textBetween( - editor.state.selection.from, - editor.state.selection.to, - ); - navigator.clipboard.writeText(selectedText); - event.preventDefault(); - } else if ((event.metaKey || event.ctrlKey) && event.key === "v") { - // Paste - // Let the paste event handlers deal with it if there's data in the clipboard - const clipboardItems = await navigator.clipboard.read(); - const hasImage = await Promise.all( - clipboardItems.map(async item => { - return item.types.some(type => type.startsWith('image/')); - }) - ).then(results => results.some(Boolean)); - - if (!hasImage) { - // Only handle text if there's no image - event.preventDefault(); - const clipboardText = await navigator.clipboard.readText(); - editor.commands.insertContent(clipboardText); - } - } - }; - - document.addEventListener("keydown", handleKeyDown); - - return () => { - document.removeEventListener("keydown", handleKeyDown); - }; - } - const handleKeyDown = async (event: KeyboardEvent) => { if (!editor || !editorFocusedRef.current) { return; } - - if (event.metaKey && event.key === "x") { + if ((event.metaKey || event.ctrlKey) && event.key === "v") { + // Let the native paste event handle it if there are images + // This allows the paste handler in editorProps to process images + return; + } + // Handle other keyboard shortcuts... + if ((event.metaKey || event.ctrlKey) && event.key === "x") { document.execCommand("cut"); event.stopPropagation(); event.preventDefault(); - } else if (event.metaKey && event.key === "v") { - document.execCommand("paste"); - event.stopPropagation(); - event.preventDefault(); - } else if (event.metaKey && event.key === "c") { + } else if ((event.metaKey || event.ctrlKey) && event.key === "c") { document.execCommand("copy"); - event.stopPropagation(); + event.stopPropagation(); event.preventDefault(); } }; - + document.addEventListener("keydown", handleKeyDown); - + return () => { document.removeEventListener("keydown", handleKeyDown); }; From 183c2ade94926216a382ff911f7e3f007655e79a Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Sun, 24 Nov 2024 06:24:28 -0500 Subject: [PATCH 08/10] Removed extra spaces. --- gui/src/components/mainInput/TipTapEditor.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index f65020f1ab..9c6cb25a99 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -576,13 +576,13 @@ const TipTapEditor = memo(function TipTapEditor({ event.preventDefault(); } else if ((event.metaKey || event.ctrlKey) && event.key === "c") { document.execCommand("copy"); - event.stopPropagation(); + event.stopPropagation(); event.preventDefault(); } }; - + document.addEventListener("keydown", handleKeyDown); - + return () => { document.removeEventListener("keydown", handleKeyDown); }; From 448ac93a64617867734cd0e580b5666a6fe63002 Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Sun, 24 Nov 2024 07:39:09 -0500 Subject: [PATCH 09/10] Combined logic into one paste handler. Better checks for the selected model. --- gui/src/components/mainInput/TipTapEditor.tsx | 109 ++++++++---------- 1 file changed, 45 insertions(+), 64 deletions(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 9c6cb25a99..983e9119d6 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -239,6 +239,7 @@ const TipTapEditor = memo(function TipTapEditor({ (store: RootState) => store.state.contextItems, ); const defaultModel = useSelector(defaultModelSelector); + const defaultModelRef = useUpdatingRef(defaultModel); const getSubmenuContextItemsRef = useUpdatingRef(getSubmenuContextItems); const availableContextProvidersRef = useUpdatingRef(availableContextProviders) @@ -321,54 +322,9 @@ const TipTapEditor = memo(function TipTapEditor({ extensions: [ Document, History, - Image.extend({ - addProseMirrorPlugins() { - const plugin = new Plugin({ - props: { - handleDOMEvents: { - paste(view, event) { - const items = event.clipboardData.items; - - const hasImageItem = Array.from(items).some( - item => item.type.startsWith('image/') - ); - - // Only log and process if we actually have an image - if (hasImageItem) { - for (const item of items) { - if (!item.type.startsWith('image/')) continue; - - const file = item.getAsFile(); - if (!file) continue; - - if (modelSupportsImages( - defaultModel.provider, - defaultModel.model, - defaultModel.title, - defaultModel.capabilities, - )) { - event.preventDefault(); - - handleImageFile(file).then((resp) => { - if (!resp) return; - - const [img, dataUrl] = resp; - const { schema } = view.state; - const node = schema.nodes.image.create({ - src: dataUrl, - }); - const tr = view.state.tr.insert(0, node); - view.dispatch(tr); - }); - } - } - } - } - }, - }, - }); - return [plugin]; - }, + Image.configure({ + inline: true, + allowBase64: true, }), Placeholder.configure({ placeholder: () => getPlaceholder(historyLengthRef.current, location), @@ -502,18 +458,48 @@ const TipTapEditor = memo(function TipTapEditor({ class: "outline-none -mt-1 mb-1 overflow-hidden", style: `font-size: ${getFontSize()}px;`, }, - handlePaste(view, event) { - const items = event.clipboardData.items; - const hasImageItem = Array.from(items).some(item => item.type.startsWith('image/')); - - if (!hasImageItem) { + handlePaste: (view, event) => { + const items = event.clipboardData?.items; + const currentModel = defaultModelRef.current; + if (!items || !currentModel) return false; + + // Check for image items and model support + for (const item of items) { + if (item.type.startsWith('image/') && + modelSupportsImages( + currentModel.provider, + currentModel.model, + currentModel.title, + currentModel.capabilities + )) { + event.preventDefault(); + const file = item.getAsFile(); + if (!file) continue; + + // Handle image file asynchronously but return synchronously + handleImageFile(file).then(result => { + if (result) { + const [image, dataUrl] = result; + const { state } = view; + const { tr } = state; + const pos = state.selection.from; + tr.replaceSelectionWith(state.schema.nodes.image.create({ src: dataUrl })); + view.dispatch(tr); + } + }); + return true; + } + } + + // Handle text paste + const text = event.clipboardData.getData('text/plain'); + if (text) { event.preventDefault(); - const text = event.clipboardData.getData('text/plain'); const lines = text.split(/\r?\n/); const { tr } = view.state; const { schema } = view.state; let pos = view.state.selection.from; - + // Delete the selected text before inserting new text tr.delete(view.state.selection.from, view.state.selection.to); @@ -524,10 +510,11 @@ const TipTapEditor = memo(function TipTapEditor({ tr.insertText(line, pos); pos += line.length; }); - + view.dispatch(tr); return true; } + return false; }, }, @@ -564,12 +551,6 @@ const TipTapEditor = memo(function TipTapEditor({ if (!editor || !editorFocusedRef.current) { return; } - if ((event.metaKey || event.ctrlKey) && event.key === "v") { - // Let the native paste event handle it if there are images - // This allows the paste handler in editorProps to process images - return; - } - // Handle other keyboard shortcuts... if ((event.metaKey || event.ctrlKey) && event.key === "x") { document.execCommand("cut"); event.stopPropagation(); @@ -580,9 +561,9 @@ const TipTapEditor = memo(function TipTapEditor({ event.preventDefault(); } }; - + document.addEventListener("keydown", handleKeyDown); - + return () => { document.removeEventListener("keydown", handleKeyDown); }; From e6d83665b985124d987a9f3cc90e700f8fab089b Mon Sep 17 00:00:00 2001 From: Charles Williams Date: Sun, 24 Nov 2024 07:43:47 -0500 Subject: [PATCH 10/10] Removed spaces --- gui/src/components/mainInput/TipTapEditor.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 983e9119d6..f0254ce34a 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -561,9 +561,9 @@ const TipTapEditor = memo(function TipTapEditor({ event.preventDefault(); } }; - + document.addEventListener("keydown", handleKeyDown); - + return () => { document.removeEventListener("keydown", handleKeyDown); };