feat: AI block UI

.gitignore

@@ -33,3 +33,5 @@ release
 /test-results/
 /playwright-report/
 /playwright/.cache/
+.env
+*.pem

.todo.txt

@@ -0,0 +1,7 @@
+
+review toolbar code
+fix design of menus (smaller items, border?)
+
+see if addToHistory: false is needed
+track last affected position to update toolbar
+enable / disable show selection plugin

docs/pages/examples/_meta.json

@@ -7,5 +7,6 @@
   "interoperability": "Interoperability",
   "custom-schema": "Custom Schemas",
   "collaboration": "Collaboration",
-  "extensions": "Extensions"
+  "extensions": "Extensions",
+  "ai": "AI"
 }

examples/03-ui-components/02-formatting-toolbar-buttons/BlueButton.tsx

@@ -1,9 +1,9 @@
+import "@blocknote/mantine/style.css";
 import {
   useBlockNoteEditor,
   useComponentsContext,
   useEditorContentOrSelectionChange,
 } from "@blocknote/react";
-import "@blocknote/mantine/style.css";
 import { useState } from "react";
 
 // Custom Formatting Toolbar Button to toggle blue text & background color.

examples/03-ui-components/06-suggestion-menus-slash-menu-items/App.tsx

@@ -4,18 +4,19 @@ import {
   PartialBlock,
 } from "@blocknote/core";
 import "@blocknote/core/fonts/inter.css";
+import { BlockNoteView } from "@blocknote/mantine";
+import "@blocknote/mantine/style.css";
 import {
   DefaultReactSuggestionItem,
   getDefaultReactSlashMenuItems,
   SuggestionMenuController,
   useCreateBlockNote,
 } from "@blocknote/react";
-import { BlockNoteView } from "@blocknote/mantine";
-import "@blocknote/mantine/style.css";
 import { HiOutlineGlobeAlt } from "react-icons/hi";
 
 // Custom Slash Menu item to insert a block after the current one.
 const insertHelloWorldItem = (editor: BlockNoteEditor) => ({
+  key: "hello_world",
   title: "Insert Hello World",
   onItemClick: () => {
     // Block that the text cursor is currently in.

examples/06-custom-schema/01-alert-block/App.tsx

@@ -29,6 +29,7 @@ const schema = BlockNoteSchema.create({
 
 // Slash menu item to insert an Alert block
 const insertAlert = (editor: typeof schema.BlockNoteEditor) => ({
+  key: "alert",
   title: "Alert",
   onItemClick: () => {
     insertOrUpdateBlock(editor, {

examples/06-custom-schema/02-suggestion-menus-mentions/App.tsx

@@ -12,7 +12,7 @@ import {
   useCreateBlockNote,
 } from "@blocknote/react";
 
-import { Mention } from "./Mention";
+import { Mention } from "./Mention.js";
 
 // Our schema with inline content specs, which contain the configs and
 // implementations for inline content  that we want our editor to use.
@@ -32,6 +32,7 @@ const getMentionMenuItems = (
   const users = ["Steve", "Bob", "Joe", "Mike"];
 
   return users.map((user) => ({
+    key: user.toLowerCase(),
     title: user,
     onItemClick: () => {
       editor.insertInlineContent([

examples/06-custom-schema/04-pdf-file-block/App.tsx

@@ -30,6 +30,7 @@ const schema = BlockNoteSchema.create({
 
 // Slash menu item to insert a PDF block
 const insertPDF = (editor: typeof schema.BlockNoteEditor) => ({
+  key: "pdf",
   title: "PDF",
   onItemClick: () => {
     insertOrUpdateBlock(editor, {

examples/06-custom-schema/react-custom-styles/App.tsx

@@ -1,17 +1,17 @@
 import { BlockNoteSchema, defaultStyleSpecs } from "@blocknote/core";
 import "@blocknote/core/fonts/inter.css";
+import { BlockNoteView } from "@blocknote/mantine";
+import "@blocknote/mantine/style.css";
 import {
-  createReactStyleSpec,
   FormattingToolbar,
   FormattingToolbarController,
   FormattingToolbarProps,
+  createReactStyleSpec,
   useActiveStyles,
   useBlockNoteEditor,
   useComponentsContext,
   useCreateBlockNote,
 } from "@blocknote/react";
-import { BlockNoteView } from "@blocknote/mantine";
-import "@blocknote/mantine/style.css";
 
 const small = createReactStyleSpec(
   {

examples/09-ai/01-minimal/.bnexample.json

@@ -0,0 +1,13 @@
+{
+  "playground": true,
+  "docs": true,
+  "author": "yousefed",
+  "tags": ["AI", "llm"],
+  "dependencies": {
+    "@blocknote/xl-ai": "latest",
+    "@mantine/core": "^7.10.1",
+    "ai": "^4.1.0",
+    "@ai-sdk/openai": "^1.1.0",
+    "@ai-sdk/groq": "^1.1.0"
+  }
+}

examples/09-ai/01-minimal/App.tsx

@@ -0,0 +1,184 @@
+import {
+  AIShowSelectionPlugin,
+  AIToolbarButton,
+  BlockNoteAIContextProvider,
+  BlockNoteAIUI,
+  locales as aiLocales,
+  createBlockNoteAIClient,
+  getAISlashMenuItems,
+  useBlockNoteAIContext,
+} from "@blocknote/xl-ai";
+
+import { createGroq } from "@ai-sdk/groq";
+import { createOpenAI } from "@ai-sdk/openai";
+import {
+  BlockNoteEditor,
+  BlockNoteSchema,
+  defaultBlockSpecs,
+  filterSuggestionItems,
+  locales,
+} from "@blocknote/core";
+import "@blocknote/core/fonts/inter.css";
+import { BlockNoteView } from "@blocknote/mantine";
+import "@blocknote/mantine/style.css";
+import {
+  FormattingToolbar,
+  FormattingToolbarController,
+  SuggestionMenuController,
+  blockTypeSelectItems,
+  getDefaultReactSlashMenuItems,
+  getFormattingToolbarItems,
+  useCreateBlockNote,
+} from "@blocknote/react";
+import "@blocknote/xl-ai/style.css";
+import { Fieldset, Switch } from "@mantine/core";
+import { useMemo, useState } from "react";
+import { BasicAutocomplete } from "./AutoComplete.js";
+import RadioGroupComponent from "./components/RadioGroupComponent.js";
+
+const schema = BlockNoteSchema.create({
+  blockSpecs: {
+    ...defaultBlockSpecs,
+    // ai: AIBlock,
+  },
+});
+
+const client = createBlockNoteAIClient({
+  apiKey: import.meta.env.VITE_BLOCKNOTE_AI_SERVER_API_KEY || "PLACEHOLDER",
+  baseURL:
+    import.meta.env.VITE_BLOCKNOTE_AI_SERVER_BASE_URL ||
+    "https://localhost:3000/ai",
+});
+
+export default function App() {
+  const [aiModelString, setAiModelString] = useState(
+    "openai/gpt-4o-2024-08-06"
+  );
+  // Creates a new editor instance.
+  const editor = useCreateBlockNote({
+    schema,
+    dictionary: {
+      ...locales.en,
+      ai: aiLocales.en,
+    } as any,
+    _extensions: {
+      // TODO: things will break when user provides different keys. Define name on plugins instead?
+      // aiBlockToolbar: new AIBlockToolbarProsemirrorPlugin(),
+      aiSelection: new AIShowSelectionPlugin(),
+    },
+  });
+
+  const model = useMemo(() => {
+    const [provider, ...modelNameParts] = aiModelString.split("/");
+    const modelName = modelNameParts.join("/");
+    if (provider === "openai") {
+      return createOpenAI({
+        ...client.getProviderSettings("openai"),
+      })(modelName, {});
+    } else if (provider === "groq") {
+      return createGroq({
+        ...client.getProviderSettings("groq"),
+      })(modelName);
+    } else if (provider === "albert-etalab") {
+      return createOpenAI({
+        // albert-etalab/neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8
+        baseURL: "https://albert.api.staging.etalab.gouv.fr/v1",
+        ...client.getProviderSettings("albert-etalab"),
+        compatibility: "compatible",
+      })(modelName);
+    }
+    throw new Error(`Unknown model: ${aiModelString}`);
+  }, [aiModelString]);
+
+  const [dataFormat, setDataFormat] = useState<"json" | "markdown">("json");
+
+  const [streamStateValue, setStream] = useState(true);
+
+  const stream = dataFormat === "markdown" ? false : streamStateValue;
+
+  // Renders the editor instance using a React component.
+  return (
+    <div>
+      <Fieldset legend="Model settings" style={{ maxWidth: "500px" }}>
+        <BasicAutocomplete value={aiModelString} onChange={setAiModelString} />
+
+        <RadioGroupComponent
+          label="Data format"
+          items={[
+            { name: "JSON", description: "JSON format", value: "json" },
+            {
+              name: "Markdown",
+              description: "Markdown format",
+              value: "markdown",
+            },
+          ]}
+          value={dataFormat}
+          onChange={(value) => setDataFormat(value as "json" | "markdown")}
+        />
+
+        <Switch
+          checked={stream}
+          disabled={dataFormat === "markdown"}
+          onChange={(e) => setStream(e.target.checked)}
+          label="Streaming"
+        />
+      </Fieldset>
+
+      <BlockNoteView
+        editor={editor}
+        formattingToolbar={false}
+        slashMenu={false}>
+        <BlockNoteAIContextProvider
+          model={model}
+          dataFormat={dataFormat}
+          // TODO: doesn't work well with typings
+          stream={streamStateValue as any}>
+          <BlockNoteAIUI />
+          <FormattingToolbarController
+            formattingToolbar={() => (
+              <FormattingToolbar>
+                {...getFormattingToolbarItems([
+                  ...blockTypeSelectItems(editor.dictionary),
+                  // ...aiBlockTypeSelectItems(aiLocales.en),
+                ])}
+                <AIToolbarButton />
+              </FormattingToolbar>
+            )}
+          />
+          <SuggestionMenu editor={editor} />
+        </BlockNoteAIContextProvider>
+        {/* TODO: Side Menu customization */}
+      </BlockNoteView>
+    </div>
+  );
+}
+
+function SuggestionMenu(props: { editor: BlockNoteEditor<any, any, any> }) {
+  const ctx = useBlockNoteAIContext();
+  return (
+    <SuggestionMenuController
+      triggerCharacter="/"
+      getItems={async (query) =>
+        filterSuggestionItems(
+          [
+            ...getDefaultReactSlashMenuItems(props.editor),
+            ...getAISlashMenuItems(props.editor, ctx),
+          ],
+          query
+        )
+      }
+    />
+  );
+}
+
+/**
+ * - correct commands
+ * - FIX UI
+ *
+ * API:
+ * - context (which part of document to pass along)
+ * - based on selection
+ *
+ * - box with proposal
+ *
+ */

examples/09-ai/01-minimal/AutoComplete.tsx

@@ -0,0 +1,60 @@
+import { Combobox, TextInput, useCombobox } from "@mantine/core";
+import { AI_MODELS } from "./data/aimodels.js";
+
+// https://mantine.dev/combobox/?e=BasicAutocomplete
+export function BasicAutocomplete(props: {
+  value: string;
+  onChange: (value: string) => void;
+}) {
+  const { value, onChange } = props;
+  const combobox = useCombobox();
+
+  const shouldFilterOptions = !AI_MODELS.some((item) => item === value);
+  const filteredOptions = shouldFilterOptions
+    ? AI_MODELS.filter((item) =>
+        item.toLowerCase().includes(value.toLowerCase().trim())
+      )
+    : AI_MODELS;
+
+  const options = filteredOptions.map((item) => (
+    <Combobox.Option value={item} key={item}>
+      {item}
+    </Combobox.Option>
+  ));
+
+  return (
+    <Combobox
+      onOptionSubmit={(optionValue) => {
+        onChange(optionValue);
+        combobox.closeDropdown();
+      }}
+      store={combobox}
+      withinPortal={false}>
+      <Combobox.Target>
+        <TextInput
+          label="Select model or type `<provider>/<model>`:"
+          placeholder="Select model or type `<provider>/<model>`"
+          value={value}
+          onChange={(event) => {
+            onChange(event.currentTarget.value);
+            combobox.openDropdown();
+            combobox.updateSelectedOptionIndex();
+          }}
+          onClick={() => combobox.openDropdown()}
+          onFocus={() => combobox.openDropdown()}
+          onBlur={() => combobox.closeDropdown()}
+        />
+      </Combobox.Target>
+
+      <Combobox.Dropdown>
+        <Combobox.Options mah={200} style={{ overflowY: "auto" }}>
+          {options.length === 0 ? (
+            <Combobox.Empty>Nothing found</Combobox.Empty>
+          ) : (
+            options
+          )}
+        </Combobox.Options>
+      </Combobox.Dropdown>
+    </Combobox>
+  );
+}

examples/09-ai/01-minimal/README.md

@@ -0,0 +1,7 @@
+# AI integration
+
+This example shows how to use the `@blocknote/xl-ai` package to add AI capabilities to BlockNote.
+
+**Relevant Docs:**
+
+- TODO