Merge pull request #2036 from undb-io/release/v1.0.0-75

Release version v1.0.0-75

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v1.0.0-75
+
+
+### 🩹 Fixes
+
+- Fix bulk insert large data set ([d87e02f](https://github.com/undb-io/undb/commit/d87e02f))
+
+### ❤️ Contributors
+
+- Nichenqin ([@nichenqin](http://github.com/nichenqin))
+
 ## v1.0.0-74
 
 

apps/frontend/src/lib/components/blocks/create-table/create-table-sheet.svelte

@@ -9,6 +9,8 @@
   import { LoaderCircleIcon } from "lucide-svelte"
 
   const isCreating = useIsMutating({ mutationKey: ["createTable"] })
+
+  export let tableNames: string[]
 </script>
 
 <Sheet.Root
@@ -33,7 +35,7 @@
     </Sheet.Header>
 
     <ScrollArea class="flex-1">
-      <CreateTable />
+      <CreateTable {tableNames} />
     </ScrollArea>
 
     <Sheet.Footer>

apps/frontend/src/lib/components/blocks/create-table/create-table.svelte

@@ -13,8 +13,10 @@
   import { FieldIdVo } from "@undb/table"
   import { CREATE_TABLE_MODAL, closeModal } from "$lib/store/modal.store"
   import { baseId, currentBase } from "$lib/store/base.store"
+  import { getNextName } from "@undb/utils"
 
   const schema = createTableCommand.omit({ baseId: true })
+  export let tableNames: string[]
 
   const mutation = createMutation({
     mutationFn: trpc.table.create.mutate,
@@ -34,7 +36,7 @@
   const form = superForm(
     defaults(
       {
-        name: "",
+        name: getNextName(tableNames, "table"),
         schema: [
           {
             id: FieldIdVo.create().value,
@@ -54,7 +56,6 @@
       resetForm: false,
       invalidateAll: true,
       onSubmit(input) {
-        console.log({ input })
         validateForm({ update: true })
       },
       async onUpdate(event) {

apps/frontend/src/lib/components/blocks/import-table/import-table.svelte

@@ -86,19 +86,10 @@
     },
   })
 
-  async function onChange(e: Event) {
-    const target = e.target as HTMLInputElement
-    const files = target.files
-    if (!files?.length) return
-
-    const [f] = files
-    tableId = TableIdVo.create().value
-    file = f
-    tableName = getNextName(tableNames, file.name)
-
-    let parsed = await parse(file)
-    console.log(parsed)
+  async function setData() {
+    if (!file) return
 
+    const parsed = await parse(file)
     if (firstRowAsHeader) {
       const names = parsed.data[0].reduce((acc, cur) => {
         if (!cur) {
@@ -121,6 +112,19 @@
     }
   }
 
+  async function onChange(e: Event) {
+    const target = e.target as HTMLInputElement
+    const files = target.files
+    if (!files?.length) return
+
+    const [f] = files
+    tableId = TableIdVo.create().value
+    file = f
+    tableName = getNextName(tableNames, file.name)
+
+    await setData()
+  }
+
   function removeFile() {
     file = undefined
   }
@@ -153,16 +157,14 @@
     }
   }
 
-  $: transposed = firstRowAsHeader
-    ? unzip(data?.data.slice(1)).slice(0, inferFieldTypeCount)
-    : unzip(data?.data).slice(0, inferFieldTypeCount)
+  $: transposed = unzip(data?.data.slice(1)).slice(0, inferFieldTypeCount)
 
-  $: schema = data?.data[0].map((header, i) => ({
+  $: schema = (data?.data[0].map((header, i) => ({
     ...inferCreateFieldType(transposed[i]),
     name: header,
     id: FieldIdVo.create().value,
     display: i === 0,
-  })) as ICreateSchemaDTO
+  })) ?? []) as ICreateSchemaDTO
 </script>
 
 {#if step === 0}
@@ -188,76 +190,103 @@
     </div>
   {/if}
   <Label class="flex items-center gap-2">
-    <Checkbox bind:checked={firstRowAsHeader} />
+    <Checkbox
+      disabled={$createTable.isPending || $createRecords.isPending}
+      bind:checked={firstRowAsHeader}
+      onCheckedChange={setData}
+    />
     First row as header
   </Label>
   <Label class="flex items-center gap-2">
-    <Checkbox bind:checked={importData} />
+    <Checkbox disabled={$createTable.isPending || $createRecords.isPending} bind:checked={importData} />
     Import Data
   </Label>
 {:else if step === 1}
+  <Label class="flex items-center gap-2">
+    <Checkbox
+      disabled={$createTable.isPending || $createRecords.isPending}
+      bind:checked={firstRowAsHeader}
+      onCheckedChange={setData}
+    />
+    First row as header
+  </Label>
+  <Label class="flex items-center gap-2">
+    <Checkbox disabled={$createTable.isPending || $createRecords.isPending} bind:checked={importData} />
+    Import Data
+  </Label>
   {#if data && file}
+    <div class="p-3">
+      <Label class="flex items-center gap-2">
+        <div>Name</div>
+        <Input disabled={$createTable.isPending || $createRecords.isPending} class="text-sm" bind:value={tableName} />
+      </Label>
+    </div>
     <div class="rounded-sm border">
-      <div class="border-b p-3">
-        <Label class="flex items-center gap-2">
-          <div>Name</div>
-          <Input class="text-sm" bind:value={tableName} />
-        </Label>
-      </div>
-      <div>
-        <Table.Root>
-          <Table.Header>
-            <Table.Row>
-              <Table.Head class="w-[200px]">Field Name</Table.Head>
-              <Table.Head class="flex-1">Field Type</Table.Head>
-              <Table.Head class="text-right">Action</Table.Head>
+      <Table.Root>
+        <Table.Header>
+          <Table.Row>
+            <Table.Head class="w-[200px]">Field Name</Table.Head>
+            <Table.Head class="flex-1">Field Type</Table.Head>
+            <Table.Head class="text-right">Action</Table.Head>
+          </Table.Row>
+        </Table.Header>
+        <Table.Body>
+          {#each schema as field, idx}
+            <Table.Row class="group">
+              <Table.Cell class="font-medium">
+                <Input
+                  class="text-sm"
+                  bind:value={data.data[0][idx]}
+                  disabled={$createTable.isPending || $createRecords.isPending}
+                />
+              </Table.Cell>
+              <Table.Cell>
+                <div class="flex items-center">
+                  <FieldIcon type={field.type} class="mr-2 h-4 w-4" />
+                  {field.type}
+                </div>
+              </Table.Cell>
+              <Table.Cell class="text-right">
+                <button
+                  disabled={$createTable.isPending || $createRecords.isPending}
+                  on:click={() => removeField(idx)}
+                  class="rounded-full p-1 opacity-0 transition-colors hover:bg-gray-200 group-hover:opacity-100"
+                >
+                  <XIcon class="text-muted-foreground h-4 w-4" />
+                </button>
+              </Table.Cell>
             </Table.Row>
-          </Table.Header>
-          <Table.Body>
-            {#each schema as field, idx}
-              <Table.Row class="group">
-                <Table.Cell class="font-medium">
-                  <Input class="text-sm" bind:value={data.data[0][idx]} />
-                </Table.Cell>
-                <Table.Cell>
-                  <div class="flex items-center">
-                    <FieldIcon type={field.type} class="mr-2 h-4 w-4" />
-                    {field.type}
-                  </div>
-                </Table.Cell>
-                <Table.Cell class="text-right">
-                  <button
-                    on:click={() => removeField(idx)}
-                    class="rounded-full p-1 opacity-0 transition-colors hover:bg-gray-200 group-hover:opacity-100"
-                  >
-                    <XIcon class="text-muted-foreground h-4 w-4" />
-                  </button>
-                </Table.Cell>
-              </Table.Row>
-            {/each}
-          </Table.Body>
-        </Table.Root>
-      </div>
+          {/each}
+        </Table.Body>
+      </Table.Root>
     </div>
   {/if}
 {/if}
 
 <div class="flex justify-end gap-2">
   {#if step === 1}
-    <Button variant="outline" on:click={() => (step = 0)} size="sm">
+    <Button
+      disabled={$createTable.isPending || $createRecords.isPending}
+      variant="outline"
+      on:click={() => (step = 0)}
+      size="sm"
+    >
       <ArrowLeftIcon class="mr-2 h-4 w-4" />
       Back
     </Button>
   {/if}
   <Button
-    disabled={(step === 0 && !file) || (step === 1 && schema.length < 1) || $createTable.isPending}
+    disabled={(step === 0 && !file) ||
+      (step === 1 && schema.length < 1) ||
+      $createTable.isPending ||
+      $createRecords.isPending}
     on:click={handleClickImport}
     size="sm"
   >
     {#if step === 0}
       Next step <ArrowRightIcon class="ml-2 h-4 w-4" />
     {:else}
-      {#if $createTable.isPending}
+      {#if $createTable.isPending || $createRecords.isPending}
         <LoaderCircleIcon class="mr-2 h-4 w-4 animate-spin" />
       {/if}
       Import

apps/frontend/src/lib/components/blocks/option/option.svelte

@@ -9,7 +9,7 @@
 <span
   data-option-id={option.id}
   class={cn(
-    "inline-flex items-center justify-start rounded-md border bg-gray-50 px-2 py-0.5 text-left text-xs font-medium text-gray-600 ring-1 ring-inset ring-gray-500/10",
+    "inline-flex items-center justify-start truncate rounded-md border bg-gray-50 px-2 py-0.5 text-left text-xs font-medium text-gray-600 ring-1 ring-inset ring-gray-500/10",
     getBgColor(option.color),
     getTextColor(option.color),
     getRingColor(option.color),

apps/frontend/src/routes/(authed)/(space)/+layout.svelte

@@ -11,7 +11,7 @@
   import { CREATE_TABLE_MODAL, toggleModal } from "$lib/store/modal.store"
   import Command from "$lib/components/blocks/command/command.svelte"
   import { role } from "$lib/store/space-member.store"
-  import { bases as basesStore } from "$lib/store/base.store"
+  import { baseId, bases as basesStore } from "$lib/store/base.store"
   import NavTools from "$lib/components/blocks/nav/nav-tools.svelte"
   import { onMount } from "svelte"
   import type { ComponentType } from "svelte"
@@ -42,7 +42,9 @@
   let tables = derived(indexDataStore, ($indexDataStore) => $indexDataStore.data?.tables?.filter(Boolean) ?? [])
   let bases = derived(indexDataStore, ($indexDataStore) => $indexDataStore.data?.bases?.filter(Boolean) ?? [])
   let baseNames = derived(bases, ($bases) => $bases.map((base) => base?.name).filter(Boolean) as string[])
-  let baseTables = derived(tables, ($tables) => $tables.filter((table) => table?.baseId === $page.params.baseId))
+  let baseTables = derived([tables, page, baseId], ([$tables, $page, $baseId]) =>
+    $tables.filter((table) => table?.baseId === $page.params.baseId || table?.baseId === $baseId),
+  )
   let baseTableNames = derived(
     baseTables,
     ($baseTables) => $baseTables.map((table) => table?.name).filter(Boolean) as string[],
@@ -106,7 +108,7 @@
   </Resizable.Pane>
 </Resizable.PaneGroup>
 
-<CreateTableSheet />
+<CreateTableSheet tableNames={$baseTableNames} />
 <ImportTableDialog tableNames={$baseTableNames} />
 {#if CreateBaseDialog}
   <CreateBaseDialog baseNames={$baseNames} />

package.json

@@ -1,6 +1,6 @@
 {
   "name": "undb",
-  "version": "1.0.0-74",
+  "version": "1.0.0-75",
   "private": true,
   "scripts": {
     "build": "NODE_ENV=production bun --bun turbo build",

packages/persistence/package.json

@@ -26,6 +26,7 @@
     "@undb/webhook": "workspace:*",
     "date-fns": "^3.6.0",
     "drizzle-orm": "^0.33.0",
+    "es-toolkit": "^1.19.0",
     "kysely": "^0.27.4",
     "kysely-bun-sqlite": "^0.3.2",
     "radash": "^12.1.0",

packages/persistence/src/record/record.repository.ts

@@ -18,6 +18,7 @@ import {
   type RecordId,
   type TableDo,
 } from "@undb/table"
+import { chunk } from "es-toolkit/array"
 import { sql, type CompiledQuery, type ExpressionBuilder } from "kysely"
 import { getAnonymousTransaction } from "../ctx"
 import type { IRecordQueryBuilder } from "../qb"
@@ -87,7 +88,7 @@ export class RecordRepository implements IRecordRepository {
     await this.outboxService.save(record)
   }
 
-  async bulkInsert(table: TableDo, records: RecordDO[]): Promise<void> {
+  async #bulkInsert(table: TableDo, records: RecordDO[]): Promise<void> {
     const trx = getAnonymousTransaction()
     const context = executionContext.getStore()
     const userId = context?.user?.userId!
@@ -137,6 +138,13 @@ export class RecordRepository implements IRecordRepository {
     await this.outboxService.saveMany(records)
   }
 
+  async bulkInsert(table: TableDo, records: RecordDO[]): Promise<void> {
+    const chunked = chunk(records, 5000)
+    for (const chunk of chunked) {
+      await this.#bulkInsert(table, chunk)
+    }
+  }
+
   async findOne(table: TableDo, spec: Option<RecordComositeSpecification>): Promise<Option<RecordDO>> {
     const foreignTables = await this.getForeignTables(table, table.schema.fields)
     const qb = this.helper.createQuery(table, foreignTables, table.schema.fields, spec)

packages/table/src/methods/duplicate-view.method.ts

@@ -10,7 +10,6 @@ export function duplicateViewMethod(this: TableDo, dto: IDuplicateViewDTO): Opti
   const spec = view.$duplicate(dto)
 
   const names = this.views.views.map((v) => v.name.value)
-  console.log(names)
   applyRules(new ViewNameShouldBeUnique(names))
 
   if (spec.isSome()) {

packages/table/src/modules/schema/fields/field-value.factory.ts

@@ -37,7 +37,6 @@ export class FieldValueFactory {
       .with({ type: "string" }, (field) => Some(new StringFieldValue(field.valueSchema.parse(value))))
       .with({ type: "select" }, (field) => {
         const parsedValue = SelectFieldValue.parseValue(value, field as SelectField)
-        console.log(parsedValue)
         return Some(new SelectFieldValue(field.valueSchema.parse(parsedValue)))
       })
       .with({ type: "reference" }, (field) => Some(new ReferenceFieldValue(field.valueSchema.parse(value))))