todo parsing using rust

.gitignore

@@ -0,0 +1,2 @@
+xotodo-backend/pkg
+xotodo-backend/target

file-parser.test.ts

@@ -0,0 +1,32 @@
+import { assertEquals } from "https://deno.land/[email protected]/testing/asserts.ts"
+import { parseFile } from "./file-parser.ts"
+import init from "./xotodo-backend/pkg/xotodo_backend.js"
+
+async function setup(): Promise<string> {
+  await init(Deno.readFile("xotodo-backend/pkg/xotodo_backend_bg.wasm"))
+  const data = await Deno.readFile("./test.md")
+  const decoder = new TextDecoder("utf-8")
+  const text = decoder.decode(data)
+  return text
+}
+
+Deno.test({
+  name: "parsing multiple lines",
+  fn: async () => {
+    const text = await setup()
+    const todos = parseFile(text)
+    assertEquals(todos.length, 3)
+    assertEquals(todos[2].status, 'closed')
+    assertEquals(todos[2].lineNumber, 23)
+  },
+})
+
+Deno.test({
+  name: "parsing invalid date",
+  fn: async () => {
+    await setup()
+    const todos = parseFile("OTODO: test @due:2021-99-32")
+    assertEquals(todos.length, 1)
+    assertEquals(todos[0].dueDate, undefined)
+  },
+})

file-parser.ts

@@ -0,0 +1,23 @@
+import { v1 } from "https://deno.land/[email protected]/uuid/mod.ts"
+import { parse } from "./xotodo-backend/pkg/xotodo_backend.js"
+
+export interface Todo {
+  title: string
+  status: 'open' | 'closed'
+  id: string
+  tsIndexed: Date
+  filePath?: string
+  shortPath?: string
+  lineNumber: number
+  dueDate?: Date
+}
+
+export function parseFile(text: string): Todo[] {
+  const todos = parse(text) as Todo[]
+  todos.forEach((t) => {
+    t.tsIndexed = new Date(t.tsIndexed)
+    t.id = v1.generate().toString()
+    t.dueDate = t.dueDate ? new Date(t.dueDate) : undefined
+  })
+  return todos
+}

main.ts

@@ -1,5 +1,7 @@
-import { Todo, watch } from "./watcher.ts"
+import { watch } from "./watcher.ts"
 import { serve } from "https://deno.land/[email protected]/http/server.ts"
+import init from "./xotodo-backend/pkg/xotodo_backend.js"
+import { Todo } from "./file-parser.ts"
 
 let websocket: WebSocket
 const todoMap = new Map<string, Todo[]>()
@@ -74,6 +76,10 @@ function onUpdate() {
 
 // start
 serve(handleRequest)
+
+// https://github.com/rustwasm/wasm-pack/issues/672
+await init(Deno.readFile("xotodo-backend/pkg/xotodo_backend_bg.wasm"))
+
 watch(onUpdate)
 onUpdate()
 

test.md

@@ -0,0 +1,30 @@
+The capacity of a vector is the amount of space allocated for any future elements that will be added onto the vector. This is not to be confused with the length of a vector, which specifies the number of actual elements within the vector. 
+If a vector’s length exceeds its capacity, its capacity will automatically be increased, but its elements will have to be reallocated.
+
+For example, a vector with capacity 10 and length 0 would be an empty vector with space for 10 more elements. 
+Pushing 10 or fewer elements onto the vector will not change its capacity or cause reallocation to occur. 
+However, if the vector’s length is increased to 11, it will have to reallocate, which can be slow. 
+For this reason, it is recommended to use Vec::with_capacity whenever possible to specify how big the vector is expected to get.
+OTODO: learn this chapter @due:2022-01-01
+The capacity of a vector is the amount of space allocated for any future elements that will be added onto the vector. This is not to be confused with the length of a vector, which specifies the number of actual elements within the vector. 
+If a vector’s length exceeds its capacity, its capacity will automatically be increased, but its elements will have to be reallocated.
+
+For example, a vector with capacity 10 and length 0 would be an empty vector with space for 10 more elements. 
+Pushing 10 or fewer elements onto the vector will not change its capacity or cause reallocation to occur. 
+However, if the vector’s length is increased to 11, it will have to reallocate, which can be slow. 
+For this reason, it is recommended to use Vec::with_capacity whenever possible to specify how big the vector is expected to get.
+The capacity of a vector is the amount of space allocated for any future elements that will be added onto the vector. This is not to be confused with the length of a vector, which specifies the number of actual elements within the vector. 
+If a vector’s length exceeds its capacity, its capacity will automatically be increased, but its elements will have to be reallocated.
+OTODO: learn this chapter too @due: 2022-01-01
+For example, a vector with capacity 10 and length 0 would be an empty vector with space for 10 more elements. 
+Pushing 10 or fewer elements onto the vector will not change its capacity or cause reallocation to occur. 
+However, if the vector’s length is increased to 11, it will have to reallocate, which can be slow. 
+For this reason, it is recommended to use Vec::with_capacity whenever possible to specify how big the vector is expected to get.
+XTODO: this is done
+The capacity of a vector is the amount of space allocated for any future elements that will be added onto the vector. This is not to be confused with the length of a vector, which specifies the number of actual elements within the vector. 
+If a vector’s length exceeds its capacity, its capacity will automatically be increased, but its elements will have to be reallocated.
+
+For example, a vector with capacity 10 and length 0 would be an empty vector with space for 10 more elements. 
+Pushing 10 or fewer elements onto the vector will not change its capacity or cause reallocation to occur. 
+However, if the vector’s length is increased to 11, it will have to reallocate, which can be slow. 
+For this reason, it is recommended to use Vec::with_capacity whenever possible to specify how big the vector is expected to get.

watcher.ts

@@ -1,18 +1,5 @@
 import { exists } from "https://deno.land/std/fs/mod.ts"
-import { v1 } from "https://deno.land/[email protected]/uuid/mod.ts"
-
-export interface Todo {
-  title: string
-  status: 'open' | 'closed' | 'removed'
-  id: string
-  tsIndexed: Date
-  filePath: string
-  shortPath: string
-  lineNumber: number
-  dueDate?: Date
-}
-
-type NumberedLine = [lineNumber: number, line: string]
+import { parseFile } from "./file-parser.ts"
 
 let currentGoogleDrivePath = ''
 setTimeout(checkGoogleDrivePath, 60 * 1000)
@@ -46,7 +33,7 @@ function existsFixed(path: string): Promise<boolean> {
 }
 
 const WATCHED_FOLDERS = [`${currentGoogleDrivePath}/My Drive/Research`, "/Users/roy"]
-const IGNORED_PATH_SEGMENTS = ["xotodo", "/Library", ".webpack", ".seafile-data", ".git"]
+const IGNORED_PATH_SEGMENTS = ["/xotodo", "/Library", "/."]
 
 // cleanup in case the any new ignored path segment was added
 for (const path of Object.keys({ ...localStorage })) {
@@ -89,7 +76,24 @@ export async function watch(onUpdate: () => void) {
           const data = await Deno.readFile(path)
           const text = decoder.decode(data)
           console.log("parsing", path)
-          parseFile(text, path, onUpdate)
+          const todos = parseFile(text)
+
+          WATCHED_FOLDERS.forEach((dir: string) => {
+            todos.forEach(t => {
+              t.filePath = path
+              t.shortPath = path.replace(dir, '')
+            })
+          })
+
+          if (todos.length > 0) {
+            localStorage.setItem(path, JSON.stringify(todos))
+            onUpdate()
+          } else if (localStorage.getItem(path)) {
+            // here, we previously had todos, but now we don't
+            localStorage.removeItem(path)
+            onUpdate()
+          }
+
         } catch (_) {
           console.log("parsing failed")
         }
@@ -98,48 +102,4 @@ export async function watch(onUpdate: () => void) {
   } catch (error) {
     console.error(error)
   }
-}
-
-function parseFile(text: string, filePath: string, onUpdate: () => void) {
-  const numberedLines = text.split('\n').map((l, i) => [i, l] as NumberedLine)
-  const openTodoLines = numberedLines.filter(nl => nl[1].includes('OTODO: '))
-  const openTodos = parseTodoLinesWith(openTodoLines, filePath, /^(.*)(OTODO: )(.*)/, 'open')
-  const closedTodoLines = numberedLines.filter(nl => nl[1].includes('XTODO: '))
-  const closedTodos = parseTodoLinesWith(closedTodoLines, filePath, /^(.*)(XTODO: )(.*)/, 'closed')
-  const todos = openTodos.concat(closedTodos)
-  if (todos.length > 0) {
-    localStorage.setItem(filePath, JSON.stringify(openTodos.concat(closedTodos)))
-    onUpdate()
-  } else if (localStorage.getItem(filePath)) {
-    // here, we previously had todos, but now we don't
-    localStorage.removeItem(filePath)
-    onUpdate()
-  }
-}
-
-function parseTodoLinesWith(lines: NumberedLine[], filePath: string, regex: RegExp, status: 'open' | 'closed' | 'removed'): Todo[] {
-  const numberedTodoStrings: [number, string][] = lines.map(nl => {
-    const str = (regex.exec(nl[1]) as string[])[3]
-    return [nl[0], str]
-  })
-
-  return numberedTodoStrings.map(numberedTodo => {
-    const lineNumber = numberedTodo[0] + 1
-    const str = numberedTodo[1]
-    const dateRegex = /@due: (\d\d\d\d-\d\d-\d\d)/
-    let dueDate: Date | undefined
-    if (dateRegex.test(str)) {
-      const dateStr = (dateRegex.exec(str) as string[])[1]
-      dueDate = new Date(dateStr)
-    }
-    const title = str.replace(dateRegex, '')
-    const id = v1.generate().toString()
-    const tsIndexed = new Date()
-    let shortPath = filePath
-    WATCHED_FOLDERS.forEach(dir => {
-      shortPath = shortPath.replace(dir, '')
-    })
-    console.log(`\textracted: ${title} @L${lineNumber}`)
-    return { title, status, id, tsIndexed, filePath, shortPath, lineNumber, dueDate }
-  })
-}
+}