wip

provider/docs/README.md

@@ -63,3 +63,4 @@ TODOs:
 - make it slurp up gdocs/confluence/markdown in repos
 - show OCG annotations (but in a way that doesn't overlay lines in the file, is more passive?)
 - show a demo of Cody working with this
+- show docs most relevant to the current visible portion or the selection, not the whole file

provider/docs/package.json

@@ -28,6 +28,7 @@
     "@mozilla/readability": "^0.5.0",
     "@opencodegraph/provider": "workspace:*",
     "@xenova/transformers": "^2.12.1",
+    "better-localstorage": "^1.0.5",
     "buffer": "^6.0.3",
     "env-paths": "^3.0.0",
     "jsdom": "^23.0.1",

provider/docs/src/corpus/cache/indexedDB.ts

@@ -0,0 +1,36 @@
+/// <reference lib="dom" />
+
+import IndexedDBStorage from 'better-localstorage'
+import { type ContentID, type CorpusCache } from './cache'
+
+/**
+ * Create a {@link CorpusCache} that stores cache data using IndexedDB.
+ */
+export function createIndexedDBCorpusCache(keyPrefix: string): CorpusCache {
+    const storage = new IndexedDBStorage(keyPrefix)
+
+    function storageKey(contentID: ContentID, key: string): string {
+        return `${keyPrefix}:${contentID}:${key}`
+    }
+
+    return {
+        async get(contentID, key) {
+            const k = storageKey(contentID, key)
+            const data = await storage.getItem(k)
+            try {
+                return data ?? null
+            } catch (error) {
+                // TODO(sqs): cast because https://github.com/dreamsavior/Better-localStorage/pull/1
+                await (storage as any).delete(k)
+                throw error
+            }
+        },
+        async set(contentID, key, value) {
+            try {
+                await storage.setItem(storageKey(contentID, key), value)
+            } catch (error) {
+                console.error(`failed to store data for ${contentID}:${key}`, error)
+            }
+        },
+    }
+}

provider/docs/src/corpus/cache/localStorage.ts

@@ -3,7 +3,7 @@
 import { type ContentID, type CorpusCache } from './cache'
 
 /**
- * Create a {@link CorpusCache} that stores cache data localStorage (using the Web Storage API).
+ * Create a {@link CorpusCache} that stores cache data in localStorage (using the Web Storage API).
  */
 export function createWebStorageCorpusCache(storage: Storage, keyPrefix: string): CorpusCache {
     function storageKey(contentID: ContentID, key: string): string {

provider/docs/src/corpus/index.ts

@@ -55,8 +55,7 @@ export async function indexCorpus(
     const indexedDocs: IndexedDoc[] = []
 
     for (const doc of data.docs) {
-        const USE_NOOP_CACHE = typeof window !== 'undefined' // TODO(sqs): dont cache in browser
-        const content = await cachedExtractContent(USE_NOOP_CACHE ? noopCache : cache, contentExtractor, doc)
+        const content = await cachedExtractContent(cache, contentExtractor, doc)
 
         const chunks = chunk(content?.content ?? doc.text, { isMarkdown: doc.text.includes('##') })
 

provider/docs/src/provider/provider.ts

@@ -6,6 +6,7 @@ import {
     type CapabilitiesResult,
 } from '@opencodegraph/provider'
 import { indexCorpus } from '../corpus'
+import { createIndexedDBCorpusCache } from '../corpus/cache/indexedDB'
 import { createWebStorageCorpusCache } from '../corpus/cache/localStorage'
 import { corpusData } from '../corpus/data'
 import { extractContentUsingMozillaReadability } from '../corpus/doc/contentExtractor'
@@ -25,7 +26,11 @@ export interface Settings {
 }
 
 const CORPUS_CACHE =
-    typeof localStorage !== 'undefined' ? createWebStorageCorpusCache(localStorage, 'ocg-provider-docs') : undefined
+    typeof indexedDB !== 'undefined'
+        ? createIndexedDBCorpusCache('ocg-provider-docs')
+        : typeof localStorage !== 'undefined'
+        ? createWebStorageCorpusCache(localStorage, 'ocg-provider-docs')
+        : undefined
 
 /**
  * An [OpenCodeGraph](https://opencodegraph.org) provider that adds contextual documentation to your