wip

client/vscode/test/fixtures/workspace/.vscode/settings.json

@@ -3,8 +3,9 @@
   "opencodegraph.debug": true,
   "cody.autocomplete.experimental.graphContext": "opencodegraph",
   "opencodegraph.providers": {
-    "../../../../../../provider/docs/dist/provider.js": {
-      "index": "http://localhost:5900/@fs/home/sqs/tmp/ocg-provider-docs/vite-docs-web.index.json",
+    "../../../../../../provider/docs/dist/provider.cjs": {
+      // "index": "http://localhost:5900/@fs/home/sqs/tmp/ocg-provider-docs/vite-docs-web.index.json",
+      "index": "file:///home/sqs/tmp/ocg-provider-docs/vite-docs-web.index.json",
     },
   },
   "opencodegraph.providers2": {

provider/docs/bin/create-archive.ts

@@ -1,8 +1,8 @@
 import { readFile } from 'fs/promises'
 import path from 'path'
-import { CorpusArchive, createCorpusArchive } from '../src/corpus/archive/corpusArchive'
-import { createWebCorpusArchive, WebCorpusArchiveOptions } from '../src/corpus/archive/web/webCorpusArchive'
-import { Doc } from '../src/corpus/doc/doc'
+import { createCorpusArchive, type CorpusArchive } from '../src/corpus/archive/corpusArchive.ts'
+import { createWebCorpusArchive, type WebCorpusArchiveOptions } from '../src/corpus/archive/web/webCorpusArchive.ts'
+import { type Doc } from '../src/corpus/doc/doc.ts'
 
 type ArchiveKind = 'web' | 'file'
 const ARCHIVE_KINDS: Record<

provider/docs/bin/create-index.ts

@@ -1,7 +1,7 @@
 import path from 'path'
-import { CorpusArchive } from '../src/corpus/archive/corpusArchive'
-import { extractContentUsingMozillaReadability } from '../src/corpus/doc/contentExtractor'
-import { createCorpusIndex } from '../src/corpus/index/corpusIndex'
+import { type CorpusArchive } from '../src/corpus/archive/corpusArchive.ts'
+import { extractContentUsingMozillaReadability } from '../src/corpus/doc/contentExtractor.ts'
+import { createCorpusIndex } from '../src/corpus/index/corpusIndex.ts'
 
 function usage(): void {
     console.error()

provider/docs/bin/search.ts

@@ -1,7 +1,7 @@
 import { readFile } from 'fs/promises'
 import path from 'path'
-import { createClient } from '../src/client/client'
-import { fromJSON } from '../src/corpus/index/corpusIndex'
+import { createClient } from '../src/client/client.ts'
+import { fromJSON } from '../src/corpus/index/corpusIndex.ts'
 
 const args = process.argv.slice(2)
 

provider/docs/package.json

@@ -17,7 +17,7 @@
   ],
   "sideEffects": false,
   "scripts": {
-    "build": "tsc --build && esbuild src/provider/provider.ts --bundle --outfile=dist/provider.js --platform=browser --format=esm --sourcemap --external:node:crypto --external:fs --external:node:os --define:self=global --loader:.node=file --banner:js=\"const require = (await import('node:module')).createRequire(import.meta.url);const __filename = (await import('node:url')).fileURLToPath(import.meta.url);const __dirname = (await import('node:path')).dirname(__filename);\"",
+    "build": "tsc --build && esbuild src/provider/provider.ts --bundle --outfile=dist/provider.cjs --platform=node --format=cjs --sourcemap --define:self=global --loader:.node=file --alias:sharp=/dev/null --define:import.meta.url=import_meta_url --inject:./polyfill1.js",
     "test": "vitest",
     "create-archive": "node --no-warnings=ExperimentalWarning --experimental-specifier-resolution=node --loader ts-node/esm/transpile-only bin/create-archive.ts",
     "create-index": "node --no-warnings=ExperimentalWarning --experimental-specifier-resolution=node --loader ts-node/esm/transpile-only bin/create-index.ts",
@@ -26,10 +26,8 @@
   "dependencies": {
     "@mozilla/readability": "^0.5.0",
     "@opencodegraph/provider": "workspace:*",
-    "@xenova/transformers": "^2.12.1",
-    "better-localstorage": "^1.0.5",
+    "@xenova/transformers": "^2.13.4",
     "buffer": "^6.0.3",
-    "env-paths": "^3.0.0",
     "jsdom": "^23.0.1",
     "lru-cache": "^10.1.0",
     "onnxruntime-web": "*"

provider/docs/polyfill1.js

@@ -0,0 +1,3 @@
+export const import_meta_url =
+  typeof document === 'undefined' ? new (require('url'.replace('', '')).URL)('file:' + __filename).href :
+    (document.currentScript && document.currentScript.src || new URL('main.js', document.baseURI).href)

provider/docs/src/client/client.ts

@@ -1,8 +1,8 @@
-import { type DocID } from '../corpus/doc/doc'
-import { type CorpusIndex, type IndexedDoc } from '../corpus/index/corpusIndex'
-import { type Logger } from '../logger'
-import { type Query, type SearchResult } from '../search/types'
-import { search } from './search'
+import { type DocID } from '../corpus/doc/doc.ts'
+import { type CorpusIndex, type IndexedDoc } from '../corpus/index/corpusIndex.ts'
+import { type Logger } from '../logger.ts'
+import { type Query, type SearchResult } from '../search/types.ts'
+import { search } from './search.ts'
 
 /**
  * A client for searching a {@link CorpusIndex}.

provider/docs/src/client/search.ts

@@ -1,10 +1,10 @@
-import { type ChunkIndex } from '../corpus/doc/chunks'
-import { type DocID } from '../corpus/doc/doc'
-import { type CorpusIndex } from '../corpus/index/corpusIndex'
-import { type Logger } from '../logger'
-import { embeddingsSearch } from '../search/embeddings'
-import { keywordSearch } from '../search/keyword'
-import { type Query, type SearchResult } from '../search/types'
+import { type ChunkIndex } from '../corpus/doc/chunks.ts'
+import { type DocID } from '../corpus/doc/doc.ts'
+import { type CorpusIndex } from '../corpus/index/corpusIndex.ts'
+import { type Logger } from '../logger.ts'
+import { embeddingsSearch } from '../search/embeddings.ts'
+import { keywordSearch } from '../search/keyword.ts'
+import { type Query, type SearchResult } from '../search/types.ts'
 
 export interface SearchOptions {
     logger?: Logger

provider/docs/src/corpus/archive/corpusArchive.ts

@@ -1,5 +1,5 @@
-import { contentID } from '../cache/contentID'
-import { type Doc } from '../doc/doc'
+import { contentID } from '../cache/contentID.ts'
+import { type Doc } from '../doc/doc.ts'
 
 export interface CorpusArchive {
     /**

provider/docs/src/corpus/archive/web/webCorpusArchive.test.ts

@@ -1,7 +1,7 @@
 import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from 'vitest'
 import createFetchMock from 'vitest-fetch-mock'
-import { type Doc } from '../../doc/doc'
-import { createWebCorpusArchive, urlHasPrefix } from './webCorpusArchive'
+import { type Doc } from '../../doc/doc.ts'
+import { createWebCorpusArchive, urlHasPrefix } from './webCorpusArchive.ts'
 
 describe('createWebCorpusSource', () => {
     const fetchMocker = createFetchMock(vi)

provider/docs/src/corpus/archive/web/webCorpusArchive.ts

@@ -1,8 +1,8 @@
-import { parseDOM } from '../../../dom'
-import { type Logger } from '../../../logger'
-import { type Doc } from '../../doc/doc'
-import { createCorpusArchive, type CorpusArchive } from '../corpusArchive'
-import { createCrawlQueue } from './crawlQueue'
+import { parseDOM } from '../../../dom.ts'
+import { type Logger } from '../../../logger.ts'
+import { type Doc } from '../../doc/doc.ts'
+import { createCorpusArchive, type CorpusArchive } from '../corpusArchive.ts'
+import { createCrawlQueue } from './crawlQueue.ts'
 
 export interface WebCorpusArchiveOptions {
     /**

provider/docs/src/corpus/cache/contentID.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from 'vitest'
-import { contentID } from './contentID'
+import { contentID } from './contentID.ts'
 
 describe('contentID', () => {
     test('returns the content ID', async () => {