Merge pull request #23 from hadynz/stemming

Fix suggestion menu display. Made highlighting faster

manifest.json

@@ -2,7 +2,7 @@
   "id": "obsidian-sidekick",
   "name": "Sidekick",
   "description": "A companion to identify hidden connections that match your tags and pages",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "minAppVersion": "0.13.8",
   "author": "Hady Osman",
   "authorUrl": "https://hady.geek.nz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "obsidian-sidekick",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "A companion to identify hidden connections that match your tags and pages",
   "main": "src/index.ts",
   "repository": {
@@ -23,7 +23,7 @@
     "prettier": "prettier --write '**/*.{ts,js,css,html}'",
     "lint": "tsc --noemit && eslint . --ext .ts",
     "clean": "rimraf dist main.js*",
-    "test": "jest --passWithNoTests",
+    "test": "jest --passWithNoTests --verbose",
     "test-watch": "jest --watch",
     "dev": "NODE_ENV=development webpack && cp ./dist/main.js* .",
     "build": "NODE_ENV=production webpack",
@@ -70,8 +70,10 @@
   },
   "dependencies": {
     "@tanishiking/aho-corasick": "^0.0.1",
+    "@types/natural": "^5.1.0",
     "lodash": "^4.17.21",
     "lokijs": "^1.5.12",
+    "natural": "^5.1.13",
     "tiny-typed-emitter": "^2.1.0"
   }
 }

src/index.ts

@@ -14,7 +14,6 @@ export default class TagsAutosuggestPlugin extends Plugin {
 
     const pluginHelper = new PluginHelper(this);
     const indexer = new Indexer(pluginHelper);
-    const search = new Search(indexer);
 
     this.registerEditorExtension(this.editorExtension);
 
@@ -23,13 +22,15 @@ export default class TagsAutosuggestPlugin extends Plugin {
     pluginHelper.onFileMetadataChanged((file) => indexer.replaceFileIndices(file));
 
     // Re/load highlighting extension after any changes to index
-    indexer.on('indexRebuilt', () =>
-      this.updateEditorExtension(suggestionsExtension(search, this.app))
-    );
-
-    indexer.on('indexUpdated', () =>
-      this.updateEditorExtension(suggestionsExtension(search, this.app))
-    );
+    indexer.on('indexRebuilt', () => {
+      const search = new Search(indexer);
+      this.updateEditorExtension(suggestionsExtension(search, this.app));
+    });
+
+    indexer.on('indexUpdated', () => {
+      const search = new Search(indexer);
+      this.updateEditorExtension(suggestionsExtension(search, this.app));
+    });
 
     // Build search index on startup (very expensive process)
     pluginHelper.onLayoutReady(() => indexer.buildIndex());

src/indexing/indexer.ts

@@ -2,11 +2,12 @@ import lokijs from 'lokijs';
 import { TypedEmitter } from 'tiny-typed-emitter';
 import type { TFile } from 'obsidian';
 
+import { tokenize } from './utils';
 import type { PluginHelper } from '../plugin-helper';
 
 type Document = {
   fileCreationTime: number;
-  type: 'tag' | 'alias' | 'page';
+  type: 'tag' | 'alias' | 'page' | 'page-token';
   keyword: string;
   replaceText: string;
 };
@@ -65,6 +66,15 @@ export class Indexer extends TypedEmitter<IndexerEvents> {
       replaceText: `[[${file.basename}]]`,
     });
 
+    tokenize(file.basename).forEach((token) => {
+      this.documents.insert({
+        fileCreationTime: file.stat.ctime,
+        type: 'page-token',
+        keyword: token,
+        replaceText: `[[${file.basename}]]`,
+      });
+    });
+
     this.pluginHelper.getAliases(file).forEach((alias) => {
       this.documents.insert({
         fileCreationTime: file.stat.ctime,

src/indexing/utils.spec.ts

@@ -0,0 +1,29 @@
+import { tokenize } from './utils';
+
+describe('tokenize', () => {
+  const dataSet = [
+    {
+      sentence: 'The quick brown fox jumps over the lazy dog.',
+      expected: ['quick', 'brown', 'fox', 'jump', 'lazi', 'dog'],
+    },
+    {
+      sentence: 'GitHub Forks',
+      expected: ['github', 'fork'],
+    },
+    {
+      sentence: 'John    Doe',
+      expected: ['john', 'doe'],
+    },
+    {
+      sentence: 'Approximate Inference',
+      expected: ['approxim', 'infer'],
+    },
+  ];
+
+  dataSet.forEach(({ sentence, expected }) => {
+    it(`Tokenizes and removes stop words ("${sentence}", [${expected}]`, () => {
+      const tokens = tokenize(sentence);
+      expect(tokens).toEqual(expected);
+    });
+  });
+});

src/indexing/utils.ts

@@ -0,0 +1,5 @@
+import natural from 'natural';
+
+export const tokenize = (text: string): string[] => {
+  return natural.PorterStemmer.tokenizeAndStem(text);
+};

src/search/index.ts

@@ -14,25 +14,28 @@ const isEqual = (a: Emit, b: Emit) => {
 };
 
 export default class Search {
-  constructor(private indexer: Indexer) {}
+  private trie: Trie;
 
-  public getReplacementSuggestions(keyword: string): string[] {
-    const keywords = this.indexer.getDocumentsByKeyword(keyword).map((doc) => doc.replaceText);
-    return _.uniq(keywords);
-  }
-
-  public find(text: string): SearchResult[] {
+  constructor(private indexer: Indexer) {
     const keywords = this.indexer.getKeywords();
 
-    const trie = new Trie(keywords, {
+    // Generating the Trie is expensive, so we only do it once
+    this.trie = new Trie(keywords, {
       allowOverlaps: false,
       onlyWholeWords: true,
       caseInsensitive: true,
     });
+  }
 
+  public getReplacementSuggestions(keyword: string): string[] {
+    const keywords = this.indexer.getDocumentsByKeyword(keyword).map((doc) => doc.replaceText);
+    return _.uniq(keywords);
+  }
+
+  public find(text: string): SearchResult[] {
     const redactedText = this.redactText(text); // Redact text that we don't want to be searched
 
-    const results = trie.parseText(redactedText);
+    const results = this.trie.parseText(redactedText);
 
     return this.mapToSearchResults(results);
   }