From 53629c9c07392b5379700ca1799d57a6553a0b60 Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Mon, 23 Sep 2024 15:55:19 -0700 Subject: [PATCH 1/3] wip web --- recipes/llm-voice-assistant/web/package.json | 5 ++- .../web/public/controller.js | 17 +++++--- .../llm-voice-assistant/web/public/index.html | 1 + recipes/llm-voice-assistant/web/server.js | 43 +++++++++++++++++++ recipes/llm-voice-assistant/web/src/main.ts | 24 ++++++++--- recipes/llm-voice-assistant/web/yarn.lock | 16 +++++-- 6 files changed, 89 insertions(+), 17 deletions(-) create mode 100644 recipes/llm-voice-assistant/web/server.js diff --git a/recipes/llm-voice-assistant/web/package.json b/recipes/llm-voice-assistant/web/package.json index d2128a9..86bdce3 100644 --- a/recipes/llm-voice-assistant/web/package.json +++ b/recipes/llm-voice-assistant/web/package.json @@ -8,7 +8,7 @@ "build:types": "tsc --declaration --declarationMap --emitDeclarationOnly --outDir ./types", "build": "npm-run-all --parallel build:**", "lint": "eslint . --ext .js,.ts", - "start": "yarn run http-server -a localhost -p 5000" + "start": "node server.js" }, "keywords": [ "Picovoice", @@ -21,7 +21,7 @@ "dependencies": { "@picovoice/cheetah-web": "^2.0.0", "@picovoice/orca-web": "^1.0.0", - "@picovoice/picollm-web": "=1.0.7", + "@picovoice/picollm-web": "file:../../../../picollm/binding/web", "@picovoice/porcupine-web": "^3.0.3", "@picovoice/web-voice-processor": "^4.0.9" }, @@ -39,6 +39,7 @@ "async-mutex": "^0.5.0", "eslint": "^8.57.0", "http-server": "^14.1.1", + "mime-types": "^2.1.35", "npm-run-all": "^4.1.5", "prettier": "^3.2.5", "rollup": "^4.18.0", diff --git a/recipes/llm-voice-assistant/web/public/controller.js b/recipes/llm-voice-assistant/web/public/controller.js index 4922d0c..2fbb23c 100644 --- a/recipes/llm-voice-assistant/web/public/controller.js +++ b/recipes/llm-voice-assistant/web/public/controller.js @@ -95,6 +95,7 @@ window.onload = () => { let completeTranscript = ""; let audioStream; let streamCalls = 0; + let isDetected = false; try { await Picovoice.init(accessKey.value, @@ -104,6 +105,7 @@ window.onload = () => { }, { onDetection: () => { + isDetected = true; message.innerText = "Wake word detected, utter your request or question..."; humanElem = startHumanMessage(); }, @@ -111,14 +113,15 @@ window.onload = () => { completeTranscript += transcript; addMessage(humanElem, transcript); }, - onEndpoint: async () => { - message.innerHTML = "Generating
"; + onEndpoint: () => { + isDetected = false; + message.innerHTML = "Generating
say `Picovoice` to interrupt"; llmElem = startLLMMessage(); }, onText: (text) => { addMessage(llmElem, text); }, - onStream: async (pcm) => { + onStream: (pcm) => { audioStream.stream(pcm); if (streamCalls > 1) { audioStream.play(); @@ -126,14 +129,16 @@ window.onload = () => { streamCalls++; } }, - onComplete: async () => { + onComplete: async (interrupted) => { if (streamCalls <= 2) { audioStream.play(); } await audioStream.waitPlayback(); - await Picovoice.start(); - message.innerText = "Say `Picovoice`" + if (!interrupted && !isDetected) { + await Picovoice.start(); + message.innerText = "Say `Picovoice`" + } streamCalls = 0; }, }); diff --git a/recipes/llm-voice-assistant/web/public/index.html b/recipes/llm-voice-assistant/web/public/index.html index c976345..9919e01 100644 --- a/recipes/llm-voice-assistant/web/public/index.html +++ b/recipes/llm-voice-assistant/web/public/index.html @@ -129,6 +129,7 @@ height: 12pt; animation: spin 2s linear infinite; margin-left: 10px; + margin-right: 10px; } @keyframes spin { diff --git a/recipes/llm-voice-assistant/web/server.js b/recipes/llm-voice-assistant/web/server.js new file mode 100644 index 0000000..91f6ea8 --- /dev/null +++ b/recipes/llm-voice-assistant/web/server.js @@ -0,0 +1,43 @@ +const http = require('http'); +const fs = require('fs'); +const path = require('path'); +const mime = require('mime-types'); + +const PORT = process.env.PORT || 5000; +const HOST = '127.0.0.1'; // Listen on localhost +const publicDir = path.join(__dirname, 'public'); + +const server = http.createServer((req, res) => { + const url = (req.url === '/') ? '/index.html' : req.url; + const filePath = path.join(publicDir, url); + const contentType = mime.lookup(filePath) || 'application/octet-stream'; + + console.log(url) + + fs.readFile(filePath, (err, content) => { + if (err) { + if (err.code === 'ENOENT') { + // File not found + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('404 Not Found'); + } else { + // Server error + res.writeHead(500, { 'Content-Type': 'text/plain' }); + res.end(`500 Internal Server Error: ${err.code}`); + } + } else { + // Success + res.writeHead(200, { + 'Content-Type': contentType, + 'Content-Length': content.length, + 'Cross-Origin-Opener-Policy': 'same-origin', + 'Cross-Origin-Embedder-Policy': 'require-corp' + }); + res.end(content); + } + }); +}); + +server.listen(PORT, HOST, () => { + console.log(`Server is running on http://${HOST}:${PORT}`); +}); diff --git a/recipes/llm-voice-assistant/web/src/main.ts b/recipes/llm-voice-assistant/web/src/main.ts index 581d317..b8fa421 100644 --- a/recipes/llm-voice-assistant/web/src/main.ts +++ b/recipes/llm-voice-assistant/web/src/main.ts @@ -3,7 +3,7 @@ import { Mutex } from 'async-mutex'; import { BuiltInKeyword, PorcupineDetection, PorcupineWorker } from '@picovoice/porcupine-web'; import { CheetahTranscript, CheetahWorker } from '@picovoice/cheetah-web'; import { OrcaStreamWorker, OrcaWorker } from '@picovoice/orca-web'; -import { Dialog, PicoLLMModel, PicoLLMWorker } from '@picovoice/picollm-web'; +import { Dialog, PicoLLMEndpoint, PicoLLMModel, PicoLLMWorker } from '@picovoice/picollm-web'; import { WebVoiceProcessor } from '@picovoice/web-voice-processor'; type PvObject = { @@ -21,7 +21,7 @@ type PvCallback = { onEndpoint: () => void; onText: (text: string) => void; onStream: (pcm: Int16Array) => void; - onComplete: () => void; + onComplete: (interrupted: boolean) => Promise; } let object: PvObject | null = null; @@ -43,9 +43,14 @@ const init = async ( const detectionCallback = async (detection: PorcupineDetection): Promise => { if (detection.index === 0) { + pllm.interrupt(); + await WebVoiceProcessor.subscribe(cheetah); await WebVoiceProcessor.unsubscribe(porcupine); + + const release = await mutex.acquire(); onDetection(detection); + release(); } }; @@ -59,6 +64,7 @@ const init = async ( const transcriptCallback = async (transcript: CheetahTranscript): Promise => { if (transcript.isEndpoint) { + await WebVoiceProcessor.subscribe(porcupine); await WebVoiceProcessor.unsubscribe(cheetah); cheetah.flush(); } @@ -102,6 +108,7 @@ const init = async ( '', // Gemma '<|endoftext|>', // Phi-2 '<|eot_id|>', // Llama-3 + '<|end|>', '<|user|>', '<|assistant|>' ]; const onCheetahFlushed = async (): Promise => { @@ -109,15 +116,15 @@ const init = async ( transcripts = []; dialog.addHumanRequest(prompt); - const { completion, completionTokens } = await pllm.generate(dialog.prompt(), { + const release = await mutex.acquire(); + + const { completion, completionTokens, endpoint } = await pllm.generate(dialog.prompt(), { completionTokenLimit: 128, stopPhrases: stopPhrases, streamCallback: async token => { if (!stopPhrases.includes(token)) { onText(token); - const release = await mutex.acquire(); const pcm = await stream.synthesize(token); - release(); synthesized++; if (pcm !== null) { onStream(pcm); @@ -127,6 +134,9 @@ const init = async ( } } }); + + release(); + dialog.addLLMResponse(completion); const waitForSynthesize = (): Promise => new Promise(resolve => { @@ -145,7 +155,9 @@ const init = async ( } synthesized = 0; stopTokens = 0; - onComplete(); + + const interrupted = (endpoint === PicoLLMEndpoint.INTERRUPTED); + await onComplete(interrupted); }; object = { diff --git a/recipes/llm-voice-assistant/web/yarn.lock b/recipes/llm-voice-assistant/web/yarn.lock index 8681cd8..4c579da 100644 --- a/recipes/llm-voice-assistant/web/yarn.lock +++ b/recipes/llm-voice-assistant/web/yarn.lock @@ -1069,10 +1069,8 @@ dependencies: "@picovoice/web-utils" "=1.4.2" -"@picovoice/picollm-web@=1.0.7": +"@picovoice/picollm-web@file:../../../../picollm/binding/web": version "1.0.7" - resolved "https://registry.yarnpkg.com/@picovoice/picollm-web/-/picollm-web-1.0.7.tgz#98111e9d28f149e6f95e802b1d32e5114413662d" - integrity sha512-vllhlFaDhhsKuuUCidq60YXH7DTwvJBRTI1QUphxfRGCtm7Th/DF9lX6e48PIFFavT+VIeG/1AgJ9oT9Z/Z9og== dependencies: "@picovoice/web-utils" "~1.4.1" @@ -2629,6 +2627,18 @@ micromatch@^4.0.4: braces "^3.0.3" picomatch "^2.3.1" +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.35: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mime@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" From 09af44c23e8dbc6fbfcdda4618b178318714fa4d Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Mon, 23 Sep 2024 15:57:23 -0700 Subject: [PATCH 2/3] more checks --- recipes/llm-voice-assistant/web/public/controller.js | 1 + recipes/llm-voice-assistant/web/src/audio_stream.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/recipes/llm-voice-assistant/web/public/controller.js b/recipes/llm-voice-assistant/web/public/controller.js index 2fbb23c..489e2eb 100644 --- a/recipes/llm-voice-assistant/web/public/controller.js +++ b/recipes/llm-voice-assistant/web/public/controller.js @@ -135,6 +135,7 @@ window.onload = () => { } await audioStream.waitPlayback(); + audioStream.clear(); if (!interrupted && !isDetected) { await Picovoice.start(); message.innerText = "Say `Picovoice`" diff --git a/recipes/llm-voice-assistant/web/src/audio_stream.ts b/recipes/llm-voice-assistant/web/src/audio_stream.ts index c0c641c..adf9067 100644 --- a/recipes/llm-voice-assistant/web/src/audio_stream.ts +++ b/recipes/llm-voice-assistant/web/src/audio_stream.ts @@ -58,6 +58,11 @@ class AudioStream { }); } + public clear(): void { + this._audioBuffers = []; + this._isPlaying = false; + } + private createBuffer(pcm: Int16Array): AudioBuffer { const buffer = this._audioContext.createBuffer( 1, From cd58ce0d5a7e265cd456324640a56717fa7fdfd3 Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Tue, 1 Oct 2024 17:12:37 -0700 Subject: [PATCH 3/3] update pico cookbook --- recipes/llm-voice-assistant/web/package.json | 5 +-- .../web/public/controller.js | 6 +-- recipes/llm-voice-assistant/web/server.js | 43 ------------------- recipes/llm-voice-assistant/web/src/main.ts | 2 +- recipes/llm-voice-assistant/web/yarn.lock | 29 ++++++------- 5 files changed, 19 insertions(+), 66 deletions(-) delete mode 100644 recipes/llm-voice-assistant/web/server.js diff --git a/recipes/llm-voice-assistant/web/package.json b/recipes/llm-voice-assistant/web/package.json index 86bdce3..54d1842 100644 --- a/recipes/llm-voice-assistant/web/package.json +++ b/recipes/llm-voice-assistant/web/package.json @@ -8,7 +8,7 @@ "build:types": "tsc --declaration --declarationMap --emitDeclarationOnly --outDir ./types", "build": "npm-run-all --parallel build:**", "lint": "eslint . --ext .js,.ts", - "start": "node server.js" + "start": "yarn run http-server -a localhost -p 5000" }, "keywords": [ "Picovoice", @@ -21,7 +21,7 @@ "dependencies": { "@picovoice/cheetah-web": "^2.0.0", "@picovoice/orca-web": "^1.0.0", - "@picovoice/picollm-web": "file:../../../../picollm/binding/web", + "@picovoice/picollm-web": "~1.1.0", "@picovoice/porcupine-web": "^3.0.3", "@picovoice/web-voice-processor": "^4.0.9" }, @@ -39,7 +39,6 @@ "async-mutex": "^0.5.0", "eslint": "^8.57.0", "http-server": "^14.1.1", - "mime-types": "^2.1.35", "npm-run-all": "^4.1.5", "prettier": "^3.2.5", "rollup": "^4.18.0", diff --git a/recipes/llm-voice-assistant/web/public/controller.js b/recipes/llm-voice-assistant/web/public/controller.js index 489e2eb..fb59753 100644 --- a/recipes/llm-voice-assistant/web/public/controller.js +++ b/recipes/llm-voice-assistant/web/public/controller.js @@ -130,12 +130,12 @@ window.onload = () => { } }, onComplete: async (interrupted) => { - if (streamCalls <= 2) { - audioStream.play(); + audioStream.play(); + if (interrupted && isDetected) { + audioStream.clear(); } await audioStream.waitPlayback(); - audioStream.clear(); if (!interrupted && !isDetected) { await Picovoice.start(); message.innerText = "Say `Picovoice`" diff --git a/recipes/llm-voice-assistant/web/server.js b/recipes/llm-voice-assistant/web/server.js deleted file mode 100644 index 91f6ea8..0000000 --- a/recipes/llm-voice-assistant/web/server.js +++ /dev/null @@ -1,43 +0,0 @@ -const http = require('http'); -const fs = require('fs'); -const path = require('path'); -const mime = require('mime-types'); - -const PORT = process.env.PORT || 5000; -const HOST = '127.0.0.1'; // Listen on localhost -const publicDir = path.join(__dirname, 'public'); - -const server = http.createServer((req, res) => { - const url = (req.url === '/') ? '/index.html' : req.url; - const filePath = path.join(publicDir, url); - const contentType = mime.lookup(filePath) || 'application/octet-stream'; - - console.log(url) - - fs.readFile(filePath, (err, content) => { - if (err) { - if (err.code === 'ENOENT') { - // File not found - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('404 Not Found'); - } else { - // Server error - res.writeHead(500, { 'Content-Type': 'text/plain' }); - res.end(`500 Internal Server Error: ${err.code}`); - } - } else { - // Success - res.writeHead(200, { - 'Content-Type': contentType, - 'Content-Length': content.length, - 'Cross-Origin-Opener-Policy': 'same-origin', - 'Cross-Origin-Embedder-Policy': 'require-corp' - }); - res.end(content); - } - }); -}); - -server.listen(PORT, HOST, () => { - console.log(`Server is running on http://${HOST}:${PORT}`); -}); diff --git a/recipes/llm-voice-assistant/web/src/main.ts b/recipes/llm-voice-assistant/web/src/main.ts index b8fa421..f9d537a 100644 --- a/recipes/llm-voice-assistant/web/src/main.ts +++ b/recipes/llm-voice-assistant/web/src/main.ts @@ -141,7 +141,7 @@ const init = async ( const waitForSynthesize = (): Promise => new Promise(resolve => { const interval = setInterval(() => { - if (synthesized === (completionTokens.length - stopTokens)) { + if ((synthesized === (completionTokens.length - stopTokens)) || endpoint === PicoLLMEndpoint.INTERRUPTED) { clearInterval(interval); resolve(); } diff --git a/recipes/llm-voice-assistant/web/yarn.lock b/recipes/llm-voice-assistant/web/yarn.lock index 4c579da..631c5e8 100644 --- a/recipes/llm-voice-assistant/web/yarn.lock +++ b/recipes/llm-voice-assistant/web/yarn.lock @@ -1069,10 +1069,12 @@ dependencies: "@picovoice/web-utils" "=1.4.2" -"@picovoice/picollm-web@file:../../../../picollm/binding/web": - version "1.0.7" +"@picovoice/picollm-web@~1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@picovoice/picollm-web/-/picollm-web-1.1.0.tgz#e8706e134787d6738765ec81a1f29ac7668a4982" + integrity sha512-3Nxc9rLCf2/AmWIvB6pnsWDX/Q399bLT0sBR1pVJpxcS3ZMh5BLVHx7599bhAVBqBYUY6DZRBLRmmH1Yhk0Ycw== dependencies: - "@picovoice/web-utils" "~1.4.1" + "@picovoice/web-utils" "~1.4.3" "@picovoice/porcupine-web@^3.0.3": version "3.0.3" @@ -1088,13 +1090,20 @@ dependencies: commander "^9.2.0" -"@picovoice/web-utils@=1.4.2", "@picovoice/web-utils@~1.4.1": +"@picovoice/web-utils@=1.4.2": version "1.4.2" resolved "https://registry.yarnpkg.com/@picovoice/web-utils/-/web-utils-1.4.2.tgz#2ddc44552d15fa1a4958e0c3384e58545255eea1" integrity sha512-pF5Uw3Vm4mOWJ2H3Zc7E/nDr/O7OhbvgEK6W7cx9MNNK3qq51MqiGluPpZ8a2K61BuIzxcNMC1mXWpmIAWVolA== dependencies: commander "^10.0.1" +"@picovoice/web-utils@~1.4.3": + version "1.4.3" + resolved "https://registry.yarnpkg.com/@picovoice/web-utils/-/web-utils-1.4.3.tgz#1de0b20d6080c18d295c6df37c09d88bf7c4f555" + integrity sha512-7JN3YYsSD9Gtce6YKG3XqpX49dkeu7jTdbox7rHQA/X/Q3zxopXA9zlCKSq6EIjFbiX2iuzDKUx1XrFa3d8c0w== + dependencies: + commander "^10.0.1" + "@picovoice/web-voice-processor@^4.0.9": version "4.0.9" resolved "https://registry.yarnpkg.com/@picovoice/web-voice-processor/-/web-voice-processor-4.0.9.tgz#23aabbc85a0290546df5b8cdb0ca3b44707106ac" @@ -2627,18 +2636,6 @@ micromatch@^4.0.4: braces "^3.0.3" picomatch "^2.3.1" -mime-db@1.52.0: - version "1.52.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" - integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== - -mime-types@^2.1.35: - version "2.1.35" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" - integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== - dependencies: - mime-db "1.52.0" - mime@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"