browser VAD

JarbasHiveMind · Nov 24, 2023 · 0a31f0b · 0a31f0b
1 parent 2a37fcc
commit 0a31f0b
Show file tree

Hide file tree

Showing 6 changed files with 196 additions and 0 deletions.
diff --git a/build.sh b/build.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+rm -rf ./dist
+mkdir ./dist
+
+cd ./src
+
+find . -name "*.js" |
+  xargs -I {} sh -c 'outfile="../dist/{}"; npx esbuild "{}" --bundle --sourcemap --outfile="${outfile%.*}.js"'
+
+cp ./*.html ../dist
diff --git a/dist/index.html b/dist/index.html
@@ -0,0 +1,37 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>HiveMind Speech WebClient</title>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <link
+      rel="stylesheet"
+      href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.3/css/bulma.min.css"
+    />
+    <script src="https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@ricky0123/[email protected]/dist/bundle.min.js"></script>
+    <script type="module" src="index.js"></script>
+  </head>
+  <body>
+    <section class="section">
+      <div class="container">
+        <h1 class="title">HiveMind Speech WebClient</h1>
+
+        <div class="block is-inline-flex">
+          <button
+            id="toggleVAD"
+            class="button is-primary is-loading"
+            onclick="window.toggleVAD()"
+            disabled
+          >
+            Start VAD
+          </button>
+        </div>
+
+        <div class="block">
+          <ul id="audio-list"></ul>
+        </div>
+      </div>
+    </section>
+  </body>
+</html>
diff --git a/dist/index.js b/dist/index.js
diff --git a/dist/index.js.map b/dist/index.js.map
diff --git a/src/index.html b/src/index.html
@@ -0,0 +1,37 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>HiveMind Speech WebClient</title>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <link
+      rel="stylesheet"
+      href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.3/css/bulma.min.css"
+    />
+    <script src="https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@ricky0123/[email protected]/dist/bundle.min.js"></script>
+    <script type="module" src="index.js"></script>
+  </head>
+  <body>
+    <section class="section">
+      <div class="container">
+        <h1 class="title">HiveMind Speech WebClient</h1>
+
+        <div class="block is-inline-flex">
+          <button
+            id="toggleVAD"
+            class="button is-primary is-loading"
+            onclick="window.toggleVAD()"
+            disabled
+          >
+            Start VAD
+          </button>
+        </div>
+
+        <div class="block">
+          <ul id="audio-list"></ul>
+        </div>
+      </div>
+    </section>
+  </body>
+</html>
diff --git a/src/index.js b/src/index.js
@@ -0,0 +1,54 @@
+// @ts-nocheck
+
+function getToggleButton() {
+  return document.getElementById("toggleVAD")
+}
+
+async function main() {
+  try {
+    const myvad = await vad.MicVAD.new({
+      onSpeechStart: () => {
+        console.log("Speech start")
+      },
+      onSpeechEnd: (arr) => {
+        console.log("Speech end")
+        const wavBuffer = vad.utils.encodeWAV(arr)
+        const base64 = vad.utils.arrayBufferToBase64(wavBuffer)
+        const url = `data:audio/wav;base64,${base64}`
+        const el = addAudio(url)
+        const speechList = document.getElementById("audio-list")
+        speechList.prepend(el)
+        // TODO - send to hivemind here
+      },
+    })
+
+    window.myvad = myvad
+    getToggleButton().classList.remove("is-loading")
+
+    window.toggleVAD = () => {
+      console.log("ran toggle vad")
+      if (myvad.listening === false) {
+        myvad.start()
+        getToggleButton().textContent = "Stop VAD"
+      } else {
+        myvad.pause()
+        getToggleButton().textContent = "Start VAD"
+      }
+    }
+    window.toggleVAD()
+    getToggleButton().disabled = false
+  } catch (e) {
+    console.error("Failed:", e)
+  }
+
+  function addAudio(audioUrl) {
+    const entry = document.createElement("li")
+    const audio = document.createElement("audio")
+    audio.controls = true
+    audio.src = audioUrl
+    entry.appendChild(audio)
+    return entry
+  }
+}
+
+main()