feat: desktop tamagotchi

package.json

@@ -15,7 +15,9 @@
     "typecheck": "pnpm -r --filter=./packages/* run build",
     "dev": "pnpm packages:dev",
     "build": "pnpm packages:build",
+    "dev:tamagotchi": "pnpm packages:dev:tamagotchi",
     "packages:dev": "pnpm -r --filter=./packages/* --parallel run dev",
+    "packages:dev:tamagotchi": "pnpm -r --filter=./packages/* --parallel run dev:tamagotchi",
     "packages:stub": "pnpm -r --filter=./packages/* run stub",
     "packages:build": "pnpm -r --filter=./packages/* run build",
     "packages:publish": "pnpm -r --filter=./packages/* run package:publish",

packages/stage/package.json

@@ -15,7 +15,9 @@
     "dev": "vite",
     "lint": "eslint .",
     "preview": "vite preview",
-    "typecheck": "vue-tsc --noEmit"
+    "typecheck": "vue-tsc --noEmit",
+    "dev:tamagotchi": "vite --mode tamagotchi",
+    "build:tamagotchi": "vite build --mode tamagotchi"
   },
   "dependencies": {
     "@11labs/client": "^0.0.4",
@@ -54,6 +56,7 @@
     "@xsai/shared-chat": "^0.0.22",
     "@xsai/stream-text": "^0.0.22",
     "defu": "^6.1.4",
+    "jszip": "^3.10.1",
     "nprogress": "^0.2.0",
     "ofetch": "^1.4.1",
     "onnxruntime-web": "^1.20.1",

packages/stage/src/components/Layouts/MobileInteractiveArea.vue

@@ -10,6 +10,7 @@ import { useMicVAD } from '../../composables/micvad'
 // import { useAudioContext } from '../../stores/audio'
 import { useChatStore } from '../../stores/chat'
 import { useSettings } from '../../stores/settings'
+import BasicTextarea from '../BasicTextarea.vue'
 import MobileChatHistory from '../Widgets/MobileChatHistory.vue'
 import MobileSettings from '../Widgets/MobileSettings.vue'
 

packages/stage/src/components/Layouts/TamagotchiInteractiveArea.vue

@@ -0,0 +1,129 @@
+<script setup lang="ts">
+// import { useDevicesList } from '@vueuse/core'
+import { storeToRefs } from 'pinia'
+
+import { DrawerContent, DrawerPortal, DrawerRoot, DrawerTrigger } from 'vaul-vue'
+import { onMounted, ref, watch } from 'vue'
+import { useI18n } from 'vue-i18n'
+
+import { useMicVAD } from '../../composables/micvad'
+// import { useAudioContext } from '../../stores/audio'
+import { useChatStore } from '../../stores/chat'
+import { useSettings } from '../../stores/settings'
+import BasicTextarea from '../BasicTextarea.vue'
+import TamagotchiChatHistory from '../Widgets/TamagotchiChatHistory.vue'
+import TamagotchiSettings from '../Widgets/TamagotchiSettings.vue'
+
+const messageInput = ref('')
+const listening = ref(false)
+
+// const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
+// const { selectedAudioDevice, isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings())
+const { isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings())
+const { send, onAfterSend } = useChatStore()
+const { t } = useI18n()
+
+async function handleSend() {
+  if (!messageInput.value.trim()) {
+    return
+  }
+
+  await send(messageInput.value)
+}
+
+const { destroy, start } = useMicVAD(selectedAudioDeviceId, {
+  onSpeechStart: () => {
+    // TODO: interrupt the playback
+    // TODO: interrupt any of the ongoing TTS
+    // TODO: interrupt any of the ongoing LLM requests
+    // TODO: interrupt any of the ongoing animation of Live2D or VRM
+    // TODO: once interrupted, we should somehow switch to listen or thinking
+    //       emotion / expression?
+    listening.value = true
+  },
+  // VAD misfire means while speech end is detected but
+  // the frames of the segment of the audio buffer
+  // is not enough to be considered as a speech segment
+  // which controlled by the `minSpeechFrames` parameter
+  onVADMisfire: () => {
+    // TODO: do audio buffer send to whisper
+    listening.value = false
+  },
+  onSpeechEnd: (buffer) => {
+    // TODO: do audio buffer send to whisper
+    listening.value = false
+    handleTranscription(buffer)
+  },
+  auto: false,
+})
+
+function handleTranscription(_buffer: Float32Array<ArrayBufferLike>) {
+  // eslint-disable-next-line no-alert
+  alert('Transcription is not implemented yet')
+}
+
+// async function handleAudioInputChange(event: Event) {
+//   const target = event.target as HTMLSelectElement
+//   const found = audioInputs.value.find(d => d.deviceId === target.value)
+//   if (!found) {
+//     selectedAudioDevice.value = undefined
+//     return
+//   }
+
+//   selectedAudioDevice.value = found
+// }
+
+watch(isAudioInputOn, async (value) => {
+  if (value === 'false') {
+    destroy()
+  }
+})
+
+onAfterSend(async () => {
+  messageInput.value = ''
+})
+
+onMounted(() => {
+  start()
+})
+</script>
+
+<template>
+  <div>
+    <div relative w-full flex gap-1>
+      <TamagotchiChatHistory absolute left-0 top-0 transform="translate-y-[-100%]" w-full />
+      <div flex flex-1>
+        <BasicTextarea
+          v-model="messageInput"
+          :placeholder="t('stage.message')"
+          border="solid 2 pink-100"
+          text="pink-400 hover:pink-600  placeholder:pink-400 placeholder:hover:pink-600"
+          bg="pink-50 dark:[#3c2632]" max-h="[10lh]" min-h="[1lh]"
+          w-full resize-none overflow-y-scroll rounded-l-xl p-2 font-medium outline-none
+          transition="all duration-250 ease-in-out placeholder:all placeholder:duration-250 placeholder:ease-in-out"
+          @submit="handleSend"
+        />
+      </div>
+      <DrawerRoot should-scale-background>
+        <DrawerTrigger
+          class="px-4 py-2.5"
+          border="solid 2 pink-100 "
+          text="lg pink-400 hover:pink-600  placeholder:pink-400 placeholder:hover:pink-600"
+          bg="pink-50 dark:[#3c2632]" max-h="[10lh]" min-h="[1lh]" rounded-r-xl
+        >
+          <div i-solar:settings-bold-duotone />
+        </DrawerTrigger>
+        <DrawerPortal>
+          <DrawerContent
+            max-h="[90%]"
+            fixed bottom-0 left-0 right-0 z-50 mt-24 h-full flex flex-col rounded-t-lg bg="[#fffbff] dark:[#1f1a1d]"
+          >
+            <div class="flex flex-1 flex-col rounded-t-lg p-5" bg="[#fffbff] dark:[#1f1a1d]" gap-2>
+              <TamagotchiSettings />
+            </div>
+          </DrawerContent>
+        </DrawerPortal>
+      </DrawerRoot>
+    </div>
+  </div>
+</template>

packages/stage/src/components/Widgets/Stage.vue

@@ -9,15 +9,18 @@ import { useQueue } from '../../composables/queue'
 import { useDelayMessageQueue, useEmotionsMessageQueue, useMessageContentQueue } from '../../composables/queues'
 import { llmInferenceEndToken } from '../../constants'
 import { Voice } from '../../constants/elevenlabs'
-import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
 
+import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
 import { useAudioContext, useSpeakingStore } from '../../stores/audio'
 import { useChatStore } from '../../stores/chat'
 import { useLLM } from '../../stores/llm'
 import { useSettings } from '../../stores/settings'
 import Live2DScene from '../Scenes/Live2D.vue'
+
 import VRMScene from '../Scenes/VRM.vue'
 
+import '../../utils/live2d-zip-loader'
+
 const live2DViewerRef = ref<{ setMotion: (motionName: string) => Promise<void> }>()
 const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
 
@@ -183,7 +186,7 @@ onUnmounted(() => {
         v-if="stageView === '2d'"
         ref="live2DViewerRef"
         :mouth-open-size="mouthOpenSize"
-        model="/assets/live2d/models/hiyori_pro_zh/runtime/hiyori_pro_t11.model3.json"
+        model="./assets/live2d/models/hiyori_pro_zh.zip"
         min-w="50% <lg:full" min-h="100 sm:100" h-full w-full flex-1
       />
       <VRMScene

packages/stage/src/components/Widgets/TamagotchiChatHistory.vue

@@ -0,0 +1,76 @@
+<script setup lang="ts">
+import { useElementBounding, useScroll } from '@vueuse/core'
+import { storeToRefs } from 'pinia'
+
+import { nextTick, ref } from 'vue'
+import { useMarkdown } from '../../composables/markdown'
+import { useChatStore } from '../../stores/chat'
+
+const chatHistoryRef = ref<HTMLDivElement>()
+
+const { messages } = storeToRefs(useChatStore())
+const bounding = useElementBounding(chatHistoryRef, { immediate: true, windowScroll: true, windowResize: true })
+const { y: chatHistoryContainerY } = useScroll(chatHistoryRef)
+
+const { process } = useMarkdown()
+const { onBeforeMessageComposed, onTokenLiteral } = useChatStore()
+
+onBeforeMessageComposed(async () => {
+  // Scroll down to the new sent message
+  nextTick().then(() => {
+    bounding.update()
+    chatHistoryContainerY.value = bounding.height.value
+  })
+})
+
+onTokenLiteral(async () => {
+  // Scroll down to the new responding message
+  nextTick().then(() => {
+    bounding.update()
+    chatHistoryContainerY.value = bounding.height.value
+  })
+})
+</script>
+
+<template>
+  <div py="1" flex="~ col" rounded="lg" overflow-hidden>
+    <div flex-1 /> <!-- spacer -->
+    <div ref="chatHistoryRef" v-auto-animate h-full w-full max-h="30vh" flex="~ col" overflow-scroll>
+      <div flex-1 /> <!-- spacer -->
+      <div v-for="(message, index) in messages" :key="index" mb-2>
+        <div v-if="message.role === 'assistant'" flex mr="12">
+          <div
+            flex="~ col"
+            border="4 solid pink-200"
+            shadow="md pink-200/50"
+            min-w-20 rounded-lg px-2 py-1
+            h="fit"
+            bg="pink-100"
+          >
+            <div>
+              <span text-xs text="pink-400/90" font-semibold class="inline hidden">Airi</span>
+            </div>
+            <div v-if="message.content" class="markdown-content" text="xs pink-400" v-html="process(message.content as string)" />
+            <div v-else i-eos-icons:three-dots-loading />
+          </div>
+        </div>
+        <div v-else-if="message.role === 'user'" flex="~">
+          <div
+            flex="~ col"
+            border="4 solid cyan-200"
+            shadow="md cyan-200/50"
+            px="2"
+            h="fit" min-w-20 rounded-lg px-2 py-1
+            bg="cyan-100"
+          >
+            <div>
+              <span text-xs text="cyan-600/90" font-semibold class="hidden">You</span>
+            </div>
+            <div v-if="message.content" class="markdown-content" text="xs cyan-600" v-html="process(message.content as string)" />
+            <div v-else />
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</template>