using openai for recognition

SQUASHED: AUTO-COMMIT-demos-stefan-webxr-openai-recognition.md,AUTO-COMMIT-src-components-widgets-ubg-card.js,AUTO-COMMIT-src-components-widgets-ubg-card.js.l4a,AUTO-COMMIT-src-components-widgets-ubg-cards.html,AUTO-COMMIT-src-components-widgets-ubg-cards.js,AUTO-COMMIT-src-components-widgets-ubg-cards-scanner.html,AUTO-COMMIT-src-components-widgets-ubg-cards-scanner.js,AUTO-COMMIT-src-components-widgets-ubg-cards-scanner.js.l4a,AUTO-COMMIT-src-external-fuse.js,

demos/stefan/webxr/openai-recognition.md

@@ -0,0 +1,127 @@
+# OpenAI-powered Recognition
+
+- no continuous mode
+
+<video id='videoInput' autoplay style='border: blue 1px solid;position: relative; display: inline-block;'></video>
+
+<div id='log'>starting</div>;
+
+<canvas id="output" width='320' height='240' style='border: red 1px solid;'></canvas>
+
+<script>
+import OpenAI from "demos/openai/openai.js"
+
+async function cardNamesFromFoto(url) {
+  let prompt =  {
+    "model": "gpt-4o", 
+    "max_tokens": 256,
+    "temperature": 1,
+    "top_p": 1,
+    "n": 1,
+    "stream": false,
+    // "stop": "VANILLA",
+    "frequency_penalty": 0,
+    "presence_penalty": 0,
+    "messages": [
+      { "role": "system", "content": `You are a special-purpose recognition software that recognizes card from an upcoming trading card game in fotos. Cards have their name in the top, however, cards may have rotated or be skewed in the foto.
+
+Given a foto, only answer with the list of names you detected. Print only one name each line.` },
+      { "role": "user", "content":  [
+        {
+          "type": "image_url",
+          "image_url": {
+            "url": url,
+            "detail": "low" // high
+          }
+        }
+      ]}
+    ]
+  }
+
+  let json = await OpenAI.openAIRequest(prompt).then(r => r.json())
+  return json.choices[0].message.content
+}
+</script>
+
+<script>
+async function run() {
+  const get = id => lively.query(this, '#' + id)
+
+  const log = get('log')
+  let video = get("videoInput"); // video is the id of video tag
+
+  const canvas = get('output')
+  const context = canvas.getContext('2d');
+
+  let videoAccess;
+  let stream;
+  const captureVideo = () => {
+    videoAccess = navigator.mediaDevices.getUserMedia({
+      video: {
+        facingMode: {
+          ideal: "environment"
+        }
+      },
+      audio: false
+    })
+      .then(function(s) {
+      stream = s
+      video.srcObject = s;
+      video.play();
+    })
+      .catch(function(err) {
+      console.log("An error occurred! " + err);
+    });
+  };
+  captureVideo()
+
+  await videoAccess.then(() => lively.sleep(1000))
+
+  const stopVideo = () => {
+    if (stream) {
+      const tracks = stream.getTracks();
+      tracks.forEach(track => track.stop());
+      stream = null;
+    }
+
+    video.pause()
+    video.srcObject = null; // Stop the video element
+  };
+
+  const askOpenAI = async () => {
+    const dataURL = canvas.toDataURL('image/png');
+    const nameString = await cardNamesFromFoto(dataURL)
+    log.innerText = nameString
+  };
+
+  video.addEventListener('click', async evt => {
+    if (video.readyState !== video.HAVE_ENOUGH_DATA) {
+      lively.error('HAVE NOT ENOUGH DATA')
+      return
+    }
+
+    canvas.height = video.videoHeight;
+    canvas.width = video.videoWidth;
+    context.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+    stopVideo()
+
+    await askOpenAI()
+
+    captureVideo()
+  })
+
+  const stopOnDetach = () => {
+    if(!lively.allParents(lively.query(this, '*'), [], true).includes(document.body)) {
+      stopVideo()
+      lively.warn('BREAK')
+    } else {
+      requestAnimationFrame(stopOnDetach);
+    }
+  }
+  requestAnimationFrame(stopOnDetach);
+}
+
+run.call(this)
+</script>
+

src/components/widgets/ubg-card.js

@@ -1910,6 +1910,7 @@ font-family: "${font}";
   }
 
   renderQRCode(cardDesc, qrAnchor, outsideBorder) {
+    return;
     const canvas = <canvas id='canvasOutput' style={`
 position: absolute;
 top: ${qrAnchor.y}mm;

src/components/widgets/ubg-cards-scanner.html

@@ -0,0 +1,46 @@
+<template id="ubg-cards-scanner" >
+<style data-src="/src/external/font-awesome/css/font-awesome.css"></style>
+<style data-src="/templates/livelystyle.css"></style>
+<style>
+  :host {
+
+  }
+  ul {
+    position: absolute;
+    top: 23px;
+    left: 0;
+    list-style-type: none;
+    padding-left: 0px;
+  }
+  li {
+    width: 300px;
+    background: color-mix(in srgb, steelblue 50%, transparent 50%);
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+
+    text-indent: 10px;
+  }
+  .button-group {
+    position: absolute;
+    top: 3px;
+    left: 50%;
+    transform: translate(-50%);
+  }
+  #log {
+    position: absolute;
+    bottom: 3px;
+    right: 3px;
+  }
+</style>
+  <video id='videoInput' autoplay style='border: blue 1px solid;position: relative; display: inline-block;'></video>
+  <canvas id="output" width='320' height='240' style='border: red 1px solid;'></canvas>
+  <ul id='list'></ul>
+  <div class="button-group">
+    <button id="startButton"><i class="fa fa-play icon" aria-hidden="true"></i> START</button>
+    <button id="stopButton"><i class="fa fa-stop icon" aria-hidden="true"></i> STOP</button>
+    <button id="transferButton"><i class="fa fa-exchange icon" aria-hidden="true"></i> TRANSFER</button>
+    <button id="clearButton"><i class="fa fa-trash icon" aria-hidden="true"></i> Clear</button>
+  </div>
+  <div id='log'>starting</div>
+</template>