Merge branch 'AI_anim' into 'master'

Ai anim

See merge request crimata-ai/crimata-electron!7

src/background.ts

@@ -6,7 +6,9 @@ import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
 const isDevelopment = process.env.NODE_ENV !== "production";
 import * as path from "path";
 import { SpeechRecorder } from "speech-recorder";
-import WebSocket from "ws";
+
+// TODO: connect To Crimata-BE
+// import WebSocket from "ws";
 
 // const ws = new WebSocket("ws://localhost:8080");
 // ws.on("open", function open() {
@@ -16,18 +18,8 @@ import WebSocket from "ws";
 //   console.log("received message", message);
 // });
 //
-// const recorder = new SpeechRecorder();
-//
-// recorder.start({
-//   onAudio: (audio: object, speech: boolean) => {
-//     if (speech) {
-//       console.log(audio);
-//       ws.send(audio);
-//     }
-//     console.log("recording");
-//     // writeStream.write(audio);
-//   }
-// });
+
+const recorder = new SpeechRecorder({ sampleRate: 16000, framesPerBuffer: 320 });
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
@@ -42,7 +34,7 @@ function createWindow() {
   // Create the browser window.
   win = new BrowserWindow({
     width: 800,
-    height: 600,
+    height: 1200,
     webPreferences: {
       // Use pluginOptions.nodeIntegration, leave this alone
       // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info
@@ -62,6 +54,25 @@ function createWindow() {
     win.loadURL("app://./index.html");
   }
 
+  win.webContents.on('did-finish-load', () => {
+    if (win) {
+      recorder.start({
+        onAudio: (audio: object, speech: boolean) => {
+          if (win) {
+            win.webContents.send('render-audio', {
+              audio: audio,
+              speech: speech
+            })
+          }
+          if (speech) {
+            // TODO: send audio to crimata BE
+            // ws.send(audio);
+          }
+        }
+      });
+    }
+  })
+
   win.on("closed", () => {
     win = null;
   });

src/components/AI/index.vue

@@ -3,13 +3,13 @@
     <radialGradient
       id="radial-gradient"
       cx="0.5"
-      cy="1"
-      r="0.54"
+      cy="0.7"
+      :r="radius"
       gradientTransform="translate(-1.489 1.5) rotate(-90) scale(1 1.989)"
       gradientUnits="objectBoundingBox"
     >
       <stop offset="0" stop-color="#fff" />
-      <stop offset="1" stop-color="#e6e6e6" />
+      <stop id="sound" offset="1" stop-color="rgb(230, 230, 230)" />
     </radialGradient>
     <filter id="Rectangle_410" x="0" y="0" width="380" height="530" filterUnits="userSpaceOnUse">
       <feOffset dy="3" input="SourceAlpha" />
@@ -19,6 +19,7 @@
       <feComposite in="SourceGraphic" />
     </filter>
   </defs>
+
   <g transform="matrix(1, 0, 0, 1, 0, 0)" filter="url(#Rectangle_410)">
     <rect
       id="Rectangle_410-2"
@@ -32,8 +33,98 @@
 </template>
 
 <script>
-import { defineComponent } from "vue";
+import { defineComponent, onMounted, ref, inject, watch } from "vue";
 export default defineComponent({
   name: "AI",
+  setup() {
+    const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
+    const analyzer = audioCtx.createAnalyser();
+    analyzer.fftSize = 2048;
+    const xInitial = 24;
+    const xFinal = 640;
+    // analyzer.minDecibels = -90;
+    analyzer.smoothingTimeConstant = 0.85;
+    const dataArray = new Uint8Array(analyzer.frequencyBinCount);
+    const sampleRate = 16000;
+    let gradient;
+    const radius = ref(0);
+    const think = ref(false);
+    const aiOffset = ref(1.0);
+
+    const anime = inject("animejs");
+    let thinkAnimation;
+
+    onMounted(() => {
+      gradient = document.getElementById("radial-gradient");
+
+      thinkAnimation = anime({
+        targets: gradient,
+        loop: true,
+        cy: "1.3",
+        direction: "alternate",
+        easing: "easeInOutCirc",
+      });
+    });
+
+    function convertBlock(buffer) {
+      // incoming data is an ArrayBuffer
+      const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
+      const l = incomingData.length; // length, we need this for the loop
+      const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
+      for (let i = 0; i < l; i++) {
+        outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
+      }
+      return outputData; // return the Float32Array
+    }
+
+    window.ipcRenderer.on("render-audio", (event, payload) => {
+      const floatArray = convertBlock(payload.audio.buffer);
+      const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
+      buffer.copyToChannel(floatArray, 0, 0);
+      const source = audioCtx.createBufferSource();
+      source.buffer = buffer;
+      source.connect(analyzer);
+      source.start();
+      analyzer.getByteFrequencyData(dataArray);
+      const dataView = dataArray.slice(xInitial, xFinal);
+      const sum = dataView.reduce((a, b) => a + b);
+      const average = sum / dataView.length;
+      const scaled = average / 255;
+      const zeroed = scaled - 0.6;
+      if (zeroed > 0) {
+        radius.value = zeroed;
+      }
+      if (payload.speech === think.value) {
+        return;
+      } else {
+        think.value = payload.speech;
+      }
+    });
+
+    watch(think, async (think, prevThink) => {
+      if (think) {
+        console.log("think", think);
+        anime({
+          targets: gradient,
+          cy: "0.7",
+        });
+        thinkAnimation.play();
+      } else {
+        console.log("stop think");
+        thinkAnimation.pause();
+        // thinkAnimation.reset();
+        anime({
+          targets: gradient,
+          cy: "1",
+        });
+
+        // thinkAnimation.reset();
+      }
+    });
+    return {
+      radius,
+      aiOffset,
+    };
+  },
 });
 </script>

src/components/UI/UIDetails/messageRenderer.vue

@@ -1,4 +1,10 @@
 <template>
+<svg
+  version="1.1"
+  xmlns="http://www.w3.org/2000/svg"
+  xmlns:xlink="http://www.w3.org/1999/xlink"
+  viewBox="0 0 350 500"
+>
   <g id="messageRenderer">
     <g
       v-for="(item, index) in renderArr"
@@ -17,6 +23,7 @@
       </transition>
     </g>
   </g>
+</svg>
 </template>
 
 <script lang="ts">

src/preload.ts

@@ -1,27 +1,8 @@
 // All of the Node.js APIs are available in the preload process.
 // It has the same sandbox as a Chrome extension.
-import ipcRenderer from "electron";
+import { ipcRenderer } from "electron";
 
-process.once("loaded", () => {
-  window.addEventListener("message", event => {
-    // do something with custom event
-    const message = event.data;
-
-    if (message.myTypeField === "handle-audio-stream") {
-      ipcRenderer.send("audio-stream", message);
-    }
-  });
-});
-
-window.addEventListener("DOMContentLoaded", () => {
-  const replaceText = (selector: string, text: string) => {
-    const element = document.getElementById(selector);
-    if (element) {
-      element.innerText = text;
-    }
-  };
-
-  for (const type of ["chrome", "node", "electron"]) {
-    replaceText(`${type}-version`, (process.versions as any)[type]);
-  }
-});
+interface Window {
+  ipcRenderer: typeof ipcRenderer;
+}
+window.ipcRenderer = ipcRenderer;