From 230efb325ce09e805e03ea966cb5d91158cb6127 Mon Sep 17 00:00:00 2001
From: riqo <hernandeze2@xavier.edu>
Date: Fri, 6 Nov 2020 16:06:08 -0600
Subject: [PATCH 1/6] add ipcRenderer render-audio event to window object

---
 src/background.ts           | 34 +++++++++++++++++++++-------------
 src/components/AI/index.vue |  6 ++++++
 src/preload.ts              | 29 +++++------------------------
 3 files changed, 32 insertions(+), 37 deletions(-)

diff --git a/src/background.ts b/src/background.ts
index 9204046..2aaec65 100644
--- a/src/background.ts
+++ b/src/background.ts
@@ -6,7 +6,9 @@ import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
 const isDevelopment = process.env.NODE_ENV !== "production";
 import * as path from "path";
 import { SpeechRecorder } from "speech-recorder";
-import WebSocket from "ws";
+
+// TODO: connect To Crimata-BE
+// import WebSocket from "ws";
 
 // const ws = new WebSocket("ws://localhost:8080");
 // ws.on("open", function open() {
@@ -16,18 +18,8 @@ import WebSocket from "ws";
 //   console.log("received message", message);
 // });
 //
-// const recorder = new SpeechRecorder();
-//
-// recorder.start({
-//   onAudio: (audio: object, speech: boolean) => {
-//     if (speech) {
-//       console.log(audio);
-//       ws.send(audio);
-//     }
-//     console.log("recording");
-//     // writeStream.write(audio);
-//   }
-// });
+
+const recorder = new SpeechRecorder();
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
@@ -62,6 +54,22 @@ function createWindow() {
     win.loadURL("app://./index.html");
   }
 
+  win.webContents.on('did-finish-load', () => {
+    if (win) {
+      recorder.start({
+        onAudio: (audio: object, speech: boolean) => {
+          if (win) {
+            win.webContents.send('render-audio', { data: audio })
+          }
+          if (speech) {
+            // TODO: send audio to crimata BE
+            // ws.send(audio);
+          }
+        }
+      });
+    }
+  })
+
   win.on("closed", () => {
     win = null;
   });
diff --git a/src/components/AI/index.vue b/src/components/AI/index.vue
index 9139461..02bb6a9 100644
--- a/src/components/AI/index.vue
+++ b/src/components/AI/index.vue
@@ -19,6 +19,7 @@
       <feComposite in="SourceGraphic" />
     </filter>
   </defs>
+
   <g transform="matrix(1, 0, 0, 1, 0, 0)" filter="url(#Rectangle_410)">
     <rect
       id="Rectangle_410-2"
@@ -35,5 +36,10 @@
 import { defineComponent } from "vue";
 export default defineComponent({
   name: "AI",
+  setup() {
+    window.ipcRenderer.on("render-audio", (event, audio) => {
+      console.log(audio);
+    });
+  },
 });
 </script>
diff --git a/src/preload.ts b/src/preload.ts
index a5c911d..9e33238 100644
--- a/src/preload.ts
+++ b/src/preload.ts
@@ -1,27 +1,8 @@
 // All of the Node.js APIs are available in the preload process.
 // It has the same sandbox as a Chrome extension.
-import ipcRenderer from "electron";
+import { ipcRenderer } from "electron";
 
-process.once("loaded", () => {
-  window.addEventListener("message", event => {
-    // do something with custom event
-    const message = event.data;
-
-    if (message.myTypeField === "handle-audio-stream") {
-      ipcRenderer.send("audio-stream", message);
-    }
-  });
-});
-
-window.addEventListener("DOMContentLoaded", () => {
-  const replaceText = (selector: string, text: string) => {
-    const element = document.getElementById(selector);
-    if (element) {
-      element.innerText = text;
-    }
-  };
-
-  for (const type of ["chrome", "node", "electron"]) {
-    replaceText(`${type}-version`, (process.versions as any)[type]);
-  }
-});
+interface Window {
+  ipcRenderer: typeof ipcRenderer;
+}
+window.ipcRenderer = ipcRenderer;

From 938eaff11d21d1773d7cfcc38356926de35fb6a0 Mon Sep 17 00:00:00 2001
From: riqo <hernandeze2@xavier.edu>
Date: Sat, 7 Nov 2020 17:04:49 -0600
Subject: [PATCH 2/6] add audio analyser fft data

---
 src/background.ts           |  4 +-
 src/components/AI/index.vue | 80 +++++++++++++++++++++++++++++++++++--
 2 files changed, 78 insertions(+), 6 deletions(-)

diff --git a/src/background.ts b/src/background.ts
index 2aaec65..e07342e 100644
--- a/src/background.ts
+++ b/src/background.ts
@@ -19,7 +19,7 @@ import { SpeechRecorder } from "speech-recorder";
 // });
 //
 
-const recorder = new SpeechRecorder();
+const recorder = new SpeechRecorder({ sampleRate: 16000, framesPerBuffer: 320 });
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
@@ -59,7 +59,7 @@ function createWindow() {
       recorder.start({
         onAudio: (audio: object, speech: boolean) => {
           if (win) {
-            win.webContents.send('render-audio', { data: audio })
+            win.webContents.send('render-audio', audio)
           }
           if (speech) {
             // TODO: send audio to crimata BE
diff --git a/src/components/AI/index.vue b/src/components/AI/index.vue
index 02bb6a9..2afa6f1 100644
--- a/src/components/AI/index.vue
+++ b/src/components/AI/index.vue
@@ -9,7 +9,7 @@
       gradientUnits="objectBoundingBox"
     >
       <stop offset="0" stop-color="#fff" />
-      <stop offset="1" stop-color="#e6e6e6" />
+      <stop id="sound" offset="1" stop-color="#e6e6e6" />
     </radialGradient>
     <filter id="Rectangle_410" x="0" y="0" width="380" height="530" filterUnits="userSpaceOnUse">
       <feOffset dy="3" input="SourceAlpha" />
@@ -33,13 +33,85 @@
 </template>
 
 <script>
-import { defineComponent } from "vue";
+import { defineComponent, onMounted, ref } from "vue";
 export default defineComponent({
   name: "AI",
   setup() {
-    window.ipcRenderer.on("render-audio", (event, audio) => {
-      console.log(audio);
+    const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
+    const analyzer = audioCtx.createAnalyser();
+    analyzer.fftSize = 256;
+    const dataArray = new Uint8Array(analyzer.frequencyBinCount);
+    const sampleRate = 16000;
+    let gradient;
+    const amplitude = ref(0);
+
+    onMounted(() => {
+      gradient = document.getElementById("sound");
     });
+
+    function convertBlock(buffer) {
+      // incoming data is an ArrayBuffer
+      const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
+      let i,
+        l = incomingData.length; // length, we need this for the loop
+      const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
+      for (i = 0; i < l; i++) {
+        outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
+      }
+      return outputData; // return the Float32Array
+    }
+
+    window.addEventListener("load", function (e) {
+      window.ipcRenderer.on("render-audio", (event, audio) => {
+        const floatArray = convertBlock(audio.buffer);
+        const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
+        buffer.copyToChannel(floatArray, 0, 0);
+        const source = audioCtx.createBufferSource();
+        source.buffer = buffer;
+        source.connect(analyzer);
+        source.start();
+        visualize();
+      });
+    });
+
+    const render = () => {
+      // whatever you want to do per frame of animation
+
+      const range = 64;
+      const interval = range / 6;
+
+      const sensitivity = 0.25;
+
+      const min = 0;
+      const max = 25;
+
+      let channel = 1;
+
+      const to = Math.floor(interval * channel);
+      const from = Math.ceil(to - interval);
+      const segment = dataArray.slice(from, to);
+
+      const sum = segment.reduce((a, b) => a + b);
+      const average = sum / segment.length;
+      console.log(average);
+      // let height = sensitivity * average;
+      //
+      // height = Math.max(min, Math.min(max, height));
+      //
+      // height = height + 230;
+      //
+      // // we have to write a function what scales between
+      // gradient.setAttribute(
+      //   "stop-color",
+      //   `rgb(${height}, ${height}, ${height})`
+      // );
+    };
+
+    function visualize() {
+      requestAnimationFrame(visualize);
+      analyzer.getByteFrequencyData(dataArray);
+      render();
+    }
   },
 });
 </script>

From e225fff2647af6e55fd920072a0569d644507f49 Mon Sep 17 00:00:00 2001
From: riqo <hernandeze2@xavier.edu>
Date: Sun, 8 Nov 2020 16:12:05 -0600
Subject: [PATCH 3/6] add think animation

---
 src/background.ts           |   7 ++-
 src/components/AI/index.vue | 109 +++++++++++++++++-------------------
 2 files changed, 57 insertions(+), 59 deletions(-)

diff --git a/src/background.ts b/src/background.ts
index e07342e..e08bc2a 100644
--- a/src/background.ts
+++ b/src/background.ts
@@ -34,7 +34,7 @@ function createWindow() {
   // Create the browser window.
   win = new BrowserWindow({
     width: 800,
-    height: 600,
+    height: 1200,
     webPreferences: {
       // Use pluginOptions.nodeIntegration, leave this alone
       // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info
@@ -59,7 +59,10 @@ function createWindow() {
       recorder.start({
         onAudio: (audio: object, speech: boolean) => {
           if (win) {
-            win.webContents.send('render-audio', audio)
+            win.webContents.send('render-audio', {
+              audio: audio,
+              speech: speech
+            })
           }
           if (speech) {
             // TODO: send audio to crimata BE
diff --git a/src/components/AI/index.vue b/src/components/AI/index.vue
index 2afa6f1..edcc65f 100644
--- a/src/components/AI/index.vue
+++ b/src/components/AI/index.vue
@@ -4,12 +4,12 @@
       id="radial-gradient"
       cx="0.5"
       cy="1"
-      r="0.54"
+      :r="radius"
       gradientTransform="translate(-1.489 1.5) rotate(-90) scale(1 1.989)"
       gradientUnits="objectBoundingBox"
     >
       <stop offset="0" stop-color="#fff" />
-      <stop id="sound" offset="1" stop-color="#e6e6e6" />
+      <stop id="sound" offset="1" stop-color="rgb(230, 230, 230)" />
     </radialGradient>
     <filter id="Rectangle_410" x="0" y="0" width="380" height="530" filterUnits="userSpaceOnUse">
       <feOffset dy="3" input="SourceAlpha" />
@@ -33,85 +33,80 @@
 </template>
 
 <script>
-import { defineComponent, onMounted, ref } from "vue";
+import { defineComponent, onMounted, ref, inject, watch } from "vue";
 export default defineComponent({
   name: "AI",
   setup() {
     const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
     const analyzer = audioCtx.createAnalyser();
-    analyzer.fftSize = 256;
+    analyzer.fftSize = 2048;
+    const xInitial = 24;
+    const xFinal = 640;
+    // analyzer.minDecibels = -90;
+    analyzer.smoothingTimeConstant = 0.85;
     const dataArray = new Uint8Array(analyzer.frequencyBinCount);
     const sampleRate = 16000;
     let gradient;
-    const amplitude = ref(0);
+    const radius = ref(0);
+    const think = ref(false);
+    const aiOffset = ref(1);
+
+    const anime = inject("animejs");
 
     onMounted(() => {
-      gradient = document.getElementById("sound");
+      gradient = document.getElementById("radial-gradient");
     });
 
     function convertBlock(buffer) {
       // incoming data is an ArrayBuffer
       const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
-      let i,
-        l = incomingData.length; // length, we need this for the loop
+      const l = incomingData.length; // length, we need this for the loop
       const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
-      for (i = 0; i < l; i++) {
+      for (let i = 0; i < l; i++) {
         outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
       }
       return outputData; // return the Float32Array
     }
 
-    window.addEventListener("load", function (e) {
-      window.ipcRenderer.on("render-audio", (event, audio) => {
-        const floatArray = convertBlock(audio.buffer);
-        const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
-        buffer.copyToChannel(floatArray, 0, 0);
-        const source = audioCtx.createBufferSource();
-        source.buffer = buffer;
-        source.connect(analyzer);
-        source.start();
-        visualize();
-      });
+    window.ipcRenderer.on("think", (event, think) => {
+      console.log("think", think);
     });
 
-    const render = () => {
-      // whatever you want to do per frame of animation
-
-      const range = 64;
-      const interval = range / 6;
-
-      const sensitivity = 0.25;
-
-      const min = 0;
-      const max = 25;
-
-      let channel = 1;
-
-      const to = Math.floor(interval * channel);
-      const from = Math.ceil(to - interval);
-      const segment = dataArray.slice(from, to);
-
-      const sum = segment.reduce((a, b) => a + b);
-      const average = sum / segment.length;
-      console.log(average);
-      // let height = sensitivity * average;
-      //
-      // height = Math.max(min, Math.min(max, height));
-      //
-      // height = height + 230;
-      //
-      // // we have to write a function what scales between
-      // gradient.setAttribute(
-      //   "stop-color",
-      //   `rgb(${height}, ${height}, ${height})`
-      // );
-    };
-
-    function visualize() {
-      requestAnimationFrame(visualize);
+    window.ipcRenderer.on("render-audio", (event, payload) => {
+      const floatArray = convertBlock(payload.audio.buffer);
+      const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
+      buffer.copyToChannel(floatArray, 0, 0);
+      const source = audioCtx.createBufferSource();
+      source.buffer = buffer;
+      source.connect(analyzer);
+      source.start();
       analyzer.getByteFrequencyData(dataArray);
-      render();
-    }
+      const dataView = dataArray.slice(xInitial, xFinal);
+      const sum = dataView.reduce((a, b) => a + b);
+      const average = sum / dataView.length;
+      const scaled = average / 255;
+      const zeroed = scaled - 0.6;
+      if (zeroed > 0) {
+        radius.value = zeroed;
+      }
+      const amplified = zeroed;
+      if (payload.speech === think) {
+        return;
+      } else {
+        think.value = payload.speech;
+      }
+    });
+
+    watch(think, (think, prevThink) => {
+      if (think) {
+        console.log("think");
+      } else {
+        console.log("dont think");
+      }
+    });
+    return {
+      radius,
+    };
   },
 });
 </script>

From e8413604d60c8b42ace7ed759db7b4f131772fa6 Mon Sep 17 00:00:00 2001
From: Enrique Hernandez <hernandeze2@xavier.edu>
Date: Tue, 10 Nov 2020 09:00:11 -0600
Subject: [PATCH 4/6] add thinking animation

---
 src/components/AI/index.vue | 38 ++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/src/components/AI/index.vue b/src/components/AI/index.vue
index edcc65f..f03f7b1 100644
--- a/src/components/AI/index.vue
+++ b/src/components/AI/index.vue
@@ -9,7 +9,7 @@
       gradientUnits="objectBoundingBox"
     >
       <stop offset="0" stop-color="#fff" />
-      <stop id="sound" offset="1" stop-color="rgb(230, 230, 230)" />
+      <stop id="sound" offset="1" stop-color="rgb(230, 230, 23)" />
     </radialGradient>
     <filter id="Rectangle_410" x="0" y="0" width="380" height="530" filterUnits="userSpaceOnUse">
       <feOffset dy="3" input="SourceAlpha" />
@@ -55,6 +55,14 @@ export default defineComponent({
 
     onMounted(() => {
       gradient = document.getElementById("radial-gradient");
+
+      // anime({
+      //   targets: gradient,
+      //   cy: "1.3",
+      //   loop: true,
+      //   direction: "alternate",
+      //   easing: "easeInOutCirc",
+      // });
     });
 
     function convertBlock(buffer) {
@@ -99,13 +107,37 @@ export default defineComponent({
 
     watch(think, (think, prevThink) => {
       if (think) {
-        console.log("think");
+        aiOffset.value = 0.7;
+        anime({
+          targets: gradient,
+          cy: "0.7",
+          complete: function () {
+            anime({
+              targets: gradient,
+              cy: "1.3",
+              loop: true,
+              direction: "alternate",
+              easing: "easeInOutCirc",
+            });
+          },
+          // loop: true,
+          // direction: "alternate",
+          // easing: "easeInOutCirc",
+        });
       } else {
-        console.log("dont think");
+        aiOffset.value = 1;
+        anime({
+          targets: gradient,
+          cy: "1",
+          loop: false,
+          easing: "easeInOutCirc",
+          offset: "-=100",
+        });
       }
     });
     return {
       radius,
+      aiOffset,
     };
   },
 });

From d634f362c304109d2652e15992d4d5603eb83a0e Mon Sep 17 00:00:00 2001
From: riqo <hernandeze2@xavier.edu>
Date: Thu, 12 Nov 2020 16:55:47 -0600
Subject: [PATCH 5/6] thinking animation v1

---
 src/components/AI/index.vue | 49 ++++++++++++++-----------------------
 1 file changed, 18 insertions(+), 31 deletions(-)

diff --git a/src/components/AI/index.vue b/src/components/AI/index.vue
index f03f7b1..d81a08f 100644
--- a/src/components/AI/index.vue
+++ b/src/components/AI/index.vue
@@ -3,7 +3,7 @@
     <radialGradient
       id="radial-gradient"
       cx="0.5"
-      cy="1"
+      cy="0.7"
       :r="radius"
       gradientTransform="translate(-1.489 1.5) rotate(-90) scale(1 1.989)"
       gradientUnits="objectBoundingBox"
@@ -49,20 +49,21 @@ export default defineComponent({
     let gradient;
     const radius = ref(0);
     const think = ref(false);
-    const aiOffset = ref(1);
+    const aiOffset = ref(1.0);
 
     const anime = inject("animejs");
+    let thinkAnimation;
 
     onMounted(() => {
       gradient = document.getElementById("radial-gradient");
 
-      // anime({
-      //   targets: gradient,
-      //   cy: "1.3",
-      //   loop: true,
-      //   direction: "alternate",
-      //   easing: "easeInOutCirc",
-      // });
+      thinkAnimation = anime({
+        targets: gradient,
+        loop: true,
+        cy: "1.3",
+        direction: "alternate",
+        easing: "easeInOutCirc",
+      });
     });
 
     function convertBlock(buffer) {
@@ -76,10 +77,6 @@ export default defineComponent({
       return outputData; // return the Float32Array
     }
 
-    window.ipcRenderer.on("think", (event, think) => {
-      console.log("think", think);
-    });
-
     window.ipcRenderer.on("render-audio", (event, payload) => {
       const floatArray = convertBlock(payload.audio.buffer);
       const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
@@ -105,34 +102,24 @@ export default defineComponent({
       }
     });
 
-    watch(think, (think, prevThink) => {
+    watch(think, async (think, prevThink) => {
       if (think) {
-        aiOffset.value = 0.7;
+        console.log("think", think);
         anime({
           targets: gradient,
           cy: "0.7",
-          complete: function () {
-            anime({
-              targets: gradient,
-              cy: "1.3",
-              loop: true,
-              direction: "alternate",
-              easing: "easeInOutCirc",
-            });
-          },
-          // loop: true,
-          // direction: "alternate",
-          // easing: "easeInOutCirc",
         });
+        thinkAnimation.play();
       } else {
-        aiOffset.value = 1;
+        console.log("stop think");
+        thinkAnimation.pause();
+        // thinkAnimation.reset();
         anime({
           targets: gradient,
           cy: "1",
-          loop: false,
-          easing: "easeInOutCirc",
-          offset: "-=100",
         });
+
+        // thinkAnimation.reset();
       }
     });
     return {

From b5b766f6994aa8d61e745d8f257216b148196408 Mon Sep 17 00:00:00 2001
From: riqo <hernandeze2@xavier.edu>
Date: Sun, 15 Nov 2020 10:46:26 -0600
Subject: [PATCH 6/6] add viewBox to messageRenderer

---
 src/components/AI/index.vue                     | 5 ++---
 src/components/UI/UIDetails/messageRenderer.vue | 7 +++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/components/AI/index.vue b/src/components/AI/index.vue
index d81a08f..a4b36ed 100644
--- a/src/components/AI/index.vue
+++ b/src/components/AI/index.vue
@@ -9,7 +9,7 @@
       gradientUnits="objectBoundingBox"
     >
       <stop offset="0" stop-color="#fff" />
-      <stop id="sound" offset="1" stop-color="rgb(230, 230, 23)" />
+      <stop id="sound" offset="1" stop-color="rgb(230, 230, 230)" />
     </radialGradient>
     <filter id="Rectangle_410" x="0" y="0" width="380" height="530" filterUnits="userSpaceOnUse">
       <feOffset dy="3" input="SourceAlpha" />
@@ -94,8 +94,7 @@ export default defineComponent({
       if (zeroed > 0) {
         radius.value = zeroed;
       }
-      const amplified = zeroed;
-      if (payload.speech === think) {
+      if (payload.speech === think.value) {
         return;
       } else {
         think.value = payload.speech;
diff --git a/src/components/UI/UIDetails/messageRenderer.vue b/src/components/UI/UIDetails/messageRenderer.vue
index c3f7795..0b9cce3 100644
--- a/src/components/UI/UIDetails/messageRenderer.vue
+++ b/src/components/UI/UIDetails/messageRenderer.vue
@@ -1,4 +1,10 @@
 <template>
+<svg
+  version="1.1"
+  xmlns="http://www.w3.org/2000/svg"
+  xmlns:xlink="http://www.w3.org/1999/xlink"
+  viewBox="0 0 350 500"
+>
   <g id="messageRenderer">
     <g
       v-for="(item, index) in renderArr"
@@ -17,6 +23,7 @@
       </transition>
     </g>
   </g>
+</svg>
 </template>
 
 <script lang="ts">