From 5a3cd6004655a1ff1d9b56f28f475e5bb179d37c Mon Sep 17 00:00:00 2001
From: Enrique Hernandez <hernandeze2@xavier.edu>
Date: Fri, 11 Dec 2020 10:41:52 -0600
Subject: [PATCH] add audioRenderer + ipc event listeners

---
 src/background.ts                             |  57 ++++++----
 src/components/UI/UIDetails/inputItem.vue     |  46 +++++---
 .../audioRenderer/useAudioRenderer.ts         |  35 ++----
 .../streamRecorder/useStreamRecorder.ts       | 103 ++++++++++++++++--
 src/preload.ts                                |  12 ++
 5 files changed, 180 insertions(+), 73 deletions(-)

diff --git a/src/background.ts b/src/background.ts
index 53227e0..6b1284a 100644
--- a/src/background.ts
+++ b/src/background.ts
@@ -1,6 +1,6 @@
 "use strict";
 
-import { app, protocol, BrowserWindow } from "electron";
+import { app, protocol, BrowserWindow, ipcMain } from "electron";
 import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
 import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
 const isDevelopment = process.env.NODE_ENV !== "production";
@@ -56,26 +56,41 @@ function createWindow() {
     win.loadURL("app://./index.html");
   }
 
-  // win.webContents.on('did-finish-load', () => {
-  //   if (win) {
-  //     recorder.start({
-  //       onAudio: (audio: object, speech: boolean) => {
-  //         if (win) {
-  //           win.setTitle("Listening...");
-  //           // win.webContents.send('render-audio', {
-  //           //   audio: audio,
-  //           //   speech: speech
-  //           // })
-  //           if (speech) {
-  //             win.setTitle("Recording...");
-  //             // TODO: send audio to crimata BE
-  //             // ws.send(audio);
-  //           }
-  //         }
-  //       }
-  //     });
-  //   }
-  // })
+
+  win.webContents.on('did-finish-load', () => {
+    // ipcMain.on('audio-stream', (Event: any, Payload: any) => {
+    //   if (Payload.stream) {
+    //     console.log('begin streaming');
+    //     recorder.start({
+    //       onAudio: (audio: object, speech: boolean) => {
+    //         console.log(audio)
+    //       }
+    //     });
+    //   } else {
+    //     recorder.stop();
+    //     console.log('stop streaming')
+    //   }
+    // })
+
+    // if (win) {
+    //   recorder.start({
+    //     onAudio: (audio: object, speech: boolean) => {
+    //       if (win) {
+    //         win.setTitle("Listening...");
+    //         // win.webContents.send('render-audio', {
+    //         //   audio: audio,
+    //         //   speech: speech
+    //         // })
+    //         if (speech) {
+    //           win.setTitle("Recording...");
+    //           // TODO: send audio to crimata BE
+    //           // ws.send(audio);
+    //         }
+    //       }
+    //     }
+    //   });
+    // }
+  })
 
   win.on("closed", () => {
     win = null;
diff --git a/src/components/UI/UIDetails/inputItem.vue b/src/components/UI/UIDetails/inputItem.vue
index f4e2f36..7e69a15 100644
--- a/src/components/UI/UIDetails/inputItem.vue
+++ b/src/components/UI/UIDetails/inputItem.vue
@@ -11,31 +11,36 @@
 <script lang="ts">
 import useInputRenderer from "@/composables/inputItem/useInputRenderer";
 import useStreamRecorder from "@/composables/streamRecorder/useStreamRecorder";
-import {
-  defineComponent,
-  watch,
-  onMounted
-} from "vue";
+import { defineComponent, watch, onMounted, ref } from "vue";
 export default defineComponent({
   name: "InputItem",
   setup() {
+    const record = ref(false);
     const { inputXoffset, input } = useInputRenderer();
-    const { initRecorder } = useStreamRecorder();
+    const { initRecorder } = useStreamRecorder(record);
 
     onMounted(async () => {
-
       // step get streamRecorder reference
       const { streamRecorder } = await initRecorder();
-      
+
       // add event listener for space keyup
       window.addEventListener("keyup", (e) => {
         if (e.key === " " && streamRecorder.state === "recording") {
-          console.log("stop recording")
+          console.log("stop recording");
+          record.value = false;
           streamRecorder.stop();
-          console.log('recorder is', streamRecorder.state);
+
+          // window.postMessage(
+          //   {
+          //     myTypeField: "audio-stream",
+          //     stream: false,
+          //   },
+          //   "*"
+          // );
+          console.log("recorder is", streamRecorder.state);
           input.value = "";
         }
-      })
+      });
 
       // begin recording on space
       watch(input, (input, prevInput) => {
@@ -43,14 +48,23 @@ export default defineComponent({
           return;
         }
         if (input === " ") {
-          console.log("record")
+          console.log("record");
+          record.value = true;
+
+          // window.postMessage(
+          //   {
+          //     myTypeField: "audio-stream",
+          //     stream: true,
+          //   },
+          //   "*"
+          // );
           streamRecorder.start();
-          console.log('recorder is', streamRecorder.state);
+          console.log("recorder is", streamRecorder.state);
         }
-      })
-    })
+      });
+    });
 
-   return {
+    return {
       input,
       inputXoffset,
     };
diff --git a/src/composables/audioRenderer/useAudioRenderer.ts b/src/composables/audioRenderer/useAudioRenderer.ts
index 9047c69..bf44178 100644
--- a/src/composables/audioRenderer/useAudioRenderer.ts
+++ b/src/composables/audioRenderer/useAudioRenderer.ts
@@ -1,37 +1,22 @@
-export default function useAudioRenderer() {
+export default function useAudioRenderer(stream: MediaStream) {
     const audioCtx = new (window.AudioContext)();
     const analyzer = audioCtx.createAnalyser();
+    const source = audioCtx.createMediaStreamSource(stream);
+    source.connect(analyzer);
+
     analyzer.fftSize = 2048;
     const xInitial = 24;
     const xFinal = 640;
-    // analyzer.minDecibels = -90;
+    analyzer.minDecibels = -90;
     analyzer.smoothingTimeConstant = 0.85;
     const dataArray = new Uint8Array(analyzer.frequencyBinCount);
-    const sampleRate = 16000;
-
-    function convertBlock(buffer: ArrayBuffer): Float32Array {
-        // incoming data is an ArrayBuffer
-        const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
-        const l = incomingData.length; // length, we need this for the loop
-        const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
-        for (let i = 0; i < l; i++) {
-            outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
-        }
-        return outputData; // return the Float32Array
-    }
-
-    const getAudioFrequencyData = () => {
-
-    }
-
-    const renderAudio = (payload) => {
+    // const sampleRate = 16000;
 
-        // step 1: convert audio from Uint8 to Float32Array
-        const floatArray = convertBlock(payload.audio.buffer);
+    const renderAudio = (audio: Float32Array, sampleRate: number) => {
 
         // step 2: source audio frequency {ffft} data
-        const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
-        buffer.copyToChannel(floatArray, 0, 0);
+        const buffer = audioCtx.createBuffer(1, audio.length, sampleRate);
+        buffer.copyToChannel(audio, 0, 0);
         const source = audioCtx.createBufferSource();
         source.buffer = buffer;
         source.connect(analyzer);
@@ -44,7 +29,7 @@ export default function useAudioRenderer() {
         const average = sum / dataView.length;
         const scaled = average / 255;
         const zeroed = scaled - 0.6;
-        return zeroed;
+        return average;
     }
 
     return {
diff --git a/src/composables/streamRecorder/useStreamRecorder.ts b/src/composables/streamRecorder/useStreamRecorder.ts
index fe99561..2d9d102 100644
--- a/src/composables/streamRecorder/useStreamRecorder.ts
+++ b/src/composables/streamRecorder/useStreamRecorder.ts
@@ -1,20 +1,101 @@
-export default function useStreamRecorder() {
+import { watch } from "vue";
+import useAudioRenderer from "@/composables/audioRenderer/useAudioRenderer";
+export default function useStreamRecorder(record: any) {
     let streamRecorder: MediaRecorder;
     const chunks: Blob[] = [];
+
+    window.AudioContext = window.AudioContext;
+    let audio_context = new AudioContext();
+    const analyzer = audio_context.createAnalyser();
+    analyzer.fftSize = 256;
+    var bufferLengthAlt = analyzer.frequencyBinCount;
+    var dataArrayAlt = new Uint8Array(bufferLengthAlt);
+
+    let source;
+
+
     const initRecorder = async () => {
-           const stream = await  navigator.mediaDevices.getUserMedia({ audio: true });
-           streamRecorder = new MediaRecorder(stream);
-           streamRecorder.ondataavailable = (e: any) =>  { chunks.push(e.data) }
-           streamRecorder.onstop = (e: any) => {
-             const audioBlob = new Blob(chunks, {'type': 'audio/ogg; codecs=opus'} );
-             // do something with blob
-           }
-          return {
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+        handleStream(stream);
+        streamRecorder = new MediaRecorder(stream);
+        streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
+        streamRecorder.onstop = (e: any) => {
+            const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' });
+            // do something with blob
+        }
+        return {
             streamRecorder
-          }
+        }
+    }
+
+    interface Window {
+        AudioContext: any;
+        webkitAudioContext: any;
+    }
+
+    interface Event {
+        inputBuffer: AudioBuffer;
+    }
+
+    let bufferLen = 4096;
+    let numChannels = 1;
+    function visualize() {
+
+        function drawAlt() {
+
+            const drawVisual = requestAnimationFrame(drawAlt);
+            analyzer.getByteFrequencyData(dataArrayAlt);
+
+            for (var i = 0; i < bufferLengthAlt; i++) {
+                console.log(dataArrayAlt[i])
+            }
+        }
+        drawAlt();
+    }
+
+    function handleStream(stream: MediaStream) {
+        source = audio_context.createMediaStreamSource(stream);
+        source.connect(analyzer);
+        visualize();
+        //
+        // const context = source.context;
+        // const sampleRate = context.sampleRate;
+
+        // const node = context.createScriptProcessor.call(
+        //     context,
+        //     bufferLen,
+        //     numChannels,
+        //     numChannels
+        // );
+
+        // source.connect(node);
+
+        // node.onaudioprocess = (e: Event) => {
+        //     const inputBuffer = new Float32Array(bufferLen);
+
+        //     e.inputBuffer.copyFromChannel(inputBuffer, 0);
+
+        //     // const int16 = new Int16Array(inputBuffer.buffer);
+
+        //     // window.postMessage(
+        //     //     {
+        //     //         myTypeField: "handle-audio-stream",
+        //     //         audioData: int16
+        //     //     },
+        //     //     "*"
+        //     // );
+        // };
+
+        // watch(record, (rec) => {
+        //     if (rec) {
+        //         node.connect(context.destination);
+        //     } else {
+        //         node.disconnect(context.destination);
+        //     }
+        // })
     }
 
     return {
-      initRecorder
+        initRecorder
     }
 }
diff --git a/src/preload.ts b/src/preload.ts
index 9e33238..82f6314 100644
--- a/src/preload.ts
+++ b/src/preload.ts
@@ -6,3 +6,15 @@ interface Window {
   ipcRenderer: typeof ipcRenderer;
 }
 window.ipcRenderer = ipcRenderer;
+
+process.once("loaded", () => {
+  window.addEventListener("message", event => {
+    // do something with custom event
+    const message = event.data;
+
+    if (message.myTypeField === "audio-stream") {
+      ipcRenderer.send("audio-stream", message);
+    }
+  });
+});
+
-- 
2.43.0