From ee63bb0ac1b9993c3f4791cb4f63603cb7ff613a Mon Sep 17 00:00:00 2001
From: Andrew Gundersen <gundersena@xavier.edu>
Date: Sat, 29 May 2021 10:36:43 -0500
Subject: [PATCH] more audio changes

---
 src/background/audio.ts 14-00-16-259.ts       |  37 ----
 src/background/init.ts                        |   9 +-
 src/background/session.ts                     |   7 +-
 .../inputitem/controllers/audio-control.ts    | 111 ------------
 .../inputitem/helpers/animations.ts           |  26 +++
 .../inputitem/helpers/audiocontrol.ts         | 113 ++++++++++++
 .../inputitem/helpers/mediarecorder.ts        |  74 ++++++++
 .../textcontrol.ts}                           |   0
 src/components/inputitem/inputItem.vue        |   8 +-
 src/components/messenger.vue                  |   2 +-
 src/modules/electron-audio.ts                 | 170 ------------------
 11 files changed, 222 insertions(+), 335 deletions(-)
 delete mode 100644 src/background/audio.ts 14-00-16-259.ts
 delete mode 100644 src/components/inputitem/controllers/audio-control.ts
 create mode 100644 src/components/inputitem/helpers/animations.ts
 create mode 100644 src/components/inputitem/helpers/audiocontrol.ts
 create mode 100644 src/components/inputitem/helpers/mediarecorder.ts
 rename src/components/inputitem/{controllers/text-control.ts => helpers/textcontrol.ts} (100%)
 delete mode 100644 src/modules/electron-audio.ts

diff --git a/src/background/audio.ts 14-00-16-259.ts b/src/background/audio.ts 14-00-16-259.ts
deleted file mode 100644
index d4d905c..0000000
--- a/src/background/audio.ts 14-00-16-259.ts	
+++ /dev/null
@@ -1,37 +0,0 @@
-import { ipcMain } from "electron";
-import electronAudio from "@/modules/electron-audio";
-
-export default function initAudioIO () {
-
-    console.log('[AUDIO]initializing audio');
-
-    // initialize the audio interface
-    const {
-        read, 
-        stopRead, 
-        write, 
-        shutdown 
-    } = electronAudio();
-
-    // event driven controls
-
-    // begin capuring audio from input stream
-    ipcMain.on("start-recording", () => {
-        console.log("[AUDIO]Reading...");
-        read();
-    });
-
-    // write a string of audio to the output stream
-    ipcMain.on("write-audio", (_e: any, audio: string) => write(audio));
-
-    // close the audio streams
-    ipcMain.on("shutdown-audio", shutdown);
-
-    // stop capturing audio and return it to caller
-    ipcMain.handle("stop-recording", () => {
-        console.log("[AUDIO]Stopping read...");
-        const audio = stopRead();
-        return audio;
-    });
-
-}
\ No newline at end of file
diff --git a/src/background/init.ts b/src/background/init.ts
index e11e34c..b3eff82 100644
--- a/src/background/init.ts
+++ b/src/background/init.ts
@@ -4,7 +4,6 @@
 import { app, dialog } from "electron";
 import { createWindow } from './window';
 import { initSession } from './session';
-import initAudioIO from './audio';
 import { backgroundMitt } from '@/modules/emitter';
 
 let win: boolean;
@@ -43,13 +42,10 @@ async function main(dev: boolean) {
     console.log("MAIN:Initializing Electron App.")
 
     // Must wait til window is created.
-    // await createWindow();
-
-    // Begin audio stream.
-    initAudioIO();
+    await createWindow();
 
     // Instantiate socket session with crimata-platorm.
-    // initSession(dev);
+    initSession(dev);
 
 }
 
@@ -64,7 +60,6 @@ export function initApp(dev: boolean): void {
 
     // Must keep to ensure app doesn't quit on close.
     app.on("before-quit", async () => {
-        await stopStream();
     });
 
     // Must keep to ensure app doesn't quit on close.
diff --git a/src/background/session.ts b/src/background/session.ts
index 633050a..82d1f4b 100644
--- a/src/background/session.ts
+++ b/src/background/session.ts
@@ -14,7 +14,6 @@ import { ipcMain, IpcMainEvent, IpcMainInvokeEvent } from "electron";
 
 import useWebSockets from "@/modules/websockets";
 
-import { play } from "./audio";
 import { renderMessage } from "@/modules/message";
 import { AuthProtocol, SessionState, Profile } from "@/types"; 
 
@@ -88,9 +87,9 @@ const onMessage = (data: string) => {
 
         if (win) {
             console.log("SESS:Emitting standard message.")
-            if (message.audio) {
-                play(message.audio)
-            }
+            // if (message.audio) {
+            //     play(message.audio)
+            // }
             ipcEmit("render-message", message)
         }
         
diff --git a/src/components/inputitem/controllers/audio-control.ts b/src/components/inputitem/controllers/audio-control.ts
deleted file mode 100644
index c9efaa9..0000000
--- a/src/components/inputitem/controllers/audio-control.ts
+++ /dev/null
@@ -1,111 +0,0 @@
-import anime from "animejs";
-import useMitt from "@/modules/mitt";
-import { useIpc } from '@/modules/ipc';
-import { onMounted, onUnmounted, ref, Ref } from "vue";
-import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
-import { renderMessage, clientMessage } from '@/modules/message';
-
-
-function showRecIcon () {
-
-    anime({
-        targets: '#recIcon',
-        opacity: [0, 0.75],
-        scale: [0.0, 1],
-        duration: 250,
-        easing: 'linear', 
-    })
-
-}
-
-function hideRecIcon () {
-
-    anime({
-        targets: '#recIcon',
-        opacity: [0.75, 0],
-        scale: [1, 0],
-        duration: 250,
-        easing: 'linear', 
-    })
-
-}
-
-
-export default function useAudioInputController (typing: Ref) {
-
-    // For sending messages.
-    const { post, invoke } = useIpc();
-    const { emitter } = useMitt();
-
-    // Keepp track of when we are recording.
-    const recording = ref(false);
-
-    //---Callbacks-----------------------------------------------
-
-    const onKeyDown = (e: KeyboardEvent) => {
-        const cmd = keyboardNameMap[e.keyCode];
-        // console.log(cmd)
-
-        // Start recording on space bar.
-        if (cmd == "SPACE" && !recording.value && !typing.value) {
-
-            console.log("INPT:Starting record.")
-            post("start-recording", "");
-
-            showRecIcon()
-            recording.value = true;
-
-        }
-
-    }
-
-    const onKeyUp = async (e: KeyboardEvent) => {
-        const cmd = keyboardNameMap[e.keyCode];
-
-        // Stop recording on space up.
-        if (cmd == "SPACE" && recording.value) {
-
-            // Create a message.
-            const message = renderMessage(
-                "", 
-                "", 
-                "", 
-                "sf"
-            )
-
-            // Render it immediately.
-            emitter.emit("self-message", message);
-
-            // Stop recording and get audio from recorder.
-            console.log("INPT:Stopping record.")
-            const audio = await invoke("stop-recording", "");
-            
-            // Send message to the backend for processing.
-            const clientM = clientMessage("", audio, message.uid);
-            post('client-message', clientM);
-
-            hideRecIcon()
-            recording.value = false;
-
-        }
-
-    }
-
-    //-----------------------------------------------------------
-
-    onMounted(() => {
-        window.addEventListener("keydown", onKeyDown);
-        window.addEventListener("keyup", onKeyUp);
-    })
-
-    onUnmounted(() => {
-        window.removeEventListener("keydown", onKeyDown);
-        window.removeEventListener("keyup", onKeyUp);
-    });
-
-
-    return {
-        recording
-    }
-
-}
diff --git a/src/components/inputitem/helpers/animations.ts b/src/components/inputitem/helpers/animations.ts
new file mode 100644
index 0000000..9683acc
--- /dev/null
+++ b/src/components/inputitem/helpers/animations.ts
@@ -0,0 +1,26 @@
+import anime from "animejs";
+
+
+export function showRecIcon () {
+
+    anime({
+        targets: '#recIcon',
+        opacity: [0, 0.75],
+        scale: [0.0, 1],
+        duration: 250,
+        easing: 'linear', 
+    })
+
+}
+
+export function hideRecIcon () {
+
+    anime({
+        targets: '#recIcon',
+        opacity: [0.75, 0],
+        scale: [1, 0],
+        duration: 250,
+        easing: 'linear', 
+    })
+
+}
\ No newline at end of file
diff --git a/src/components/inputitem/helpers/audiocontrol.ts b/src/components/inputitem/helpers/audiocontrol.ts
new file mode 100644
index 0000000..f7335b6
--- /dev/null
+++ b/src/components/inputitem/helpers/audiocontrol.ts
@@ -0,0 +1,113 @@
+import useMitt from "@/modules/mitt";
+import { useIpc } from '@/modules/ipc';
+import { showRecIcon, hideRecIcon } from "./animations";
+// import useMediaRecorder from "./mediarecorder";
+import { onMounted, onUnmounted, ref, Ref } from "vue";
+import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
+import { renderMessage, clientMessage } from '@/modules/message';
+import { AudioData } from "@/types";
+
+
+
+export default function useAudioInputController (typing: Ref) {
+
+    let mediaRecorder: any;
+    let buffer = '';
+
+    // For sending messages.
+    const { post, invoke } = useIpc();
+    const { emitter } = useMitt();
+
+    const recording = ref(false);
+
+    // initiate media recorder
+    if (navigator.mediaDevices) {
+        console.log("Initializing media recorder.");
+
+        const usrOptions: any = {
+            audio: true,
+            video: false
+        }
+
+        navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => {
+
+            const context = new AudioContext();
+            const source = context.createMediaStreamSource(stream);
+            const processor = context.createScriptProcessor(1024, 1, 1);
+
+            source.connect(processor);
+            processor.connect(context.destination);
+
+            processor.onaudioprocess = (e: AudioProcessingEvent) => {
+                if (recording.value) {
+                    const arr = new Uint16Array(e.inputBuffer);
+                    buffer += String.fromCharCode.apply(null, new Float32Array(e.inputBuffer));
+                } else buffer = '';
+            };
+
+        });
+
+    }
+
+    //---Callbacks-----------------------------------------------
+
+    const onKeyDown = (e: KeyboardEvent) => {
+        const cmd = keyboardNameMap[e.keyCode];
+
+        // Start recording on space bar.
+        if (cmd == "SPACE" && !typing.value && !recording.value) {
+  
+                    console.log("INPT:Starting record, filling buffer.")
+                    recording.value = true;
+                    showRecIcon();
+
+        }
+
+    }
+
+    const onKeyUp = async (e: KeyboardEvent) => {
+        const cmd = keyboardNameMap[e.keyCode];
+
+        // Stop recording on space up.
+        if (cmd == "SPACE" && recording.value) {
+
+            // get audio from buffer then stop recording
+            console.log("INPT:Stopping record.")
+            const audio = buffer;
+            recording.value = false;
+            hideRecIcon()
+
+            // render a place holder message
+            const message = renderMessage(
+                "", 
+                "", 
+                "", 
+                "sf"
+            );
+
+            emitter.emit("self-message", message); 
+
+            console.log(`Posting audio: ${audio}`);
+            console.log(typeof audio);
+
+        }
+
+    }
+
+    //-----------------------------------------------------------
+
+    onMounted(() => {
+        window.addEventListener("keydown", onKeyDown);
+        window.addEventListener("keyup", onKeyUp);
+    })
+
+    onUnmounted(() => {
+        window.removeEventListener("keydown", onKeyDown);
+        window.removeEventListener("keyup", onKeyUp);
+    });
+
+    return {
+        recording
+    }
+
+}
diff --git a/src/components/inputitem/helpers/mediarecorder.ts b/src/components/inputitem/helpers/mediarecorder.ts
new file mode 100644
index 0000000..9d78709
--- /dev/null
+++ b/src/components/inputitem/helpers/mediarecorder.ts
@@ -0,0 +1,74 @@
+
+
+
+if (navigator.mediaDevices) {
+  console.log('getUserMedia supported.');
+
+  var constraints = { audio: true };
+  var chunks = [];
+
+  navigator.mediaDevices.getUserMedia(constraints)
+  .then(function(stream) {
+
+    var mediaRecorder = new MediaRecorder(stream);
+
+    visualize(stream);
+
+    record.onclick = function() {
+      mediaRecorder.start();
+      console.log(mediaRecorder.state);
+      console.log("recorder started");
+      record.style.background = "red";
+      record.style.color = "black";
+    }
+
+    stop.onclick = function() {
+      mediaRecorder.stop();
+      console.log(mediaRecorder.state);
+      console.log("recorder stopped");
+      record.style.background = "";
+      record.style.color = "";
+    }
+
+    mediaRecorder.onstop = function(e) {
+      console.log("data available after MediaRecorder.stop() called.");
+
+      var clipName = prompt('Enter a name for your sound clip');
+
+      var clipContainer = document.createElement('article');
+      var clipLabel = document.createElement('p');
+      var audio = document.createElement('audio');
+      var deleteButton = document.createElement('button');
+
+      clipContainer.classList.add('clip');
+      audio.setAttribute('controls', '');
+      deleteButton.innerHTML = "Delete";
+      clipLabel.innerHTML = clipName;
+
+      clipContainer.appendChild(audio);
+      clipContainer.appendChild(clipLabel);
+      clipContainer.appendChild(deleteButton);
+      soundClips.appendChild(clipContainer);
+
+      audio.controls = true;
+      var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });
+      chunks = [];
+      var audioURL = URL.createObjectURL(blob);
+      audio.src = audioURL;
+      console.log("recorder stopped");
+
+      deleteButton.onclick = function(e) {
+        evtTgt = e.target;
+        evtTgt.parentNode.parentNode.removeChild(evtTgt.parentNode);
+      }
+    }
+
+    mediaRecorder.ondataavailable = function(e) {
+      chunks.push(e.data);
+    }
+  })
+  .catch(function(err) {
+    console.log('The following error occurred: ' + err);
+  })
+}
+
diff --git a/src/components/inputitem/controllers/text-control.ts b/src/components/inputitem/helpers/textcontrol.ts
similarity index 100%
rename from src/components/inputitem/controllers/text-control.ts
rename to src/components/inputitem/helpers/textcontrol.ts
diff --git a/src/components/inputitem/inputItem.vue b/src/components/inputitem/inputItem.vue
index 4c9b021..861b9d8 100644
--- a/src/components/inputitem/inputItem.vue
+++ b/src/components/inputitem/inputItem.vue
@@ -26,12 +26,10 @@
 import { defineComponent } from "vue";
 import draggify from "@/modules/draggify";
 
-import TextInput from "@/components/textInput.vue";
+import TextInput from "./textinput.vue";
 
-import useTextInputController from 
-  "@/components/controllers/text-control";
-import useAudioInputController from 
-  "@/components/controllers/audio-control";
+import useTextInputController from "./helpers/textcontrol";
+import useAudioInputController from "./helpers/audiocontrol";
 
 export default defineComponent({
   name: "InputItem",
diff --git a/src/components/messenger.vue b/src/components/messenger.vue
index edc50a1..3719730 100644
--- a/src/components/messenger.vue
+++ b/src/components/messenger.vue
@@ -18,7 +18,7 @@
 <script lang="ts">
 import { defineComponent, onMounted, onUnmounted } from "vue";
 import Message from "@/components/message.vue";
-import InputItem from "@/components/inputItem.vue";
+import InputItem from "@/components/inputitem/inputitem.vue";
 import Settings from "@/components/settings.vue";
 import useMitt from "@/modules/mitt";
 import useMessages from "@/modules/messages";
diff --git a/src/modules/electron-audio.ts b/src/modules/electron-audio.ts
deleted file mode 100644
index a7a710b..0000000
--- a/src/modules/electron-audio.ts
+++ /dev/null
@@ -1,170 +0,0 @@
-const portAudio = require('naudiodon');
-const stream = require('stream');
-const fs = require('fs');
-
-import { AudioData } from "@/types";
-
-
-// split Buffer into an array of len-sized Buffers
-const bufSplit = (buf: Buffer, len: number) => {
-    const chunks = [];
-    let i = 0;
-    let L = len;
-
-    while(i < buf.byteLength) {
-      chunks.push(buf.slice(i, L));
-      i = L;
-      L += len;
-    }
-
-    return chunks;
-}
-
-
-export default function electronAudio () {
-
-    // current config
-    let currentHost: number;
-    let currentInputDevice: number;
-    let currentOutputDevice: number;
-
-    // audio streams
-    let ai: any;
-    let ao: any;
-
-    // write audio here
-    let record = false;
-    let readAudio = '';
-
-    console.log(portAudio.getHostAPIs());
-
-    // get info for given host api
-    const getHostInfo = (id: number) => {
-        return portAudio.getHostAPIs().HostAPIs.filter((host: any) =>
-            host.id === id)[0];
-    }
-
-    // get info for given device
-    const getDeviceInfo = (id: number) => {
-        return portAudio.getDevices().filter((device: any) =>
-            device.id === id)[0];
-    };
-
-    // constructor to start audio streams
-    const setup = () => {
-        console.log("Checking audio devices.");
-
-        currentHost = portAudio.getHostAPIs().defaultHostAPI;
-        const hostInfo = getHostInfo(currentHost);
-
-        if (currentInputDevice !== hostInfo.defaultInput) {
-            currentInputDevice = hostInfo.defaultInput;
-            ai = setInputStream();
-            ai.start();
-        }
-
-        if (currentOutputDevice !== hostInfo.defaultOutput) {
-            currentOutputDevice = hostInfo.defaultOutput;
-            ao = setOutputStream()
-            ao.start();
-        }
-
-    };
-
-    // init and start input stream
-    const setInputStream = () => {
-
-        const inputDeviceInfo = getDeviceInfo(currentInputDevice);
-        console.log(inputDeviceInfo);
-
-        return new portAudio.AudioIO({
-            inOptions: {
-                channelCount: inputDeviceInfo.maxInputChannels,
-                sampleFormat: portAudio.SampleFormat16Bit,
-                sampleRate: inputDeviceInfo.defaultSampleRate,
-                deviceId: inputDeviceInfo.id,
-                closeOnError: false
-            }
-        })
-        .setEncoding("hex")
-        .on('error', () => console.log("input stream error"))
-        .on('data', (chunk: Buffer) => {
-            console.log("data")
-            if (record) readAudio += chunk.toString();
-        });
-
-    }
-
-    // init and start output stream
-    const setOutputStream = () => {
-
-        const outputDeviceInfo = getDeviceInfo(currentOutputDevice);
-        console.log(outputDeviceInfo);
-
-        return new portAudio.AudioIO({
-            outOptions: {
-                channelCount: outputDeviceInfo.maxOutputChannels,
-                sampleFormat: portAudio.SampleFormat16Bit,
-                sampleRate: outputDeviceInfo.defaultSampleRate,
-                deviceId: outputDeviceInfo.id,
-                closeOnError: false
-            }
-        })
-        .on('error', () => console.log("output stream error"))
-        
-    }
-
-    // pipe ai to writable
-    const read = () => {
-        record = true;
-    };
-
-    // remove pipe and return audio contents
-    const stopRead = (): AudioData => {
-        const deviceInfo = getDeviceInfo(currentInputDevice);
-
-        const audio: AudioData = {
-            hex: readAudio,
-            channels: deviceInfo.maxInputChannels,
-            fs: deviceInfo.defaultSampleRate
-        }
-
-        record = false;
-        readAudio = '';
-
-        return audio;
-    };
-
-    // write a string of audio to stream
-    const write = (audio: string) => {
-
-        // convert audio into chunks of buffers
-        const chunks = bufSplit(
-            Buffer.from(audio, 'hex'),
-            8192 // chunk size
-        );
-
-        // write the chunks
-        const readable = new stream.Readable()
-        readable.pipe(ao);
-        chunks.forEach(chunk => readable.push(chunk))
-
-    };
-
-    // quit both streams
-    const shutdown = () => {
-        if (ao) ao.quit();
-        if (ai) ai.quit();
-    };
-
-    // run
-    setInterval(setup, 2000);
-
-    return {
-        read,
-        stopRead,
-        write,
-        shutdown
-    };
-
-}
\ No newline at end of file
-- 
2.43.0