prototype browser audio code

package.json

@@ -26,7 +26,7 @@
     "electron-is-dev": "^2.0.0",
     "electron-updater": "^4.3.8",
     "mitt": "^2.1.0",
-    "naudiodon": "^2.3.2",
+    "naudiodon": "^2.3.5",
     "node-record-lpcm16": "^1.0.1",
     "update-electron-app": "^2.0.1",
     "uuid": "^8.3.2",
@@ -57,7 +57,7 @@
     "eslint": "^6.7.2",
     "eslint-plugin-vue": "^7.0.0-0",
     "lint-staged": "^9.5.0",
-    "node-sass": "^4.12.0",
+    "node-sass": "4",
     "optimize-wasm-webpack-plugin": "^1.0.12",
     "sass-loader": "^8.0.2",
     "spectron": "11.0.0",

src/background/audio.ts

@@ -1,188 +0,0 @@
-/* eslint @typescript-eslint/no-var-requires: "off" */
-
-"use strict";
-
-import { ipcMain } from "electron";
-import { backgroundMitt } from '@/modules/emitter';
-
-const portAudio = require('naudiodon');
-
-// Audio in and out stream objects.
-let ai: typeof portAudio.AudioIO | boolean = false;
-let ao: typeof portAudio.AudioIO | boolean = false;
-
-// Whether activly recording.
-let record = false;
-
-const audioContainer = {
-    input: '',
-}
-
-const audioOptions = {
-    channelCount: 1,
-    sampleFormat: 16,
-    sampleRate: 16000,
-    deviceId: -1,
-    closeOnError: false,
-}
-
-// Toggles record to true to begin capturing chunks.
-const onRecordingStart = (_event: any, _payload: any) => {
-    console.log("AUDIO: Beginning audio capture.")
-    record = true;
-}
-
-// Returns recorded audio to frontend and sets record to false.
-const onRecordingEnd = async (_event: any, payload: any) => {
-    console.log("AUDIO:Sending audio to browser.")
-
-    return new Promise((resolve, reject) => {
-
-        try {
-            resolve(audioContainer.input);
-            record = false;
-        } catch (e) {
-            reject()
-        }
-
-    });
-};
-
-
-// Main audio function run by run.ts module.
-export function initAudioIO(): void {
-    console.log("AUDIO:Starting io streams.")
-
-    if (!ai) {
-
-        // Initialize and start input stream.
-        ai = new portAudio.AudioIO({ inOptions: audioOptions });
-        ai.setEncoding("hex");
-        ai.start();
-
-        // On each data chunk...
-        ai.on('data', (chunk: string) => {
-
-            // If recording, we capture the data.
-            if (record) {
-                console.log('AUDIO:Recording...')
-                audioContainer.input += chunk;
-            }
-
-            // Else, we don't capture and also clear audioContainer.
-            else {
-                if (audioContainer.input.length) {
-                    audioContainer.input = "";
-                }
-            }
-
-        });
-    }
-
-    if (!ao) {
-
-        // Initialize and start input stream.
-        ao = new portAudio.AudioIO({ outOptions: audioOptions });
-        ao.start();
-
-    }
-
-    // Listen to record.
-    console.log("AUDIO:Adding recording listeners.")
-
-    ipcMain.removeAllListeners("start-recording");
-    ipcMain.on("start-recording", onRecordingStart);
-
-    ipcMain.removeHandler("stop-recording");
-    ipcMain.handle("stop-recording", onRecordingEnd);
-}
-
-
-// ---Audio playback--------------------------------------------
-
-// Split Buffer into an array of len-sized Buffers.
-function bufSplit(buf: Buffer, len: number): Array<Buffer> {
-    const chunks = [];
-    let i = 0;
-    let L = len;
-
-    while(i < buf.byteLength) {
-      chunks.push(buf.slice(i, L));
-      i = L;
-      L += len;
-    }
-
-    return chunks;
-}
-
-// Audio playback.
-export function play(input: string): void {
-
-    // Format the audio.
-    const audio = bufSplit(
-        Buffer.from(input as string, 'hex'),
-        8192
-    );
-
-    // Called on end of write.
-    const callback = () => {
-
-        // We stop audio playback anim.
-        backgroundMitt.emit('ipc-renderer', {
-            endpoint: 'stop-playback-anim'
-        });
-
-    }
-
-    write();
-
-    // Iterate through audio array and write buffers to portAudio writable.
-    function write() {
-        let chunk: Buffer;
-        let ok = true;
-        let i = 0;
-
-        do {
-            chunk = audio[i];
-            if (i === audio.length - 1) {
-                // write last chunk.
-                ao.write(chunk, null, callback);
-            } else {
-                // check for backpreassure.
-                ok = ao.write(chunk, null);
-            }
-            i++;
-        } while (i < audio.length && ok);
-
-        if (i < audio.length) {
-            // Had to stop early!
-            // Write some more once it drains.
-            ao.once('drain', write);
-        }
-    }
-}
-
-// -------------------------------------------------------------
-
-// Get's called on window close.
-export async function stopStream() {
-    console.log("AUDIO:Stopping audio stream.")
-    if (ai) {
-        try {
-            await ai.quit()
-        } catch(e){
-            console.log('AUDIO: Failed to shutdown audio input.');
-            throw e;
-        }
-    }
-    if (ao) {
-        try {
-            await ao.quit()
-        } catch(e){
-            console.log('AUDIO: Failed to shutdown audio output.');
-            throw e;
-        }
-    }
-}
-
-

src/background/init.ts

@@ -4,7 +4,6 @@
 import { app, dialog } from "electron";
 import { createWindow } from './window';
 import { initSession } from './session';
-import { initAudioIO, stopStream } from './audio';
 import { backgroundMitt } from '@/modules/emitter';
 
 let win: boolean;
@@ -48,9 +47,6 @@ async function main(dev: boolean) {
     // Instantiate socket session with crimata-platorm.
     initSession(dev);
 
-    // Begin audio stream.
-    initAudioIO();
-
 }
 
 // Root function of app.
@@ -64,7 +60,6 @@ export function initApp(dev: boolean): void {
 
     // Must keep to ensure app doesn't quit on close.
     app.on("before-quit", async () => {
-        await stopStream();
     });
 
     // Must keep to ensure app doesn't quit on close.

src/background/session.ts

@@ -14,7 +14,6 @@ import { ipcMain, IpcMainEvent, IpcMainInvokeEvent } from "electron";
 
 import useWebSockets from "@/modules/websockets";
 
-import { play } from "./audio";
 import { renderMessage } from "@/modules/message";
 import { AuthProtocol, SessionState, Profile } from "@/types"; 
 
@@ -88,9 +87,9 @@ const onMessage = (data: string) => {
 
         if (win) {
             console.log("SESS:Emitting standard message.")
-            if (message.audio) {
+            // if (message.audio) {
-                play(message.audio)
+            //     play(message.audio)
-            }
+            // }
             ipcEmit("render-message", message)
         }
         
@@ -126,12 +125,12 @@ const { createSocket, sendMessage } = useWebSockets(onMessage, onOpen);
 
 // Handle messages from window/client.
 const onClientMessage = (_event: IpcMainEvent, payload: any) => {
-    console.log("New client message")
+    console.log("New client message");
 
-    const success = sendMessage(payload)
+    const success = sendMessage(payload);
 
     if (!success) {
-        console.log("Unable to send message: ", payload)
+        console.log("Unable to send message: ", payload);
     }
     
 }
@@ -150,14 +149,14 @@ export const initSession = (dev: boolean) => {
     // Open socket connection.
     let url = "ws://crimata.com:8760";
     if (dev) url = "ws://localhost:8760";
-    createSocket(url)
+    createSocket(url);
 
     // Attack browser window init listener.
-    ipcMain.removeAllListeners("app-mounted")
+    ipcMain.removeAllListeners("app-mounted");
     ipcMain.on("app-mounted", onNewBrowserWindow);
 
     // Attach listeners for frontend.
-    ipcMain.removeAllListeners("client-message")
+    ipcMain.removeAllListeners("client-message");
     ipcMain.on("client-message", onClientMessage);
 
     // Keep win up-to-date.

src/components/controllers/audioCtrl.ts

@@ -1,111 +0,0 @@
-import anime from "animejs";
-import useMitt from "@/modules/mitt";
-import { useIpc } from '@/modules/ipc';
-import { onMounted, onUnmounted, ref, Ref } from "vue";
-import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
-import { renderMessage, clientMessage } from '@/modules/message';
-
-
-function showRecIcon () {
-
-    anime({
-        targets: '#recIcon',
-        opacity: [0, 0.75],
-        scale: [0.0, 1],
-        duration: 250,
-        easing: 'linear', 
-    })
-
-}
-
-function hideRecIcon () {
-
-    anime({
-        targets: '#recIcon',
-        opacity: [0.75, 0],
-        scale: [1, 0],
-        duration: 250,
-        easing: 'linear', 
-    })
-
-}
-
-
-export default function useAudioInputController (typing: Ref) {
-
-    // For sending messages.
-    const { post, invoke } = useIpc();
-    const { emitter } = useMitt();
-
-    // Keepp track of when we are recording.
-    const recording = ref(false);
-
-    //---Callbacks-----------------------------------------------
-
-    const onKeyDown = (e: KeyboardEvent) => {
-        const cmd = keyboardNameMap[e.keyCode];
-        // console.log(cmd)
-
-        // Start recording on space bar.
-        if (cmd == "SPACE" && !recording.value && !typing.value) {
-
-            console.log("INPT:Starting record.")
-            post("start-recording", "");
-
-            showRecIcon()
-            recording.value = true;
-
-        }
-
-    }
-
-    const onKeyUp = async (e: KeyboardEvent) => {
-        const cmd = keyboardNameMap[e.keyCode];
-
-        // Stop recording on space up.
-        if (cmd == "SPACE" && recording.value) {
-
-            // Create a message.
-            const message = renderMessage(
-                "", 
-                "", 
-                "", 
-                "sf"
-            )
-
-            // Render it immediately.
-            emitter.emit("self-message", message);
-
-            // Stop recording and get audio from recorder.
-            console.log("INPT:Stopping record.")
-            const audio = await invoke("stop-recording", "");
-            
-            // Send message to the backend for processing.
-            const clientM = clientMessage("", audio, message.uid);
-            post('client-message', clientM);
-
-            hideRecIcon()
-            recording.value = false;
-
-        }
-
-    }
-
-    //-----------------------------------------------------------
-
-    onMounted(() => {
-        window.addEventListener("keydown", onKeyDown);
-        window.addEventListener("keyup", onKeyUp);
-    })
-
-    onUnmounted(() => {
-        window.removeEventListener("keydown", onKeyDown);
-        window.removeEventListener("keyup", onKeyUp);
-    });
-
-
-    return {
-        recording
-    }
-
-}

src/components/inputitem/animations.ts

@@ -0,0 +1,26 @@
+import anime from "animejs";
+
+
+export function showRecIcon () {
+
+    anime({
+        targets: '#recIcon',
+        opacity: [0, 0.75],
+        scale: [0.0, 1],
+        duration: 250,
+        easing: 'linear', 
+    })
+
+}
+
+export function hideRecIcon () {
+
+    anime({
+        targets: '#recIcon',
+        opacity: [0.75, 0],
+        scale: [1, 0],
+        duration: 250,
+        easing: 'linear', 
+    })
+
+}

src/components/inputitem/audiocontrol.ts

@@ -0,0 +1,261 @@
+import useMitt from "@/modules/mitt";
+import { useIpc } from '@/modules/ipc';
+import { showRecIcon, hideRecIcon } from "./animations";
+import { onMounted, onUnmounted, ref, Ref } from "vue";
+import keyboardNameMap from "./keyBoardMaps/keyboardNameMap";
+import { renderMessage, clientMessage } from '@/modules/message';
+
+const { post, invoke } = useIpc();
+
+
+function floatTo16bPCM(output: DataView, offset: number, input: Float32Array) {
+  for (var i = 0; i < input.length; i++, offset += 2) {
+    var s = Math.max(-1, Math.min(1, input[i]));
+    output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+  }
+}
+
+function writeFloat32 (output: DataView, offset: number, input: Float32Array) {
+  for (var i = 0; i < input.length; i++, offset += 4) {
+    output.setFloat32(offset, input[i], true)
+  }
+}
+
+function writeString(view: DataView, offset: number, string: string) {
+  for (var i = 0; i < string.length; i++) {
+    view.setUint8(offset + i, string.charCodeAt(i));
+  }
+}
+
+function encodeWAV (samples: Float32Array, format: number, sampleRate: number, numChannels: number, bitDepth: number) {
+
+  var bytesPerSample = bitDepth / 8
+  var blockAlign = numChannels * bytesPerSample
+
+  var buffer = new ArrayBuffer(44 + samples.length * bytesPerSample)
+  var view = new DataView(buffer)
+
+  /* RIFF identifier */
+  writeString(view, 0, 'RIFF')
+  /* RIFF chunk length */
+  view.setUint32(4, 36 + samples.length * bytesPerSample, true)
+  /* RIFF type */
+  writeString(view, 8, 'WAVE')
+  /* format chunk identifier */
+  writeString(view, 12, 'fmt ')
+  /* format chunk length */
+  view.setUint32(16, 16, true)
+  /* sample format (raw) */
+  view.setUint16(20, format, true)
+  /* channel count */
+  view.setUint16(22, numChannels, true)
+  /* sample rate */
+  view.setUint32(24, sampleRate, true)
+  /* byte rate (sample rate * block align) */
+  view.setUint32(28, sampleRate * blockAlign, true)
+  /* block align (channel count * bytes per sample) */
+  view.setUint16(32, blockAlign, true)
+  /* bits per sample */
+  view.setUint16(34, bitDepth, true)
+  /* data chunk identifier */
+  writeString(view, 36, 'data')
+  /* data chunk length */
+  view.setUint32(40, samples.length * bytesPerSample, true)
+  /* write data */
+  if (format === 1) {
+    floatTo16bPCM(view, 44, samples)
+  } else {
+    writeFloat32(view, 44, samples)
+  }
+
+  return buffer
+}
+
+function mergeBuffers(bufferArray: Float32Array[], recLength: number) {
+
+  // initialize array to hold all samples
+  var result = new Float32Array(recLength);
+
+  // for each array in bufferArray, append values to result
+  var offset = 0;
+
+  for (var i = 0; i < bufferArray.length; i++) {
+    result.set(bufferArray[i], offset);
+    offset += bufferArray[i].length;
+  }
+
+  return result;
+}
+
+
+function exportBuffer(recBuffer: Float32Array[], recLength: number, exportSampleRate: number) {
+  var mergedBuffers = mergeBuffers(recBuffer, recLength);
+  var encodedWav = encodeWAV(mergedBuffers, 1, exportSampleRate, 1, 16);
+  var audioBlob = new Blob([encodedWav], {type: 'audio/wav'});
+  return audioBlob;
+}
+
+function postAudioBlob(blob: Blob, uid: string) {
+  const reader = new FileReader();
+  reader.readAsDataURL(blob);
+  reader.onload = (e: ProgressEvent<FileReader>) => {
+      post("client-message", clientMessage("", reader.result as string, uid))
+  }
+}
+
+export default function useAudioInputController (typing: Ref) {
+
+    let audioContext: AudioContext;
+
+    let buffer: Float32Array[] = [];
+    let bufferLen = 0;
+    let sampleRate = 16000;
+    let numChannels = 1;
+
+    // For sending messages.
+    const { emitter } = useMitt();
+
+    const recording = ref(false);
+
+    // initiate media recorder
+    if (navigator.mediaDevices) {
+        console.log("Initializing media recorder.");
+
+        const usrOptions: any = {
+            audio: true,
+            video: false
+        }
+
+        navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => {
+
+            audioContext = new AudioContext();
+            const source = audioContext.createMediaStreamSource(stream);
+            const processor = audioContext.createScriptProcessor(4096, 1, 1);
+
+            sampleRate = audioContext.sampleRate;
+
+            source.connect(processor);
+            processor.connect(audioContext.destination);
+
+            processor.onaudioprocess = (e: AudioProcessingEvent) => {
+
+                if (recording.value) {
+
+                    const data: Float32Array = e.inputBuffer.getChannelData(0);
+
+                    buffer.push(data);
+                    bufferLen += data.length;
+
+                } else {
+                    buffer = [];
+                    bufferLen = 0;
+                }
+
+            };
+
+        });
+
+    }
+
+    const play = (dataUrl: string) => {
+      const snd = new Audio(dataUrl);
+      snd.addEventListener("canplaythrough", (event: any) => {
+        console.log("Playing");
+        snd.play();
+      });
+    }
+
+    // var play = function (blob: Blob) {
+    //   // We'll use a FileReader to create and ArrayBuffer out of the audio response.
+    //   var fileReader = new FileReader();
+    //   fileReader.onload = function() {
+    //     // Once we have an ArrayBuffer we can create our BufferSource and decode the result as an AudioBuffer.
+    //     const playbackSource = audioContext.createBufferSource();
+    //     audioContext.decodeAudioData(fileReader.result as Buffer, function(audioBuffer) {
+    //       console.log(audioBuffer.length);
+    //       console.log(audioBuffer.sampleRate);
+    //       console.log(audioBuffer.numberOfChannels);
+    //       console.log(audioBuffer.duration);
+
+    //       // Set the source buffer as our new AudioBuffer.
+    //       playbackSource.buffer = audioBuffer;
+    //       // Set the destination (the actual audio-rendering device--your device's speakers).
+    //       playbackSource.connect(audioContext.destination);
+    //       // Add an "on ended" callback.
+    //       playbackSource.onended = function(event) {
+    //         console.log("Playback ended");
+    //       };
+    //       // Start the playback.
+    //       playbackSource.start(0);
+    //     });
+    //   };
+    //   fileReader.readAsArrayBuffer(blob);
+    // };
+
+    //---Callbacks-----------------------------------------------
+
+    const onKeyDown = (e: KeyboardEvent) => {
+        const cmd = keyboardNameMap[e.keyCode];
+
+        // Start recording on space bar.
+        if (cmd == "SPACE" && !typing.value && !recording.value) {
+  
+                    console.log("INPT:Starting record, filling buffer.")
+                    recording.value = true;
+                    showRecIcon();
+
+        }
+
+    }
+
+    const onKeyUp = async (e: KeyboardEvent) => {
+        const cmd = keyboardNameMap[e.keyCode];
+
+        // Stop recording on space up.
+        if (cmd == "SPACE" && recording.value) {
+
+            // get audio from buffer then stop recording
+            console.log("INPT:Stopping record.")
+            hideRecIcon()
+
+            // render a place holder message
+            const message = renderMessage(
+                "", 
+                "", 
+                "", 
+                "sf"
+            );
+
+            emitter.emit("self-message", message);
+
+            const blob: Blob = exportBuffer(buffer, bufferLen, sampleRate);
+            postAudioBlob(blob, message.uid);
+            recording.value = false;
+
+            // const reader = new FileReader();
+            // reader.readAsDataURL(blob);
+            // reader.onload = (e: ProgressEvent<FileReader>) => {
+            //   play(reader.result as string);
+            // }
+
+        }
+
+    }
+
+    //-----------------------------------------------------------
+
+    onMounted(() => {
+        window.addEventListener("keydown", onKeyDown);
+        window.addEventListener("keyup", onKeyUp);
+    })
+
+    onUnmounted(() => {
+        window.removeEventListener("keydown", onKeyDown);
+        window.removeEventListener("keyup", onKeyUp);
+    });
+
+    return {
+        recording
+    }
+
+}

src/components/inputitem/inputItem.vue

@@ -11,6 +11,7 @@
     <!-- Recording animation on space bar -->
     <span v-if="recording" class="play"></span>
     <span v-if="recording" class="pause"></span>
+    <span id="audio"></span>
 
     <!-- Show text input on key-down -->
     <TextInput />
@@ -26,12 +27,10 @@
 import { defineComponent } from "vue";
 import draggify from "@/modules/draggify";
 
-import TextInput from "@/components/textInput.vue";
+import TextInput from "./textinput.vue";
 
-import useTextInputController from 
+import useTextInputController from "./textcontrol";
-  "@/components/controllers/textCtrl";
+import useAudioInputController from "./audiocontrol";
-import useAudioInputController from 
-  "@/components/controllers/audioCtrl";
 
 export default defineComponent({
   name: "InputItem",

src/components/inputitem/textcontrol.ts

@@ -2,7 +2,7 @@ import anime from "animejs";
 import useMitt from "@/modules/mitt";
 import { useIpc } from '@/modules/ipc';
 import { Ref, ref, watch, onMounted, onUnmounted } from "vue";
-import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
+import keyboardNameMap from "./keyBoardMaps/keyboardNameMap";
 import { clientMessage, renderMessage } from '@/modules/message';
 
 //---Animations-----------------------------------------------
@@ -96,7 +96,7 @@ export default function useTextInputController(elementX: Ref) {
             emitter.emit("self-message", message);
 
             // Send it to the backend for processing.
-            const clientM = clientMessage(textInput.value, false, message.uid);
+            const clientM = clientMessage(textInput.value, null, message.uid);
             post('client-message', clientM);
 
             clearInput()

src/components/message.vue

@@ -47,7 +47,7 @@
 
         <!-- Show question mark if transcription comes back as unknown. -->
         <div
-          v-else-if="message.content.text === false"
+          v-else-if="message.content.text == false"
           class="questionMark"
         >
         ?

src/components/messenger.vue

@@ -18,7 +18,7 @@
 <script lang="ts">
 import { defineComponent, onMounted, onUnmounted } from "vue";
 import Message from "@/components/message.vue";
-import InputItem from "@/components/inputItem.vue";
+import InputItem from "@/components/inputitem/inputitem.vue";
 import Settings from "@/components/settings.vue";
 import useMitt from "@/modules/mitt";
 import useMessages from "@/modules/messages";

src/modules/message.ts

@@ -3,7 +3,7 @@ import {
   ClientMessage, 
   ClientRequest, 
   AuthRequest, 
-  LogoutRequest 
+  LogoutRequest
 } from "@/types";
 
 import { v4 as uuidv4 } from 'uuid';
@@ -30,7 +30,7 @@ export const renderMessage = (text: boolean | string, audio: boolean | string, c
   }
 )
 
-export const clientMessage = (text: string, audio: string | boolean, uid: string): ClientMessage => (
+export const clientMessage = (text: string, audio: string | null, uid: string): ClientMessage => (
   {
     audio,
     text,

src/types.ts

@@ -12,9 +12,10 @@ export interface RenderMessage {
     newMessage: boolean;
 }
 
+
 export interface ClientMessage {
     text: string;
-    audio: boolean | string;
+    audio: string | null;
     uid: string;
 }