Merge remote-tracking branch 'origin/main' into main

Merge audio code into new UI code

src/App.vue

@@ -1,5 +1,5 @@
 <template>
-  <div id="app" v-if="!loading">
+  <div id="app" v-if="ready">
 
     <button class="titlebar">
 
@@ -19,7 +19,7 @@
 </template>
 
 <script lang="ts">
-import {defineComponent} from 'vue';
+import { defineComponent, onMounted, onUnmounted, ref } from 'vue';
 import { useIpc } from "@/modules/ipc";
 import { useAuth } from "@/modules/auth";
 import Splash from "@/components/splash.vue"
@@ -28,9 +28,18 @@ export default defineComponent({
   components: { Splash },
 
   setup() {
-
     const { invoke } = useIpc();
-    const { loading } = useAuth();
+    const ready = ref(false);
+
+    onMounted(() => {
+        window.ipcRenderer.on('window-ready', (_event, payload: {message: boolean}) => {
+            ready.value = payload.message;
+        })
+    });
+
+    onUnmounted(() => {
+        window.ipcRenderer.removeAllListeners('window-ready')
+    })
 
     const callNavbar = (action: string) => {
       invoke('nav-bar', action);
@@ -38,7 +47,7 @@ export default defineComponent({
 
     return {
         callNavbar,
-        loading
+        ready
     }
   }
 })

src/background/audio.ts

@@ -3,11 +3,15 @@
 "use strict";
 
 import { sendAudio } from './session'
+import { ipcMain } from "electron";
 const portAudio = require('naudiodon');
 
-let ai: typeof portAudio.AudioIO;
-let ao: typeof portAudio.AudioIO;
-let audioInput: Buffer[] = [];
+let ai: typeof portAudio.AudioIO | null = null;
+let ao: typeof portAudio.AudioIO | null = null;
+let record = false;
+const audioContainer = {
+    input: '',
+}
 const encoding = "base64";
 const audioOptions = {
     channelCount: 1,
@@ -17,53 +21,55 @@ const audioOptions = {
     closeOnError: false,
 }
 
-// run every time the main function launches.
-export const initAudioIO = () => {
-  // init portAudio readable stream once.
-  if (!ai) {
-    ai = new portAudio.AudioIO({ inOptions: audioOptions });
+// callback run on space bar key up and down.
+const updateRecorder = (_event, payload: boolean): void => {
+    record = payload;
+    if (!record) {
+        sendAudio(Buffer.from(audioContainer.input, 'base64'));
+    }
+};
 
-    // base64 encoding needed for google speech to text.
-    ai.setEncoding(encoding);
+// listen for space key up/down event.
+ipcMain.removeAllListeners('update-recorder');
+ipcMain.on('update-recorder', updateRecorder);
 
-    // pause the portAudio data flow as soon as stream is initiated.
-    ai.start();
-    ai.pause();
-
-    ai.on('error', (e: Error) => {
-      console.log('Error recording audio', + e);
+// utility function used by play func.
+function bufSplit(input: Array<Buffer>): Array<Buffer> {
+    const result: Buffer[] = [];
+    input.forEach((b: Buffer) => {
+        // split buffer into two.
+        result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length));
     });
-    ai.on('data', (chunk: string) => {
-      // turn string base64 data into buffer object.
-      console.log('received string chunk of length', chunk.length)
-      const buf = Buffer.from(chunk, encoding);
-      audioInput.push(buf);
-    });
-  }
-
-  // init portAudio writable stream once.
-  if (!ao) {
-      ao = new portAudio.AudioIO({ outOptions: audioOptions });
-      ao.start();
-  }
+    return result;
 }
 
-// run this function on space bar key up and down.
-export const updateRecorder = (_event, record: boolean): void => {
-    if (record) {
-        // resume mic data flow on space bar key down.
-        ai.resume();
-    } else {
-        // stop mic data flow on space bar key up.
-        ai.pause();
-        const audio = Buffer.concat(audioInput);
-        sendAudio(audio);
-        audioInput = [];
+// main audio function run by run.ts module.
+export function initAudioIO(): void {
+
+    if (!ai) {
+        ai = new portAudio.AudioIO({ inOptions: audioOptions });
+
+        // base64 encoding needed for google speech to text.
+        ai.setEncoding(encoding);
+        ai.start();
+
+        ai.on('data', (chunk: string) => {
+            if (record) {
+                audioContainer.input += chunk;
+            } else {
+                if (audioContainer.input.length) audioContainer.input = "";
+            }
+        });
+    }
+
+    if (!ao) {
+        ao = new portAudio.AudioIO({ outOptions: audioOptions });
+        ao.start();
     }
 }
 
 // play audio buffers.
-export const play = (input: Array<Buffer>) => {
+export function play(input: Array<Buffer>): void {
     let i = 0;
     // format buffers into half their size to account for writable highwaterMark.
     const audio = bufSplit(input);
@@ -76,6 +82,7 @@ export const play = (input: Array<Buffer>) => {
     function write() {
         let chunk: Buffer;
         let ok = true;
+
         do {
             chunk = audio[i];
             if (i === audio.length - 1) {
@@ -96,12 +103,14 @@ export const play = (input: Array<Buffer>) => {
     }
 }
 
-// utility function used by play func
-function bufSplit(input: Array<Buffer>): Array<Buffer> {
-    let result: Buffer[] = [];
-    input.forEach((b: Buffer) => {
-        // split buffer into two.
-        result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length));
-    });
-    return result;
+export function stopStream() {
+    if(ai != null) {
+        ai.quit();
+        ai = null;
+    }
+    if (ao != null) {
+        ao.quit();
+        ao = null;
+    }
 }
+

src/background/init.ts

@@ -1,8 +1,10 @@
+
 "use strict";
 
 import { app } from "electron";
 import { main } from './run';
 import { backgroundMitt } from './emitter';
+import { stopStream } from './audio';
 
 let winActive: boolean;
 
@@ -19,12 +21,11 @@ export function initApp(dev: boolean): void {
     // Quit when all windows are closed.
     app.on("window-all-closed", () => {
       if (process.platform !== "darwin") {
+        stopStream();
         app.quit();
       }
     });
 
-    // TODO: Debug activate functionaity: currently not working.
-    // Might have to de-reference window object
     app.on("activate", () => {
       if (winActive === false) {
         main();

src/background/window.ts

@@ -3,7 +3,6 @@
 import { BrowserWindow, ipcMain } from "electron";
 import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
 import { backgroundMitt } from './emitter';
-import { updateRecorder } from './audio';
 import * as path from "path";
 
 interface WindowSettings {
@@ -28,31 +27,48 @@ interface IpcRendererPayload {
 
 let win: BrowserWindow | null;
 
+// Util function to handle window close & minimize.
+const navBarHandler = (_event, action: string): void => {
+    switch(action) {
+        case 'close':
+            if (win) win.close();
+        break;
+        case 'min':
+            if (win) win.minimize();
+        break;
+    }
+}
+
+// Util function to render message on ipc-renderer event.
+const renderMessage = (payload: IpcRendererPayload): void => {
+    if (win) {
+        win.webContents.send(payload.endpoint, {
+            message: payload.message
+        });
+    }
+}
+
+// Do this on window mount.
 const windowMount = (): void => {
-  backgroundMitt.emit('window-active', true);
-  ipcMain.removeAllListeners('update-recorder');
-  ipcMain.on('update-recorder', updateRecorder);
-  // handle renderer auth-token event
-  ipcMain.removeHandler('nav-bar'); // avoid setting duplicate handlers
-  ipcMain.handle('nav-bar', navBarHandler);
+    backgroundMitt.emit('window-active', true);
+    // handle win nav-bar event.
+    ipcMain.removeHandler('nav-bar'); // avoid setting duplicate handlers
+    ipcMain.handle('nav-bar', navBarHandler);
+    // render messages through ipc-renderer.
+    backgroundMitt.on('ipc-renderer', renderMessage);
 }
 
+// do this on window dismount.
 const windowDismount = (): void => {
-  win = null;
-  backgroundMitt.emit('window-active', false);
-  ipcMain.removeAllListeners('update-recorder');
+    win = null;
+    backgroundMitt.emit('window-active', false);
 }
 
-backgroundMitt.on('ipc-renderer', (payload: IpcRendererPayload) => {
-  if (win)
-    win.webContents.send(payload.endpoint, {
-      message: payload.message
-    });
-});
-
-export const createWindow = async (options: WindowSettings): Promise<void> => {
+// function used by run.ts to create the main window.
+export async function createWindow(options: WindowSettings): Promise<void> {
   return new Promise((resolve, _reject) => {
 
+    // avoid creating duplicate windows.
     if (win) resolve();
 
     win = new BrowserWindow({
@@ -72,6 +88,7 @@ export const createWindow = async (options: WindowSettings): Promise<void> => {
       }
     });
 
+
     if (process.env.WEBPACK_DEV_SERVER_URL) {
       // Load the url of the dev server if in development mode
       win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string);
@@ -89,21 +106,12 @@ export const createWindow = async (options: WindowSettings): Promise<void> => {
     })
 
     win.webContents.on('did-finish-load', () => {
-      windowMount();      
+        if (win) win.webContents.send('window-ready', {
+            message: true
+        });
+      windowMount();
       resolve();
     });
 
   });
-};
-
-// call this function on nav bar click
-function navBarHandler(_event, action: string) {
-    switch(action) {
-        case 'close':
-            if (win) win.close();
-            break;
-        case 'min':
-            if (win) win.minimize();
-            break;
-    }
 }

src/main.ts

@@ -7,6 +7,7 @@ import mitt from "mitt";
 import anime from "animejs";
 import { createApp } from "vue";
 
+
 // Handle events.
 const emitter = mitt();
 

src/modules/auth.ts

@@ -11,8 +11,6 @@ const state = reactive<AuthState>({
     error: undefined,
 });
 
-const loading = ref(true);
-
 const AUTH_KEY = 'crimata_token';
 
 const token = window.localStorage.getItem(AUTH_KEY);
@@ -22,7 +20,6 @@ if (token) {
 }
 
 const authToken = async () => {
-   loading.value = true;
    const { invoke  } = useIpc();
     try {
         const res = await invoke('auth-session', token);
@@ -35,7 +32,6 @@ const authToken = async () => {
         window.localStorage.removeItem(AUTH_KEY);
         state.accessToken = null;
     }
-    loading.value = false;
 }
 
 // authenticate on auth event
@@ -60,6 +56,5 @@ export const useAuth = () => {
         setToken,
         logout,
         ...toRefs(state), // accessToken, error
-        loading
     }
 }

src/views/messenger.vue

@@ -4,7 +4,7 @@
     <!-- <InputItem /> -->
 
     <!-- List of message bubbles. -->
-    <div id="messenger" >
+    <div id="messenger">
       <MessageItem v-for="message in messages" :message="message" :key="message.id"/>
     </div>
  

tests/audio.js

@@ -10,6 +10,11 @@ const encoding = 'LINEAR16';
 const sampleRateHertz = 16000;
 const languageCode = 'en-US';
 
+const audioContainer = {
+    input: '',
+    buffers: []
+}
+
 const config = {
   encoding: encoding,
   sampleRateHertz: sampleRateHertz,
@@ -20,8 +25,6 @@ const config = {
  * Note that transcription is limited to 60 seconds audio.
  * Use a GCS file for audio longer than 1 minute.
  */
-let audio;
-
 async function transcribeSpeech (audio) {
   const request = {
     config: config,
@@ -41,7 +44,8 @@ async function transcribeSpeech (audio) {
   console.log(`Transcription: ${transcription}`);
 }
 
-let audioInput = [];
+let record = true;
+
 // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream
 const ia = new portAudio.AudioIO({
   inOptions: {
@@ -54,42 +58,116 @@ const ia = new portAudio.AudioIO({
 });
 ia.setEncoding('base64');
 ia.start();
-ia.on('error', (e) => {
-    console.log('error recording audio', + e);
-});
 ia.on('data', (chunk) => {
-
-    const buf = Buffer.from(chunk, 'base64');
-    audioInput.push(buf);
+    if (record) {
+        console.log('recording data')
+        audioContainer.input += chunk;
+        // audioContainer.buffers.push(Buffer.from(chunk, 'base64'));
+    } else {
+        if (audioContainer.input.length) audioContainer.input = "";
+    }
 });
 
-async function processAudio() {
+const ao = new portAudio.AudioIO({
+    outOptions: {
+        sampleFormat: 16,
+        channelCount: 1,
+        sampleRate: 16000,
+        deviceId: -1,
+        closeOnError: false,
+    }
+});
+ao.start();
 
-    const buf = Buffer.concat(audioInput);
+let counter = 0;
+const tests = [];
 
-    audioInput = [];
+function testCallback() {
+    tests.forEach(t => console.log(t))
+    console.log(tests[0])
+    play(tests[0])
+}
 
-   transcribeSpeech({
-       content: buf
-   });
+function play(bufArray) {
+    let i = 0;
+    // format buffers into half their size to account for writable highwaterMark.
+    const audio = bufSplit(bufArray);
+    // call this fuction after last audio chunk has been written.
+    const callback = () => {
+        // TODO: clear portAudio writable buffer on write end.
+        console.log('Finished writing test number %d', counter)
+        if (counter === 3) {
+            console.log('finished test writing!')
+        }
+    }
+    write();
+    // iterate through audio array and write buffers to portAudio writable.
+    function write() {
+        let ok = true;
+        do {
+            if (i === audio.length - 1) {
+                // write last chunk.
+                ao.write(audio[i], null, callback);
+            } else {
+                // check for backpreassure.
+                ok = ao.write(audio[i], null);
+            }
+            i++;
+        } while (i < audio.length && ok);
+
+        if (i < audio.length) {
+            // Had to stop early!
+            // Write some more once it drains.
+            ao.once('drain', write);
+        }
+    }
+}
+
+// utility function used by play func
+function bufSplit(input){
+    const result = [];
+    input.forEach((b) => {
+        // split buffer into two.
+        result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length));
+    });
+    return result;
+}
+
+async function test() {
+    transcribeSpeech({
+        content: Buffer.from(audioContainer.input, 'base64')
+    });
+    tests.push(audioContainer.buffers)
+    counter++;
+    console.log('audio string length:', audioContainer.input.length)
+    console.log('buffers written: ', audioContainer.buffers.length)
 }
 
 setTimeout(async () => {
-    ia.pause();
-    processAudio();
-    // ia.resume();
+    record = false;
+    test()
 }, 3000);
 
 setTimeout(() => {
-    ia.resume();
-}, 5000)
+    record = true;
+}, 6000)
 
 setTimeout(async () => {
-    ia.pause();
-    processAudio();
-    // ia.resume();
-}, 8000);
+    record = false;
+    test();
+}, 9000);
 
 setTimeout(() => {
-    ia.pause();
-}, 9000)
+    record = true;
+}, 11000)
+
+setTimeout(async () => {
+    record = false;
+    test();
+}, 14000);
+
+setTimeout(async () => {
+    ia.quit();
+    // testCallback()
+    return;
+}, 16000);