add audio playback

src/background/audio.ts

@@ -1,50 +1,104 @@
-"use strict";
 /* eslint @typescript-eslint/no-var-requires: "off" */
+
+"use strict";
+
 import { sendAudio } from './session'
 const portAudio = require('naudiodon');
 
-let ia: typeof portAudio.AudioIO;
-let audioInput: string[] = [];
-
-function processAudio(audioInput: Array<string>): Buffer {
-    let audio = '';
-    audioInput.forEach(chunk => audio += chunk);
-    return  Buffer.from(audio, "base64");
+let ai: typeof portAudio.AudioIO;
+let ao: typeof portAudio.AudioIO;
+let audioInput: Buffer[] = [];
+const encoding = "base64";
+const audioOptions = {
+    channelCount: 1,
+    sampleFormat: 16,
+    sampleRate: 16000,
+    deviceId: -1,
+    closeOnError: false,
 }
 
-export const updateRecorder = (_event, record: boolean): void => {
-  if (record) {
-      ia.resume();
-  } else {
-      ia.pause();
-      const audio = processAudio(audioInput);
-      sendAudio(audio);
-      audioInput = [];
+// run every time the main function launches.
+export const initAudioIO = () => {
+  // init portAudio readable stream once.
+  if (!ai) {
+    ai = new portAudio.AudioIO({ inOptions: audioOptions });
+
+    // base64 encoding needed for google speech to text.
+    ai.setEncoding(encoding);
+
+    // pause the portAudio data flow as soon as stream is initiated.
+    ai.start();
+    ai.pause();
+
+    ai.on('error', (e: Error) => {
+      console.log('Error recording audio', + e);
+    });
+    ai.on('data', (chunk: string) => {
+      // turn string base64 data into buffer object.
+      audioInput.push(Buffer.from(chunk, encoding));
+    });
+  }
+
+  // init portAudio writable stream once.
+  if (!ao) {
+      ao = new portAudio.AudioIO({ outOptions: audioOptions });
+      ao.start();
   }
 }
 
-export const initRecorder = () => {
-
-    if (!ia) {
-        ia = new portAudio.AudioIO({
-          inOptions: {
-            channelCount: 1,
-            sampleFormat: 16,
-            sampleRate: 16000,
-            deviceId: -1,
-            closeOnError: false,
-          }
-        });
-        ia.setEncoding('base64');
-        ia.start();
-        ia.pause();
-        ia.on('error', (e: Error) => {
-            console.log('Error recording audio', + e);
-        });
-        ia.on('data', (chunk: string) => {
-          audioInput.push(chunk);
-        });
+// run this function on space bar key up and down.
+export const updateRecorder = (_event, record: boolean): void => {
+    if (record) {
+        // resume mic data flow on space bar key down.
+        ai.resume();
+    } else {
+        // stop mic data flow on space bar key up.
+        ai.pause();
+        sendAudio(Buffer.concat(audioInput));
+        audioInput = [];
     }
 }
 
+// play audio buffers.
+export const play = (input: Array<Buffer>) => {
+    let i = 0;
+    // format buffers into half their size to account for writable highwaterMark.
+    const audio = bufSplit(input);
+    // call this fuction after last audio chunk has been written.
+    const callback = () => {
+        // TODO: clear portAudio writable buffer on write end.
+    }
+    write();
+    // iterate through audio array and write buffers to portAudio writable.
+    function write() {
+        let chunk: Buffer;
+        let ok = true;
+        do {
+            chunk = audio[i];
+            if (i === audio.length - 1) {
+                // write last chunk.
+                ao.write(chunk, null, callback);
+            } else {
+                // check for backpreassure.
+                ok = ao.write(chunk, null);
+            }
+            i++;
+        } while (i < audio.length && ok);
 
+        if (i < audio.length) {
+            // Had to stop early!
+            // Write some more once it drains.
+            ao.once('drain', write);
+        }
+    }
+}
+
+// utility function used by play func
+function bufSplit(input: Array<Buffer>): Array<Buffer> {
+    let result: Buffer[] = [];
+    input.forEach((b: Buffer) => {
+        // split buffer into two.
+        result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length));
+    });
+    return result;
+}

src/background/run.ts

@@ -2,7 +2,7 @@
 
 import { createWindow } from './window';
 import { initSession } from './session';
-import { initRecorder } from './audio';
+import { initAudioIO } from './audio';
 
 let socket = false;
 
@@ -23,6 +23,6 @@ export async function main() {
     socket = true;
 
     // Begin audio stream.
-    // initRecorder();
+    initAudioIO();
 
 }

src/background/session.ts

@@ -5,7 +5,10 @@ import { backgroundMitt } from './emitter';
 import { ipcMain } from "electron";
 
 interface RenderMessage {
-    content: string;
+    content: {
+        text: string;
+        audio: boolean | Buffer;
+    };
     avatar: string;
     context: string;
     subContext: string;
@@ -24,7 +27,7 @@ interface TextMessage {
     content: string;
 }
 
-const ip = 'ws://127.0.0.1';
+const ip = 'ws://127.0. 0.1';
 const port = 8760;
 const reconnectTimeout = 3000; //ms
 let socket: WebSocket;
@@ -67,6 +70,9 @@ const receiveMessage = (message: string): void => {
     }
 
     const parsed: RenderMessage = JSON.parse(message);
+    console.log('message received', parsed);
+    if (parsed.content.audio) {
+    }
     backgroundMitt.emit('ipc-renderer', {
         endpoint: 'render-message',
         message: parsed

src/components/messageView/index.vue

@@ -0,0 +1,111 @@
+<template>
+  <svg
+    version="1.1"
+    xmlns="http://www.w3.org/2000/svg"
+    xmlns:xlink="http://www.w3.org/1999/xlink"
+    :viewBox="scrollView"
+    @click="emittMessage"
+  >
+    <g id="messageView" transform="translate(0, 20)">
+      <g v-for="(item, index) in renderArr" :key="index" preserverAspectRatio="none">
+        <transition
+          appear
+          v-on:before-appear="beforeEnter"
+          v-on:appear="enter"
+          v-on:after-enter="afterEnter"
+        >
+          <TextBubble :item="item" :key="index" />
+        </transition>
+      </g>
+    </g>
+  </svg>
+</template>
+
+<script lang="ts">
+import TextBubble from "@/components/textBubble/index.vue";
+import { Message } from "@/types/message/index";
+import { useIpc } from "@/modules/ipc";
+import { Queue } from "@/modules/queue";
+import useTransitions from "./viewDetails/transitions";
+import useScrollView from "@/modules/scrollView";
+import useMitt from "@/modules/mitt";
+import {
+  defineComponent,
+  reactive,
+  onMounted,
+  onUnmounted,
+  watch,
+} from "vue";
+
+export default defineComponent({
+  name: "MessageView",
+  components: { TextBubble },
+
+  setup() {
+    const { emitter } = useMitt();
+    const renderArr: Message[] = reactive([]);
+    const renderQ = new Queue();
+
+    const { invoke } = useIpc("message");
+
+    // Each method will be called on their respective lifecycle event.
+    const { beforeEnter, enter, afterEnter, rendering } = useTransitions();
+    const { scrollView } = useScrollView({
+      targetId: "messageView",
+    });
+
+    const render = (m: Message) => {
+      if (rendering.value) {
+        renderQ.enqueue(m);
+      } else {
+        renderArr.push(m);
+      }
+    };
+
+    watch(rendering, (rendering, _prevRendering) => {
+      if (!rendering) {
+        const m = renderQ.dequeue() as Message | null;
+        if (m) renderArr.push(m);
+      }
+    });
+
+    onMounted(() => {
+      // Call render function on event "render-message."
+      window.ipcRenderer.on("render-message", (event, payload) => {
+        const m = payload.message;
+        if (m.content) render(m);
+      });
+
+      emitter.on("user-message", async (message) => {
+        try {
+          const res = await invoke(message);
+          render(res);
+        } catch (e) {
+          console.log("No response. Do not render.");
+        }
+      });
+    });
+
+    onUnmounted(() => {
+      window.ipcRenderer.removeAllListeners("render-message");
+      emitter.all.clear();
+    });
+
+    return {
+      renderArr,
+      beforeEnter,
+      enter,
+      afterEnter,
+      scrollView,
+    };
+  },
+});
+</script>
+
+<style lang="scss" scoped>
+#messageView {
+  width: 1000px;
+  height: 100vh;
+  color: red;
+}
+</style>

src/components/taInput/inputController.ts

@@ -34,7 +34,7 @@ export default function useInputController() {
         emitter.emit("animate-send");
 
         // send message to backend
-        post('message', message);
+        if (input) post('message', message);
     }
 
     const {

src/components/textBubble/bubbleDetails/drawer.ts

@@ -0,0 +1,50 @@
+import { ref, onMounted } from "vue";
+import { Ref } from "@/types/vueRef/index";
+
+/**
+ * handles visualization of individual svg text bubbles
+ */
+export default function useBubbleDrawer({
+    mr,
+    tr,
+}: {
+    mr: Ref<string>;
+    tr: Ref<string>;
+}) {
+    const signatureTranslate = ref("");
+    let messageRect: SVGSVGElement & SVGRectElement;
+    let messageText: SVGSVGElement & SVGTextElement;
+
+    onMounted(async () => {
+        messageRect = (mr.value as unknown) as SVGSVGElement & SVGRectElement;
+        messageText = (tr.value as unknown) as SVGSVGElement & SVGTextElement;
+    });
+
+    function adjustMessageRect(): void {
+        // must get dimensions of text box to fit bubble around
+        const padding = 12.5;
+        const textBox = messageText.getBBox();
+        messageRect.setAttribute("x", String(textBox.x - padding));
+        messageRect.setAttribute("y", String(textBox.y - padding));
+        messageRect.setAttribute("width", String(textBox.width + 2 * padding));
+        messageRect.setAttribute("height", String(textBox.height + 2 * padding));
+    }
+
+    function translateMsgSignature(): void {
+        if (messageRect.getAttribute("height") !== null) {
+            const height = Number(messageRect.getAttribute("height"));
+            signatureTranslate.value = `translate(-10 ${height})`;
+        }
+    }
+
+    async function drawMessage() {
+        // ignore lint error: await needed for proper render
+        await adjustMessageRect();
+        translateMsgSignature();
+    }
+
+    return {
+        drawMessage,
+        signatureTranslate,
+    };
+}