add audio worklet

Working protocols with platform

See merge request crimata/electron-app!6

package.json

@@ -66,25 +66,9 @@
     "spectron": "11.0.0",
     "typescript": "~3.9.3",
     "vue-cli-plugin-electron-builder": "~2.0.0-rc.6",
-    "vue-jest": "^5.0.0-0"
-  },
-  "vue": {
-    "lintOnSave": false,
-    "pluginOptions": {
-      "electronBuilder": {
-        "mainProcessFile": "./src/init.ts",
-        "rendererProcessFile": "./src/render/main.ts",
-        "preload": "./src/render/preload.ts",
-        "builderOptions": {
-          "appId": "com.crimata.ElectronUpdaterApp",
-          "artifactName": "${productName}-${version}.${ext}",
-          "publish": {
-            "provider": "generic",
-            "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build"
-          }
-        }
-      }
-    }
+    "vue-jest": "^5.0.0-0",
+    "worker-loader": "^3.0.8",
+    "worklet-loader": "^1.0.0"
   },
   "gitHooks": {
     "pre-commit": "lint-staged"

public/processor.js

@@ -0,0 +1,12 @@
+
+class AudioProcessor extends AudioWorkletProcessor {
+  process (inputs, outputs, parameters) {
+    console.log(inputs);
+    console.log(outputs);
+    console.log(parameters);
+    return true
+  }
+}
+
+registerProcessor('processor', AudioProcessor)
+

public/worklet/audioProcessor.ts

@@ -0,0 +1,33 @@
+interface AudioWorkletProcessor {
+  readonly port: MessagePort;
+  process(
+    inputs: Float32Array[][],
+    outputs: Float32Array[][],
+    parameters: Record<string, Float32Array>
+  ): boolean;
+}
+
+declare let AudioWorkletProcessor: {
+  prototype: AudioWorkletProcessor;
+  new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor;
+};
+
+declare function registerProcessor(
+  name: string,
+  processorCtor: (new (
+    options?: AudioWorkletNodeOptions
+  ) => AudioWorkletProcessor) & {
+    parameterDescriptors?: AudioParamDescriptor[];
+  }
+): undefined;
+
+class AudioProcessor extends AudioWorkletProcessor {
+  process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>) {
+    console.log(inputs);
+    console.log(outputs);
+    console.log(parameters);
+    return true
+  }
+}
+
+registerProcessor('audio-processor', AudioProcessor)

src/account.ts

@@ -1,7 +1,7 @@
 
 import { postAuth, postLogin, postLogout } from "@/api/account";
 import { endSession, launchSession } from "@/session";
-import { getToken,  setToken, setProfile, getProfile, clearStore } from "./store";
+import { getToken,  setToken, setProfile, clearStore } from "./store";
 import { parseAuthRes } from "./auth";
 import { ipcEmit } from "@/composables/useEmitter";
 
@@ -83,13 +83,4 @@ export const accountLogout = async (): Promise<Error | void> => {
 
 }
 
-export const updateAppState = (): void => {
-
-    const profile = getProfile();
-
-    ipcEmit("set-profile", profile);
-
-    // ipcEmit('messages') etc
-
-}
 

src/composables/useIpcMain.ts

@@ -71,12 +71,10 @@ export class IpcListener<InputParam> implements IIpcListener<InputParam> {
       ipcMain.removeAllListeners(this.channel);
   }
 
-  private _onPost = (_e: IpcMainEvent, payload?: string | null): void => {
+  private _onPost = (_e: IpcMainEvent, payload: InputParam): void => {
 
       console.log(`[IPC] Post: ${this.channel}`);
-
-      const params = payload ? JSON.parse(payload) : null;
-      this._listenerCallback(params);
+      this._listenerCallback(payload);
   }
 
 }

src/composables/useWebsockets.ts

@@ -11,7 +11,13 @@ let _connectionCheckInterval: ReturnType<typeof setTimeout>;
 
 export default function useWebSockets(
     messageCallback: (message: string) => void,
-    connectionStatusCallback: (alive: boolean) => void,
+    connectionStatusCallback: (status: string) => void,
+    statusOptions?: {
+       openMessage: string,
+       pongMessage: string,
+       closeMessage: string,
+       pingErrorMessage: string,
+    },
 ) {
 
     let socket: WebSocket;
@@ -41,13 +47,15 @@ export default function useWebSockets(
 
             socket.send(JSON.stringify({key: secret}));
 
+            connectionStatusCallback(statusOptions ? statusOptions.openMessage : "Connection Opened");
+
             // ping server
             _connectionCheckInterval = setInterval(() => {
 
                 socket.ping(null, true, (e: Error) => {
                     if (e) {
                         socket.close();
-                        connectionStatusCallback(false);
+                        connectionStatusCallback(statusOptions ? statusOptions.pingErrorMessage : "Connection Lost");
                         setTimeout(() => connect(socketUrl, secret), _reconnectTimeout);
                     }
                 });
@@ -62,15 +70,20 @@ export default function useWebSockets(
         });
 
         socket.on("close", (event: WebSocket.CloseEvent) => {
-            connectionStatusCallback(false);
+            connectionStatusCallback(statusOptions ? statusOptions.closeMessage : "Connection Closed");
             clearInterval(_connectionCheckInterval);
             if (!event.wasClean) {
-                setTimeout(() => connect(socketUrl, secret), _reconnectTimeout);
+                setTimeout(() => {
+                    connect(socketUrl, secret), _reconnectTimeout
+                });
+
             }
 
         });
 
-        socket.on("pong", () => connectionStatusCallback(true));
+        socket.on("pong", () => connectionStatusCallback(statusOptions ? statusOptions.pongMessage : "Pong"));
+
+        socket.on('error', () => {});
 
     }
 
@@ -78,7 +91,7 @@ export default function useWebSockets(
         if (socket) {
             socket.close();
         }
-    }
+    };
 
     return {
         connect,

src/ipc/handlers.ts

@@ -1,7 +1,7 @@
 
 "use strict";
 
-import { accountLogin, accountLogout, accountProfile } from "@/account";
+import { accountLogin, accountLogout } from "@/account";
 import { IpcHandler } from "@/composables/useIpcMain";
 import { flush } from "@/audio";
 

src/ipc/listeners.ts

@@ -2,11 +2,14 @@
 import { IpcListener } from "@/composables/useIpcMain"
 import { sendMessage } from '@/session';
 import { collect } from "@/audio";
-import { updateAppState } from "@/account";
+import { updateAppState } from "@/session";
+import { onNavBar } from "@/window";
 
 const CLIENT_MESSAGE_CHANNEL = "post-session-send"
 const GET_AUDIO_CHANNEL = "post-audio-collect";
 const APP_MOUNT_CHANNEL = "post-app-mount";
+const NAV_BAR_CHANNEL = "post-nav-bar";
+const WINDOW_BLUR_CHANNEL = "post-window-focus";
 
 export const messageListener = new IpcListener<Message>({
     channel: CLIENT_MESSAGE_CHANNEL,
@@ -22,3 +25,22 @@ export const appMountListener = new IpcListener<null>({
     channel: APP_MOUNT_CHANNEL,
     listenerCallback: updateAppState
 });
+
+export const navBarListener = new IpcListener({
+    channel: NAV_BAR_CHANNEL,
+    listenerCallback: onNavBar
+});
+
+
+const onWindowFocus: IpcListenerCallback<{ isFocused: boolean }> = (payload) => {
+    if (payload.isFocused) {
+        console.log('window is focused')
+    } else {
+        console.log('window is blurred')
+    }
+}
+
+export const windowFocusListener = new IpcListener({
+    channel: WINDOW_BLUR_CHANNEL,
+    listenerCallback: onWindowFocus
+});

src/main.ts

@@ -10,8 +10,8 @@
  */
 
 import initIpcMain from "@/ipc/index";
-import  { accountAuth, updateAppState } from "./account";
-import { launchSession } from "./session";
+import  { accountAuth } from "./account";
+import { launchSession, updateAppState } from "./session";
 import createWindow from "./window";
 
 let authState: AuthState | null;

src/messages.ts

@@ -0,0 +1,67 @@
+import { ipcEmit } from "@/composables/useEmitter";
+
+export default class Messages {
+
+    messages: Message[];
+
+    constructor(messages: Message[]) {
+        console.log(messages);
+        this.messages = messages;
+        this.emit();
+    }
+
+    emit() {
+        ipcEmit("init-messages", this.messages);
+    }
+
+    update(update: Update) {
+
+        if (update.name === "add") {
+            this.messages.push(update.data as Message);
+            ipcEmit("add-message", update.data);
+        } 
+
+        else if (update.name === "annotate") {
+            this._annotate(update.data as Annotation);
+            ipcEmit("update-message", update.data);
+        }
+
+        else {
+            this._delete(update.data as string);
+            ipcEmit("delete-message", update.data);
+        }
+
+    }
+
+    _annotate(annotation: Annotation) {
+
+        for (let i in this.messages) {
+
+            if (this.messages[i].uid == annotation.uid) {
+
+                if (annotation.name == "content") {
+                    this.messages[i].content = annotation.data as string[];
+                }
+
+                if (annotation.name == "context") {
+                    this.messages[i].context = annotation.data as string;
+                }
+
+                ipcEmit("update-message", annotation);
+
+            }
+
+        }
+
+    }
+
+    _delete(uid: string) {
+        for (var i = 0; i < this.messages.length; i++) {
+            if (this.messages[i].uid === uid) {
+                this.messages.splice(i, 1);
+                break;
+            }
+        }
+    }
+    
+}

src/render/App.vue

@@ -21,7 +21,7 @@
 
 <script lang="ts">
 
-import { defineComponent, onMounted } from "vue";
+import { defineComponent, onMounted, onUnmounted } from "vue";
 import { IpcRendererEvent } from "electron";
 
 import Splash from "@/render/components/splash.vue";
@@ -29,7 +29,7 @@ import Messenger from "@/render/components/messenger.vue";
 import Login from "@/render/components/login.vue";
 import Header from "@/render/components/header.vue";
 
-import { profile, authComplete } from "@/render/composables/useProfile";
+import { profile, authComplete } from "@/render/shared/profile";
 import { postAppMount } from "@/render/ipc";
 
 export default defineComponent({
@@ -43,7 +43,26 @@ export default defineComponent({
 
   setup() {
 
-    onMounted(() => postAppMount());
+    const onFocus = () => {
+        console.log('window is focused');
+    }
+
+    const onBlur = () => {
+        console.log('window is blurred');
+    }
+
+    onMounted(() => {
+        // subscribe to visibility change events
+        window.addEventListener('blur', onBlur);
+        window.addEventListener('focus', onFocus);
+
+        postAppMount()
+    });
+
+    onUnmounted(() => {
+        window.removeEventListener("blur", onBlur);
+        window.removeEventListener("focus", onFocus);
+    });
 
     return {
         authComplete,

src/render/audio/audio.worklet.ts

@@ -0,0 +1,33 @@
+interface AudioWorkletProcessor {
+  readonly port: MessagePort;
+  process(
+    inputs: Float32Array[][],
+    outputs: Float32Array[][],
+    parameters: Record<string, Float32Array>
+  ): boolean;
+}
+
+declare let AudioWorkletProcessor: {
+  prototype: AudioWorkletProcessor;
+  new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor;
+};
+
+declare function registerProcessor(
+  name: string,
+  processorCtor: (new (
+    options?: AudioWorkletNodeOptions
+  ) => AudioWorkletProcessor) & {
+    parameterDescriptors?: AudioParamDescriptor[];
+  }
+): undefined;
+
+class AudioProcessor extends AudioWorkletProcessor {
+  process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>) {
+    console.log(inputs);
+    console.log(outputs);
+    console.log(parameters);
+    return true
+  }
+}
+
+registerProcessor('audio-processor', AudioProcessor)

src/render/audio/audioNode.ts

@@ -0,0 +1,9 @@
+
+// MyWorkletNode.js
+export default class MyWorkletNode extends AudioWorkletNode {
+   constructor(context) {
+    super(context, '-processor')
+    console.log(this.channelCount)
+  }
+}
+

src/render/audio/setupAudio.ts

@@ -0,0 +1,74 @@
+import AudioProcessor from "./audio.worklet.ts";
+console.log('YAYAYAYYA', AudioProcessor)
+//TODO: need to set export directory for audio worklet
+// webpack!!!
+
+ const constraints = {
+      audio: {
+        echoCancellation: true,
+        autoGainControl: true,
+        noiseSuppression: true,
+        channelCount: 1
+      },
+      video: false
+    }
+async function getWebAudioMediaStream() {
+  if (!window.navigator.mediaDevices) {
+    throw new Error(
+      "This browser does not support web audio or it is not enabled."
+    );
+  }
+
+  try {
+    const result = await window.navigator.mediaDevices.getUserMedia(constraints);
+
+    return result;
+  } catch (e) {
+    switch (e.name) {
+      case "NotAllowedError":
+        throw new Error(
+          "A recording device was found but has been disallowed for this application. Enable the device in the browser settings."
+        );
+
+      case "NotFoundError":
+        throw new Error(
+          "No recording device was found. Please attach a microphone and click Retry."
+        );
+
+      default:
+        throw e;
+    }
+  }
+}
+
+export async function setupAudio() {
+  // Get the browser audio. Awaits user "allowing" it for the current tab.
+  const mediaStream = await getWebAudioMediaStream();
+
+  const context = new window.AudioContext();
+  const audioSource = context.createMediaStreamSource(mediaStream);
+
+  let node;
+
+    // Add our audio processor worklet to the context.
+
+    try {
+        await context.audioWorklet.addModule("audio.worklet.js");
+    } catch (e) {
+      throw new Error(
+        `Failed to load audio analyzer worklet. Further info: ${e.message}`
+      );
+    }
+
+    node = new AudioWorkletNode(context, 'audio-processor')
+
+    // Connect the audio source (microphone output) to our analysis node.
+    audioSource.connect(node);
+
+    // Connect our analysis node to the output. Required even though we do not
+    // output any audio. Allows further downstream audio processing or output to
+    // occur.
+    node.connect(context.destination);
+
+  return { context, node };
+}

src/render/components/bubble.vue

@@ -1,53 +1,17 @@
 <template>
-  <div :class="`${type}-message`">
-
-    <!-- message bubble -->
-    <div :class="`${type}-${child}-bubble`">
-      <div class="notification-dot"/>
-      <div class="error-dot"/>
-      {{ text }}
-    </div>
-
-    <!-- message context -->
-    <span class="context">
-      <div :class="`${type}-icon`"/>
-      {{ context }}
-    </span>
-
+  <div :class="`bubble ${modifier}-bubble ${modifier}-${child}`">
+    {{ content }}
   </div>
 </template>
 
 <script lang='ts'>
 
- import { defineComponent, ref, onMounted } from 'vue';
+ import { defineComponent } from 'vue';
 
  export default defineComponent({
   name: "Bubble",
 
-  props: ["text", "context", "type", "position"],
-
-  setup() {
-
-    const seen = ref(false);
-    /* initialize seen state */
-    if (document.visibilityState === "visible") {
-        seen.value = true;
-    } else seen.value = false;
-
-    onMounted(() => {
-
-      if(seen.value)
-      document.addEventListener("visibilitychange", () => {
-          seen.value = true
-      });
-
-    });
-
-    return {
-      seen
-    };
-
-  }
+  props: ["modifier", "content", "child"],
 
  })
 
@@ -55,277 +19,55 @@
 
 <style lang="scss" scoped>
 
-  .message {
-    width: 100vw;
-    display: flex;
-    flex-direction: column;
-    padding-top: 9px;
-    padding-bottom: 9px;
-  }
-
-  .message:first-child {
-    margin-top: 55px;
-  }
-
-  .message:last-child {
-    margin-bottom: 18px;
-  }
-
-  .sf-message {
-    @extend .message;
-    justify-content: flex-end;
-  }
-
-  .ai-message, .fr-message {
-    @extend .message;
-    justify-content: flex-start;
-  }
-
-  .bubble {
-    position: relative;
-    max-width: 66vw;
-    font-family: "SF Pro Text";
-    font-size: 14px;
-    padding: 10px;
-    border-radius: 18px;
-  }
-
-  .sf-bubble {
-    @extend .bubble;
-    background-color: #58c4fd;
-    color: white;
-  }
-
-
-
-
-
-
-
-
-  .firstChildMessage {
-    padding-bottom: 2px;
-  }
-
-  .middleChildMessage {
-    padding-top: 2px;
-    padding-bottom: 2px;
-  }
-
-  .lastChildMessage {
-    padding-top: 2px;
-  }
-
-  #aiMessage {
-    align-items: flex-start;
-  }
-
-  #sfMessage {
-    align-items: flex-end;
-  }
-
-  #frMessage {
-    align-items: flex-start;
-  }
-
-  .messageBox {
-    position: relative;
-
-    display: flex;
-    flex-direction: column;
-
-    // Animate on render.
-    animation-name: appear;
-    animation-duration: 0.25s;
-  }
-
-  #aiMessageBox {
-    margin-left: 20px;
-    align-items: flex-start;
-  }
-
-  #sfMessageBox {
-    margin-right: 20px;
-    align-items: flex-end;
-  }
-
-  #frMessageBox {
-    margin-left: 20px;
-    align-items: flex-start;
-  }
-
-  .bubble {
-    position: relative;
-
-    max-width: 66vw;
-
-    display: flex;
-    flex-direction: column;
-
-    font-family: "SF Pro Text";
-    font-size: 14px;
-
-    padding: 10px;
-
-    border-radius: 18px;
-  }
-
-  @keyframes appear {
-    10% {
-      transform: scale(0.3);
-    }
-    100% {
-      transform: scale(1);
-    }
-  }
-
-  #aiBubble {
-    background-color: white;
-  }
-
-  #sfBubble {
-    background-color: #58c4fd;
-    color: white;
-  }
-
-  #frBubble {
-    background-color: white;
-  }
-
-  .sf-firstChildBubble {
-    border-bottom-right-radius: 9px;
-  }
-
-  .sf-middleChildBubble {
-    border-top-right-radius: 9px;
-    border-bottom-right-radius: 9px;
-  }
-
-  .sf-lastChildBubble {
-    border-top-right-radius: 9px;
-  }
-
-  .ai-firstChildBubble, .fr-firstChildBubble {
-    border-bottom-left-radius: 9px;
-  }
-
-  .ai-middleChildBubble, .fr-middleChildBubble {
-    border-top-left-radius: 9px;
-    border-bottom-left-radius: 9px;
-  }
-
-  .ai-lastChildBubble, .fr-lastChildBubble {
-    border-top-left-radius: 9px;
-  }
-
-  .notify {
-    position: absolute;
-    width: 12px;
-    height: 12px;
-    border-radius: 50%;
-    background-color: #58D9FF;
-    top: -5px;
-    left: -5px;
-    border: 2px solid #EBEBEB;
-    transform: scale(0);
-
-    animation-name: notify-anim;
-    animation-duration: 5s;
-  }
-
-  @keyframes notify-anim {
-    0%, 90% {
-      transform: scale(1);
-    }
-    100% {
-      transform: scale(0);
-    }
-  }
-
-  .context {
-    position: relative;
-
-    display: flex;
-    align-items: center;
-
-    font-family: "SF Compact Display";
-    font-size: 12px;
-    font-weight: bold;
-    margin-top: 5px;
-  }
-
-  .photo {
-    display: flex;
-    justify-content: center;
-    align-items: center;
-
-    margin-right: 5px;
-
-    width: 30px;
-    height: 30px;
-
-    font-size: 14px;
-
-    background-color: white;
-    border-radius: 15px;
-  }
-
-  // Apply for audio message playback.
-  .playing {
-    animation-name: circle1;
-    animation-duration: 2s;
-    animation-iteration-count: infinite;
-  }
-
-  @keyframes circle1 {
-    0%,
-    100% {
-      box-shadow: 0 0 0 0.4em rgba(195, 195, 195, 0.4), 0 0 0 0.25em rgba(195, 195, 195, 0.15);
-    }
-    25% {
-      box-shadow: 0 0 0 0.15em rgba(195, 195, 195, 0.15), 0 0 0 0.4em rgba(195, 195, 195, 0.3);
-    }
-    50% {
-      box-shadow: 0 0 0 0.55em rgba(195, 195, 195, 0.55), 0 0 0 0.15em rgba(195, 195, 195, 0.05);
-    }
-    75% {
-      box-shadow: 0 0 0 0.25em rgba(195, 195, 195, 0.25), 0 0 0 0.55em rgba(195, 195, 195, 0.45);
-    }
-  }
-
-  .contentLoader {
-    display: flex;
-  }
-
-  .contentLoaderDot {
-    width: 5px;
-    height: 5px;
-
-    margin: 2px;
-    border-radius: 2.5px;
-    background-color: #357CA2;
-  }
-
-  .questionMark {
-    font-weight: 900;
-    color: #357CA2;
-  }
-
-  // .divider {
-  //   width: 100vw;
-  //   display: flex;
-  //   justify-content: center;
-  //   align-items: center;
-
-  //   font-family: "SF Compact Display";
-  //   font-size: 12px;
-  //   font-weight: bold;
-  //   color: #9B9B9B;
-
-  //   margin-bottom: 18px;
-  // }
-
-
-</style>
-
-
+.bubble {
+  position: relative;
+  max-width: 66vw;
+  font-family: "SF Pro Text";
+  font-size: 14px;
+  padding: 10px;
+  border-radius: 18px;
+  margin-bottom: 4px;
+}
+
+.ai-bubble {
+  background-color: #FFFFFF;
+  margin-left: 15px;
+}
+
+.client-bubble {
+  color: white;
+  background-color: #58C4FD;
+  margin-right: 15px;
+}
+
+.ai-first-child {
+  border-bottom-left-radius: 9px;
+}
+
+.ai-middle-child {
+  border-top-left-radius: 9px;
+  border-bottom-left-radius: 9px;
+}
+
+.ai-last-child {
+  border-top-left-radius: 9px;
+}
+
+.client-first-child {
+  border-bottom-right-radius: 9px;
+}
+
+.client-middle-child {
+  border-top-right-radius: 9px;
+  border-bottom-right-radius: 9px;
+}
+
+.client-last-child {
+  border-top-right-radius: 9px;
+}
+
+.ai-none-child, .client-none-child {
+  border-top-right-radius: 9px;
+}
+
+</style>

src/render/components/connectionStatus.vue

@@ -0,0 +1,37 @@
+<template>
+    <div class="statusContainer">
+        <div>{{status}}</div>
+    </div>
+
+</template>
+
+
+<script lang="ts">
+
+import { defineComponent } from 'vue'
+import { status } from "@/render/shared/connectionStatus";
+
+export default defineComponent({
+    name: "ConnectionStatus",
+
+    setup() {
+        return {
+            status
+        }
+    }
+
+});
+
+</script>
+
+
+<style lang="scss">
+.statusContainer {
+    width: 100vw;
+    display: flex;
+    justify-content: center;
+    :first-child {
+        margin-right: 2px;
+    }
+}
+</style>

src/render/components/context.vue

@@ -0,0 +1,88 @@
+<template>
+  <span :class="`context ${modifier}-context`">
+    <div v-if="modifier==='ai'" class="avatar"></div>
+    {{ contextRef }}
+  </span>
+</template>
+
+<script lang='ts'>
+
+ import { defineComponent, ref, watch } from 'vue';
+ import { annotateAnim } from "@/render/components/controllers/helpers";
+
+ export default defineComponent({
+  name: "Context",
+
+  props: ["modifier", "context", "uid"],
+
+  setup(props) {
+
+    const contextRef = ref(" ");
+    contextRef.value = props.context;
+
+    watch(() => props.context, (val: string, _oldval: string) => {
+      annotateAnim(props.uid, val, contextRef);
+    });
+
+    return {
+      contextRef
+    };
+
+  }
+
+ })
+
+</script>
+
+<style lang="scss" scoped>
+
+.context {
+  position: relative;
+  min-height: 14px;
+  display: flex;
+  align-items: center;
+  font-family: "SF Compact Display";
+  font-size: 12px;
+  font-weight: bold;
+  margin-top: 5px;
+}
+
+.ai-context {
+  margin-left: 15px;
+}
+
+.client-context {
+  margin-right: 15px;
+}
+
+.avatar {
+  margin-right: 5px;
+  width: 30px;
+  height: 30px;
+  background-color: white;
+  border-radius: 15px;
+}
+
+.playing {
+  animation-name: message-playback-anim;
+  animation-duration: 2s;
+  animation-iteration-count: infinite;
+}
+
+@keyframes message-playback-anim {
+  0%,
+  100% {
+    box-shadow: 0 0 0 0.4em rgba(195, 195, 195, 0.4), 0 0 0 0.25em rgba(195, 195, 195, 0.15);
+  }
+  25% {
+    box-shadow: 0 0 0 0.15em rgba(195, 195, 195, 0.15), 0 0 0 0.4em rgba(195, 195, 195, 0.3);
+  }
+  50% {
+    box-shadow: 0 0 0 0.55em rgba(195, 195, 195, 0.55), 0 0 0 0.15em rgba(195, 195, 195, 0.05);
+  }
+  75% {
+    box-shadow: 0 0 0 0.25em rgba(195, 195, 195, 0.25), 0 0 0 0.55em rgba(195, 195, 195, 0.45);
+  }
+}
+
+</style>

src/render/components/controllers/helpers.ts

@@ -1,6 +1,5 @@
-import { ref } from "vue";
+import { ref, Ref } from "vue";
 import anime from "animejs";
-import { v4 as uuidv4 } from 'uuid';
 
 
 export function animateTextInput () {
@@ -86,17 +85,40 @@ export function animateAudioInput () {
 
 }
 
+// animate a message annotation.
+//! probably could be improved.
+export function annotateAnim (uid: string, val: string, contextRef: Ref) {
 
-export function newMessage ({
-    text=false,
-    audio=false,
-    context=false,
-    uid=uuidv4()
-}) {
-    return {
-        text: text,
-        audio: audio,
-        context: context,
-        uid: uid
-    };
+  anime({
+    targets: `#${uid} span`,
+    opacity: [1, 0],
+    duration: 250,
+    easing: 'easeOutExpo'
+  })
+
+  .finished.then(() => {
+    contextRef.value = val; 
+  })
+
+  .then(() => {
+    anime({
+      targets: `#${uid} span`,
+      opacity: [0, 1],
+      duration: 250,
+      easing: 'easeOutExpo'
+    })
+  })
 }
+
+// Given index and length of content, return message child status.
+export function calcChild (index: number, len: number) {
+  if (len === 1) {
+    return "none";
+  } else if (index === 0) {
+    return "first-child";
+  } else if (index === len - 1) {
+    return "last-child";
+  } else {
+    return "middle-child";
+  }
+}

src/render/components/controllers/inputItem.control.audio.ts

@@ -1,6 +1,6 @@
 import { onMounted, onUnmounted, ref, Ref } from "vue";
 import { postMessage } from "@/render/ipc";
-import { newMessage, animateAudioInput } from "./helpers";
+import { animateAudioInput } from "./helpers";
 import { invokeReturnAudio, postAudioChunk } from "@/render/ipc";
 
 export default function useAudioInputController (typing: Ref) {

src/render/components/controllers/inputItem.control.text.ts

@@ -1,6 +1,6 @@
 import { Ref, ref, watch, onMounted, onUnmounted } from "vue";
 import { postMessage } from "@/render/ipc";
-import { newMessage, animateTextInput } from "./helpers";
+import { animateTextInput } from "./helpers";
 
 
 export default function useTextInputController(elementX: Ref) {
@@ -36,11 +36,12 @@ export default function useTextInputController(elementX: Ref) {
         if (textInput) {
 
             // Send it to the backend for processing.
-            const message = newMessage({
-                text: false
-            });
+            const message: Raw = {
+                text: textInput.value,
+                audio: false
+            };
 
-            // postMessage(message);
+            postMessage(message);
 
             clearInput()
         }

src/render/components/controllers/messenger.control.ts

@@ -1,38 +0,0 @@
-import { ref } from 'vue';
-import useScroll from "@/render/composables/useScroll";
-
-const messagesRef = ref();
-
-/* seed the canvas with messages */
-const seedCanvas = (messages: Message[]) => {
-  messagesRef.value = messages;
-}
-
-const addMessage = (message: Message) => {
-  messagesRef.value.push(message);
-}
-
-const updateMessage = (message: Message) => {
-
-  let target_message = messagesRef.value.filter((m: Message) => {
-      return m.uid = message.uid;
-  })[0];
-
-  if (target_message) {
-    target_message = message;
-  }
-
-}
-
-export default function useMessages() {
-
-  const { updateScrollRef, adjustScroll } = useScroll("messenger");
-
-  return {
-    messagesRef,
-    seedCanvas,
-    addMessage,
-    updateMessage
-  };
-
-}

src/render/components/header.vue

@@ -9,6 +9,7 @@
         class="menuButton minimizeButton"
         @click.prevent="postNavBarMin"
       />
+      <connection-status />
     </span>
 </template>
 
@@ -16,16 +17,20 @@
 
     // import { postNavBarExit, postNavBarMin } from "@/render/ipc";
     import { defineComponent } from "vue";
+    import { postNavBar } from "@/render/ipc";
+
+    import ConnectionStatus from "./connectionStatus.vue";
 
     export default defineComponent({
         name: "Header",
+        components: { ConnectionStatus },
         setup() {
-        const postNavBarExit = () => {};
-        const postNavBarMin = () => {};
-        return {
-            postNavBarExit,
-            postNavBarMin
-        }
+            const postNavBarExit = () => postNavBar('close');
+            const postNavBarMin = () => postNavBar('min');
+            return {
+                postNavBarExit,
+                postNavBarMin
+            }
         }
 
     });

src/render/components/inputItem.vue

@@ -32,7 +32,7 @@ import draggify from "@/render/composables/useDraggify";
 
 import useTextInputController from
   "@/render/components/controllers/inputItem.control.text";
-import useAudioInputController from
+// import useAudioInputController from
   "@/render/components/controllers/inputItem.control.audio";
 
 export default defineComponent({
@@ -51,7 +51,8 @@ export default defineComponent({
 
     // Controllers for text and audio.
     const { typing } = useTextInputController(elementX)
-    const { recording } = useAudioInputController(typing)
+    // const { recording } = useAudioInputController(typing)
+    const recording = false;
 
     return {
       elementX,

src/render/components/login.vue

@@ -35,7 +35,7 @@
 <script lang="ts">
 
 import { defineComponent, ref } from "vue";
-import { setProfile } from '@/render/composables/useProfile';
+import { setProfile } from '@/render/shared/profile';
 import { invokeLogin } from "@/render/ipc";
 
 export default defineComponent({

src/render/components/message.vue

@@ -0,0 +1,89 @@
+<template>
+  <div :id="uid" :class="`${modifier}-message`">
+
+    <Bubble
+      v-for="(val, index) in content"
+      :modifier="modifier"
+      :content="val"
+      :child="calcChild(index, content.length)"
+    />
+
+    <Context
+      :modifier="modifier"
+      :context="context" 
+      :uid="uid"
+    />
+
+  </div>
+</template>
+
+<script lang='ts'>
+
+ import { defineComponent } from 'vue';
+ import Bubble from "@/render/components/bubble.vue";
+ import Context from "@/render/components/context.vue";
+ import { calcChild } from "@/render/components/controllers/helpers";
+
+ export default defineComponent({
+  name: "Message",
+
+  props: ["modifier", "content", "context", "uid"],
+
+  components: {
+    Bubble,
+    Context
+  },
+
+  setup() {
+
+    return {
+      calcChild
+    };
+
+  }
+
+ })
+
+</script>
+
+<style lang="scss" scoped>
+
+.message {
+  width: 100vw;
+  display: flex;
+  flex-direction: column;
+  padding-top: 9px;
+  padding-bottom: 9px;
+  animation-name: message-init-anim;
+  animation-duration: 0.25s; 
+}
+
+@keyframes message-init-anim {
+  from {
+    opacity: 0;
+  } to {
+    opacity: 1;
+  }
+}
+
+.message:first-child {
+  margin-top: 55px;
+}
+
+.message:last-child {
+  margin-bottom: 6px;
+}
+
+.client-message {
+  @extend .message;
+  align-items: flex-end;
+}
+
+.ai-message {
+  @extend .message;
+  align-items: flex-start;
+}
+
+</style>
+
+

src/render/components/messenger.vue

@@ -7,11 +7,12 @@
 
   <!-- List of message bubbles. -->
   <div id="messenger">
-    <Bubble
+    <Message
       v-for="message in messages"
-      :text="message.content.text"
+      :modifier="message.modifier"
+      :content="message.content"
       :context="message.context"
-      :key="message[0]"
+      :uid="message.uid"
     />
   </div>
 
@@ -19,13 +20,13 @@
 
 <script lang="ts">
 
-import { defineComponent, onMounted, onUnmounted } from "vue";
+import { defineComponent } from "vue";
 import InputItem from "@/render/components/inputItem.vue";
 import Settings from "@/render/components/settings.vue";
-import Bubble from "@/render/components/bubble.vue";
+import Message from "@/render/components/message.vue";
 
-import { profile } from "@/render/composables/useProfile";
-import { messages } from "@/render/composables/useMessages";
+import { profile } from "@/render/shared/profile";
+import { messages } from "@/render/shared/messages";
 
 export default defineComponent({
   name: "Messenger",
@@ -33,7 +34,7 @@ export default defineComponent({
   components: {
     InputItem,
     Settings,
-    Bubble
+    Message
   },
 
   setup() {

src/render/components/settings.vue

@@ -28,7 +28,7 @@
 <script lang="ts">
 
   import { defineComponent, ref } from "vue";
-  import { clearProfile } from "@/render/composables/useProfile"
+  import { clearProfile } from "@/render/shared/profile"
   import { invokeLogout } from "@/render/ipc";
 
   export default defineComponent({

src/render/composables/useIpcRend.ts

@@ -25,7 +25,7 @@ export class IpcRendererListener<InputParam> implements IIpcListener<InputParam>
   }
 
   private _onPost = (_e: IpcRendererEvent, payload: InputParam): void => {
-      console.log(`[IPC] Post: ${this.channel}`);
+      console.log(`[IPC] Post: ${this.channel}`, payload);
       this._listenerCallback(payload);
   }
 
@@ -34,16 +34,16 @@ export class IpcRendererListener<InputParam> implements IIpcListener<InputParam>
 
 export default function useIpcRenderer () {
 
-    const invoke = async (endpoint: string, payload: any) => {
+    const invoke = async <T>(endpoint: string, payload: T) => {
         try {
-            const res = await window.ipcRenderer.invoke(endpoint, payload);
+            const res = await window.ipcRenderer.invoke(endpoint, payload? JSON.stringify(payload) : null);
             return res;
         } catch (e) {
             throw e;
         }
     }
 
-    const post = (endpoint: string, payload: any) => {
+    const post = <T>(endpoint: string, payload: T) => {
         window.ipcRenderer.send(endpoint, payload);
     };
 

src/render/composables/useMessages.ts

@@ -1,33 +0,0 @@
-// shared
-import { ref, Ref } from "vue";
-import useScroll from "@/render/composables/useScroll";
-
-export const messages: Ref<Array<Message>> = ref([]);
-
-export const setMessages: IpcListenerCallback<Array<Message>> = (payload) => {
-    messages.value = payload as Array<Message>;
-}
-
-export const addMessage: IpcListenerCallback<Message> = (payload) => {
-    messages.value.push(payload as Message);
-}
-
-export const updateMessage: IpcListenerCallback<Message> = (payload) => {
-    const message = payload as Message;
-
-    let targetMessage = messages.value.filter((m: Message) => {
-        return m.uid = message.uid;
-    })[0];
-
-    if (targetMessage) {
-        targetMessage = message;
-    }
-
-}
-
-export default {
-    messages,
-    setMessages,
-    addMessage,
-    updateMessage
-};

src/render/composables/useScroll.ts

@@ -1,28 +1,32 @@
 
 
-export default function useScroll(element: string) {
-
-  let isScrolledToBottom: boolean;
-  const view = document.getElementById(element)
+export default function useScroll(elementId: string) {
 
   // Update isScrolledToBottom
-  const updateScrollRef = () => {
-    if (view) isScrolledToBottom = view.scrollHeight - view.clientHeight <= view.scrollTop + 1;
-    return isScrolledToBottom;
+  const isScrolledToBottom = () => {
+    const el = document.getElementById(elementId);
+
+    if (el) {
+      return el.scrollHeight - el.clientHeight <= el.scrollTop + 30;
+    }
+
   }
 
   // Adjust scroll after we add content to the messenger.
   const adjustScroll = () => {
-    if (view) {
-      view.scrollTo({
-        top: view.scrollHeight - view.clientHeight,
+    const el = document.getElementById(elementId);
+
+    if (el) {
+      el.scrollTo({
+        top: el.scrollHeight - el.clientHeight,
         behavior: 'smooth'
       });
     }
+
   }
 
   return {
-    updateScrollRef,
+    isScrolledToBottom,
     adjustScroll, 
   };
   

src/render/ipc.ts

@@ -14,7 +14,7 @@ const { post, invoke } = useIpc();
 export const invokeLogin = async (
     payload: LoginPayload
 ): Promise<Profile | Error> => (
-    await invoke('invoke-account-login', JSON.stringify(payload))
+    await invoke('invoke-account-login', payload)
 );
 
 export const invokeLogout = async (): Promise<void> => (
@@ -41,11 +41,7 @@ export const invokeReturnAudio = async (): Promise<ArrayBuffer[] | Error> => (
  *
  */
 
-export const invokeSession = async (cid: string): Promise<Profile | Error> => (
-    await invoke("messenger-init", cid)
-);
-
-export const postMessage = (payload: Message): void => (
+export const postMessage = (payload: Raw): void => (
     post('post-session-send', payload)
 );
 
@@ -53,6 +49,25 @@ export const postAppMount = (): void => (
     post('post-app-mount', null)
 );
 
+/**
+ *
+ * Nav bar endpoints
+ *
+ */
+
+export const postNavBar = (payload: string): void => (
+    post('post-nav-bar', payload)
+);
+
+/**
+ *
+ * Window focus endpoint
+ *
+ */
+
+export const postWindowBlur = (payload: { isFocused: boolean }): void => (
+    post('post-window-focus', payload)
+);
 
 /**
  *

src/render/listeners.ts

@@ -1,13 +1,19 @@
 
 import { IpcRendererListener } from "./composables/useIpcRend"
-import { setProfile } from "./composables/useProfile";
-import { setMessages, addMessage, updateMessage } from "./composables/useMessages";
+/*
+ * Shared state imports
+ * */
+import { setProfile } from "./shared/profile";
+import { setMessages, addMessage, updateMessage, deleteMessage } from "./shared/messages";
+import { setConnectionStatus } from "./shared/connectionStatus";
 
 const SET_PROFILE_CHANNEL = "set-profile";
 
 const INIT_MESSAGES_CHANNEL = "init-messages";
 const ADD_MESSAGE_CHANNEL = "add-message";
 const UPDATE_MESSAGE_CHANNEL = "update-message";
+const DELETE_MESSAGE_CHANNEL = "delete-message";
+const CONNECTION_STATUS_CHANNEL = "set-connection-status";
 
 export const setProfileListener = new IpcRendererListener({
     channel: SET_PROFILE_CHANNEL,
@@ -30,3 +36,12 @@ export const updateMessagesListener = new IpcRendererListener({
     listenerCallback: updateMessage
 });
 
+export const deleteMessagesListener = new IpcRendererListener({
+    channel: DELETE_MESSAGE_CHANNEL,
+    listenerCallback: deleteMessage
+});
+
+export const connectionStatusListener = new IpcRendererListener({
+    channel: CONNECTION_STATUS_CHANNEL,
+    listenerCallback: setConnectionStatus
+});

src/render/main.ts

@@ -3,6 +3,7 @@
 
 import App from "./App.vue";
 
+import { setupAudio } from "./audio/setupAudio";
 import mitt from "mitt";
 import { createApp } from "vue";
 
@@ -12,6 +13,8 @@ import { initIpcRendererListeners } from "./ipc"
 // Handle ipcMain events.
 initIpcRendererListeners();
 
+setupAudio();
+
 // Handle events.
 const emitter = mitt();
 

src/render/shared/connectionStatus.ts

@@ -0,0 +1,12 @@
+// shared
+import { ref } from "vue";
+
+export const status = ref("");
+
+export const setConnectionStatus: IpcListenerCallback<string> = (payload) => {
+    status.value = payload;
+};
+
+export default {
+    status
+};

src/render/shared/messages.ts

@@ -0,0 +1,64 @@
+// shared
+import { ref, Ref } from "vue";
+import useScroll from "@/render/composables/useScroll";
+
+const { isScrolledToBottom, adjustScroll } = useScroll("messenger");
+
+export const messages: Ref<Array<Message>> = ref([]);
+
+export const setMessages: IpcListenerCallback<Array<Message>> = (payload) => {
+    messages.value = payload as Array<Message>;
+    setTimeout(() => {
+        adjustScroll();
+    }, 1000);
+}
+
+export const addMessage: IpcListenerCallback<Message> = (payload) => {
+    const bottom = isScrolledToBottom();
+    messages.value.push(payload as Message);
+    if (bottom) {
+        setTimeout(() => {
+            adjustScroll();
+        }, 10);
+    }
+}
+
+export const updateMessage: IpcListenerCallback<Annotation> = (payload) => {
+    const annotation = payload as Annotation;
+
+    for (const i in messages.value) {
+
+        if (messages.value[i].uid == annotation.uid) {
+
+            if (annotation.name == "content") {
+                messages.value[i].content = annotation.data as string[];
+            }
+
+            if (annotation.name == "context") {
+                messages.value[i].context = annotation.data as string;
+            }
+
+        }
+
+    }
+
+}
+
+export const deleteMessage: IpcListenerCallback<string> = (payload) => {
+    const uid = payload as string;
+
+    for (let i = 0; i < messages.value.length; i++) {
+        if (messages.value[i].uid === uid) {
+            messages.value.splice(i, 1);
+            break;
+        }
+    }
+
+}
+
+export default {
+    messages,
+    setMessages,
+    addMessage,
+    updateMessage
+};

src/render/shims-vue.d.ts

@@ -11,3 +11,8 @@ declare module "anime-js" {
     export = anime;
   }
 }
+
+declare module "*.worklet.ts" {
+  const exportString: string;
+  export default exportString;
+}

src/session.ts

@@ -3,72 +3,64 @@
 
 
 // import useAudio from "@/audio";
-import { ipcEmit } from "@/composables/useEmitter";
+import Messages from "./messages";
+import { getProfile } from "./store";
 import useWebsockets from "./composables/useWebsockets";
 import {config} from "@/config";
-
-/* data structure of messages that's tied to the UI */
-const uiState: any | null = null;
+import { ipcEmit } from "@/composables/useEmitter";
 
 /* start and stop audio functionality */
 // const { initAudio, closeAudio } = useAudio();
 
-const isInitMessage = (message: any): boolean => {
-    return true;
-};
-
-const deauthenticate = (): void => {
-    console.log('deauthenticating')
-};
-
-const isAddMessage = (message: any): boolean => {
-    return true;
-};
-
-/**
- * Controls for interfacing with the platform.
- * Takes an onMessage callback which we define below.
- */
+let messages: Messages;
 
 
 const onMessageCallback = (payload: string) => {
 
-    const message  = JSON.parse(payload);
-    console.log(typeof message);
+    const message = JSON.parse(payload);
 
-    /* if the platform fails to authenticate, we must back down */
     if (message === "CLOSE_AUTH_FAIL") {
-        deauthenticate();
-        return;
+        console.log("deauthenticating");
     }
 
-    ipcEmit("add-message", message)
-    return
-
-    /* on init, platform sends state, used to init canvas */
-    if (isInitMessage(message)) {
-        ipcEmit("init-messages", message)
-    }
-
-
-    else if (isAddMessage(message)) {
-        ipcEmit("add-messages", message)
+    else if (message.header === "init") {
+        messages = new Messages(message.body);
     }
 
     else {
-        ipcEmit("update-messages", message)
+        if (messages) {
+            messages.update(message.body);
+        }
     }
 
 }
 
-const onConnectionStatusCallback = (alive: boolean) => {
-    // console.log('[Session]: Connection Alive: ', alive);
-    // ipcEmit('connection-state', alive);
+const connectionStatusCallback = (status: string) => {
+    ipcEmit('set-connection-status', status);
+}
+
+
+export const updateAppState = (): void => {
+    const profile = getProfile();
+
+    ipcEmit("set-profile", profile);
+
+    if (messages)
+    messages.emit();
+
+}
+
+const statusOptions = {
+   openMessage: "Connected",
+   pongMessage: "Nominal",
+   closeMessage: "Reconnecting",
+   pingErrorMessage: "Connection Lost",
 }
 
 const { connect, send, close } = useWebsockets(
     onMessageCallback,
-    onConnectionStatusCallback
+    connectionStatusCallback,
+    statusOptions
 );
 
 /* send a message to the platform */

src/types.ts

@@ -1,14 +1,25 @@
 
+interface Update {
+    name: string;
+    data: Message[] | Message | Annotation | string;
+}
+
 interface Message {
-    text: boolean | string;
+    modifier: string;
+    content: string[];
     context: boolean | string;
-    audio: boolean | string;
-    type: 1 | 2 | 3;
     uid: string;
 }
 
-interface ViewMessage extends Message {
-    child: string;
+interface Annotation {
+    name: string;
+    data: string | string[];
+    uid: string;
+}
+
+interface Raw {
+    text: string | boolean;
+    audio: string | boolean;
 }
 
 interface WindowState {
@@ -47,7 +58,7 @@ interface IpcHandlerCallback<I, O> {
 }
 
 interface IpcListenerCallback<T> {
-    (payload: T | null): void;
+    (payload: T): void;
 }
 
 interface IIpcHandler<I, O> {

src/window.ts

@@ -33,20 +33,20 @@ const loadWinState = (fileName: string): WindowState => {
 }
 
 // Called when a NavBar button is pressed.
-const onNavBar = (_event: any, action: string): void => {
+export const onNavBar: IpcListenerCallback<string> = (payload): void => {
   if (win) {
-    if (action === "close") {
-      win.close()
+    if (payload === "close") {
+      win.close();
     } else {
-      win.minimize()
+      win.minimize();
     }
   }
 }
 
 // Util function to render message on ipc-renderer event.
-const postToWindow = <T>(event: IpcRendererEvent<T>): void => {
+const postToWindow = <T>(e: IpcRendererEvent<T>): void => {
     if (win) {
-        win.webContents.send(event.channel, event.payload);
+        win.webContents.send(e.channel, e.payload);
     }
 }
 
@@ -75,10 +75,6 @@ const onWindowMount = (): void => {
     // Must tell initApp that window exists.
     backgroundMitt.emit('window-active', true);
 
-    // Handle win nav-bar event.
-    ipcMain.removeAllListeners("nav-bar") // avoid setting duplicate handlers
-    ipcMain.on("nav-bar", onNavBar);
-
     // Gateway for messages to the frontend.
     backgroundMitt.removeAllListeners("ipc-renderer")
     backgroundMitt.on("ipc-renderer", postToWindow);

tests/audio.js

@@ -1,84 +1,38 @@
 
-const speech = require('@google-cloud/speech');
+// const speech = require('@google-cloud/speech');
 const portAudio = require('naudiodon');
 // const rs = fs.createReadStream('rawAudio.wav');
 
-// Creates a client
-const client = new speech.SpeechClient();
-
-const encoding = 'LINEAR16';
-const sampleRateHertz = 16000;
-const languageCode = 'en-US';
-
 const audioContainer = {
     input: '',
     buffers: []
 }
 
-const config = {
-  encoding: encoding,
-  sampleRateHertz: sampleRateHertz,
-  languageCode: languageCode,
-};
-
-/**
- * Note that transcription is limited to 60 seconds audio.
- * Use a GCS file for audio longer than 1 minute.
- */
-async function transcribeSpeech (audio) {
-  const request = {
-    config: config,
-    audio: audio,
-  };
-
-  // Detects speech in the audio file. This creates a recognition job that you
-  // can wait for now, or get its result later.
-  const [operation] = await client.longRunningRecognize(request);
-
-  // Get a Promise representation of the final result of the job
-  const [response] = await operation.promise();
-
-  const transcription = response.results
-    .map(result => result.alternatives[0].transcript)
-    .join('\n');
-  console.log(`Transcription: ${transcription}`);
-}
 
 let record = true;
 
 // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream
-const ia = new portAudio.AudioIO({
+const aio = new portAudio.AudioIO({
   inOptions: {
-    channelCount: 1,
-    sampleFormat: 16,
-    sampleRate: 16000,
-    deviceId: -1,
-    closeOnError: false,
+    channelCount: 2,
+    sampleFormat: portAudio.SampleFormat16Bit,
+    sampleRate: 44100,
+    deviceId: -1 // Use -1 or omit the deviceId to select the default device
   },
-});
-ia.setEncoding('base64');
-ia.start();
-ia.on('data', (chunk) => {
-    if (record) {
-        console.log('recording data');
-        audioContainer.input += chunk;
-    } else {
-        if (audioContainer.input.length) audioContainer.input = "";
-    }
+  outOptions: {
+    channelCount: 2,
+    sampleFormat: portAudio.SampleFormat16Bit,
+    sampleRate: 44100,
+    deviceId: -1 // Use -1 or omit the deviceId to select the default device
+  }
 });
 
-const ao = new portAudio.AudioIO({
-    outOptions: {
-        sampleFormat: 16,
-        channelCount: 1,
-        sampleRate: 16000,
-        deviceId: -1,
-        closeOnError: false,
-    }
-});
-ao.start();
 
-let counter = 0;
+aio.start()
+aio.read()
+aio.on('data', buf => console.log(buf.timestamp));
+
+const counter = 0;
 const tests = [];
 
 function testCallback() {
@@ -143,13 +97,6 @@ function bufSplit(input){
 }
 
 async function test() {
-    transcribeSpeech({
-        content: Buffer.from(audioContainer.input, 'base64')
-    });
-    tests.push(audioContainer.input)
-    counter++;
-    console.log('audio string length:', audioContainer.input.length)
-    console.log('buffers written: ', audioContainer.buffers.length)
 }
 
 setTimeout(async () => {
@@ -157,26 +104,6 @@ setTimeout(async () => {
     test()
 }, 4000);
 
-// setTimeout(() => {
-//     record = true;
-// }, 6000)
-//
-// setTimeout(async () => {
-//     record = false;
-//     test();
-// }, 9000);
-//
-// setTimeout(() => {
-//     record = true;
-// }, 11000)
-//
-// setTimeout(async () => {
-//     record = false;
-//     test();
-// }, 14000);
-//
 setTimeout(async () => {
-    ia.quit();
-    testCallback()
     return;
 }, 6000);

tests/transcribe.js

@@ -0,0 +1,38 @@
+
+const speech = require('@google-cloud/speech');
+
+// Creates a client
+const client = new speech.SpeechClient();
+
+const encoding = 'LINEAR16';
+const sampleRateHertz = 16000;
+const languageCode = 'en-US';
+
+const config = {
+  encoding: encoding,
+  sampleRateHertz: sampleRateHertz,
+  languageCode: languageCode,
+};
+
+/**
+ * Note that transcription is limited to 60 seconds audio.
+ * Use a GCS file for audio longer than 1 minute.
+ */
+async function transcribeSpeech (audio) {
+  const request = {
+    config: config,
+    audio: audio,
+  };
+
+  // Detects speech in the audio file. This creates a recognition job that you
+  // can wait for now, or get its result later.
+  const [operation] = await client.longRunningRecognize(request);
+
+  // Get a Promise representation of the final result of the job
+  const [response] = await operation.promise();
+
+  const transcription = response.results
+    .map(result => result.alternatives[0].transcript)
+    .join('\n');
+  console.log(`Transcription: ${transcription}`);
+}

tsconfig.worklet.json

@@ -0,0 +1,23 @@
+{
+    "compilerOptions": {
+        "target": "ES2018",
+        "module": "esnext",
+        "strict": true,
+        "importHelpers": true,
+        "moduleResolution": "node",
+        "resolveJsonModule": true,
+        "experimentalDecorators": true,
+        "skipLibCheck": true,
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
+        "sourceMap": true,
+        "baseUrl": ".",
+        "types": ["webpack-env"],
+        "paths": {
+            "@/*": ["src/*"]
+        },
+        "lib": ["esnext", "scripthost"]
+    },
+    "include": ["src/**/*.worklet.ts"],
+    "exclude": ["node_modules"]
+}

vue.config.js

@@ -0,0 +1,50 @@
+
+const path = require("path");
+
+module.exports = {
+  assetsDir: "../../static/SPA"
+}
+module.exports = {
+    lintOnSave: false,
+    pluginOptions: {
+        electronBuilder: {
+          mainProcessFile: "./src/init.ts",
+          rendererProcessFile: "./src/render/main.ts",
+          preload: "./src/render/preload.ts",
+
+
+            builderOptions: {
+              "appId": "com.crimata.ElectronUpdaterApp",
+              "artifactName": "${productName}-${version}.${ext}",
+              "publish": {
+                "provider": "generic",
+                "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build"
+              }
+            },
+
+
+          chainWebpackRendererProcess: (config) => {
+            // Chain webpack config for electron renderer process only (won't be applied to web builds)
+            config.outputDir = path.resolve(__dirnamei, "js");
+            config.module
+              .rule('worklet')
+              .test(/\.worklet\.ts$/)
+              .use('worklet-loader')
+                .loader('worklet-loader')
+                .tap(options => {
+                    options.name = "js/[hash].worklet.js";
+                    return options
+                })
+                .end()
+              // Add another loader
+              .use('ts-loader')
+                .loader('ts-loader')
+                .tap(options => {
+                    options.configFile = "tsconfig.worklet.json";
+                        return options
+                    })
+                    .end()
+          },
+        }
+      }
+};

yarn.lock

@@ -1456,6 +1456,11 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0"
   integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw==
 
+"@types/json-schema@^7.0.6":
+  version "7.0.7"
+  resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad"
+  integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==
+
 "@types/long@^4.0.0", "@types/long@^4.0.1":
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
@@ -2367,7 +2372,7 @@ ajv-keywords@^3.1.0, ajv-keywords@^3.4.1, ajv-keywords@^3.5.2:
   resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d"
   integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==
 
-ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4:
+ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
   integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==
@@ -4898,11 +4903,6 @@ dotenv-expand@^5.1.0:
   resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0"
   integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==
 
-dotenv@^10.0.0:
-  version "10.0.0"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81"
-  integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==
-
 dotenv@^8.2.0:
   version "8.2.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
@@ -6649,6 +6649,11 @@ hmac-drbg@^1.0.0:
     minimalistic-assert "^1.0.0"
     minimalistic-crypto-utils "^1.0.1"
 
+hoek@^4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb"
+  integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==
+
 hoopy@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/hoopy/-/hoopy-0.1.4.tgz#609207d661100033a9a9402ad3dea677381c1b1d"
@@ -8447,7 +8452,7 @@ loader-utils@^0.2.16:
     json5 "^0.5.0"
     object-assign "^4.0.1"
 
-loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0:
+loader-utils@^1.0.0, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613"
   integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==
@@ -11154,6 +11159,14 @@ schema-utils@2.7.0:
     ajv "^6.12.2"
     ajv-keywords "^3.4.1"
 
+schema-utils@^0.4.0:
+  version "0.4.7"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187"
+  integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==
+  dependencies:
+    ajv "^6.1.0"
+    ajv-keywords "^3.1.0"
+
 schema-utils@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770"
@@ -11172,6 +11185,15 @@ schema-utils@^2.0.0, schema-utils@^2.5.0, schema-utils@^2.6.1, schema-utils@^2.6
     ajv "^6.12.4"
     ajv-keywords "^3.5.2"
 
+schema-utils@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.0.0.tgz#67502f6aa2b66a2d4032b4279a2944978a0913ef"
+  integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==
+  dependencies:
+    "@types/json-schema" "^7.0.6"
+    ajv "^6.12.5"
+    ajv-keywords "^3.5.2"
+
 scss-tokenizer@^0.2.3:
   version "0.2.3"
   resolved "https://registry.yarnpkg.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz#8eb06db9a9723333824d3f5530641149847ce5d1"
@@ -13300,6 +13322,14 @@ worker-farm@^1.7.0:
   dependencies:
     errno "~0.1.7"
 
+worker-loader@^3.0.8:
+  version "3.0.8"
+  resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-3.0.8.tgz#5fc5cda4a3d3163d9c274a4e3a811ce8b60dbb37"
+  integrity sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==
+  dependencies:
+    loader-utils "^2.0.0"
+    schema-utils "^3.0.0"
+
 worker-rpc@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5"
@@ -13307,6 +13337,15 @@ worker-rpc@^0.1.0:
   dependencies:
     microevent.ts "~0.1.1"
 
+worklet-loader@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/worklet-loader/-/worklet-loader-1.0.0.tgz#17e2eef75981de469c1e1e200ad1ffb54efe7a29"
+  integrity sha512-4yFqiGDwICoJB4ZbWHzCzyTyDrRnCU1XfvSJtjiBBDreuWDYpA6wf8yqQjNckcjL1jm/sT9ocSvP5tJnmsMOLA==
+  dependencies:
+    hoek "^4.2.1"
+    loader-utils "^1.0.0"
+    schema-utils "^0.4.0"
+
 wrap-ansi@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba"