Merge remote-tracking branch 'origin/main'

src/background/session.ts

@@ -4,9 +4,8 @@ import WebSocket from 'ws';
 import { backgroundMitt } from './emitter';
 import { ipcMain } from "electron";
 import { play } from './audio';
-import { UserCreds, ClientMessage } from "@/types/message/index";
-import { clientMessage } from '@/modules/message';
-import { v4 as uuidv4 } from 'uuid';
+import { UserCreds, ClientMessage, RenderMessage, Annotation } from "@/types/message/index";
+import { clientMessage, renderMessage } from '@/modules/message';
 
 const ip = 'ws://127.0.0.1';
 const port = 8760;
@@ -15,7 +14,7 @@ let socket: WebSocket;
 let success = false;
 let auth = false;
 
-const onAuthSession = async (_event, payload: string | UserCreds | null): Promise<string> => {
+const onAuthSession = async (e: any, payload: string | UserCreds | null): Promise<string> => {
     return new Promise((resolve, reject) => {
 
         console.log('authenticating...');
@@ -43,40 +42,48 @@ const onAuthSession = async (_event, payload: string | UserCreds | null): Promis
     });
 };
 
-const onMessage = (message: string): void => {
+const onMessage = (messageStr: string): void => {
 
     while (!auth) {
-        backgroundMitt.emit('auth-res', message);
+        backgroundMitt.emit('auth-res', messageStr);
         return;
     }
 
-    // Parse the message.
-    const m = JSON.parse(message);
+    // Decode the message.
+    const m = JSON.parse(messageStr)
+    let message: RenderMessage | Annotation;
 
-    // check to see if this is a Render Message.
+    // Check to see if this is a Render Message.
     if (m.content) {
+         message = renderMessage({
+            text: m.content.text,
+            audio: m.content.audio,
+            context: m.context,
+            modifier: m.modifier,
+        });
 
-        // If there is audio, play it.
-        if (m.content.audio) {
+        // Play audio if any.
+        if (message.content.audio) {
             const audioBytes = Buffer.from(m.content.audio as string, 'hex');
             m.content.audio = true;
             play(audioBytes);
-        } else {
-            m.content.audio = false;
         }
 
-        // if there is no unique id, add it.
-        if(!m.uid) m.uid = uuidv4();
     }
 
+    else {
+        message = m // Annotation
+    }
+
+    // Send it to the frontend.
     backgroundMitt.emit('ipc-renderer', {
         endpoint: 'render-message',
-        message: m
+        message: message 
     });
 
 };
 
-const onClientMessage = (_event, payload: ClientMessage): void => {
+const onClientMessage = (e: any, payload: ClientMessage): void => {
     console.log('sending message');
     socket.send(JSON.stringify(payload));
 }
@@ -150,8 +157,7 @@ export function initSession(): void {
 }
 
 
-export function sendAudio(content: string, uid: string): void {
-    const m = clientMessage("", content, uid);
+export function sendAudio(audio: string, uid: string): void {
+    const m = clientMessage("", audio, uid);
     socket.send(JSON.stringify(m));
-}
-
+}

src/background/window.ts

@@ -3,6 +3,7 @@
 import { BrowserWindow, ipcMain } from "electron";
 import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
 import { backgroundMitt } from './emitter';
+import { RenderMessage } from "@/types/message/index";
 import * as path from "path";
 
 interface WindowSettings {
@@ -11,15 +12,6 @@ interface WindowSettings {
   resizable: boolean;
 }
 
-interface RenderMessage {
-  content: string;
-  context: string;
-  subContext: string;
-  modifiers: string;
-  time: string;
-  id: string;
-}
-
 interface IpcRendererPayload {
   endpoint: string;
   message: RenderMessage | null;

src/components/inputItem/inputController.ts

@@ -10,14 +10,10 @@ import useMitt from "@/modules/mitt";
 import { useIpc } from '@/modules/ipc';
 import keyboardNameMap from "./keyBoardMaps/keyboardNameMap";
 import keyboardCharMap from "./keyBoardMaps/keyboardCharMap";
-import { RenderMessage, ClientMessage } from "@/types/message/index";
-import { v4 as uuidv4 } from 'uuid';
-import {clientMessage, renderMessage} from '@/modules/message';
 import anime from "animejs";
+import { clientMessage, renderMessage } from '@/modules/message';
 
 let textInput: HTMLInputElement | null;
-let m: RenderMessage | ClientMessage;
-let uid: string;
 
 // Animation functions for text input element.
 function expandInput() {
@@ -50,24 +46,27 @@ export default function useInputController() {
     const input = ref("");
     const capsLock = ref(false);
 
+    // Utility function to render message.
+    const render = (text: string, audio: boolean | string, context: string, modifier: string): string => {
+        const m = renderMessage({ text, audio, context, modifier });
+        emitter.emit("self-message", m);
+        console.log(m.time)
+        return m.uid;
+    }
+
     // send message on ENTER.
     const onEnter = () => {
 
         if (input.value) {
 
-            uid = uuidv4();
+            // Render the message immediately and get its unique id.
+            const uid = render(input.value, false, "", "sf");
 
-            m = renderMessage(input.value, false, "sf", uid);
+            // Then send it to the backend for processing.
+            const clientM = clientMessage(input.value, false, uid);
+            post('client-message', clientM);
 
-            // render user message.
-            emitter.emit("self-message", m);
-
-            m = clientMessage(m.content.text, m.content.audio, uid);
-
-            // send message to main process.
-            post('client-message', m);
-
-            // reset text input.
+            // Reset the text input.
             if (textInput) textInput.value = ''; 
             input.value = " ";
         }
@@ -127,18 +126,15 @@ export default function useInputController() {
 
             isRecording.value = false;
 
-            uid = uuidv4();
-
-            // render audio message.
-            m = renderMessage("", false, "sf", uid);
-            emitter.emit("self-message", m);
-
-            // stop stream recording.
+            // // Render the message immediately and get its unique id.
+            const uid = render("", "", "", "sf");
+            // Notify main process to stop recording and send send audio to backend for analysis.
             post('update-recorder', {
                 isRecording: isRecording.value,
                 uid
             }); 
 
+            // Reset input.
             input.value = ''
         }
         else if (e.code === "CapsLock") {
@@ -158,7 +154,6 @@ export default function useInputController() {
         window.removeEventListener("keyup", onKeyUp);
     });
 
-
     // watch keyboard input & update inputItem state.
     watch(input, (input, prevInput) => {
         if (isTyping.value && input.length === 0) {

src/components/messageItem.vue

@@ -1,6 +1,7 @@
 <template>  
   <div
     class="message"
+    :class="`${message.isChild}ChildMessage`"
     :id="`${message.modifier}Message`"
     :key="message.id"
   >
@@ -9,11 +10,15 @@
     <!-- !denoted by the modifier attribute -->
 
     <!-- Includes bubble and context. -->
-    <div class="messageBox" :id="`${message.modifier}MessageBox`">
+    <div 
+      :id="`${message.modifier}MessageBox`"
+      class="messageBox"
+    >
 
       <!-- message bubble -->
       <div
         class="bubble"
+        :class="`${message.modifier}-${message.isChild}ChildBubble`"
         :id="`${message.modifier}Bubble`"
       >
         <!-- Show loader when audio is being transcribed. -->
@@ -42,7 +47,7 @@
       </div>
 
       <!-- message context -->
-      <span class="context">
+      <span v-if="message.isChild === 'last' || message.isChild === 'none'" class="context">
 
         <!-- Render Crimata icon or friend's initials. -->
         <div v-if="message.modifier == 'ai'" class="photo">
@@ -69,6 +74,7 @@
 
  import {defineComponent, ref, onMounted} from 'vue';
 
+
  export default defineComponent({
   name: "MessageItem",
 
@@ -127,6 +133,19 @@
     margin-bottom: 18px;
   }
 
+  .firstChildMessage {
+    padding-bottom: 2px;
+  }
+
+  .middleChildMessage {
+    padding-top: 2px;
+    padding-bottom: 2px;
+  }
+
+  .lastChildMessage {
+    padding-top: 2px;
+  }
+
   #aiMessage {
     align-items: flex-start;
   }
@@ -199,6 +218,32 @@
     background-color: white;
   }
 
+  .sf-firstChildBubble {
+    border-bottom-right-radius: 9px;
+  }
+
+  .sf-middleChildBubble {
+    border-top-right-radius: 9px;
+    border-bottom-right-radius: 9px;
+  }
+
+  .sf-lastChildBubble {
+    border-top-right-radius: 9px;
+  }
+
+  .ai-firstChildBubble, .fr-firstChildBubble {
+    border-bottom-left-radius: 9px;
+  }
+
+  .ai-middleChildBubble, .fr-middleChildBubble {
+    border-top-left-radius: 9px;
+    border-bottom-left-radius: 9px;
+  }
+
+  .ai-lastChildBubble, .fr-lastChildBubble {
+    border-top-left-radius: 9px;
+  }
+
   .context {
     display: flex;
     align-items: center;

src/modules/message.ts

@@ -1,14 +1,30 @@
 import { RenderMessage, ClientMessage } from "@/types/message/index";
+import { v4 as uuidv4 } from 'uuid';
 
-export const renderMessage = (text: string, audio: string | boolean, modifier: string, uid: string): RenderMessage => (
+interface RenderParams {
+  text: string;
+  audio: boolean | string;
+  context: string;
+  modifier: string;
+}
+
+function getTimeStamp(): number {  
+  const currentdate = new Date();
+  return currentdate.getTime();
+}
+
+// Create a RenderMessage object.
+export const renderMessage = (params: RenderParams): RenderMessage => (
   {
     content: {
-        audio,
-        text
+        text: params.text,
+        audio: params.audio
     },
-    context: "",
-    modifier,
-    uid
+    context: params.context,
+    modifier: params.modifier,
+    time: getTimeStamp(),
+    uid: uuidv4(),
+    isChild: "none"
   }
 )
 

src/modules/messages.ts

@@ -28,18 +28,21 @@ function deleteOldest() {
 export default function useMessages() {
 
     const addMessage = (m: RenderMessage) => {
+      console.log(`Adding message: ${m.content.text}`)
+
       // check for message limit before setting a new message.
       if (messages.value.size >= sizeLimit) deleteOldest();
       
       messages.value.set(m.uid, m);
     }
 
-    const updateMessage = (m: Annotation) => {
+    const updateMessage = (a: Annotation) => {
         // update message if in the map.
-        if (messages.value.has(m.uid)) {
-            messages.value.get(m.uid).context = m.context;
-            messages.value.get(m.uid).content.text = m.text;
-        }
+        const m = messages.value.get(a.uid)
+        m.context = a.context
+        m.text = a.text
+
+        return m
     }
 
     const saveMessages = () => {

src/types/message/index.ts

@@ -10,7 +10,9 @@ export interface RenderMessage {
     };
     context: string;
     modifier: string;
+    time: number;
     uid: string;
+    isChild: string;
 }
 
 export interface ClientMessage {
@@ -23,4 +25,4 @@ export interface Annotation {
     text: string;
     context: string;
     uid: string;
-}
+}

src/views/messenger.vue

@@ -29,22 +29,62 @@ export default defineComponent({
     const { emitter } = useMitt();
     const { messages, addMessage, updateMessage } = useMessages();
 
-    const onRenderMessage = (_event: any, payload: any) => { 
+    let prevRef: string; // ref to previous message
+    let activeGroup = false; // if active group exists.
 
-      // render message if there is content.
-      if (payload.message.content) {
-        addMessage(payload.message);
+    // When backend has new message.
+    const onRenderMessage = (_event: any, payload: any) => { 
+      let message = payload.message; // cleaner
+
+      // Check type and handle accordingly.
+      if (message.content) {
+        addMessage(message);
       } else {
-        // must be an annotation, update message.
-        updateMessage(payload.message);
+        message = updateMessage(message);
       }
 
+      // Check for grouping.
+      if (prevRef) {
+        const prev = messages.value.get(prevRef)
+        const current = messages.value.get(message.uid)
+
+        console.log(`${message.modifier}, ${prev.modifier}`)
+        console.log(`${message.context}, ${prev.context}`)
+        
+        // If last two messages go together...group accordingly.
+        if (current.time < prev.time + 20000 && current.modifier === prev.modifier && current.context === prev.context) {
+
+          if (activeGroup) {
+            prev.isChild = "middle"
+          } else {
+            prev.isChild = "first"
+          }
+
+          // Current always last child.
+          current.isChild = "last"
+          activeGroup = true
+        }
+
+        // Else...reset activeGroup.
+        else {
+          activeGroup = false
+        }
+
+      }
+
+      // Set previous message for next time.
+      prevRef = message.uid
+
     }
 
     onMounted(() => {
-      // Listen for new messages to render.
+
+      // Front end listener.
       emitter.on('self-message', (message) => addMessage(message));
+
+      // Back end listener.
       window.ipcRenderer.on("render-message", onRenderMessage);
+
     });
 
     onUnmounted(() => {