From c0b71a4a6f01b00bbdd7f8dcc4888f8b35fffcd2 Mon Sep 17 00:00:00 2001 From: Andrew Gundersen Date: Fri, 12 Mar 2021 14:01:51 -0600 Subject: [PATCH] trying to fix freeze bug --- src/background/session.ts | 42 +++++++++--------- src/background/window.ts | 10 +---- src/components/inputItem/inputController.ts | 36 ++++++---------- src/components/messageItem.vue | 3 +- src/modules/message.ts | 47 +++++++++++++++++---- src/types/message/index.ts | 4 +- src/views/messenger.vue | 24 ++++++++--- 7 files changed, 100 insertions(+), 66 deletions(-) diff --git a/src/background/session.ts b/src/background/session.ts index 5948978..861eaea 100644 --- a/src/background/session.ts +++ b/src/background/session.ts @@ -4,8 +4,8 @@ import WebSocket from 'ws'; import { backgroundMitt } from './emitter'; import { ipcMain } from "electron"; import { play } from './audio'; -import { UserCreds, ClientMessage } from "@/types/message/index"; -import { clientMessage } from '@/modules/message'; +import { UserCreds, ClientMessage, RenderMessage, Annotation } from "@/types/message/index"; +import { clientMessage, initRenderMessageFromString } from '@/modules/message'; import { v4 as uuidv4 } from 'uuid'; const ip = 'ws://127.0.0.1'; @@ -15,7 +15,7 @@ let socket: WebSocket; let success = false; let auth = false; -const onAuthSession = async (_event, payload: string | UserCreds | null): Promise => { +const onAuthSession = async (e: any, payload: string | UserCreds | null): Promise => { return new Promise((resolve, reject) => { console.log('authenticating...'); @@ -43,40 +43,43 @@ const onAuthSession = async (_event, payload: string | UserCreds | null): Promis }); }; -const onMessage = (message: string): void => { +const onMessage = (message_str: string): void => { while (!auth) { - backgroundMitt.emit('auth-res', message); + backgroundMitt.emit('auth-res', message_str); return; } - // Parse the message. - const m = JSON.parse(message); + // Decode the message. + const m = JSON.parse(message_str) + let message: RenderMessage | Annotation; - // check to see if this is a Render Message. + // Check to see if this is a Render Message. if (m.content) { + message = initRenderMessageFromString(message_str) - // If there is audio, play it. - if (m.content.audio) { + // Play audio if any. + if (message.content.audio) { const audioBytes = Buffer.from(m.content.audio as string, 'hex'); m.content.audio = true; play(audioBytes); - } else { - m.content.audio = false; } - // if there is no unique id, add it. - if(!m.uid) m.uid = uuidv4(); } + else { + message = m // Annotation + } + + // Send it to the frontend. backgroundMitt.emit('ipc-renderer', { endpoint: 'render-message', - message: m + message: message }); }; -const onClientMessage = (_event, payload: ClientMessage): void => { +const onClientMessage = (e: any, payload: ClientMessage): void => { console.log('sending message'); socket.send(JSON.stringify(payload)); } @@ -150,8 +153,7 @@ export function initSession(): void { } -export function sendAudio(content: string, uid: string): void { - const m = clientMessage("", content, uid); +export function sendAudio(audio: string, uid: string): void { + const m = clientMessage("", audio, uid); socket.send(JSON.stringify(m)); -} - +} \ No newline at end of file diff --git a/src/background/window.ts b/src/background/window.ts index 31c8850..27aac38 100644 --- a/src/background/window.ts +++ b/src/background/window.ts @@ -3,6 +3,7 @@ import { BrowserWindow, ipcMain } from "electron"; import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; import { backgroundMitt } from './emitter'; +import { RenderMessage } from "@/types/message/index"; import * as path from "path"; interface WindowSettings { @@ -11,15 +12,6 @@ interface WindowSettings { resizable: boolean; } -interface RenderMessage { - content: string; - context: string; - subContext: string; - modifiers: string; - time: string; - id: string; -} - interface IpcRendererPayload { endpoint: string; message: RenderMessage | null; diff --git a/src/components/inputItem/inputController.ts b/src/components/inputItem/inputController.ts index 45c762c..5bfefb5 100644 --- a/src/components/inputItem/inputController.ts +++ b/src/components/inputItem/inputController.ts @@ -12,10 +12,9 @@ import keyboardNameMap from "./keyBoardMaps/keyboardNameMap"; import keyboardCharMap from "./keyBoardMaps/keyboardCharMap"; import { RenderMessage, ClientMessage } from "@/types/message/index"; import { v4 as uuidv4 } from 'uuid'; -import {clientMessage, renderMessage} from '@/modules/message'; +import {clientMessage, initRenderMessage} from '@/modules/message'; let textInput: HTMLInputElement | null; -let m: RenderMessage | ClientMessage; let uid: string; export default function useInputController() { @@ -33,19 +32,15 @@ export default function useInputController() { // remove first character of input. const text = input.value.substring(1); - uid = uuidv4(); + // Render the message immediately. + const message = initRenderMessage(text, "", "", "sf"); + emitter.emit("self-message", message); - m = renderMessage(text, false, "sf", uid); + // Then send it to the backend for processing. + const clientMm = clientMessage(text, false, message.uid); + post('client-message', message); - // render user message. - emitter.emit("self-message", m); - - m = clientMessage(m.content.text, m.content.audio, uid); - - // send message to backend. - post('client-message', m); - - // reset text input. + // Reset the text input. if (textInput) textInput.value = ''; input.value = " "; } @@ -98,18 +93,17 @@ export default function useInputController() { isRecording.value = false; - uid = uuidv4(); - - // render audio message. - m = renderMessage("", false, "sf", uid); - emitter.emit("self-message", m); + // Render the message immediately. + const audioMessage = initRenderMessage("", "", "", "sf") + emitter.emit("self-message", audioMessage); - // stop stream recording. + // Notify backend to stop recording and send audio to python backend for audio processing. post('update-recorder', { isRecording: isRecording.value, - uid + uid: audioMessage.uid }); + // Reset input. input.value = '' } } @@ -129,7 +123,6 @@ export default function useInputController() { // watch keyboard input & update current state. watch(input, (input, prevInput) => { - // if (prevInput !== "" || prevInput.length > 0) { if (!isRecording.value) { isTyping.value = true; @@ -137,7 +130,6 @@ export default function useInputController() { return; } - // if (input === " " && isRecording.value === false) { isTyping.value = false; isRecording.value = true; diff --git a/src/components/messageItem.vue b/src/components/messageItem.vue index aab3827..5139176 100644 --- a/src/components/messageItem.vue +++ b/src/components/messageItem.vue @@ -42,7 +42,7 @@ - +
@@ -69,6 +69,7 @@ import {defineComponent, ref, onMounted} from 'vue'; + export default defineComponent({ name: "MessageItem", diff --git a/src/modules/message.ts b/src/modules/message.ts index 37c6ba9..58233a3 100644 --- a/src/modules/message.ts +++ b/src/modules/message.ts @@ -1,16 +1,47 @@ import { RenderMessage, ClientMessage } from "@/types/message/index"; +import { v4 as uuidv4 } from 'uuid'; -export const renderMessage = (text: string, audio: string | boolean, modifier: string, uid: string): RenderMessage => ( - { +// Create a RenderMessage object directly from json string. +export function initRenderMessageFromString(message_str: string) { + + // Parse the string. + const m = JSON.parse(message_str); + + // Auto set uid and time. + let currentdate = new Date(); + + const message: RenderMessage = { + content: m.content, + context: m.context, + modifier: m.modifier, + time: currentdate.getTime(), + uid: uuidv4(), + isChild: false + } + + return message +} + +// Create a RenderMessage object. +export function initRenderMessage(text: string, audio: string, context: string, modifier: string) { + + // Auto set uid and time. + let currentdate = new Date(); + + const message: RenderMessage = { content: { - audio, - text + text: text, + audio: audio }, - context: "", - modifier, - uid + context: context, + modifier: modifier, + time: currentdate.getTime(), + uid: uuidv4(), + isChild: false } -) + + return message +} export const clientMessage = (text: string, audio: string | boolean, uid: string): ClientMessage => ( { diff --git a/src/types/message/index.ts b/src/types/message/index.ts index 80e3a97..7c74fca 100644 --- a/src/types/message/index.ts +++ b/src/types/message/index.ts @@ -10,7 +10,9 @@ export interface RenderMessage { }; context: string; modifier: string; + time: number; uid: string; + isChild: boolean; } export interface ClientMessage { @@ -23,4 +25,4 @@ export interface Annotation { text: string; context: string; uid: string; -} +} \ No newline at end of file diff --git a/src/views/messenger.vue b/src/views/messenger.vue index a74fe94..28c1352 100644 --- a/src/views/messenger.vue +++ b/src/views/messenger.vue @@ -11,6 +11,7 @@