From d06fb395b92b6cf40abfc608c9be83e8b055663e Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Tue, 23 Feb 2021 14:33:58 -0600 Subject: [PATCH] add audio playback --- gcloud.json | 12 ++ src/background/audio.ts | 128 +++++++++++++----- src/background/run.ts | 4 +- src/background/session.ts | 10 +- src/components/messageView/index.vue | 111 +++++++++++++++ src/components/taInput/inputController.ts | 2 +- .../textBubble/bubbleDetails/drawer.ts | 50 +++++++ tests/audio.js | 77 +++++++++-- tests/server.js | 6 +- 9 files changed, 342 insertions(+), 58 deletions(-) create mode 100644 gcloud.json create mode 100644 src/components/messageView/index.vue create mode 100644 src/components/textBubble/bubbleDetails/drawer.ts diff --git a/gcloud.json b/gcloud.json new file mode 100644 index 0000000..5c68549 --- /dev/null +++ b/gcloud.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "crimata", + "private_key_id": "3a324fa3fcca4cea589e4b5007f4142db4b3a84d", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCxFJOjhIkYlyhn\nHjtpFC5EOwzNFZem6k6SS7zcgWbNLe9mcBsiTKCjpxItzkIo0pnT32yqw+AKsqpv\n6+LVyzzjfB7vtEUflvg6KgmYMS4ixOTCEbN3e2qj1t2RnITXZR2VGSGdoKKyqZm0\ntWrmexQvSN20cByL1vfxD3E69oYFGhVraa0zmm94WXHAzofT6DkvGdrbK1s1G9jI\nG3mdQ1jN71tghyloeHL+R/1iKQoZFYTkVe6yWvccSNAZelt/5sciCKRTCArXj89X\nLwU6jGOKIsyqx3BeUyWaSS59xpnGX3meSf5Q9U+j45ePUiqO6TTleF+kWiBwjUpp\nfeYcwEIDAgMBAAECggEAElkN9R7p75zV6F1XDYu0QYiWyncmx/o2Gu1zC6vywWa1\ni/korpSe/mX0ub9J1p3/p1balRUHlUQu6brYvYs56140fGTC1sOXQ7uQU+8glySs\niTk5TbOBeKluOsSrdP/6oTTB6Wm4AegVz4YOpgPxsvaLVkNZidnDxfvyIQhjGYsO\nsvRaw48eEmGOa1uvDhgvPOxiDp5YEdxqSue/7ylzjBdKecDyLgTSDDf/p+FwIx5B\n2d/0uRgQBaAk6D/DNOvN8GrYyakFmlDjE72nrBGAJq50UyV6THOSDk+8cK+2QiS1\nEDCFufj+7Dk1EBfBFUFY8ZBB/pphpEWyLoFl7NzGCQKBgQDaj/KOOcj6BkkTj1WM\nRW3KhOE3nhsCQEgQGCflpycEAZj6J7ktBG2/4teDt9sr+Epx8ymnQCZa9pmysdrD\nU0aWku1qUx0IUKTtZbAEbB+AnrWYDYkIWU8kRUUvNHNTBg43VyuohV9wprPih138\nHWMeNac82/XICz2A3rv+Xkx8KwKBgQDPaaEEGWVs3nIhf/p3M3/xWNKt8fhAERpG\n0YpTjgHW3FQFYjiCTTIcuLO3qsDGmfIAB9tL37s2Y3e2wKBGUj/o69Fr7sWVPdsX\nJtUc0+KiG2Bs0hsyZPdk9UQiQX8y2eq1NZW7vEoxWso1bM64WURB5N8ocWePNJJx\nL2Oe75rtiQKBgQDLfQQOiRxmNF3rOSMkAywyRr9NQgXRdbniSiszNQotP7OHDF7q\n29m2suOGfjIv4O6m4wdf8WkEfd4hsleETc9Ft6wVtyYrrLGxWWCk7WnzHVDjLY7s\n2AHIOjostf+9R8EKoz1BnFN8laibev71EQNMiBWZow1VX6m2hymurWs2mwKBgQCb\nuY3n2v14qOb92e1+U89KsEq1yMd/qpeU9jwqEaO14wS+aglNY5ItWEuuqWhFdE3q\n0ftHUzpnUnUOZD+xrI1JXsyEgegc7i0xi7lUBI3S8kUKTxGWW5IXXcKDCbPrxQtg\ndFPweSUnOyg4xnHKnVMPOjyGS+bZ8TnF+zOLoBAtKQKBgF6hAOIwF1DheXT4p8xS\nlSGhl8dBxjsJDVwbnFWQceGzGnq1e3ncy/UItt42UCDVRWTH4r7pQ9eKyd/3foUD\n6KDIcCrYo2KJh6ZNFJ67XF30U+TuEg5U/9jTbWqDOwYVXcMcnzQE4ZiGBxVHL6xr\nNc8FRW5+7xh4zNfzhGUGt5kt\n-----END PRIVATE KEY-----\n", + "client_email": "crimata-service-account@crimata.iam.gserviceaccount.com", + "client_id": "102914612880037864172", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/crimata-service-account%40crimata.iam.gserviceaccount.com" +} diff --git a/src/background/audio.ts b/src/background/audio.ts index 78fcff5..3263582 100644 --- a/src/background/audio.ts +++ b/src/background/audio.ts @@ -1,50 +1,104 @@ -"use strict"; /* eslint @typescript-eslint/no-var-requires: "off" */ + +"use strict"; + import { sendAudio } from './session' const portAudio = require('naudiodon'); -let ia: typeof portAudio.AudioIO; -let audioInput: string[] = []; - -function processAudio(audioInput: Array): Buffer { - let audio = ''; - audioInput.forEach(chunk => audio += chunk); - return Buffer.from(audio, "base64"); +let ai: typeof portAudio.AudioIO; +let ao: typeof portAudio.AudioIO; +let audioInput: Buffer[] = []; +const encoding = "base64"; +const audioOptions = { + channelCount: 1, + sampleFormat: 16, + sampleRate: 16000, + deviceId: -1, + closeOnError: false, } -export const updateRecorder = (_event, record: boolean): void => { - if (record) { - ia.resume(); - } else { - ia.pause(); - const audio = processAudio(audioInput); - sendAudio(audio); - audioInput = []; +// run every time the main function launches. +export const initAudioIO = () => { + // init portAudio readable stream once. + if (!ai) { + ai = new portAudio.AudioIO({ inOptions: audioOptions }); + + // base64 encoding needed for google speech to text. + ai.setEncoding(encoding); + + // pause the portAudio data flow as soon as stream is initiated. + ai.start(); + ai.pause(); + + ai.on('error', (e: Error) => { + console.log('Error recording audio', + e); + }); + ai.on('data', (chunk: string) => { + // turn string base64 data into buffer object. + audioInput.push(Buffer.from(chunk, encoding)); + }); + } + + // init portAudio writable stream once. + if (!ao) { + ao = new portAudio.AudioIO({ outOptions: audioOptions }); + ao.start(); } } -export const initRecorder = () => { - - if (!ia) { - ia = new portAudio.AudioIO({ - inOptions: { - channelCount: 1, - sampleFormat: 16, - sampleRate: 16000, - deviceId: -1, - closeOnError: false, - } - }); - ia.setEncoding('base64'); - ia.start(); - ia.pause(); - ia.on('error', (e: Error) => { - console.log('Error recording audio', + e); - }); - ia.on('data', (chunk: string) => { - audioInput.push(chunk); - }); +// run this function on space bar key up and down. +export const updateRecorder = (_event, record: boolean): void => { + if (record) { + // resume mic data flow on space bar key down. + ai.resume(); + } else { + // stop mic data flow on space bar key up. + ai.pause(); + sendAudio(Buffer.concat(audioInput)); + audioInput = []; } } +// play audio buffers. +export const play = (input: Array) => { + let i = 0; + // format buffers into half their size to account for writable highwaterMark. + const audio = bufSplit(input); + // call this fuction after last audio chunk has been written. + const callback = () => { + // TODO: clear portAudio writable buffer on write end. + } + write(); + // iterate through audio array and write buffers to portAudio writable. + function write() { + let chunk: Buffer; + let ok = true; + do { + chunk = audio[i]; + if (i === audio.length - 1) { + // write last chunk. + ao.write(chunk, null, callback); + } else { + // check for backpreassure. + ok = ao.write(chunk, null); + } + i++; + } while (i < audio.length && ok); + + if (i < audio.length) { + // Had to stop early! + // Write some more once it drains. + ao.once('drain', write); + } + } +} +// utility function used by play func +function bufSplit(input: Array): Array { + let result: Buffer[] = []; + input.forEach((b: Buffer) => { + // split buffer into two. + result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length)); + }); + return result; +} diff --git a/src/background/run.ts b/src/background/run.ts index d0cfde1..a92e941 100644 --- a/src/background/run.ts +++ b/src/background/run.ts @@ -2,7 +2,7 @@ import { createWindow } from './window'; import { initSession } from './session'; -import { initRecorder } from './audio'; +import { initAudioIO } from './audio'; let socket = false; @@ -23,6 +23,6 @@ export async function main() { socket = true; // Begin audio stream. - // initRecorder(); + initAudioIO(); } diff --git a/src/background/session.ts b/src/background/session.ts index 71e91ee..0b4a741 100644 --- a/src/background/session.ts +++ b/src/background/session.ts @@ -5,7 +5,10 @@ import { backgroundMitt } from './emitter'; import { ipcMain } from "electron"; interface RenderMessage { - content: string; + content: { + text: string; + audio: boolean | Buffer; + }; avatar: string; context: string; subContext: string; @@ -24,7 +27,7 @@ interface TextMessage { content: string; } -const ip = 'ws://127.0.0.1'; +const ip = 'ws://127.0. 0.1'; const port = 8760; const reconnectTimeout = 3000; //ms let socket: WebSocket; @@ -67,6 +70,9 @@ const receiveMessage = (message: string): void => { } const parsed: RenderMessage = JSON.parse(message); + console.log('message received', parsed); + if (parsed.content.audio) { + } backgroundMitt.emit('ipc-renderer', { endpoint: 'render-message', message: parsed diff --git a/src/components/messageView/index.vue b/src/components/messageView/index.vue new file mode 100644 index 0000000..e231033 --- /dev/null +++ b/src/components/messageView/index.vue @@ -0,0 +1,111 @@ + + + + + diff --git a/src/components/taInput/inputController.ts b/src/components/taInput/inputController.ts index facbe1f..1f8df0b 100644 --- a/src/components/taInput/inputController.ts +++ b/src/components/taInput/inputController.ts @@ -34,7 +34,7 @@ export default function useInputController() { emitter.emit("animate-send"); // send message to backend - post('message', message); + if (input) post('message', message); } const { diff --git a/src/components/textBubble/bubbleDetails/drawer.ts b/src/components/textBubble/bubbleDetails/drawer.ts new file mode 100644 index 0000000..8343fb6 --- /dev/null +++ b/src/components/textBubble/bubbleDetails/drawer.ts @@ -0,0 +1,50 @@ +import { ref, onMounted } from "vue"; +import { Ref } from "@/types/vueRef/index"; + +/** + * handles visualization of individual svg text bubbles + */ +export default function useBubbleDrawer({ + mr, + tr, +}: { + mr: Ref; + tr: Ref; +}) { + const signatureTranslate = ref(""); + let messageRect: SVGSVGElement & SVGRectElement; + let messageText: SVGSVGElement & SVGTextElement; + + onMounted(async () => { + messageRect = (mr.value as unknown) as SVGSVGElement & SVGRectElement; + messageText = (tr.value as unknown) as SVGSVGElement & SVGTextElement; + }); + + function adjustMessageRect(): void { + // must get dimensions of text box to fit bubble around + const padding = 12.5; + const textBox = messageText.getBBox(); + messageRect.setAttribute("x", String(textBox.x - padding)); + messageRect.setAttribute("y", String(textBox.y - padding)); + messageRect.setAttribute("width", String(textBox.width + 2 * padding)); + messageRect.setAttribute("height", String(textBox.height + 2 * padding)); + } + + function translateMsgSignature(): void { + if (messageRect.getAttribute("height") !== null) { + const height = Number(messageRect.getAttribute("height")); + signatureTranslate.value = `translate(-10 ${height})`; + } + } + + async function drawMessage() { + // ignore lint error: await needed for proper render + await adjustMessageRect(); + translateMsgSignature(); + } + + return { + drawMessage, + signatureTranslate, + }; +} diff --git a/tests/audio.js b/tests/audio.js index 070960d..8fde8eb 100644 --- a/tests/audio.js +++ b/tests/audio.js @@ -50,43 +50,90 @@ const ia = new portAudio.AudioIO({ sampleRate: 16000, deviceId: -1, closeOnError: false, - } + }, }); -ia.setEncoding('base64'); +// ia.setEncoding('base64'); ia.start(); ia.on('error', (e) => { console.log('error recording audio', + e); }); ia.on('data', (chunk) => { - audioInput.push(chunk); - console.log('Got %d characters of string data:', chunk.length); + audioInput.push(chunk); +}); + +const ao = new portAudio.AudioIO({ + outOptions: { + channelCount: 1, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: 16000, + deviceId: -1, // Use -1 or omit the deviceId to select the default device + closeOnError: true // Close the stream if an audio error is detected, if set false then just log the error + } }); +ao.start(); + +const callback = () => { + console.log('yayayay') +} + +const play = (audio) => { + + console.log('audio length', audio.length); + let i = 0; + write(); + + function write() { + let ok = true; + do { + const chunk = audio[i] + console.log(chunk, i) + + i++; + if (i === audio.length - 1) { + // Last time! + ao.write(chunk, null, callback); + } else { + // See if we should continue, or wait. + // Don't pass the callback, because we're not done yet. + ok = ao.write(chunk, null); + } + } while (i < audio.length - 1 && ok); + if (i < audio.length - 1) { + // Had to stop early! + // Write some more once it drains. + ao.on('drain', write); + } + } + +} + async function processAudio() { - audio = ''; - audioInput.forEach(s => { - audio += s; - }) - const buffer = Buffer.from(audio, 'base64'); + + const buf = Buffer.concat(audioInput); + + play(audioInput); + audioInput = []; - await transcribeSpeech({ - content: buffer - }); + + transcribeSpeech({ + content: buf + }); } setTimeout(async () => { ia.pause(); - await processAudio(); + processAudio(); // ia.resume(); }, 3000); setTimeout(() => { ia.resume(); -}, 4000) +}, 5000) setTimeout(async () => { ia.pause(); - await processAudio(); + processAudio(); // ia.resume(); }, 8000); diff --git a/tests/server.js b/tests/server.js index 01358e8..0440047 100644 --- a/tests/server.js +++ b/tests/server.js @@ -9,8 +9,12 @@ wss.on("connection", function connection(ws, req) { console.log(message) if (message === 'no-token' || message === 'undefined') { - console.log('no token to authenticate'); ws.send('success'); + } else if (message instanceof Buffer) { + // audio message + + } else { + const parsed = JSON.parse(message); } }); -- 2.43.0