From 31e097b27fbf742dc90c68582a67d11adcb296af Mon Sep 17 00:00:00 2001 From: riqo Date: Thu, 4 Mar 2021 16:56:59 -0600 Subject: [PATCH] add audio playback + buf split function --- src/background/audio.ts | 27 +++++++++----- src/background/session.ts | 9 +++-- tests/audio.js | 75 ++++++++++++++++++++++----------------- 3 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/background/audio.ts b/src/background/audio.ts index 124811d..313f6f1 100644 --- a/src/background/audio.ts +++ b/src/background/audio.ts @@ -34,13 +34,18 @@ ipcMain.removeAllListeners('update-recorder'); ipcMain.on('update-recorder', updateRecorder); // utility function used by play func. -function bufSplit(input: Array): Array { - const result: Buffer[] = []; - input.forEach((b: Buffer) => { - // split buffer into two. - result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length)); - }); - return result; +function bufSplit(buf: Buffer, len: number): Array { + let chunks = []; + let i = 0; + let L = len; + + while(i < buf.byteLength) { + chunks.push(buf.slice(i, L)); + i = L; + L += len; + } + + return chunks; } // main audio function run by run.ts module. @@ -55,6 +60,7 @@ export function initAudioIO(): void { ai.on('data', (chunk: string) => { if (record) { + console.log('recording...') audioContainer.input += chunk; } else { if (audioContainer.input.length) audioContainer.input = ""; @@ -69,10 +75,13 @@ export function initAudioIO(): void { } // play audio buffers. -export function play(input: Array): void { +export function play(input: string): void { let i = 0; + console.log(Buffer.from(input, encoding)); // format buffers into half their size to account for writable highwaterMark. - const audio = bufSplit(input); + const audio = bufSplit(Buffer.from(input), 8192); + console.log(audio); + // call this fuction after last audio chunk has been written. const callback = () => { // TODO: clear portAudio writable buffer on write end. diff --git a/src/background/session.ts b/src/background/session.ts index dd5597c..bd42b34 100644 --- a/src/background/session.ts +++ b/src/background/session.ts @@ -3,11 +3,12 @@ import WebSocket from 'ws'; import { backgroundMitt } from './emitter'; import { ipcMain } from "electron"; +import { play } from './audio'; interface RenderMessage { content: { text: string; - audio: boolean | Buffer; + audio: string; }; context: string; modifier: string; @@ -67,8 +68,10 @@ const receiveMessage = (message: string): void => { } const parsed: RenderMessage = JSON.parse(message); - console.log('message received', parsed); + console.log('message received', parsed.content.text); if (parsed.content.audio) { + console.log('audio received'); + play(parsed.content.audio); } backgroundMitt.emit('ipc-renderer', { endpoint: 'render-message', @@ -78,10 +81,12 @@ const receiveMessage = (message: string): void => { }; const sendMessage = (_event, payload: TextMessage): void => { + console.log('sending message') socket.send(JSON.stringify(payload)); } export const sendAudio = (content: Buffer) => { + console.log('sending audio') socket.send(content); } diff --git a/tests/audio.js b/tests/audio.js index b3008ff..d0f3595 100644 --- a/tests/audio.js +++ b/tests/audio.js @@ -60,9 +60,8 @@ ia.setEncoding('base64'); ia.start(); ia.on('data', (chunk) => { if (record) { - console.log('recording data') + console.log('recording data'); audioContainer.input += chunk; - // audioContainer.buffers.push(Buffer.from(chunk, 'base64')); } else { if (audioContainer.input.length) audioContainer.input = ""; } @@ -83,22 +82,32 @@ let counter = 0; const tests = []; function testCallback() { - tests.forEach(t => console.log(t)) - console.log(tests[0]) play(tests[0]) } -function play(bufArray) { +function splitArrayIntoChunksOfLen(buf, len) { + let chunks = []; + let i = 0; + let L = len; + console.log(buf.byteLength) + while(i < buf.byteLength) { + chunks.push(buf.slice(i, L)); + i = L; + L += len; + } + + chunks.forEach(c => console.log(c)) + return chunks; +} + +function play(input) { let i = 0; + const buf = Buffer.from(input, 'base64'); // format buffers into half their size to account for writable highwaterMark. - const audio = bufSplit(bufArray); + const audio = splitArrayIntoChunksOfLen(buf, 8192); // call this fuction after last audio chunk has been written. const callback = () => { // TODO: clear portAudio writable buffer on write end. - console.log('Finished writing test number %d', counter) - if (counter === 3) { - console.log('finished test writing!') - } } write(); // iterate through audio array and write buffers to portAudio writable. @@ -137,7 +146,7 @@ async function test() { transcribeSpeech({ content: Buffer.from(audioContainer.input, 'base64') }); - tests.push(audioContainer.buffers) + tests.push(audioContainer.input) counter++; console.log('audio string length:', audioContainer.input.length) console.log('buffers written: ', audioContainer.buffers.length) @@ -146,28 +155,28 @@ async function test() { setTimeout(async () => { record = false; test() -}, 3000); - -setTimeout(() => { - record = true; -}, 6000) - -setTimeout(async () => { - record = false; - test(); -}, 9000); - -setTimeout(() => { - record = true; -}, 11000) - -setTimeout(async () => { - record = false; - test(); -}, 14000); - +}, 4000); + +// setTimeout(() => { +// record = true; +// }, 6000) +// +// setTimeout(async () => { +// record = false; +// test(); +// }, 9000); +// +// setTimeout(() => { +// record = true; +// }, 11000) +// +// setTimeout(async () => { +// record = false; +// test(); +// }, 14000); +// setTimeout(async () => { ia.quit(); - // testCallback() + testCallback() return; -}, 16000); +}, 6000); -- 2.43.0