From 5671c8854adc31bfe62419141ba9af711a20a206 Mon Sep 17 00:00:00 2001 From: Andrew Gundersen Date: Wed, 26 May 2021 11:56:33 -0500 Subject: [PATCH] proof concept new audio interface --- src/background/audio.ts | 211 +++++----------------------------- src/background/init.ts | 4 +- src/background/initAudio.ts | 18 --- src/background/session.ts | 12 +- src/modules/audio.ts | 129 --------------------- src/modules/electron-audio.ts | 156 +++++++++++++++++++++++++ 6 files changed, 190 insertions(+), 340 deletions(-) delete mode 100644 src/background/initAudio.ts delete mode 100644 src/modules/audio.ts create mode 100644 src/modules/electron-audio.ts diff --git a/src/background/audio.ts b/src/background/audio.ts index 97a1576..b96f510 100644 --- a/src/background/audio.ts +++ b/src/background/audio.ts @@ -1,196 +1,37 @@ -/* eslint @typescript-eslint/no-var-requires: "off" */ - -"use strict"; - import { ipcMain } from "electron"; -import { backgroundMitt } from '@/modules/emitter'; - -const portAudio = require('naudiodon'); - -// Audio in and out stream objects. -let ai: typeof portAudio.AudioIO | boolean = false; -let ao: typeof portAudio.AudioIO | boolean = false; - -// Whether activly recording. -let record = false; +import electronAudio from "@/modules/electron-audio"; -const audioContainer = { - input: '', -} +export default function initAudioIO () { -const audioOptions = { - channelCount: 1, - sampleRate: 44100, - sampleFormat: portAudio.SampleFormat16Bit, - deviceId: -1, - closeOnError: false -} + console.log('[AUDIO]initializing audio'); -// Toggles record to true to begin capturing chunks. -const onRecordingStart = (_event: any, _payload: any) => { - console.log("AUDIO: Beginning audio capture.") - record = true; -} + // initialize the audio interface + const { + read, + stopRead, + write, + shutdown + } = electronAudio(); -// Returns recorded audio to frontend and sets record to false. -const onRecordingEnd = async (_event: any, payload: any) => { - console.log("AUDIO:Sending audio to browser.") - - return new Promise((resolve, reject) => { - - try { - console.log("Resolving audio"); - console.log(audioContainer.input); - resolve(audioContainer.input); - record = false; - } catch (e) { - reject() - } + // event driven controls + // begin capuring audio from input stream + ipcMain.on("start-recording", () => { + console.log("[AUDIO]Reading..."); + read(); }); -}; - - -// Main audio function run by run.ts module. -export function initAudioIO(): void { - console.log("AUDIO:Starting io streams.") - - if (!ai) { - - // Initialize and start input stream. - ai = new portAudio.AudioIO({ inOptions: audioOptions }); - ai.setEncoding("hex"); - ai.start(); - - // On each data chunk... - ai.on('data', (chunk: string) => { - - // If recording, we capture the data. - if (record) { - console.log('AUDIO:Recording...') - audioContainer.input += chunk; - } - - // Else, we don't capture and also clear audioContainer. - else { - if (audioContainer.input.length) { - audioContainer.input = ""; - } - } - - }); - } - - if (!ao) { - - // Initialize and start input stream. - ao = new portAudio.AudioIO({ outOptions: audioOptions }); - ao.start(); - - } - - // Listen to record. - console.log("AUDIO:Adding recording listeners.") - - ipcMain.removeAllListeners("start-recording"); - ipcMain.on("start-recording", onRecordingStart); - ipcMain.removeHandler("stop-recording"); - ipcMain.handle("stop-recording", onRecordingEnd); -} + // write a string of audio to the output stream + ipcMain.on("write-audio", (_e: any, audio: string) => write(audio)); + // close the audio streams + ipcMain.on("shutdown-audio", shutdown); -// ---Audio playback-------------------------------------------- - -// Split Buffer into an array of len-sized Buffers. -function bufSplit(buf: Buffer, len: number): Array { - const chunks = []; - let i = 0; - let L = len; - - while(i < buf.byteLength) { - chunks.push(buf.slice(i, L)); - i = L; - L += len; - } - - return chunks; -} - -// Audio playback. -export function play(input: string): void { - - // Format the audio. - const audio = bufSplit( - Buffer.from(input as string, 'hex'), - 8192 - ); - - // Called on end of write. - const callback = () => { - - // We stop audio playback anim. - backgroundMitt.emit('ipc-renderer', { - endpoint: 'stop-playback-anim' - }); - - } - - write(); - - // Iterate through audio array and write buffers to portAudio writable. - function write() { - let chunk: Buffer; - let ok = true; - let i = 0; - - do { - - chunk = audio[i]; - - // On last chunk - if (i === audio.length - 1) { - ao.write(chunk, null, callback); - } - - else { - // check for backpreassure. - ok = ao.write(chunk, null); - } - - i++; - - } while (i < audio.length && ok); - - if (i < audio.length) { - // Had to stop early! - // Write some more once it drains. - ao.once('drain', write); - } - } -} - -// ------------------------------------------------------------- - -// Get's called on window close. -export async function stopStream() { - console.log("AUDIO:Stopping audio stream.") - if (ai) { - try { - await ai.quit() - } catch(e){ - console.log('AUDIO: Failed to shutdown audio input.'); - throw e; - } - } - if (ao) { - try { - await ao.quit() - } catch(e){ - console.log('AUDIO: Failed to shutdown audio output.'); - throw e; - } - } -} - + // stop capturing audio and return it to caller + ipcMain.handle("stop-recording", () => { + console.log("[AUDIO]Stopping read..."); + const audio = stopRead(); + return audio; + }); +} \ No newline at end of file diff --git a/src/background/init.ts b/src/background/init.ts index b206393..fe8fb87 100644 --- a/src/background/init.ts +++ b/src/background/init.ts @@ -4,7 +4,7 @@ import { app, dialog } from "electron"; import { createWindow } from './window'; import { initSession } from './session'; -import initAudioIO from './initAudio'; +import initAudioIO from './audio'; import { backgroundMitt } from '@/modules/emitter'; let win: boolean; @@ -64,7 +64,7 @@ export function initApp(dev: boolean): void { // Must keep to ensure app doesn't quit on close. app.on("before-quit", async () => { - // await stopStream(); + await stopStream(); }); // Must keep to ensure app doesn't quit on close. diff --git a/src/background/initAudio.ts b/src/background/initAudio.ts deleted file mode 100644 index 27cc11b..0000000 --- a/src/background/initAudio.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { ipcMain } from "electron"; -import audioInterface from "@/modules/audio"; - -export default function initAudioIO () { - console.log('[AUDIO]initializing audio'); - - // initialize the audio interface - const { read, stopRead, write, shutdown } = audioInterface(); - - ipcMain.on("start-recording", read); - - ipcMain.on("shutdown-audio", shutdown); - - ipcMain.handle("stop-recording", () => { - return stopRead(); - }); - -} \ No newline at end of file diff --git a/src/background/session.ts b/src/background/session.ts index 89a8802..633050a 100644 --- a/src/background/session.ts +++ b/src/background/session.ts @@ -126,12 +126,12 @@ const { createSocket, sendMessage } = useWebSockets(onMessage, onOpen); // Handle messages from window/client. const onClientMessage = (_event: IpcMainEvent, payload: any) => { - console.log("New client message") + console.log("New client message"); - const success = sendMessage(payload) + const success = sendMessage(payload); if (!success) { - console.log("Unable to send message: ", payload) + console.log("Unable to send message: ", payload); } } @@ -150,14 +150,14 @@ export const initSession = (dev: boolean) => { // Open socket connection. let url = "ws://crimata.com:8760"; if (dev) url = "ws://localhost:8760"; - createSocket(url) + createSocket(url); // Attack browser window init listener. - ipcMain.removeAllListeners("app-mounted") + ipcMain.removeAllListeners("app-mounted"); ipcMain.on("app-mounted", onNewBrowserWindow); // Attach listeners for frontend. - ipcMain.removeAllListeners("client-message") + ipcMain.removeAllListeners("client-message"); ipcMain.on("client-message", onClientMessage); // Keep win up-to-date. diff --git a/src/modules/audio.ts b/src/modules/audio.ts deleted file mode 100644 index 853243a..0000000 --- a/src/modules/audio.ts +++ /dev/null @@ -1,129 +0,0 @@ -const portAudio = require('naudiodon'); -const stream = require('stream'); -const fs = require('fs'); - - -// current config -let currentHost: number; -let currentInputDevice: number; -let currentOutputDevice: number; - -// audio streams -let ai: any; -let ao: any; - -// write audio here -const path = 'rawAudio.raw' -let writable = fs.createWriteStream(path); - -// get info for given host api -const getHostInfo = (id: number) => { - return portAudio.getHostAPIs().HostAPIs.filter(host => - host.id === id)[0]; -} - -// get info for given device -const getDeviceInfo = (id: number) => { - return portAudio.getDevices().filter(device => - device.id === id)[0]; -}; - -// constructor to start audio streams -const setup = () => { - - currentHost = portAudio.getHostAPIs().defaultHostAPI; - const hostInfo = getHostInfo(currentHost); - - if (currentInputDevice !== hostInfo.defaultInput) { - currentInputDevice = hostInfo.defaultInput; - ai = setInputStream(); - ai.start(); - } - - if (currentOutputDevice !== hostInfo.defaultOutput) { - currentOutputDevice = hostInfo.defaultOutput; - ao = setOutputStream() - ao.start(); - } - -}; - -// init and start input stream -const setInputStream = () => { - - const inputDeviceInfo = getDeviceInfo(currentInputDevice); - - return new portAudio.AudioIO({ - inOptions: { - channelCount: inputDeviceInfo.maxInputChannels, - sampleFormat: portAudio.SampleFormat16Bit, - sampleRate: inputDeviceInfo.defaultSampleRate, - deviceId: inputDeviceInfo.id, - closeOnError: false - } - }) - .on('error', () => console.log("input stream error")) - -} - -// init and start output stream -const setOutputStream = () => { - - const outputDeviceInfo = getDeviceInfo(currentOutputDevice); - - return new portAudio.AudioIO({ - outOptions: { - channelCount: outputDeviceInfo.maxOutputChannels, - sampleFormat: portAudio.SampleFormat16Bit, - sampleRate: outputDeviceInfo.defaultSampleRate, - deviceId: outputDeviceInfo.id, - closeOnError: false - } - }) - .on('error', () => console.log("output stream error")) - -} - -// pipe ai to writable -const read = () => { - if (ai) { - ai.pipe(writable); - } -}; - -// remove pipe and return audio contents -const stopRead = () => { - if (ai) { - ai.unpipe(writable); - return fs.readFileSync(path); - }; -}; - -// write a buffer of audio to stream -const write = (chunks: Buffer) => { - const readable = new stream.Readable() - readable.pipe(ao); - chunks.forEach(item => readable.push(item)) -}; - -// quit both streams -const shutdown = () => { - if (ao) ao.quit(); - if (ai) ai.quit(); -}; - - -// run in loop and return controls -export default function audioInterface () { - setInterval(setup, 2000); - return { - read, - stopRead, - write, - shutdown - } -} - - - - diff --git a/src/modules/electron-audio.ts b/src/modules/electron-audio.ts new file mode 100644 index 0000000..37fd5f2 --- /dev/null +++ b/src/modules/electron-audio.ts @@ -0,0 +1,156 @@ +const portAudio = require('naudiodon'); +const stream = require('stream'); +const fs = require('fs'); + + +// split Buffer into an array of len-sized Buffers +const bufSplit = (buf: Buffer, len: number) => { + const chunks = []; + let i = 0; + let L = len; + + while(i < buf.byteLength) { + chunks.push(buf.slice(i, L)); + i = L; + L += len; + } + + return chunks; +} + + +export default function electronAudio () { + + // current config + let currentHost: number; + let currentInputDevice: number; + let currentOutputDevice: number; + + // audio streams + let ai: any; + let ao: any; + + // write audio here + let record = false; + let readAudio = ''; + + // get info for given host api + const getHostInfo = (id: number) => { + return portAudio.getHostAPIs().HostAPIs.filter((host: any) => + host.id === id)[0]; + } + + // get info for given device + const getDeviceInfo = (id: number) => { + return portAudio.getDevices().filter((device: any) => + device.id === id)[0]; + }; + + // constructor to start audio streams + const setup = () => { + + currentHost = portAudio.getHostAPIs().defaultHostAPI; + const hostInfo = getHostInfo(currentHost); + + if (currentInputDevice !== hostInfo.defaultInput) { + currentInputDevice = hostInfo.defaultInput; + ai = setInputStream(); + ai.start(); + } + + if (currentOutputDevice !== hostInfo.defaultOutput) { + currentOutputDevice = hostInfo.defaultOutput; + ao = setOutputStream() + ao.start(); + } + + }; + + // init and start input stream + const setInputStream = () => { + + const inputDeviceInfo = getDeviceInfo(currentInputDevice); + console.log(inputDeviceInfo); + + return new portAudio.AudioIO({ + inOptions: { + channelCount: inputDeviceInfo.maxInputChannels, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: inputDeviceInfo.defaultSampleRate, + deviceId: inputDeviceInfo.id, + closeOnError: false + } + }) + .setEncoding("hex") + .on('error', () => console.log("input stream error")) + .on('data', (chunk: Buffer) => { + if (record) readAudio += chunk.toString(); + }); + + } + + // init and start output stream + const setOutputStream = () => { + + const outputDeviceInfo = getDeviceInfo(currentOutputDevice); + console.log(outputDeviceInfo); + + return new portAudio.AudioIO({ + outOptions: { + channelCount: outputDeviceInfo.maxOutputChannels, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: outputDeviceInfo.defaultSampleRate, + deviceId: outputDeviceInfo.id, + closeOnError: false + } + }) + .on('error', () => console.log("output stream error")) + + } + + // pipe ai to writable + const read = () => { + record = true; + }; + + // remove pipe and return audio contents + const stopRead = () => { + const audio = readAudio; + record = false; + readAudio = ''; + return audio; + }; + + // write a string of audio to stream + const write = (audio: string) => { + + // convert audio into chunks of buffers + const chunks = bufSplit( + Buffer.from(audio, 'hex'), + 8192 // chunk size + ); + + // write the chunks + const readable = new stream.Readable() + readable.pipe(ao); + chunks.forEach(chunk => readable.push(chunk)) + + }; + + // quit both streams + const shutdown = () => { + if (ao) ao.quit(); + if (ai) ai.quit(); + }; + + // run + setInterval(setup, 2000); + + return { + read, + stopRead, + write, + shutdown + }; + +} \ No newline at end of file -- 2.43.0