From 37764bb3a0fa496e8e25c660de8720c3af3e5d60 Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Thu, 4 Feb 2021 12:34:33 -0600 Subject: [PATCH 1/4] update recorder inputs --- crash.log | 4 ++++ src/background.ts | 16 +++++++------ src/background/recorder.ts | 47 ++++++++++++++++++++------------------ 3 files changed, 38 insertions(+), 29 deletions(-) create mode 100644 crash.log diff --git a/crash.log b/crash.log new file mode 100644 index 0000000..cfecd62 --- /dev/null +++ b/crash.log @@ -0,0 +1,4 @@ +PID 335844 received SIGSEGV for address: 0xc +/home/enrique/Developer/electron-app/node_modules/segfault-handler/build/Release/segfault-handler.node(+0x37a5)[0x7f1c071387a5] +/lib/x86_64-linux-gnu/libpthread.so.0(+0x153c0)[0x7f1c10ffa3c0] +/home/enrique/Developer/electron-app/node_modules/electron/dist/electron dist_electron(+0x72d9550)[0x55ccd61fe550] diff --git a/src/background.ts b/src/background.ts index d3cecd9..4cae5f4 100644 --- a/src/background.ts +++ b/src/background.ts @@ -7,17 +7,19 @@ var portAudio = require('naudiodon'); // NOTE Program Begins Here (async () => { + console.log(portAudio.getDevices()) const isDevelopment = process.env.NODE_ENV !== "production"; - const recorder = new Recorder({ - channelCount: 1, - sampleFormat: portAudio.SampleFormat16Bit, - sampleRate: 16000, - deviceId: -1, // Use -1 or omit the deviceId to select the default device - closeOnError: true - }, 'binary'); try { + + const recorder = new Recorder({ + channelCount: 1, + sampleFormat: 16, + sampleRate: 16000, + deviceId: 24, // Use -1 or omit the deviceId to select the default device + closeOnError: true + }, 'binary'); // await authenticate method initApp(isDevelopment); } catch (e) { diff --git a/src/background/recorder.ts b/src/background/recorder.ts index 36f93ba..d078585 100644 --- a/src/background/recorder.ts +++ b/src/background/recorder.ts @@ -11,6 +11,14 @@ interface RecorderI { pause(): void; } +type inputOptions = { + channelCount: number, + sampleFormat: number, + sampleRate: number, + deviceId: number, + closeOnError: boolean +} + class StringWritable extends Writable { constructor(options: { @@ -37,15 +45,6 @@ class StringWritable extends Writable { } } -type inputOptions = {clear - - channelCount: number, - sampleFormat: string, - sampleRate: number, - deviceId: number, - closeOnError: boolean -} - export class Recorder implements RecorderI { private ia: any; @@ -93,21 +92,25 @@ export class Recorder implements RecorderI { private _update = (Event: any, record: boolean) => { if (record) { - console.log('recording'); - this.micWritable.data = ''; - if(this.ia._readableState.paused){ - this.resume(); - return; - } - this.ia.pipe(this.micWritable); + // check for stream status + console.log('recording') - console.log(this.ia._readableState.paused); + + // this.ia.pipe(this.micWritable); } else { - console.log('stop recording'); - this.ia.unpipe(this.micWritable); - this.micWritable.on('finish', () => { - console.log('hello'); - }) + console.log('stop recording') + // unpipe + // this.ia.unpipe(this.micWritable); + + // stop stream + // this.ia.quit(); + // this.ia = null; + + // get stream data + // this.micWritable.on('finish', () => { + // console.log('finished writing data'); + // }) + // this.restart(); } } } From a534964b968203156a9d81e1bf5aa184bfb3409f Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Thu, 4 Feb 2021 13:15:04 -0600 Subject: [PATCH 2/4] add stream restart to recorder --- src/background/recorder.ts | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/background/recorder.ts b/src/background/recorder.ts index d078585..d1fd614 100644 --- a/src/background/recorder.ts +++ b/src/background/recorder.ts @@ -12,11 +12,11 @@ interface RecorderI { } type inputOptions = { - channelCount: number, - sampleFormat: number, - sampleRate: number, - deviceId: number, - closeOnError: boolean + channelCount: number; + sampleFormat: number; + sampleRate: number; + deviceId: number; + closeOnError: boolean; } class StringWritable extends Writable { @@ -55,9 +55,13 @@ export class Recorder implements RecorderI { this.recorderOptions = options; this.micWritable = new StringWritable({ defaultEncoding: encoding - }) - this.restart(); - ipcMain.on('update-recorder', this._update); + }); + try { + this.restart(); + ipcMain.on('update-recorder', this._update); + } catch(e) { + console.log('MEGA ERROR!'); + } } restart() { @@ -94,23 +98,22 @@ export class Recorder implements RecorderI { if (record) { // check for stream status console.log('recording') - - + this.resume(); // this.ia.pipe(this.micWritable); } else { console.log('stop recording') // unpipe - // this.ia.unpipe(this.micWritable); + this.ia.unpipe(this.micWritable); // stop stream - // this.ia.quit(); - // this.ia = null; + this.ia.quit(); + this.ia = null; // get stream data - // this.micWritable.on('finish', () => { - // console.log('finished writing data'); - // }) - // this.restart(); + this.micWritable.on('finish', () => { + console.log('finished writing data'); + }) + this.restart(); } } } From 9143689c90986c71a77c564cfcf33114a5943fad Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Thu, 4 Feb 2021 16:36:50 -0600 Subject: [PATCH 3/4] add main function for window --- src/background/initApp.ts | 2 -- src/background/main.ts | 39 +++++++++++++++++++++++++++++++++++ src/background/recorder.ts | 42 ++++++++------------------------------ 3 files changed, 47 insertions(+), 36 deletions(-) create mode 100644 src/background/main.ts diff --git a/src/background/initApp.ts b/src/background/initApp.ts index e0ea203..01910c5 100644 --- a/src/background/initApp.ts +++ b/src/background/initApp.ts @@ -26,8 +26,6 @@ function createWindow() { if (process.env.WEBPACK_DEV_SERVER_URL) { // Load the url of the dev server if in development mode win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); - // NOTE uncomment for dev tools on app launch - if (!process.env.IS_TEST) win.webContents.openDevTools(); } else { createProtocol("app"); // Load the index.html when not in development diff --git a/src/background/main.ts b/src/background/main.ts new file mode 100644 index 0000000..b36af61 --- /dev/null +++ b/src/background/main.ts @@ -0,0 +1,39 @@ +import {Recorder} from './recorder'; +import { ipcMain } from "electron"; +/* + * this code will be run after window has finished loading + */ +export function main() { + // connect to crimata-platform ws server + + // start recorder + const recorder = new Recorder({ + channelCount: 1, + sampleFormat: 16, + sampleRate: 16000, + deviceId: -1, + closeOnError: false, + }, 'binary'); + + const updateRecorder = (Event: any, record: boolean) => { + if (record) { + console.log('recording') + recorder.resume(); + } else { + console.log('stop recording') + recorder.pause(); + const data = recorder.getData(); + // TODO: send data() + } + } + + const sendMessage = (Event: any, payload: any) => { + + } + + // event handlers + ipcMain.on('update-recorder', updateRecorder); + ipcMain.on('send-message', updateRecorder); + // send message + +} diff --git a/src/background/recorder.ts b/src/background/recorder.ts index d1fd614..cc4570a 100644 --- a/src/background/recorder.ts +++ b/src/background/recorder.ts @@ -57,18 +57,19 @@ export class Recorder implements RecorderI { defaultEncoding: encoding }); try { - this.restart(); - ipcMain.on('update-recorder', this._update); + this.start(); } catch(e) { console.log('MEGA ERROR!'); } } - restart() { + private start(): void { this.ia = new portAudio.AudioIO({ inOptions: this.recorderOptions }); - this._start(); + this.ia.pipe(this.micWritable); + this.ia.start(); + this.pause(); } pause(): void { @@ -80,40 +81,13 @@ export class Recorder implements RecorderI { } getData(): string { - return this.micWritable.data; + const data = this.micWritable.data; + this.micWritable.data = ''; + return data; } stop(): string { this.ia.quit(); return this.getData(); } - - private _start(): void { - this.ia.pipe(this.micWritable); - this.ia.start(); - this.pause(); - } - - private _update = (Event: any, record: boolean) => { - if (record) { - // check for stream status - console.log('recording') - this.resume(); - // this.ia.pipe(this.micWritable); - } else { - console.log('stop recording') - // unpipe - this.ia.unpipe(this.micWritable); - - // stop stream - this.ia.quit(); - this.ia = null; - - // get stream data - this.micWritable.on('finish', () => { - console.log('finished writing data'); - }) - this.restart(); - } - } } From 29dc7de3e403603915617962eae5c682e3955cfd Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Thu, 4 Feb 2021 20:23:34 -0600 Subject: [PATCH 4/4] add window emitters and stuff --- src/background.ts | 40 +++++++--------- src/background/createWindow.ts | 38 +++++++++++++++ src/background/initApp.ts | 80 +++++++------------------------ src/background/main.ts | 39 ---------------- src/background/recorder.ts | 3 +- src/background/run.ts | 83 +++++++++++++++++++++++++++++++++ src/background/windowEmitter.ts | 5 ++ 7 files changed, 161 insertions(+), 127 deletions(-) create mode 100644 src/background/createWindow.ts delete mode 100644 src/background/main.ts create mode 100644 src/background/run.ts create mode 100644 src/background/windowEmitter.ts diff --git a/src/background.ts b/src/background.ts index 4cae5f4..5b5ceec 100644 --- a/src/background.ts +++ b/src/background.ts @@ -1,29 +1,25 @@ -"use strict"; -import {initApp} from './background/initApp'; -import {Recorder} from './background/recorder'; -var portAudio = require('naudiodon'); +/* + * Entry point for Crimata electron app. + * "Look on my Works, ye Mighty, and despair!" + */ +"use strict"; + +import { protocol } from "electron"; +import { initApp } from './background/initApp'; + +// Scheme must be registered before the app is ready +protocol.registerSchemesAsPrivileged([ + { scheme: "app", privileges: { secure: true, standard: true } } +]); + +// Load environment variable +const isDevelopment = process.env.NODE_ENV !== "production"; // NOTE Program Begins Here (async () => { - console.log(portAudio.getDevices()) - const isDevelopment = process.env.NODE_ENV !== "production"; - - - try { - - const recorder = new Recorder({ - channelCount: 1, - sampleFormat: 16, - sampleRate: 16000, - deviceId: 24, // Use -1 or omit the deviceId to select the default device - closeOnError: true - }, 'binary'); - // await authenticate method - initApp(isDevelopment); - } catch (e) { - console.log("Error", e); - } + console.log('Starting Crimata electron app.'); + initApp(isDevelopment); })(); diff --git a/src/background/createWindow.ts b/src/background/createWindow.ts new file mode 100644 index 0000000..02aadf9 --- /dev/null +++ b/src/background/createWindow.ts @@ -0,0 +1,38 @@ +"use strict"; + +import { BrowserWindow } from "electron"; +import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; +import * as path from "path"; + +export function createWindow(options: { + width: number; + height: number; + resizable: boolean; +}): BrowserWindow { + + const win: BrowserWindow = new BrowserWindow({ + width: options.width, + height: options.height, + resizable: options.resizable, + webPreferences: { + // Use pluginOptions.nodeIntegration, leave this alone + // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info + nodeIntegration: (process.env + .ELECTRON_NODE_INTEGRATION as unknown) as boolean, + preload: path.join(__dirname, "preload.js") + } + }); + + if (process.env.WEBPACK_DEV_SERVER_URL) { + // Load the url of the dev server if in development mode + win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); + } else { + createProtocol("app"); + // Load the index.html when not in development + win.loadURL("app://./index.html"); + } + + return win; +} + + diff --git a/src/background/initApp.ts b/src/background/initApp.ts index 01910c5..40de858 100644 --- a/src/background/initApp.ts +++ b/src/background/initApp.ts @@ -1,87 +1,39 @@ + "use strict"; -import { app, protocol, BrowserWindow, ipcMain } from "electron"; -import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; -import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer"; -import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; -import * as path from "path"; +import { app } from "electron"; +import { main } from './run'; +import { windowEmitter } from './windowEmitter'; -var win: BrowserWindow | null; +export function initApp(dev: boolean): void { -function createWindow() { - // Create the browser window. - win = new BrowserWindow({ - width: 450, - height: 1025, - resizable: true, - webPreferences: { - // Use pluginOptions.nodeIntegration, leave this alone - // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info - nodeIntegration: (process.env - .ELECTRON_NODE_INTEGRATION as unknown) as boolean, - preload: path.join(__dirname, "preload.js") - } - }); + let winActive: boolean; - if (process.env.WEBPACK_DEV_SERVER_URL) { - // Load the url of the dev server if in development mode - win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); - } else { - createProtocol("app"); - // Load the index.html when not in development - win.loadURL("app://./index.html"); - } + windowEmitter.on('window-active', (state: boolean) => { + winActive = state; + }); - win.webContents.on('did-finish-load', () => { - // do stuff once window has finished loading - }) - - win.on("closed", () => { - win = null; - }); -} - -export function initApp(isDev: boolean){ - - // Scheme must be registered before the app is ready - protocol.registerSchemesAsPrivileged([ - { scheme: "app", privileges: { secure: true, standard: true } } - ]); - - // This method will be called when Electron has finished - // initialization and is ready to create browser windows. - // Some APIs can only be used after this event occurs. - app.on("ready", async () => { - if (isDev && !process.env.IS_TEST) { - // Install Vue Devtools - try { - await installExtension(VUEJS_DEVTOOLS); - } catch (e) { - console.error("Vue Devtools failed to install:", e.toString()); - } - } - createWindow(); + app.on("ready", () => { + main(); }); // Quit when all windows are closed. app.on("window-all-closed", () => { - // On macOS it is common for applications and their menu bar - // to stay active until the user quits explicitly with Cmd + Q if (process.platform !== "darwin") { app.quit(); } }); + // TODO: Debug activate functionaity: currently not working. + // Might have to de-reference window object app.on("activate", () => { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (win === null) { - createWindow(); + if (winActive === false) { + main(); } }); // Exit cleanly on request from parent process in development mode. - if (isDev) { + if (dev) { if (process.platform === "win32") { process.on("message", data => { if (data === "graceful-exit") { diff --git a/src/background/main.ts b/src/background/main.ts deleted file mode 100644 index b36af61..0000000 --- a/src/background/main.ts +++ /dev/null @@ -1,39 +0,0 @@ -import {Recorder} from './recorder'; -import { ipcMain } from "electron"; -/* - * this code will be run after window has finished loading - */ -export function main() { - // connect to crimata-platform ws server - - // start recorder - const recorder = new Recorder({ - channelCount: 1, - sampleFormat: 16, - sampleRate: 16000, - deviceId: -1, - closeOnError: false, - }, 'binary'); - - const updateRecorder = (Event: any, record: boolean) => { - if (record) { - console.log('recording') - recorder.resume(); - } else { - console.log('stop recording') - recorder.pause(); - const data = recorder.getData(); - // TODO: send data() - } - } - - const sendMessage = (Event: any, payload: any) => { - - } - - // event handlers - ipcMain.on('update-recorder', updateRecorder); - ipcMain.on('send-message', updateRecorder); - // send message - -} diff --git a/src/background/recorder.ts b/src/background/recorder.ts index cc4570a..4b741d6 100644 --- a/src/background/recorder.ts +++ b/src/background/recorder.ts @@ -4,7 +4,6 @@ const portAudio = require('naudiodon'); const { Writable } = require('stream'); // eslint-disable-next-line @typescript-eslint/no-var-requires const { StringDecoder } = require('string_decoder'); -import { ipcMain } from "electron"; interface RecorderI { stop(): string; @@ -59,7 +58,7 @@ export class Recorder implements RecorderI { try { this.start(); } catch(e) { - console.log('MEGA ERROR!'); + console.log('RECORDER ERROR!', e); } } diff --git a/src/background/run.ts b/src/background/run.ts new file mode 100644 index 0000000..63ad873 --- /dev/null +++ b/src/background/run.ts @@ -0,0 +1,83 @@ +import { createWindow } from './createWindow'; +import WebSocket from 'ws'; +import { Recorder } from './recorder'; +import { ipcMain, BrowserWindow } from "electron"; +import { windowEmitter } from './windowEmitter'; + +/* + * The main function will be run after electron app is ready. + */ +export function main(): void { + + // create main window. + let win: BrowserWindow | null; + win = createWindow({ width: 350, height: 525, resizable: false }); + + // Instantiate ws session with crimata-platorm. + const ws = new WebSocket('ws://localhost:8080'); + + // Instantiate portAudio recorder. + const recorder = new Recorder({ + channelCount: 1, + sampleFormat: 16, + sampleRate: 16000, + deviceId: -1, + closeOnError: false, + }, 'binary'); + + const initSession = (): void => { + ws.send('hernandeze2@xavier.edu'); + } + + const sendMessage = (Event: any, payload: any): void => { + ws.send(JSON.stringify(payload)); + } + + const sendAudio = (data: string): void => { + ws.send(data); + } + + const receiveMessage = (message: string): void => { + // NOTE assuming message is JSON string + const parsed = JSON.parse(message); + win.webContents.send('render-message', { + message: parsed + }); + } + + const updateRecorder = (Event: any, record: boolean): void => { + if (record) { + recorder.resume(); + } else { + recorder.pause(); + const data = recorder.getData(); + sendAudio(data); + } + } + + const windowMount = (): void => { + console.log('testing mount'); + // ipc event handlers + windowEmitter.emit('window-active', true); + ipcMain.on('update-recorder', updateRecorder); + ipcMain.on('send-message', sendMessage); + } + + const windowDismount = (): void => { + console.log('testing dismount'); + // TODO: Gracefully close ws connection + win = null; + windowEmitter.emit('window-active', false); + ipcMain.removeListener('update-recorder', updateRecorder); + ipcMain.removeListener('send-message', sendMessage); + } + + // ws handlers + ws.on('connect', initSession); + ws.on("message", receiveMessage); + + // Handle window mount and dismount. + win.webContents.on('did-finish-load', windowMount); + win.on("closed", windowDismount); + +} diff --git a/src/background/windowEmitter.ts b/src/background/windowEmitter.ts new file mode 100644 index 0000000..65bf9ae --- /dev/null +++ b/src/background/windowEmitter.ts @@ -0,0 +1,5 @@ +const EventEmitter = require('events'); + +class WindowEmitter extends EventEmitter {} + +export const windowEmitter = new WindowEmitter();