From: Andrew Gundersen Date: Thu, 4 Feb 2021 20:26:17 +0000 (-0600) Subject: merging recorder into login X-Git-Tag: v0.9~100^2~3 X-Git-Url: https://andrewgundersen.net/repos?a=commitdiff_plain;h=5e709daf79fd01dd7cc8b212a9106ca7c9ffd72b;p=mime-chat merging recorder into login --- 5e709daf79fd01dd7cc8b212a9106ca7c9ffd72b diff --cc src/background.ts index 41b6579,d3cecd9..b374d27 --- a/src/background.ts +++ b/src/background.ts @@@ -1,152 -1,27 +1,27 @@@ - import { - app, - protocol, - BrowserWindow, - ipcMain - } from "electron"; - import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; - import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer"; - import * as path from "path"; - import WebSocket from "ws"; + "use strict"; + import {initApp} from './background/initApp'; + import {Recorder} from './background/recorder'; -var portAudio = require('naudiodon'); ++// const portAudio = require('naudiodon'); - const isDevelopment = process.env.NODE_ENV !== "production"; + // NOTE Program Begins Here + (async () => { - // Keep a global reference of the window object, if you don't, the window will - // be closed automatically when the JavaScript object is garbage collected. - let win: BrowserWindow | null; + const isDevelopment = process.env.NODE_ENV !== "production"; - // Scheme must be registered before the app is ready - protocol.registerSchemesAsPrivileged([ - { scheme: "app", privileges: { secure: true, standard: true } } - ]); - const recorder = new Recorder({ - channelCount: 1, - sampleFormat: portAudio.SampleFormat16Bit, - sampleRate: 16000, - deviceId: -1, // Use -1 or omit the deviceId to select the default device - closeOnError: true - }, 'binary'); ++ // const recorder = new Recorder({ ++ // channelCount: 1, ++ // sampleFormat: portAudio.SampleFormat16Bit, ++ // sampleRate: 16000, ++ // deviceId: -1, // Use -1 or omit the deviceId to select the default device ++ // closeOnError: true ++ // }, 'binary'); - function createWindow() { - - // Create the browser window. - win = new BrowserWindow({ - width: 350, - height: 500, - resizable: true, - maximizable: false, - - // Postition at launch - x: 10, - y: 10, - - webPreferences: { - devTools: false, - - nodeIntegration: - (process.env.ELECTRON_NODE_INTEGRATION as unknown) as boolean, - - preload: path.join(__dirname, "preload.js") - - } - - }); - - - if (process.env.WEBPACK_DEV_SERVER_URL) { - // Load the url of the dev server if in development mode - win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); - // NOTE uncomment for dev tools on app launch - if (!process.env.IS_TEST) win.webContents.openDevTools(); - } else { - createProtocol("app"); - // Load the index.html when not in development - win.loadURL("app://./index.html"); - } - - - // Websockets config. - win.webContents.on('did-finish-load', () => { - - // Define the websocket - const ws = new WebSocket("ws://localhost:8080"); - - // Send ID on open. - ws.on("open", function open() { - ws.send("gundersena@crimata.com"); - }); - - ws.on("message", (message: any) => { - const parsed = JSON.parse(message); - console.log('new message', message) - if (win) { - win.webContents.send('render-message', { - message: parsed - }) - } - }); - - ipcMain.on('update-menu-bar', (Event: any, payload: any) => { - // win.setTitle("Listening..."); - }) - - ipcMain.on('send-message', (Event: any, payload: any) => { - ws.send(JSON.stringify(payload)); - // win.setTitle("Listening..."); - }) - - ipcMain.on('update-recorder', (Event: any, record: boolean) => { - // ai.start(); - }) - }) - - win.on("closed", () => { - win = null; - }); - - } - - // Quit when all windows are closed. - app.on("window-all-closed", () => { - // On macOS it is common for applications and their menu bar - // to stay active until the user quits explicitly with Cmd + Q - if (process.platform !== "darwin") { - app.quit(); - } - }); - - app.on("activate", () => { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (win === null) { - createWindow(); - } - }); - - // This method will be called when Electron has finished - // initialization and is ready to create browser windows. - // Some APIs can only be used after this event occurs. - app.on("ready", async () => { - if (isDevelopment && !process.env.IS_TEST) { - // Install Vue Devtools try { - await installExtension(VUEJS_DEVTOOLS); + // await authenticate method + initApp(isDevelopment); } catch (e) { - console.error("Vue Devtools failed to install:", e.toString()); + console.log("Error", e); } - } - createWindow(); - }); - - // Exit cleanly on request from parent process in development mode. - if (isDevelopment) { - - // For windows. - if (process.platform === "win32") { - process.on("message", data => { - if (data === "graceful-exit") { - app.quit(); - } - }); - } - else { - process.on("SIGTERM", () => { - app.quit(); - }); - } - } -})(); ++})(); diff --cc src/background/initApp.ts index 0000000,e0ea203..89fc68d mode 000000,100644..100644 --- a/src/background/initApp.ts +++ b/src/background/initApp.ts @@@ -1,0 -1,100 +1,100 @@@ + "use strict"; + -import { app, protocol, BrowserWindow, ipcMain } from "electron"; ++import { app, protocol, BrowserWindow } from "electron"; + import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; + import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer"; + import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; + import * as path from "path"; + -var win: BrowserWindow | null; ++let win: BrowserWindow | null; + + function createWindow() { + // Create the browser window. + win = new BrowserWindow({ + width: 450, + height: 1025, + resizable: true, + webPreferences: { + // Use pluginOptions.nodeIntegration, leave this alone + // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info + nodeIntegration: (process.env + .ELECTRON_NODE_INTEGRATION as unknown) as boolean, + preload: path.join(__dirname, "preload.js") + } + }); + + if (process.env.WEBPACK_DEV_SERVER_URL) { + // Load the url of the dev server if in development mode + win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); + // NOTE uncomment for dev tools on app launch + if (!process.env.IS_TEST) win.webContents.openDevTools(); + } else { + createProtocol("app"); + // Load the index.html when not in development + win.loadURL("app://./index.html"); + } + + win.webContents.on('did-finish-load', () => { + // do stuff once window has finished loading + }) + + win.on("closed", () => { + win = null; + }); + } + + export function initApp(isDev: boolean){ + + // Scheme must be registered before the app is ready + protocol.registerSchemesAsPrivileged([ + { scheme: "app", privileges: { secure: true, standard: true } } + ]); + + // This method will be called when Electron has finished + // initialization and is ready to create browser windows. + // Some APIs can only be used after this event occurs. + app.on("ready", async () => { + if (isDev && !process.env.IS_TEST) { + // Install Vue Devtools + try { + await installExtension(VUEJS_DEVTOOLS); + } catch (e) { + console.error("Vue Devtools failed to install:", e.toString()); + } + } + createWindow(); + }); + + // Quit when all windows are closed. + app.on("window-all-closed", () => { + // On macOS it is common for applications and their menu bar + // to stay active until the user quits explicitly with Cmd + Q + if (process.platform !== "darwin") { + app.quit(); + } + }); + + app.on("activate", () => { + // On macOS it's common to re-create a window in the app when the + // dock icon is clicked and there are no other windows open. + if (win === null) { + createWindow(); + } + }); + + // Exit cleanly on request from parent process in development mode. + if (isDev) { + if (process.platform === "win32") { + process.on("message", data => { + if (data === "graceful-exit") { + app.quit(); + } + }); + } else { + process.on("SIGTERM", () => { + app.quit(); + }); + } + } + } + diff --cc src/background/recorder.ts index 0000000,36f93ba..2897d69 mode 000000,100644..100644 --- a/src/background/recorder.ts +++ b/src/background/recorder.ts @@@ -1,0 -1,113 +1,113 @@@ + // eslint-disable-next-line @typescript-eslint/no-var-requires + const portAudio = require('naudiodon'); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { Writable } = require('stream'); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { StringDecoder } = require('string_decoder'); + import { ipcMain } from "electron"; + + interface RecorderI { + stop(): string; + pause(): void; + } + + class StringWritable extends Writable { + + constructor(options: { + defaultEncoding: string; + }) { + super(options); + this._decoder = new StringDecoder(options && options.defaultEncoding); + this.data = ''; + } + + _write(chunk: Buffer, encoding: string, callback: (error? : Error) => void) { + if (encoding === 'buffer') { + chunk = this._decoder.write(chunk); + } + this.data += chunk; + console.log('writing'); + callback(); + } + + _final(callback: (error? : Error) => void) { + console.log('yayayay'); + this.data += this._decoder.end(); + callback(); + } + } + -type inputOptions = {clear ++type inputOptions = {clear; + - channelCount: number, - sampleFormat: string, - sampleRate: number, - deviceId: number, - closeOnError: boolean ++ channelCount: number; ++ sampleFormat: string; ++ sampleRate: number; ++ deviceId: number; ++ closeOnError: boolean; + } + + export class Recorder implements RecorderI { + - private ia: any; ++ private ia: boolean; + private micWritable: StringWritable; + private recorderOptions: inputOptions; + + constructor(options: inputOptions, encoding: string) { + this.recorderOptions = options; + this.micWritable = new StringWritable({ + defaultEncoding: encoding + }) + this.restart(); + ipcMain.on('update-recorder', this._update); + } + + restart() { + this.ia = new portAudio.AudioIO({ + inOptions: this.recorderOptions + }); + this._start(); + } + + pause(): void { + this.ia.pause(); + } + + resume(){ + this.ia.resume(); + } + + getData(): string { + return this.micWritable.data; + } + + stop(): string { + this.ia.quit(); + return this.getData(); + } + + private _start(): void { + this.ia.pipe(this.micWritable); + this.ia.start(); + this.pause(); + } + - private _update = (Event: any, record: boolean) => { ++ private _update = (Event: boolean, record: boolean) => { + if (record) { + console.log('recording'); + this.micWritable.data = ''; + if(this.ia._readableState.paused){ + this.resume(); + return; + } + this.ia.pipe(this.micWritable); + + console.log(this.ia._readableState.paused); + } else { + console.log('stop recording'); + this.ia.unpipe(this.micWritable); + this.micWritable.on('finish', () => { + console.log('hello'); + }) + } + } + } diff --cc testAudio.js index 0000000,c2c291b..815964a mode 000000,100644..100644 --- a/testAudio.js +++ b/testAudio.js @@@ -1,0 -1,87 +1,87 @@@ + + const fs = require('fs'); + const speech = require('@google-cloud/speech'); + const portAudio = require('naudiodon'); -const rs = fs.createReadStream('rawAudio.wav'); ++// const rs = fs.createReadStream('rawAudio.wav'); + const WebSocket = require("ws") + const {Writable} = require('stream'); + + // Creates a client + const client = new speech.SpeechClient(); + + // connect to test server -const ws = new WebSocket("ws://localhost:8080"); ++// const ws = new WebSocket("ws://localhost:8080"); + + // google cloud speech to text settings + const encoding = 'LINEAR16'; + const sampleRateHertz = 44100; + const languageCode = 'en-US'; + + const audioChunks = []; + + const audioInputStreamTransform = new Writable({ + write(chunk, encoding, next) { + console.log(chunk); + audioChunks.push(chunk); + next(); + }, + + final() { + console.log(audioChunks); + }, + }); + + const config = { + encoding: encoding, + sampleRateHertz: sampleRateHertz, + languageCode: languageCode, + }; + + const request = { + config, + interimResults: true, + }; + + const speechCallback = (data) => { + console.log( + `Transcription: ${data.results[0].alternatives[0].transcript}` + ); + } + -const recognizeStream = client -.streamingRecognize(request) -.on('error', err => { - if (err.code === 11) { - // restartStream(); - } else { - console.error('API request error ' + err); - } -}) -.on('data', speechCallback); ++// const recognizeStream = client ++// .streamingRecognize(request) ++// .on('error', err => { ++// if (err.code === 11) { ++// // restartStream(); ++// } else { ++// console.error('API request error ' + err); ++// } ++// }) ++// .on('data', speechCallback); + + // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream + const recorderOptions = { + channelCount: 1, + sampleFormat: portAudio.sampleFormat16Bit, + sampleRate: 16000, + deviceId: -1, // Use -1 or omit the deviceId to select the default device + closeOnError: false // Close the stream if an audio error is detected, if set false then just log the error + } + + const ai = new portAudio.AudioIO({ + inOptions: recorderOptions + }); + + // manipulate individual chunks as they're available + // const audioData = []; + // ai.on('data', (d) => { + // audioData.push(d.toString('binary')) + // }) + + ai.pipe(audioInputStreamTransform) + ai.start(); + setTimeout(() => { + ai.quit(); + }, 2000) + +