me/mime-chat
Watch
1
0
Fork
You've already forked mime-chat
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

refactor tests for readability + linting fixes

Browse source
This commit is contained in:
Enrique Hernandez 2021-02-10 23:04:53 -06:00
commit 57fce6d6cc
12 changed files with 123 additions and 318 deletions
Download patch file Download diff file Expand all files Collapse all files

202
src/audio.js
View file

@ -1,202 +0,0 @@
function main(
encoding = 'LINEAR16',
sampleRateHertz = 16000,
languageCode = 'en-US',
streamingLimit = 290000
) {
// [START speech_transcribe_infinite_streaming]
// const encoding = 'LINEAR16';
// const sampleRateHertz = 16000;
// const languageCode = 'en-US';
// const streamingLimit = 10000; // ms - set to low number for demo purposes
const chalk = require('chalk');
const {Writable} = require('stream');
const recorder = require('node-record-lpcm16');
const fs = require('fs');
const writeStream = fs.createWriteStream('test.wav', { encoding: 'binary'});
// // Imports the Google Cloud client library
// // Currently, only v1p1beta1 contains result-end-time
const speech = require('@google-cloud/speech').v1p1beta1;
const client = new speech.SpeechClient();
const config = {
encoding: encoding,
sampleRateHertz: sampleRateHertz,
languageCode: languageCode,
};
const request = {
config,
interimResults: true,
};
let recognizeStream = null;
let restartCounter = 0;
let audioInput = [];
let lastAudioInput = [];
let resultEndTime = 0;
let isFinalEndTime = 0;
let finalRequestEndTime = 0;
let newStream = true;
let bridgingOffset = 0;
let lastTranscriptWasFinal = false;
function startStream() {
// Clear current audioInput
audioInput = [];
// Initiate (Reinitiate) a recognize stream
recognizeStream = client
.streamingRecognize(request)
.on('error', err => {
if (err.code === 11) {
// restartStream();
} else {
console.error('API request error ' + err);
}
})
.on('data', speechCallback);
// Restart stream when streamingLimit expires
setTimeout(restartStream, streamingLimit);
}
const speechCallback = stream => {
// Convert API result end time from seconds + nanoseconds to milliseconds
resultEndTime =
stream.results[0].resultEndTime.seconds * 1000 +
Math.round(stream.results[0].resultEndTime.nanos / 1000000);
// Calculate correct time based on offset from audio sent twice
const correctedTime =
resultEndTime - bridgingOffset + streamingLimit * restartCounter;
process.stdout.clearLine();
process.stdout.cursorTo(0);
let stdoutText = '';
if (stream.results[0] && stream.results[0].alternatives[0]) {
stdoutText =
correctedTime + ': ' + stream.results[0].alternatives[0].transcript;
}
if (stream.results[0].isFinal) {
process.stdout.write(chalk.green(`${stdoutText}\n`));
isFinalEndTime = resultEndTime;
lastTranscriptWasFinal = true;
} else {
// Make sure transcript does not exceed console character length
if (stdoutText.length > process.stdout.columns) {
stdoutText =
stdoutText.substring(0, process.stdout.columns - 4) + '...';
}
process.stdout.write(chalk.red(`${stdoutText}`));
lastTranscriptWasFinal = false;
}
};
const audioInputStreamTransform = new Writable({
write(chunk, encoding, next) {
if (newStream && lastAudioInput.length !== 0) {
// Approximate math to calculate time of chunks
const chunkTime = streamingLimit / lastAudioInput.length;
if (chunkTime !== 0) {
if (bridgingOffset < 0) {
bridgingOffset = 0;
}
if (bridgingOffset > finalRequestEndTime) {
bridgingOffset = finalRequestEndTime;
}
const chunksFromMS = Math.floor(
(finalRequestEndTime - bridgingOffset) / chunkTime
);
bridgingOffset = Math.floor(
(lastAudioInput.length - chunksFromMS) * chunkTime
);
for (let i = chunksFromMS; i < lastAudioInput.length; i++) {
recognizeStream.write(lastAudioInput[i]);
}
}
newStream = false;
}
audioInput.push(chunk);
if (recognizeStream) {
recognizeStream.write(chunk);
}
next();
},
final() {
if (recognizeStream) {
// recognize event is a stream readable object
recognizeStream.end();
}
},
});
function restartStream() {
if (recognizeStream) {
recognizeStream.end();
recognizeStream.removeListener('data', speechCallback);
recognizeStream = null;
}
if (resultEndTime > 0) {
finalRequestEndTime = isFinalEndTime;
}
resultEndTime = 0;
lastAudioInput = [];
lastAudioInput = audioInput;
restartCounter++;
if (!lastTranscriptWasFinal) {
process.stdout.write('\n');
}
process.stdout.write(
chalk.yellow(`${streamingLimit * restartCounter}: RESTARTING REQUEST\n`)
);
newStream = true;
startStream();
}
// Start recording and send the microphone input to the Speech API
recorder
.record({
sampleRateHertz: sampleRateHertz,
threshold: 0, // Silence threshold
silence: 1000,
keepSilence: true,
recordProgram: 'rec', // Try also "arecord" or "sox"
})
.stream()
.on('error', err => {
console.error('Audio recording error ' + err);
})
.pipe(audioInputStreamTransform);
console.log('');
console.log('Listening, press Ctrl+C to stop.');
console.log('');
console.log('End (ms) Transcript Results/Status');
console.log('=========================================================');
// startStream();
// [END speech_transcribe_infinite_streaming]
}
// process.on('unhandledRejection', err => {
// console.error(err.message);
// process.exitCode = 1;
// });
main(...process.argv.slice(2));

5
src/background.ts
View file

@ -4,8 +4,9 @@
*/
"use strict";
import { protocol } from "electron";
import { initApp } from './background/initApp';
import { initApp } from './background/init';
// Scheme must be registered before the app is ready
protocol.registerSchemesAsPrivileged([
@ -21,4 +22,4 @@ const isDevelopment = process.env.NODE_ENV !== "production";
console.log('Starting Crimata electron app.');
initApp(isDevelopment);
})();
})();

55
src/background/createWindow.ts
View file

@ -1,55 +0,0 @@
"use strict";
import { BrowserWindow, ipcMain } from "electron";
import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
import { windowEmitter } from './windowEmitter';
import { sendMessage } from './session';
import * as path from "path";
interface WindowSettings {
width: number;
height: number;
resizable: boolean;
}
const handleMessage = (_event, arg: any) => {
sendMessage(arg);
}
const windowMount = (): void => {
windowEmitter.emit('window-active', true);
ipcMain.on('send-message', handleMessage);
}
export const createWindow = async (options: WindowSettings): Promise<BrowserWindow> => {
return new Promise((resolve, _reject) => {
const win: BrowserWindow = new BrowserWindow({
width: options.width,
height: options.height,
resizable: options.resizable,
webPreferences: {
// Use pluginOptions.nodeIntegration, leave this alone
// See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info
nodeIntegration: (process.env
.ELECTRON_NODE_INTEGRATION as unknown) as boolean,
preload: path.join(__dirname, "preload.js")
}
});
if (process.env.WEBPACK_DEV_SERVER_URL) {
// Load the url of the dev server if in development mode
win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string);
} else {
createProtocol("app");
// Load the index.html when not in development
win.loadURL("app://./index.html");
}
win.webContents.on('did-finish-load', () => {
windowMount();
resolve(win);
});
});
};

6
src/background/emitter.ts Normal file
View file

@ -0,0 +1,6 @@
/* eslint-disable */
const EventEmitter = require('events');
class BackgroundMitt extends EventEmitter { }
export const backgroundMitt = new BackgroundMitt();

4
src/background/initApp.ts → src/background/init.ts
View file

@ -2,11 +2,11 @@
import { app } from "electron";
import { main } from './run';
import { windowEmitter } from './windowEmitter';
import { backgroundMitt } from './emitter';
let winActive: boolean;
windowEmitter.on('window-active', (state: boolean) => {
backgroundMitt.on('window-active', (state: boolean) => {
winActive = state;
});

19
src/background/run.ts
View file

@ -1,20 +1,29 @@
"use strict";
import { createWindow } from './createWindow';
import { createWindow } from './window';
import { ipcMain, BrowserWindow } from "electron";
import { initSession } from './session';
import { windowEmitter } from './windowEmitter';
import { backgroundMitt } from './emitter';
// const portAudio = require('naudiodon');
interface RenderMessage {
content: string;
context: string;
subContext: string;
modifiers: string;
time: string;
id: string;
}
interface IpcRendererPayload {
endpoint: string;
message: any;
message: RenderMessage | null;
}
let win: BrowserWindow | null;
let activeSession = false;
windowEmitter.on('ipc-renderer', (payload: IpcRendererPayload) => {
backgroundMitt.on('ipc-renderer', (payload: IpcRendererPayload) => {
if (win)
win.webContents.send(payload.endpoint, {
message: payload.message
@ -23,7 +32,7 @@ windowEmitter.on('ipc-renderer', (payload: IpcRendererPayload) => {
const windowDismount = (): void => {
win = null;
windowEmitter.emit('window-active', false);
backgroundMitt.emit('window-active', false);
ipcMain.removeAllListeners('send-message');
}

71
src/background/session.ts
View file

@ -1,16 +1,16 @@
"use strict";
import WebSocket from 'ws';
import { windowEmitter } from './windowEmitter';
import { ipcMain } from "electron";
import { backgroundMitt } from './emitter';
import { ipcMain } from "electron";
interface RenderMessage {
content: string;
context: string;
subContext: string;
modifiers: string;
time: string;
id: string;
interface RenderMessage {
content: string;
context: string;
subContext: string;
modifiers: string;
time: string;
id: string;
}
interface UserCreds {
@ -18,6 +18,11 @@ interface UserCreds {
password: string;
}
interface TextMessage {
audio: 0;
content: string;
}
const ip = 'ws://localhost';
const port = 8081;
const reconnectTimeout = 3000; //ms
@ -25,25 +30,10 @@ let socket: WebSocket;
let success = false;
let auth = false;
const receiveMessage = (message: string): void => {
while(!auth) {
windowEmitter.emit('auth-res', message);
return;
}
const parsed: RenderMessage = JSON.parse(message);
windowEmitter.emit('ipc-renderer', {
endpoint: 'render-message',
message: parsed
});
};
const authSession = async (_event, payload: string | UserCreds | null): Promise<string> => {
return new Promise((resolve, reject) => {
console.log('authenticating...');
windowEmitter.on('auth-res', (res: string) => {
backgroundMitt.on('auth-res', (res: string) => {
if (res === 'locked') {
reject(res);
} else {
@ -61,10 +51,25 @@ const authSession = async (_event, payload: string | UserCreds | null): Promise<
});
};
export const sendMessage = (content: string | Buffer) => {
const receiveMessage = (message: string): void => {
while (!auth) {
backgroundMitt.emit('auth-res', message);
return;
}
const parsed: RenderMessage = JSON.parse(message);
backgroundMitt.emit('ipc-renderer', {
endpoint: 'render-message',
message: parsed
});
};
export const sendMessage = (content: TextMessage | Buffer) => {
if (content instanceof Buffer) {
socket.send(content);
} else if (content){
} else if (content) {
socket.send(JSON.stringify(content));
}
}
@ -77,8 +82,6 @@ export const initSession = () => {
success = false;
}
console.log('Connecting to Crimata Servers...');
socket = new WebSocket(`${ip}:${port}`);
socket.binaryType = 'arraybuffer';
@ -87,10 +90,10 @@ export const initSession = () => {
ipcMain.handle('auth-user', authSession);
socket.on('open', () => {
console.log('Success! Connected to Crimata.');
console.log('Success! Connected to Crimata Servers.');
// fetch token from renderer
windowEmitter.emit('ipc-renderer', {
backgroundMitt.emit('ipc-renderer', {
endpoint: 'fetch-token',
message: null
});
@ -102,9 +105,9 @@ export const initSession = () => {
console.log('ERROR: Failed to connect.');
socket.removeAllListeners();
socket.close();
setTimeout(()=> {
setTimeout(() => {
if (!success) {
console.log('Attempting reconnect.');
console.log('Reconnecting...');
initSession();
}
}, reconnectTimeout);
@ -112,7 +115,7 @@ export const initSession = () => {
});
socket.on('close', () => {
console.log('Connection droped. Restarting.')
console.log('Connection droped! Restarting...')
initSession();
});

60
src/background/window.ts Normal file
View file

@ -0,0 +1,60 @@
"use strict";
import { BrowserWindow, ipcMain } from "electron";
import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
import { backgroundMitt } from './emitter';
import { sendMessage } from './session';
import * as path from "path";
interface WindowSettings {
width: number;
height: number;
resizable: boolean;
}
interface TextMessage {
audio: 0;
content: string;
}
const handleMessage = (_event, arg: TextMessage | Buffer ) => {
sendMessage(arg);
}
const windowMount = (): void => {
backgroundMitt.emit('window-active', true);
ipcMain.on('send-message', handleMessage);
}
export const createWindow = async (options: WindowSettings): Promise<BrowserWindow> => {
return new Promise((resolve, _reject) => {
const win: BrowserWindow = new BrowserWindow({
width: options.width,
height: options.height,
resizable: options.resizable,
webPreferences: {
// Use pluginOptions.nodeIntegration, leave this alone
// See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info
nodeIntegration: (process.env
.ELECTRON_NODE_INTEGRATION as unknown) as boolean,
preload: path.join(__dirname, "preload.js")
}
});
if (process.env.WEBPACK_DEV_SERVER_URL) {
// Load the url of the dev server if in development mode
win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string);
} else {
createProtocol("app");
// Load the index.html when not in development
win.loadURL("app://./index.html");
}
win.webContents.on('did-finish-load', () => {
windowMount();
resolve(win);
});
});
};

5
src/background/windowEmitter.ts
View file

@ -1,5 +0,0 @@
const EventEmitter = require('events');
class WindowEmitter extends EventEmitter {}
export const windowEmitter = new WindowEmitter();

3
testAudio.js → tests/audio.js
View file

@ -1,10 +1,7 @@
const fs = require('fs');
const speech = require('@google-cloud/speech');
const portAudio = require('naudiodon');
// const rs = fs.createReadStream('rawAudio.wav');
const WebSocket = require("ws")
const {Writable} = require('stream');
// Creates a client
const client = new speech.SpeechClient();

9
testServer.js → tests/server.js
View file

@ -1,13 +1,5 @@
const WebSocket = require("ws");
const testMessage = {
content: 'Hello from test server!' ,
context: 'test message',
subContext: '',
modifiers: 'ai',
id: "0",
time: 'test'
}
const wss = new WebSocket.Server({
port: 8081
});
@ -25,5 +17,4 @@ wss.on("connection", function connection(ws, req) {
const ip = req.socket.remoteAddress;
console.log("received connection from", ip);
// ws.send(JSON.stringify(testMessage));
});

0
ws.py → tests/ws.py
View file