me/mime-chat
Watch
1
0
Fork
You've already forked mime-chat
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add speech recorder + ws

Browse source
This commit is contained in:
Enrique Hernandez 2020-10-13 20:19:32 -05:00
commit abbfe42093
7 changed files with 125 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

12
package.json
View file

@ -15,11 +15,16 @@
},
"main": "background.js",
"dependencies": {
"@types/bindings": "^1.3.0",
"@types/uuid": "^8.3.0",
"@types/ws": "^7.2.7",
"animejs": "^3.2.0",
"core-js": "^3.6.5",
"speech-recorder": "^1.2.1",
"vue": "^3.0.0-0",
"vue-router": "^4.0.0-0",
"vuex": "^4.0.0-0"
"vuex": "^4.0.0-0",
"ws": "^7.3.1"
},
"devDependencies": {
"@types/animejs": "^3.1.2",
@ -54,6 +59,9 @@
"pre-commit": "lint-staged"
},
"lint-staged": {
"*.{js,jsx,vue,ts,tsx}": ["vue-cli-service lint", "git add"]
"*.{js,jsx,vue,ts,tsx}": [
"vue-cli-service lint",
"git add"
]
}
}

23
src/background.ts
View file

@ -5,6 +5,29 @@ import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
const isDevelopment = process.env.NODE_ENV !== "production";
import * as path from "path";
import { SpeechRecorder } from "speech-recorder";
import WebSocket from "ws";
const ws = new WebSocket("ws://localhost:8080");
ws.on("open", function open() {
ws.send("hello fro client");
});
ws.on("message", (message: any) => {
console.log("received message", message);
});
const recorder = new SpeechRecorder();
recorder.start({
onAudio: (audio: object, speech: boolean) => {
if (speech) {
console.log(audio);
ws.send(audio);
}
console.log("recording");
// writeStream.write(audio);
}
});
// Keep a global reference of the window object, if you don't, the window will
// be closed automatically when the JavaScript object is garbage collected.

59
src/components/microphone/index.vue
View file

@ -7,64 +7,7 @@
<script>
import { defineComponent } from 'vue';
export default defineComponent({
name: 'Microphone',
async setup() {
const bufferLen = 4096;
const numChannels = 1;
window.AudioContext = window.AudioContext || window.webkitAudioContext;
const audioContext = new AudioContext();
let result;
let cWasm;
const getCwasm = async () => {
await import('../../../crate/pkg').then(async module => {
cWasm = module;
});
}
await getCwasm();
function handleStream(stream) {
const source = audioContext.createMediaStreamSource(stream);
const context = source.context;
// NOTE might come in handy for VAD
// const sampleRate = context.sampleRate;
const node = context.createScriptProcessor.call(
context,
bufferLen,
numChannels,
numChannels
);
node.onaudioprocess = async e => {
const inputBuff = new Float32Array(bufferLen);
// get data from audio event and copy into inputBuff
e.inputBuffer.copyFromChannel(inputBuff, 0);
const int16 = new Int16Array(inputBuff.buffer);
result = await cWasm.foo(int16);
// TODO add handle audio stream to main process?
// window.postMessage(
// {
// myTypeField: "handle-audio-stream",
// audioData: int16,
// },
// "*"
// );
};
source.connect(node);
node.connect(context.destination);
}
navigator.mediaDevices
.getUserMedia({ audio: true })
.then(s => {
handleStream(s);
}).catch(error => {
throw error;
});
}
name: 'Microphone',
})
</script>

2
src/main.ts
View file

@ -1,4 +1,4 @@
import { createApp, provide, inject } from "vue";
import { createApp } from "vue";
import App from "./App.vue";
import router from "./router";
import store from "./store";

34
testServer.js Normal file
View file

@ -0,0 +1,34 @@
const WebSocket = require("ws");
const wss = new WebSocket.Server({
port: 8080
// perMessageDeflate: {
// zlibDeflateOptions: {
// // See zlib defaults.
// chunkSize: 1024,
// memLevel: 7,
// level: 3
// },
// zlibInflateOptions: {
// chunkSize: 10 * 1024
// },
// // Other options settable:
// clientNoContextTakeover: true, // Defaults to negotiated value.
// serverNoContextTakeover: true, // Defaults to negotiated value.
// serverMaxWindowBits: 10, // Defaults to negotiated value.
// // Below options specified as default values.
// concurrencyLimit: 10, // Limits zlib concurrency for perf.
// threshold: 1024 // Size (in bytes) below which messages
// // should not be compressed.
// }
});
wss.on("connection", function connection(ws, req) {
ws.on("message", function incoming(message) {
console.log("received: %s:", message);
});
const ip = req.socket.remoteAddress;
console.log("received connection from", ip);
ws.send("hello from server!");
});

24
vue.config.js
View file

@ -1,18 +1,20 @@
const path = require("path");
const WasmPackPlugin = require("@wasm-tool/wasm-pack-plugin");
const OptimizeWasmPlugin = require("optimize-wasm-webpack-plugin");
// NOTE needed imports for rust wasm
// const path = require("path");
// const WasmPackPlugin = require("@wasm-tool/wasm-pack-plugin");
// const OptimizeWasmPlugin = require("optimize-wasm-webpack-plugin");
module.exports = {
lintOnSave: false,
configureWebpack: {
optimization: {
minimizer: [new OptimizeWasmPlugin()]
},
plugins: [
new WasmPackPlugin({
crateDirectory: path.resolve(__dirname, "crate")
})
]
// NOTE uncomment to use rust wasm
// optimization: {
// minimizer: [new OptimizeWasmPlugin()]
// },
// plugins: [
// new WasmPackPlugin({
// crateDirectory: path.resolve(__dirname, "crate")
// })
// ]
},
pluginOptions: {
electronBuilder: {

46
yarn.lock
View file

@ -1216,6 +1216,11 @@
dependencies:
"@babel/types" "^7.3.0"
"@types/bindings@^1.3.0":
version "1.3.0"
resolved "https://registry.yarnpkg.com/@types/bindings/-/bindings-1.3.0.tgz#e9cd75a96d7abc1ecba0dc7eecb09a9f96cd417c"
integrity sha512-mTWOE6wC64MoEpv33otJNpQob81l5Pi+NsUkdiiP8EkESraQM94zuus/2s/Vz2Idy1qQkctNINYDZ61nfG1ngQ==
"@types/body-parser@*":
version "1.19.0"
resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.0.tgz#0685b3c47eb3006ffed117cdd55164b61f80538f"
@ -1469,6 +1474,11 @@
dependencies:
source-map "^0.6.1"
"@types/uuid@^8.3.0":
version "8.3.0"
resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.0.tgz#215c231dff736d5ba92410e6d602050cce7e273f"
integrity sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==
"@types/webpack-dev-server@^3.11.0":
version "3.11.0"
resolved "https://registry.yarnpkg.com/@types/webpack-dev-server/-/webpack-dev-server-3.11.0.tgz#bcc3b85e7dc6ac2db25330610513f2228c2fcfb2"
@ -1506,6 +1516,13 @@
"@types/webpack-sources" "*"
source-map "^0.6.0"
"@types/ws@^7.2.7":
version "7.2.7"
resolved "https://registry.yarnpkg.com/@types/ws/-/ws-7.2.7.tgz#362ad1a1d62721bdb725e72c8cccf357078cf5a3"
integrity sha512-UUFC/xxqFLP17hTva8/lVT0SybLUrfSD9c+iapKb0fEiC8uoDbA+xuZ3pAN603eW+bY8ebSMLm9jXdIPnD0ZgA==
dependencies:
"@types/node" "*"
"@types/yargs-parser@*":
version "15.0.0"
resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d"
@ -2836,7 +2853,7 @@ binaryen@^85.0.0:
resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-85.0.0.tgz#72c385d558862e52b75918bbce748d4627a98499"
integrity sha512-01z4ZvA1O1hhNH+R4QQHuqzslecdHyzHYeuPMm4rB3hEiCP6BQlXZhHs68FgKfKT199YOt1uy5bmAIsilaOQ9g==
bindings@^1.5.0:
bindings@^1.3.0, bindings@^1.5.0:
version "1.5.0"
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==
@ -8636,6 +8653,11 @@ nan@^2.12.1, nan@^2.13.2:
resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01"
integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==
nan@^2.14.1:
version "2.14.2"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"
integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==
nanomatch@^1.2.9:
version "1.2.13"
resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119"
@ -11164,6 +11186,16 @@ spectron@^11.0.0:
split "^1.0.0"
webdriverio "^6.1.20"
speech-recorder@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/speech-recorder/-/speech-recorder-1.2.1.tgz#db050c39db951db099ae3ee07f4eba574faab770"
integrity sha512-kILQF+5ZS6XNgZo6zul+6PPMSYbryPaLs5qWLnjR2KtbCZGxNwuuPTiMaJYvvQRdFmJlJOSgsGylZYk1sAwR4Q==
dependencies:
bindings "^1.3.0"
nan "^2.14.1"
uuid "^3.3.3"
webrtcvad "^1.0.1"
split-string@^3.0.1, split-string@^3.0.2:
version "3.1.0"
resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2"
@ -12312,7 +12344,7 @@ utils-merge@1.0.1:
resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
uuid@^3.3.2, uuid@^3.4.0:
uuid@^3.3.2, uuid@^3.3.3, uuid@^3.4.0:
version "3.4.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
@ -12711,6 +12743,14 @@ webpack@^4.0.0, webpack@^4.18.0:
watchpack "^1.7.4"
webpack-sources "^1.4.1"
webrtcvad@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/webrtcvad/-/webrtcvad-1.0.1.tgz#862b35c4ed2e7de3f39b268f48ef890ac85e02ce"
integrity sha512-oLfReCmGMpRducFWKP+o0GpKZLPj0u6qkln3P7wGaEzyjxtBiFuzvK28pmKF5SSihEkC1RGKO7Pi9C4VfY1q4Q==
dependencies:
bindings "^1.3.0"
node-addon-api "^1.7.1"
websocket-driver@0.6.5:
version "0.6.5"
resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.6.5.tgz#5cb2556ceb85f4373c6d8238aa691c8454e13a36"
@ -12885,7 +12925,7 @@ ws@^6.0.0, ws@^6.2.1:
dependencies:
async-limiter "~1.0.0"
ws@^7.0.0, ws@^7.2.3:
ws@^7.0.0, ws@^7.2.3, ws@^7.3.1:
version "7.3.1"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.3.1.tgz#d0547bf67f7ce4f12a72dfe31262c68d7dc551c8"
integrity sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==