add speech recorder + ws

diff --git a/package.json b/package.json
index f965baf66fc2faa9bfbef269bbb485f11072b097..9c86aab6c020c12eba3a16c4cb5521cf0df5c1a2 100644 (file)
--- a/package.json
+++ b/package.json
@@ -15,11 +15,16 @@
   },
   "main": "background.js",
   "dependencies": {
+    "@types/bindings": "^1.3.0",
+    "@types/uuid": "^8.3.0",
+    "@types/ws": "^7.2.7",
     "animejs": "^3.2.0",
     "core-js": "^3.6.5",
+    "speech-recorder": "^1.2.1",
     "vue": "^3.0.0-0",
     "vue-router": "^4.0.0-0",
-    "vuex": "^4.0.0-0"
+    "vuex": "^4.0.0-0",
+    "ws": "^7.3.1"
   },
   "devDependencies": {
     "@types/animejs": "^3.1.2",
@@ -54,6 +59,9 @@
     "pre-commit": "lint-staged"
   },
   "lint-staged": {
-    "*.{js,jsx,vue,ts,tsx}": ["vue-cli-service lint", "git add"]
+    "*.{js,jsx,vue,ts,tsx}": [
+      "vue-cli-service lint",
+      "git add"
+    ]
   }
 }

diff --git a/src/background.ts b/src/background.ts
index defc4f6fb65223acaf2caf52b44a43eadb08dbf9..b461c753f9a827a1ce10e863a350ff2abfafd4c7 100644 (file)
--- a/src/background.ts
+++ b/src/background.ts
@@ -5,6 +5,29 @@ import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
 import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
 const isDevelopment = process.env.NODE_ENV !== "production";
 import * as path from "path";
+import { SpeechRecorder } from "speech-recorder";
+import WebSocket from "ws";
+
+const ws = new WebSocket("ws://localhost:8080");
+ws.on("open", function open() {
+  ws.send("hello fro client");
+});
+ws.on("message", (message: any) => {
+  console.log("received message", message);
+});
+
+const recorder = new SpeechRecorder();
+
+recorder.start({
+  onAudio: (audio: object, speech: boolean) => {
+    if (speech) {
+      console.log(audio);
+      ws.send(audio);
+    }
+    console.log("recording");
+    // writeStream.write(audio);
+  }
+});
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.

diff --git a/src/components/microphone/index.vue b/src/components/microphone/index.vue
index d46b91f54164f149f92b363c6159b5984d5c5121..d2e7f2ba6c5a45c0f863df02da0fe95a10123d11 100644 (file)
--- a/src/components/microphone/index.vue
+++ b/src/components/microphone/index.vue
@@ -7,64 +7,7 @@
 <script>
  import { defineComponent } from 'vue';
  export default defineComponent({
-   name: 'Microphone',
-   async setup() {
-     const bufferLen = 4096;
-     const numChannels = 1;
-     window.AudioContext = window.AudioContext || window.webkitAudioContext;
-     const audioContext = new AudioContext();
-
-      let result;
-      let cWasm;
-      const getCwasm = async () => {
-        await import('../../../crate/pkg').then(async module => {
-            cWasm = module;
-        });
-      }
-     await getCwasm();
-
-     function handleStream(stream) {
-       const source = audioContext.createMediaStreamSource(stream);
-       const context = source.context;
-       // NOTE might come in handy for VAD
-       // const sampleRate = context.sampleRate;
-
-       const node = context.createScriptProcessor.call(
-         context,
-         bufferLen,
-         numChannels,
-         numChannels
-       );
-
-       node.onaudioprocess = async e => {
-         const inputBuff = new Float32Array(bufferLen);
-         // get data from audio event and copy into inputBuff
-         e.inputBuffer.copyFromChannel(inputBuff, 0);
-         const int16 = new Int16Array(inputBuff.buffer);
-         result = await cWasm.foo(int16);
-
-           // TODO add handle audio stream to main process?
-           // window.postMessage(
-           //   {
-           //     myTypeField: "handle-audio-stream",
-           //     audioData: int16,
-           //   },
-           //   "*"
-           // );
-       };
-       source.connect(node);
-       node.connect(context.destination);
-     }
-
-       navigator.mediaDevices
-         .getUserMedia({ audio: true })
-           .then(s => {
-             handleStream(s);
-           }).catch(error => {
-             throw error;
-
-           });
-   }
+     name: 'Microphone',
  })
 
 </script>

diff --git a/src/main.ts b/src/main.ts
index 325b85c987a4a170ff511e8361871ce53984491a..a04047e9dbdc0a6814174854ecd4edce9267b359 100644 (file)
--- a/src/main.ts
+++ b/src/main.ts
@@ -1,4 +1,4 @@
-import { createApp, provide, inject } from "vue";
+import { createApp } from "vue";
 import App from "./App.vue";
 import router from "./router";
 import store from "./store";

diff --git a/testServer.js b/testServer.js
new file mode 100644 (file)
index 0000000..acf809e
--- /dev/null
+++ b/testServer.js
@@ -0,0 +1,34 @@
+const WebSocket = require("ws");
+
+const wss = new WebSocket.Server({
+  port: 8080
+  // perMessageDeflate: {
+  //   zlibDeflateOptions: {
+  //     // See zlib defaults.
+  //     chunkSize: 1024,
+  //     memLevel: 7,
+  //     level: 3
+  //   },
+  //   zlibInflateOptions: {
+  //     chunkSize: 10 * 1024
+  //   },
+  //   // Other options settable:
+  //   clientNoContextTakeover: true, // Defaults to negotiated value.
+  //   serverNoContextTakeover: true, // Defaults to negotiated value.
+  //   serverMaxWindowBits: 10, // Defaults to negotiated value.
+  //   // Below options specified as default values.
+  //   concurrencyLimit: 10, // Limits zlib concurrency for perf.
+  //   threshold: 1024 // Size (in bytes) below which messages
+  //   // should not be compressed.
+  // }
+});
+
+wss.on("connection", function connection(ws, req) {
+  ws.on("message", function incoming(message) {
+    console.log("received: %s:", message);
+  });
+
+  const ip = req.socket.remoteAddress;
+  console.log("received connection from", ip);
+  ws.send("hello from server!");
+});

diff --git a/vue.config.js b/vue.config.js
index a40877b5fda9feacd88dcc3b0a56601c120954e2..c1f417074ae3992916a63701a34141f1aaf09ca6 100644 (file)
--- a/vue.config.js
+++ b/vue.config.js
@@ -1,18 +1,20 @@
-const path = require("path");
-const WasmPackPlugin = require("@wasm-tool/wasm-pack-plugin");
-const OptimizeWasmPlugin = require("optimize-wasm-webpack-plugin");
+// NOTE needed imports for rust wasm
+// const path = require("path");
+// const WasmPackPlugin = require("@wasm-tool/wasm-pack-plugin");
+// const OptimizeWasmPlugin = require("optimize-wasm-webpack-plugin");
 
 module.exports = {
   lintOnSave: false,
   configureWebpack: {
-    optimization: {
-      minimizer: [new OptimizeWasmPlugin()]
-    },
-    plugins: [
-      new WasmPackPlugin({
-        crateDirectory: path.resolve(__dirname, "crate")
-      })
-    ]
+    // NOTE uncomment to use rust wasm
+    // optimization: {
+    //   minimizer: [new OptimizeWasmPlugin()]
+    // },
+    // plugins: [
+    //   new WasmPackPlugin({
+    //     crateDirectory: path.resolve(__dirname, "crate")
+    //   })
+    // ]
   },
   pluginOptions: {
     electronBuilder: {

diff --git a/yarn.lock b/yarn.lock
index 694fda418f3f5ff5982b93e23606455271ff6f1e..8d2a5a4bd827486acff4d5deec62c3f1bc372131 100644 (file)
--- a/yarn.lock
+++ b/yarn.lock
@@ -1216,6 +1216,11 @@
   dependencies:
     "@babel/types" "^7.3.0"
 
+"@types/bindings@^1.3.0":
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/@types/bindings/-/bindings-1.3.0.tgz#e9cd75a96d7abc1ecba0dc7eecb09a9f96cd417c"
+  integrity sha512-mTWOE6wC64MoEpv33otJNpQob81l5Pi+NsUkdiiP8EkESraQM94zuus/2s/Vz2Idy1qQkctNINYDZ61nfG1ngQ==
+
 "@types/body-parser@*":
   version "1.19.0"
   resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.0.tgz#0685b3c47eb3006ffed117cdd55164b61f80538f"
@@ -1469,6 +1474,11 @@
   dependencies:
     source-map "^0.6.1"
 
+"@types/uuid@^8.3.0":
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.0.tgz#215c231dff736d5ba92410e6d602050cce7e273f"
+  integrity sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==
+
 "@types/webpack-dev-server@^3.11.0":
   version "3.11.0"
   resolved "https://registry.yarnpkg.com/@types/webpack-dev-server/-/webpack-dev-server-3.11.0.tgz#bcc3b85e7dc6ac2db25330610513f2228c2fcfb2"
@@ -1506,6 +1516,13 @@
     "@types/webpack-sources" "*"
     source-map "^0.6.0"
 
+"@types/ws@^7.2.7":
+  version "7.2.7"
+  resolved "https://registry.yarnpkg.com/@types/ws/-/ws-7.2.7.tgz#362ad1a1d62721bdb725e72c8cccf357078cf5a3"
+  integrity sha512-UUFC/xxqFLP17hTva8/lVT0SybLUrfSD9c+iapKb0fEiC8uoDbA+xuZ3pAN603eW+bY8ebSMLm9jXdIPnD0ZgA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/yargs-parser@*":
   version "15.0.0"
   resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d"
@@ -2836,7 +2853,7 @@ binaryen@^85.0.0:
   resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-85.0.0.tgz#72c385d558862e52b75918bbce748d4627a98499"
   integrity sha512-01z4ZvA1O1hhNH+R4QQHuqzslecdHyzHYeuPMm4rB3hEiCP6BQlXZhHs68FgKfKT199YOt1uy5bmAIsilaOQ9g==
 
-bindings@^1.5.0:
+bindings@^1.3.0, bindings@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
   integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==
@@ -8636,6 +8653,11 @@ nan@^2.12.1, nan@^2.13.2:
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01"
   integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==
 
+nan@^2.14.1:
+  version "2.14.2"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"
+  integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==
+
 nanomatch@^1.2.9:
   version "1.2.13"
   resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119"
@@ -11164,6 +11186,16 @@ spectron@^11.0.0:
     split "^1.0.0"
     webdriverio "^6.1.20"
 
+speech-recorder@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/speech-recorder/-/speech-recorder-1.2.1.tgz#db050c39db951db099ae3ee07f4eba574faab770"
+  integrity sha512-kILQF+5ZS6XNgZo6zul+6PPMSYbryPaLs5qWLnjR2KtbCZGxNwuuPTiMaJYvvQRdFmJlJOSgsGylZYk1sAwR4Q==
+  dependencies:
+    bindings "^1.3.0"
+    nan "^2.14.1"
+    uuid "^3.3.3"
+    webrtcvad "^1.0.1"
+
 split-string@^3.0.1, split-string@^3.0.2:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2"
@@ -12312,7 +12344,7 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
 
-uuid@^3.3.2, uuid@^3.4.0:
+uuid@^3.3.2, uuid@^3.3.3, uuid@^3.4.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
@@ -12711,6 +12743,14 @@ webpack@^4.0.0, webpack@^4.18.0:
     watchpack "^1.7.4"
     webpack-sources "^1.4.1"
 
+webrtcvad@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/webrtcvad/-/webrtcvad-1.0.1.tgz#862b35c4ed2e7de3f39b268f48ef890ac85e02ce"
+  integrity sha512-oLfReCmGMpRducFWKP+o0GpKZLPj0u6qkln3P7wGaEzyjxtBiFuzvK28pmKF5SSihEkC1RGKO7Pi9C4VfY1q4Q==
+  dependencies:
+    bindings "^1.3.0"
+    node-addon-api "^1.7.1"
+
 websocket-driver@0.6.5:
   version "0.6.5"
   resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.6.5.tgz#5cb2556ceb85f4373c6d8238aa691c8454e13a36"
@@ -12885,7 +12925,7 @@ ws@^6.0.0, ws@^6.2.1:
   dependencies:
     async-limiter "~1.0.0"
 
-ws@^7.0.0, ws@^7.2.3:
+ws@^7.0.0, ws@^7.2.3, ws@^7.3.1:
   version "7.3.1"
   resolved "https://registry.yarnpkg.com/ws/-/ws-7.3.1.tgz#d0547bf67f7ce4f12a72dfe31262c68d7dc551c8"
   integrity sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==