From 5872d245f0cc881363601a3f39ad95055a865f65 Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Wed, 7 Jul 2021 07:10:13 -0500 Subject: [PATCH] add audio worklet --- audio/audioNode.ts | 0 package.json | 22 +----- public/processor.js | 12 ++++ public/worklet/audioProcessor.ts | 33 +++++++++ src/ipc/handlers.ts | 2 +- src/render/audio/audio.worklet.ts | 33 +++++++++ src/render/audio/audioNode.ts | 9 +++ src/render/audio/setupAudio.ts | 74 +++++++++++++++++++++ src/render/main.ts | 3 + src/render/shims-vue.d.ts | 5 ++ tests/audio.js | 107 +++++------------------------- tests/transcribe.js | 38 +++++++++++ tsconfig.worklet.json | 23 +++++++ vue.config.js | 50 ++++++++++++++ yarn.lock | 53 +++++++++++++-- 15 files changed, 347 insertions(+), 117 deletions(-) create mode 100644 audio/audioNode.ts create mode 100644 public/processor.js create mode 100644 public/worklet/audioProcessor.ts create mode 100644 src/render/audio/audio.worklet.ts create mode 100644 src/render/audio/audioNode.ts create mode 100644 src/render/audio/setupAudio.ts create mode 100644 tests/transcribe.js create mode 100644 tsconfig.worklet.json create mode 100644 vue.config.js diff --git a/audio/audioNode.ts b/audio/audioNode.ts new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json index 24d13b1..081698f 100644 --- a/package.json +++ b/package.json @@ -66,25 +66,9 @@ "spectron": "11.0.0", "typescript": "~3.9.3", "vue-cli-plugin-electron-builder": "~2.0.0-rc.6", - "vue-jest": "^5.0.0-0" - }, - "vue": { - "lintOnSave": false, - "pluginOptions": { - "electronBuilder": { - "mainProcessFile": "./src/init.ts", - "rendererProcessFile": "./src/render/main.ts", - "preload": "./src/render/preload.ts", - "builderOptions": { - "appId": "com.crimata.ElectronUpdaterApp", - "artifactName": "${productName}-${version}.${ext}", - "publish": { - "provider": "generic", - "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build" - } - } - } - } + "vue-jest": "^5.0.0-0", + "worker-loader": "^3.0.8", + "worklet-loader": "^1.0.0" }, "gitHooks": { "pre-commit": "lint-staged" diff --git a/public/processor.js b/public/processor.js new file mode 100644 index 0000000..df735a5 --- /dev/null +++ b/public/processor.js @@ -0,0 +1,12 @@ + +class AudioProcessor extends AudioWorkletProcessor { + process (inputs, outputs, parameters) { + console.log(inputs); + console.log(outputs); + console.log(parameters); + return true + } +} + +registerProcessor('processor', AudioProcessor) + diff --git a/public/worklet/audioProcessor.ts b/public/worklet/audioProcessor.ts new file mode 100644 index 0000000..8c8336f --- /dev/null +++ b/public/worklet/audioProcessor.ts @@ -0,0 +1,33 @@ +interface AudioWorkletProcessor { + readonly port: MessagePort; + process( + inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record + ): boolean; +} + +declare let AudioWorkletProcessor: { + prototype: AudioWorkletProcessor; + new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor; +}; + +declare function registerProcessor( + name: string, + processorCtor: (new ( + options?: AudioWorkletNodeOptions + ) => AudioWorkletProcessor) & { + parameterDescriptors?: AudioParamDescriptor[]; + } +): undefined; + +class AudioProcessor extends AudioWorkletProcessor { + process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record) { + console.log(inputs); + console.log(outputs); + console.log(parameters); + return true + } +} + +registerProcessor('audio-processor', AudioProcessor) diff --git a/src/ipc/handlers.ts b/src/ipc/handlers.ts index 5529662..7a9ba49 100644 --- a/src/ipc/handlers.ts +++ b/src/ipc/handlers.ts @@ -1,7 +1,7 @@ "use strict"; -import { accountLogin, accountLogout, accountProfile } from "@/account"; +import { accountLogin, accountLogout } from "@/account"; import { IpcHandler } from "@/composables/useIpcMain"; import { flush } from "@/audio"; diff --git a/src/render/audio/audio.worklet.ts b/src/render/audio/audio.worklet.ts new file mode 100644 index 0000000..8c8336f --- /dev/null +++ b/src/render/audio/audio.worklet.ts @@ -0,0 +1,33 @@ +interface AudioWorkletProcessor { + readonly port: MessagePort; + process( + inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record + ): boolean; +} + +declare let AudioWorkletProcessor: { + prototype: AudioWorkletProcessor; + new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor; +}; + +declare function registerProcessor( + name: string, + processorCtor: (new ( + options?: AudioWorkletNodeOptions + ) => AudioWorkletProcessor) & { + parameterDescriptors?: AudioParamDescriptor[]; + } +): undefined; + +class AudioProcessor extends AudioWorkletProcessor { + process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record) { + console.log(inputs); + console.log(outputs); + console.log(parameters); + return true + } +} + +registerProcessor('audio-processor', AudioProcessor) diff --git a/src/render/audio/audioNode.ts b/src/render/audio/audioNode.ts new file mode 100644 index 0000000..245bfed --- /dev/null +++ b/src/render/audio/audioNode.ts @@ -0,0 +1,9 @@ + +// MyWorkletNode.js +export default class MyWorkletNode extends AudioWorkletNode { + constructor(context) { + super(context, '-processor') + console.log(this.channelCount) + } +} + diff --git a/src/render/audio/setupAudio.ts b/src/render/audio/setupAudio.ts new file mode 100644 index 0000000..f69d4b0 --- /dev/null +++ b/src/render/audio/setupAudio.ts @@ -0,0 +1,74 @@ +import AudioProcessor from "./audio.worklet.ts"; +console.log('YAYAYAYYA', AudioProcessor) +//TODO: need to set export directory for audio worklet +// webpack!!! + + const constraints = { + audio: { + echoCancellation: true, + autoGainControl: true, + noiseSuppression: true, + channelCount: 1 + }, + video: false + } +async function getWebAudioMediaStream() { + if (!window.navigator.mediaDevices) { + throw new Error( + "This browser does not support web audio or it is not enabled." + ); + } + + try { + const result = await window.navigator.mediaDevices.getUserMedia(constraints); + + return result; + } catch (e) { + switch (e.name) { + case "NotAllowedError": + throw new Error( + "A recording device was found but has been disallowed for this application. Enable the device in the browser settings." + ); + + case "NotFoundError": + throw new Error( + "No recording device was found. Please attach a microphone and click Retry." + ); + + default: + throw e; + } + } +} + +export async function setupAudio() { + // Get the browser audio. Awaits user "allowing" it for the current tab. + const mediaStream = await getWebAudioMediaStream(); + + const context = new window.AudioContext(); + const audioSource = context.createMediaStreamSource(mediaStream); + + let node; + + // Add our audio processor worklet to the context. + + try { + await context.audioWorklet.addModule("audio.worklet.js"); + } catch (e) { + throw new Error( + `Failed to load audio analyzer worklet. Further info: ${e.message}` + ); + } + + node = new AudioWorkletNode(context, 'audio-processor') + + // Connect the audio source (microphone output) to our analysis node. + audioSource.connect(node); + + // Connect our analysis node to the output. Required even though we do not + // output any audio. Allows further downstream audio processing or output to + // occur. + node.connect(context.destination); + + return { context, node }; +} diff --git a/src/render/main.ts b/src/render/main.ts index 5375889..9f986ed 100644 --- a/src/render/main.ts +++ b/src/render/main.ts @@ -3,6 +3,7 @@ import App from "./App.vue"; +import { setupAudio } from "./audio/setupAudio"; import mitt from "mitt"; import { createApp } from "vue"; @@ -12,6 +13,8 @@ import { initIpcRendererListeners } from "./ipc" // Handle ipcMain events. initIpcRendererListeners(); +setupAudio(); + // Handle events. const emitter = mitt(); diff --git a/src/render/shims-vue.d.ts b/src/render/shims-vue.d.ts index 35a61a9..35b2544 100644 --- a/src/render/shims-vue.d.ts +++ b/src/render/shims-vue.d.ts @@ -11,3 +11,8 @@ declare module "anime-js" { export = anime; } } + +declare module "*.worklet.ts" { + const exportString: string; + export default exportString; +} diff --git a/tests/audio.js b/tests/audio.js index b8ab22f..ba21748 100644 --- a/tests/audio.js +++ b/tests/audio.js @@ -1,84 +1,38 @@ -const speech = require('@google-cloud/speech'); +// const speech = require('@google-cloud/speech'); const portAudio = require('naudiodon'); // const rs = fs.createReadStream('rawAudio.wav'); -// Creates a client -const client = new speech.SpeechClient(); - -const encoding = 'LINEAR16'; -const sampleRateHertz = 16000; -const languageCode = 'en-US'; - const audioContainer = { input: '', buffers: [] } -const config = { - encoding: encoding, - sampleRateHertz: sampleRateHertz, - languageCode: languageCode, -}; - -/** - * Note that transcription is limited to 60 seconds audio. - * Use a GCS file for audio longer than 1 minute. - */ -async function transcribeSpeech (audio) { - const request = { - config: config, - audio: audio, - }; - - // Detects speech in the audio file. This creates a recognition job that you - // can wait for now, or get its result later. - const [operation] = await client.longRunningRecognize(request); - - // Get a Promise representation of the final result of the job - const [response] = await operation.promise(); - - const transcription = response.results - .map(result => result.alternatives[0].transcript) - .join('\n'); - console.log(`Transcription: ${transcription}`); -} let record = true; // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream -const ia = new portAudio.AudioIO({ +const aio = new portAudio.AudioIO({ inOptions: { - channelCount: 1, - sampleFormat: 16, - sampleRate: 16000, - deviceId: -1, - closeOnError: false, + channelCount: 2, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: 44100, + deviceId: -1 // Use -1 or omit the deviceId to select the default device }, -}); -ia.setEncoding('base64'); -ia.start(); -ia.on('data', (chunk) => { - if (record) { - console.log('recording data'); - audioContainer.input += chunk; - } else { - if (audioContainer.input.length) audioContainer.input = ""; - } + outOptions: { + channelCount: 2, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: 44100, + deviceId: -1 // Use -1 or omit the deviceId to select the default device + } }); -const ao = new portAudio.AudioIO({ - outOptions: { - sampleFormat: 16, - channelCount: 1, - sampleRate: 16000, - deviceId: -1, - closeOnError: false, - } -}); -ao.start(); -let counter = 0; +aio.start() +aio.read() +aio.on('data', buf => console.log(buf.timestamp)); + +const counter = 0; const tests = []; function testCallback() { @@ -143,13 +97,6 @@ function bufSplit(input){ } async function test() { - transcribeSpeech({ - content: Buffer.from(audioContainer.input, 'base64') - }); - tests.push(audioContainer.input) - counter++; - console.log('audio string length:', audioContainer.input.length) - console.log('buffers written: ', audioContainer.buffers.length) } setTimeout(async () => { @@ -157,26 +104,6 @@ setTimeout(async () => { test() }, 4000); -// setTimeout(() => { -// record = true; -// }, 6000) -// -// setTimeout(async () => { -// record = false; -// test(); -// }, 9000); -// -// setTimeout(() => { -// record = true; -// }, 11000) -// -// setTimeout(async () => { -// record = false; -// test(); -// }, 14000); -// setTimeout(async () => { - ia.quit(); - testCallback() return; }, 6000); diff --git a/tests/transcribe.js b/tests/transcribe.js new file mode 100644 index 0000000..11b38d7 --- /dev/null +++ b/tests/transcribe.js @@ -0,0 +1,38 @@ + +const speech = require('@google-cloud/speech'); + +// Creates a client +const client = new speech.SpeechClient(); + +const encoding = 'LINEAR16'; +const sampleRateHertz = 16000; +const languageCode = 'en-US'; + +const config = { + encoding: encoding, + sampleRateHertz: sampleRateHertz, + languageCode: languageCode, +}; + +/** + * Note that transcription is limited to 60 seconds audio. + * Use a GCS file for audio longer than 1 minute. + */ +async function transcribeSpeech (audio) { + const request = { + config: config, + audio: audio, + }; + + // Detects speech in the audio file. This creates a recognition job that you + // can wait for now, or get its result later. + const [operation] = await client.longRunningRecognize(request); + + // Get a Promise representation of the final result of the job + const [response] = await operation.promise(); + + const transcription = response.results + .map(result => result.alternatives[0].transcript) + .join('\n'); + console.log(`Transcription: ${transcription}`); +} diff --git a/tsconfig.worklet.json b/tsconfig.worklet.json new file mode 100644 index 0000000..f304b84 --- /dev/null +++ b/tsconfig.worklet.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2018", + "module": "esnext", + "strict": true, + "importHelpers": true, + "moduleResolution": "node", + "resolveJsonModule": true, + "experimentalDecorators": true, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "sourceMap": true, + "baseUrl": ".", + "types": ["webpack-env"], + "paths": { + "@/*": ["src/*"] + }, + "lib": ["esnext", "scripthost"] + }, + "include": ["src/**/*.worklet.ts"], + "exclude": ["node_modules"] +} diff --git a/vue.config.js b/vue.config.js new file mode 100644 index 0000000..2f1f526 --- /dev/null +++ b/vue.config.js @@ -0,0 +1,50 @@ + +const path = require("path"); + +module.exports = { + assetsDir: "../../static/SPA" +} +module.exports = { + lintOnSave: false, + pluginOptions: { + electronBuilder: { + mainProcessFile: "./src/init.ts", + rendererProcessFile: "./src/render/main.ts", + preload: "./src/render/preload.ts", + + + builderOptions: { + "appId": "com.crimata.ElectronUpdaterApp", + "artifactName": "${productName}-${version}.${ext}", + "publish": { + "provider": "generic", + "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build" + } + }, + + + chainWebpackRendererProcess: (config) => { + // Chain webpack config for electron renderer process only (won't be applied to web builds) + config.outputDir = path.resolve(__dirnamei, "js"); + config.module + .rule('worklet') + .test(/\.worklet\.ts$/) + .use('worklet-loader') + .loader('worklet-loader') + .tap(options => { + options.name = "js/[hash].worklet.js"; + return options + }) + .end() + // Add another loader + .use('ts-loader') + .loader('ts-loader') + .tap(options => { + options.configFile = "tsconfig.worklet.json"; + return options + }) + .end() + }, + } + } +}; diff --git a/yarn.lock b/yarn.lock index 465adc9..15c2b5a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1456,6 +1456,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0" integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== +"@types/json-schema@^7.0.6": + version "7.0.7" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" + integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== + "@types/long@^4.0.0", "@types/long@^4.0.1": version "4.0.1" resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" @@ -2367,7 +2372,7 @@ ajv-keywords@^3.1.0, ajv-keywords@^3.4.1, ajv-keywords@^3.5.2: resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== -ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4: +ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -4898,11 +4903,6 @@ dotenv-expand@^5.1.0: resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0" integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA== -dotenv@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" - integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== - dotenv@^8.2.0: version "8.2.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a" @@ -6649,6 +6649,11 @@ hmac-drbg@^1.0.0: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.1" +hoek@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" + integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== + hoopy@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/hoopy/-/hoopy-0.1.4.tgz#609207d661100033a9a9402ad3dea677381c1b1d" @@ -8447,7 +8452,7 @@ loader-utils@^0.2.16: json5 "^0.5.0" object-assign "^4.0.1" -loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0: +loader-utils@^1.0.0, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613" integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA== @@ -11154,6 +11159,14 @@ schema-utils@2.7.0: ajv "^6.12.2" ajv-keywords "^3.4.1" +schema-utils@^0.4.0: + version "0.4.7" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187" + integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ== + dependencies: + ajv "^6.1.0" + ajv-keywords "^3.1.0" + schema-utils@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770" @@ -11172,6 +11185,15 @@ schema-utils@^2.0.0, schema-utils@^2.5.0, schema-utils@^2.6.1, schema-utils@^2.6 ajv "^6.12.4" ajv-keywords "^3.5.2" +schema-utils@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.0.0.tgz#67502f6aa2b66a2d4032b4279a2944978a0913ef" + integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA== + dependencies: + "@types/json-schema" "^7.0.6" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + scss-tokenizer@^0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz#8eb06db9a9723333824d3f5530641149847ce5d1" @@ -13300,6 +13322,14 @@ worker-farm@^1.7.0: dependencies: errno "~0.1.7" +worker-loader@^3.0.8: + version "3.0.8" + resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-3.0.8.tgz#5fc5cda4a3d3163d9c274a4e3a811ce8b60dbb37" + integrity sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g== + dependencies: + loader-utils "^2.0.0" + schema-utils "^3.0.0" + worker-rpc@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5" @@ -13307,6 +13337,15 @@ worker-rpc@^0.1.0: dependencies: microevent.ts "~0.1.1" +worklet-loader@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/worklet-loader/-/worklet-loader-1.0.0.tgz#17e2eef75981de469c1e1e200ad1ffb54efe7a29" + integrity sha512-4yFqiGDwICoJB4ZbWHzCzyTyDrRnCU1XfvSJtjiBBDreuWDYpA6wf8yqQjNckcjL1jm/sT9ocSvP5tJnmsMOLA== + dependencies: + hoek "^4.2.1" + loader-utils "^1.0.0" + schema-utils "^0.4.0" + wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba" -- 2.43.0