From: riqo Date: Wed, 30 Dec 2020 16:55:27 +0000 (-0500) Subject: add base64 audio recorder X-Git-Tag: v0.9~119 X-Git-Url: https://andrewgundersen.net/repos?a=commitdiff_plain;h=5b0d462d1dd8d1b5b75ffc3a8f01c3002cd5a21f;p=mime-chat add base64 audio recorder --- diff --git a/build/config.gypi b/build/config.gypi new file mode 100644 index 0000000..6f84ed7 --- /dev/null +++ b/build/config.gypi @@ -0,0 +1,79 @@ +# Do not edit. File was generated by node-gyp's "configure" step +{ + "target_defaults": { + "cflags": [], + "default_configuration": "Release", + "defines": [], + "include_dirs": [], + "libraries": [] + }, + "variables": { + "asan": 0, + "build_v8_with_gn": "false", + "coverage": "false", + "dcheck_always_on": 0, + "debug_nghttp2": "false", + "debug_node": "false", + "enable_lto": "false", + "enable_pgo_generate": "false", + "enable_pgo_use": "false", + "error_on_warn": "false", + "force_dynamic_crt": 0, + "host_arch": "x64", + "icu_data_in": "../../deps/icu-tmp/icudt67l.dat", + "icu_endianness": "l", + "icu_gyp_path": "tools/icu/icu-generic.gyp", + "icu_path": "deps/icu-small", + "icu_small": "false", + "icu_ver_major": "67", + "is_debug": 0, + "llvm_version": "0.0", + "napi_build_version": "6", + "node_byteorder": "little", + "node_debug_lib": "false", + "node_enable_d8": "false", + "node_install_npm": "true", + "node_module_version": 83, + "node_no_browser_globals": "false", + "node_prefix": "/", + "node_release_urlbase": "https://nodejs.org/download/release/", + "node_shared": "false", + "node_shared_brotli": "false", + "node_shared_cares": "false", + "node_shared_http_parser": "false", + "node_shared_libuv": "false", + "node_shared_nghttp2": "false", + "node_shared_openssl": "false", + "node_shared_zlib": "false", + "node_tag": "", + "node_target_type": "executable", + "node_use_bundled_v8": "true", + "node_use_dtrace": "true", + "node_use_etw": "false", + "node_use_node_code_cache": "true", + "node_use_node_snapshot": "true", + "node_use_openssl": "true", + "node_use_v8_platform": "true", + "node_with_ltcg": "false", + "node_without_node_options": "false", + "openssl_fips": "", + "openssl_is_fips": "false", + "shlib_suffix": "83.dylib", + "target_arch": "x64", + "v8_enable_31bit_smis_on_64bit_arch": 0, + "v8_enable_gdbjit": 0, + "v8_enable_i18n_support": 1, + "v8_enable_inspector": 1, + "v8_enable_pointer_compression": 0, + "v8_no_strict_aliasing": 1, + "v8_optimized_debug": 1, + "v8_promise_internal_field_count": 1, + "v8_random_seed": 0, + "v8_trace_maps": 0, + "v8_use_siphash": 1, + "want_separate_host_toolset": 0, + "xcode_version": "11.0", + "nodedir": "/Users/Enrique/Library/Caches/node-gyp/14.4.0", + "standalone_static_library": 1 + } +} diff --git a/package.json b/package.json index 2292dc9..3f760b1 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "animejs": "^3.2.0", "core-js": "^3.6.5", "mitt": "^2.1.0", - "speech-recorder": "^1.2.1", + "naudiodon": "^2.3.2", "vue": "^3.0.0-0", "vue-router": "^4.0.0-0", "vuex": "^4.0.0-0", diff --git a/rawAudio.raw b/rawAudio.raw new file mode 100644 index 0000000..e69de29 diff --git a/src/background.ts b/src/background.ts index 0ac5900..169469a 100644 --- a/src/background.ts +++ b/src/background.ts @@ -5,20 +5,29 @@ import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer"; const isDevelopment = process.env.NODE_ENV !== "production"; import * as path from "path"; -import { SpeechRecorder } from "speech-recorder"; const fs = require('fs'); -// TODO: connect To Crimata-BE -import WebSocket from "ws"; +const portAudio = require('naudiodon'); + +// Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream +const audioOptions = { + channelCount: 2, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: 44100, + deviceId: -1, // Use -1 or omit the deviceId to select the default device + closeOnError: true // Close the stream if an audio error is detected, if set false then just log the error +} +let ai = new portAudio.AudioIO({ + inOptions: audioOptions +}); -const recorder = new SpeechRecorder({ sampleRate: 16000, framesPerBuffer: 320, level: 0 }); -// const writeStream = fs.createWriteStream("audio.raw"); +import WebSocket from "ws"; +const ws = new WebSocket("ws://localhost:8080"); // Keep a global reference of the window object, if you don't, the window will // be closed automatically when the JavaScript object is garbage collected. let win: BrowserWindow | null; -let ws: any; // Scheme must be registered before the app is ready protocol.registerSchemesAsPrivileged([ @@ -53,7 +62,6 @@ function createWindow() { win.webContents.on('did-finish-load', () => { - ws = new WebSocket("ws://localhost:8080"); ws.on("open", function open() { // ws.send("hernandeze2@xavier.edu"); }); @@ -67,44 +75,40 @@ function createWindow() { } }); - ipcMain.on('send-message', (Event: any, message: any) => { - const stringMessage = JSON.stringify(message) - ws.send(stringMessage); + ipcMain.on('update-menu-bar', (Event: any, payload: any) => { + // win.setTitle("Listening..."); }) - ipcMain.on('update-menu-bar', (Event: any, payload: any) => { + ipcMain.on('send-message', (Event: any, payload: any) => { + ws.send(JSON.stringify(payload)); // win.setTitle("Listening..."); }) - let audioData: any[] = []; + let audioData = ""; ipcMain.on('update-recorder', (Event: any, record: boolean) => { - console.log('updating recorder', record) if (record) { - recorder.start({ - onAudio: (audio: any) => { - audioData.push(audio.buffer) - // writeStream.write(audio); - } - }) + ai.start(); + ai.on('data', (buf: Buffer) => { + const base64Data = buf.toString('base64'); + audioData += base64Data; + }); } else { - recorder.stop() - // send audio to BE + ai.quit(); const message = { - text: '', + text: "", audio: audioData } ws.send(JSON.stringify(message)) - console.log('sent', message) - - // clear audio.raw file - audioData = []; + ai = new portAudio.AudioIO({ + inOptions: audioOptions + }) + audioData = ""; } }) }) win.on("closed", () => { win = null; - ws = null; }); } diff --git a/src/core/UI/input/audio/useAudioRecorder.ts b/src/core/UI/input/audio/useAudioRecorder.ts index 289d647..52ea69f 100644 --- a/src/core/UI/input/audio/useAudioRecorder.ts +++ b/src/core/UI/input/audio/useAudioRecorder.ts @@ -17,13 +17,12 @@ export default function useAudioRecorder(record: Ref, stream: MediaStre const bufferLen = 4096; const numChannels = 1; - window.AudioContext = window.AudioContext; + //window.AudioContext = window.AudioContext; const audioContext = new AudioContext({ sampleRate: 16000 }); const source = audioContext.createMediaStreamSource(stream); const context = source.context; const sampleRate = context.sampleRate; - console.log(sampleRate) const node = context.createScriptProcessor.call( context, @@ -47,12 +46,12 @@ export default function useAudioRecorder(record: Ref, stream: MediaStre node.disconnect(context.destination); const message = { - text: '', - audio: { - sampleRate: sampleRate, - channelCount: numChannels, - audio: audioArray - } + text: '', + audio: { + sampleRate: sampleRate, + channelCount: numChannels, + audio: audioArray + } } window.postMessage({ myTypeField: 'send-message', @@ -65,4 +64,4 @@ export default function useAudioRecorder(record: Ref, stream: MediaStre source.connect(node); -} \ No newline at end of file +} diff --git a/src/core/UI/input/audio/useStreamRecord.ts b/src/core/UI/input/audio/useStreamRecord.ts index 87dae82..4380d23 100644 --- a/src/core/UI/input/audio/useStreamRecord.ts +++ b/src/core/UI/input/audio/useStreamRecord.ts @@ -1,4 +1,4 @@ -import {inject} from 'vue'; +import { inject } from 'vue'; export default function useStreamRecord(): { initRecorder: (stream: MediaStream) => MediaRecorder; } { @@ -9,23 +9,23 @@ export default function useStreamRecord(): { mimeType: "audio/webm; codecs=opus", } - console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; ')) + // console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; ')) const initRecorder = (stream: MediaStream) => { - const audioTrack = stream.getAudioTracks()[0]; - const audioSettings = audioTrack.getSettings() + const audioTrack = stream.getAudioTracks()[0]; + const audioSettings = audioTrack.getSettings() streamRecorder = new MediaRecorder(stream, options); - streamRecorder.ondataavailable = (e: any) => { - if (e.data.size > 0) { + streamRecorder.ondataavailable = (e: any) => { + if (e.data.size > 0) { chunks.push(e.data); - } else { + } else { // ... - } + } //chunks.push(e.data) } streamRecorder.onstop = async (e: any) => { - const audioBlob = new Blob(chunks, { + const audioBlob = new Blob(chunks, { type: options.mimeType }); const arrayBuffer = await audioBlob.arrayBuffer() @@ -41,8 +41,8 @@ export default function useStreamRecord(): { } } window.postMessage({ - myTypeField: 'send-message', - message: message + myTypeField: 'send-message', + message: message }, '*') } diff --git a/src/core/UI/input/useInputController.ts b/src/core/UI/input/useInputController.ts index 0a94fc9..b5f6f84 100644 --- a/src/core/UI/input/useInputController.ts +++ b/src/core/UI/input/useInputController.ts @@ -37,12 +37,10 @@ export default function useInputController() { function keyupHandler(e: any) { if (e.key === " " && visualizeStream.value) { console.log("stop recording"); - // recorder.stop(); - - // window.postMessage({ - // myTypeField: 'update-recorder', - // record: false - // }, '*') + window.postMessage({ + myTypeField: 'update-recorder', + record: false + }, '*') visualizeStream.value = false; paths.value = [] @@ -73,12 +71,10 @@ export default function useInputController() { if (input === " " && visualizeStream.value === false) { console.log("record MediaStream"); visualizeStream.value = true; - // recorder.start(); - - // window.postMessage({ - // myTypeField: 'update-recorder', - // record: true - // }, '*') + window.postMessage({ + myTypeField: 'update-recorder', + record: true + }, '*') expandBubble(audioBubbleWidth); visualizeStreamAsPaths(stream, paths, audioBubbleWidth); diff --git a/src/core/UI/scrollView/useScrollView.ts b/src/core/UI/scrollView/useScrollView.ts index 6d3a5e3..859b704 100644 --- a/src/core/UI/scrollView/useScrollView.ts +++ b/src/core/UI/scrollView/useScrollView.ts @@ -1,56 +1,57 @@ -import {ref, computed, onMounted} from "vue"; -export default function useScrollView({ targetId }: {targetId: string}) { - const scroll = ref(0); - let scrollTarget: HTMLElement | SVGElement | null; - const targetHeight = ref(0); - let topBound: number; +import { ref, computed, onMounted } from "vue"; +export default function useScrollView({ targetId }: { targetId: string }) { + const scroll = ref(0); + let scrollTarget: HTMLElement | SVGElement | null; + const targetHeight = ref(0); + let topBound: number; + const scrollStartHeight = 445; - const clampedScroll = computed(() => { - if (targetHeight.value === 0) { return 0; } - topBound = -targetHeight.value + 450; - return Math.max(topBound, Math.min(scroll.value, 0)) as number; - }) + const clampedScroll = computed(() => { + if (targetHeight.value === 0) { return 0; } + topBound = -targetHeight.value + 450; + return Math.max(topBound, Math.min(scroll.value, 0)) as number; + }) - const scrollView = computed(() => { - if (clampedScroll.value === 0) { - scroll.value = 0; - } else if (clampedScroll.value === topBound) { - scroll.value = topBound; - } - return `0 ${clampedScroll.value} 350 500`; - }); - - const getHeight = () => { - if (scrollTarget) { - return scrollTarget.getBoundingClientRect().height as number; - } + const scrollView = computed(() => { + if (clampedScroll.value === 0) { + scroll.value = 0; + } else if (clampedScroll.value === topBound) { + scroll.value = topBound; } + return `0 ${clampedScroll.value} 350 500`; + }); - const handleScrollEvent = (verticalScroll: number) => { - if (scrollTarget) { - // only scroll if renderer height is greater than main window - const heightResult = getHeight(); - if (heightResult) { - if (heightResult > 500) { - targetHeight.value = heightResult; - scroll.value += verticalScroll; - } - } - } + const getHeight = () => { + if (scrollTarget) { + return scrollTarget.getBoundingClientRect().height as number; } + } - onMounted(() => { - scrollTarget = document.getElementById(targetId); - if (scrollTarget) { - window.addEventListener("wheel", (e) => { - const verticalScroll = e.deltaY * 0.25; - handleScrollEvent(verticalScroll); - }) + const handleScrollEvent = (verticalScroll: number) => { + if (scrollTarget) { + const heightResult = getHeight(); + if (heightResult) { + // only scroll if renderer scrollStart + if (heightResult > scrollStartHeight) { + targetHeight.value = heightResult; + scroll.value += verticalScroll; + } } - }); + } + } - return { - scrollView + onMounted(() => { + scrollTarget = document.getElementById(targetId); + if (scrollTarget) { + window.addEventListener("wheel", (e) => { + const verticalScroll = e.deltaY * 0.25; + handleScrollEvent(verticalScroll); + }) } + }); + + return { + scrollView + } -} \ No newline at end of file +} diff --git a/src/core/messenger/message/messageDetails/useComputed.ts b/src/core/messenger/message/messageDetails/useComputed.ts index b5eed34..dc4cfaf 100644 --- a/src/core/messenger/message/messageDetails/useComputed.ts +++ b/src/core/messenger/message/messageDetails/useComputed.ts @@ -49,7 +49,6 @@ export default function useMessageItemComp(m: string, id: string) { if (messageRenderer) { const rect = messageRenderer.getBoundingClientRect(); let Y = 600; - console.log('renderer height',rect.height) if (rect.height > 500) { Y = rect.height + messageMarginTop; } diff --git a/src/core/messenger/messageView/useMessageView.ts b/src/core/messenger/messageView/useMessageView.ts index c7685bf..e1eae8d 100644 --- a/src/core/messenger/messageView/useMessageView.ts +++ b/src/core/messenger/messageView/useMessageView.ts @@ -54,7 +54,6 @@ export default function useMessageView(): { // get message positioning & shift state before animating if (message) { messageOffset = calculateMessageOffsets(message); - console.log(messageOffset.y) shift = messageOffset.y > 400 ? true : false; } if (el) { diff --git a/testServer.js b/testServer.js index 3907aaf..2cf1d80 100644 --- a/testServer.js +++ b/testServer.js @@ -34,10 +34,7 @@ const wss = new WebSocket.Server({ wss.on("connection", function connection(ws, req) { ws.on("message", function incoming(message) { const parsed = JSON.parse(message) - if (parsed.audio) { - console.log(parsed.audio) - } - console.log("received: %s:", parsed); + console.log(parsed) }); const ip = req.socket.remoteAddress; diff --git a/yarn.lock b/yarn.lock index ebae7d4..40823de 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2903,7 +2903,7 @@ binaryen@^85.0.0: resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-85.0.0.tgz#72c385d558862e52b75918bbce748d4627a98499" integrity sha512-01z4ZvA1O1hhNH+R4QQHuqzslecdHyzHYeuPMm4rB3hEiCP6BQlXZhHs68FgKfKT199YOt1uy5bmAIsilaOQ9g== -bindings@^1.3.0, bindings@^1.5.0: +bindings@^1.2.1, bindings@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== @@ -8742,7 +8742,7 @@ mz@^2.4.0: object-assign "^4.0.1" thenify-all "^1.0.0" -nan@^2.12.1, nan@^2.13.2, nan@^2.14.1: +nan@^2.12.1, nan@^2.13.2, nan@^2.14.0: version "2.14.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== @@ -8769,6 +8769,14 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= +naudiodon@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/naudiodon/-/naudiodon-2.3.2.tgz#e1b14902e830e0140dcbfd80d81f7cf75a777c79" + integrity sha512-ugxUwZNs9mvY4p4bhKLftX+06cYDN6XLjcXZiW3B72Mzce63OXmynSemeUaCqLpTXQRipyaySc61dNwtLrxfng== + dependencies: + bindings "^1.5.0" + segfault-handler "^1.3.0" + negotiator@0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" @@ -10863,6 +10871,14 @@ scss-tokenizer@^0.2.3: js-base64 "^2.1.8" source-map "^0.4.2" +segfault-handler@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/segfault-handler/-/segfault-handler-1.3.0.tgz#054bc847832fa14f218ba6a79e42877501c8870e" + integrity sha512-p7kVHo+4uoYkr0jmIiTBthwV5L2qmWtben/KDunDZ834mbos+tY+iO0//HpAJpOFSQZZ+wxKWuRo4DxV02B7Lg== + dependencies: + bindings "^1.2.1" + nan "^2.14.0" + select-hose@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -11282,16 +11298,6 @@ spectron@^11.0.0: split "^1.0.0" webdriverio "^6.1.20" -speech-recorder@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/speech-recorder/-/speech-recorder-1.2.1.tgz#db050c39db951db099ae3ee07f4eba574faab770" - integrity sha512-kILQF+5ZS6XNgZo6zul+6PPMSYbryPaLs5qWLnjR2KtbCZGxNwuuPTiMaJYvvQRdFmJlJOSgsGylZYk1sAwR4Q== - dependencies: - bindings "^1.3.0" - nan "^2.14.1" - uuid "^3.3.3" - webrtcvad "^1.0.1" - split-string@^3.0.1, split-string@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" @@ -12445,7 +12451,7 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= -uuid@^3.3.2, uuid@^3.3.3, uuid@^3.4.0: +uuid@^3.3.2, uuid@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== @@ -12850,14 +12856,6 @@ webpack@^4.0.0, webpack@^4.18.0: watchpack "^1.7.4" webpack-sources "^1.4.1" -webrtcvad@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/webrtcvad/-/webrtcvad-1.0.1.tgz#862b35c4ed2e7de3f39b268f48ef890ac85e02ce" - integrity sha512-oLfReCmGMpRducFWKP+o0GpKZLPj0u6qkln3P7wGaEzyjxtBiFuzvK28pmKF5SSihEkC1RGKO7Pi9C4VfY1q4Q== - dependencies: - bindings "^1.3.0" - node-addon-api "^1.7.1" - websocket-driver@0.6.5: version "0.6.5" resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.6.5.tgz#5cb2556ceb85f4373c6d8238aa691c8454e13a36"