From 706bde4d586b6563fd4ad5d54229d85e837f16d0 Mon Sep 17 00:00:00 2001 From: riqo Date: Wed, 30 Dec 2020 15:24:39 -0500 Subject: [PATCH] add 16int audio recorder --- src/background.ts | 6 +- src/core/UI/input/audio/useAudioRecorder.ts | 68 +++++++++++++++++++ src/core/UI/input/audio/useStreamRecord.ts | 62 +++++++++-------- .../UI/input/audio/useStreamVisualizer.ts | 1 - src/core/UI/input/useInputController.ts | 22 +++--- src/preload.ts | 2 +- testServer.js | 6 +- 7 files changed, 125 insertions(+), 42 deletions(-) create mode 100644 src/core/UI/input/audio/useAudioRecorder.ts diff --git a/src/background.ts b/src/background.ts index dc05c35..0ac5900 100644 --- a/src/background.ts +++ b/src/background.ts @@ -55,7 +55,7 @@ function createWindow() { ws = new WebSocket("ws://localhost:8080"); ws.on("open", function open() { - ws.send("hernandeze2@xavier.edu"); + // ws.send("hernandeze2@xavier.edu"); }); ws.on("message", (message: any) => { const parsed = JSON.parse(message); @@ -67,8 +67,8 @@ function createWindow() { } }); - ipcMain.on('send-message', (Event: any, payload: any) => { - const stringMessage = JSON.stringify(payload.message) + ipcMain.on('send-message', (Event: any, message: any) => { + const stringMessage = JSON.stringify(message) ws.send(stringMessage); }) diff --git a/src/core/UI/input/audio/useAudioRecorder.ts b/src/core/UI/input/audio/useAudioRecorder.ts new file mode 100644 index 0000000..289d647 --- /dev/null +++ b/src/core/UI/input/audio/useAudioRecorder.ts @@ -0,0 +1,68 @@ + +interface Window { + AudioContext: any; + webkitAudioContext: any; +} + +interface Event { + inputBuffer: AudioBuffer; +} + +import { Ref } from '@/types/vueRef/index'; +import { watch } from 'vue'; + +export default function useAudioRecorder(record: Ref, stream: MediaStream) { + + let audioArray: Int16Array[] = []; + const bufferLen = 4096; + const numChannels = 1; + + window.AudioContext = window.AudioContext; + const audioContext = new AudioContext({ sampleRate: 16000 }); + + const source = audioContext.createMediaStreamSource(stream); + const context = source.context; + const sampleRate = context.sampleRate; + console.log(sampleRate) + + const node = context.createScriptProcessor.call( + context, + bufferLen, + numChannels, + numChannels + ); + + node.onaudioprocess = (e: Event) => { + const inputBuffer = new Float32Array(bufferLen); + + e.inputBuffer.copyFromChannel(inputBuffer, 0); + const int16 = new Int16Array(inputBuffer.buffer); + audioArray.push(int16); + }; + + watch(record, (beginRecording) => { + if (beginRecording) { + node.connect(context.destination); + } else { + node.disconnect(context.destination); + + const message = { + text: '', + audio: { + sampleRate: sampleRate, + channelCount: numChannels, + audio: audioArray + } + } + window.postMessage({ + myTypeField: 'send-message', + message: message + }, '*') + + audioArray = []; + } + }) + + source.connect(node); + +} \ No newline at end of file diff --git a/src/core/UI/input/audio/useStreamRecord.ts b/src/core/UI/input/audio/useStreamRecord.ts index 7cdd393..87dae82 100644 --- a/src/core/UI/input/audio/useStreamRecord.ts +++ b/src/core/UI/input/audio/useStreamRecord.ts @@ -5,37 +5,45 @@ export default function useStreamRecord(): { let streamRecorder: MediaRecorder; const chunks: Blob[] = []; const emitter: any = inject("mitt"); + const options = { + mimeType: "audio/webm; codecs=opus", + } + console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; ')) const initRecorder = (stream: MediaStream) => { - streamRecorder = new MediaRecorder(stream); - streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) } - streamRecorder.onstop = (e: any) => { - // send audio to BE - const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' }); - // emitter.emit("newSelfMessage", audioBlob); - const blobToBase64 = (blob: any) => { - return new Promise((resolve) => { - const reader = new FileReader(); - reader.readAsDataURL(blob); - reader.onloadend = function () { - resolve(reader.result); - }; - }); - }; - (async () => { - const b64 = await blobToBase64(audioBlob); - const jsonString = JSON.stringify({blob: b64}); - const message = { - text: '', - audio: b64 + const audioTrack = stream.getAudioTracks()[0]; + const audioSettings = audioTrack.getSettings() + + streamRecorder = new MediaRecorder(stream, options); + streamRecorder.ondataavailable = (e: any) => { + if (e.data.size > 0) { + chunks.push(e.data); + } else { + // ... + } + //chunks.push(e.data) + } + streamRecorder.onstop = async (e: any) => { + const audioBlob = new Blob(chunks, { + type: options.mimeType + }); + const arrayBuffer = await audioBlob.arrayBuffer() + console.log(arrayBuffer) + const message = { + text: '', + audio: { + sampleRate: audioSettings.sampleRate, + sampleSize: audioSettings.sampleSize, + channelCount: audioSettings.channelCount, + mimeType: options.mimeType, + arrayBuffer: arrayBuffer } - window.postMessage({ - myTypeField: 'send-message', - message: message - }, '*') - })(); - + } + window.postMessage({ + myTypeField: 'send-message', + message: message + }, '*') } return streamRecorder diff --git a/src/core/UI/input/audio/useStreamVisualizer.ts b/src/core/UI/input/audio/useStreamVisualizer.ts index 5cd8d4a..c31bf6c 100644 --- a/src/core/UI/input/audio/useStreamVisualizer.ts +++ b/src/core/UI/input/audio/useStreamVisualizer.ts @@ -11,7 +11,6 @@ interface AudioPath { } export default function useStreamVisualizer(draw: Ref) { const audioContext = new AudioContext(); - console.log('audi sample rate', audioContext.sampleRate) const analyzer = audioContext.createAnalyser(); analyzer.fftSize = 128; const bufferLength = analyzer.frequencyBinCount; diff --git a/src/core/UI/input/useInputController.ts b/src/core/UI/input/useInputController.ts index 8876dae..0a94fc9 100644 --- a/src/core/UI/input/useInputController.ts +++ b/src/core/UI/input/useInputController.ts @@ -5,6 +5,7 @@ import useStreamRecord from "./audio/useStreamRecord"; import useStreamVisualizer from './audio/useStreamVisualizer'; import useTextRender from './text/useTextRender'; import useAudioAnimations from './audio/audioDetails/useAnimations'; +import useAudioRecorder from './audio/useAudioRecorder'; /* * Input controller logic @@ -37,10 +38,12 @@ export default function useInputController() { if (e.key === " " && visualizeStream.value) { console.log("stop recording"); // recorder.stop(); - window.postMessage({ - myTypeField: 'update-recorder', - record: false - }, '*') + + // window.postMessage({ + // myTypeField: 'update-recorder', + // record: false + // }, '*') + visualizeStream.value = false; paths.value = [] audioBubbleWidth.value = 10; @@ -55,7 +58,7 @@ export default function useInputController() { // get stream & initialize recorder object on mount onMounted(async () => { stream = await useMediaStream(); - console.log(stream) + useAudioRecorder(visualizeStream, stream) recorder = initRecorder(stream); }) @@ -72,10 +75,11 @@ export default function useInputController() { visualizeStream.value = true; // recorder.start(); - window.postMessage({ - myTypeField: 'update-recorder', - record: true - }, '*') + // window.postMessage({ + // myTypeField: 'update-recorder', + // record: true + // }, '*') + expandBubble(audioBubbleWidth); visualizeStreamAsPaths(stream, paths, audioBubbleWidth); return; diff --git a/src/preload.ts b/src/preload.ts index 242516a..87f3b68 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -15,7 +15,7 @@ process.once("loaded", () => { const message = event.data; if (message.myTypeField === "send-message") { - ipcRenderer.send("send-message", message); + ipcRenderer.send("send-message", message.message); } if (message.myTypeField === "update-menu-bar") { diff --git a/testServer.js b/testServer.js index 1d6d422..3907aaf 100644 --- a/testServer.js +++ b/testServer.js @@ -33,7 +33,11 @@ const wss = new WebSocket.Server({ wss.on("connection", function connection(ws, req) { ws.on("message", function incoming(message) { - console.log("received: %s:", message); + const parsed = JSON.parse(message) + if (parsed.audio) { + console.log(parsed.audio) + } + console.log("received: %s:", parsed); }); const ip = req.socket.remoteAddress; -- 2.43.0