From: Andrew Gundersen Date: Thu, 3 Jun 2021 00:14:48 +0000 (-0500) Subject: prototype browser audio code X-Git-Url: https://andrewgundersen.net/repos?a=commitdiff_plain;h=fd7235ae5b67b6e1cf62a8babe449666c44a63cc;p=mime-chat prototype browser audio code --- diff --git a/src/components/inputitem/helpers/animations.ts b/src/components/inputitem/animations.ts similarity index 100% rename from src/components/inputitem/helpers/animations.ts rename to src/components/inputitem/animations.ts diff --git a/src/components/inputitem/audiocontrol.ts b/src/components/inputitem/audiocontrol.ts new file mode 100644 index 0000000..1b46436 --- /dev/null +++ b/src/components/inputitem/audiocontrol.ts @@ -0,0 +1,261 @@ +import useMitt from "@/modules/mitt"; +import { useIpc } from '@/modules/ipc'; +import { showRecIcon, hideRecIcon } from "./animations"; +import { onMounted, onUnmounted, ref, Ref } from "vue"; +import keyboardNameMap from "./keyBoardMaps/keyboardNameMap"; +import { renderMessage, clientMessage } from '@/modules/message'; + +const { post, invoke } = useIpc(); + + +function floatTo16bPCM(output: DataView, offset: number, input: Float32Array) { + for (var i = 0; i < input.length; i++, offset += 2) { + var s = Math.max(-1, Math.min(1, input[i])); + output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + } +} + +function writeFloat32 (output: DataView, offset: number, input: Float32Array) { + for (var i = 0; i < input.length; i++, offset += 4) { + output.setFloat32(offset, input[i], true) + } +} + +function writeString(view: DataView, offset: number, string: string) { + for (var i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +} + +function encodeWAV (samples: Float32Array, format: number, sampleRate: number, numChannels: number, bitDepth: number) { + + var bytesPerSample = bitDepth / 8 + var blockAlign = numChannels * bytesPerSample + + var buffer = new ArrayBuffer(44 + samples.length * bytesPerSample) + var view = new DataView(buffer) + + /* RIFF identifier */ + writeString(view, 0, 'RIFF') + /* RIFF chunk length */ + view.setUint32(4, 36 + samples.length * bytesPerSample, true) + /* RIFF type */ + writeString(view, 8, 'WAVE') + /* format chunk identifier */ + writeString(view, 12, 'fmt ') + /* format chunk length */ + view.setUint32(16, 16, true) + /* sample format (raw) */ + view.setUint16(20, format, true) + /* channel count */ + view.setUint16(22, numChannels, true) + /* sample rate */ + view.setUint32(24, sampleRate, true) + /* byte rate (sample rate * block align) */ + view.setUint32(28, sampleRate * blockAlign, true) + /* block align (channel count * bytes per sample) */ + view.setUint16(32, blockAlign, true) + /* bits per sample */ + view.setUint16(34, bitDepth, true) + /* data chunk identifier */ + writeString(view, 36, 'data') + /* data chunk length */ + view.setUint32(40, samples.length * bytesPerSample, true) + /* write data */ + if (format === 1) { + floatTo16bPCM(view, 44, samples) + } else { + writeFloat32(view, 44, samples) + } + + return buffer +} + +function mergeBuffers(bufferArray: Float32Array[], recLength: number) { + + // initialize array to hold all samples + var result = new Float32Array(recLength); + + // for each array in bufferArray, append values to result + var offset = 0; + + for (var i = 0; i < bufferArray.length; i++) { + result.set(bufferArray[i], offset); + offset += bufferArray[i].length; + } + + return result; +} + + +function exportBuffer(recBuffer: Float32Array[], recLength: number, exportSampleRate: number) { + var mergedBuffers = mergeBuffers(recBuffer, recLength); + var encodedWav = encodeWAV(mergedBuffers, 1, exportSampleRate, 1, 16); + var audioBlob = new Blob([encodedWav], {type: 'audio/wav'}); + return audioBlob; +} + +function postAudioBlob(blob: Blob, uid: string) { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onload = (e: ProgressEvent) => { + post("client-message", clientMessage("", reader.result as string, uid)) + } +} + +export default function useAudioInputController (typing: Ref) { + + let audioContext: AudioContext; + + let buffer: Float32Array[] = []; + let bufferLen = 0; + let sampleRate = 16000; + let numChannels = 1; + + // For sending messages. + const { emitter } = useMitt(); + + const recording = ref(false); + + // initiate media recorder + if (navigator.mediaDevices) { + console.log("Initializing media recorder."); + + const usrOptions: any = { + audio: true, + video: false + } + + navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => { + + audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(stream); + const processor = audioContext.createScriptProcessor(4096, 1, 1); + + sampleRate = audioContext.sampleRate; + + source.connect(processor); + processor.connect(audioContext.destination); + + processor.onaudioprocess = (e: AudioProcessingEvent) => { + + if (recording.value) { + + const data: Float32Array = e.inputBuffer.getChannelData(0); + + buffer.push(data); + bufferLen += data.length; + + } else { + buffer = []; + bufferLen = 0; + } + + }; + + }); + + } + + const play = (dataUrl: string) => { + const snd = new Audio(dataUrl); + snd.addEventListener("canplaythrough", (event: any) => { + console.log("Playing"); + snd.play(); + }); + } + + // var play = function (blob: Blob) { + // // We'll use a FileReader to create and ArrayBuffer out of the audio response. + // var fileReader = new FileReader(); + // fileReader.onload = function() { + // // Once we have an ArrayBuffer we can create our BufferSource and decode the result as an AudioBuffer. + // const playbackSource = audioContext.createBufferSource(); + // audioContext.decodeAudioData(fileReader.result as Buffer, function(audioBuffer) { + // console.log(audioBuffer.length); + // console.log(audioBuffer.sampleRate); + // console.log(audioBuffer.numberOfChannels); + // console.log(audioBuffer.duration); + + // // Set the source buffer as our new AudioBuffer. + // playbackSource.buffer = audioBuffer; + // // Set the destination (the actual audio-rendering device--your device's speakers). + // playbackSource.connect(audioContext.destination); + // // Add an "on ended" callback. + // playbackSource.onended = function(event) { + // console.log("Playback ended"); + // }; + // // Start the playback. + // playbackSource.start(0); + // }); + // }; + // fileReader.readAsArrayBuffer(blob); + // }; + + //---Callbacks----------------------------------------------- + + const onKeyDown = (e: KeyboardEvent) => { + const cmd = keyboardNameMap[e.keyCode]; + + // Start recording on space bar. + if (cmd == "SPACE" && !typing.value && !recording.value) { + + console.log("INPT:Starting record, filling buffer.") + recording.value = true; + showRecIcon(); + + } + + } + + const onKeyUp = async (e: KeyboardEvent) => { + const cmd = keyboardNameMap[e.keyCode]; + + // Stop recording on space up. + if (cmd == "SPACE" && recording.value) { + + // get audio from buffer then stop recording + console.log("INPT:Stopping record.") + hideRecIcon() + + // render a place holder message + const message = renderMessage( + "", + "", + "", + "sf" + ); + + emitter.emit("self-message", message); + + const blob: Blob = exportBuffer(buffer, bufferLen, sampleRate); + postAudioBlob(blob, message.uid); + recording.value = false; + + // const reader = new FileReader(); + // reader.readAsDataURL(blob); + // reader.onload = (e: ProgressEvent) => { + // play(reader.result as string); + // } + + } + + } + + //----------------------------------------------------------- + + onMounted(() => { + window.addEventListener("keydown", onKeyDown); + window.addEventListener("keyup", onKeyUp); + }) + + onUnmounted(() => { + window.removeEventListener("keydown", onKeyDown); + window.removeEventListener("keyup", onKeyUp); + }); + + return { + recording + } + +} diff --git a/src/components/inputitem/helpers/audiocontrol.ts b/src/components/inputitem/helpers/audiocontrol.ts deleted file mode 100644 index 637409e..0000000 --- a/src/components/inputitem/helpers/audiocontrol.ts +++ /dev/null @@ -1,138 +0,0 @@ -import useMitt from "@/modules/mitt"; -import { useIpc } from '@/modules/ipc'; -import { showRecIcon, hideRecIcon } from "./animations"; -import { onMounted, onUnmounted, ref, Ref } from "vue"; -import keyboardNameMap from "../keyBoardMaps/keyboardNameMap"; -import { renderMessage, clientMessage } from '@/modules/message'; -import { AudioData } from "@/types"; - - -function arrayBufferToBase64 (buffer: any) { - var binary = ''; - var bytes = new Uint8Array(buffer); - var len = bytes.byteLength; - for (var i = 0; i < len; i++) { - binary += String.fromCharCode( bytes[ i ] ); - } - return window.btoa( binary ); -} - - - -export default function useAudioInputController (typing: Ref) { - - let mediaRecorder: any; - let buffer = ''; - let sampleRate = 44100; // default - let channels = 1; - - // For sending messages. - const { post, invoke } = useIpc(); - const { emitter } = useMitt(); - - const recording = ref(false); - - // initiate media recorder - if (navigator.mediaDevices) { - console.log("Initializing media recorder."); - - const usrOptions: any = { - audio: true, - video: false - } - - navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => { - - const context = new AudioContext(); - const source = context.createMediaStreamSource(stream); - const processor = context.createScriptProcessor(1024, 1, 1); - - source.connect(processor); - processor.connect(context.destination); - - processor.onaudioprocess = (e: AudioProcessingEvent) => { - - if (recording.value) { - - const data = e.inputBuffer.getChannelData(0).buffer; - - buffer += arrayBufferToBase64(data); - - sampleRate = e.inputBuffer.sampleRate; - - } else buffer = ''; - - }; - - }); - - } - - //---Callbacks----------------------------------------------- - - const onKeyDown = (e: KeyboardEvent) => { - const cmd = keyboardNameMap[e.keyCode]; - - // Start recording on space bar. - if (cmd == "SPACE" && !typing.value && !recording.value) { - - console.log("INPT:Starting record, filling buffer.") - recording.value = true; - showRecIcon(); - - } - - } - - const onKeyUp = async (e: KeyboardEvent) => { - const cmd = keyboardNameMap[e.keyCode]; - - // Stop recording on space up. - if (cmd == "SPACE" && recording.value) { - - // get audio from buffer then stop recording - console.log("INPT:Stopping record.") - recording.value = false; - hideRecIcon() - - // render a place holder message - const message = renderMessage( - "", - "", - "", - "sf" - ); - - emitter.emit("self-message", message); - - // send audio to backend - const audio: AudioData = { - content: buffer, - channels: channels, - fs: sampleRate - } - - console.log("INPT:Sending audio to backend"); - post("client-message", clientMessage("", audio, message.uid)); - - } - - } - - //----------------------------------------------------------- - - onMounted(() => { - window.addEventListener("keydown", onKeyDown); - window.addEventListener("keyup", onKeyUp); - }) - - onUnmounted(() => { - window.removeEventListener("keydown", onKeyDown); - window.removeEventListener("keyup", onKeyUp); - }); - - return { - recording - } - -} diff --git a/src/components/inputitem/helpers/mediarecorder.ts b/src/components/inputitem/helpers/mediarecorder.ts deleted file mode 100644 index 9d78709..0000000 --- a/src/components/inputitem/helpers/mediarecorder.ts +++ /dev/null @@ -1,74 +0,0 @@ - - - -if (navigator.mediaDevices) { - console.log('getUserMedia supported.'); - - var constraints = { audio: true }; - var chunks = []; - - navigator.mediaDevices.getUserMedia(constraints) - .then(function(stream) { - - var mediaRecorder = new MediaRecorder(stream); - - visualize(stream); - - record.onclick = function() { - mediaRecorder.start(); - console.log(mediaRecorder.state); - console.log("recorder started"); - record.style.background = "red"; - record.style.color = "black"; - } - - stop.onclick = function() { - mediaRecorder.stop(); - console.log(mediaRecorder.state); - console.log("recorder stopped"); - record.style.background = ""; - record.style.color = ""; - } - - mediaRecorder.onstop = function(e) { - console.log("data available after MediaRecorder.stop() called."); - - var clipName = prompt('Enter a name for your sound clip'); - - var clipContainer = document.createElement('article'); - var clipLabel = document.createElement('p'); - var audio = document.createElement('audio'); - var deleteButton = document.createElement('button'); - - clipContainer.classList.add('clip'); - audio.setAttribute('controls', ''); - deleteButton.innerHTML = "Delete"; - clipLabel.innerHTML = clipName; - - clipContainer.appendChild(audio); - clipContainer.appendChild(clipLabel); - clipContainer.appendChild(deleteButton); - soundClips.appendChild(clipContainer); - - audio.controls = true; - var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' }); - chunks = []; - var audioURL = URL.createObjectURL(blob); - audio.src = audioURL; - console.log("recorder stopped"); - - deleteButton.onclick = function(e) { - evtTgt = e.target; - evtTgt.parentNode.parentNode.removeChild(evtTgt.parentNode); - } - } - - mediaRecorder.ondataavailable = function(e) { - chunks.push(e.data); - } - }) - .catch(function(err) { - console.log('The following error occurred: ' + err); - }) -} - diff --git a/src/components/inputitem/inputItem.vue b/src/components/inputitem/inputItem.vue index 861b9d8..9c05a0d 100644 --- a/src/components/inputitem/inputItem.vue +++ b/src/components/inputitem/inputItem.vue @@ -11,6 +11,7 @@ + @@ -28,8 +29,8 @@ import draggify from "@/modules/draggify"; import TextInput from "./textinput.vue"; -import useTextInputController from "./helpers/textcontrol"; -import useAudioInputController from "./helpers/audiocontrol"; +import useTextInputController from "./textcontrol"; +import useAudioInputController from "./audiocontrol"; export default defineComponent({ name: "InputItem", diff --git a/src/components/inputitem/helpers/textcontrol.ts b/src/components/inputitem/textcontrol.ts similarity index 98% rename from src/components/inputitem/helpers/textcontrol.ts rename to src/components/inputitem/textcontrol.ts index 2e9fb2c..19a9414 100644 --- a/src/components/inputitem/helpers/textcontrol.ts +++ b/src/components/inputitem/textcontrol.ts @@ -2,7 +2,7 @@ import anime from "animejs"; import useMitt from "@/modules/mitt"; import { useIpc } from '@/modules/ipc'; import { Ref, ref, watch, onMounted, onUnmounted } from "vue"; -import keyboardNameMap from "../keyBoardMaps/keyboardNameMap"; +import keyboardNameMap from "./keyBoardMaps/keyboardNameMap"; import { clientMessage, renderMessage } from '@/modules/message'; //---Animations----------------------------------------------- diff --git a/src/modules/message.ts b/src/modules/message.ts index acc5447..493c2a7 100644 --- a/src/modules/message.ts +++ b/src/modules/message.ts @@ -3,8 +3,7 @@ import { ClientMessage, ClientRequest, AuthRequest, - LogoutRequest, - AudioData + LogoutRequest } from "@/types"; import { v4 as uuidv4 } from 'uuid'; @@ -31,7 +30,7 @@ export const renderMessage = (text: boolean | string, audio: boolean | string, c } ) -export const clientMessage = (text: string, audio: AudioData | null, uid: string): ClientMessage => ( +export const clientMessage = (text: string, audio: string | null, uid: string): ClientMessage => ( { audio, text, diff --git a/src/types.ts b/src/types.ts index ebebe1a..44a978c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,15 +12,10 @@ export interface RenderMessage { newMessage: boolean; } -export interface AudioData { - content: string; - channels: number; - fs: number; -} export interface ClientMessage { text: string; - audio: AudioData | null; + audio: string | null; uid: string; }