me/mime-chat
Watch
1
0
Fork
You've already forked mime-chat
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add audioRenderer + ipc event listeners

Browse source
This commit is contained in:
Enrique Hernandez 2020-12-11 10:41:52 -06:00
commit 5a3cd60046
5 changed files with 180 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

57
src/background.ts
View file

@ -1,6 +1,6 @@
"use strict";
import { app, protocol, BrowserWindow } from "electron";
import { app, protocol, BrowserWindow, ipcMain } from "electron";
import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
const isDevelopment = process.env.NODE_ENV !== "production";
@ -56,26 +56,41 @@ function createWindow() {
win.loadURL("app://./index.html");
}
// win.webContents.on('did-finish-load', () => {
// if (win) {
// recorder.start({
// onAudio: (audio: object, speech: boolean) => {
// if (win) {
// win.setTitle("Listening...");
// // win.webContents.send('render-audio', {
// // audio: audio,
// // speech: speech
// // })
// if (speech) {
// win.setTitle("Recording...");
// // TODO: send audio to crimata BE
// // ws.send(audio);
// }
// }
// }
// });
// }
// })
win.webContents.on('did-finish-load', () => {
// ipcMain.on('audio-stream', (Event: any, Payload: any) => {
// if (Payload.stream) {
// console.log('begin streaming');
// recorder.start({
// onAudio: (audio: object, speech: boolean) => {
// console.log(audio)
// }
// });
// } else {
// recorder.stop();
// console.log('stop streaming')
// }
// })
// if (win) {
// recorder.start({
// onAudio: (audio: object, speech: boolean) => {
// if (win) {
// win.setTitle("Listening...");
// // win.webContents.send('render-audio', {
// // audio: audio,
// // speech: speech
// // })
// if (speech) {
// win.setTitle("Recording...");
// // TODO: send audio to crimata BE
// // ws.send(audio);
// }
// }
// }
// });
// }
})
win.on("closed", () => {
win = null;

50
src/components/UI/UIDetails/inputItem.vue
View file

@ -11,31 +11,36 @@
<script lang="ts">
import useInputRenderer from "@/composables/inputItem/useInputRenderer";
import useStreamRecorder from "@/composables/streamRecorder/useStreamRecorder";
import {
defineComponent,
watch,
onMounted
} from "vue";
import { defineComponent, watch, onMounted, ref } from "vue";
export default defineComponent({
name: "InputItem",
setup() {
const record = ref(false);
const { inputXoffset, input } = useInputRenderer();
const { initRecorder } = useStreamRecorder();
const { initRecorder } = useStreamRecorder(record);
onMounted(async () => {
// step get streamRecorder reference
const { streamRecorder } = await initRecorder();
// add event listener for space keyup
window.addEventListener("keyup", (e) => {
if (e.key === " " && streamRecorder.state === "recording") {
console.log("stop recording")
console.log("stop recording");
record.value = false;
streamRecorder.stop();
console.log('recorder is', streamRecorder.state);
// window.postMessage(
// {
// myTypeField: "audio-stream",
// stream: false,
// },
// "*"
// );
console.log("recorder is", streamRecorder.state);
input.value = "";
}
})
});
// begin recording on space
watch(input, (input, prevInput) => {
@ -43,14 +48,23 @@ export default defineComponent({
return;
}
if (input === " ") {
console.log("record")
streamRecorder.start();
console.log('recorder is', streamRecorder.state);
}
})
})
console.log("record");
record.value = true;
return {
// window.postMessage(
// {
// myTypeField: "audio-stream",
// stream: true,
// },
// "*"
// );
streamRecorder.start();
console.log("recorder is", streamRecorder.state);
}
});
});
return {
input,
inputXoffset,
};

35
src/composables/audioRenderer/useAudioRenderer.ts
View file

@ -1,37 +1,22 @@
export default function useAudioRenderer() {
export default function useAudioRenderer(stream: MediaStream) {
const audioCtx = new (window.AudioContext)();
const analyzer = audioCtx.createAnalyser();
const source = audioCtx.createMediaStreamSource(stream);
source.connect(analyzer);
analyzer.fftSize = 2048;
const xInitial = 24;
const xFinal = 640;
// analyzer.minDecibels = -90;
analyzer.minDecibels = -90;
analyzer.smoothingTimeConstant = 0.85;
const dataArray = new Uint8Array(analyzer.frequencyBinCount);
const sampleRate = 16000;
// const sampleRate = 16000;
function convertBlock(buffer: ArrayBuffer): Float32Array {
// incoming data is an ArrayBuffer
const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
const l = incomingData.length; // length, we need this for the loop
const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
for (let i = 0; i < l; i++) {
outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
}
return outputData; // return the Float32Array
}
const getAudioFrequencyData = () => {
}
const renderAudio = (payload) => {
// step 1: convert audio from Uint8 to Float32Array
const floatArray = convertBlock(payload.audio.buffer);
const renderAudio = (audio: Float32Array, sampleRate: number) => {
// step 2: source audio frequency {ffft} data
const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
buffer.copyToChannel(floatArray, 0, 0);
const buffer = audioCtx.createBuffer(1, audio.length, sampleRate);
buffer.copyToChannel(audio, 0, 0);
const source = audioCtx.createBufferSource();
source.buffer = buffer;
source.connect(analyzer);
@ -44,7 +29,7 @@ export default function useAudioRenderer() {
const average = sum / dataView.length;
const scaled = average / 255;
const zeroed = scaled - 0.6;
return zeroed;
return average;
}
return {

103
src/composables/streamRecorder/useStreamRecorder.ts
View file

@ -1,20 +1,101 @@
export default function useStreamRecorder() {
import { watch } from "vue";
import useAudioRenderer from "@/composables/audioRenderer/useAudioRenderer";
export default function useStreamRecorder(record: any) {
let streamRecorder: MediaRecorder;
const chunks: Blob[] = [];
window.AudioContext = window.AudioContext;
let audio_context = new AudioContext();
const analyzer = audio_context.createAnalyser();
analyzer.fftSize = 256;
var bufferLengthAlt = analyzer.frequencyBinCount;
var dataArrayAlt = new Uint8Array(bufferLengthAlt);
let source;
const initRecorder = async () => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
streamRecorder = new MediaRecorder(stream);
streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
streamRecorder.onstop = (e: any) => {
const audioBlob = new Blob(chunks, {'type': 'audio/ogg; codecs=opus'} );
// do something with blob
}
return {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
handleStream(stream);
streamRecorder = new MediaRecorder(stream);
streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
streamRecorder.onstop = (e: any) => {
const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' });
// do something with blob
}
return {
streamRecorder
}
}
}
interface Window {
AudioContext: any;
webkitAudioContext: any;
}
interface Event {
inputBuffer: AudioBuffer;
}
let bufferLen = 4096;
let numChannels = 1;
function visualize() {
function drawAlt() {
const drawVisual = requestAnimationFrame(drawAlt);
analyzer.getByteFrequencyData(dataArrayAlt);
for (var i = 0; i < bufferLengthAlt; i++) {
console.log(dataArrayAlt[i])
}
}
drawAlt();
}
function handleStream(stream: MediaStream) {
source = audio_context.createMediaStreamSource(stream);
source.connect(analyzer);
visualize();
//
// const context = source.context;
// const sampleRate = context.sampleRate;
// const node = context.createScriptProcessor.call(
// context,
// bufferLen,
// numChannels,
// numChannels
// );
// source.connect(node);
// node.onaudioprocess = (e: Event) => {
// const inputBuffer = new Float32Array(bufferLen);
// e.inputBuffer.copyFromChannel(inputBuffer, 0);
// // const int16 = new Int16Array(inputBuffer.buffer);
// // window.postMessage(
// // {
// // myTypeField: "handle-audio-stream",
// // audioData: int16
// // },
// // "*"
// // );
// };
// watch(record, (rec) => {
// if (rec) {
// node.connect(context.destination);
// } else {
// node.disconnect(context.destination);
// }
// })
}
return {
initRecorder
initRecorder
}
}

12
src/preload.ts
View file

@ -6,3 +6,15 @@ interface Window {
ipcRenderer: typeof ipcRenderer;
}
window.ipcRenderer = ipcRenderer;
process.once("loaded", () => {
window.addEventListener("message", event => {
// do something with custom event
const message = event.data;
if (message.myTypeField === "audio-stream") {
ipcRenderer.send("audio-stream", message);
}
});
});