"use strict";
-import { app, protocol, BrowserWindow } from "electron";
+import { app, protocol, BrowserWindow, ipcMain } from "electron";
import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
const isDevelopment = process.env.NODE_ENV !== "production";
win.loadURL("app://./index.html");
}
- // win.webContents.on('did-finish-load', () => {
- // if (win) {
- // recorder.start({
- // onAudio: (audio: object, speech: boolean) => {
- // if (win) {
- // win.setTitle("Listening...");
- // // win.webContents.send('render-audio', {
- // // audio: audio,
- // // speech: speech
- // // })
- // if (speech) {
- // win.setTitle("Recording...");
- // // TODO: send audio to crimata BE
- // // ws.send(audio);
- // }
- // }
- // }
- // });
- // }
- // })
+
+ win.webContents.on('did-finish-load', () => {
+ // ipcMain.on('audio-stream', (Event: any, Payload: any) => {
+ // if (Payload.stream) {
+ // console.log('begin streaming');
+ // recorder.start({
+ // onAudio: (audio: object, speech: boolean) => {
+ // console.log(audio)
+ // }
+ // });
+ // } else {
+ // recorder.stop();
+ // console.log('stop streaming')
+ // }
+ // })
+
+ // if (win) {
+ // recorder.start({
+ // onAudio: (audio: object, speech: boolean) => {
+ // if (win) {
+ // win.setTitle("Listening...");
+ // // win.webContents.send('render-audio', {
+ // // audio: audio,
+ // // speech: speech
+ // // })
+ // if (speech) {
+ // win.setTitle("Recording...");
+ // // TODO: send audio to crimata BE
+ // // ws.send(audio);
+ // }
+ // }
+ // }
+ // });
+ // }
+ })
win.on("closed", () => {
win = null;
<script lang="ts">
import useInputRenderer from "@/composables/inputItem/useInputRenderer";
import useStreamRecorder from "@/composables/streamRecorder/useStreamRecorder";
-import {
- defineComponent,
- watch,
- onMounted
-} from "vue";
+import { defineComponent, watch, onMounted, ref } from "vue";
export default defineComponent({
name: "InputItem",
setup() {
+ const record = ref(false);
const { inputXoffset, input } = useInputRenderer();
- const { initRecorder } = useStreamRecorder();
+ const { initRecorder } = useStreamRecorder(record);
onMounted(async () => {
-
// step get streamRecorder reference
const { streamRecorder } = await initRecorder();
-
+
// add event listener for space keyup
window.addEventListener("keyup", (e) => {
if (e.key === " " && streamRecorder.state === "recording") {
- console.log("stop recording")
+ console.log("stop recording");
+ record.value = false;
streamRecorder.stop();
- console.log('recorder is', streamRecorder.state);
+
+ // window.postMessage(
+ // {
+ // myTypeField: "audio-stream",
+ // stream: false,
+ // },
+ // "*"
+ // );
+ console.log("recorder is", streamRecorder.state);
input.value = "";
}
- })
+ });
// begin recording on space
watch(input, (input, prevInput) => {
return;
}
if (input === " ") {
- console.log("record")
+ console.log("record");
+ record.value = true;
+
+ // window.postMessage(
+ // {
+ // myTypeField: "audio-stream",
+ // stream: true,
+ // },
+ // "*"
+ // );
streamRecorder.start();
- console.log('recorder is', streamRecorder.state);
+ console.log("recorder is", streamRecorder.state);
}
- })
- })
+ });
+ });
- return {
+ return {
input,
inputXoffset,
};
-export default function useAudioRenderer() {
+export default function useAudioRenderer(stream: MediaStream) {
const audioCtx = new (window.AudioContext)();
const analyzer = audioCtx.createAnalyser();
+ const source = audioCtx.createMediaStreamSource(stream);
+ source.connect(analyzer);
+
analyzer.fftSize = 2048;
const xInitial = 24;
const xFinal = 640;
- // analyzer.minDecibels = -90;
+ analyzer.minDecibels = -90;
analyzer.smoothingTimeConstant = 0.85;
const dataArray = new Uint8Array(analyzer.frequencyBinCount);
- const sampleRate = 16000;
-
- function convertBlock(buffer: ArrayBuffer): Float32Array {
- // incoming data is an ArrayBuffer
- const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
- const l = incomingData.length; // length, we need this for the loop
- const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
- for (let i = 0; i < l; i++) {
- outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
- }
- return outputData; // return the Float32Array
- }
-
- const getAudioFrequencyData = () => {
-
- }
-
- const renderAudio = (payload) => {
+ // const sampleRate = 16000;
- // step 1: convert audio from Uint8 to Float32Array
- const floatArray = convertBlock(payload.audio.buffer);
+ const renderAudio = (audio: Float32Array, sampleRate: number) => {
// step 2: source audio frequency {ffft} data
- const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
- buffer.copyToChannel(floatArray, 0, 0);
+ const buffer = audioCtx.createBuffer(1, audio.length, sampleRate);
+ buffer.copyToChannel(audio, 0, 0);
const source = audioCtx.createBufferSource();
source.buffer = buffer;
source.connect(analyzer);
const average = sum / dataView.length;
const scaled = average / 255;
const zeroed = scaled - 0.6;
- return zeroed;
+ return average;
}
return {
-export default function useStreamRecorder() {
+import { watch } from "vue";
+import useAudioRenderer from "@/composables/audioRenderer/useAudioRenderer";
+export default function useStreamRecorder(record: any) {
let streamRecorder: MediaRecorder;
const chunks: Blob[] = [];
+
+ window.AudioContext = window.AudioContext;
+ let audio_context = new AudioContext();
+ const analyzer = audio_context.createAnalyser();
+ analyzer.fftSize = 256;
+ var bufferLengthAlt = analyzer.frequencyBinCount;
+ var dataArrayAlt = new Uint8Array(bufferLengthAlt);
+
+ let source;
+
+
const initRecorder = async () => {
- const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
- streamRecorder = new MediaRecorder(stream);
- streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
- streamRecorder.onstop = (e: any) => {
- const audioBlob = new Blob(chunks, {'type': 'audio/ogg; codecs=opus'} );
- // do something with blob
- }
- return {
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ handleStream(stream);
+ streamRecorder = new MediaRecorder(stream);
+ streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
+ streamRecorder.onstop = (e: any) => {
+ const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' });
+ // do something with blob
+ }
+ return {
streamRecorder
- }
+ }
+ }
+
+ interface Window {
+ AudioContext: any;
+ webkitAudioContext: any;
+ }
+
+ interface Event {
+ inputBuffer: AudioBuffer;
+ }
+
+ let bufferLen = 4096;
+ let numChannels = 1;
+ function visualize() {
+
+ function drawAlt() {
+
+ const drawVisual = requestAnimationFrame(drawAlt);
+ analyzer.getByteFrequencyData(dataArrayAlt);
+
+ for (var i = 0; i < bufferLengthAlt; i++) {
+ console.log(dataArrayAlt[i])
+ }
+ }
+ drawAlt();
+ }
+
+ function handleStream(stream: MediaStream) {
+ source = audio_context.createMediaStreamSource(stream);
+ source.connect(analyzer);
+ visualize();
+ //
+ // const context = source.context;
+ // const sampleRate = context.sampleRate;
+
+ // const node = context.createScriptProcessor.call(
+ // context,
+ // bufferLen,
+ // numChannels,
+ // numChannels
+ // );
+
+ // source.connect(node);
+
+ // node.onaudioprocess = (e: Event) => {
+ // const inputBuffer = new Float32Array(bufferLen);
+
+ // e.inputBuffer.copyFromChannel(inputBuffer, 0);
+
+ // // const int16 = new Int16Array(inputBuffer.buffer);
+
+ // // window.postMessage(
+ // // {
+ // // myTypeField: "handle-audio-stream",
+ // // audioData: int16
+ // // },
+ // // "*"
+ // // );
+ // };
+
+ // watch(record, (rec) => {
+ // if (rec) {
+ // node.connect(context.destination);
+ // } else {
+ // node.disconnect(context.destination);
+ // }
+ // })
}
return {
- initRecorder
+ initRecorder
}
}
ipcRenderer: typeof ipcRenderer;
}
window.ipcRenderer = ipcRenderer;
+
+process.once("loaded", () => {
+ window.addEventListener("message", event => {
+ // do something with custom event
+ const message = event.data;
+
+ if (message.myTypeField === "audio-stream") {
+ ipcRenderer.send("audio-stream", message);
+ }
+ });
+});
+
projects / mime-chat / commitdiff