add 16int audio recorder

src/background.ts

@@ -55,7 +55,7 @@ function createWindow() {
 
     ws = new WebSocket("ws://localhost:8080");
     ws.on("open", function open() {
-      ws.send("hernandeze2@xavier.edu");
+      // ws.send("hernandeze2@xavier.edu");
     });
     ws.on("message", (message: any) => {
       const parsed = JSON.parse(message);
@@ -67,8 +67,8 @@ function createWindow() {
       }
     });
 
-    ipcMain.on('send-message', (Event: any, payload: any) => {
-      const stringMessage = JSON.stringify(payload.message)
+    ipcMain.on('send-message', (Event: any, message: any) => {
+      const stringMessage = JSON.stringify(message)
       ws.send(stringMessage);
     })
 

src/core/UI/input/audio/useAudioRecorder.ts

@@ -0,0 +1,68 @@
+
+interface Window {
+  AudioContext: any;
+  webkitAudioContext: any;
+}
+
+interface Event {
+  inputBuffer: AudioBuffer;
+}
+
+import { Ref } from '@/types/vueRef/index';
+import { watch } from 'vue';
+
+export default function useAudioRecorder(record: Ref<boolean>, stream: MediaStream) {
+
+  let audioArray: Int16Array[] = [];
+  const bufferLen = 4096;
+  const numChannels = 1;
+
+  window.AudioContext = window.AudioContext;
+  const audioContext = new AudioContext({ sampleRate: 16000 });
+
+  const source = audioContext.createMediaStreamSource(stream);
+  const context = source.context;
+  const sampleRate = context.sampleRate;
+  console.log(sampleRate)
+
+  const node = context.createScriptProcessor.call(
+    context,
+    bufferLen,
+    numChannels,
+    numChannels
+  );
+
+  node.onaudioprocess = (e: Event) => {
+    const inputBuffer = new Float32Array(bufferLen);
+
+    e.inputBuffer.copyFromChannel(inputBuffer, 0);
+    const int16 = new Int16Array(inputBuffer.buffer);
+    audioArray.push(int16);
+  };
+
+  watch(record, (beginRecording) => {
+    if (beginRecording) {
+      node.connect(context.destination);
+    } else {
+      node.disconnect(context.destination);
+
+      const message = {
+          text: '',
+          audio: {
+              sampleRate: sampleRate,
+              channelCount: numChannels,
+              audio: audioArray
+          }
+      }
+      window.postMessage({
+        myTypeField: 'send-message',
+        message: message
+      }, '*')
+
+      audioArray = [];
+    }
+  })
+
+  source.connect(node);
+
+}

src/core/UI/input/audio/useStreamRecord.ts

@@ -5,37 +5,45 @@ export default function useStreamRecord(): {
     let streamRecorder: MediaRecorder;
     const chunks: Blob[] = [];
     const emitter: any = inject("mitt");
+    const options = {
+        mimeType: "audio/webm; codecs=opus",
+    }
 
+    console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; '))
     const initRecorder = (stream: MediaStream) => {
-        streamRecorder = new MediaRecorder(stream);
-        streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
-        streamRecorder.onstop = (e: any) => {
-            // send audio to BE
-            const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' });
-            // emitter.emit("newSelfMessage", audioBlob);
-            const blobToBase64 = (blob: any) => {
-               return new Promise((resolve) => {
-                const reader = new FileReader();
-                reader.readAsDataURL(blob);
-                reader.onloadend = function () {
-                  resolve(reader.result);
-                };
-            });
-           };
 
-            (async () => {
-                const b64 = await blobToBase64(audioBlob);
-                const jsonString = JSON.stringify({blob: b64});
-                const message = {
-                    text: '',
-                    audio: b64
+       const audioTrack = stream.getAudioTracks()[0]; 
+       const audioSettings = audioTrack.getSettings()
+
+        streamRecorder = new MediaRecorder(stream, options);
+        streamRecorder.ondataavailable = (e: any) => { 
+              if (e.data.size > 0) {
+                chunks.push(e.data);
+              } else {
+                // ...
+              }
+            //chunks.push(e.data) 
+        }
+        streamRecorder.onstop = async (e: any) => {
+            const audioBlob = new Blob(chunks, { 
+                type: options.mimeType
+            });
+            const arrayBuffer = await audioBlob.arrayBuffer()
+            console.log(arrayBuffer)
+            const message = {
+                text: '',
+                audio: {
+                    sampleRate: audioSettings.sampleRate,
+                    sampleSize: audioSettings.sampleSize,
+                    channelCount: audioSettings.channelCount,
+                    mimeType: options.mimeType,
+                    arrayBuffer: arrayBuffer
                 }
-                window.postMessage({
-                  myTypeField: 'send-message',
-                  message: message
-                }, '*')
-            })();
- 
+            }
+            window.postMessage({
+              myTypeField: 'send-message',
+              message: message
+            }, '*')
 
         }
         return streamRecorder

src/core/UI/input/audio/useStreamVisualizer.ts

@@ -11,7 +11,6 @@ interface AudioPath {
 }
 export default function useStreamVisualizer(draw: Ref<boolean>) {
     const audioContext = new AudioContext();
-    console.log('audi sample rate', audioContext.sampleRate)
     const analyzer = audioContext.createAnalyser();
     analyzer.fftSize = 128;
     const bufferLength = analyzer.frequencyBinCount;

src/core/UI/input/useInputController.ts

@@ -5,6 +5,7 @@ import useStreamRecord from "./audio/useStreamRecord";
 import useStreamVisualizer from './audio/useStreamVisualizer';
 import useTextRender from './text/useTextRender';
 import useAudioAnimations from './audio/audioDetails/useAnimations';
+import useAudioRecorder from './audio/useAudioRecorder';
 
 /* 
 * Input controller logic
@@ -37,10 +38,12 @@ export default function useInputController() {
     if (e.key === " " && visualizeStream.value) {
       console.log("stop recording");
       // recorder.stop();
-      window.postMessage({
-        myTypeField: 'update-recorder',
-        record: false
-      }, '*')
+
+      // window.postMessage({
+      //   myTypeField: 'update-recorder',
+      //   record: false
+      // }, '*')
+
       visualizeStream.value = false;
       paths.value = []
       audioBubbleWidth.value = 10;
@@ -55,7 +58,7 @@ export default function useInputController() {
   // get stream & initialize recorder object on mount
   onMounted(async () => {
     stream = await useMediaStream();
-    console.log(stream)
+    useAudioRecorder(visualizeStream, stream)
     recorder = initRecorder(stream);
   })
 
@@ -72,10 +75,11 @@ export default function useInputController() {
       visualizeStream.value = true;
       // recorder.start();
 
-      window.postMessage({
-        myTypeField: 'update-recorder',
-        record: true
-      }, '*')
+      // window.postMessage({
+      //   myTypeField: 'update-recorder',
+      //   record: true
+      // }, '*')
+
       expandBubble(audioBubbleWidth);
       visualizeStreamAsPaths(stream, paths, audioBubbleWidth);
       return;

src/preload.ts

@@ -15,7 +15,7 @@ process.once("loaded", () => {
     const message = event.data;
 
     if (message.myTypeField === "send-message") {
-      ipcRenderer.send("send-message", message);
+      ipcRenderer.send("send-message", message.message);
     }
 
     if (message.myTypeField === "update-menu-bar") {

testServer.js

@@ -33,7 +33,11 @@ const wss = new WebSocket.Server({
 
 wss.on("connection", function connection(ws, req) {
   ws.on("message", function incoming(message) {
-    console.log("received: %s:", message);
+    const parsed = JSON.parse(message)
+    if (parsed.audio) {
+      console.log(parsed.audio)
+    }
+    console.log("received: %s:", parsed);
   });
 
   const ip = req.socket.remoteAddress;