add audio analyser fft data

diff --git a/src/background.ts b/src/background.ts
index 2aaec657e8478927f2597dd1701c9c1979ccad8f..e07342e454f47d133ca8e04de63ae15d7d5409f2 100644 (file)
--- a/src/background.ts
+++ b/src/background.ts
@@ -19,7 +19,7 @@ import { SpeechRecorder } from "speech-recorder";
 // });
 //
 
-const recorder = new SpeechRecorder();
+const recorder = new SpeechRecorder({ sampleRate: 16000, framesPerBuffer: 320 });
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
@@ -59,7 +59,7 @@ function createWindow() {
       recorder.start({
         onAudio: (audio: object, speech: boolean) => {
           if (win) {
-            win.webContents.send('render-audio', { data: audio })
+            win.webContents.send('render-audio', audio)
           }
           if (speech) {
             // TODO: send audio to crimata BE

diff --git a/src/components/AI/index.vue b/src/components/AI/index.vue
index 02bb6a9575be824b5e4295fb38c97006850bde0c..2afa6f1040bae79b8864dfc73482a8fb26aca63d 100644 (file)
--- a/src/components/AI/index.vue
+++ b/src/components/AI/index.vue
@@ -9,7 +9,7 @@
       gradientUnits="objectBoundingBox"
     >
       <stop offset="0" stop-color="#fff" />
-      <stop offset="1" stop-color="#e6e6e6" />
+      <stop id="sound" offset="1" stop-color="#e6e6e6" />
     </radialGradient>
     <filter id="Rectangle_410" x="0" y="0" width="380" height="530" filterUnits="userSpaceOnUse">
       <feOffset dy="3" input="SourceAlpha" />
@@ -33,13 +33,85 @@
 </template>
 
 <script>
-import { defineComponent } from "vue";
+import { defineComponent, onMounted, ref } from "vue";
 export default defineComponent({
   name: "AI",
   setup() {
-    window.ipcRenderer.on("render-audio", (event, audio) => {
-      console.log(audio);
+    const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
+    const analyzer = audioCtx.createAnalyser();
+    analyzer.fftSize = 256;
+    const dataArray = new Uint8Array(analyzer.frequencyBinCount);
+    const sampleRate = 16000;
+    let gradient;
+    const amplitude = ref(0);
+
+    onMounted(() => {
+      gradient = document.getElementById("sound");
+    });
+
+    function convertBlock(buffer) {
+      // incoming data is an ArrayBuffer
+      const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
+      let i,
+        l = incomingData.length; // length, we need this for the loop
+      const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
+      for (i = 0; i < l; i++) {
+        outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
+      }
+      return outputData; // return the Float32Array
+    }
+
+    window.addEventListener("load", function (e) {
+      window.ipcRenderer.on("render-audio", (event, audio) => {
+        const floatArray = convertBlock(audio.buffer);
+        const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
+        buffer.copyToChannel(floatArray, 0, 0);
+        const source = audioCtx.createBufferSource();
+        source.buffer = buffer;
+        source.connect(analyzer);
+        source.start();
+        visualize();
+      });
     });
+
+    const render = () => {
+      // whatever you want to do per frame of animation
+
+      const range = 64;
+      const interval = range / 6;
+
+      const sensitivity = 0.25;
+
+      const min = 0;
+      const max = 25;
+
+      let channel = 1;
+
+      const to = Math.floor(interval * channel);
+      const from = Math.ceil(to - interval);
+      const segment = dataArray.slice(from, to);
+
+      const sum = segment.reduce((a, b) => a + b);
+      const average = sum / segment.length;
+      console.log(average);
+      // let height = sensitivity * average;
+      //
+      // height = Math.max(min, Math.min(max, height));
+      //
+      // height = height + 230;
+      //
+      // // we have to write a function what scales between
+      // gradient.setAttribute(
+      //   "stop-color",
+      //   `rgb(${height}, ${height}, ${height})`
+      // );
+    };
+
+    function visualize() {
+      requestAnimationFrame(visualize);
+      analyzer.getByteFrequencyData(dataArray);
+      render();
+    }
   },
 });
 </script>