--- /dev/null
+{
+ "type": "service_account",
+ "project_id": "crimata",
+ "private_key_id": "3a324fa3fcca4cea589e4b5007f4142db4b3a84d",
+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCxFJOjhIkYlyhn\nHjtpFC5EOwzNFZem6k6SS7zcgWbNLe9mcBsiTKCjpxItzkIo0pnT32yqw+AKsqpv\n6+LVyzzjfB7vtEUflvg6KgmYMS4ixOTCEbN3e2qj1t2RnITXZR2VGSGdoKKyqZm0\ntWrmexQvSN20cByL1vfxD3E69oYFGhVraa0zmm94WXHAzofT6DkvGdrbK1s1G9jI\nG3mdQ1jN71tghyloeHL+R/1iKQoZFYTkVe6yWvccSNAZelt/5sciCKRTCArXj89X\nLwU6jGOKIsyqx3BeUyWaSS59xpnGX3meSf5Q9U+j45ePUiqO6TTleF+kWiBwjUpp\nfeYcwEIDAgMBAAECggEAElkN9R7p75zV6F1XDYu0QYiWyncmx/o2Gu1zC6vywWa1\ni/korpSe/mX0ub9J1p3/p1balRUHlUQu6brYvYs56140fGTC1sOXQ7uQU+8glySs\niTk5TbOBeKluOsSrdP/6oTTB6Wm4AegVz4YOpgPxsvaLVkNZidnDxfvyIQhjGYsO\nsvRaw48eEmGOa1uvDhgvPOxiDp5YEdxqSue/7ylzjBdKecDyLgTSDDf/p+FwIx5B\n2d/0uRgQBaAk6D/DNOvN8GrYyakFmlDjE72nrBGAJq50UyV6THOSDk+8cK+2QiS1\nEDCFufj+7Dk1EBfBFUFY8ZBB/pphpEWyLoFl7NzGCQKBgQDaj/KOOcj6BkkTj1WM\nRW3KhOE3nhsCQEgQGCflpycEAZj6J7ktBG2/4teDt9sr+Epx8ymnQCZa9pmysdrD\nU0aWku1qUx0IUKTtZbAEbB+AnrWYDYkIWU8kRUUvNHNTBg43VyuohV9wprPih138\nHWMeNac82/XICz2A3rv+Xkx8KwKBgQDPaaEEGWVs3nIhf/p3M3/xWNKt8fhAERpG\n0YpTjgHW3FQFYjiCTTIcuLO3qsDGmfIAB9tL37s2Y3e2wKBGUj/o69Fr7sWVPdsX\nJtUc0+KiG2Bs0hsyZPdk9UQiQX8y2eq1NZW7vEoxWso1bM64WURB5N8ocWePNJJx\nL2Oe75rtiQKBgQDLfQQOiRxmNF3rOSMkAywyRr9NQgXRdbniSiszNQotP7OHDF7q\n29m2suOGfjIv4O6m4wdf8WkEfd4hsleETc9Ft6wVtyYrrLGxWWCk7WnzHVDjLY7s\n2AHIOjostf+9R8EKoz1BnFN8laibev71EQNMiBWZow1VX6m2hymurWs2mwKBgQCb\nuY3n2v14qOb92e1+U89KsEq1yMd/qpeU9jwqEaO14wS+aglNY5ItWEuuqWhFdE3q\n0ftHUzpnUnUOZD+xrI1JXsyEgegc7i0xi7lUBI3S8kUKTxGWW5IXXcKDCbPrxQtg\ndFPweSUnOyg4xnHKnVMPOjyGS+bZ8TnF+zOLoBAtKQKBgF6hAOIwF1DheXT4p8xS\nlSGhl8dBxjsJDVwbnFWQceGzGnq1e3ncy/UItt42UCDVRWTH4r7pQ9eKyd/3foUD\n6KDIcCrYo2KJh6ZNFJ67XF30U+TuEg5U/9jTbWqDOwYVXcMcnzQE4ZiGBxVHL6xr\nNc8FRW5+7xh4zNfzhGUGt5kt\n-----END PRIVATE KEY-----\n",
+ "client_email": "crimata-service-account@crimata.iam.gserviceaccount.com",
+ "client_id": "102914612880037864172",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/crimata-service-account%40crimata.iam.gserviceaccount.com"
+}
-"use strict";
/* eslint @typescript-eslint/no-var-requires: "off" */
+
+"use strict";
+
import { sendAudio } from './session'
const portAudio = require('naudiodon');
-let ia: typeof portAudio.AudioIO;
-let audioInput: string[] = [];
-
-function processAudio(audioInput: Array<string>): Buffer {
- let audio = '';
- audioInput.forEach(chunk => audio += chunk);
- return Buffer.from(audio, "base64");
+let ai: typeof portAudio.AudioIO;
+let ao: typeof portAudio.AudioIO;
+let audioInput: Buffer[] = [];
+const encoding = "base64";
+const audioOptions = {
+ channelCount: 1,
+ sampleFormat: 16,
+ sampleRate: 16000,
+ deviceId: -1,
+ closeOnError: false,
}
-export const updateRecorder = (_event, record: boolean): void => {
- if (record) {
- ia.resume();
- } else {
- ia.pause();
- const audio = processAudio(audioInput);
- sendAudio(audio);
- audioInput = [];
+// run every time the main function launches.
+export const initAudioIO = () => {
+ // init portAudio readable stream once.
+ if (!ai) {
+ ai = new portAudio.AudioIO({ inOptions: audioOptions });
+
+ // base64 encoding needed for google speech to text.
+ ai.setEncoding(encoding);
+
+ // pause the portAudio data flow as soon as stream is initiated.
+ ai.start();
+ ai.pause();
+
+ ai.on('error', (e: Error) => {
+ console.log('Error recording audio', + e);
+ });
+ ai.on('data', (chunk: string) => {
+ // turn string base64 data into buffer object.
+ audioInput.push(Buffer.from(chunk, encoding));
+ });
+ }
+
+ // init portAudio writable stream once.
+ if (!ao) {
+ ao = new portAudio.AudioIO({ outOptions: audioOptions });
+ ao.start();
}
}
-export const initRecorder = () => {
-
- if (!ia) {
- ia = new portAudio.AudioIO({
- inOptions: {
- channelCount: 1,
- sampleFormat: 16,
- sampleRate: 16000,
- deviceId: -1,
- closeOnError: false,
- }
- });
- ia.setEncoding('base64');
- ia.start();
- ia.pause();
- ia.on('error', (e: Error) => {
- console.log('Error recording audio', + e);
- });
- ia.on('data', (chunk: string) => {
- audioInput.push(chunk);
- });
+// run this function on space bar key up and down.
+export const updateRecorder = (_event, record: boolean): void => {
+ if (record) {
+ // resume mic data flow on space bar key down.
+ ai.resume();
+ } else {
+ // stop mic data flow on space bar key up.
+ ai.pause();
+ sendAudio(Buffer.concat(audioInput));
+ audioInput = [];
}
}
+// play audio buffers.
+export const play = (input: Array<Buffer>) => {
+ let i = 0;
+ // format buffers into half their size to account for writable highwaterMark.
+ const audio = bufSplit(input);
+ // call this fuction after last audio chunk has been written.
+ const callback = () => {
+ // TODO: clear portAudio writable buffer on write end.
+ }
+ write();
+ // iterate through audio array and write buffers to portAudio writable.
+ function write() {
+ let chunk: Buffer;
+ let ok = true;
+ do {
+ chunk = audio[i];
+ if (i === audio.length - 1) {
+ // write last chunk.
+ ao.write(chunk, null, callback);
+ } else {
+ // check for backpreassure.
+ ok = ao.write(chunk, null);
+ }
+ i++;
+ } while (i < audio.length && ok);
+
+ if (i < audio.length) {
+ // Had to stop early!
+ // Write some more once it drains.
+ ao.once('drain', write);
+ }
+ }
+}
+// utility function used by play func
+function bufSplit(input: Array<Buffer>): Array<Buffer> {
+ let result: Buffer[] = [];
+ input.forEach((b: Buffer) => {
+ // split buffer into two.
+ result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length));
+ });
+ return result;
+}
import { createWindow } from './window';
import { initSession } from './session';
-import { initRecorder } from './audio';
+import { initAudioIO } from './audio';
let socket = false;
socket = true;
// Begin audio stream.
- // initRecorder();
+ initAudioIO();
}
import { ipcMain } from "electron";
interface RenderMessage {
- content: string;
+ content: {
+ text: string;
+ audio: boolean | Buffer;
+ };
avatar: string;
context: string;
subContext: string;
content: string;
}
-const ip = 'ws://127.0.0.1';
+const ip = 'ws://127.0. 0.1';
const port = 8760;
const reconnectTimeout = 3000; //ms
let socket: WebSocket;
}
const parsed: RenderMessage = JSON.parse(message);
+ console.log('message received', parsed);
+ if (parsed.content.audio) {
+ }
backgroundMitt.emit('ipc-renderer', {
endpoint: 'render-message',
message: parsed
--- /dev/null
+<template>
+ <svg
+ version="1.1"
+ xmlns="http://www.w3.org/2000/svg"
+ xmlns:xlink="http://www.w3.org/1999/xlink"
+ :viewBox="scrollView"
+ @click="emittMessage"
+ >
+ <g id="messageView" transform="translate(0, 20)">
+ <g v-for="(item, index) in renderArr" :key="index" preserverAspectRatio="none">
+ <transition
+ appear
+ v-on:before-appear="beforeEnter"
+ v-on:appear="enter"
+ v-on:after-enter="afterEnter"
+ >
+ <TextBubble :item="item" :key="index" />
+ </transition>
+ </g>
+ </g>
+ </svg>
+</template>
+
+<script lang="ts">
+import TextBubble from "@/components/textBubble/index.vue";
+import { Message } from "@/types/message/index";
+import { useIpc } from "@/modules/ipc";
+import { Queue } from "@/modules/queue";
+import useTransitions from "./viewDetails/transitions";
+import useScrollView from "@/modules/scrollView";
+import useMitt from "@/modules/mitt";
+import {
+ defineComponent,
+ reactive,
+ onMounted,
+ onUnmounted,
+ watch,
+} from "vue";
+
+export default defineComponent({
+ name: "MessageView",
+ components: { TextBubble },
+
+ setup() {
+ const { emitter } = useMitt();
+ const renderArr: Message[] = reactive([]);
+ const renderQ = new Queue();
+
+ const { invoke } = useIpc("message");
+
+ // Each method will be called on their respective lifecycle event.
+ const { beforeEnter, enter, afterEnter, rendering } = useTransitions();
+ const { scrollView } = useScrollView({
+ targetId: "messageView",
+ });
+
+ const render = (m: Message) => {
+ if (rendering.value) {
+ renderQ.enqueue(m);
+ } else {
+ renderArr.push(m);
+ }
+ };
+
+ watch(rendering, (rendering, _prevRendering) => {
+ if (!rendering) {
+ const m = renderQ.dequeue() as Message | null;
+ if (m) renderArr.push(m);
+ }
+ });
+
+ onMounted(() => {
+ // Call render function on event "render-message."
+ window.ipcRenderer.on("render-message", (event, payload) => {
+ const m = payload.message;
+ if (m.content) render(m);
+ });
+
+ emitter.on("user-message", async (message) => {
+ try {
+ const res = await invoke(message);
+ render(res);
+ } catch (e) {
+ console.log("No response. Do not render.");
+ }
+ });
+ });
+
+ onUnmounted(() => {
+ window.ipcRenderer.removeAllListeners("render-message");
+ emitter.all.clear();
+ });
+
+ return {
+ renderArr,
+ beforeEnter,
+ enter,
+ afterEnter,
+ scrollView,
+ };
+ },
+});
+</script>
+
+<style lang="scss" scoped>
+#messageView {
+ width: 1000px;
+ height: 100vh;
+ color: red;
+}
+</style>
emitter.emit("animate-send");
// send message to backend
- post('message', message);
+ if (input) post('message', message);
}
const {
--- /dev/null
+import { ref, onMounted } from "vue";
+import { Ref } from "@/types/vueRef/index";
+
+/**
+ * handles visualization of individual svg text bubbles
+ */
+export default function useBubbleDrawer({
+ mr,
+ tr,
+}: {
+ mr: Ref<string>;
+ tr: Ref<string>;
+}) {
+ const signatureTranslate = ref("");
+ let messageRect: SVGSVGElement & SVGRectElement;
+ let messageText: SVGSVGElement & SVGTextElement;
+
+ onMounted(async () => {
+ messageRect = (mr.value as unknown) as SVGSVGElement & SVGRectElement;
+ messageText = (tr.value as unknown) as SVGSVGElement & SVGTextElement;
+ });
+
+ function adjustMessageRect(): void {
+ // must get dimensions of text box to fit bubble around
+ const padding = 12.5;
+ const textBox = messageText.getBBox();
+ messageRect.setAttribute("x", String(textBox.x - padding));
+ messageRect.setAttribute("y", String(textBox.y - padding));
+ messageRect.setAttribute("width", String(textBox.width + 2 * padding));
+ messageRect.setAttribute("height", String(textBox.height + 2 * padding));
+ }
+
+ function translateMsgSignature(): void {
+ if (messageRect.getAttribute("height") !== null) {
+ const height = Number(messageRect.getAttribute("height"));
+ signatureTranslate.value = `translate(-10 ${height})`;
+ }
+ }
+
+ async function drawMessage() {
+ // ignore lint error: await needed for proper render
+ await adjustMessageRect();
+ translateMsgSignature();
+ }
+
+ return {
+ drawMessage,
+ signatureTranslate,
+ };
+}
sampleRate: 16000,
deviceId: -1,
closeOnError: false,
- }
+ },
});
-ia.setEncoding('base64');
+// ia.setEncoding('base64');
ia.start();
ia.on('error', (e) => {
console.log('error recording audio', + e);
});
ia.on('data', (chunk) => {
- audioInput.push(chunk);
- console.log('Got %d characters of string data:', chunk.length);
+ audioInput.push(chunk);
+});
+
+const ao = new portAudio.AudioIO({
+ outOptions: {
+ channelCount: 1,
+ sampleFormat: portAudio.SampleFormat16Bit,
+ sampleRate: 16000,
+ deviceId: -1, // Use -1 or omit the deviceId to select the default device
+ closeOnError: true // Close the stream if an audio error is detected, if set false then just log the error
+ }
});
+ao.start();
+
+const callback = () => {
+ console.log('yayayay')
+}
+
+const play = (audio) => {
+
+ console.log('audio length', audio.length);
+ let i = 0;
+ write();
+
+ function write() {
+ let ok = true;
+ do {
+ const chunk = audio[i]
+ console.log(chunk, i)
+
+ i++;
+ if (i === audio.length - 1) {
+ // Last time!
+ ao.write(chunk, null, callback);
+ } else {
+ // See if we should continue, or wait.
+ // Don't pass the callback, because we're not done yet.
+ ok = ao.write(chunk, null);
+ }
+ } while (i < audio.length - 1 && ok);
+ if (i < audio.length - 1) {
+ // Had to stop early!
+ // Write some more once it drains.
+ ao.on('drain', write);
+ }
+ }
+
+}
+
async function processAudio() {
- audio = '';
- audioInput.forEach(s => {
- audio += s;
- })
- const buffer = Buffer.from(audio, 'base64');
+
+ const buf = Buffer.concat(audioInput);
+
+ play(audioInput);
+
audioInput = [];
- await transcribeSpeech({
- content: buffer
- });
+
+ transcribeSpeech({
+ content: buf
+ });
}
setTimeout(async () => {
ia.pause();
- await processAudio();
+ processAudio();
// ia.resume();
}, 3000);
setTimeout(() => {
ia.resume();
-}, 4000)
+}, 5000)
setTimeout(async () => {
ia.pause();
- await processAudio();
+ processAudio();
// ia.resume();
}, 8000);
console.log(message)
if (message === 'no-token' || message === 'undefined') {
- console.log('no token to authenticate');
ws.send('success');
+ } else if (message instanceof Buffer) {
+ // audio message
+
+ } else {
+ const parsed = JSON.parse(message);
}
});
projects / mime-chat / commitdiff