stream path animation v4
update input items styling add ipc event handler for message add send-message ipc handler add audio to send-message ipc remove extra JSON stringify statement for audio message add render bug fix for add contacts
This commit is contained in:
Enrique Hernandez
riqo
parent
52f6893c50
commit
1627e07964
23 changed files with 450 additions and 295 deletions
|
|
@ -8,22 +8,15 @@ import * as path from "path";
|
|||
import { SpeechRecorder } from "speech-recorder";
|
||||
|
||||
// TODO: connect To Crimata-BE
|
||||
// import WebSocket from "ws";
|
||||
import WebSocket from "ws";
|
||||
|
||||
// const ws = new WebSocket("ws://localhost:8080");
|
||||
// ws.on("open", function open() {
|
||||
// ws.send("hello fro client");
|
||||
// });
|
||||
// ws.on("message", (message: any) => {
|
||||
// console.log("received message", message);
|
||||
// });
|
||||
//
|
||||
|
||||
// const recorder = new SpeechRecorder({ sampleRate: 16000, framesPerBuffer: 320 });
|
||||
|
||||
// Keep a global reference of the window object, if you don't, the window will
|
||||
// be closed automatically when the JavaScript object is garbage collected.
|
||||
let win: BrowserWindow | null;
|
||||
let ws: any;
|
||||
|
||||
// Scheme must be registered before the app is ready
|
||||
protocol.registerSchemesAsPrivileged([
|
||||
|
|
@ -56,44 +49,35 @@ function createWindow() {
|
|||
win.loadURL("app://./index.html");
|
||||
}
|
||||
|
||||
|
||||
win.webContents.on('did-finish-load', () => {
|
||||
// ipcMain.on('audio-stream', (Event: any, Payload: any) => {
|
||||
// if (Payload.stream) {
|
||||
// console.log('begin streaming');
|
||||
// recorder.start({
|
||||
// onAudio: (audio: object, speech: boolean) => {
|
||||
// console.log(audio)
|
||||
// }
|
||||
// });
|
||||
// } else {
|
||||
// recorder.stop();
|
||||
// console.log('stop streaming')
|
||||
// }
|
||||
// })
|
||||
|
||||
// if (win) {
|
||||
// recorder.start({
|
||||
// onAudio: (audio: object, speech: boolean) => {
|
||||
// if (win) {
|
||||
// win.setTitle("Listening...");
|
||||
// // win.webContents.send('render-audio', {
|
||||
// // audio: audio,
|
||||
// // speech: speech
|
||||
// // })
|
||||
// if (speech) {
|
||||
// win.setTitle("Recording...");
|
||||
// // TODO: send audio to crimata BE
|
||||
// // ws.send(audio);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
ws = new WebSocket("ws://localhost:8080");
|
||||
ws.on("open", function open() {
|
||||
ws.send("hernandeze2@xavier.edu");
|
||||
});
|
||||
ws.on("message", (message: any) => {
|
||||
const parsed = JSON.parse(message);
|
||||
console.log('new message', message)
|
||||
if (win) {
|
||||
win.webContents.send('render-message', {
|
||||
message: parsed
|
||||
})
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.on('send-message', (Event: any, payload: any) => {
|
||||
const stringMessage = JSON.stringify(payload.message)
|
||||
ws.send(stringMessage);
|
||||
})
|
||||
|
||||
ipcMain.on('update-menu-bar', (Event: any, payload: any) => {
|
||||
// win.setTitle("Listening...");
|
||||
})
|
||||
})
|
||||
|
||||
win.on("closed", () => {
|
||||
win = null;
|
||||
ws = null;
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@
|
|||
class="messageList"
|
||||
transform="translate(0, 25)"
|
||||
preserverAspectRatio="none"
|
||||
|
||||
>
|
||||
<transition
|
||||
appear
|
||||
|
|
@ -28,13 +27,9 @@
|
|||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import useRenderView from '@/core/messenger/renderView/useRenderView';
|
||||
import useMessageView from "@/core/messenger/messageView/useMessageView";
|
||||
import useScrollView from "@/core/UI/scrollView/useScrollView";
|
||||
import {
|
||||
defineComponent,
|
||||
reactive,
|
||||
inject,
|
||||
} from "vue";
|
||||
import { defineComponent, reactive, inject } from "vue";
|
||||
import MessageItem from "./messageItem.vue";
|
||||
import { Message } from "@/types/message/index";
|
||||
import { Mitt } from "@/types/mitt/index";
|
||||
|
|
@ -54,9 +49,9 @@ export default defineComponent({
|
|||
});
|
||||
}
|
||||
|
||||
const { beforeEnter, enter, afterEnter } = useRenderView();
|
||||
const { scrollView } = useScrollView({
|
||||
targetId: "messageRenderer"
|
||||
const { beforeEnter, enter, afterEnter } = useMessageView();
|
||||
const { scrollView } = useScrollView({
|
||||
targetId: "messageRenderer",
|
||||
});
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -2,35 +2,35 @@ import { Message } from "@/types/message/index";
|
|||
|
||||
const Messages: Message[] = [
|
||||
{
|
||||
content: "Sounds good, see you Thursday.",
|
||||
context: "Message from Josie Wright",
|
||||
subContext: "Subject: Chick File",
|
||||
content: "Welcome",
|
||||
context: "ai",
|
||||
subContext: "",
|
||||
modifier: "ai",
|
||||
id: "2",
|
||||
time: ""
|
||||
},
|
||||
{
|
||||
content:
|
||||
"I can be there at 5 pm tomorrow to greet you at a Chick file with my almond and chive salad.",
|
||||
context: "You to Josie Wright",
|
||||
subContext: "josie@gmail.com",
|
||||
"add friend",
|
||||
context: "launch",
|
||||
subContext: "",
|
||||
modifier: "sf",
|
||||
id: "1",
|
||||
time: ""
|
||||
},
|
||||
{
|
||||
content: "Sweet, thanks!",
|
||||
context: "Message from Tommy McConville",
|
||||
content: "Did you mean to addcontact?",
|
||||
context: "addcontact",
|
||||
subContext: "",
|
||||
modifier: "ai",
|
||||
id: "5",
|
||||
time: ""
|
||||
},
|
||||
{
|
||||
content: "Hey do you know Enrique Hernandez?",
|
||||
context: "Message from Crimata AI",
|
||||
content: "yes",
|
||||
context: "fulfill",
|
||||
subContext: "",
|
||||
modifier: "ai",
|
||||
modifier: "sf",
|
||||
id: "6",
|
||||
time: ""
|
||||
},
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ export default defineComponent({
|
|||
}
|
||||
}
|
||||
|
||||
window.ipcRenderer.on("render-message", (event, payload) => {
|
||||
console.log(payload.message)
|
||||
emitter.emit("renderMessage", payload.message);
|
||||
})
|
||||
|
||||
return {
|
||||
emittMessage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,53 +1,68 @@
|
|||
<template>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" :width="bubbleWidth" height="34" :x="center" y="450" >
|
||||
<g id="Group_126" data-name="Group 126" transform="translate(0 0)">
|
||||
<rect id="Rectangle_478" data-name="Rectangle 478" :width="bubbleWidth" height="34" rx="10" transform="translate(0 0)" fill="#b9b9b9"/>
|
||||
<g id="Group_125" data-name="Group 125">
|
||||
|
||||
<path v-for="(path, index) in paths"
|
||||
:key="index"
|
||||
:id="path.id"
|
||||
:data-name="path.name"
|
||||
:d="path.d"
|
||||
:transform="path.transform"
|
||||
fill="none"
|
||||
stroke="#000"
|
||||
stroke-linecap="round"
|
||||
stroke-width="1.5"
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
:width="bubbleWidth"
|
||||
height="34"
|
||||
:x="center"
|
||||
y="440"
|
||||
rx="10"
|
||||
class="audioInput"
|
||||
>
|
||||
<g
|
||||
id="Group_126"
|
||||
data-name="Group 126"
|
||||
transform="translate(0 0)"
|
||||
rx="10"
|
||||
>
|
||||
<rect
|
||||
id="Rectangle_478"
|
||||
data-name="Rectangle 478"
|
||||
:width="bubbleWidth"
|
||||
height="34"
|
||||
rx="10"
|
||||
transform="translate(0 0)"
|
||||
fill="#b9b9b9"
|
||||
/>
|
||||
|
||||
<!-- <path id="Path_328" data-name="Path 328" d="M0,25v-10" transform="translate(10 0)" fill="none" stroke="#000" stroke-linecap="round" stroke-width="1.5"/>
|
||||
<path id="Path_328" data-name="Path 328" d="M0,25v-15" transform="translate(15 0)" fill="none" stroke="#000" stroke-linecap="round" stroke-width="1.5"/>
|
||||
<path id="Path_328" data-name="Path 328" d="M0,25v-20" transform="translate(20 0)" fill="none" stroke="#000" stroke-linecap="round" stroke-width="1.5"/>
|
||||
|
||||
<path id="Path_328" data-name="Path 328" d="M0,25v-15" transform="translate(25 0)" fill="none" stroke="#000" stroke-linecap="round" stroke-width="1.5"/>
|
||||
<path id="Path_328" data-name="Path 328" d="M0,25v-10" transform="translate(30 0)" fill="none" stroke="#000" stroke-linecap="round" stroke-width="1.5"/>
|
||||
|
||||
<path id="Path_328" data-name="Path 328" d="M0,25v-5" transform="translate(120 0)" fill="none" stroke="#000" stroke-linecap="round" stroke-width="1.5"/>
|
||||
--> </g>
|
||||
</g>
|
||||
</svg>
|
||||
|
||||
<svg>
|
||||
<g rx="20" id="Group_125" data-name="Group 125" v-for="(path, index) in paths">
|
||||
<path
|
||||
:key="index"
|
||||
id="path"
|
||||
:data-name="path.identifier"
|
||||
:d="path.d"
|
||||
:transform="`translate(${path.offset} 0)`"
|
||||
fill="none"
|
||||
:stroke="path.color"
|
||||
stroke-linecap="round"
|
||||
stroke-width="1.5"
|
||||
/>
|
||||
</g>
|
||||
</svg>
|
||||
</g>
|
||||
</svg>
|
||||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import { defineComponent, computed, ref } from 'vue'
|
||||
export default defineComponent ({
|
||||
name: 'AudioInput',
|
||||
import { defineComponent, computed, inject } from "vue";
|
||||
export default defineComponent({
|
||||
name: "AudioInput",
|
||||
props: {
|
||||
bubbleWidth: Number,
|
||||
paths: Array
|
||||
paths: Array,
|
||||
},
|
||||
setup(props){
|
||||
setup(props) {
|
||||
|
||||
// need audioPath type/interface
|
||||
const emitter: any = inject("mitt");
|
||||
function audioSend() {
|
||||
|
||||
}
|
||||
|
||||
const center = computed(() => {
|
||||
if (props.bubbleWidth) {
|
||||
return 175 - props.bubbleWidth / 2;
|
||||
}
|
||||
})
|
||||
return { center }
|
||||
}
|
||||
})
|
||||
</script>
|
||||
});
|
||||
return { center };
|
||||
},
|
||||
});
|
||||
</script>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
<template>
|
||||
<foreignObject class="inputContainer" :x="textXoffset" y="450" width="55%" height="50%">
|
||||
<foreignObject
|
||||
class="inputContainer"
|
||||
:x="textXoffset"
|
||||
y="450"
|
||||
width="55%"
|
||||
height="50%"
|
||||
>
|
||||
<div xmlns="http://www.w3.org/1999/xhtml">
|
||||
<div class="inputBubble" contenteditable v-show="input.length > 0">
|
||||
<p>{{ input }}</p>
|
||||
|
|
@ -28,15 +34,15 @@ div[contenteditable] {
|
|||
display: table;
|
||||
justify-items: center;
|
||||
border: 0;
|
||||
border-radius: 20px;
|
||||
color: #fff;
|
||||
background-color: #585858;
|
||||
border-radius: 10px;
|
||||
color: #000;
|
||||
background-color: #b9b9b9;
|
||||
font-size: 14;
|
||||
text-align: left;
|
||||
p {
|
||||
max-width: 150px;
|
||||
overflow-wrap: break-word;
|
||||
padding: 10px 15px 10px 15px;
|
||||
padding: 7px 10px 5px 10px;
|
||||
margin: 0;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,32 @@
|
|||
<template>
|
||||
<audio-input
|
||||
v-if="renderAudio"
|
||||
:bubbleWidth="reactiveWidth"
|
||||
:paths="paths"
|
||||
/>
|
||||
<text-input
|
||||
v-else
|
||||
:input="input"
|
||||
:textXoffset="textInputXoffset"
|
||||
/>
|
||||
<audio-input v-if="visualizeStream" :bubbleWidth="audioBubbleWidth" :paths="paths" />
|
||||
<text-input v-else :input="input" :textXoffset="textInputXoffset" />
|
||||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import { defineComponent } from 'vue'
|
||||
import TextInput from './inputDetails/textInput.vue';
|
||||
import AudioInput from './inputDetails/audioInput.vue';
|
||||
import { defineComponent } from "vue";
|
||||
import TextInput from "./inputDetails/textInput.vue";
|
||||
import AudioInput from "./inputDetails/audioInput.vue";
|
||||
import useInputController from "@/core/UI/input/useInputController";
|
||||
export default defineComponent({
|
||||
name: "InputItem",
|
||||
components: {TextInput, AudioInput},
|
||||
components: { TextInput, AudioInput },
|
||||
setup() {
|
||||
|
||||
const {
|
||||
const {
|
||||
input,
|
||||
renderAudio,
|
||||
reactiveWidth,
|
||||
visualizeStream,
|
||||
textInputXoffset,
|
||||
paths } = useInputController();
|
||||
audioBubbleWidth,
|
||||
paths,
|
||||
} = useInputController();
|
||||
|
||||
return {
|
||||
input,
|
||||
textInputXoffset,
|
||||
renderAudio,
|
||||
reactiveWidth,
|
||||
paths
|
||||
}
|
||||
}
|
||||
})
|
||||
</script>
|
||||
visualizeStream,
|
||||
audioBubbleWidth,
|
||||
paths,
|
||||
};
|
||||
},
|
||||
});
|
||||
</script>
|
||||
|
|
|
|||
28
src/core/UI/input/audio/audioDetails/useAnimations.ts
Normal file
28
src/core/UI/input/audio/audioDetails/useAnimations.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import AnimeFunc from "@/types/animejs/index";
|
||||
import {inject} from "vue";
|
||||
import {Ref} from "@/types/vueRef/index";
|
||||
|
||||
export default function useAudioAnimations() {
|
||||
// imports animejs safely
|
||||
let anime: AnimeFunc;
|
||||
const animeInject: AnimeFunc | undefined = inject("animejs");
|
||||
if (animeInject) anime = animeInject;
|
||||
|
||||
function animateSend(el: Element, trigger: Ref<boolean>) {
|
||||
console.log(el);
|
||||
anime({
|
||||
targets: el,
|
||||
opacity: 0,
|
||||
translateY: -100,
|
||||
duration: 0,
|
||||
complete: function() {
|
||||
trigger.value = false;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
animateSend
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,8 +1,10 @@
|
|||
import {inject} from 'vue';
|
||||
export default function useStreamRecord(): {
|
||||
initRecorder: (stream: MediaStream) => MediaRecorder;
|
||||
} {
|
||||
let streamRecorder: MediaRecorder;
|
||||
const chunks: Blob[] = [];
|
||||
const emitter: any = inject("mitt");
|
||||
|
||||
const initRecorder = (stream: MediaStream) => {
|
||||
streamRecorder = new MediaRecorder(stream);
|
||||
|
|
@ -10,6 +12,31 @@ export default function useStreamRecord(): {
|
|||
streamRecorder.onstop = (e: any) => {
|
||||
// send audio to BE
|
||||
const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' });
|
||||
// emitter.emit("newSelfMessage", audioBlob);
|
||||
const blobToBase64 = (blob: any) => {
|
||||
return new Promise((resolve) => {
|
||||
const reader = new FileReader();
|
||||
reader.readAsDataURL(blob);
|
||||
reader.onloadend = function () {
|
||||
resolve(reader.result);
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
(async () => {
|
||||
const b64 = await blobToBase64(audioBlob);
|
||||
const jsonString = JSON.stringify({blob: b64});
|
||||
const message = {
|
||||
text: '',
|
||||
audio: b64
|
||||
}
|
||||
window.postMessage({
|
||||
myTypeField: 'send-message',
|
||||
message: message
|
||||
}, '*')
|
||||
})();
|
||||
|
||||
|
||||
}
|
||||
return streamRecorder
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
import {Ref} from '@/types/vueRef/index';
|
||||
export default function useStreamRender(render: Ref<boolean>) {
|
||||
|
||||
// window.AudioContext = window.AudioContext;
|
||||
const audioContext = new AudioContext();
|
||||
const analyzer = audioContext.createAnalyser();
|
||||
analyzer.fftSize = 2048;
|
||||
const xInitial = 24;
|
||||
const xFinal = 640;
|
||||
const dataArray = new Uint8Array(analyzer.frequencyBinCount);
|
||||
|
||||
interface AmplitudePath {
|
||||
id: string;
|
||||
name: string;
|
||||
d: string;
|
||||
transform: string;
|
||||
}
|
||||
|
||||
const createAudioPath = (a: number, offset: number) => {
|
||||
const identifier = `path_${offset}`
|
||||
const path: AmplitudePath = {
|
||||
id: identifier,
|
||||
name: identifier,
|
||||
d: `M0,25v${-a}`,
|
||||
transform: `translate(${offset} 0)`
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
|
||||
function visualize(w: Ref<number>, p: Ref<number>, paths: Ref<any>) {
|
||||
|
||||
const amplitude = () => {
|
||||
const dataView = dataArray.slice(xInitial, xFinal);
|
||||
const sum = dataView.reduce((a, b) => a + b);
|
||||
const average = sum / dataView.length;
|
||||
const scaled = average * 0.45;
|
||||
return scaled;
|
||||
}
|
||||
|
||||
function draw() {
|
||||
if (!render.value) {
|
||||
return;
|
||||
}
|
||||
requestAnimationFrame(draw);
|
||||
analyzer.getByteFrequencyData(dataArray);
|
||||
|
||||
// max-width
|
||||
if (w.value === 300) {
|
||||
return;
|
||||
}
|
||||
|
||||
// increase width as time passes
|
||||
|
||||
// get amplitude
|
||||
const a = amplitude();
|
||||
|
||||
if (a > 1) {
|
||||
|
||||
w.value++;
|
||||
p.value += 5;
|
||||
console.log(a)
|
||||
const path = createAudioPath(a, p.value);
|
||||
paths.value.push(path)
|
||||
}
|
||||
|
||||
// create path item
|
||||
}
|
||||
draw();
|
||||
}
|
||||
|
||||
function renderStream(s: MediaStream, w: Ref<number>, pathOffset: Ref<number>, paths: Ref<any>) {
|
||||
const source = audioContext.createMediaStreamSource(s);
|
||||
source.connect(analyzer);
|
||||
visualize(w, pathOffset, paths);
|
||||
}
|
||||
|
||||
return {
|
||||
renderStream
|
||||
}
|
||||
}
|
||||
148
src/core/UI/input/audio/useStreamVisualizer.ts
Normal file
148
src/core/UI/input/audio/useStreamVisualizer.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import { Ref } from '@/types/vueRef/index';
|
||||
import AnimeFunc from "@/types/animejs/index";
|
||||
import { inject, ref } from "vue";
|
||||
|
||||
interface AudioPath {
|
||||
identifier: string;
|
||||
d: string;
|
||||
offset: number;
|
||||
color: string;
|
||||
draw: boolean;
|
||||
}
|
||||
export default function useStreamVisualizer(draw: Ref<boolean>) {
|
||||
const audioContext = new AudioContext();
|
||||
console.log('audi sample rate', audioContext.sampleRate)
|
||||
const analyzer = audioContext.createAnalyser();
|
||||
analyzer.fftSize = 128;
|
||||
const bufferLength = analyzer.frequencyBinCount;
|
||||
const dataArray = new Uint8Array(bufferLength);
|
||||
|
||||
// imports animejs safely
|
||||
let anime: AnimeFunc;
|
||||
const animeInject: AnimeFunc | undefined = inject("animejs");
|
||||
if (animeInject) anime = animeInject;
|
||||
|
||||
/*
|
||||
* Draws MediaStream audioTrack as 'AudioPath' objects
|
||||
*/
|
||||
function drawStreamAs(paths: Ref<any>, bubbleWidth: Ref<number>): void {
|
||||
const xInitial = ref(10);
|
||||
const maxAmplitude = 15;
|
||||
const maxBubbleWidth = 300;
|
||||
|
||||
function shiftPathStream(pathList: NodeList): void {
|
||||
for (let i = 0; i < pathList.length; i++) {
|
||||
const stick = pathList[i] as HTMLElement;
|
||||
const computedQuery = window.getComputedStyle(stick);
|
||||
const matrix = new WebKitCSSMatrix(computedQuery.webkitTransform);
|
||||
anime({
|
||||
targets: stick,
|
||||
translateX: [matrix.m41, matrix.m41 - 6],
|
||||
})
|
||||
}
|
||||
}
|
||||
function filterPathList(pathList: NodeList) {
|
||||
for (let i = 0; i < pathList.length; i++) {
|
||||
const stick = pathList[i] as HTMLElement;
|
||||
const computedQuery = window.getComputedStyle(stick);
|
||||
const matrix = new WebKitCSSMatrix(computedQuery.webkitTransform);
|
||||
if (matrix.m41 < 10) {
|
||||
anime({
|
||||
targets: stick,
|
||||
opacity: 0,
|
||||
duration: 0
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function drawNewPath(): void {
|
||||
|
||||
const calculateAmplitude = (): number => (
|
||||
Math.min(Math.max(
|
||||
(dataArray.reduce((a, b) => a + b) / bufferLength) * 0.45,
|
||||
0), maxAmplitude)
|
||||
)
|
||||
|
||||
const createAudioPath = (a: number, o: number): AudioPath => ({
|
||||
identifier: `path_${o}`,
|
||||
d: `M0,25v${-a}`,
|
||||
offset: o,
|
||||
color: '#000',
|
||||
draw: true
|
||||
})
|
||||
|
||||
const amplitude = calculateAmplitude();
|
||||
const p: AudioPath = createAudioPath(amplitude, xInitial.value);
|
||||
paths.value.push(p);
|
||||
}
|
||||
|
||||
paths.value = [];
|
||||
function drawPathStream(): void {
|
||||
|
||||
let streamPath;
|
||||
if (!draw.value) {
|
||||
window.clearTimeout(streamPath)
|
||||
return;
|
||||
}
|
||||
setTimeout(() => {
|
||||
requestAnimationFrame(drawPathStream);
|
||||
analyzer.getByteFrequencyData(dataArray);
|
||||
}, 1000 / 10)
|
||||
|
||||
/*
|
||||
* To Draw pathStream
|
||||
*/
|
||||
if (bubbleWidth.value === 300) {
|
||||
xInitial.value = bubbleWidth.value - 10;
|
||||
} else if (bubbleWidth.value > 20) {
|
||||
xInitial.value = bubbleWidth.value;
|
||||
} else {
|
||||
xInitial.value +=3;
|
||||
}
|
||||
streamPath = setTimeout(async () => {
|
||||
const pathList = document.querySelectorAll('path');
|
||||
filterPathList(pathList);
|
||||
drawNewPath();
|
||||
if (bubbleWidth.value === maxBubbleWidth) {
|
||||
shiftPathStream(pathList);
|
||||
}
|
||||
}, 200)
|
||||
}
|
||||
drawPathStream();
|
||||
}
|
||||
|
||||
function increaseBubbleWidth(bubbleRef: Ref<number>) {
|
||||
const increase = () => {
|
||||
|
||||
let animation;
|
||||
if (!draw.value) {
|
||||
if (animation) {
|
||||
cancelAnimationFrame(animation)
|
||||
}
|
||||
return;
|
||||
}
|
||||
animation = requestAnimationFrame(increase);
|
||||
if (bubbleRef.value < 300) {
|
||||
bubbleRef.value++;
|
||||
}
|
||||
}
|
||||
increase();
|
||||
}
|
||||
|
||||
function expandBubble(bubbleWidth: Ref<number>) {
|
||||
increaseBubbleWidth(bubbleWidth);
|
||||
}
|
||||
|
||||
function visualizeStreamAsPaths(s: MediaStream, paths: Ref<any>, bubbleWidth: Ref<number>): void {
|
||||
const source = audioContext.createMediaStreamSource(s);
|
||||
source.connect(analyzer);
|
||||
drawStreamAs(paths, bubbleWidth);
|
||||
}
|
||||
|
||||
return {
|
||||
visualizeStreamAsPaths,
|
||||
expandBubble
|
||||
}
|
||||
}
|
||||
|
|
@ -35,6 +35,7 @@ export default function useTextInputAnims() {
|
|||
anime({
|
||||
targets: el,
|
||||
translateY: -10,
|
||||
duration: 0
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,33 +1,35 @@
|
|||
import {ref, computed, inject} from 'vue';
|
||||
import { ref, computed, inject } from 'vue';
|
||||
import useTextInputAnims from "./textDetails/useAnimations";
|
||||
import { Ref } from "@/types/vueRef/index";
|
||||
/*
|
||||
* text rendering logic used by inputController
|
||||
*/
|
||||
export default function useTextRender(input: Ref<string>) {
|
||||
|
||||
const emitter: any = inject("mitt");
|
||||
const {animateSend, verticalShiftInput, inputAppear} = useTextInputAnims();
|
||||
|
||||
const inputHeight = ref(42);
|
||||
const inputWidth = ref(0);
|
||||
const renderTextInput = async() => {
|
||||
const inputDivs = await document.getElementsByClassName("inputBubble");
|
||||
const height = inputDivs[0].getBoundingClientRect().height;
|
||||
inputWidth.value = inputDivs[0].getBoundingClientRect().width;
|
||||
const foreignObjectDiv = document.getElementsByClassName(
|
||||
"inputContainer"
|
||||
);
|
||||
if (height !== inputHeight.value) {
|
||||
inputHeight.value = height;
|
||||
}
|
||||
const foreignEl = foreignObjectDiv[0] as HTMLElement;
|
||||
if (inputHeight.value === 0) {
|
||||
inputAppear(foreignEl)
|
||||
} else if (inputHeight.value > 36) {
|
||||
const yTrans = -inputHeight.value + 30;
|
||||
verticalShiftInput(foreignEl, yTrans);
|
||||
}
|
||||
|
||||
const emitter: any = inject("mitt");
|
||||
const { animateSend, verticalShiftInput, inputAppear } = useTextInputAnims();
|
||||
|
||||
async function renderTextInput() {
|
||||
// first, get input dimensions
|
||||
const inputDivs = await document.getElementsByClassName("inputBubble");
|
||||
const height = inputDivs[0].getBoundingClientRect().height;
|
||||
inputWidth.value = inputDivs[0].getBoundingClientRect().width;
|
||||
// then, query for foreignObject div
|
||||
const foreignObjectDiv = document.getElementsByClassName(
|
||||
"inputContainer"
|
||||
);
|
||||
const foreignEl = foreignObjectDiv[0] as HTMLElement;
|
||||
if (height !== inputHeight.value) {
|
||||
inputHeight.value = height;
|
||||
}
|
||||
if (inputHeight.value === 0) {
|
||||
inputAppear(foreignEl)
|
||||
} else if (inputHeight.value > 36) {
|
||||
const yTrans = -inputHeight.value + 30;
|
||||
verticalShiftInput(foreignEl, yTrans);
|
||||
}
|
||||
}
|
||||
|
||||
emitter.on("newSelfMessage", async (payload: any) => {
|
||||
|
|
@ -44,4 +46,4 @@ export default function useTextRender(input: Ref<string>) {
|
|||
textInputXoffset,
|
||||
renderTextInput
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,73 +1,77 @@
|
|||
import {ref, watch, onUnmounted, onMounted } from "vue";
|
||||
import { ref, watch, onUnmounted, onMounted } from "vue";
|
||||
import useKeyDownHandler from "@/utils/keyDownHandler/useKeyDownHandler";
|
||||
import useMediaStream from "@/utils/mediaStream/useMediaStream";
|
||||
import useStreamRecord from "./audio/useStreamRecord";
|
||||
import useStreamRender from './audio/useStreamRender';
|
||||
import useStreamVisualizer from './audio/useStreamVisualizer';
|
||||
import useTextRender from './text/useTextRender';
|
||||
import useAudioAnimations from './audio/audioDetails/useAnimations';
|
||||
|
||||
/*
|
||||
* Input controller logic
|
||||
* Will render text input by default as user types or
|
||||
* if user holds space bar down, audio media-stream
|
||||
* will be rendered & recorded
|
||||
* Will visualize text input by default as user types or
|
||||
* if user holds space bar down, input audio stream
|
||||
* will be visualized & recorded
|
||||
*/
|
||||
// NOTE: input params should probably be input and MediaStream
|
||||
export default function useInputController() {
|
||||
// program ref varaiables
|
||||
const renderAudio = ref(false);
|
||||
const reactiveWidth = ref(10);
|
||||
const pathOffset = ref(10);
|
||||
const visualizeStream = ref(false);
|
||||
const paths = ref([]);
|
||||
// audioStream variables
|
||||
const audioBubbleWidth = ref(10);
|
||||
let stream: MediaStream;
|
||||
let recorder: MediaRecorder;
|
||||
|
||||
// use input ref & keydown handler function
|
||||
|
||||
const { keyDownHandler, input } = useKeyDownHandler();
|
||||
// use text Offset computed prop and & textRender function
|
||||
const { textInputXoffset, renderTextInput } = useTextRender(input);
|
||||
// use stream render and record logic
|
||||
const { renderStream } = useStreamRender(renderAudio);
|
||||
const { visualizeStreamAsPaths, expandBubble } = useStreamVisualizer(visualizeStream);
|
||||
const { initRecorder } = useStreamRecord()
|
||||
const { animateSend } = useAudioAnimations();
|
||||
|
||||
function audioSend() {
|
||||
const domQuery = document.getElementsByClassName("audioInput");
|
||||
const inputEl = domQuery[0];
|
||||
animateSend(inputEl, visualizeStream);
|
||||
}
|
||||
|
||||
// stops stream recording on space key up
|
||||
function keyupHandler(e: any) {
|
||||
if (e.key === " " && visualizeStream.value) {
|
||||
console.log("stop recording");
|
||||
recorder.stop();
|
||||
visualizeStream.value = false;
|
||||
paths.value = []
|
||||
audioBubbleWidth.value = 10;
|
||||
input.value = ''
|
||||
}
|
||||
}
|
||||
|
||||
// add window event keyboard listeners
|
||||
window.addEventListener("keydown", keyDownHandler);
|
||||
window.addEventListener('keyup', keyupHandler);
|
||||
|
||||
// get stream & initialize recorder object on mount
|
||||
onMounted(async () => {
|
||||
stream = await useMediaStream();
|
||||
console.log(stream)
|
||||
recorder = initRecorder(stream);
|
||||
})
|
||||
|
||||
// callback function called on key up event
|
||||
function keyupHandler(e: any) {
|
||||
// stop stream recording on space key up
|
||||
if (e.key === " " && renderAudio.value) {
|
||||
console.log("stop recording");
|
||||
recorder.stop();
|
||||
// reset controller ref variables
|
||||
renderAudio.value = false;
|
||||
input.value = "";
|
||||
reactiveWidth.value = 10;
|
||||
pathOffset.value = 10;
|
||||
}
|
||||
}
|
||||
|
||||
// step 1: add window event keyboard listeners
|
||||
window.addEventListener("keydown", keyDownHandler);
|
||||
window.addEventListener('keyup', keyupHandler);
|
||||
|
||||
// step 2: determine whether to render text or audio based one the keyboard input
|
||||
// determine whether to render text or audio based one the keyboard input
|
||||
watch(input, (input, prevInput) => {
|
||||
if (prevInput !== "" || prevInput.length > 0) {
|
||||
if (!renderAudio.value) {
|
||||
if (!visualizeStream.value) {
|
||||
renderTextInput();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (input === " " && renderAudio.value === false) {
|
||||
console.log("record");
|
||||
renderAudio.value = true;
|
||||
if (input === " " && visualizeStream.value === false) {
|
||||
console.log("record MediaStream");
|
||||
visualizeStream.value = true;
|
||||
recorder.start();
|
||||
renderStream(stream, reactiveWidth, pathOffset, paths);
|
||||
expandBubble(audioBubbleWidth);
|
||||
visualizeStreamAsPaths(stream, paths, audioBubbleWidth);
|
||||
return;
|
||||
}
|
||||
}
|
||||
renderTextInput();
|
||||
});
|
||||
|
||||
|
|
@ -79,10 +83,10 @@ export default function useInputController() {
|
|||
|
||||
return {
|
||||
input,
|
||||
renderAudio,
|
||||
reactiveWidth,
|
||||
visualizeStream,
|
||||
textInputXoffset,
|
||||
audioBubbleWidth,
|
||||
paths
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ export default function useMessageItemComp(m: string, id: string) {
|
|||
if (messageRenderer) {
|
||||
const rect = messageRenderer.getBoundingClientRect();
|
||||
let Y = 600;
|
||||
console.log('renderer height',rect.height)
|
||||
if (rect.height > 500) {
|
||||
Y = rect.height + messageMarginTop;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,21 @@
|
|||
import useRendererAnims from "./renderDetails/useAnimations";
|
||||
import useOffsetCalculator from './renderDetails/useOffsetCalculator';
|
||||
import useMessageViewAnims from "./viewDetails/useAnimations";
|
||||
import useOffsetCalculator from './viewDetails/useOffsetCalculator';
|
||||
import { VueTransitionCallback } from '@/types/vue3/vueTransitionCallback/index';
|
||||
|
||||
|
||||
/**
|
||||
* renderView component logic
|
||||
* handles rendering of exisiting user messages
|
||||
* messageView component logic
|
||||
* handles visualization of exisiting user messages
|
||||
* as well as incoming messages
|
||||
*/
|
||||
export default function useRenderView(): {
|
||||
export default function useMessageView(): {
|
||||
beforeEnter: VueTransitionCallback;
|
||||
enter: VueTransitionCallback;
|
||||
afterEnter: VueTransitionCallback;
|
||||
} {
|
||||
// uses offsetCalculator logic and renderer animations
|
||||
const { calculateMessageOffsets } = useOffsetCalculator();
|
||||
const { shiftAnimate, stackAnimate } = useRendererAnims();
|
||||
const { shiftAnimate, stackAnimate } = useMessageViewAnims();
|
||||
|
||||
const messagePadding = 15;
|
||||
let messageList: NodeList;
|
||||
|
|
@ -54,11 +54,18 @@ export default function useRenderView(): {
|
|||
// get message positioning & shift state before animating
|
||||
if (message) {
|
||||
messageOffset = calculateMessageOffsets(message);
|
||||
shift = messageOffset.y > 445 ? true : false;
|
||||
console.log(messageOffset.y)
|
||||
shift = messageOffset.y > 400 ? true : false;
|
||||
}
|
||||
if (el) {
|
||||
if (shift) {
|
||||
shiftOffset -= el.getBBox().height + messagePadding;
|
||||
|
||||
console.log(messageOffset.y)
|
||||
if (messageOffset.y < 445) {
|
||||
shiftOffset -= 15;
|
||||
} else {
|
||||
shiftOffset -= el.getBBox().height + messagePadding;
|
||||
}
|
||||
}
|
||||
animateMessage(el);
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import { MessageAnimeFunc } from '@/types/message/index';
|
|||
/**
|
||||
* anime js animations used by message renderer
|
||||
*/
|
||||
export default function useRendererAnims(): {
|
||||
export default function useMessageViewAnims(): {
|
||||
shiftAnimate: MessageAnimeFunc;
|
||||
stackAnimate: MessageAnimeFunc;
|
||||
} {
|
||||
|
|
@ -2,8 +2,10 @@
|
|||
// It has the same sandbox as a Chrome extension.
|
||||
import { ipcRenderer } from "electron";
|
||||
|
||||
interface Window {
|
||||
ipcRenderer: typeof ipcRenderer;
|
||||
declare global {
|
||||
interface Window {
|
||||
ipcRenderer: typeof ipcRenderer;
|
||||
}
|
||||
}
|
||||
window.ipcRenderer = ipcRenderer;
|
||||
|
||||
|
|
@ -12,8 +14,12 @@ process.once("loaded", () => {
|
|||
// do something with custom event
|
||||
const message = event.data;
|
||||
|
||||
if (message.myTypeField === "audio-stream") {
|
||||
ipcRenderer.send("audio-stream", message);
|
||||
if (message.myTypeField === "send-message") {
|
||||
ipcRenderer.send("send-message", message);
|
||||
}
|
||||
|
||||
if (message.myTypeField === "update-menu-bar") {
|
||||
ipcRenderer.send("update-menu-bar", message);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
|
||||
interface AnimeFuncParams {
|
||||
targets: Element | HTMLElement | SVGGElement | SVGElement;
|
||||
translateY?: number;
|
||||
translateX?: number | any[] | undefined;
|
||||
transform?: string;
|
||||
easing?: string;
|
||||
duration?: number;
|
||||
offset?: number;
|
||||
begin?: () => void;
|
||||
complete?: () => void;
|
||||
stroke?: string;
|
||||
opacity?: number;
|
||||
}
|
||||
|
||||
export default interface AnimeFunc {
|
||||
|
|
|
|||
|
|
@ -19,15 +19,16 @@ export default function useKeyDownHandler() {
|
|||
return;
|
||||
}
|
||||
const message = {
|
||||
content: input.value,
|
||||
signature: "EnriqueH",
|
||||
outgoing: true,
|
||||
modifier: "sf",
|
||||
imgURL: "/test/path",
|
||||
id: null
|
||||
text: input.value,
|
||||
audio: 0
|
||||
};
|
||||
// fire event with message payload
|
||||
emitter.emit("newSelfMessage", message);
|
||||
|
||||
window.postMessage({
|
||||
myTypeField: 'send-message',
|
||||
message: message
|
||||
}, '*')
|
||||
}
|
||||
|
||||
function keyDownHandler(e: KeyDownEvent) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
export default async function useMediaStream() {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
// var audioTrack = stream.getAudioTracks()[0];
|
||||
// console.log(audioTrack.getSettings())
|
||||
return stream;
|
||||
}
|
||||
|
|
@ -1,5 +1,13 @@
|
|||
const WebSocket = require("ws");
|
||||
|
||||
const testMessage = {
|
||||
content: 'Hello from test server!' ,
|
||||
context: 'test message',
|
||||
subContext: '',
|
||||
modifiers: 'ai',
|
||||
id: "0",
|
||||
time: 'test'
|
||||
}
|
||||
const wss = new WebSocket.Server({
|
||||
port: 8080
|
||||
// perMessageDeflate: {
|
||||
|
|
@ -30,5 +38,5 @@ wss.on("connection", function connection(ws, req) {
|
|||
|
||||
const ip = req.socket.remoteAddress;
|
||||
console.log("received connection from", ip);
|
||||
ws.send("hello from server!");
|
||||
ws.send(JSON.stringify(testMessage));
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue