me/mime-chat
Watch
1
0
Fork
You've already forked mime-chat
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

added streamRender animation

Browse source
This commit is contained in:
riqo 2020-12-11 16:20:41 -06:00
commit 899d6ffe7f
7 changed files with 169 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/components/UI/UIDetails/audioInput.vue Normal file
View file

@ -0,0 +1,20 @@
<template>
<foreignObject x="100" y="450" width="55%" height="50%">
<div xmlns="http://www.w3.org/1999/xhtml">
<div>
<canvas width="640" height="100"></canvas>
</div>
</div>
</foreignObject>
</template>
<script lang="ts">
import {defineComponent} from 'vue'
import useStreamRecorder from "@/composables/streamRecorder/useStreamRecorder";
export default defineComponent ({
name: 'AudioInput',
setup() {
}
})
</script>

51
src/components/UI/UIDetails/inputItem.vue
View file

@ -1,8 +1,9 @@
<template>
<foreignObject class="inputContainer" :x="inputXoffset" y="450" width="55%" height="50%">
<foreignObject class="inputContainer" :x="renderAudio? reactiveWidth: inputXoffset" y="450" width="55%" height="50%">
<div xmlns="http://www.w3.org/1999/xhtml">
<div class="inputBubble" contenteditable v-show="input.length > 0">
<p>{{ input }}</p>
<p v-if="!renderAudio">{{ input }}</p>
<canvas v-else class="audioCanvas" height="20" width="10" ></canvas>
</div>
</div>
</foreignObject>
@ -16,57 +17,20 @@ export default defineComponent({
name: "InputItem",
setup() {
const record = ref(false);
const { inputXoffset, input } = useInputRenderer();
const { inputXoffset, input, renderAudio, reactiveWidth } = useInputRenderer();
const { initRecorder } = useStreamRecorder(record);
onMounted(async () => {
// step get streamRecorder reference
const { streamRecorder } = await initRecorder();
// add event listener for space keyup
window.addEventListener("keyup", (e) => {
if (e.key === " " && streamRecorder.state === "recording") {
console.log("stop recording");
record.value = false;
streamRecorder.stop();
// window.postMessage(
// {
// myTypeField: "audio-stream",
// stream: false,
// },
// "*"
// );
console.log("recorder is", streamRecorder.state);
input.value = "";
}
});
// begin recording on space
watch(input, (input, prevInput) => {
if (prevInput !== "" || prevInput.length > 0) {
return;
}
if (input === " ") {
console.log("record");
record.value = true;
// window.postMessage(
// {
// myTypeField: "audio-stream",
// stream: true,
// },
// "*"
// );
streamRecorder.start();
console.log("recorder is", streamRecorder.state);
}
});
});
return {
input,
inputXoffset,
renderAudio,
reactiveWidth
};
},
});
@ -86,6 +50,9 @@ div[contenteditable] {
background-color: #585858;
font-size: 14;
text-align: left;
.audioCanvas {
max-width: 190px
}
p {
max-width: 150px;
overflow-wrap: break-word;

4
src/components/UI/index.vue
View file

@ -8,6 +8,7 @@
>
<!-- <AI /> -->
<MessageRenderer />
<AudioInput />
<InputItem />
<!-- <TitleBar /> -->
</svg>
@ -19,11 +20,12 @@ import MessageRenderer from "./UIDetails/messageRenderer.vue";
import InputItem from "./UIDetails/inputItem.vue";
import TitleBar from "./UIDetails/titleBar.vue";
import Messages from "./UIDetails/messages";
import AudioInput from './UIDetails/audioInput.vue';
import { Mitt } from "@/types/mitt/index";
import { defineComponent, ref, inject } from "vue";
export default defineComponent({
name: "UI",
components: { MessageRenderer, InputItem },
components: { MessageRenderer, InputItem, AudioInput },
setup() {
const messages = Messages;
const count = ref(0);

59
src/composables/inputItem/useInputRenderer.ts
View file

@ -1,20 +1,48 @@
import {ref, computed, inject, watch, onUnmounted} from "vue";
import {ref, computed, inject, watch, onUnmounted, onMounted} from "vue";
import useInputAnims from "./useInputAnims";
import useKeyDownHandler from "@/composables/keyDownHandler/useKeyDownHandler";
import useMediaStream from "@/composables/mediaStream/useMediaStream";
import useStreamRecorder from "@/composables/streamRecorder/useStreamRecorder";
export default function useInputRenderer() {
const { keyDownHandler, input } = useKeyDownHandler();
let stream: any;
const renderAudio = ref(false);
const {handleStream} = useStreamRecorder(renderAudio)
onMounted(async () => {
stream = await useMediaStream()
})
function keyupHandler(e: any) {
if (e.key === " " && renderAudio.value) {
console.log("stop recording");
renderAudio.value = false;
// window.postMessage(
// {
// myTypeField: "audio-stream",
// stream: false,
// },
// "*"
// );
input.value = "";
reactiveWidth.value = 0;
}
}
// add window event listener
window.addEventListener("keydown", keyDownHandler);
window.addEventListener('keyup', keyupHandler);
// remove Event Listener on component unMount
onUnmounted(() => {
window.removeEventListener("keydown", keyDownHandler);
window.removeEventListener("keyup", keyupHandler);
});
const emitter: any = inject("mitt");
const {animateSend, verticalShiftInput, inputAppear} = useInputAnims();
const inputHeight = ref(42);
const inputWidth = ref(0);
const renderInput = async() => {
@ -36,7 +64,28 @@ export default function useInputRenderer() {
}
}
watch(input, () => {
const reactiveWidth = ref(0);
watch(input, async (input, prevInput) => {
if (prevInput !== "" || prevInput.length > 0) {
renderInput();
return;
}
if (input === " ") {
console.log("record");
renderAudio.value = true;
const canvas = await document.getElementsByClassName('audioCanvas');
const test = canvas[0] as HTMLCanvasElement;
handleStream(stream.stream, test, reactiveWidth);
// window.postMessage(
// {
// myTypeField: "audio-stream",
// stream: true,
// },
// "*"
// );
//console.log("recorder is", streamRecorder.state);
}
renderInput();
});
@ -52,7 +101,9 @@ export default function useInputRenderer() {
return {
inputXoffset,
input
input,
renderAudio,
reactiveWidth
}
}

32
src/composables/inputItem/useTextInputRender.ts Normal file
View file

@ -0,0 +1,32 @@
import {ref} from 'vue';
import useInputAnims from "./useInputAnims";
export default function useTextRenderInput() {
const inputHeight = ref(42);
const inputWidth = ref(0);
const { verticalShiftInput, inputAppear } = useInputAnims();
const renderTextInput = async() => {
const inputDivs = await document.getElementsByClassName("inputBubble");
const height = inputDivs[0].getBoundingClientRect().height;
inputWidth.value = inputDivs[0].getBoundingClientRect().width;
const foreignObjectDiv = document.getElementsByClassName(
"inputContainer"
);
if (height !== inputHeight.value) {
inputHeight.value = height;
}
const foreignEl = foreignObjectDiv[0] as HTMLElement;
if (inputHeight.value === 0) {
inputAppear(foreignEl)
} else if (inputHeight.value > 36) {
const yTrans = -inputHeight.value + 30;
verticalShiftInput(foreignEl, yTrans);
}
}
return {
renderTextInput
}
}

6
src/composables/mediaStream/useMediaStream.ts Normal file
View file

@ -0,0 +1,6 @@
export default async function useMediaStream() {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
return {
stream
}
}

106
src/composables/streamRecorder/useStreamRecorder.ts
View file

@ -1,9 +1,15 @@
import { watch } from "vue";
import { watch, ref, inject } from "vue";
import useAudioRenderer from "@/composables/audioRenderer/useAudioRenderer";
export default function useStreamRecorder(record: any) {
import AnimeFunc from "@/types/animejs/index";
export default function useStreamRecorder(render: any) {
let streamRecorder: MediaRecorder;
const chunks: Blob[] = [];
// imports animejs safely
let anime: AnimeFunc;
const animeInject: AnimeFunc | undefined = inject("animejs");
if (animeInject) anime = animeInject;
window.AudioContext = window.AudioContext;
let audio_context = new AudioContext();
const analyzer = audio_context.createAnalyser();
@ -11,91 +17,67 @@ export default function useStreamRecorder(record: any) {
var bufferLengthAlt = analyzer.frequencyBinCount;
var dataArrayAlt = new Uint8Array(bufferLengthAlt);
let source;
let source: any;
const initRecorder = async () => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
handleStream(stream);
streamRecorder = new MediaRecorder(stream);
streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
streamRecorder.onstop = (e: any) => {
// send audio to BE
const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' });
// do something with blob
}
return {
streamRecorder
}
}
interface Window {
AudioContext: any;
webkitAudioContext: any;
}
interface Event {
inputBuffer: AudioBuffer;
}
let bufferLen = 4096;
let numChannels = 1;
function visualize() {
function visualize(canvas:any, canvasCtx: any, w: any) {
const WIDTH = canvas.width;
const HEIGHT = canvas.height;
function drawAlt() {
if (!render.value) {
return;
}
const drawVisual = requestAnimationFrame(drawAlt);
analyzer.getByteFrequencyData(dataArrayAlt);
for (var i = 0; i < bufferLengthAlt; i++) {
console.log(dataArrayAlt[i])
// canvasCtx.fillStyle = 'rgb(0, 0, 0)';
// canvasCtx.fillRect(0, 0, WIDTH, HEIGHT);
var barWidth = (WIDTH / bufferLengthAlt) * 5;
var barHeight;
var x = 0;
// max-width
if (canvas.width === 190) {
return;
}
// increase width and center as time passes
canvas.width++;
const center = 175 - canvas.width / 2;
w.value= center;
// fill bubble with frequency bar
for(var i = 0; i < bufferLengthAlt; i++) {
barHeight = dataArrayAlt[i];
canvasCtx.fillStyle = 'rgb(' + (barHeight+100) + ',50,50)';
canvasCtx.fillRect(x,HEIGHT-barHeight/2,barWidth,barHeight/2);
x += barWidth + 1;
}
}
drawAlt();
}
function handleStream(stream: MediaStream) {
source = audio_context.createMediaStreamSource(stream);
function handleStream(s: MediaStream, c: HTMLCanvasElement, w: any) {
source = audio_context.createMediaStreamSource(s);
source.connect(analyzer);
visualize();
//
// const context = source.context;
// const sampleRate = context.sampleRate;
// const node = context.createScriptProcessor.call(
// context,
// bufferLen,
// numChannels,
// numChannels
// );
// source.connect(node);
// node.onaudioprocess = (e: Event) => {
// const inputBuffer = new Float32Array(bufferLen);
// e.inputBuffer.copyFromChannel(inputBuffer, 0);
// // const int16 = new Int16Array(inputBuffer.buffer);
// // window.postMessage(
// // {
// // myTypeField: "handle-audio-stream",
// // audioData: int16
// // },
// // "*"
// // );
// };
// watch(record, (rec) => {
// if (rec) {
// node.connect(context.destination);
// } else {
// node.disconnect(context.destination);
// }
// })
const canvasCtx = c.getContext('2d');
visualize(c, canvasCtx, w);
}
return {
initRecorder
initRecorder,
handleStream
}
}