From: Andrew Gundersen Date: Tue, 24 Aug 2021 20:04:21 +0000 (-0500) Subject: WIP:playback queue and tray icon states X-Git-Tag: v2.0.1~3 X-Git-Url: https://andrewgundersen.net/repos?a=commitdiff_plain;h=a4d6a883b0f43cacfa92756dba204d338b575767;p=mime-chat WIP:playback queue and tray icon states --- diff --git a/build/icon_128x128.png b/build/icon_128x128.png deleted file mode 100644 index bc835c0..0000000 Binary files a/build/icon_128x128.png and /dev/null differ diff --git a/build/icon_128x128@2x.png b/build/icon_128x128@2x.png deleted file mode 100644 index 0e47443..0000000 Binary files a/build/icon_128x128@2x.png and /dev/null differ diff --git a/build/icon_16x16.png b/build/icon_16x16.png deleted file mode 100644 index b925c06..0000000 Binary files a/build/icon_16x16.png and /dev/null differ diff --git a/build/icon_16x16@2x.png b/build/icon_16x16@2x.png deleted file mode 100644 index 7bf880c..0000000 Binary files a/build/icon_16x16@2x.png and /dev/null differ diff --git a/build/icon_256x256.png b/build/icon_256x256.png deleted file mode 100644 index 0e47443..0000000 Binary files a/build/icon_256x256.png and /dev/null differ diff --git a/build/icon_256x256@2x.png b/build/icon_256x256@2x.png deleted file mode 100644 index 3126e1d..0000000 Binary files a/build/icon_256x256@2x.png and /dev/null differ diff --git a/build/icon_32x32.png b/build/icon_32x32.png deleted file mode 100644 index 7bf880c..0000000 Binary files a/build/icon_32x32.png and /dev/null differ diff --git a/build/icon_32x32@2x.png b/build/icon_32x32@2x.png deleted file mode 100644 index 2a9f925..0000000 Binary files a/build/icon_32x32@2x.png and /dev/null differ diff --git a/build/icon_512x512.png b/build/icon_512x512.png deleted file mode 100644 index 3126e1d..0000000 Binary files a/build/icon_512x512.png and /dev/null differ diff --git a/build/icon_512x512@2x.png b/build/icon_512x512@2x.png deleted file mode 100644 index 7568cb1..0000000 Binary files a/build/icon_512x512@2x.png and /dev/null differ diff --git a/build/tray/000_16x16@2x.png b/build/tray/000_16x16@2x.png new file mode 100644 index 0000000..6d61d7f Binary files /dev/null and b/build/tray/000_16x16@2x.png differ diff --git a/build/tray/001_16x16@2x.png b/build/tray/001_16x16@2x.png new file mode 100644 index 0000000..b7f0e2c Binary files /dev/null and b/build/tray/001_16x16@2x.png differ diff --git a/build/tray/010_16x16@2x.png b/build/tray/010_16x16@2x.png new file mode 100644 index 0000000..6b97193 Binary files /dev/null and b/build/tray/010_16x16@2x.png differ diff --git a/build/tray/011_16x16@2x.png b/build/tray/011_16x16@2x.png new file mode 100644 index 0000000..befcd5f Binary files /dev/null and b/build/tray/011_16x16@2x.png differ diff --git a/build/tray/100_16x16@2x.png b/build/tray/100_16x16@2x.png new file mode 100644 index 0000000..2d26144 Binary files /dev/null and b/build/tray/100_16x16@2x.png differ diff --git a/build/tray/101_16x16@2x.png b/build/tray/101_16x16@2x.png new file mode 100644 index 0000000..91a15be Binary files /dev/null and b/build/tray/101_16x16@2x.png differ diff --git a/build/tray/110_16x16@2x.png b/build/tray/110_16x16@2x.png new file mode 100644 index 0000000..fd073dd Binary files /dev/null and b/build/tray/110_16x16@2x.png differ diff --git a/build/tray/111_16x16@2x.png b/build/tray/111_16x16@2x.png new file mode 100644 index 0000000..2bc77fd Binary files /dev/null and b/build/tray/111_16x16@2x.png differ diff --git a/package.json b/package.json index e7c7cfe..2cad1d5 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,6 @@ "electron-is-dev": "^2.0.0", "electron-store": "^8.0.0", "electron-updater": "^4.3.8", - "iohook": "^0.9.3", "mitt": "^2.1.0", "update-electron-app": "^2.0.1", "vue": "^3.0.0-0", diff --git a/src/account.ts b/src/account.ts index 6ea1ac4..7324e6f 100644 --- a/src/account.ts +++ b/src/account.ts @@ -114,8 +114,6 @@ export const updateAppState = () => { ipcEmit("set-version", app.getVersion()); - updateAppUI(); - } diff --git a/src/audio.ts b/src/audio.ts index b1a6297..2dcc9cd 100644 --- a/src/audio.ts +++ b/src/audio.ts @@ -1,5 +1,8 @@ -const nodeAudio = require('@crimata/nodeaudio'); import { globalShortcut } from "electron"; +const nodeAudio = require('@crimata/nodeaudio'); + +import { updateTray } from "@/tray"; +import { sendMessage } from "@/session"; import { backgroundMitt, ipcEmit } from '@/composables/useEmitter'; let inputDevice: number; @@ -7,68 +10,100 @@ let outputDevice: number; let setStreamsId: ReturnType; +let playbackEl; +let writeToOBuffer = false; let recording = false; const chunks: Int16Array[] = []; -// backgroundMitt.on("data", (int16Arr: Int16Array) => { -// if (recording) { -// chunks.push(int16Arr); -// ipcEmit("recording", int16Arr); // for recording animation -// } -// }); - -// const setStreams = () => { - -// const defaultInput = nodeAudio.GetDefaultInputDevice(); -// const defaultOutput = nodeAudio.GetDefaultOutputDevice(); - -// if (inputDevice !== defaultInput) { -// inputDevice = defaultInput; -// nodeAudio.CloseInputStream(inputDevice); -// nodeAudio.OpenInputStream(inputDevice); -// } - -// if (outputDevice !== defaultOutput) { -// outputDevice = defaultOutput; -// nodeAudio.CloseOutputStream(outputDevice); -// nodeAudio.OpenOutputStream(outputDevice); -// } -// } - -// const startRecording = () => { -// recording = true; -// } - -// const stopRecording = () => { -// recording = false; -// const audio: Int16Array = nodeAudio.MergeChunks(chunks); -// backgroundMitt.emit("audio", audio); // emit audio event -// chunks.length = 0; -// } +backgroundMitt.on("data", (int16Arr: Int16Array) => { + if (recording) { + chunks.push(int16Arr); + ipcEmit("recording", int16Arr); + } +}); -export const initAudio = () => { - // nodeAudio.Initialize(backgroundMitt.emit.bind(backgroundMitt)); - // setStreamsId = setInterval(setStreams, 2000); // sense device +backgroundMitt.on("playback", (uid: string, aplitude: number) => { + ipcEmit("animate-playback", { uid: uid, aplitude: aplitude }); +}); - const ret = globalShortcut.register('CommandOrControl+R', () => { - console.log('Toggle record...'); - }); +backgroundMitt.on("write", (status: boolean) => { + writeToOBuffer = status; +}); + +const setStreams = () => { - if (!ret) { - console.log('registration failed'); + const defaultInput = nodeAudio.core.GetDefaultInputDevice(); + const defaultOutput = nodeAudio.core.GetDefaultOutputDevice(); + + if (inputDevice !== defaultInput) { + inputDevice = defaultInput; + nodeAudio.core.CloseInputStream(inputDevice); + nodeAudio.core.OpenInputStream(inputDevice); + } + + if (outputDevice !== defaultOutput) { + outputDevice = defaultOutput; + nodeAudio.core.CloseOutputStream(outputDevice); + nodeAudio.core.OpenOutputStream(outputDevice); } +} - // Check whether a shortcut is registered. - console.log(globalShortcut.isRegistered('CommandOrControl+R')); +const startRecording = () => { + /* If playback is happening, we kill it */ + if (playback) nodeAudio.core.ClearOutputBuffer(); + + recording = true; + updateTray("recording", recording); } -// export const playback = (audio: Int16Array) => { -// console.log("Playing audio"); -// // nodeAudio.WriteToOutputStream(audio); -// } +const stopRecording = () => { + recording = false; + updateTray("recording", recording); + + /* Send the data to the platform right away */ + sendMessage({ + text: false, + audio: nodeAudio.utils.mergeChunks(chunks) + } as Raw); + + chunks.length = 0; +} -// export const terminateAudio = () => { -// console.log("Terminating audio"); -// // clearInterval(setStreamsId); -// } \ No newline at end of file +export const initAudio = () => { + nodeAudio.core.Initialize(backgroundMitt.emit.bind(backgroundMitt)); + setStreamsId = setInterval(setStreams, 2000); // sense device + + /* Toggle recording switch with global shortcut */ + globalShortcut.register('CommandOrControl+R', () => { + if (recording) { + stopRecording(); + } else { + startRecording(); + }; + }); + +} + +export const playback = (audio: Int16Array, source: string) => { + + /* If existing playback is happening, we kill it */ + if (playback) nodeAudio.core.ClearOutputBuffer(); + + const playbackEl = uid; + + nodeAudio.core.WriteToOutputStream(audio.buffer); +} + +export const terminateAudio = () => { + clearInterval(setStreamsId); + nodeAudio.core.Terminate(); +} + +export const getStreamState = () => { + let busy = false; + if (playback || recording) { + busy = true; + } + return busy; +} diff --git a/src/init.ts b/src/init.ts index 3dddc73..5e3cbad 100644 --- a/src/init.ts +++ b/src/init.ts @@ -10,6 +10,7 @@ import createWindow from "./window"; import main from "./main"; import { backgroundMitt } from '@/composables/useEmitter'; import { setWindowOpen, getWindowOpen } from './store'; +import { terminateAudio } from './audio'; console.log('Starting Crimata electron app.'); @@ -31,6 +32,7 @@ app.on("ready", async () => { }); app.on("will-quit", () => { + terminateAudio(); globalShortcut.unregisterAll(); }); diff --git a/src/ipc/listeners.ts b/src/ipc/listeners.ts index ef0f965..5f63838 100644 --- a/src/ipc/listeners.ts +++ b/src/ipc/listeners.ts @@ -1,13 +1,15 @@ -import { IpcListener } from "@/composables/useIpcMain" -import { sendMessage } from '@/session'; -import { updateAppState } from "@/account"; +import { playback } from "@/audio"; import { onNavBar } from "@/window"; +import { sendMessage } from '@/session'; import { setWindowFocus } from '@/store'; +import { updateAppState } from "@/account"; +import { IpcListener } from "@/composables/useIpcMain" const CLIENT_MESSAGE_CHANNEL = "post-session-send" const APP_MOUNT_CHANNEL = "post-app-mount"; const NAV_BAR_CHANNEL = "post-nav-bar"; const POST_WINDOW_FOCUS = 'post-window-focus'; +const PLAYBACK_CHANNEL = "post-playback;" export const messageListener = new IpcListener({ channel: CLIENT_MESSAGE_CHANNEL, @@ -28,3 +30,8 @@ export const windowFocusListener = new IpcListener({ channel: POST_WINDOW_FOCUS, listenerCallback: ({ isFocused }) => setWindowFocus(isFocused) }); + +export const playbackListener = new IpcListener({ + channel: PLAYBACK_CHANNEL, + listenerCallback: playback +}); diff --git a/src/main.ts b/src/main.ts index 2d9e5fe..8ae759b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,18 +1,7 @@ -/** - * Where the background logic really begins, gets called by app.onReady(). - * - * Handles authentication. If profile is set, we launch a session, which consis - * of opening a connection with the platform, initializing the audio streams. - * - * The session is primarily an interface between the frontend and the platform, - * relaying messages from one to the other. - * - */ - +import { initTray } from '@/tray'; +import createWindow from "@/window"; import initIpcMain from "@/ipc/index"; -import { accountAuth } from "./account"; -import createWindow from "./window"; -import { initTray } from './tray'; +import { accountAuth } from "@/account"; export default async function main() { diff --git a/src/render/audioVisualizer.js b/src/render/audioVisualizer.js new file mode 100644 index 0000000..8f1fe13 --- /dev/null +++ b/src/render/audioVisualizer.js @@ -0,0 +1,81 @@ +class AudioVisualizer { + constructor( audioContext, processFrame, processError ) { + this.audioContext = audioContext; + this.processFrame = processFrame; + this.connectStream = this.connectStream.bind( this ); + navigator.mediaDevices.getUserMedia( { audio: true, video: false } ) + .then( this.connectStream ) + .catch( ( error ) => { + if ( processError ) { + processError( error ); + } + } ); + } + + connectStream( stream ) { + this.analyser = this.audioContext.createAnalyser(); + const source = this.audioContext.createMediaStreamSource( stream ); + source.connect( this.analyser ); + this.analyser.smoothingTimeConstant = 0.5; + this.analyser.fftSize = 32; + + this.initRenderLoop( this.analyser ); + } + + initRenderLoop() { + const frequencyData = new Uint8Array( this.analyser.frequencyBinCount ); + const processFrame = this.processFrame || ( () => {} ); + + const renderFrame = () => { + this.analyser.getByteFrequencyData( frequencyData ); + processFrame( frequencyData ); + + requestAnimationFrame( renderFrame ); + }; + requestAnimationFrame( renderFrame ); + } +} + +const visualMainElement = document.querySelector( 'main' ); +const visualValueCount = 16; +let visualElements; +const createDOMElements = () => { + let i; + for ( i = 0; i < visualValueCount; ++i ) { + const elm = document.createElement( 'div' ); + visualMainElement.appendChild( elm ); + } + + visualElements = document.querySelectorAll( 'main div' ); +}; +createDOMElements(); + +const init = () => { + // Creating initial DOM elements + const audioContext = new AudioContext(); + const initDOM = () => { + visualMainElement.innerHTML = ''; + createDOMElements(); + }; + initDOM(); + + // Swapping values around for a better visual effect + const dataMap = { 0: 15, 1: 10, 2: 8, 3: 9, 4: 6, 5: 5, 6: 2, 7: 1, 8: 0, 9: 4, 10: 3, 11: 7, 12: 11, 13: 12, 14: 13, 15: 14 }; + const processFrame = ( data ) => { + const values = Object.values( data ); + let i; + for ( i = 0; i < visualValueCount; ++i ) { + const value = values[ dataMap[ i ] ] / 255; + const elmStyles = visualElements[ i ].style; + elmStyles.transform = `scaleY( ${ value } )`; + elmStyles.opacity = Math.max( .25, value ); + } + }; + + const processError = () => { + visualMainElement.classList.add( 'error' ); + visualMainElement.innerText = 'Please allow access to your microphone in order to see this demo.\nNothing bad is going to happen... hopefully :P'; + } + + const a = new AudioVisualizer( audioContext, processFrame, processError ); +}; \ No newline at end of file diff --git a/src/render/components/context.vue b/src/render/components/context.vue index 1092eb5..d351462 100644 --- a/src/render/components/context.vue +++ b/src/render/components/context.vue @@ -2,8 +2,8 @@ -
- +
+
{{ context.img }}
diff --git a/src/render/components/controllers/helpers.ts b/src/render/components/controllers/helpers.ts index 2fb1161..97480f3 100644 --- a/src/render/components/controllers/helpers.ts +++ b/src/render/components/controllers/helpers.ts @@ -85,6 +85,14 @@ export function animateAudioInput () { } +/* TODO: animation function that takes in a message ID, and data about the + * playedback audio. We target the element with the UID and animate it according + * to the data. - still very hypothetical... + */ +export function animatePlayback (uid: string, meta: number) { + console.log("animating playback!") +} + // Given index and length of content, return message child status. export function calcChild (index: number, len: number) { if (len === 1) { diff --git a/src/render/components/controllers/inputItem.control.audio.ts b/src/render/components/controllers/inputItem.control.audio.ts deleted file mode 100644 index 9852e53..0000000 --- a/src/render/components/controllers/inputItem.control.audio.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { onMounted, onUnmounted, ref, Ref } from "vue"; -import { postMessage } from "@/render/ipc"; -import { animateAudioInput } from "./helpers"; -import { invokeReturnAudio, postAudioChunk } from "@/render/ipc"; - -export default function useAudioInputController (typing: Ref) { - - const recording = ref(false); - let mediaRecorder: MediaRecorder; - - const { show, hide } = animateAudioInput(); - - // initialize audio - const conf = {audio: true, video: false} - navigator.mediaDevices.getUserMedia(conf).then((stream: MediaStream) => { - - const options = {mimeType: 'audio/webm'}; - mediaRecorder = new MediaRecorder(stream, options); - - // post any mew audio to backend - mediaRecorder.addEventListener('dataavailable', (e: BlobEvent) => { - e.data.arrayBuffer().then((buff: ArrayBuffer) => { - postAudioChunk(buff); - }); - }); - - // get audio and post new message to backend - mediaRecorder.addEventListener('stop', (_e: Event) => { - invokeReturnAudio().then((audio: ArrayBuffer[] | Error) => { - console.log(audio); - // postMessage(newMessage({audio: audio})); - }); - }); - - }); - - // start recording on space bar - const record = () => { - console.log("INPT:Capturing audio...") - mediaRecorder.start(); - recording.value = true; - show() - } - - // stop recording and send on release - const stop = () => { - console.log("INPT:Stopping record.") - mediaRecorder.stop(); - recording.value = false; - hide(); - } - - const onKeyDown = (e: KeyboardEvent) => { - if (e.keyCode == 32 && !typing.value) record(); - } - - const onKeyUp = (e: KeyboardEvent) => { - if (e.keyCode == 32 && recording.value) stop(); - } - - //----------------------------------------------------------- - - onMounted(() => { - window.addEventListener("keydown", onKeyDown); - window.addEventListener("keyup", onKeyUp); - }); - - onUnmounted(() => { - window.removeEventListener("keydown", onKeyDown); - window.removeEventListener("keyup", onKeyUp); - }) - - return { - recording - } - -} diff --git a/src/render/components/inputItem.vue b/src/render/components/inputItem.vue index 5b9c403..7efa6a5 100644 --- a/src/render/components/inputItem.vue +++ b/src/render/components/inputItem.vue @@ -19,21 +19,19 @@ type="text" /> - - -