html bubbles, more simplifications, new session protocol

This commit is contained in:
Andrew Gundersen 2022-01-08 16:53:45 -06:00
commit d3f99d6133
21 changed files with 340 additions and 169 deletions

View file

@ -5,9 +5,8 @@ const { postLogin } = require("./api");
const { ipcEmit } = require("./utils/emitter");
const { launchSession, endSession } = require("./session");
store.set("account", "smith12@gmail.com");
// store.set("account", "adgundersen@gmail.com");
// store.delete("account");
// store.set("account", "smith12@gmail.com");
store.set("account", "adgundersen@gmail.com");
async function login(_e, email, password)
{

View file

@ -1,9 +1,9 @@
const nodeAudio = require("@crimata/nodeaudio");
const { encode, decode } = require("base64-arraybuffer");
const { globalShortcut, ipcMain } = require("electron");
const { sendMessage } = require("./io");
const { updateTray } = require("./tray");
const { encode, decode } = require("./codec");
const { backgroundMitt, ipcEmit } = require("./utils/emitter");
let inputDevice;
@ -67,8 +67,9 @@ function stopRecording()
setRecordingStatus(false);
sendMessage({
text: false,
audio: encode(nodeAudio.utils.mergeChunks(chunks).buffer)
category: "audio/L16",
text: null,
blob: encode(nodeAudio.utils.mergeChunks(chunks).buffer)
});
chunks.length = 0;
@ -129,3 +130,8 @@ exports.initAudio = initAudio;
exports.terminateAudio = terminateAudio;
exports.playback = playback;
exports.streamState = streamState;
// setWriteId = setTimeout(() => {
// setPlaybackStatus(false);
// }, 500);

13
src/codec.js Normal file
View file

@ -0,0 +1,13 @@
const { ipcMain } = require("electron");
const { encode, decode } = require("base64-arraybuffer");
ipcMain.handle("encode", (_e, arrayBuffer) => {
return encode(arrayBuffer);
});
ipcMain.handle("decode", (_e, b64string) => {
return decode(b64string);
});
exports.encode = encode;
exports.decode = decode;

View file

@ -1,7 +1,7 @@
const WebSocket = require("ws");
const { ipcMain } = require("electron");
const config = require("./config");
const { ipcMain } = require("electron");
const { updateTray } = require("./tray");
const { backgroundMitt, ipcEmit } = require("./utils/emitter");
@ -22,7 +22,7 @@ function connectToPlatform(account)
.on("error", () => {}) /** keep silent on error */
.on("message", (payload) => backgroundMitt.emit("ws-message", JSON.parse(payload)))
.on("message", (payload) => backgroundMitt.emit("message", JSON.parse(payload)))
.on("close", (code, reason) => {
setConnectionStatus(1);
@ -55,4 +55,26 @@ ipcMain.on("message", (_e, message) => sendMessage(message));
exports.connectToPlatform = connectToPlatform;
exports.sendMessage = sendMessage;
exports.disconnectFromPlatform = disconnectFromPlatform;
exports.disconnectFromPlatform = disconnectFromPlatform;
/* ------------- Test Messages------------- */
setTimeout(() => {
// sendMessage({
// category: "text/plain",
// text: "show contacts",
// blob: null
// })
sendMessage({
category: "text/plain",
text: "add contact John Smith, john.smith97@gmail.com",
blob: null
})
}, 2000);

View file

@ -20,7 +20,7 @@ app.on("ready", () =>
initAccount();
});
app.on("will-quit", terminateAudio);
// app.on("will-quit", terminateAudio);
// When user clicks app icon (re-open)
app.on("activate", createWin);

52
src/preview.html Normal file
View file

@ -0,0 +1,52 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<style>
html, body {
margin: 0;
padding: 0;
font-family: "SF Pro Text";
font-size: 14px;
}
#titlebar {
position: fixed;
width: 100vw;
height: 54px;
opacity: 0;
-webkit-app-region: drag;
border: none;
outline: none;
z-index: 1;
}
.building {
width: 250px;
height: 150px;
}
.email {
padding: 15px;
}
.email-subject {
font-size: 16px;
margin-top: 10px;
}
</style>
</head>
<body>
<button id="titlebar"></button>
<div id="content"></div>
<script>
window.mainApi.on("preview-init", (content) => {
document.getElementById("content").innerHTML = content;
});
</script>
</body>
</html>

View file

@ -1,10 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="30" viewBox="0 0 30 30">
<g id="Group_604" data-name="Group 604" transform="translate(-366 -464)">
<circle id="Ellipse_69" data-name="Ellipse 69" cx="15" cy="15" r="15" transform="translate(366 464)" fill="#fff"/>
<g id="Group_603" data-name="Group 603">
<circle id="Ellipse_50" data-name="Ellipse 50" cx="3.116" cy="3.116" r="3.116" transform="translate(374 479.252)" fill="#383838"/>
<circle id="Ellipse_51" data-name="Ellipse 51" cx="3.116" cy="3.116" r="3.116" transform="translate(381.789 479.252)" fill="#383838"/>
<circle id="Ellipse_52" data-name="Ellipse 52" cx="3.116" cy="3.116" r="3.116" transform="translate(377.889 472.517)" fill="#383838"/>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 766 B

View file

@ -1,11 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="12" viewBox="0 0 16 12">
<g id="Group_618" data-name="Group 618" transform="translate(-777 -3651)">
<g id="Rectangle_1621" data-name="Rectangle 1621" transform="translate(777 3651)" fill="none" stroke="#58c4fd" stroke-width="2.3">
<rect width="16" height="12" rx="4" stroke="none"/>
<rect x="1.15" y="1.15" width="13.7" height="9.7" rx="2.85" fill="none"/>
</g>
<g id="Group_617" data-name="Group 617">
<path id="Polygon_6" data-name="Polygon 6" d="M6.419.415a1,1,0,0,1,1.162,0l3.88,2.771A1,1,0,0,1,10.88,5H3.12a1,1,0,0,1-.581-1.814Z" transform="translate(792 3656) rotate(180)" fill="#58c4fd"/>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 710 B

View file

@ -1,6 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="13.691" viewBox="0 0 16 13.691">
<g id="Group_754" data-name="Group 754" transform="translate(-4671 -921)">
<ellipse id="Ellipse_114" data-name="Ellipse 114" cx="8" cy="6" rx="8" ry="6" transform="translate(4671 921)" fill="#58fd84"/>
<path id="Path_193" data-name="Path 193" d="M4693.211,699.285v3.906l3.449-2.126Z" transform="translate(-19.217 231)" fill="#58fd84" stroke="#58fd84" stroke-linejoin="round" stroke-width="1"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 510 B

View file

@ -1,7 +1,6 @@
import Splash from "./splash.js";
import Messenger from "./messenger.js";
import Login from "./login.js";
import Header from "./header.js";
const account = Vue.ref(null);
@ -10,8 +9,7 @@ const App =
components: {
Splash,
Messenger,
Login,
Header
Login
},
setup()
@ -22,7 +20,7 @@ const App =
template: `
<main id="app" v-if="account !== null">
<Header />
<button id="header-titlebar" />
<Messenger v-if="account"/>
<Login v-else />

View file

@ -1,23 +1,36 @@
const Bubble =
{
props: ["modifier", "text", "child"],
props: ["category", "text", "blob", "modifier", "child"],
setup(props)
{
const getUrl = () => {
return `data:${props.category};base64,${props.blob}`;
}
Vue.onMounted(() => {
const messengerEl = document.getElementById("messenger");
if (messengerEl) {
messengerEl.dispatchEvent(new CustomEvent('adjust-scroll'));
}
document.getElementById("messenger").dispatchEvent(new CustomEvent('adjust-scroll'));
});
return { getUrl };
},
template: `
<div :class="'bubble ' + modifier + '-bubble ' + modifier +'-'+ child" v-html="text">
</div> `
<div :class="'bubble ' + modifier + '-bubble ' + modifier +'-'+ child">
<p v-if="category=='text/plain'">{{ text }}</p>
<p v-else-if="category=='audio/L16'">{{ text }}</p>
<div v-else-if="category=='text/html'" v-html="text"></div>
<img v-else-if="category.startsWith('image/')" :src="getUrl()" />
<a v-else :href="getUrl()">{{ text }}</a>
</div>`
};
export default Bubble;
// TODO: should include other styling/actions for audio bubbles

View file

@ -1,7 +1,7 @@
const Context =
{
props: ["modifier", "context", "avatar", "audio", "id"],
props: ["modifier", "context", "avatar", "id"],
setup(props) {
const playback = Vue.ref(false);
@ -9,7 +9,6 @@ const Context =
Vue.onMounted(() => {
window.mainApi.on("playback", (id, status) => {
if (id === props.id) {
console.log(id, status);
playback.value = status;
}
});
@ -25,10 +24,10 @@ const Context =
v-show="avatar"
class="context-avatar"
:class="{ audio: playback }"
>
>
<img :src="'data:image/png;base64,' + avatar" />
<img :src="avatar">
</div>
{{ context }}

View file

@ -47,7 +47,10 @@ function useText(elementId, left, typing)
if (e.key === "Enter" && el.value)
{
window.mainApi.send("message", {
text: el.value, audio: null });
category: "text/plain",
text: el.value,
blob: null
});
show.value = false;
}
});
@ -70,4 +73,4 @@ function useText(elementId, left, typing)
return { leftSide, show };
}
export default useText;
export default useText;

View file

@ -53,5 +53,3 @@ function getInitials(name)
}
export default Input;

View file

@ -8,7 +8,7 @@ const Message =
Context
},
props: ["modifier", "content", "context", "avatar", "audio", "id"],
props: ["modifier", "content", "context", "avatar", "id"],
setup(props)
{
@ -19,9 +19,11 @@ const Message =
<div :id="id" :class="'message ' + modifier + '-message'">
<Bubble
v-for="(text, index) in content"
v-for="(c, index) in content"
:category="c.category"
:text="c.text"
:blob="c.blob"
:modifier="modifier"
:text="text"
:child="calcChild(index, content.length)"
/>
@ -29,7 +31,6 @@ const Message =
:modifier="modifier"
:context="context"
:avatar="avatar"
:audio="audio"
:id="id"
/>
@ -56,4 +57,4 @@ function calcChild(index, len)
}
}
export default Message;
export default Message;

View file

@ -17,33 +17,54 @@ const Messenger = {
setup()
{
const person = Vue.ref("");
const messages = Vue.ref([]);
const messages = Vue.ref([{
modifier: "ai",
content: [{
category: "text/html",
text: `<div class="mail">
<h4>About HTML Parser</h4>
<p>Return the text of the most recently opened start tag. This should not normally be needed for structured processing, but may be useful in dealing with HTML as deployed or for re-generating input with minimal changes (whitespace between attributes can be preserved, etc.).</p>
</div>`,
blob: null
}],
context: "Email from David Spade",
avatar: null,
id: 0
}]);
const onDrop = async (e) => {
const file = e.dataTransfer.items[0].getAsFile();
const buffer = await file.arrayBuffer();
const b64String = await window.mainApi.invoke("encode", buffer);
window.mainApi.send("message", {
category: file.type,
text: file.name,
blob: b64String
});
}
Vue.onMounted(() => {
window.mainApi.on("ui-init", (ui) => {
person.value = ui.person;
messages.value = ui.messages;
});
window.mainApi.on("ui-update", (message) => {
if (message.hasOwnProperty("person"))
window.mainApi.on("message", (message) => {
if (message.person)
{
person.value = message.person;
return;
messages.value = message.messages;
}
let index = messages.value.findIndex(m => m.id===message.id);
if (index >= 0)
else if (message.name)
{
messages.value[index] = message;
}
else
person.value = message;
}
else if (message.content)
{
messages.value.push(message);
}
else
{
messages.value[messages.value.length - 1].content.push(message);
}
});
initScroll("messenger");
@ -54,7 +75,7 @@ const Messenger = {
messages.value = [];
});
return { person, messages };
return { person, messages, onDrop };
},
template: `
@ -62,28 +83,20 @@ const Messenger = {
<WS />
<Input :person="person" />
<div id="messenger">
<div id="messenger"
@drop="onDrop($event)"
@dragover.prevent
@dragenter.prevent
>
<Message
v-for="message in messages"
:modifier="message.modifier"
:content="message.content"
:context="message.context"
:avatar="message.avatar"
:audio="message.audio"
:id="message.id"
/>
</div> `
}
export default Messenger;
// {
// modifier: "ai",
// content: [
// "This is test 2\r\n"
// ],
// context: "Email from adgundersen@gmail.com",
// avatar: "./assets/mailAvatar.svg",
// audio: null,
// id: 0
// }

View file

@ -4,23 +4,15 @@ const Settings =
setup() {
const active = Vue.ref(false);
function hideSettings(e) {
if (e.key == "Escape") {
active.value = false;
window.removeEventListener("keydown", hideSettings);
}
}
window.addEventListener("keydown", (e) => {
console.log("toggle settings");
})
function showSettings() {
active.value = true;
window.addEventListener("keydown", hideSettings);
}
async function logout() {
const logout = async () => {
await window.mainApi.send("logout");
}
return { showSettings, active, logout };
return { active, logout };
},
template: `
@ -30,11 +22,7 @@ const Settings =
Logout
</button>
</div>
<button v-else class="settings-icon" @click="showSettings">
<img src="./assets/settingsIcon.svg">
</button> `
</div>`
}
export default Settings;

View file

@ -33,6 +33,53 @@ html, body {
z-index: 1;
}
.contacts {
margin: 0;
padding: 0;
border-radius: inherit;
background-color: #DBDBDB;
}
.contacts > li {
padding: 12px;
display: flex;
}
.contacts img {
border-radius: 50%;
}
.contacts .info {
margin-left: 10px;
margin-right: 10px;
}
.contacts .name {
}
.contacts .email {
font-family: SF Compact Display;
font-weight: bold;
font-size: 12px;
color: #898989;
}
.mail {
padding: 12px;
margin: 0px;
background-color: #D6F2FF;
border-radius: inherit;
}
.mail h4 {
margin-top: 0px;
}
.mail p {
margin-bottom: 0px;
}
#header-menu {
position: fixed;
margin-left: 20px;
@ -389,7 +436,6 @@ html, body {
max-width: 66vw;
font-family: "SF Pro Text";
font-size: 14px;
padding: 10px;
border-radius: 18px;
margin-bottom: 4px;
}
@ -431,15 +477,29 @@ html, body {
border-top-right-radius: 9px;
}
.bubble-notify {
position: absolute;
width: 12px;
height: 12px;
border-radius: 50%;
background-color: #58D9FF;
top: -5px;
left: -5px;
border: 2px solid #EBEBEB;
.bubble > p {
margin: 0px;
padding: 10px;
}
.bubble > div {
border-radius: inherit;
}
.bubble > img {
border-radius: inherit;
display: block;
max-width: inherit;
}
.bubble a {
border-radius: inherit;
font-family: "SF Compact Rounded";
background-color: #D9D9D9;
font-weight: bold;
padding: 12px;
color: #727272;
margin: 0px;
}
.context {
@ -472,6 +532,10 @@ html, body {
margin-right: 5px;
}
.context-avatar img {
border-radius: 50%;
}
/* ---------- shared between components ---------- */
.button {

View file

@ -4,8 +4,7 @@ const { contextBridge, ipcRenderer } = require("electron");
const validFromMainChannels = [
"set-account",
"ui-init",
"ui-update",
"message",
"ws-status",
"record",
"playback"
@ -14,9 +13,10 @@ const validFromMainChannels = [
const validToMain = [
"messenger",
"message",
"nav-bar",
"login",
"logout"
"logout",
"encode",
"decode"
]
ipcRenderer.setMaxListeners(250);

View file

@ -7,78 +7,91 @@ const { initAudio, terminateAudio, playback, streamState } = require("./audio");
let ui; /* wait to receive ui from the platform */
const updateQueue = [];
const messageQueue = [];
let processContentId;
backgroundMitt.on("ws-message", (message) =>
{
if (message.hasOwnProperty("ui"))
{
ui = message.ui;
ipcEmit("ui-init", ui);
}
else if (message.hasOwnProperty("person"))
{
ui.person = message.person;
ipcEmit("ui-update", message);
}
else
{
updateQueue.push(message);
}
});
/* Possible messages:
* - New UI (set ui)
* - New message (add message to ui.messages)
* - New content (add content to existing message)
* - Update person
*/
backgroundMitt.on("message", (message) => messageQueue.push(message));
function handleMessage()
function handleMessageQueue()
{
if (!streamState.pb && !streamState.rec) /* if user is not "busy" */
if (!streamState.pb && !streamState.rec)
{
const message = updateQueue.shift();
if (message)
const update = messageQueue.shift();
if (update)
{
let index = ui.messages.findIndex(m => m.id === message.id);
console.log("Handling UI Update:", update);
if (index >= 0)
if (update.person)
{
ui.messages[index] = message;
ui = update;
}
else if (update.name)
{
ui.person = update;
}
else if (update.content)
{
ui.messages.push(update);
}
else
{
ui.messages.push(message)
ui.messages[ui.messages.length - 1].content.push(update);
}
if (message.modifier === "ai" && !store.get("focus"))
{
new Notification({title: message.context, body: message.content[message.content.length - 1]}).show();
}
ipcEmit("message", update);
ipcEmit("ui-update", message);
if (message.audio) playback(message.audio, message.id);
// notify();
}
}
}
function notify()
{
const message = ui.messages[ui.messages.length - 1];
const content = message.content[message.content.length -1];
new Notification(
{
title: message.context,
body: content.text
}
).show();
if (content.category == "audio/L16") playback(content.blob, message.id);
}
function launchSession(account)
{
connectToPlatform(account);
initAudio();
processContentId = setInterval(handleMessage, 100);
// initAudio();
processContentId = setInterval(handleMessageQueue, 100);
}
function endSession()
{
clearInterval(processContentId);
terminateAudio();
// terminateAudio();
disconnectFromPlatform();
ui = null;
}
ipcMain.on("messenger", () => {
if (ui) {
ipcEmit("ui-init", ui);
ipcEmit("message", ui);
}
});
exports.launchSession = launchSession;
exports.endSession = endSession;
// ui.messages[ui.messages.findIndex(m => m.id === message.id)].content.push(message);

View file

@ -20,12 +20,29 @@ function updateWindowPosition()
})
}
// Called when a NavBar button is pressed.
function onNavBar(_e, cmd)
function createPreviewWin(_e, content)
{
cmd === "close" ? win.close() : win.minimize();
console.log("Creating preview window");
const preview = new BrowserWindow({
width: 500,
height: 300,
x: null,
y: null,
resizable: true,
frame: false,
webPreferences: { preload: path.join(__dirname, "/render/preload.js") }
});
preview.loadFile(path.join(__dirname, "preview.html"));
preview.webContents.on("did-finish-load", () => {
preview.webContents.send("preview-init", content);
});
}
function createWin()
{
if (win) return;
@ -75,7 +92,6 @@ function createWin()
});
}
ipcMain.on("nav-bar", onNavBar);
ipcMain.on("create-win", createWin);
ipcMain.on("preview", createPreviewWin);
exports.createWin = createWin;