Video in calls (#135)

#135

This PR adds basic video functionality to our voice rooms. Again I followed the Discord UX for inspiration, so all video calls start as voice-only calls that gracefully upgrade (and downgrade) when someone turns on a video or starts screen sharing.

When a video feed is detected the Room page will change to display a grid of feeds. The grid logic is very basic, that's definitely an area to improve in the future. You can open the chat part of the room with a new button on the VoiceWidget - on the desktop layout this creates a split view with video on the left and chat on the right, but on mobile it switches to chat fullscreen. I also added a little pin icon you can use to focus on a single video feed (useful for screen sharing). There is a lot of tailwind I don't understand here, but it seems to work well enough.

I moved voice.ts into a new `call` folder and moved some of its stores into `call/stores.ts` which allowed me to keep most of the video logic in `call/video.ts`. It's not a perfect encapsulation as voice.ts does subscribe to some of the hooks for the livekit calls and passes some of the signals onto `video.ts`. This could probably be broken up better but for this PR I'd rather not focus on making it perfect if that's ok. Partly for the sake of time but also because I envision another PR that renames/reorganizes things and I think a larger UX evaluation is necessary and should include real user feedback. I'm not confident tha""t the Voice Room concept as a whole will stick going forward. Maybe all rooms in a livekit enabled server should be able to host a call (like a slack huddle), maybe users want to be able to schedule calls as events, or even have them start with an ad-hoc set of participants completely outside of a NIP-29 group, etc.

Co-authored-by: mplorentz <mplorentz@noreply.gitea.coracle.social>
Reviewed-on: #135
Co-authored-by: Matt Lorentz <mplorentz@noreply.coracle.social>
Co-committed-by: Matt Lorentz <mplorentz@noreply.coracle.social>
This commit was merged in pull request #135.
This commit is contained in:
2026-04-08 17:10:20 +00:00
committed by hodlbod
parent 65ca8a7fd8
commit f4ebc4e99e
17 changed files with 943 additions and 209 deletions
+278
View File
@@ -0,0 +1,278 @@
<script lang="ts">
import cx from "classnames"
import {Track} from "livekit-client"
import {displayProfileByPubkey, loadProfile} from "@welshman/app"
import Pin from "@assets/icons/pin.svg?dataurl"
import Button from "@lib/components/Button.svelte"
import Icon from "@lib/components/Icon.svelte"
import ProfileCircle from "@app/components/ProfileCircle.svelte"
import VideoCallTile from "@app/components/VideoCallTile.svelte"
import VoiceWidget from "@app/components/VoiceWidget.svelte"
import {get} from "svelte/store"
import {
VideoCallLayout,
isDesktopLayout,
toggleVideoPrimaryTile,
videoCallLayout,
videoCallViewportSync,
ViewportSize,
videoPrimaryTileKey,
} from "@app/call/video"
import {currentVoiceSession, currentVoiceRoom, pubkeyFromLiveKitIdentity} from "@app/call/stores"
type Props = {
layout: VideoCallLayout
mobile?: boolean
url: string
h: string
class?: string
}
type VideoTileData = {
identity: string
isLocal: boolean
trackSid: string
track: Track | undefined
source: Track.Source.Camera | Track.Source.ScreenShare
}
type TileLayout = "spotlight" | "default" | "strip"
const {layout, mobile = false, url, h, class: className = ""}: Props = $props()
$effect(() => {
const currentLayout = isDesktopLayout.current ? ViewportSize.Desktop : ViewportSize.Mobile
const {previousLayout} = videoCallViewportSync
if (previousLayout === undefined) {
videoCallViewportSync.previousLayout = currentLayout
return
}
if (previousLayout === currentLayout) return
const p = get(videoCallLayout)
if (previousLayout === ViewportSize.Desktop && currentLayout === ViewportSize.Mobile) {
if (p === VideoCallLayout.Split) videoCallLayout.set(VideoCallLayout.Video)
} else if (previousLayout === ViewportSize.Mobile && currentLayout === ViewportSize.Desktop) {
if (p === VideoCallLayout.Chat) videoCallLayout.set(VideoCallLayout.Split)
}
videoCallViewportSync.previousLayout = currentLayout
})
const isViewingCurrentCallRoom = $derived(
$currentVoiceRoom?.url === url && $currentVoiceRoom?.h === h,
)
const showVideoContent = $derived(
isViewingCurrentCallRoom &&
(mobile
? layout === VideoCallLayout.Video
: layout === VideoCallLayout.Split || layout === VideoCallLayout.Video),
)
const videoTiles = $derived.by(() => {
const session = $currentVoiceSession
if (!session || $currentVoiceRoom?.url !== url || $currentVoiceRoom?.h !== h) {
return []
}
const room = session.room
const videoTiles: VideoTileData[] = []
const user = room.localParticipant
if (session.cameraOn) {
const localPub = user.getTrackPublication(Track.Source.Camera)
videoTiles.push({
identity: user.identity,
isLocal: true,
trackSid: localPub?.trackSid ?? "local-camera",
track: localPub?.track,
source: Track.Source.Camera,
})
}
if (session.screenShareOn) {
const localPub = user.getTrackPublication(Track.Source.ScreenShare)
videoTiles.push({
identity: user.identity,
isLocal: true,
trackSid: localPub?.trackSid ?? "local-screen",
track: localPub?.track,
source: Track.Source.ScreenShare,
})
}
for (const rp of room.remoteParticipants.values()) {
const camPub = rp.getTrackPublication(Track.Source.Camera)
if (camPub?.isSubscribed && camPub.track) {
videoTiles.push({
identity: rp.identity,
isLocal: false,
trackSid: camPub.trackSid,
track: camPub.track,
source: Track.Source.Camera,
})
}
const screenPub = rp.getTrackPublication(Track.Source.ScreenShare)
if (screenPub?.isSubscribed && screenPub.track) {
videoTiles.push({
identity: rp.identity,
isLocal: false,
trackSid: screenPub.trackSid,
track: screenPub.track,
source: Track.Source.ScreenShare,
})
}
}
return videoTiles
})
/** Identity + source only — LiveKit can change trackSid after publish, which broke spotlight + stale-key effect. */
const tileKey = (t: VideoTileData) => `${t.identity}\x1f${t.source}`
const primaryTile = $derived.by(() => {
const k = $videoPrimaryTileKey
if (k === undefined) return undefined
return videoTiles.find(t => tileKey(t) === k)
})
const secondaryTiles = $derived.by(() => {
const p = primaryTile
if (p === undefined) return videoTiles
const pk = tileKey(p)
return videoTiles.filter(t => tileKey(t) !== pk)
})
const useSpotlightLayout = $derived(primaryTile !== undefined)
const useMultiGrid = $derived(!useSpotlightLayout && videoTiles.length > 2)
$effect(() => {
const k = $videoPrimaryTileKey
if (k === undefined) return
if (!videoTiles.some(t => tileKey(t) === k)) {
videoPrimaryTileKey.set(undefined)
}
})
$effect(() => {
for (const t of videoTiles) {
const pk = pubkeyFromLiveKitIdentity(t.identity)
if (pk) loadProfile(pk)
}
})
const labelFor = (identity: string, source: VideoTileData["source"]) => {
const pk = pubkeyFromLiveKitIdentity(identity)
const name = pk ? displayProfileByPubkey(pk) : "Unknown"
return source === Track.Source.ScreenShare ? `${name} · screen` : name
}
const showTileGrid = $derived(videoTiles.length > 0)
const spotlightHandlerFor = (key: string) => () => {
toggleVideoPrimaryTile(key)
}
const panelChrome = $derived(
cx(
mobile &&
"flex min-h-0 w-full flex-1 flex-col gap-2 overflow-y-auto overflow-x-hidden bg-base-200 px-2 pt-4 md:hidden pb-[calc(3.5rem+var(--saib))]",
!mobile &&
"flex min-h-0 w-full min-w-0 flex-1 flex-col gap-2 overflow-hidden bg-base-200 px-2 pb-2 pt-4",
className,
),
)
</script>
{#snippet videoTile(tile: VideoTileData, layout: TileLayout)}
<div
class={cx(
"relative isolate overflow-hidden rounded-box shadow-sm",
layout === "spotlight" && "min-h-0 flex-1",
layout === "default" && "aspect-video w-full min-h-0",
layout === "strip" && "aspect-video w-44 shrink-0",
tile.source === Track.Source.ScreenShare ? "bg-black" : "bg-base-100",
)}>
{#if tile.track}
<VideoCallTile
track={tile.track}
muted={tile.isLocal}
fit={tile.source === Track.Source.ScreenShare ? "contain" : "cover"}
class="pointer-events-none absolute inset-0" />
{:else}
<div class="absolute inset-0 flex items-center justify-center">
<ProfileCircle pubkey={pubkeyFromLiveKitIdentity(tile.identity)} {url} size={14} />
</div>
{/if}
<span
class="pointer-events-none absolute bottom-1 left-1 max-w-[calc(100%-0.5rem)] truncate rounded bg-base-100/80 px-1.5 py-0.5 text-xs">
{labelFor(tile.identity, tile.source)}{tile.isLocal ? " (you)" : ""}
</span>
{#if videoTiles.length > 1}
{@const pinned = $videoPrimaryTileKey === tileKey(tile)}
<Button
data-tip={pinned ? "Exit spotlight" : "Spotlight"}
aria-pressed={pinned}
class={cx(
"absolute right-1 top-1 z-20 btn btn-xs btn-square btn-ghost",
pinned ? "btn-active bg-primary/25 text-primary" : "bg-base-100/70",
)}
onclick={spotlightHandlerFor(tileKey(tile))}>
<Icon icon={Pin} size={3} />
</Button>
{/if}
</div>
{/snippet}
{#snippet videoPanelBody()}
{#if showTileGrid}
{#if useSpotlightLayout && primaryTile}
<div class="flex min-h-0 flex-1 flex-col gap-2 overflow-hidden">
{@render videoTile(primaryTile, "spotlight")}
{#if secondaryTiles.length > 0}
<div
class="flex max-h-40 shrink-0 flex-row gap-2 overflow-x-auto overflow-y-hidden py-0.5">
{#each secondaryTiles as tile (tileKey(tile))}
{@render videoTile(tile, "strip")}
{/each}
</div>
{/if}
</div>
{:else if useMultiGrid}
<div
class="grid min-h-0 flex-1 grid-cols-1 content-start gap-2 overflow-y-auto sm:grid-cols-2">
{#each videoTiles as tile (tileKey(tile))}
{@render videoTile(tile, "default")}
{/each}
</div>
{:else}
<div class="flex min-h-0 flex-1 flex-col gap-2 overflow-y-auto">
{#each videoTiles as tile (tileKey(tile))}
{@render videoTile(tile, "default")}
{/each}
</div>
{/if}
{:else}
<div
class="flex min-h-[12rem] flex-1 flex-col items-center justify-center gap-2 rounded-box bg-base-200/50 p-4 text-center text-sm opacity-80">
<p>No camera or screen share yet.</p>
<p class="text-xs">Use the camera or screen share control to share video.</p>
</div>
{/if}
{/snippet}
{#if showVideoContent}
<div class={panelChrome}>
{#if mobile}
<div class="flex min-h-0 flex-1 flex-col gap-2">
<div class="min-h-0 flex-1 overflow-hidden">
{@render videoPanelBody()}
</div>
<div class="shrink-0 pb-2">
<VoiceWidget />
</div>
</div>
{:else}
{@render videoPanelBody()}
{/if}
</div>
{/if}
+31
View File
@@ -0,0 +1,31 @@
<script lang="ts">
import type {Track} from "livekit-client"
import cx from "classnames"
type Props = {
track: Track
muted?: boolean
fit?: "cover" | "contain"
class?: string
}
const {track, muted = true, fit = "cover", class: className = ""}: Props = $props()
let videoElement = $state<HTMLVideoElement | undefined>()
$effect(() => {
const element = videoElement
const activeTrack = track
if (!element) return
activeTrack.attach(element)
return () => {
activeTrack.detach(element)
}
})
</script>
<video
bind:this={videoElement}
class={cx("h-full w-full", fit === "contain" ? "object-contain" : "object-cover", className)}
playsinline
{muted}></video>
@@ -7,13 +7,8 @@
import ModalHeader from "@lib/components/ModalHeader.svelte"
import ModalSubtitle from "@lib/components/ModalSubtitle.svelte"
import ModalTitle from "@lib/components/ModalTitle.svelte"
import {
currentVoiceSession,
DeviceKind,
supportsAudioOutputSelection,
switchVoiceActiveDevice,
type VoiceSession,
} from "@app/voice"
import {currentVoiceSession, type VoiceSession} from "@app/call/stores"
import {DeviceKind, supportsAudioOutputSelection, switchVoiceActiveDevice} from "@app/call/voice"
import {popModal} from "@app/util/modal"
const selectValueForActiveDevice = (session: VoiceSession, kind: DeviceKind): string => {
@@ -26,8 +21,10 @@
let audioInputs = $state<MediaDeviceInfo[]>([])
let audioOutputs = $state<MediaDeviceInfo[]>([])
let videoInputs = $state<MediaDeviceInfo[]>([])
let selectedInput = $state("")
let selectedOutput = $state("")
let selectedVideo = $state("")
const loadDevices = async () => {
if (!navigator.mediaDevices?.enumerateDevices) return
@@ -35,9 +32,11 @@
const devices = await navigator.mediaDevices.enumerateDevices()
audioInputs = devices.filter(d => d.kind === "audioinput")
audioOutputs = devices.filter(d => d.kind === "audiooutput")
videoInputs = devices.filter(d => d.kind === "videoinput")
} catch {
audioInputs = []
audioOutputs = []
videoInputs = []
}
}
@@ -55,6 +54,7 @@
}
selectedInput = selectValueForActiveDevice(session, DeviceKind.AudioInput)
selectedOutput = selectValueForActiveDevice(session, DeviceKind.AudioOutput)
selectedVideo = selectValueForActiveDevice(session, DeviceKind.VideoInput)
})
const onInputChange = () => {
@@ -65,6 +65,10 @@
void switchVoiceActiveDevice(DeviceKind.AudioOutput, selectedOutput)
}
const onVideoChange = () => {
void switchVoiceActiveDevice(DeviceKind.VideoInput, selectedVideo)
}
const onDone = () => {
popModal()
}
@@ -76,8 +80,8 @@
<Modal>
<ModalBody>
<ModalHeader>
<ModalTitle>Audio settings</ModalTitle>
<ModalSubtitle>Choose microphone and speaker for this call.</ModalSubtitle>
<ModalTitle>Call settings</ModalTitle>
<ModalSubtitle>Microphone, speaker, and camera for this call.</ModalSubtitle>
</ModalHeader>
<div class="flex flex-col gap-4 pt-2">
<FieldInline>
@@ -120,6 +124,25 @@
{/snippet}
</FieldInline>
{/if}
<FieldInline>
{#snippet label()}
<p>Camera</p>
{/snippet}
{#snippet input()}
<select
class="select select-bordered w-full"
bind:value={selectedVideo}
onchange={onVideoChange}
aria-label="Camera">
<option value="">Default camera</option>
{#each videoInputs as d (d.deviceId)}
<option value={d.deviceId}>
{d.label || `Camera ${d.deviceId.slice(0, 8)}`}
</option>
{/each}
</select>
{/snippet}
</FieldInline>
</div>
</ModalBody>
<ModalFooter>
+3 -4
View File
@@ -12,14 +12,13 @@
import {makeRoomId} from "@app/core/state"
import {
VoiceState,
deriveVoiceParticipants,
cancelJoinVoiceRoom,
currentVoiceRoom,
voiceState,
isParticipantSpeaking,
participantKey,
voiceState,
type VoiceParticipant,
} from "@app/voice"
} from "@app/call/stores"
import {cancelJoinVoiceRoom, deriveVoiceParticipants} from "@app/call/voice"
interface Props {
url: string
@@ -14,7 +14,7 @@
import ModalTitle from "@lib/components/ModalTitle.svelte"
import {AbortError, TimeoutError} from "$lib/util"
import {displayRoom} from "@app/core/state"
import {joinVoiceRoom} from "@app/voice"
import {joinVoiceRoom} from "@app/call/voice"
import {popModal} from "@app/util/modal"
import {pushToast} from "@app/util/toast"
+137 -24
View File
@@ -1,15 +1,20 @@
<script lang="ts">
import {readable} from "svelte/store"
import {fly} from "svelte/transition"
import {fade, fly} from "svelte/transition"
import {goto} from "$app/navigation"
import {page} from "$app/stores"
import cx from "classnames"
import {displayRelayUrl} from "@welshman/util"
import Microphone from "@assets/icons/microphone.svg?dataurl"
import MicrophoneOff from "@assets/icons/microphone-off.svg?dataurl"
import Videocamera from "@assets/icons/videocamera.svg?dataurl"
import VideocameraRecord from "@assets/icons/videocamera-record.svg?dataurl"
import Monitor from "@assets/icons/monitor.svg?dataurl"
import PhoneRounded from "@assets/icons/phone-rounded.svg?dataurl"
import PhoneCallingRounded from "@assets/icons/phone-calling-rounded.svg?dataurl"
import ChatRound from "@assets/icons/chat-round.svg?dataurl"
import CloseCircle from "@assets/icons/close-circle.svg?dataurl"
import Settings from "@assets/icons/settings.svg?dataurl"
import {Capacitor} from "@capacitor/core"
import Icon from "@lib/components/Icon.svelte"
import Button from "@lib/components/Button.svelte"
import VoiceCallAudioSettingsDialog from "@app/components/VoiceCallAudioSettingsDialog.svelte"
@@ -23,16 +28,23 @@
type Room,
} from "@app/core/state"
import {pushModal} from "@app/util/modal"
import {notifications} from "@app/util/notifications"
import {makeRoomPath} from "@app/util/routes"
import {
VideoCallLayout,
isDesktopLayout,
toggleCamera,
toggleScreenShare,
videoCallLayout,
} from "@app/call/video"
import {
VoiceState,
currentVoiceSession,
currentVoiceRoom,
voiceState,
leaveVoiceRoom,
toggleMute,
cancelJoinVoiceRoom,
} from "@app/voice"
isLocalSpeaking,
} from "@app/call/stores"
import {cancelJoinVoiceRoom, leaveVoiceRoom, toggleMute} from "@app/call/voice"
const {relay, h} = $derived($page.params)
const url = $derived(relay ? decodeRelay(relay) : undefined)
@@ -41,6 +53,14 @@
)
const routeDisplayedRoom = $derived($displayedRoomStore)
const isViewingCurrentVoiceRoom = $derived(
$currentVoiceRoom !== undefined &&
url !== undefined &&
typeof h === "string" &&
$currentVoiceRoom.url === url &&
$currentVoiceRoom.h === h,
)
const targetRoom = $derived.by((): Room | undefined => {
if ($voiceState === VoiceState.Joining || $voiceState === VoiceState.Connected) {
return $currentVoiceRoom
@@ -66,9 +86,45 @@
pushModal(VoiceRoomJoinDialog, {url: targetRoom.url, h: targetRoom.h})
}
const openAudioSettings = () => {
const goToRoom = () => {
if (!targetRoom) return
const path = makeRoomPath(targetRoom.url, targetRoom.h)
if ($page.url.pathname !== path) {
void goto(path)
}
}
const openCallSettings = () => {
pushModal(VoiceCallAudioSettingsDialog)
}
const showChatButton = $derived($voiceState === VoiceState.Connected && isViewingCurrentVoiceRoom)
const isChatPanelActive = $derived(
showChatButton &&
(isDesktopLayout.current
? $videoCallLayout === VideoCallLayout.Split
: $videoCallLayout === VideoCallLayout.Chat),
)
const onChatToggle = () => {
if (!showChatButton) return
if (isDesktopLayout.current) {
videoCallLayout.update(p =>
p === VideoCallLayout.Split ? VideoCallLayout.Video : VideoCallLayout.Split,
)
} else {
videoCallLayout.update(p =>
p === VideoCallLayout.Video ? VideoCallLayout.Chat : VideoCallLayout.Video,
)
}
}
const chatUnread = $derived(
targetRoom !== undefined && $notifications.has(makeRoomPath(targetRoom.url, targetRoom.h)),
)
const mediaToggleClass = "center tooltip tooltip-top btn btn-sm btn-square btn-ghost"
</script>
{#if targetRoom}
@@ -76,19 +132,47 @@
in:fly={{y: 60, duration: 350}}
out:fly={{y: 60, duration: 250}}
class="flex flex-col gap-2 rounded-box bg-base-100 p-3">
<div class="flex flex-col gap-0.5">
{#if $voiceState === VoiceState.Joining}
<span class="text-sm font-semibold text-warning">Joining...</span>
{:else if $voiceState === VoiceState.Connected}
<span class="text-sm font-semibold text-success">Voice Connected</span>
{:else}
<span class="text-sm font-semibold text-neutral-content">Disconnected</span>
<div class="flex items-start justify-between gap-2">
<button
type="button"
class="min-w-0 flex-1 rounded-lg px-1 py-0.5 text-left outline-none hover:bg-base-200/60 focus-visible:ring-2 focus-visible:ring-primary focus-visible:ring-offset-2 focus-visible:ring-offset-base-100"
onclick={goToRoom}
aria-label="Open room {roomName}">
<div class="flex flex-col gap-0.5">
{#if $voiceState === VoiceState.Joining}
<span class="text-sm font-semibold text-warning">Joining...</span>
{:else if $voiceState === VoiceState.Connected}
<span class="text-sm font-semibold text-success">Voice Connected</span>
{:else}
<span class="text-sm font-semibold text-neutral-content">Disconnected</span>
{/if}
<span class="ellipsize text-xs opacity-70">
{roomName} / {spaceName}
</span>
</div>
</button>
{#if showChatButton}
<Button
data-tip="Toggle Chat"
class={cx(
mediaToggleClass,
"relative shrink-0 overflow-visible",
isChatPanelActive && "text-primary",
)}
onclick={onChatToggle}>
<span class="relative inline-flex">
<Icon icon={ChatRound} size={4} />
{#if chatUnread}
<span
transition:fade={{duration: 150}}
class="absolute -right-0.5 -top-0.5 h-2 w-2 rounded-full bg-primary ring-2 ring-base-100"
aria-hidden="true"></span>
{/if}
</span>
</Button>
{/if}
<span class="ellipsize text-xs opacity-70">
{roomName} / {spaceName}
</span>
</div>
<div class="flex items-center gap-1">
<div class="flex flex-wrap items-center gap-2">
{#if $voiceState === VoiceState.Joining}
<span class="loading loading-spinner loading-sm"></span>
<Button
@@ -100,16 +184,45 @@
{:else if $voiceState === VoiceState.Connected && $currentVoiceSession}
<Button
data-tip={$currentVoiceSession.muted ? "Unmute" : "Mute"}
class="center tooltip tooltip-top btn btn-sm btn-square {$currentVoiceSession.muted
? 'btn-error'
: 'btn-ghost'}"
class={cx(
mediaToggleClass,
"overflow-visible",
!$currentVoiceSession.muted && $isLocalSpeaking && "text-primary",
$currentVoiceSession.muted &&
"text-error ring-1 ring-error/50 ring-offset-0 ring-offset-base-100",
)}
onclick={toggleMute}>
<Icon icon={$currentVoiceSession.muted ? MicrophoneOff : Microphone} size={4} />
<span class="relative inline-flex items-center justify-center overflow-visible">
<Icon icon={Microphone} size={4} />
{#if $currentVoiceSession.muted}
<span
class="pointer-events-none absolute inset-0 flex items-center justify-center overflow-visible"
aria-hidden="true">
<span
class="h-[1.3px] w-[150%] max-w-none shrink-0 -rotate-45 rounded-full bg-current"
></span>
</span>
{/if}
</span>
</Button>
<Button
data-tip="Audio settings"
data-tip={$currentVoiceSession.cameraOn ? "Turn off camera" : "Turn on camera"}
class={cx(mediaToggleClass, $currentVoiceSession.cameraOn && "text-primary")}
onclick={toggleCamera}>
<Icon icon={$currentVoiceSession.cameraOn ? VideocameraRecord : Videocamera} size={4} />
</Button>
{#if !Capacitor.isNativePlatform()}
<Button
data-tip={$currentVoiceSession.screenShareOn ? "Stop sharing" : "Share screen"}
class={cx(mediaToggleClass, $currentVoiceSession.screenShareOn && "text-primary")}
onclick={toggleScreenShare}>
<Icon icon={Monitor} size={4} />
</Button>
{/if}
<Button
data-tip="Call settings"
class="center tooltip tooltip-top btn btn-sm btn-square btn-ghost"
onclick={openAudioSettings}>
onclick={openCallSettings}>
<Icon icon={Settings} size={4} />
</Button>
<Button