Add media hint to RTC call notifications

spec/unit/matrixrtc/MatrixRTCSession.spec.ts

@@ -322,13 +322,13 @@
                 sess!.once(MatrixRTCSessionEvent.DidSendCallNotification, resolve);
             });
 
-            sess!.joinRoomSession([mockFocus], mockFocus, { notificationType: "ring" });
+            sess!.joinRoomSession([mockFocus], mockFocus, { notification: { type: "ring" } });
             await Promise.race([sentStateEvent, new Promise((resolve) => setTimeout(resolve, 5000))]);
             mockRoomState(mockRoom, [{ ...membershipTemplate, user_id: client.getUserId()! }]);
             sess!.onRTCSessionMemberUpdate();
             const ownMembershipId = sess?.memberships[0].eventId;
 
             expect(client.sendEvent).toHaveBeenCalledWith(mockRoom!.roomId, EventType.RTCNotification, {
                 "m.mentions": { user_ids: [], room: true },
                 "notification_type": "ring",
                 "m.relates_to": {
@@ -370,13 +370,76 @@
             );
         });
 
+        it("sends a notification with a hint when starting a call and emits DidSendCallNotification", async () => {
+            // Simulate a join, including the update to the room state
+            // Ensure sendEvent returns event IDs so the DidSendCallNotification payload includes them
+            sendEventMock
+                .mockResolvedValueOnce({ event_id: "legacy-evt" })
+                .mockResolvedValueOnce({ event_id: "new-evt" });
+            const didSendEventFn = jest.fn();
+            sess!.once(MatrixRTCSessionEvent.DidSendCallNotification, didSendEventFn);
+            // Create an additional listener to create a promise that resolves after the emission.
+            const didSendNotification = new Promise((resolve) => {
+                sess!.once(MatrixRTCSessionEvent.DidSendCallNotification, resolve);
+            });
+
+            sess!.joinRoomSession([mockFocus], mockFocus, { notification: { type: "ring", hint: "video" } });
+            await Promise.race([sentStateEvent, new Promise((resolve) => setTimeout(resolve, 5000))]);
+            mockRoomState(mockRoom, [{ ...membershipTemplate, user_id: client.getUserId()! }]);
+            sess!.onRTCSessionMemberUpdate();
+            const ownMembershipId = sess?.memberships[0].eventId;
+
+            expect(client.sendEvent).toHaveBeenCalledWith(mockRoom!.roomId, EventType.RTCNotification, {
+                "m.mentions": { user_ids: [], room: true },
+                "notification_type": "ring",
+                "media_hint": "video",
+                "m.relates_to": {
+                    event_id: ownMembershipId,
+                    rel_type: "m.reference",
+                },
+                "lifetime": 30000,
+                "sender_ts": expect.any(Number),
+            });
+
+            // Check if deprecated notify event is also sent.
+            expect(client.sendEvent).toHaveBeenCalledWith(mockRoom!.roomId, EventType.CallNotify, {
+                "application": "m.call",
+                "m.mentions": { user_ids: [], room: true },
+                "notify_type": "ring",
+                "call_id": "",
+            });
+            await didSendNotification;
+            // And ensure we emitted the DidSendCallNotification event with both payloads
+            expect(didSendEventFn).toHaveBeenCalledWith(
+                {
+                    "event_id": "new-evt",
+                    "lifetime": 30000,
+                    "m.mentions": { room: true, user_ids: [] },
+                    "m.relates_to": {
+                        event_id: expect.any(String),
+                        rel_type: "m.reference",
+                    },
+                    "notification_type": "ring",
+                    "media_hint": "video",
+                    "sender_ts": expect.any(Number),
+                },
+                {
+                    "application": "m.call",
+                    "call_id": "",
+                    "event_id": "legacy-evt",
+                    "m.mentions": { room: true, user_ids: [] },
+                    "notify_type": "ring",
+                },
+            );
+        });
+
         it("doesn't send a notification when joining an existing call", async () => {
             // Add another member to the call so that it is considered an existing call
             mockRoomState(mockRoom, [membershipTemplate]);
             sess!.onRTCSessionMemberUpdate();
 
             // Simulate a join, including the update to the room state
-            sess!.joinRoomSession([mockFocus], mockFocus, { notificationType: "ring" });
+            sess!.joinRoomSession([mockFocus], mockFocus, { notification: { type: "ring" } });
             await Promise.race([sentStateEvent, new Promise((resolve) => setTimeout(resolve, 5000))]);
             mockRoomState(mockRoom, [membershipTemplate, { ...membershipTemplate, user_id: client.getUserId()! }]);
             sess!.onRTCSessionMemberUpdate();
@@ -386,7 +449,7 @@
 
         it("doesn't send a notification when someone else starts the call faster than us", async () => {
             // Simulate a join, including the update to the room state
-            sess!.joinRoomSession([mockFocus], mockFocus, { notificationType: "ring" });
+            sess!.joinRoomSession([mockFocus], mockFocus, { notification: { type: "ring" } });
             await Promise.race([sentStateEvent, new Promise((resolve) => setTimeout(resolve, 5000))]);
             // But this time we want to simulate a race condition in which we receive a state event
             // from someone else, starting the call before our own state event has been sent

src/matrixrtc/CallMembership.ts

@@ -19,6 +19,7 @@
 import { type Focus } from "./focus.ts";
 import { isLivekitFocusActive } from "./LivekitFocus.ts";
 import { type SessionDescription } from "./MatrixRTCSession.ts";
+import { RTCCallIntent } from "./types.ts";
 
 /**
  * The default duration in milliseconds that a membership is considered valid for.
@@ -31,37 +32,37 @@
 
 /**
  * MSC4143 (MatrixRTC) session membership data.
- * Represents an entry in the memberships section of an m.call.member event as it is on the wire.
+ * Represents the `session` in the memberships section of an m.call.member event as it is on the wire.
  **/
 export type SessionMembershipData = {
     /**
      * The RTC application defines the type of the RTC session.
      */
-    application: string;
+    "application": string;
 
     /**
      * The id of this session.
      * A session can never span over multiple rooms so this id is to distinguish between
      * multiple session in one room. A room wide session that is not associated with a user,
      * and therefore immune to creation race conflicts, uses the `call_id: ""`.
      */
-    call_id: string;
+    "call_id": string;
 
     /**
      * The Matrix device ID of this session. A single user can have multiple sessions on different devices.
      */
-    device_id: string;
+    "device_id": string;
 
     /**
      * The focus selection system this user/membership is using.
      */
-    focus_active: Focus;
+    "focus_active": Focus;
 
     /**
      * A list of possible foci this uses knows about. One of them might be used based on the focus_active
      * selection system.
      */
-    foci_preferred: Focus[];
+    "foci_preferred": Focus[];
 
     /**
      * Optional field that contains the creation of the session. If it is undefined the creation
@@ -70,36 +71,50 @@
      *  - If it is undefined it can be interpreted as a "Join".
      *  - If it is defined it can be interpreted as an "Update"
      */
-    created_ts?: number;
+    "created_ts"?: number;
 
     // Application specific data
 
     /**
      * If the `application` = `"m.call"` this defines if it is a room or user owned call.
      * There can always be one room scroped call but multiple user owned calls (breakout sessions)
      */
-    scope?: CallScope;
+    "scope"?: CallScope;
 
     /**
      * Optionally we allow to define a delta to the `created_ts` that defines when the event is expired/invalid.
      * This should be set to multiple hours. The only reason it exist is to deal with failed delayed events.
      * (for example caused by a homeserver crashes)
      **/
-    expires?: number;
+    "expires"?: number;
+
+    /**
+     * The intent of the call from the perspective of this user. This may be an audio call, video call or
+     * something else.
+     */
+    "m.call.intent"?: RTCCallIntent;
 };
 
-const checkSessionsMembershipData = (data: any, errors: string[]): data is SessionMembershipData => {
+const checkSessionsMembershipData = (
+    data: Partial<Record<keyof SessionMembershipData, any>>,
+    errors: string[],
+): data is SessionMembershipData => {
     const prefix = "Malformed session membership event: ";
     if (typeof data.device_id !== "string") errors.push(prefix + "device_id must be string");
     if (typeof data.call_id !== "string") errors.push(prefix + "call_id must be string");
     if (typeof data.application !== "string") errors.push(prefix + "application must be a string");
     if (typeof data.focus_active?.type !== "string") errors.push(prefix + "focus_active.type must be a string");
     if (!Array.isArray(data.foci_preferred)) errors.push(prefix + "foci_preferred must be an array");
     // optional parameters
-    if (data.created_ts && typeof data.created_ts !== "number") errors.push(prefix + "created_ts must be number");
+    if (data.created_ts !== undefined && typeof data.created_ts !== "number")
+        errors.push(prefix + "created_ts must be number");
 
     // application specific data (we first need to check if they exist)
-    if (data.scope && typeof data.scope !== "string") errors.push(prefix + "scope must be string");
+    if (data.scope !== undefined && typeof data.scope !== "string") errors.push(prefix + "scope must be string");
+
+    if (data["m.call.intent"] !== undefined && typeof data["m.call.intent"] !== "string")
+        errors.push(prefix + "m.call.intent must be a string");
+
     return errors.length === 0;
 };
 
@@ -142,6 +157,10 @@
         return this.membershipData.device_id;
     }
 
+    public get callIntent(): RTCCallIntent | undefined {
+        return this.membershipData["m.call.intent"];
+    }
+
     public get sessionDescription(): SessionDescription {
         return {
             application: this.membershipData.application,

src/matrixrtc/MatrixRTCSession.ts

@@ -34,6 +34,7 @@ import {
     type Status,
     type IRTCNotificationContent,
     type ICallNotifyContent,
+    type RTCCallIntent,
 } from "./types.ts";
 import { RoomKeyTransport } from "./RoomKeyTransport.ts";
 import {
@@ -92,6 +93,11 @@ export interface SessionConfig {
      * @default `undefined` (no notification)
      */
     notificationType?: RTCNotificationType;
+
+    /**
+     * Determines the kind of call this will be.
+     */
+    callIntent?: RTCCallIntent;
 }
 
 /**
@@ -614,6 +620,24 @@ export class MatrixRTCSession extends TypedEventEmitter<
         return this.memberships[0];
     }
 
+    /**
+     * Get the call intent for the current call, based on what members are advertising. If one or more
+     * members disagree on the current call intent, or nobody specifies one then `undefined` is returned.
+     *
+     * If all members that specify a call intent agree, that value is returned.
+     * @returns A call intent, or `undefined` if no consensus or not given. 
+     */
+    public getConsensusCallIntent(): RTCCallIntent|undefined {
+        const getFirstCallIntent = this.memberships.find(m => !!m.callIntent)?.callIntent;
+        if (!getFirstCallIntent) {
+            return undefined;
+        }
+        if (this.memberships.some(m => !m.callIntent || m.callIntent === getFirstCallIntent)) {
+            return getFirstCallIntent;
+        }
+        return undefined;
+    }
+
     /**
      * This method is used when the user is not yet connected to the Session but wants to know what focus
      * the users in the session are using to make a decision how it wants/should connect.
@@ -667,7 +691,11 @@ export class MatrixRTCSession extends TypedEventEmitter<
     /**
      * Sends a notification corresponding to the configured notify type.
      */
-    private sendCallNotify(parentEventId: string, notificationType: RTCNotificationType): void {
+    private sendCallNotify(
+        parentEventId: string,
+        notificationType: RTCNotificationType,
+        callIntent?: RTCCallIntent,
+    ): void {
         const sendLegacyNotificationEvent = async (): Promise<{
             response: ISendEventResponse;
             content: ICallNotifyContent;
@@ -695,6 +723,9 @@ export class MatrixRTCSession extends TypedEventEmitter<
                 "sender_ts": Date.now(),
                 "lifetime": 30_000, // 30 seconds
             };
+            if (callIntent) {
+                content["m.call.intent"] = callIntent;
+            }
             const response = await this.client.sendEvent(this.roomSubset.roomId, EventType.RTCNotification, content);
             return { response, content };
         };
@@ -757,7 +788,11 @@ export class MatrixRTCSession extends TypedEventEmitter<
                 // If we're the first member in the call, we're responsible for
                 // sending the notification event
                 if (ownMembership.eventId && this.joinConfig?.notificationType) {
-                    this.sendCallNotify(ownMembership.eventId, this.joinConfig.notificationType);
+                    this.sendCallNotify(
+                        ownMembership.eventId,
+                        this.joinConfig.notificationType,
+                        ownMembership.callIntent,
+                    );
                 } else {
                     this.logger.warn("Own membership eventId is undefined, cannot send call notification");
                 }

src/matrixrtc/MembershipManager.ts

@@ -24,9 +24,9 @@
 import { type Room } from "../models/room.ts";
 import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts";
 import { type Focus } from "./focus.ts";
-import { isMyMembership, Status } from "./types.ts";
+import { isMyMembership, RTCCallIntent, Status } from "./types.ts";
 import { isLivekitFocusActive } from "./LivekitFocus.ts";
-import { type SessionDescription, type MembershipConfig } from "./MatrixRTCSession.ts";
+import { type SessionDescription, type MembershipConfig, SessionConfig } from "./MatrixRTCSession.ts";
 import { ActionScheduler, type ActionUpdate } from "./MembershipManagerActionScheduler.ts";
 import { TypedEventEmitter } from "../models/typed-event-emitter.ts";
 import {
@@ -156,6 +156,7 @@
 {
     private activated = false;
     private logger: Logger;
+    private callIntent: RTCCallIntent | undefined;
 
     public isActivated(): boolean {
         return this.activated;
@@ -281,6 +282,10 @@
         }
     }
 
+    public updateCallIntent(callIntent: RTCCallIntent) {
+        this.callIntent = callIntent;
+    }
+
     /**
      * @throws if the client does not return user or device id.
      * @param joinConfig
@@ -289,7 +294,7 @@
      * @param getOldestMembership
      */
     public constructor(
-        private joinConfig: MembershipConfig | undefined,
+        private joinConfig: (SessionConfig & MembershipConfig) | undefined,
         private room: Pick<Room, "getLiveTimeline" | "roomId" | "getVersion">,
         private client: Pick<
             MatrixClient,
@@ -311,6 +316,7 @@
         this.deviceId = deviceId;
         this.stateKey = this.makeMembershipStateKey(userId, deviceId);
         this.state = MembershipManager.defaultState;
+        this.callIntent = joinConfig?.callIntent;
         this.scheduler = new ActionScheduler((type): Promise<ActionUpdate> => {
             if (this.oldStatus) {
                 // we put this at the beginning of the actions scheduler loop handle callback since it is a loop this
@@ -743,13 +749,14 @@
     private makeMyMembership(expires: number): SessionMembershipData {
         return {
             // TODO: use the new format for m.rtc.member events where call_id becomes session.id
-            application: this.sessionDescription.application,
-            call_id: this.sessionDescription.id,
-            scope: "m.room",
-            device_id: this.deviceId,
+            "application": this.sessionDescription.application,
+            "call_id": this.sessionDescription.id,
+            "scope": "m.room",
+            "device_id": this.deviceId,
             expires,
-            focus_active: { type: "livekit", focus_selection: "oldest_membership" },
-            foci_preferred: this.fociPreferred ?? [],
+            "focus_active": { type: "livekit", focus_selection: "oldest_membership" },
+            "foci_preferred": this.fociPreferred ?? [],
+            "m.call.intent": this.callIntent,
         };
     }
 

src/matrixrtc/types.ts

@@ -96,10 +96,19 @@ export interface ICallNotifyContent {
 }
 
 export type RTCNotificationType = "ring" | "notification";
+
+/**
+ * Represents the intention of the call from the perspective of the sending user.
+ */
+export type RTCCallIntent = "audio" | "video" | string;
 export interface IRTCNotificationContent extends RelationEvent {
     "m.mentions": IMentions;
     "decline_reason"?: string;
     "notification_type": RTCNotificationType;
+    /**
+     * The initial intent of the calling user.
+     */
+    "m.call.intent"?: RTCCallIntent;
     "sender_ts": number;
     "lifetime": number;
 }