Client implemented tools (#9)

.github/workflows/test.yaml

@@ -15,6 +15,8 @@ jobs:
           channel: 'stable'
       - name: install deps
         run: flutter pub get
+      - name: generate mocks
+        run: dart run build_runner build
       - name: format
         run: dart format lib/ test/ --set-exit-if-changed
       - name: check

.gitignore

@@ -44,3 +44,6 @@ migrate_working_dir/
 *.iws
 .idea/
 .vscode/
+
+# Generated mocks
+test/**/*.mocks.dart

CHANGELOG.md

@@ -15,4 +15,8 @@
 ## 0.0.4
 
 * Changed implementation of mute/unmute. It's now `micMuted` and `speakerMuted`
-* Added functions for toggling mute of mic (`toggleMicMuted()`) and speaker (`toggleSpeakerMuted()`)
+* Added functions for toggling mute of mic (`toggleMicMuted()`) and speaker (`toggleSpeakerMuted()`)
+
+## 0.0.5
+
+* Add client-implemented tools

example/lib/main.dart

@@ -1,4 +1,5 @@
 import 'dart:async';
+import 'dart:convert';
 
 import 'package:flutter/material.dart';
 import 'package:ultravox_client/ultravox_client.dart';
@@ -70,9 +71,26 @@ class _MyHomePageState extends State<MyHomePage> {
           UltravoxSession.create(experimentalMessages: _debug ? {"debug"} : {});
     });
     _session!.statusNotifier.addListener(_onStatusChange);
+    _session!.registerToolImplementation("getSecretMenu", _getSecretMenu);
     await _session!.joinCall(joinUrl);
   }
 
+  ClientToolResult _getSecretMenu(Object params) {
+    return ClientToolResult(json.encode({
+      "date": DateTime.now().toIso8601String(),
+      "specialItems": [
+        {
+          "name": "Banana smoothie",
+          "price": 3.99,
+        },
+        {
+          "name": "Butter pecan ice cream (one scoop)",
+          "price": 1.99,
+        }
+      ],
+    }));
+  }
+
   Future<void> _endCall() async {
     if (_session == null) {
       return;

example/pubspec.lock

@@ -475,7 +475,7 @@ packages:
       path: ".."
       relative: true
     source: path
-    version: "0.0.3"
+    version: "0.0.4"
   uuid:
     dependency: transitive
     description:

lib/src/session.dart

@@ -1,3 +1,5 @@
+import 'dart:async';
+
 import 'package:flutter/material.dart';
 import 'package:livekit_client/livekit_client.dart' as lk;
 import 'package:web_socket_channel/web_socket_channel.dart';
@@ -91,6 +93,34 @@ class Transcripts extends ChangeNotifier {
   }
 }
 
+/// The result type returned by a ClientToolImplementation.
+class ClientToolResult {
+  /// The result of the client tool.
+  ///
+  /// This is exactly the string that will be seen by the model. Often JSON.
+  final String result;
+
+  /// The type of response the tool is providing.
+  ///
+  /// Most tools simply provide information back to the model, in which case
+  /// responseType need not be set. For other tools that are instead interpreted
+  /// by the server to affect the call, responseType may be set to indicate how
+  /// the call should be altered. In this case, [result] should be JSON with
+  /// instructions for the server. The schema depends on the response type.
+  /// See https://docs.ultravox.ai/tools for more information.
+  final String? responseType;
+
+  ClientToolResult(this.result, {this.responseType});
+}
+
+/// A function that fulfills a client-implemented tool.
+///
+/// The function should take an object containing the tool's parameters (parsed
+/// from JSON) and return a [ClientToolResult] object. It may or may not be
+/// asynchronous.
+typedef ClientToolImplementation = FutureOr<ClientToolResult> Function(
+    Object data);
+
 /// Manages a single session with Ultravox.
 ///
 /// In addition to providing methods to manage a call, [UltravoxSession] exposes
@@ -180,13 +210,29 @@ class UltravoxSession {
   final lk.Room _room;
   final lk.EventsListener<lk.RoomEvent> _listener;
   late WebSocketChannel _wsChannel;
+  final _registeredTools = <String, ClientToolImplementation>{};
 
   UltravoxSession(this._room, this._experimentalMessages)
       : _listener = _room.createListener();
 
   UltravoxSession.create({Set<String>? experimentalMessages})
       : this(lk.Room(), experimentalMessages ?? {});
 
+  /// Registers a client tool implementation using the given name.
+  ///
+  /// If the call is started with a client-implemented tool, this implementation
+  /// will be invoked when the model calls the tool.
+  /// See https://docs.ultravox.ai/tools for more information.
+  void registerToolImplementation(String name, ClientToolImplementation impl) {
+    _registeredTools[name] = impl;
+  }
+
+  /// Convenience batch wrapper for [registerToolImplementation].
+  void registerToolImplementations(
+      Map<String, ClientToolImplementation> implementations) {
+    implementations.forEach(registerToolImplementation);
+  }
+
   /// Connects to a call using the given [joinUrl].
   Future<void> joinCall(String joinUrl) async {
     if (status != UltravoxSessionStatus.disconnected) {
@@ -204,7 +250,7 @@ class UltravoxSession {
     _wsChannel = WebSocketChannel.connect(url);
     await _wsChannel.ready;
     _wsChannel.stream.listen((event) async {
-      await _handleSocketMessage(event);
+      await handleSocketMessage(event);
     });
   }
 
@@ -219,8 +265,7 @@ class UltravoxSession {
       throw Exception(
           'Cannot send text while not connected. Current status: $status');
     }
-    final message = jsonEncode({'type': 'input_text_message', 'text': text});
-    _room.localParticipant?.publishData(utf8.encode(message), reliable: true);
+    await _sendData({'type': 'input_text_message', 'text': text});
   }
 
   Future<void> _disconnect() async {
@@ -235,7 +280,8 @@ class UltravoxSession {
     statusNotifier.value = UltravoxSessionStatus.disconnected;
   }
 
-  Future<void> _handleSocketMessage(dynamic event) async {
+  @visibleForTesting
+  Future<void> handleSocketMessage(dynamic event) async {
     if (event is! String) {
       throw Exception('Received unexpected message from socket');
     }
@@ -259,7 +305,7 @@ class UltravoxSession {
     await _room.startAudio();
   }
 
-  void _handleDataMessage(lk.DataReceivedEvent event) {
+  Future<void> _handleDataMessage(lk.DataReceivedEvent event) async {
     final data = jsonDecode(utf8.decode(event.data));
     switch (data['type']) {
       case 'state':
@@ -315,10 +361,53 @@ class UltravoxSession {
           }
         }
         break;
+      case 'client_tool_invocation':
+        await _invokeClientTool(data['toolName'] as String,
+            data['invocationId'] as String, data['parameters'] as Object);
       default:
         if (_experimentalMessages.isNotEmpty) {
           experimentalMessageNotifier.value = data as Map<String, dynamic>;
         }
     }
   }
+
+  Future<void> _invokeClientTool(
+      String toolName, String invocationId, Object parameters) async {
+    final tool = _registeredTools[toolName];
+    if (tool == null) {
+      await _sendData({
+        'type': 'client_tool_result',
+        'invocationId': invocationId,
+        'errorType': 'undefined',
+        'errorMessage':
+            'Client tool $toolName is not registered (Flutter client)',
+      });
+      return;
+    }
+    try {
+      final result = await tool(parameters);
+      final data = {
+        'type': 'client_tool_result',
+        'invocationId': invocationId,
+        'result': result.result,
+      };
+      if (result.responseType != null) {
+        data['responseType'] = result.responseType!;
+      }
+      await _sendData(data);
+    } catch (e) {
+      await _sendData({
+        'type': 'client_tool_result',
+        'invocationId': invocationId,
+        'errorType': 'implementation-error',
+        'errorMessage': e.toString(),
+      });
+    }
+  }
+
+  Future<void> _sendData(Object data) async {
+    final message = jsonEncode(data);
+    await _room.localParticipant
+        ?.publishData(utf8.encode(message), reliable: true);
+  }
 }

pubspec.yaml

@@ -1,6 +1,6 @@
 name: ultravox_client
 description: "Flutter client SDK for Ultravox."
-version: 0.0.4
+version: 0.0.5
 homepage: https://ultravox.ai
 repository: https://github.com/fixie-ai/ultravox-client-sdk-flutter
 topics:
@@ -30,4 +30,6 @@ dev_dependencies:
   flutter_test:
     sdk: flutter
   flutter_lints: ^4.0.0
+  mockito: ^5.4.4
+  build_runner: ^2.4.13
 

test/fake_lk.dart

@@ -0,0 +1,74 @@
+import 'dart:async';
+import 'dart:collection';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:mockito/annotations.dart';
+import 'package:livekit_client/livekit_client.dart' as lk;
+
+@GenerateNiceMocks([
+  MockSpec<lk.LocalParticipant>(),
+  MockSpec<lk.RemoteParticipant>(),
+])
+import 'fake_lk.mocks.dart';
+
+class FakeRoomEvents extends Fake implements lk.EventsListener<lk.RoomEvent> {
+  final _listeners = <FutureOr<void> Function(lk.RoomEvent)>[];
+
+  @override
+  lk.CancelListenFunc listen(FutureOr<void> Function(lk.RoomEvent) onEvent) {
+    _listeners.add(onEvent);
+    return () {};
+  }
+
+  @override // Copied from real implementation.
+  lk.CancelListenFunc on<E>(
+    FutureOr<void> Function(E) then, {
+    bool Function(E)? filter,
+  }) {
+    return listen((event) async {
+      // event must be E
+      if (event is! E) return;
+      // filter must be true (if filter is used)
+      if (filter != null && !filter(event as E)) return;
+      // cast to E
+      await then(event as E);
+    });
+  }
+
+  void emit(lk.RoomEvent event) {
+    for (final listener in _listeners) {
+      listener(event);
+    }
+  }
+}
+
+class FakeRoom extends Fake implements lk.Room {
+  final _events = FakeRoomEvents();
+
+  @override
+  lk.EventsListener<lk.RoomEvent> createListener({bool synchronized = false}) {
+    return _events;
+  }
+
+  @override
+  Future<void> connect(
+    String url,
+    String token, {
+    lk.ConnectOptions? connectOptions,
+    @Deprecated('deprecated, please use roomOptions in Room constructor')
+    lk.RoomOptions? roomOptions,
+    lk.FastConnectOptions? fastConnectOptions,
+  }) async {}
+
+  @override
+  UnmodifiableMapView<String, lk.RemoteParticipant> get remoteParticipants =>
+      UnmodifiableMapView({"remote": remoteParticipant});
+  final remoteParticipant = MockRemoteParticipant();
+
+  @override
+  final MockLocalParticipant localParticipant = MockLocalParticipant();
+
+  void emit(lk.RoomEvent event) {
+    _events.emit(event);
+  }
+}