Merge pull request #702 from opentok/develop

v2.26.1
opentok · Oct 30, 2023 · 2c3d9c4 · 2c3d9c4
2 parents a9b3f06 + dd33de3
commit 2c3d9c4
Show file tree

Hide file tree

Showing 16 changed files with 8,635 additions and 3,679 deletions.
diff --git a/@types/index.d.ts b/@types/index.d.ts
@@ -38,6 +38,7 @@ declare module "opentok-react-native" {
     name: string;
     streamId: string;
     hasAudio: boolean;
+    hasCaptions: boolean;
     hasVideo: boolean;
     sessionId: string;
     connectionId: string;
@@ -57,7 +58,7 @@ declare module "opentok-react-native" {
   interface StreamPropertyChangedEvent {
     newValue: any;
     oldValue: any;
-    changedProperty: "hasAudio" | "hasVideo" | "videoDimensions";
+    changedProperty: "hasAudio" | "hasCaptions" | "hasVideo" | "videoDimensions";
     stream: Stream;
   }
 
@@ -72,6 +73,11 @@ declare module "opentok-react-native" {
     message: string;
   }
 
+  interface SubscriberCaptionEvent {
+    text: string,
+    isFinal: boolean,
+  }
+
   interface SubscriberAudioStatsEvent {
     audioBytesReceived: number,
     audioPacketsLost: number,
@@ -356,6 +362,11 @@ declare module "opentok-react-native" {
      */
     publishAudio?: boolean;
 
+    /**
+     * Whether to publish captions.
+     */
+    publishCaptions?: boolean;
+
     /**
      * Whether to publish video.
      */
@@ -420,6 +431,11 @@ declare module "opentok-react-native" {
      * the publisher are available.
      */
     getRtcStatsReport?: () => void;
+
+    /**
+     * Sets video transformers for the publisher (or clears them if passed an empty array).
+     */
+    setVideoTransformers?: () => void;
   }
 
   interface OTSubscriberProps extends ViewProps {
@@ -460,6 +476,11 @@ declare module "opentok-react-native" {
      */
     subscribeToAudio?: boolean;
 
+    /**
+     * Whether to subscribe to captions.
+     */
+    subscribeToCaptions?: boolean;
+
     /**
      * Whether to subscribe video.
      */
@@ -482,6 +503,11 @@ declare module "opentok-react-native" {
      */
     connected?: Callback<any>;
 
+    /**
+     * Sent when the subscriber receives a caption for the stream.
+     */
+    captionReceived?: CallbackWithParam<SubscriberCaptionEvent, any>;
+
     /**
      * Called when the subscriber’s stream has been interrupted.
      */

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,25 @@
-# 2.26.0  (September 2023)
+# 2.26.1  (October 2023)
+
+- [Update]: The new `OTPublisher.setVideoTransformers()` method lets you set (and clear)
+  video transformers, such as a background blur for a publisher (issues #631 and #682).
+  For more info, see the docs: [OTPublisher](/docs/OTPublisher.md).
+
+- [Update]: Live Captions API enhancements (issue #643)
+
+  * The new OTPublisher.publishCaptions option lets you enable and disable captions for a published stream  (if captions are enabled for the session). For more info, see the docs:[OTPublisher](/docs/OTPublisher.md).
+
+  * The new OTSubscriber.subscribeToCaptions option lets you turn captions on and off for a subscriber (if captions are enabled for the session and the publisher is publishing captions). For more info, see the docs: [OTSubscriber](/docs/OTSubscriber.md).
+
+  * The new OTSubscriber captionReceived event is dispatched when a subscriber receives a caption. For more info, see the docs: [OTSubscriber](/docs/OTSubscriber.md).
+
+  * For more information, see the [Live Captions developer guide](https://tokbox.com/developer/guides/live-captions).
+
+- [Fix]: Fixes an issue in which applications could not connect to a session when
+  the `proxyUrl` option for OTSession was set. - issue #645
+
+- [Fix]: Fixes an issue a stream is not destroyed immediately after unmounting an OTSession component or when the `OTSession.disconnect()` method is called. - issues #685 and #686
+
+# 2.26.0  (October 2023)
 
 - [Update]: Update OpenTok Android SDK and OpenTok iOS SDK to version 2.26.1.
 
@@ -11,11 +32,6 @@
   [known issues](https://tokbox.com/developer/sdks/ios/release-notes.html#known-issues)
   in the OpenTok iOS SDK release notes.
 
-- [Fix]: Fixes an issue in which applications could not connect to a session when
-  the `proxyUrl` option for OTSession was set. - issue #645
-
-- [Fix]: Fixes an issue a stream is not destroyed immediately after unmounting an OTSession component or when the `OTSession.disconnect()` method is called. - issues #685 and #686
-
 # 2.25.4 (October 2023)
 
 - [Fix]: Fixes TypeScript definitions  - issue #690.
@@ -42,6 +58,7 @@ For more info, see the docs:
 * [OTSubscriber](/docs/OTSubscriber.md)
 
 - [Fix]: Fix android app crash due to permission missing.
+- [Fix]: Fix OTSubscriber audioVolume and other properties not working -- issue #694
 
 # 2.25.2 (July 5 2023)
 

diff --git a/android/src/main/java/com/opentokreactnative/OTSessionManager.java b/android/src/main/java/com/opentokreactnative/OTSessionManager.java
@@ -31,6 +31,7 @@
 import com.opentok.android.MuteForcedInfo;
 import com.opentok.android.Publisher;
 import com.opentok.android.PublisherKit;
+import com.opentok.android.PublisherKit.VideoTransformer;
 import com.opentok.android.Stream;
 import com.opentok.android.OpentokError;
 import com.opentok.android.Subscriber;
@@ -62,7 +63,9 @@ public class OTSessionManager extends ReactContextBaseJavaModule
         Session.ArchiveListener,
         Session.MuteListener,
         Session.StreamPropertiesListener,
+        Session.StreamCaptionsPropertiesListener,
         SubscriberKit.AudioLevelListener,
+        SubscriberKit.CaptionsListener,
         SubscriberKit.SubscriberRtcStatsReportListener,
         SubscriberKit.AudioStatsListener,
         SubscriberKit.VideoStatsListener,
@@ -163,6 +166,7 @@ public void initPublisher(String publisherId, ReadableMap properties, Callback c
         String resolution = properties.getString("resolution");
         Boolean publishAudio = properties.getBoolean("publishAudio");
         Boolean publishVideo = properties.getBoolean("publishVideo");
+        Boolean publishCaptions = properties.getBoolean("publishCaptions");
         String videoSource = properties.getString("videoSource");
         Boolean scalableScreenshare = properties.getBoolean("scalableScreenshare");
         Publisher mPublisher = null;
@@ -203,6 +207,7 @@ public void initPublisher(String publisherId, ReadableMap properties, Callback c
         mPublisher.setAudioFallbackEnabled(audioFallbackEnabled);
         mPublisher.setPublishVideo(publishVideo);
         mPublisher.setPublishAudio(publishAudio);
+        mPublisher.setPublishCaptions(publishCaptions);
         mPublisher.setAudioStatsListener(this);
         mPublisher.setVideoStatsListener(this);
         mPublisher.setMuteListener(this);
@@ -247,8 +252,10 @@ public void subscribeToStream(String streamId, String sessionId, ReadableMap pro
         mSubscriber.setRtcStatsReportListener(this);
         mSubscriber.setVideoListener(this);
         mSubscriber.setStreamListener(this);
+        mSubscriber.setCaptionsListener(this);
         mSubscriber.setSubscribeToAudio(properties.getBoolean("subscribeToAudio"));
         mSubscriber.setSubscribeToVideo(properties.getBoolean("subscribeToVideo"));
+        mSubscriber.setSubscribeToCaptions(properties.getBoolean("subscribeToCaptions"));
         if (properties.hasKey("preferredFrameRate")) {
             mSubscriber.setPreferredFrameRate((float) properties.getDouble("preferredFrameRate"));
         }
@@ -374,6 +381,16 @@ public void publishAudio(String publisherId, Boolean publishAudio) {
         }
     }
 
+    @ReactMethod
+    public void publishCaptions(String publisherId, Boolean publishCaptions) {
+
+        ConcurrentHashMap<String, Publisher> mPublishers = sharedState.getPublishers();
+        Publisher mPublisher = mPublishers.get(publisherId);
+        if (mPublisher != null) {
+            mPublisher.setPublishCaptions(publishCaptions);
+        }
+    }
+
     @ReactMethod
     public void publishVideo(String publisherId, Boolean publishVideo) {
 
@@ -394,6 +411,16 @@ public void getRtcStatsReport(String publisherId) {
         }
     }
 
+    @ReactMethod
+    public void setVideoTransformers(String publisherId, ReadableArray videoTransformers) {
+        ConcurrentHashMap<String, Publisher> mPublishers = sharedState.getPublishers();
+        Publisher mPublisher = mPublishers.get(publisherId);
+        if (mPublisher != null) {
+          ArrayList<VideoTransformer> nativeVideoTransformers = Utils.sanitizeVideoTransformerList(mPublisher, videoTransformers);
+          mPublisher.setVideoTransformers(nativeVideoTransformers);
+        }
+    }
+
     @ReactMethod
     public void subscribeToAudio(String streamId, Boolean subscribeToAudio) {
 
@@ -414,6 +441,15 @@ public void subscribeToVideo(String streamId, Boolean subscribeToVideo) {
         }
     }
 
+    @ReactMethod
+    public void subscribeToCaptions(String streamId, Boolean subscribeToCaptions) {
+        ConcurrentHashMap<String, Subscriber> mSubscribers = sharedState.getSubscribers();
+        Subscriber mSubscriber = mSubscribers.get(streamId);
+        if (mSubscriber != null) {
+            mSubscriber.setSubscribeToCaptions(subscribeToCaptions);
+        }
+    }
+
     @ReactMethod
     public void setPreferredResolution(String streamId, ReadableMap resolution) {
 
@@ -1149,6 +1185,22 @@ public void onVideoDataReceived(SubscriberKit subscriber) {
         }
     }
 
+    @Override
+    public void onCaptionText(SubscriberKit subscriber, String text, boolean isFinal) {
+        String streamId = Utils.getStreamIdBySubscriber(subscriber);
+        if (streamId.length() > 0) {
+            ConcurrentHashMap<String, Stream> streams = sharedState.getSubscriberStreams();
+            Stream mStream = streams.get(streamId);
+            WritableMap subscriberInfo = Arguments.createMap();
+            if (mStream != null) {
+                subscriberInfo.putMap("stream", EventUtils.prepareJSStreamMap(mStream, subscriber.getSession()));
+            }
+            subscriberInfo.putString("text", String.valueOf(text));
+            subscriberInfo.putBoolean("isFinal", isFinal);
+            sendEventMap(this.getReactApplicationContext(), subscriberPreface + "onCaptionText", subscriberInfo);
+        }
+    }
+
     @Override
     public void onStreamHasAudioChanged(Session session, Stream stream, boolean Audio) {
 
@@ -1157,6 +1209,15 @@ public void onStreamHasAudioChanged(Session session, Stream stream, boolean Audi
         printLogs("onStreamHasAudioChanged");
     }
 
+    @Override
+    public void onStreamHasCaptionsChanged(Session session, Stream stream, boolean hasCaptions) {
+        if (stream != null) {
+            WritableMap eventData = EventUtils.prepareStreamPropertyChangedEventData("hasCaptions", !hasCaptions, hasCaptions, stream, session);
+            sendEventMap(this.getReactApplicationContext(), session.getSessionId() + ":" + sessionPreface + "onStreamPropertyChanged", eventData);
+            printLogs("onStreamHasCaptionsChanged");
+        }
+    }
+
     @Override
     public void onStreamHasVideoChanged(Session session, Stream stream, boolean Video) {
 

diff --git a/android/src/main/java/com/opentokreactnative/utils/Utils.java b/android/src/main/java/com/opentokreactnative/utils/Utils.java
@@ -3,6 +3,7 @@
 import com.opentok.android.OpentokError;
 import com.opentok.android.Publisher;
 import com.opentok.android.PublisherKit;
+import com.opentok.android.PublisherKit.VideoTransformer;
 import com.opentok.android.Subscriber;
 import com.opentok.android.SubscriberKit;
 import com.opentok.android.Session.Builder.TransportPolicy;
@@ -100,6 +101,25 @@ public static List<IceServer> sanitizeIceServer(ReadableArray serverList) {
         return iceServers;
     }
 
+    public static ArrayList<VideoTransformer> sanitizeVideoTransformerList(PublisherKit publisher, ReadableArray transformerList) {
+        ArrayList<VideoTransformer> nativeVideoTransformers = new ArrayList<>();
+        if (transformerList != null) {
+            for (int i = 0; i < transformerList.size(); i++) {
+                String transformerName = transformerList.getMap(i).getString("name");
+                if (transformerName == "BackgroundReplacement" ) {
+                    // Only implemented in iOS -- ignore in Androd for now.
+                    continue;
+                }
+                VideoTransformer transformer = publisher.new VideoTransformer(
+                    transformerName,
+                    transformerList.getMap(i).getString("properties")
+                );
+                nativeVideoTransformers.add(transformer);
+            }
+        }
+        return nativeVideoTransformers;
+    }
+
     public static VideoContentHint convertVideoContentHint(String videoContentHint) {
 
         switch (videoContentHint) {

diff --git a/docs/EventData.md b/docs/EventData.md
@@ -79,6 +79,18 @@ The event object has the following properties:
   };
 ```
 
+## SubscriberCaptionEvent
+
+To get captions for a subscriber, register an event listener for the `captionReceived` event.
+The event object has the following properties: 
+
+```javascript
+  event = {
+    text: string,
+    isFinal: boolean,
+  };
+```
+
 ## ConnectionCreatedEvent
 
 You can find the structure of the object below: 

diff --git a/docs/OTPublisher.md b/docs/OTPublisher.md
@@ -14,7 +14,9 @@ class App extends Component {
     super(props);
 
     this.publisherProperties = {
-      publishAudio: false,
+      publishAudio: true,
+      publishVideo: false,
+      publishCaptions: true,
       cameraPosition: 'front'
     };
 
@@ -60,6 +62,8 @@ The OTPublisher component has the following properties, each of which is optiona
 
   * publishVideo -- Toggles video on (`true`) or off `false`.
 
+  * publishCaptions -- Toggles captions on (`true`) or off `false` for the published stream.
+
 * `eventHandlers` (Object) -- An object containing key-value pairs of event names and
 callback functions for event handlers. See [Events](#events).
 
@@ -108,6 +112,8 @@ see the Subscriber videoDisabled event and the OpenTok Media Router and media mo
 
 * **publishVideo** (Boolean) -- Whether to publish video. The default is `true`.
 
+* **publishCaptions** (Boolean) — Whether to publish captions. Note that the session must have captions enabled (using the Video API REST method or server SDK) and the publisher must be publishing audio. For more information, see the [Live Captions developer guide](https://tokbox.com/developer/guides/live-captions).
+
 * **scalableScreenshare** (Boolean) -- Whether to allow use of
 {scalable video}(https://tokbox.com/developer/guides/scalable-video/) for a screen-sharing publisher
 (true) or not (false, the default). This only applies to a publisher that has the `videoSource` set
@@ -159,5 +165,52 @@ A [streamingEvent](./EventData.md#streamingEvent) object is passed into the even
 * **streamDestroyed** (Object) -- Sent when the publisher stops streaming.
 A [streamingEvent](./EventData.md#streamingEvent) object is passed into the event handler.
 
+**setVideoTransformers()** -- Sets video transformers for the publisher. This method has one parameter -- and array of objects defining each transformer to apply to the publisher's stream. A transformer object has two properties:
+
+* `name` (String) -- Either 'BackgroundBlur' (for a background blur filter) or 'BackgroundImageReplacement' (for a background image replacement filter). Android only supports the 'BackgroundBlur' transformer (and it is a beta feature in Android).
+
+* `properties` (String) -- A JSON string with the properties of the Vonage video transformer.
+
+  For a background blur transformer, the format of the JSON is:
+
+  ```
+  `{
+     "radius" :"None"
+   }`
+   ```
+
+   Valid values for the radius property are "None", "High", and "Low".
+
+  For a custom background blur transformer, the format of the JSON is:
+
+  ```
+  `{
+    "radius": "Custom",
+    "custom_radius": "value"
+  }
+  ```
+
+  `custom_radius` can be any positive integer.
+
+  For a background replacement transformer (supported on iOS only), the format of the JSON is:
+
+  ```
+  `{
+    "image_file_path": "path/to/image"
+  }`
+  ```
+
+  Where `image_file_path` is the absolute file path of a local image to use as virtual background. Supported image formats are PNG and JPEG.
+
+*Important:* Media transformations, such as background blur and background replacement, are resource-intensive and require devices with high processing power. It is recommended to only use these transformations on supported devices. See the following documentation:
+
+* [For iOS](https://tokbox.com/developer/guides/vonage-media-processor/ios/#client-requirements)
+
+* [For Android](https://tokbox.com/developer/guides/vonage-media-processor/android/#client-requirements)
+
+For more information on transformers, see [Using the Vonage Media Processor library](https://tokbox.com/developer/guides/vonage-media-processor/)
+
 **videoNetworkStats** (Object) -- Sent periodically to report audio statistics for the publisher.
   A [PublisherVideoNetworkStatsEvent](./EventData.md#PublisherVideoNetworkStatsEvent) object is passed into the event handler.
+## Methods
+