Enable frame sync extension.

api/rtp_headers.h

@@ -22,6 +22,7 @@
 #include "api/video/color_space.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame_marking.h"
+#include "api/video/video_frame_sync.h"
 #include "api/video/video_rotation.h"
 #include "api/video/video_timing.h"
 #include "common_types.h"  // NOLINT(build/include)
@@ -159,6 +160,9 @@ struct RTPHeaderExtension {
   std::string mid;
 
   absl::optional<ColorSpace> color_space;
+
+  // Used for multi-stream sync.
+  absl::optional<FrameSync> frame_sync;
 };
 
 enum { kRtpCsrcSize = 15 };  // RFC 3550 page 13

api/rtp_parameters.cc

@@ -116,6 +116,7 @@ constexpr char RtpExtension::kColorSpaceUri[];
 constexpr char RtpExtension::kMidUri[];
 constexpr char RtpExtension::kRidUri[];
 constexpr char RtpExtension::kRepairedRidUri[];
+constexpr char RtpExtension::kVideoFrameSyncUri[];
 
 constexpr int RtpExtension::kMinId;
 constexpr int RtpExtension::kMaxId;
@@ -151,7 +152,8 @@ bool RtpExtension::IsSupportedForVideo(absl::string_view uri) {
          uri == webrtc::RtpExtension::kDependencyDescriptorUri ||
          uri == webrtc::RtpExtension::kColorSpaceUri ||
          uri == webrtc::RtpExtension::kRidUri ||
-         uri == webrtc::RtpExtension::kRepairedRidUri;
+         uri == webrtc::RtpExtension::kRepairedRidUri ||
+         uri == webrtc::RtpExtension::kVideoFrameSyncUri;
 }
 
 bool RtpExtension::IsEncryptionSupported(absl::string_view uri) {
@@ -173,7 +175,9 @@ bool RtpExtension::IsEncryptionSupported(absl::string_view uri) {
          uri == webrtc::RtpExtension::kVideoContentTypeUri ||
          uri == webrtc::RtpExtension::kMidUri ||
          uri == webrtc::RtpExtension::kRidUri ||
-         uri == webrtc::RtpExtension::kRepairedRidUri;
+         uri == webrtc::RtpExtension::kRepairedRidUri ||
+         // VideoFrameSync needs to be encrypted.
+         uri == webrtc::RtpExtension::kVideoFrameSyncUri;
 }
 
 const RtpExtension* RtpExtension::FindHeaderExtensionByUri(

api/rtp_parameters.h

@@ -360,6 +360,12 @@ struct RTC_EXPORT RtpExtension {
   static constexpr char kRepairedRidUri[] =
       "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id";
 
+  // Header extension for video frame sync
+  // https://github.com/open-webrtc-toolkit/owt-client-native/wiki/video-frame-sync
+  static constexpr char kVideoFrameSyncUri[] =
+      "https://github.com/open-webrtc-toolkit/owt-client-native/wiki/"
+      "video-frame-sync";
+
   // Inclusive min and max IDs for two-byte header extensions and one-byte
   // header extensions, per RFC8285 Section 4.2-4.3.
   static constexpr int kMinId = 1;

api/video/BUILD.gn

@@ -22,6 +22,8 @@ rtc_library("video_rtp_headers") {
     "video_content_type.cc",
     "video_content_type.h",
     "video_frame_marking.h",
+    "video_frame_sync.cc",
+    "video_frame_sync.h",
     "video_rotation.h",
     "video_timing.cc",
     "video_timing.h",

api/video/encoded_image.h

@@ -23,6 +23,7 @@
 #include "api/video/video_codec_constants.h"
 #include "api/video/video_codec_type.h"
 #include "api/video/video_content_type.h"
+#include "api/video/video_frame_sync.h"
 #include "api/video/video_frame_type.h"
 #include "api/video/video_rotation.h"
 #include "api/video/video_timing.h"
@@ -115,6 +116,14 @@ class RTC_EXPORT EncodedImage {
     color_space_ = color_space;
   }
 
+  absl::optional<webrtc::FrameSync> FrameSync() const {
+      return frame_sync_;
+  }
+
+  void SetFrameSync(const absl::optional<webrtc::FrameSync> sync_point) {
+    frame_sync_ = sync_point;
+  }
+
   const RtpPacketInfos& PacketInfos() const { return packet_infos_; }
   void SetPacketInfos(RtpPacketInfos packet_infos) {
     packet_infos_ = std::move(packet_infos);
@@ -231,6 +240,7 @@ class RTC_EXPORT EncodedImage {
   // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
   RtpPacketInfos packet_infos_;
   bool retransmission_allowed_ = true;
+  absl::optional<webrtc::FrameSync> frame_sync_;
 };
 
 }  // namespace webrtc

api/video/video_frame_sync.cc

@@ -0,0 +1,20 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/video_frame_sync.h"
+
+namespace webrtc {
+
+FrameSync::FrameSync() = default;
+FrameSync::FrameSync(const FrameSync& other) = default;
+FrameSync::FrameSync(FrameSync&& other) = default;
+FrameSync& FrameSync::operator=(const FrameSync& other) = default;
+
+}  // namespace webrtc

api/video/video_frame_sync.h

@@ -0,0 +1,52 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_VIDEO_FRAME_SYNC_H_
+#define API_VIDEO_VIDEO_FRAME_SYNC_H_
+
+#include <stdint.h>
+#include "rtc_base/system/rtc_export.h"
+
+namespace webrtc {
+
+// This class represents frame sync point information needed by SFU/MCU
+// to sync frames from different streams.
+//
+// Currently the |sync_point| is an application defined 2-byte identfier
+// that is supposed mono increase and wrap around when maximum value is
+// reached. one byte is reserved here for future extension.
+class RTC_EXPORT FrameSync {
+ public:
+  FrameSync();
+  FrameSync(const FrameSync& other);
+  FrameSync(FrameSync&& other);
+  FrameSync& operator=(const FrameSync& other);
+
+  friend bool operator==(const FrameSync& lhs, const FrameSync& rhs) {
+    return lhs.sync_point_ == rhs.sync_point_;
+  }
+  friend bool operator!=(const FrameSync& lhs, const FrameSync& rhs) {
+    return lhs.sync_point_ != rhs.sync_point_;
+  }
+
+  void set_sync_point(uint16_t sync_point) {
+      sync_point_ = sync_point;
+  }
+
+  uint16_t SyncPoint() const {
+    return sync_point_;
+  }
+
+ private:
+  uint16_t sync_point_;
+  uint8_t reserved_one_byte_;
+};
+}  // namespace webrtc
+#endif  // API_VIDEO_VIDEO_FRAME_SYNC_H_

call/rtp_payload_params.cc

@@ -195,7 +195,9 @@ RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader(
   if (generic_descriptor_experiment_)
     SetGeneric(codec_specific_info, shared_frame_id, is_keyframe,
                &rtp_video_header);
-
+  rtp_video_header.frame_sync = image.FrameSync()
+                                    ? absl::make_optional(*image.FrameSync())
+                                    : absl::nullopt;
   return rtp_video_header;
 }
 

media/engine/webrtc_video_engine.cc

@@ -591,7 +591,8 @@ WebRtcVideoEngine::GetRtpHeaderExtensions() const {
         webrtc::RtpExtension::kVideoTimingUri,
         webrtc::RtpExtension::kFrameMarkingUri,
         webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri,
-        webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) {
+        webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri,
+        webrtc::RtpExtension::kVideoFrameSyncUri}) {
     result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv);
   }
   result.emplace_back(

modules/rtp_rtcp/include/rtp_rtcp_defines.h

@@ -73,6 +73,7 @@ enum RTPExtensionType : int {
   kRtpExtensionGenericFrameDescriptor01,
   kRtpExtensionGenericFrameDescriptor02,
   kRtpExtensionColorSpace,
+  kRtpExtensionVideoFrameSync,
   kRtpExtensionNumberOfExtensions  // Must be the last entity in the enum.
 };
 

modules/rtp_rtcp/source/rtp_header_extension_map.cc

@@ -50,6 +50,7 @@ constexpr ExtensionInfo kExtensions[] = {
     CreateExtensionInfo<RtpDependencyDescriptorExtension>(),
     CreateExtensionInfo<ColorSpaceExtension>(),
     CreateExtensionInfo<InbandComfortNoiseExtension>(),
+    CreateExtensionInfo<FrameSyncExtension>(),
 };
 
 // Because of kRtpExtensionNone, NumberOfExtension is 1 bigger than the actual

modules/rtp_rtcp/source/rtp_header_extensions.cc

@@ -399,6 +399,38 @@ bool PlayoutDelayLimits::Write(rtc::ArrayView<uint8_t> data,
   return true;
 }
 
+// Video Frame Sync.
+//
+// E.g. Frame sync point: the monolic increasing index of sync point.
+// might wrap around when reaching maximum values.
+//
+//   0                   1                   2                   3
+//   0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  |  ID   | len=3 |         Frame sync point      |  Reserved     |
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+constexpr RTPExtensionType FrameSyncExtension::kId;
+constexpr uint8_t FrameSyncExtension::kValueSizeBytes;
+constexpr const char FrameSyncExtension::kUri[];
+
+bool FrameSyncExtension::Parse(rtc::ArrayView<const uint8_t> data,
+    FrameSync* frame_sync) {
+  RTC_DCHECK_EQ(data.size(), kValueSizeBytes);
+  if (data.size() != kValueSizeBytes)
+    return false;
+  uint16_t raw = ByteReader<uint16_t, 2>::ReadBigEndian(data.data());
+  frame_sync->set_sync_point(raw);
+  return true;
+}
+
+bool FrameSyncExtension::Write(rtc::ArrayView<uint8_t> data,
+    const FrameSync& frame_sync) {
+  RTC_DCHECK_EQ(data.size(), kValueSizeBytes);
+  ByteWriter<uint16_t, 2>::WriteBigEndian(data.data(), frame_sync.SyncPoint());
+  data[2] = 0;
+  return true;
+}
+
 // Video Content Type.
 //
 // E.g. default video or screenshare.

modules/rtp_rtcp/source/rtp_header_extensions.h

@@ -20,6 +20,7 @@
 #include "api/video/color_space.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame_marking.h"
+#include "api/video/video_frame_sync.h"
 #include "api/video/video_rotation.h"
 #include "api/video/video_timing.h"
 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
@@ -234,6 +235,21 @@ class FrameMarkingExtension {
   static bool IsScalable(uint8_t temporal_id, uint8_t layer_id);
 };
 
+class FrameSyncExtension {
+ public:
+  using value_type = FrameSync;
+  static constexpr RTPExtensionType kId = kRtpExtensionVideoFrameSync;
+  static constexpr uint8_t kValueSizeBytes = 3;
+  static constexpr const char kUri[] =
+      "https://github.com/open-webrtc-toolkit/owt-client-native/wiki/video-frame-sync";
+
+  static size_t ValueSize(const FrameSync& frame_sync) {
+    return kValueSizeBytes;
+  }
+  static bool Parse(rtc::ArrayView<const uint8_t> data, FrameSync* frame_sync);
+  static bool Write(rtc::ArrayView<uint8_t> data, const FrameSync& frame_sync);
+};
+
 class ColorSpaceExtension {
  public:
   using value_type = ColorSpace;

modules/rtp_rtcp/source/rtp_packet.cc

@@ -199,6 +199,7 @@ void RtpPacket::ZeroMutableExtensions() {
       case RTPExtensionType::kRtpExtensionRtpStreamId:
       case RTPExtensionType::kRtpExtensionVideoContentType:
       case RTPExtensionType::kRtpExtensionVideoRotation:
+      case RTPExtensionType::kRtpExtensionVideoFrameSync:
       case RTPExtensionType::kRtpExtensionInbandComfortNoise: {
         // Non-mutable extension. Don't change it.
         break;

modules/rtp_rtcp/source/rtp_packet_received.cc

@@ -76,6 +76,7 @@ void RtpPacketReceived::GetHeader(RTPHeader* header) const {
   GetExtension<RtpMid>(&header->extension.mid);
   GetExtension<PlayoutDelayLimits>(&header->extension.playout_delay);
   header->extension.color_space = GetExtension<ColorSpaceExtension>();
+  header->extension.frame_sync = GetExtension<FrameSyncExtension>();
 }
 
 }  // namespace webrtc

modules/rtp_rtcp/source/rtp_sender.cc

@@ -83,6 +83,7 @@ constexpr RtpExtensionSize kVideoExtensionSizes[] = {
      RtpGenericFrameDescriptorExtension00::kMaxSizeBytes},
     {RtpGenericFrameDescriptorExtension01::kId,
      RtpGenericFrameDescriptorExtension01::kMaxSizeBytes},
+    CreateExtensionSize<FrameSyncExtension>(),
 };
 
 bool HasBweExtension(const RtpHeaderExtensionMap& extensions_map) {

modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -295,6 +295,9 @@ void RTPSenderVideo::AddRtpHeaderExtensions(
   if (last_packet && set_color_space && video_header.color_space)
     packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value());
 
+  if (last_packet && video_header.frame_sync)
+    packet->SetExtension<FrameSyncExtension>(video_header.frame_sync.value());
+
   // According to
   // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
   // ts_126114v120700p.pdf Section 7.4.5:

modules/rtp_rtcp/source/rtp_utility.cc

@@ -18,6 +18,7 @@
 #include "api/array_view.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame_marking.h"
+#include "api/video/video_frame_sync.h"
 #include "api/video/video_rotation.h"
 #include "api/video/video_timing.h"
 #include "modules/rtp_rtcp/include/rtp_cvo.h"
@@ -450,6 +451,20 @@ void RtpHeaderParser::ParseOneByteExtensionHeader(
           RTC_LOG(WARNING) << "TransportSequenceNumberV2 unsupported by rtp "
                               "header parser.";
           break;
+        case kRtpExtensionVideoFrameSync: {
+          if (len != 2) {
+            RTC_LOG(LS_WARNING)
+                << "Invalid video frame sync point len: " << len;
+            return;
+          }
+          uint16_t sync_point = ptr[0] << 8;
+          sync_point += ptr[1];
+          absl::optional<webrtc::FrameSync> frame_sync =
+              absl::make_optional<webrtc::FrameSync>();
+          frame_sync->set_sync_point(sync_point);
+          header->extension.frame_sync = frame_sync;
+          break;
+        }
         case kRtpExtensionPlayoutDelay: {
           if (len != 2) {
             RTC_LOG(LS_WARNING) << "Incorrect playout delay len: " << len;

modules/rtp_rtcp/source/rtp_video_header.h

@@ -20,6 +20,7 @@
 #include "api/video/video_codec_type.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame_marking.h"
+#include "api/video/video_frame_sync.h"
 #include "api/video/video_frame_type.h"
 #include "api/video/video_rotation.h"
 #include "api/video/video_timing.h"
@@ -89,6 +90,7 @@ struct RTPVideoHeader {
   VideoSendTiming video_timing;
   FrameMarking frame_marking = {false, false, false, false, false, 0xFF, 0, 0};
   absl::optional<ColorSpace> color_space;
+  absl::optional<FrameSync> frame_sync;
   RTPVideoTypeHeader video_type_header;
 };
 

modules/video_coding/encoded_frame.h

@@ -70,6 +70,8 @@ class RTC_EXPORT VCMEncodedFrame : protected EncodedImage {
   using EncodedImage::SpatialIndex;
   using EncodedImage::SpatialLayerFrameSize;
   using EncodedImage::Timestamp;
+  using EncodedImage::FrameSync;
+  using EncodedImage::SetFrameSync;
 
   /**
    *   Get render time in milliseconds

modules/video_coding/frame_object.cc

@@ -37,6 +37,7 @@ RtpFrameObject::RtpFrameObject(
     VideoContentType content_type,
     const RTPVideoHeader& video_header,
     const absl::optional<webrtc::ColorSpace>& color_space,
+    const absl::optional<webrtc::FrameSync> frame_sync,
     RtpPacketInfos packet_infos,
     rtc::scoped_refptr<EncodedImageBuffer> image_buffer)
     : first_seq_num_(first_seq_num),
@@ -70,6 +71,7 @@ RtpFrameObject::RtpFrameObject(
 
   rotation_ = rotation;
   SetColorSpace(color_space);
+  SetFrameSync(frame_sync);
   content_type_ = content_type;
   if (timing.flags != VideoSendTiming::kInvalid) {
     // ntp_time_ms_ may be -1 if not estimated yet. This is not a problem,

modules/video_coding/frame_object.h

@@ -34,6 +34,7 @@ class RtpFrameObject : public EncodedFrame {
                  VideoContentType content_type,
                  const RTPVideoHeader& video_header,
                  const absl::optional<webrtc::ColorSpace>& color_space,
+                 const absl::optional<webrtc::FrameSync> frame_sync,
                  RtpPacketInfos packet_infos,
                  rtc::scoped_refptr<EncodedImageBuffer> image_buffer);