Remove B-frames when playing WebRTC streams (#4176)

1.  webrtc 播放移除了 h264 rtp中的b 帧
2. 增加了 rtc.bfilter 配置
ref: ISO_IEC_14496-10-AVC-2012

conf/config.ini

@@ -389,6 +389,8 @@ nackMaxCount=15
 nackIntervalRatio=1.0
 #nack包中rtp个数，减小此值可以让nack包响应更灵敏
 nackRtpSize=8
+#是否尝试过滤 b帧
+bfilter=0
 
 [srt]
 #srt播放推流、播放超时时间,单位秒

webrtc/WebRtcPlayer.cpp

@@ -18,9 +18,156 @@ using namespace std;
 
 namespace mediakit {
 
-WebRtcPlayer::Ptr WebRtcPlayer::create(const EventPoller::Ptr &poller,
-                                       const RtspMediaSource::Ptr &src,
-                                       const MediaInfo &info) {
+namespace Rtc {
+#define RTC_FIELD "rtc."
+const string kBfilter = RTC_FIELD "bfilter";
+static onceToken token([]() { mINI::Instance()[kBfilter] = 0; });
+} // namespace Rtc
+
+H264BFrameFilter::H264BFrameFilter()
+    : _last_seq(0)
+    , _last_stamp(0)
+    , _first_packet(true) {}
+
+RtpPacket::Ptr H264BFrameFilter::processPacket(const RtpPacket::Ptr &packet) {
+    if (!packet) {
+        return nullptr;
+    }
+
+    if (isH264BFrame(packet)) {
+        return nullptr;
+    }
+
+    auto cur_stamp = packet->getStamp();
+    auto cur_seq = packet->getSeq();
+
+    if (_first_packet) {
+        _first_packet = false;
+        _last_seq = cur_seq;
+        _last_stamp = cur_stamp;
+    }
+
+    // 处理时间戳连续性问题
+    if (cur_stamp < _last_stamp) {
+        return nullptr;
+    }
+    _last_stamp = cur_stamp;
+
+    // 处理 seq 连续性问题
+    if (cur_seq > _last_seq + 4) {
+        RtpHeader *header = packet->getHeader();
+        _last_seq = (_last_seq + 1) & 0xFFFF;
+        header->seq = htons(_last_seq);
+    }
+
+    return packet;
+}
+
+bool H264BFrameFilter::isH264BFrame(const RtpPacket::Ptr &packet) const {
+    uint8_t *payload = packet->getPayload();
+    size_t payload_size = packet->getPayloadSize();
+
+    if (payload_size < 1) {
+        return false;
+    }
+
+    uint8_t nal_unit_type = payload[0] & 0x1F;
+    switch (nal_unit_type) {
+        case 24: // STAP-A
+            return handleStapA(payload, payload_size);
+        case 28: // FU-A
+            return handleFua(payload, payload_size);
+        default:
+            if (nal_unit_type < 24) {
+                return isBFrameByNalType(nal_unit_type, payload + 1, payload_size - 1);
+            }
+            return false;
+    }
+}
+
+bool H264BFrameFilter::handleStapA(const uint8_t *payload, size_t payload_size) const {
+    size_t offset = 1;
+    while (offset + 2 <= payload_size) {
+        uint16_t nalu_size = (payload[offset] << 8) | payload[offset + 1];
+        offset += 2;
+        if (offset + nalu_size > payload_size || nalu_size < 1) {
+            return false;
+        }
+        uint8_t original_nal_type = payload[offset] & 0x1F;
+        if (original_nal_type < 24) {
+            if (isBFrameByNalType(original_nal_type, payload + offset + 1, nalu_size - 1)) {
+                return true;
+            }
+        }
+        offset += nalu_size;
+    }
+    return false;
+}
+
+bool H264BFrameFilter::handleFua(const uint8_t *payload, size_t payload_size) const {
+    if (payload_size < 2) {
+        return false;
+    }
+    uint8_t fu_header = payload[1];
+    uint8_t original_nal_type = fu_header & 0x1F;
+    bool start_bit = fu_header & 0x80;
+    if (start_bit) {
+        return isBFrameByNalType(original_nal_type, payload + 2, payload_size - 2);
+    }
+    return false;
+}
+
+bool H264BFrameFilter::isBFrameByNalType(uint8_t nal_type, const uint8_t *data, size_t size) const {
+    if (size < 1) {
+        return false;
+    }
+
+    if (nal_type != NAL_NIDR && nal_type != NAL_PARTITION_A && nal_type != NAL_PARTITION_B && nal_type != NAL_PARTITION_C) {
+        return false;
+    }
+
+    uint8_t slice_type = extractSliceType(data, size);
+    return slice_type == H264SliceTypeB || slice_type == H264SliceTypeB1;
+}
+
+int H264BFrameFilter::decodeExpGolomb(const uint8_t *data, size_t size, size_t &bitPos) const {
+    if (bitPos >= size * 8)
+        return -1;
+
+    int leadingZeroBits = 0;
+    while (bitPos < size * 8 && !getBit(data, bitPos++)) {
+        leadingZeroBits++;
+    }
+
+    int result = (1 << leadingZeroBits) - 1;
+    for (int i = 0; i < leadingZeroBits; i++) {
+        if (bitPos < size * 8) {
+            result += getBit(data, bitPos++) << (leadingZeroBits - i - 1);
+        }
+    }
+
+    return result;
+}
+
+int H264BFrameFilter::getBit(const uint8_t *data, size_t pos) const {
+    size_t byteIndex = pos / 8;
+    size_t bitOffset = pos % 8;
+    uint8_t byte = data[byteIndex];
+    return (byte >> (7 - bitOffset)) & 0x01;
+}
+
+uint8_t H264BFrameFilter::extractSliceType(const uint8_t *data, size_t size) const {
+    size_t bitPos = 0;
+    int first_mb_in_slice = decodeExpGolomb(data, size, bitPos);
+    int slice_type = decodeExpGolomb(data, size, bitPos);
+
+    if (slice_type >= 0 && slice_type <= 9) {
+        return static_cast<uint8_t>(slice_type);
+    }
+    return -1;
+}
+
+WebRtcPlayer::Ptr WebRtcPlayer::create(const EventPoller::Ptr &poller, const RtspMediaSource::Ptr &src, const MediaInfo &info) {
     WebRtcPlayer::Ptr ret(new WebRtcPlayer(poller, src, info), [](WebRtcPlayer *ptr) {
         ptr->onDestory();
         delete ptr;
@@ -29,22 +176,26 @@ WebRtcPlayer::Ptr WebRtcPlayer::create(const EventPoller::Ptr &poller,
     return ret;
 }
 
-WebRtcPlayer::WebRtcPlayer(const EventPoller::Ptr &poller,
-                           const RtspMediaSource::Ptr &src,
-                           const MediaInfo &info) : WebRtcTransportImp(poller) {
+WebRtcPlayer::WebRtcPlayer(const EventPoller::Ptr &poller, const RtspMediaSource::Ptr &src, const MediaInfo &info)
+    : WebRtcTransportImp(poller) {
     _media_info = info;
     _play_src = src;
     CHECK(src);
 
     GET_CONFIG(bool, direct_proxy, Rtsp::kDirectProxy);
     _send_config_frames_once = direct_proxy;
+
+    GET_CONFIG(bool, enable, Rtc::kBfilter);
+    _bfliter_flag = enable;
+    _is_h264 = false;
+    _bfilter = std::make_shared<H264BFrameFilter>();
 }
 
 void WebRtcPlayer::onStartWebRTC() {
     auto playSrc = _play_src.lock();
-    if(!playSrc){
+    if (!playSrc) {
         onShutdown(SockException(Err_shutdown, "rtsp media source was shutdown"));
-        return ;
+        return;
     }
     WebRtcTransportImp::onStartWebRTC();
     if (canSendRtp()) {
@@ -71,8 +222,18 @@ void WebRtcPlayer::onStartWebRTC() {
 
             size_t i = 0;
             pkt->for_each([&](const RtpPacket::Ptr &rtp) {
-                //TraceL<<"send track type:"<<rtp->type<<" ts:"<<rtp->getStamp()<<" ntp:"<<rtp->ntp_stamp<<" size:"<<rtp->getPayloadSize()<<" i:"<<i;
-                strong_self->onSendRtp(rtp, ++i == pkt->size());
+                if (strong_self->_bfliter_flag) {
+                    if (TrackVideo == rtp->type && strong_self->_is_h264) {
+                        auto rtp_filter = strong_self->_bfilter->processPacket(rtp);
+                        if (rtp_filter) {
+                            strong_self->onSendRtp(rtp_filter, ++i == pkt->size());
+                        }
+                    } else {
+                        strong_self->onSendRtp(rtp, ++i == pkt->size());
+                    }
+                } else {
+                    strong_self->onSendRtp(rtp, ++i == pkt->size());
+                }
             });
         });
         _reader->setDetachCB([weak_self]() {
@@ -83,7 +244,7 @@ void WebRtcPlayer::onStartWebRTC() {
             strong_self->onShutdown(SockException(Err_shutdown, "rtsp ring buffer detached"));
         });
 
-        _reader->setMessageCB([weak_self] (const toolkit::Any &data) {
+        _reader->setMessageCB([weak_self](const toolkit::Any &data) {
             auto strong_self = weak_self.lock();
             if (!strong_self) {
                 return;
@@ -118,8 +279,8 @@ void WebRtcPlayer::onDestory() {
 
 void WebRtcPlayer::onRtcConfigure(RtcConfigure &configure) const {
     auto playSrc = _play_src.lock();
-    if(!playSrc){
-        return ;
+    if (!playSrc) {
+        return;
     }
     WebRtcTransportImp::onRtcConfigure(configure);
     // 这是播放  [AUTO-TRANSLATED:d93c019e]
@@ -142,6 +303,7 @@ void WebRtcPlayer::sendConfigFrames(uint32_t before_seq, uint32_t sample_rate, u
     if (!video_track) {
         return;
     }
+    _is_h264 = video_track->getCodecId() == CodecH264;
     auto frames = video_track->getConfigFrames();
     if (frames.empty()) {
         return;

webrtc/WebRtcPlayer.h

@@ -11,10 +11,116 @@
 #ifndef ZLMEDIAKIT_WEBRTCPLAYER_H
 #define ZLMEDIAKIT_WEBRTCPLAYER_H
 
-#include "WebRtcTransport.h"
 #include "Rtsp/RtspMediaSource.h"
+#include "WebRtcTransport.h"
 
 namespace mediakit {
+/**
+ * @brief H.264 B 帧过滤器
+ * 用于从 H.264 RTP 流中移除 B 帧
+ */
+class H264BFrameFilter {
+public:
+    /**
+     * ISO_IEC_14496-10-AVC-2012
+     * Table 7-6 – Name association to slice_type
+     */
+    enum H264SliceType {
+        H264SliceTypeP = 0,
+        H264SliceTypeB = 1,
+        H264SliceTypeI = 2,
+        H264SliceTypeSP = 3,
+        H264SliceTypeSI = 4,
+        H264SliceTypeP1 = 5,
+        H264SliceTypeB1 = 6,
+        H264SliceTypeI1 = 7,
+        H264SliceTypeSP1 = 8,
+        H264SliceTypeSI1 = 9,
+    };
+
+    enum H264NALUType {
+        NAL_NIDR = 1,
+        NAL_PARTITION_A = 2,
+        NAL_PARTITION_B = 3,
+        NAL_PARTITION_C = 4,
+        NAL_IDR = 5,
+    };
+
+    H264BFrameFilter();
+
+    ~H264BFrameFilter() = default;
+
+    /**
+     * @brief 处理单个 RTP 包，移除 B 帧
+     * @param packet 输入的 RTP 包
+     * @return 如果不是 B 帧则返回原包，否则返回 nullptr
+     */
+    RtpPacket::Ptr processPacket(const RtpPacket::Ptr &packet);
+
+private:
+    /**
+     * @brief 判断 RTP 包是否包含 H.264 的 B 帧
+     * @param packet RTP 包
+     * @return 如果是 B 帧返回 true，否则返回 false
+     */
+    bool isH264BFrame(const RtpPacket::Ptr &packet) const;
+
+    /**
+     * @brief 根据 NAL 类型和数据判断是否是 B 帧
+     * @param nal_type NAL 单元类型
+     * @param data NAL 单元数据（不含 NAL 头）
+     * @param size 数据大小
+     * @return 如果是 B 帧返回 true，否则返回 false
+     */
+    bool isBFrameByNalType(uint8_t nal_type, const uint8_t *data, size_t size) const;
+
+    /**
+     * @brief 解析指数哥伦布编码
+     * @param data 数据缓冲区
+     * @param size 缓冲区大小
+     * @param bits_offset 位偏移量
+     * @return 解析出的数值
+     */
+    int decodeExpGolomb(const uint8_t *data, size_t size, size_t &bitPos) const;
+
+    /**
+     * @brief 从比特流中读取位
+     * @param data 数据缓冲区
+     * @param size 缓冲区大小
+     * @return 读取的位值（0 或 1）
+     */
+    int getBit(const uint8_t *data, size_t size) const;
+
+    /**
+     * @brief 提取切片类型值
+     * @param data 数据缓冲区
+     * @param size 缓冲区大小
+     * @return 切片类型值
+     */
+    uint8_t extractSliceType(const uint8_t *data, size_t size) const;
+
+    /**
+     * @brief 处理FU-A分片
+     * @param payload 数据缓冲区
+     * @param payload_size 缓冲区大小
+     * @return 如果是 B 帧返回 true，否则返回 false
+     */
+    bool handleFua(const uint8_t *payload, size_t payload_size) const;
+
+    /**
+   * @brief 处理 STAP-A 组合包
+   * @param payload 数据缓冲区
+   * @param payload_size 缓冲区大小
+   * @return 如果是 B 帧返回 true，否则返回 false
+   */
+    bool handleStapA(const uint8_t *payload, size_t payload_size) const;
+
+
+private:
+    uint16_t _last_seq; // 维护输出流的序列号
+    uint32_t _last_stamp; // 维护输出流的时间戳
+    bool _first_packet; // 是否是第一个包的标记
+};
 
 class WebRtcPlayer : public WebRtcTransportImp {
 public:
@@ -48,6 +154,10 @@ private:
     // 播放rtsp源的reader对象  [AUTO-TRANSLATED:7b305055]
     // Reader object for playing rtsp source
     RtspMediaSource::RingType::RingReader::Ptr _reader;
+
+    bool _is_h264 { false };
+    bool _bfliter_flag { false };
+    std::shared_ptr<H264BFrameFilter> _bfilter;
 };
 
 }// namespace mediakit