From bb25caf8147411a7657ccb56ef2b10dcfc1afe40 Mon Sep 17 00:00:00 2001 From: Grishka Date: Wed, 24 Apr 2019 19:36:32 +0300 Subject: [PATCH] fixes --- OpusEncoder.cpp | 14 ++----- VoIPController.cpp | 2 +- client/android/tg_voip_jni.cpp | 9 ++++- os/android/VideoRendererAndroid.cpp | 9 ++++- os/android/VideoRendererAndroid.h | 1 + os/android/VideoSourceAndroid.cpp | 4 ++ os/android/VideoSourceAndroid.h | 1 + video/ScreamCongestionController.cpp | 46 +++++++++++------------ video/VideoPacketSender.cpp | 56 ++++++++++++++-------------- video/VideoPacketSender.h | 2 +- 10 files changed, 78 insertions(+), 66 deletions(-) diff --git a/OpusEncoder.cpp b/OpusEncoder.cpp index d555b71..69805dd 100755 --- a/OpusEncoder.cpp +++ b/OpusEncoder.cpp @@ -41,7 +41,7 @@ tgvoip::OpusEncoder::OpusEncoder(MediaStreamItf *source, bool needSecondary):que opus_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(1)); opus_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1)); opus_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); - opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND)); + opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(OPUS_AUTO)); requestedBitrate=20000; currentBitrate=0; running=false; @@ -59,9 +59,7 @@ tgvoip::OpusEncoder::OpusEncoder(MediaStreamItf *source, bool needSecondary):que secondaryEncoder=opus_encoder_create(48000, 1, OPUS_APPLICATION_VOIP, NULL); opus_encoder_ctl(secondaryEncoder, OPUS_SET_COMPLEXITY(10)); opus_encoder_ctl(secondaryEncoder, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); - //opus_encoder_ctl(secondaryEncoder, OPUS_SET_VBR(0)); opus_encoder_ctl(secondaryEncoder, OPUS_SET_BITRATE(8000)); - opus_encoder_ctl(secondaryEncoder, OPUS_SET_BANDWIDTH(secondaryEnabledBandwidth)); }else{ secondaryEncoder=NULL; } @@ -107,9 +105,11 @@ void tgvoip::OpusEncoder::Encode(int16_t* data, size_t len){ levelMeter->Update(data, len); if(secondaryEncoderEnabled!=wasSecondaryEncoderEnabled){ wasSecondaryEncoderEnabled=secondaryEncoderEnabled; - opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(secondaryEncoderEnabled ? secondaryEnabledBandwidth : OPUS_BANDWIDTH_FULLBAND)); } int32_t r=opus_encode(enc, data, static_cast(len), buffer, 4096); +// int bw; +// opus_encoder_ctl(enc, OPUS_GET_BANDWIDTH(&bw)); +// LOGV("Opus bandwidth: %d", bw); if(r<=0){ LOGE("Error encoding: %d", r); }else if(r==1){ @@ -185,27 +185,21 @@ void tgvoip::OpusEncoder::RunThread(){ if(vadMode){ if(frameHasVoice){ opus_encoder_ctl(enc, OPUS_SET_BITRATE(currentBitrate)); - opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(vadModeVoiceBandwidth)); if(secondaryEncoder){ opus_encoder_ctl(secondaryEncoder, OPUS_SET_BITRATE(currentBitrate)); - opus_encoder_ctl(secondaryEncoder, OPUS_SET_BANDWIDTH(vadModeVoiceBandwidth)); } }else{ opus_encoder_ctl(enc, OPUS_SET_BITRATE(vadNoVoiceBitrate)); - opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(vadModeNoVoiceBandwidth)); if(secondaryEncoder){ opus_encoder_ctl(secondaryEncoder, OPUS_SET_BITRATE(vadNoVoiceBitrate)); - opus_encoder_ctl(secondaryEncoder, OPUS_SET_BANDWIDTH(vadModeNoVoiceBandwidth)); } } wasVadMode=true; }else if(wasVadMode){ wasVadMode=false; opus_encoder_ctl(enc, OPUS_SET_BITRATE(currentBitrate)); - opus_encoder_ctl(enc, OPUS_SET_BANDWIDTH(secondaryEncoderEnabled ? secondaryEnabledBandwidth : OPUS_AUTO)); if(secondaryEncoder){ opus_encoder_ctl(secondaryEncoder, OPUS_SET_BITRATE(currentBitrate)); - opus_encoder_ctl(secondaryEncoder, OPUS_SET_BANDWIDTH(secondaryEnabledBandwidth)); } } Encode(frame, 960*packetsPerFrame); diff --git a/VoIPController.cpp b/VoIPController.cpp index 3f632fc..d7e76f4 100755 --- a/VoIPController.cpp +++ b/VoIPController.cpp @@ -3303,7 +3303,7 @@ void VoIPController::SendPacketReliably(unsigned char type, unsigned char *data, void VoIPController::SendExtra(Buffer &data, unsigned char type){ ENFORCE_MSG_THREAD; - LOGV("Sending extra type %u length %lu", type, (unsigned int)data.Length()); + LOGV("Sending extra type %u length %u", type, (unsigned int)data.Length()); for(vector::iterator x=currentExtras.begin();x!=currentExtras.end();++x){ if(x->type==type){ x->firstContainingSeq=0; diff --git a/client/android/tg_voip_jni.cpp b/client/android/tg_voip_jni.cpp index 8818b6a..21bca5c 100644 --- a/client/android/tg_voip_jni.cpp +++ b/client/android/tg_voip_jni.cpp @@ -532,6 +532,10 @@ namespace tgvoip { ((video::VideoSourceAndroid*)(intptr_t)inst)->SetRotation((unsigned int)rotation); } + void VideoSource_nativeSetPaused(JNIEnv* env, jobject thiz, jlong inst, jboolean paused){ + ((video::VideoSourceAndroid*)(intptr_t)inst)->SetStreamPaused((bool)paused); + } + #pragma mark - VideoRenderer jlong VideoRenderer_nativeInit(JNIEnv* env, jobject thiz){ @@ -612,7 +616,7 @@ extern "C" void tgvoipRegisterNatives(JNIEnv* env){ if(videoRenderer){ video::VideoRendererAndroid::decodeAndDisplayMethod=env->GetMethodID(videoRenderer, "decodeAndDisplay", "(Ljava/nio/ByteBuffer;IJ)V"); video::VideoRendererAndroid::resetMethod=env->GetMethodID(videoRenderer, "reset", "(Ljava/lang/String;II[[B)V"); - video::VideoRendererAndroid::setStreamEnabledMethod=env->GetMethodID(videoRenderer, "setStreamEnabled", "(Z)V"); + video::VideoRendererAndroid::setStreamEnabledMethod=env->GetMethodID(videoRenderer, "setStreamEnabled", "(ZZ)V"); video::VideoRendererAndroid::setRotationMethod=env->GetMethodID(videoRenderer, "setRotation", "(I)V"); } } @@ -713,7 +717,8 @@ extern "C" void tgvoipRegisterNatives(JNIEnv* env){ {"nativeRelease", "(J)V", (void *) &tgvoip::VideoSource_nativeRelease}, {"nativeSetVideoStreamParameters", "(J[Ljava/nio/ByteBuffer;II)V", (void *) &tgvoip::VideoSource_nativeSetVideoStreamParameters}, {"nativeSendFrame", "(JLjava/nio/ByteBuffer;III)V", (void *) &tgvoip::VideoSource_nativeSendFrame}, - {"nativeSetRotation", "(JI)V", (void*)&tgvoip::VideoSource_nativeSetRotation} + {"nativeSetRotation", "(JI)V", (void*)&tgvoip::VideoSource_nativeSetRotation}, + {"nativeSetPaused", "(JZ)V", (void*)&tgvoip::VideoSource_nativeSetPaused} }; env->RegisterNatives(videoSource, videoSourceMethods, sizeof(videoSourceMethods)/sizeof(JNINativeMethod)); } diff --git a/os/android/VideoRendererAndroid.cpp b/os/android/VideoRendererAndroid.cpp index a407342..167beba 100644 --- a/os/android/VideoRendererAndroid.cpp +++ b/os/android/VideoRendererAndroid.cpp @@ -127,7 +127,7 @@ void VideoRendererAndroid::RunThread(){ } env->CallVoidMethod(jobj, resetMethod, env->NewStringUTF(codecStr.c_str()), (jint)width, (jint)height, jcsd); }else if(request.type==Request::Type::UpdateStreamState){ - env->CallVoidMethod(jobj, setStreamEnabledMethod, streamEnabled); + env->CallVoidMethod(jobj, setStreamEnabledMethod, streamEnabled, streamPaused); } } free(buf); @@ -150,5 +150,10 @@ void VideoRendererAndroid::SetRotation(uint16_t rotation){ } void VideoRendererAndroid::SetStreamPaused(bool paused){ - + streamPaused=paused; + Request req{ + Buffer(0), + Request::Type::UpdateStreamState + }; + queue.Put(std::move(req)); } diff --git a/os/android/VideoRendererAndroid.h b/os/android/VideoRendererAndroid.h index 7e093b8..6e8a40f 100644 --- a/os/android/VideoRendererAndroid.h +++ b/os/android/VideoRendererAndroid.h @@ -49,6 +49,7 @@ namespace tgvoip{ int width; int height; bool streamEnabled=true; + bool streamPaused=false; uint32_t codec; uint16_t rotation=0; jobject jobj; diff --git a/os/android/VideoSourceAndroid.cpp b/os/android/VideoSourceAndroid.cpp index 8431f50..7e79440 100644 --- a/os/android/VideoSourceAndroid.cpp +++ b/os/android/VideoSourceAndroid.cpp @@ -86,3 +86,7 @@ void VideoSourceAndroid::SetBitrate(uint32_t bitrate){ env->CallVoidMethod(javaObject, setBitrateMethod, (jint)bitrate); }); } + +void VideoSourceAndroid::SetStreamPaused(bool paused){ + streamStateCallback(paused); +} diff --git a/os/android/VideoSourceAndroid.h b/os/android/VideoSourceAndroid.h index b67bc8e..bac967b 100644 --- a/os/android/VideoSourceAndroid.h +++ b/os/android/VideoSourceAndroid.h @@ -23,6 +23,7 @@ namespace tgvoip{ void SetStreamParameters(std::vector csd, unsigned int width, unsigned int height); virtual void RequestKeyFrame() override; virtual void SetBitrate(uint32_t bitrate) override; + void SetStreamPaused(bool paused); static std::vector availableEncoders; private: diff --git a/video/ScreamCongestionController.cpp b/video/ScreamCongestionController.cpp index 11b5db3..2fdaaf2 100644 --- a/video/ScreamCongestionController.cpp +++ b/video/ScreamCongestionController.cpp @@ -12,28 +12,28 @@ using namespace tgvoip; using namespace tgvoip::video; namespace{ - /*static*/ constexpr float QDELAY_TARGET_LO=0.1f; // seconds - /*static*/ constexpr float QDELAY_TARGET_HI=0.4f; // seconds - /*static*/ constexpr float QDELAY_WEIGHT=0.1f; - /*static*/ constexpr float QDELAY_TREND_TH=0.2f; - /*static*/ constexpr uint32_t MIN_CWND=3000; // bytes - /*static*/ constexpr float MAX_BYTES_IN_FLIGHT_HEAD_ROOM=1.1f; - /*static*/ constexpr float GAIN=1.0f; - /*static*/ constexpr float BETA_LOSS=0.9f; - /*static*/ constexpr float BETA_ECN=0.9f; - /*static*/ constexpr float BETA_R=0.9f; - /*static*/ constexpr uint32_t MSS=1024; - /*static*/ constexpr float RATE_ADJUST_INTERVAL=0.2f; - /*static*/ constexpr uint32_t TARGET_BITRATE_MIN=50*1024; // bps - /*static*/ constexpr uint32_t TARGET_BITRATE_MAX=500*1024; // bps - /*static*/ constexpr uint32_t RAMP_UP_SPEED=200000; // bps/s - /*static*/ constexpr float PRE_CONGESTION_GUARD=0.1f; - /*static*/ constexpr float TX_QUEUE_SIZE_FACTOR=1.0f; - /*static*/ constexpr float RTP_QDELAY_TH=0.02f; // seconds - /*static*/ constexpr float TARGET_RATE_SCALE_RTP_QDELAY=0.95f; - /*static*/ constexpr float QDELAY_TREND_LO=0.2f; - /*static*/ constexpr float T_RESUME_FAST_INCREASE=5.0f; // seconds - /*static*/ constexpr uint32_t RATE_PACE_MIN=50000; // bps + constexpr float QDELAY_TARGET_LO=0.1f; // seconds + constexpr float QDELAY_TARGET_HI=0.4f; // seconds + constexpr float QDELAY_WEIGHT=0.1f; + constexpr float QDELAY_TREND_TH=0.2f; + constexpr uint32_t MIN_CWND=3000; // bytes + constexpr float MAX_BYTES_IN_FLIGHT_HEAD_ROOM=1.1f; + constexpr float GAIN=1.0f; + constexpr float BETA_LOSS=0.9f; + constexpr float BETA_ECN=0.9f; + constexpr float BETA_R=0.9f; + constexpr uint32_t MSS=1024; + constexpr float RATE_ADJUST_INTERVAL=0.2f; + constexpr uint32_t TARGET_BITRATE_MIN=50*1024; // bps + constexpr uint32_t TARGET_BITRATE_MAX=500*1024; // bps + constexpr uint32_t RAMP_UP_SPEED=1024*1024;//200000; // bps/s + constexpr float PRE_CONGESTION_GUARD=0.1f; + constexpr float TX_QUEUE_SIZE_FACTOR=1.0f; + constexpr float RTP_QDELAY_TH=0.02f; // seconds + constexpr float TARGET_RATE_SCALE_RTP_QDELAY=0.95f; + constexpr float QDELAY_TREND_LO=0.2f; + constexpr float T_RESUME_FAST_INCREASE=5.0f; // seconds + constexpr uint32_t RATE_PACE_MIN=50000; // bps } ScreamCongestionController::ScreamCongestionController() : qdelayTarget(QDELAY_TARGET_LO), cwnd(MIN_CWND) { @@ -71,7 +71,7 @@ void ScreamCongestionController::UpdateCWnd(float qdelay){ inFastIncrease=false; }else{ if((float)bytesInFlight*1.5f+bytesNewlyAcked>cwnd){ - LOGD("HERE"); + //LOGD("HERE"); cwnd+=bytesNewlyAcked; } return; diff --git a/video/VideoPacketSender.cpp b/video/VideoPacketSender.cpp index f19b3f6..687f2b1 100644 --- a/video/VideoPacketSender.cpp +++ b/video/VideoPacketSender.cpp @@ -94,6 +94,7 @@ void VideoPacketSender::SetSource(VideoSource *source){ if(!source) return; + sourceChangeTime=lastVideoResolutionChangeTime=VoIPController::GetCurrentTime(); uint32_t bitrate=videoCongestionControl.GetBitrate(); currentVideoBitrate=bitrate; source->SetBitrate(bitrate); @@ -102,9 +103,10 @@ void VideoPacketSender::SetSource(VideoSource *source){ source->SetCallback(std::bind(&VideoPacketSender::SendFrame, this, placeholders::_1, placeholders::_2, placeholders::_3)); source->SetStreamStateCallback([this](bool paused){ stm->paused=paused; - SendStreamFlags(*stm); + GetMessageThread().Post([this]{ + SendStreamFlags(*stm); + }); }); - lastVideoResolutionChangeTime=VoIPController::GetCurrentTime(); } void VideoPacketSender::SendFrame(const Buffer &_frame, uint32_t flags, uint32_t rotation){ @@ -125,7 +127,7 @@ void VideoPacketSender::SendFrame(const Buffer &_frame, uint32_t flags, uint32_t source->SetBitrate(bitrate); } int resolutionFromBitrate=GetVideoResolutionForCurrentBitrate(); - if(resolutionFromBitrate!=stm->resolution && currentTime-lastVideoResolutionChangeTime>3.0){ + if(resolutionFromBitrate!=stm->resolution && currentTime-lastVideoResolutionChangeTime>3.0 && currentTime-sourceChangeTime>10.0){ LOGI("Changing video resolution: %d -> %d", stm->resolution, resolutionFromBitrate); stm->resolution=resolutionFromBitrate; GetMessageThread().Post([this, resolutionFromBitrate]{ @@ -291,33 +293,33 @@ void VideoPacketSender::SendFrame(const Buffer &_frame, uint32_t flags, uint32_t int VideoPacketSender::GetVideoResolutionForCurrentBitrate(){ int peerMaxVideoResolution=GetProtocolInfo().maxVideoResolution; - int resolutionFromBitrate=INIT_VIDEO_RES_1080; - // TODO: probably move this to server config - if(stm->codec==CODEC_AVC || stm->codec==CODEC_VP8){ - if(currentVideoBitrate>400000){ - resolutionFromBitrate=INIT_VIDEO_RES_720; - }else if(currentVideoBitrate>250000){ - resolutionFromBitrate=INIT_VIDEO_RES_480; - }else{ - resolutionFromBitrate=INIT_VIDEO_RES_360; + if(VoIPController::GetCurrentTime()-sourceChangeTime<10.0){ + // TODO: probably move this to server config + if(stm->codec==CODEC_AVC || stm->codec==CODEC_VP8){ + if(currentVideoBitrate>400000){ + resolutionFromBitrate=INIT_VIDEO_RES_720; + }else if(currentVideoBitrate>250000){ + resolutionFromBitrate=INIT_VIDEO_RES_480; + }else{ + resolutionFromBitrate=INIT_VIDEO_RES_360; + } + }else if(stm->codec==CODEC_HEVC || stm->codec==CODEC_VP9){ + if(currentVideoBitrate>400000){ + resolutionFromBitrate=INIT_VIDEO_RES_1080; + }else if(currentVideoBitrate>250000){ + resolutionFromBitrate=INIT_VIDEO_RES_720; + }else if(currentVideoBitrate>100000){ + resolutionFromBitrate=INIT_VIDEO_RES_480; + }else{ + resolutionFromBitrate=INIT_VIDEO_RES_360; + } } - }else if(stm->codec==CODEC_HEVC || stm->codec==CODEC_VP9){ - if(currentVideoBitrate>400000){ + }else{ + if(stm->codec==CODEC_AVC || stm->codec==CODEC_VP8) + resolutionFromBitrate=INIT_VIDEO_RES_720; + else if(stm->codec==CODEC_HEVC || stm->codec==CODEC_VP9) resolutionFromBitrate=INIT_VIDEO_RES_1080; - }else if(currentVideoBitrate>250000){ - resolutionFromBitrate=INIT_VIDEO_RES_720; - }else if(currentVideoBitrate>100000){ - resolutionFromBitrate=INIT_VIDEO_RES_480; - }else{ - resolutionFromBitrate=INIT_VIDEO_RES_360; - } } return std::min(peerMaxVideoResolution, resolutionFromBitrate); } - -void VideoPacketSender::SendStreamCSD(){ - assert(stm->csdIsValid); - - //SendExtra(buf, EXTRA_TYPE_STREAM_CSD); -} diff --git a/video/VideoPacketSender.h b/video/VideoPacketSender.h index eab5dfb..ec58564 100644 --- a/video/VideoPacketSender.h +++ b/video/VideoPacketSender.h @@ -42,7 +42,6 @@ namespace tgvoip{ void SendFrame(const Buffer& frame, uint32_t flags, uint32_t rotation); int GetVideoResolutionForCurrentBitrate(); - void SendStreamCSD(); VideoSource* source=NULL; std::shared_ptr stm; @@ -55,6 +54,7 @@ namespace tgvoip{ uint32_t videoPacketLossCount=0; uint32_t currentVideoBitrate=0; double lastVideoResolutionChangeTime=0.0; + double sourceChangeTime=0.0; std::vector packetsForFEC; size_t fecFrameCount=0;