Skip to content

Commit

Permalink
Improved HoloLens performance for stability.
Browse files Browse the repository at this point in the history
  • Loading branch information
@phongcao
phongcao committed Jun 8, 2018
1 parent 0902baf commit 55c5915
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 11 deletions.
53 changes: 47 additions & 6 deletions org/webrtc/wrapper/MediaSourceHelper.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@
#include "webrtc/modules/video_coding/timing.h"
#include "third_party/winuwp_h264/H264Decoder/H264Decoder.h"

#define MAX_PREDICTION_TIMESTAMP 256
#define DISABLE_DROP_FRAMES
#define ENABLE_SAMPLE_TIME_CALCULATION

#define MAX_PREDICTION_TIMESTAMP 256
#define MIN_SAMPLE_DURATION 5 * 10000

using Microsoft::WRL::ComPtr;
using Platform::Collections::Vector;
Expand All @@ -37,7 +41,6 @@ namespace Org {
bool IsSampleIDR(IMFSample* sample);
bool DropFramesToIDR(std::list<webrtc::VideoFrame*>& frames);


SampleData::SampleData()
: sizeHasChanged(false)
, size({ -1, -1 })
Expand All @@ -53,7 +56,7 @@ namespace Org {
, _fpsCallback(fpsCallback)
, _frameType(frameType)
, _isFirstFrame(true)
, _futureOffsetMs(45)
, _futureOffsetMs(25)
, _lastSampleTime(0)
, _lastSize({ 0, 0 })
, _lastRotation(-1)
Expand Down Expand Up @@ -100,6 +103,17 @@ namespace Org {
delete frame;
}
}

#ifdef ENABLE_SAMPLE_TIME_CALCULATION
if (_lastSampleTime == 0) {
frame->set_ntp_time_ms(_futureOffsetMs);
}
else {
frame->set_ntp_time_ms(rtc::TimeMillis() - _lastSampleTime);
}

_lastSampleTime = rtc::TimeMillis();
#endif //ENABLE_SAMPLE_TIME_CALCULATION
}

std::unique_ptr<SampleData> MediaSourceHelper::DequeueFrame() {
Expand All @@ -122,14 +136,30 @@ namespace Org {
_isFirstFrame = false;
Org::WebRtc::FirstFrameRenderHelper::FireEvent(
rtc::TimeMillis() * rtc::kNumMillisecsPerSec);
#ifdef ENABLE_SAMPLE_TIME_CALCULATION
_frameTimeTotal = data->duration;
data->sample->SetSampleTime(_frameTimeTotal);
#else // ENABLE_SAMPLE_TIME_CALCULATION
LONGLONG frameTime = GetNextSampleTimeHns(data->renderTime, _frameType == FrameTypeH264);
data->sample->SetSampleTime(frameTime);
#endif // ENABLE_SAMPLE_TIME_CALCULATION
}
else {
#ifdef ENABLE_SAMPLE_TIME_CALCULATION
LONGLONG sampleDuration = data->duration / (_frames.size() + 1);
if (sampleDuration < MIN_SAMPLE_DURATION) {
sampleDuration = MIN_SAMPLE_DURATION;
}

//OutputDebugString((L"_frames.size(): " + _frames.size() + "\r\n")->Data());
//OutputDebugString((L"sampleDuration: " + (sampleDuration / 10000) + "\r\n")->Data());
_frameTimeTotal += sampleDuration;
LONGLONG frameTime = _frameTimeTotal;
#else // ENABLE_SAMPLE_TIME_CALCULATION
LONGLONG frameTime = GetNextSampleTimeHns(data->renderTime, _frameType == FrameTypeH264);
#endif // ENABLE_SAMPLE_TIME_CALCULATION

if (data->predictionTimestamp > 0)
{
if (data->predictionTimestamp > 0) {
// Injects prediction timestamp id so that we can parse it in
// IMFMediaEngine::OnVideoStreamTick() later.
data->predictionTimestampId = ++_timestampCounter % MAX_PREDICTION_TIMESTAMP;
Expand All @@ -140,13 +170,19 @@ namespace Org {

// Set the duration property
if (_frameType == FrameTypeH264) {
#ifdef ENABLE_SAMPLE_TIME_CALCULATION
data->sample->SetSampleDuration(sampleDuration);
#else // ENABLE_SAMPLE_TIME_CALCULATION
data->sample->SetSampleDuration(frameTime - _lastSampleTime);
#endif // ENABLE_SAMPLE_TIME_CALCULATION
}
else {
LONGLONG duration = (LONGLONG)((1.0 / 30) * 1000 * 1000 * 10);
data->sample->SetSampleDuration(duration);
}
#ifndef ENABLE_SAMPLE_TIME_CALCULATION
_lastSampleTime = frameTime;
#endif // #ifdef ENABLE_SAMPLE_TIME_CALCULATION
}
UpdateFrameRate();

Expand All @@ -167,14 +203,19 @@ namespace Org {

std::unique_ptr<SampleData> MediaSourceHelper::DequeueH264Frame() {

#ifndef DISABLE_DROP_FRAMES
if (_frames.size() > 15)
DropFramesToIDR(_frames);
#endif // DISABLE_DROP_FRAMES

std::unique_ptr<webrtc::VideoFrame> frame(_frames.front());
_frames.pop_front();

std::unique_ptr<SampleData> data(new SampleData);
data->predictionTimestamp = frame->prediction_timestamp();
#ifdef ENABLE_SAMPLE_TIME_CALCULATION
data->duration = frame->ntp_time_ms() * 10000;
#endif // ENABLE_SAMPLE_TIME_CALCULATION

// Get the IMFSample in the frame.
{
Expand Down Expand Up @@ -364,11 +405,11 @@ namespace Org {
void MediaSourceHelper::UpdateFrameRate() {
// Do FPS calculation and notification.
_frameCounter++;

// If we have about a second worth of frames
int64_t now = rtc::TimeMillis();
if ((now - _lastTimeFPSCalculated) > 1000) {
_fpsCallback(_frameCounter);

_frameCounter = 0;
_lastTimeFPSCalculated = now;
}
Expand Down
5 changes: 1 addition & 4 deletions org/webrtc/wrapper/MediaSourceHelper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ namespace Org {
LONGLONG renderTime;
LONGLONG predictionTimestamp;
uint8_t predictionTimestampId;
LONGLONG duration;
};

class MediaSourceHelper {
Expand Down Expand Up @@ -111,11 +112,7 @@ namespace Org {

// Prediction timestamp.
int _timestampCounter;
LONGLONG _lastSampleDuration;
LONGLONG _frameTimeTotal;
LONGLONG lastTimestampHns_;
int64_t _freezeTickTime;
bool _isFreezing;
};
}
}
Expand Down
1 change: 0 additions & 1 deletion org/webrtc/wrapper/RTMediaStreamSource.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ using Windows::System::Threading::TimerElapsedHandler;
using Windows::System::Threading::ThreadPoolTimer;

#define INITIAL_FRAME_RATE 30
#define INITIAL_SKIPPED_TIME 3000

//namespace Org {
// namespace WebRtc {
Expand Down

10 comments on commit 55c5915

@rishi-rranjan
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ I tested this with Release 2.0 but I don't see any improvement in Hololens stability. The Fractal server example still shakes a lot with this change also. Have you seen any noticeable stability with his changes?

I have not looked at at the end to end delay yet.

@phongcao
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rishi-rranjan Yes, it's more stable as we've tested on our side. We tried running the server on both local machine and Azure VM and the latency is always less than 150ms. What are the latency/fps that you see on your side?

@rishi-rranjan
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@phongcao We are testing it in on Prem scenario so our network delay is not an issue. What we have observed is following:

  • When we move our head, it takes almost 2-3 seconds before view gets updated. FPS on Hololens client drops to less than 10 FPS.
  • We are expecting an issue on client side as we see that server FPS is above 50 FPS.
  • We will have to do some more testing on our side to pinpoint the issue.

@phongcao
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rishi-rranjan Can you enable SHOW_DEBUG_INFO flag in VideoRenderer.h and let me know the latency? Can you also try running the DirectX server samples and tell me the numbers?

@rishi-rranjan
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@phongcao My Hololens upgraded to RS4 yesterday and now my client keeps crashing on Hololens. Will send the logs as soon as I figure out the crash issue.

Here is the crash logs in case you have any idea:

avcore\audiocore\client\audioclient\audioclientcore.cpp(1477)\AUDIOSES.DLL!68DFBBA3: (caller: 68DD624A) ReturnHr(1) tid(4b0) 80070490 Element not found.
(webrtcvoiceengine.cc:702): webrtc: IAudioClient::Initialize() failed:
(webrtcvoiceengine.cc:702): webrtc: no format suggested
(audio_device_wasapi_win.cc:679): Failed to configure input audio device after activate, hr=0x88890008
'StreamingDirectxHololensClient.exe' (Win32): Loaded 'C:\Windows\System32\userenv.dll'. Skipped loading symbols. Module is native, and native debugging is currently disabled.
'StreamingDirectxHololensClient.exe' (Win32): Loaded 'C:\Windows\System32\profapi.dll'. Skipped loading symbols. Module is native, and native debugging is currently disabled.
'StreamingDirectxHololensClient.exe' (Win32): Loaded 'C:\Windows\System32\profext.dll'. Skipped loading symbols. Module is native, and native debugging is currently disabled.
Debug Error!

Program: ...a49aVS.Debug_x86.VenkyDass\StreamingDirectxHololensClient.exe

abort() has been called

@phongcao
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This PR should fix that issue.

@rishi-rranjan
Copy link

@rishi-rranjan rishi-rranjan commented on 55c5915 Sep 18, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@phongcao Please find the attached video capture for FPS and Delay. This setup is running over local LAN and server is sitting right behind the WiFi AP. WiFi AP is a Linksys WRT 1900 ACS with Hololens as only client. Server is connected to router using ethernet.

Even with local LAN I don't see the delay lower than 230 ms. In this case delay is 330 ms.

Also I have confirmed no other WiFi is interfering on this channel.

capture

@phongcao
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rishi-rranjan Feel free to create an issue about it and provide us more info.

@rishi-rranjan
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@phongcao I will open an issue once I have some more idea on what's going wrong. I have isolated the issue to Hololens. I tried native app on Windows Laptop as streaming client and then Hololens UWP app back to back. Windows laptop shows consistent 20+FPS stream. I analyzed the Wireshark capture on Windows laptop and traffic is completely clean and jitter free so I have ruled out any issues with my router.

Since there is no way to run Wireshark on Hololens, I am looking at WebRTC "RTC Stats" to capture the packet headers to rule out any radio interface issue with Hololens.

I will also add some debug prints around DISABLE_DROP_FRAMES. Any reason why this was added in an interactive application? IDR will refresh the decoder so we can drop the frames if there is a long queue build up.

@phongcao
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rishi-rranjan On HoloLens, we're using IMFMediaEngine for h264 decoder and it doesn't work well if we keep dropping IDR frame. Regarding your concern about the queue, it won't build up since we calculate the sample duration basing on the current length of the queue. If you know a better way to fix it after debugging, please let us know.

Please sign in to comment.