Welcome Guest! To enable all features, please Login or Register.

Notification

Icon
Error

Options
View
Last Go to last post Unread Go to first unread post
#1 Posted : Tuesday, February 21, 2017 7:02:00 AM(UTC)
wong fei

Groups: Registered
Posts: 8


Hi, I'm integrating IltmmCapture into game engine for rendering camera video to texture.

Sometimes on big camera resolutions (>=1080p) the delay between ReceiveProc calls may rise up to 300 milliseconds.

But I can't reproduce that problem using capture_original.exe from examples, seems the problem in my code:

Code:

		HRESULT hr = CoCreateInstance(rclsid, NULL, CLSCTX_INPROC_SERVER, riid, (void**)&this->CaptureInterface);
		HR_GUARD_RET(hr, TEXT("CoCreateInstance"));
		
		this->WindowName = FString::Printf(TEXT("LtmmCaptureWindow%X"), this);
		this->Window = CreateCaptureWindow(*this->WindowName, 32, 32, CaptureWindowProc);
		if (!this->Window)
		{
			UE_LOG(LtmmLog, Error, TEXT("CreateCaptureWindow failed"));
			break;
		}

		SetWindowLongPtr((HWND)this->Window, DWLP_USER, (LONG_PTR)this);

		this->CaptureInterface->SetNotifyWindow((long)(UINT64)this->Window, WM_CAPTURENOTIFY);
		this->CaptureInterface->put_VideoWindowFrame(0);
		this->CaptureInterface->put_PreferredVideoRenderer(ltmmVideoRenderer_NULL);

		this->VideoTexture = new UVideoTexture();
		this->UserCallback = new LMVUserCallbackImpl(this);

		// Setup Video Processors

		HRESULT hr = RemoveVideoProcessors(this->CaptureInterface);
		HR_GUARD_BREAK(hr, TEXT("RemoveVideoProcessors"));

		long ProcessorId = 0;

		// LEAD Video RGB Converter
		#if 1
		{
			ScopedComPtr<IltmmProcessor> Processor;

			hr = FindVideoProcessor(this->CaptureInterface, 
				L"@device:sw:{E526D606-22E7-494C-B81E-AC0A94BFE603}\\{E2B7DB22-38C5-11D5-91F6-00104BDB8FF9}", 
				&Processor);
			HR_GUARD_BREAK(hr, TEXT("FindVideoProcessor"));

			hr = AddVideoProcessor(this->CaptureInterface, Processor, ProcessorId);
			HR_GUARD_BREAK(hr, TEXT("AddVideoProcessor"));

			ScopedComPtr<IUnknown> ProcessorObject;
			hr = GetSelectedVideoProcessor(this->CaptureInterface, ProcessorId, &ProcessorObject);
			HR_GUARD_BREAK(hr, TEXT("GetSelectedVideoProcessor"));

			ScopedComPtr<ILMVConvRGB> ProcessorInterface;
			hr = ProcessorObject->QueryInterface(IID_ILMVConvRGB, (void**)&ProcessorInterface);
			HR_GUARD_BREAK(hr, TEXT("QueryInterface IID_ILMVConvRGB"));

			hr = ProcessorInterface->SetOutputType(LM_MEDIATYPE_RGB32, false);
			HR_GUARD_BREAK(hr, TEXT("SetOutputType"));

			hr = ProcessorInterface->SetRGBOrder(false);
			HR_GUARD_BREAK(hr, TEXT("SetRGBOrder"));

			++ProcessorId;
		}
		#endif

		// LEAD Video Callback Filter (2.0)
		{
			ScopedComPtr<IltmmProcessor> Processor;

			hr = FindVideoProcessor(this->CaptureInterface, 
				L"@device:sw:{E526D606-22E7-494C-B81E-AC0A94BFE603}\\{E2B7DDB0-38C5-11D5-91F6-00104BDB8FF9}", 
				&Processor);
			HR_GUARD_BREAK(hr, TEXT("FindVideoProcessor"));

			hr = AddVideoProcessor(this->CaptureInterface, Processor, ProcessorId);
			HR_GUARD_BREAK(hr, TEXT("AddVideoProcessor"));

			ScopedComPtr<IUnknown> ProcessorObject;
			hr = GetSelectedVideoProcessor(this->CaptureInterface, ProcessorId, &ProcessorObject);
			HR_GUARD_BREAK(hr, TEXT("GetSelectedVideoProcessor"));

			hr = ProcessorObject->QueryInterface(IID_ILMVCallback2, (void**)&this->VideoCallback);
			HR_GUARD_BREAK(hr, TEXT("QueryInterface IID_ILMVCallback2"));

			this->VideoCallback->put_CallInSameThread(VARIANT_FALSE);
			this->VideoCallback->put_ReceiveProcObj(this->UserCallback);

			++ProcessorId;
		}

		HRESULT hr = this->CaptureInterface->StartCapture(ltmmCapture_Mode_Video);
		HR_GUARD_RET(hr, TEXT("IltmmCapture::StartCapture"));
		
		// SUCCESS

		
		
struct LMVUserCallbackImpl : public ILMVUserCallback2
{
	IVideoFrameHandler* Context = nullptr;
	double LastFrameTime = 0;

	LMVUserCallbackImpl(IVideoFrameHandler* ContextPtr) : Context(ContextPtr) { }

	STDMETHODIMP QueryInterface(REFIID riid, void** ppv) 
	{
		if (!ppv) return E_POINTER;

		if (riid == IID_ILMVUserCallback2)
		{
			*ppv = (ILMVUserCallback2*)this;
			return S_OK;
		}

		return E_NOINTERFACE;
	}

	STDMETHODIMP_(ULONG) AddRef() { return S_OK; }
	STDMETHODIMP_(ULONG) Release() { return S_OK; }

	STDMETHODIMP ReceiveProc(LONGLONG pData, LONG lWidth, LONG lHeight, LONG lBitCount, LONG lSize, LONG bTopDown)
	{
		#if 1
		const double ticks = FPlatformTime::Seconds();
		if (this->LastFrameTime > 0)
		{
			double delta = ticks - this->LastFrameTime;
			if (delta > (1 / 10.0)) // at least 10 FPS
			{
				UE_LOG(LtmmLog, LTMM_LOG_LEVEL, TEXT("LAG: ILMVUserCallback %.5f ms"), delta * 1000);
			}
		}
		this->LastFrameTime = ticks;
		#endif

		if (Context)
		{
			// disabled for lag test
			//Context->OnVideoFrame((void*)pData, (size_t)lSize, (int)lWidth, (int)lHeight, (int)lBitCount, (int)bTopDown);
		}
		return S_OK;
	}
};


Code:

[2017.02.20-20.28.15:176][162]LtmmLog:Warning: delta 24.51499
[2017.02.20-20.28.15:200][164]LtmmLog:Warning: delta 24.17107
[2017.02.20-20.28.15:224][166]LtmmLog:Warning: delta 23.74079
[2017.02.20-20.28.15:250][168]LtmmLog:Warning: delta 25.72558
[2017.02.20-20.28.15:273][170]LtmmLog:Warning: delta 23.20200
[2017.02.20-20.28.15:297][172]LtmmLog:Warning: delta 23.82831
[2017.02.20-20.28.15:320][174]LtmmLog:Warning: delta 23.54651
[2017.02.20-20.28.15:344][176]LtmmLog:Warning: delta 23.28310
[2017.02.20-20.28.15:379][179]LtmmLog:Warning: delta 35.22161
[2017.02.20-20.28.15:679][205]LtmmLog:Warning: delta 299.94842 // ???
[2017.02.20-20.28.16:045][237]LtmmLog:Warning: delta 366.40164 // ???
[2017.02.20-20.28.16:323][261]LtmmLog:Warning: delta 278.15159 // ???
[2017.02.20-20.28.16:464][273]LtmmLog:Warning: delta 140.28785 // ???
[2017.02.20-20.28.16:491][275]LtmmLog:Warning: delta 27.60419
[2017.02.20-20.28.16:517][277]LtmmLog:Warning: delta 25.55668
[2017.02.20-20.28.16:541][279]LtmmLog:Warning: delta 23.98962
[2017.02.20-20.28.16:565][281]LtmmLog:Warning: delta 24.40501
[2017.02.20-20.28.16:591][283]LtmmLog:Warning: delta 26.06368
[2017.02.20-20.28.16:617][285]LtmmLog:Warning: delta 25.27460
[2017.02.20-20.28.16:640][287]LtmmLog:Warning: delta 23.53105
[2017.02.20-20.28.16:664][289]LtmmLog:Warning: delta 24.20724
 

Try the latest version of LEADTOOLS for free for 60 days by downloading the evaluation: https://www.leadtools.com/downloads

Wanna join the discussion? Login to your LEADTOOLS Support accountor Register a new forum account.

#2 Posted : Friday, February 24, 2017 12:59:08 PM(UTC)

Walter  
Walter

Groups: Tech Support, Administrators
Posts: 320

Was thanked: 3 time(s) in 3 post(s)

Hello Wong,

The LEADTOOLS main capture demo does not implement the ReceiveProc for the video callback. If you disable The video callback in your application, do you see any issues? Is it only when you enable the video callback in your application you see the problem?

Unfortunately, none of the other shipping LEADTOOLS demos that utilize the video callback are designed with the capture control in mind. Theya all use the play control. So there isn't a direct 1-to-1 demo you could try here.

It looks like your log is writting out to hard disk. If you write the log to the screen, do you see the same issue? Perhaps part of the problem is that you're using the hard disk and it may be taking a long time to write to it given whatever else is happening on the machine. Since the main capture demo also isn't also doing this, that may be why it is not displaying the same issue.

Your stated output here does not make much sense given the code you have provided. You code would indicate the log should be like this:
[2017.02.20-20.28.15:379][179]LAG: ILMVUserCallback 35.22161 ms
[2017.02.20-20.28.15:679][205]LAG: ILMVUserCallback 299.94842 ms
[2017.02.20-20.28.16:045][237]LAG: ILMVUserCallback 366.40164 ms
[2017.02.20-20.28.16:323][261]LAG: ILMVUserCallback 278.15159 ms
[2017.02.20-20.28.16:464][273]LAG: ILMVUserCallback 140.28785 ms
[2017.02.20-20.28.16:491][275]LAG: ILMVUserCallback 27.60419 ms


But what you've noted is this:
[2017.02.20-20.28.15:379][179]LtmmLog:Warning: delta 35.22161
[2017.02.20-20.28.15:679][205]LtmmLog:Warning: delta 299.94842 // ???
[2017.02.20-20.28.16:045][237]LtmmLog:Warning: delta 366.40164 // ???
[2017.02.20-20.28.16:323][261]LtmmLog:Warning: delta 278.15159 // ???
[2017.02.20-20.28.16:464][273]LtmmLog:Warning: delta 140.28785 // ???
[2017.02.20-20.28.16:491][275]LtmmLog:Warning: delta 27.60419


Perhaps you copied these at different times and that's why the text doesn't match up. Whatever the case is, it caught my eye too.

Can you also provide more details about the camera input? Is this a local hardware device you are capturing from to render in game? Can you tell me what make/model is of the device so I have a better idea of your data stream?
Walter Bates
Senior Support Engineer
LEAD Technologies, Inc.
LEAD Logo
 
#3 Posted : Saturday, February 25, 2017 6:12:29 AM(UTC)
wong fei

Groups: Registered
Posts: 8


I tested other camera models and they worked well, seems it was the specific camera problem (Mobius ActionCam).
 
#4 Posted : Saturday, February 25, 2017 9:11:51 AM(UTC)
wong fei

Groups: Registered
Posts: 8


Now I'm using ILMVCallback2 and the goal is accessing in realtime (with minimal latency) each captured frame raw pixels. Is there a better solution?

Also is there a way to disable capture file?

I tried using IltmmCapture::put_TargetType(ltmmCapture_Target_Stream) with dummy IStream, it works, but hangs when calling StopCapture.

Edited by user Sunday, February 26, 2017 1:13:30 PM(UTC)  | Reason: update message

 
#5 Posted : Monday, February 27, 2017 8:57:03 PM(UTC)

Walter  
Walter

Groups: Tech Support, Administrators
Posts: 320

Was thanked: 3 time(s) in 3 post(s)

The video callback is the fastest way to get access to the raw bitmap data of every frame. The callback can be put right after the video decoder, so it is the fastest.

To avoid creating a file, you can try one of the following:
  • IltmmCapture::put_TargetType(ltmmCapture_TargetFormat_NULL);
  • IltmmCapture::put_TargetFile(L"C:\\NUL");


Setting the target format should cause null renderers to be used, and no sink filter would be needed. If that doesn't work though, you can write out to C:\NUL. This is a special DOS "throw away" file name.
Walter Bates
Senior Support Engineer
LEAD Technologies, Inc.
LEAD Logo
 
#6 Posted : Tuesday, February 28, 2017 10:18:11 AM(UTC)
wong fei

Groups: Registered
Posts: 8


Before using IStream tried setting file to NUL or different TargetType / TargetFormat options, but it doesn't help:

===

hr = Capture->put_TargetType(ltmmCapture_Target_None);
hr = Capture->put_TargetFormat(ltmmCapture_TargetFormat_NULL);

Error: StartCapture: CODE=80004005 (Unknown)

===

hr = Capture->put_TargetType(ltmmCapture_Target_File);
hr = Capture->put_TargetFormat(ltmmCapture_TargetFormat_NULL);
hr = Capture->put_TargetFile(L"C:\\NUL"); // neither NUL

Error: StartCapture: CODE=80070002 (File not found)

===

hr = Capture->put_TargetType(ltmmCapture_Target_None);
hr = Capture->put_TargetFormat(ltmmCapture_TargetFormat_NULL);
hr = Capture->put_TargetFile(L"C:\\tmp\\dummy");

OK: writes to file "C:\\tmp\\dummy"

===

hr = Capture->put_TargetType(ltmmCapture_Target_None);
hr = Capture->put_TargetFormat(ltmmCapture_TargetFormat_NULL);
hr = Capture->put_TargetFile(0);

Error: StartCapture: CODE=80070003 (Invalid path)

===

hr = Capture->put_TargetFormat(ltmmCapture_TargetFormat_NULL);

OK: writes to file "c:\Users\username\Documents\capture.avi"
 
#7 Posted : Wednesday, March 1, 2017 3:40:54 PM(UTC)

Aaron  
Aaron

Groups: Registered, Tech Support, Administrators
Posts: 71

Was thanked: 3 time(s) in 3 post(s)

Another option may be, if you don't actually need to save the video to an output file and are only using the capture control to get the frames from the capture device, then you should never need to actually call StartCapture or set a TargetType/TargetFormat/TargetFile. You could just set IltmmCapture::put_Preview to true to get the frames then false to stop getting the frames:

https://www.leadtools.co...capture-put-preview.html

The preview setting would force the capture control to begin getting the frames from the capture device (to show a preview) without actually starting the save to disk of any data. You don't necessarily need to display the preview, but just starting the preview should begin the capture of the frames from the capture device. Since the frames are now passing through the control via the preview option (the callback filter fires any time a frame passes through the control), the callback filter should fire giving you the frames themselves.
Aaron Brasington
Developer Support Engineer
LEAD Technologies, Inc.

LEAD Logo
 
#8 Posted : Thursday, March 2, 2017 3:39:09 PM(UTC)
wong fei

Groups: Registered
Posts: 8


Thanks, put_Preview is what I looked for.
 
You cannot post new topics in this forum.
You cannot reply to topics in this forum.
You cannot delete your posts in this forum.
You cannot edit your posts in this forum.
You cannot create polls in this forum.
You cannot vote in polls in this forum.

Powered by YAF.NET | YAF.NET © 2003-2017, Yet Another Forum.NET
This page was generated in 0.296 seconds.