From f87d5362291dbe80bdfd9ba6ecbf827f033800b2 Mon Sep 17 00:00:00 2001 From: "Ryan C. Gordon" Date: Sat, 16 Dec 2023 16:00:15 -0500 Subject: [PATCH] camera: Added more accurate timestamps. --- src/camera/SDL_camera.c | 18 +++++++++++++++++- src/camera/SDL_syscamera.h | 11 ++++++++++- src/camera/v4l2/SDL_camera_v4l2.c | 7 +++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/camera/SDL_camera.c b/src/camera/SDL_camera.c index d9c581b04..6cbd45362 100644 --- a/src/camera/SDL_camera.c +++ b/src/camera/SDL_camera.c @@ -109,6 +109,9 @@ static void ClosePhysicalCameraDevice(SDL_CameraDevice *device) device->filled_output_surfaces.next = NULL; device->empty_output_surfaces.next = NULL; device->app_held_output_surfaces.next = NULL; + + device->base_timestamp = 0; + device->adjust_timestamp = 0; } // this must not be called while `device` is still in a device list, or while a device's camera thread is still running. @@ -535,7 +538,12 @@ SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device) SDL_Log("CAMERA: New frame available!"); #endif - if (device->empty_output_surfaces.next == NULL) { + if (device->drop_frames > 0) { + device->drop_frames--; + camera_driver.impl.ReleaseFrame(device, device->acquire_surface); + device->acquire_surface->pixels = NULL; + device->acquire_surface->pitch = 0; + } else if (device->empty_output_surfaces.next == NULL) { // uhoh, no output frames available! Either the app is slow, or it forgot to release frames when done with them. Drop this new frame. #if DEBUG_CAMERA SDL_Log("CAMERA: No empty output surfaces! Dropping frame!"); @@ -544,6 +552,12 @@ SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device) device->acquire_surface->pixels = NULL; device->acquire_surface->pitch = 0; } else { + if (!device->adjust_timestamp) { + device->adjust_timestamp = SDL_GetTicksNS(); + device->base_timestamp = timestampNS; + } + timestampNS = (timestampNS - device->base_timestamp) + device->adjust_timestamp; + slist = device->empty_output_surfaces.next; output_surface = slist->surface; device->empty_output_surfaces.next = slist->next; @@ -828,6 +842,8 @@ SDL_Camera *SDL_OpenCameraDevice(SDL_CameraDeviceID instance_id, const SDL_Camer device->output_surfaces[i].surface = surf; } + device->drop_frames = 1; + // Start the camera thread if necessary if (!camera_driver.impl.ProvidesOwnCallbackThread) { char threadname[64]; diff --git a/src/camera/SDL_syscamera.h b/src/camera/SDL_syscamera.h index a48a15298..27672890b 100644 --- a/src/camera/SDL_syscamera.h +++ b/src/camera/SDL_syscamera.h @@ -25,7 +25,7 @@ #include "../SDL_hashtable.h" -#define DEBUG_CAMERA 1 +#define DEBUG_CAMERA 0 // !!! FIXME: update these drivers! @@ -92,6 +92,15 @@ struct SDL_CameraDevice // Driver-specific hardware data on how to open device (`hidden` is driver-specific data _when opened_). void *handle; + // Dropping the first frame(s) after open seems to help timing on some platforms. + int drop_frames; + + // Backend timestamp of first acquired frame, so we can keep these meaningful regardless of epoch. + Uint64 base_timestamp; + + // SDL timestamp of first acquired frame, so we can roughly convert to SDL ticks. + Uint64 adjust_timestamp; + // Pixel data flows from the driver into these, then gets converted for the app if necessary. SDL_Surface *acquire_surface; diff --git a/src/camera/v4l2/SDL_camera_v4l2.c b/src/camera/v4l2/SDL_camera_v4l2.c index 0eebc7196..c43762a46 100644 --- a/src/camera/v4l2/SDL_camera_v4l2.c +++ b/src/camera/v4l2/SDL_camera_v4l2.c @@ -67,7 +67,6 @@ struct SDL_PrivateCameraData io_method io; int nb_buffers; struct buffer *buffers; - int first_start; int driver_pitch; }; @@ -129,6 +128,7 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6 } } + *timestampNS = SDL_GetTicksNS(); // oh well, close enough. frame->pixels = device->hidden->buffers[0].start; frame->pitch = device->hidden->driver_pitch; break; @@ -161,6 +161,8 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6 frame->pitch = device->hidden->driver_pitch; device->hidden->buffers[buf.index].available = 1; + *timestampNS = (((Uint64) buf.timestamp.tv_sec) * SDL_NS_PER_SECOND) + SDL_US_TO_NS(buf.timestamp.tv_usec); + #if DEBUG_CAMERA SDL_Log("CAMERA: debug mmap: image %d/%d data[0]=%p", buf.index, device->hidden->nb_buffers, (void*)frame->pixels); #endif @@ -202,6 +204,8 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6 frame->pitch = device->hidden->driver_pitch; device->hidden->buffers[i].available = 1; + *timestampNS = (((Uint64) buf.timestamp.tv_sec) * SDL_NS_PER_SECOND) + SDL_US_TO_NS(buf.timestamp.tv_usec); + #if DEBUG_CAMERA SDL_Log("CAMERA: debug userptr: image %d/%d data[0]=%p", buf.index, device->hidden->nb_buffers, (void*)frame->pixels); #endif @@ -212,7 +216,6 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6 break; } - *timestampNS = SDL_GetTicksNS(); // !!! FIXME: can we get this info more accurately from v4l2? return 1; }