From 47f880407307a36dc7356412591a2019fbd0ff54 Mon Sep 17 00:00:00 2001 From: Armin Novak Date: Thu, 4 Oct 2018 12:21:28 +0200 Subject: [PATCH 1/2] Fixed audio recording with opensles. --- .../audin/client/opensles/audin_opensl_es.c | 143 ++---------- channels/audin/client/opensles/opensl_io.c | 210 +++++++++--------- channels/audin/client/opensles/opensl_io.h | 41 +--- 3 files changed, 126 insertions(+), 268 deletions(-) diff --git a/channels/audin/client/opensles/audin_opensl_es.c b/channels/audin/client/opensles/audin_opensl_es.c index 5b89e7e7e..6838b1b3a 100644 --- a/channels/audin/client/opensles/audin_opensl_es.c +++ b/channels/audin/client/opensles/audin_opensl_es.c @@ -29,8 +29,6 @@ #include #include -#include -#include #include #include @@ -56,85 +54,29 @@ typedef struct _AudinOpenSLESDevice AudinReceive receive; - HANDLE thread; - HANDLE stopEvent; - void* user_data; rdpContext* rdpcontext; wLog* log; } AudinOpenSLESDevice; -static DWORD WINAPI audin_opensles_thread_func(LPVOID arg) +static UINT audin_opensles_close(IAudinDevice* device); + +static void audin_receive(void* context, const void* data, size_t size) { - union + UINT error; + AudinOpenSLESDevice* opensles = (AudinOpenSLESDevice*) context; + + if (!opensles || !data) { - void* v; - short* s; - BYTE* b; - } buffer; - AudinOpenSLESDevice* opensles = (AudinOpenSLESDevice*) arg; - const size_t raw_size = opensles->frames_per_packet * opensles->bytes_per_channel; - int rc = CHANNEL_RC_OK; - UINT error = CHANNEL_RC_OK; - DWORD status; - WLog_Print(opensles->log, WLOG_DEBUG, "opensles=%p", (void*) opensles); - assert(opensles); - assert(opensles->frames_per_packet > 0); - assert(opensles->stopEvent); - assert(opensles->stream); - buffer.v = calloc(1, raw_size); - - if (!buffer.v) - { - error = CHANNEL_RC_NO_MEMORY; - WLog_Print(opensles->log, WLOG_ERROR, "calloc failed!"); - - if (opensles->rdpcontext) - setChannelError(opensles->rdpcontext, CHANNEL_RC_NO_MEMORY, - "audin_opensles_thread_func reported an error"); - - goto out; + WLog_ERR(TAG, "[%s] Invalid arguments context=%p, data=%p", __FUNCTION__, opensles, data); + return; } - while (1) - { - status = WaitForSingleObject(opensles->stopEvent, 0); - - if (status == WAIT_FAILED) - { - error = GetLastError(); - WLog_Print(opensles->log, WLOG_ERROR, "WaitForSingleObject failed with error %"PRIu32"!", error); - break; - } - - if (status == WAIT_OBJECT_0) - break; - - rc = android_RecIn(opensles->stream, buffer.s, raw_size); - - if (rc < 0) - { - WLog_Print(opensles->log, WLOG_ERROR, "android_RecIn %d", rc); - continue; - } - - error = opensles->receive(&opensles->format, - buffer.v, raw_size, opensles->user_data); - - if (error) - break; - } - - free(buffer.v); -out: - WLog_Print(opensles->log, WLOG_DEBUG, "thread shutdown."); + error = opensles->receive(&opensles->format, data, size, opensles->user_data); if (error && opensles->rdpcontext) - setChannelError(opensles->rdpcontext, error, "audin_opensles_thread_func reported an error"); - - ExitThread(error); - return error; + setChannelError(opensles->rdpcontext, error, "audin_receive reported an error"); } /** @@ -156,8 +98,6 @@ static UINT audin_opensles_free(IAudinDevice* device) if (!opensles) return CHANNEL_RC_OK; - assert(opensles); - assert(!opensles->stream); free(opensles->device_name); free(opensles); return CHANNEL_RC_OK; @@ -274,46 +214,26 @@ static UINT audin_opensles_open(IAudinDevice* device, AudinReceive receive, WLog_Print(opensles->log, WLOG_DEBUG, "device=%p, receive=%p, user_data=%p", (void*) device, (void*) receive, (void*) user_data); - assert(opensles); - /* The function may have been called out of order, - * ignore duplicate open requests. */ if (opensles->stream) - return CHANNEL_RC_OK; + goto error_out; if (!(opensles->stream = android_OpenRecDevice( - opensles->device_name, + opensles, audin_receive, opensles->format.nSamplesPerSec, opensles->format.nChannels, opensles->frames_per_packet, opensles->format.wBitsPerSample))) { WLog_Print(opensles->log, WLOG_ERROR, "android_OpenRecDevice failed!"); - return ERROR_INTERNAL_ERROR; + goto error_out; } opensles->receive = receive; opensles->user_data = user_data; - - if (!(opensles->stopEvent = CreateEvent(NULL, TRUE, FALSE, NULL))) - { - WLog_Print(opensles->log, WLOG_ERROR, "CreateEvent failed!"); - goto error_out; - } - - if (!(opensles->thread = CreateThread(NULL, 0, - audin_opensles_thread_func, opensles, 0, NULL))) - { - WLog_Print(opensles->log, WLOG_ERROR, "CreateThread failed!"); - goto error_out; - } - return CHANNEL_RC_OK; error_out: - android_CloseRecDevice(opensles->stream); - opensles->stream = NULL; - CloseHandle(opensles->stopEvent); - opensles->stopEvent = NULL; + audin_opensles_close(opensles); return ERROR_INTERNAL_ERROR; } @@ -322,38 +242,15 @@ error_out: * * @return 0 on success, otherwise a Win32 error code */ -static UINT audin_opensles_close(IAudinDevice* device) +UINT audin_opensles_close(IAudinDevice* device) { - UINT error; AudinOpenSLESDevice* opensles = (AudinOpenSLESDevice*) device; + + if (!opensles) + return ERROR_INVALID_PARAMETER; + WLog_Print(opensles->log, WLOG_DEBUG, "device=%p", (void*) device); - assert(opensles); - - /* The function may have been called out of order, - * ignore duplicate requests. */ - if (!opensles->stopEvent) - { - WLog_Print(opensles->log, WLOG_ERROR, "[ERROR] function called without matching open."); - return ERROR_REQUEST_OUT_OF_SEQUENCE; - } - - assert(opensles->stopEvent); - assert(opensles->thread); - assert(opensles->stream); - SetEvent(opensles->stopEvent); - - if (WaitForSingleObject(opensles->thread, INFINITE) == WAIT_FAILED) - { - error = GetLastError(); - WLog_Print(opensles->log, WLOG_ERROR, "WaitForSingleObject failed with error %"PRIu32"", error); - return error; - } - - CloseHandle(opensles->stopEvent); - CloseHandle(opensles->thread); android_CloseRecDevice(opensles->stream); - opensles->stopEvent = NULL; - opensles->thread = NULL; opensles->receive = NULL; opensles->user_data = NULL; opensles->stream = NULL; diff --git a/channels/audin/client/opensles/opensl_io.c b/channels/audin/client/opensles/opensl_io.c index f6777665a..5e7fba603 100644 --- a/channels/audin/client/opensles/opensl_io.c +++ b/channels/audin/client/opensles/opensl_io.c @@ -34,6 +34,39 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CONV16BIT 32768 #define CONVMYFLT (1./32768.) +typedef struct +{ + size_t size; + void* data; +} queue_element; + +struct opensl_stream +{ + // engine interfaces + SLObjectItf engineObject; + SLEngineItf engineEngine; + + // device interfaces + SLDeviceVolumeItf deviceVolume; + + // recorder interfaces + SLObjectItf recorderObject; + SLRecordItf recorderRecord; + SLAndroidSimpleBufferQueueItf recorderBufferQueue; + + unsigned int inchannels; + unsigned int sr; + unsigned int buffersize; + unsigned int bits_per_sample; + + queue_element* prep; + queue_element* next; + + void* context; + opensl_receive_t receive; +}; + + static void bqRecorderCallback(SLAndroidSimpleBufferQueueItf bq, void* context); // creates the OpenSL ES audio engine @@ -236,42 +269,84 @@ static void openSLDestroyEngine(OPENSL_STREAM* p) } } +static queue_element* opensles_queue_element_new(size_t size) +{ + queue_element* q = calloc(1, sizeof(queue_element)); + + if (!q) + goto fail; + + q->size = size; + q->data = malloc(size); + + if (!q->data) + goto fail; + + return q; +fail: + free(q); + return NULL; +} + +static void opensles_queue_element_free(void* obj) +{ + queue_element* e = (queue_element*)obj; + + if (e) + free(e->data); + + free(e); +} // open the android audio device for input -OPENSL_STREAM* android_OpenRecDevice(char* name, int sr, int inchannels, +OPENSL_STREAM* android_OpenRecDevice(void* context, opensl_receive_t receive, + int sr, + int inchannels, int bufferframes, int bits_per_sample) { OPENSL_STREAM* p; + + if (!context || !receive) + return NULL; + p = (OPENSL_STREAM*) calloc(1, sizeof(OPENSL_STREAM)); if (!p) return NULL; + p->context = context; + p->receive = receive; p->inchannels = inchannels; p->sr = sr; - p->queue = Queue_New(TRUE, -1, -1); p->buffersize = bufferframes; p->bits_per_sample = bits_per_sample; if ((p->bits_per_sample != 8) && (p->bits_per_sample != 16)) - { - android_CloseRecDevice(p); - return NULL; - } + goto fail; if (openSLCreateEngine(p) != SL_RESULT_SUCCESS) - { - android_CloseRecDevice(p); - return NULL; - } + goto fail; if (openSLRecOpen(p) != SL_RESULT_SUCCESS) - { - android_CloseRecDevice(p); - return NULL; - } + goto fail; + /* Create receive buffers, prepare them and start recording */ + p->prep = opensles_queue_element_new(p->buffersize * p->bits_per_sample / 8); + p->next = opensles_queue_element_new(p->buffersize * p->bits_per_sample / 8); + + if (!p->prep || !p->next) + goto fail; + + (*p->recorderBufferQueue)->Enqueue(p->recorderBufferQueue, + p->next->data, p->next->size); + (*p->recorderBufferQueue)->Enqueue(p->recorderBufferQueue, + p->prep->data, p->prep->size); + (*p->recorderRecord)->SetRecordState(p->recorderRecord, + SL_RECORDSTATE_RECORDING); return p; +fail: + android_CloseRecDevice(p); + return NULL; } // close the android audio device @@ -280,30 +355,8 @@ void android_CloseRecDevice(OPENSL_STREAM* p) if (p == NULL) return; - if (p->queue) - { - while (Queue_Count(p->queue) > 0) - { - queue_element* e = Queue_Dequeue(p->queue); - free(e->data); - free(e); - } - - Queue_Free(p->queue); - } - - if (p->next) - { - free(p->next->data); - free(p->next); - } - - if (p->prep) - { - free(p->prep->data); - free(p->prep); - } - + opensles_queue_element_free(p->next); + opensles_queue_element_free(p->prep); openSLDestroyEngine(p); free(p); } @@ -311,86 +364,25 @@ void android_CloseRecDevice(OPENSL_STREAM* p) // this callback handler is called every time a buffer finishes recording static void bqRecorderCallback(SLAndroidSimpleBufferQueueItf bq, void* context) { - queue_element* e; OPENSL_STREAM* p = (OPENSL_STREAM*) context; - assert(p); - assert(p->next); - assert(p->prep); - assert(p->queue); - e = calloc(1, sizeof(queue_element)); + queue_element* e; + + if (!p) + return; + + e = p->next; if (!e) return; - e->data = calloc(p->buffersize, p->bits_per_sample / 8); + if (!p->context || !p->receive) + WLog_WARN(TAG, "Missing receive callback=%p, context=%p", p->receive, p->context); + else + p->receive(p->context, e->data, e->size); - if (!e->data) - { - free(e); - return; - } - - e->size = p->buffersize * p->bits_per_sample / 8; - Queue_Enqueue(p->queue, p->next); p->next = p->prep; p->prep = e; (*p->recorderBufferQueue)->Enqueue(p->recorderBufferQueue, e->data, e->size); } -// gets a buffer of size samples from the device -int android_RecIn(OPENSL_STREAM* p, short* buffer, int size) -{ - queue_element* e; - int rc; - DWORD status; - assert(p); - assert(buffer); - assert(size > 0); - - /* Initial trigger for the queue. */ - if (!p->prep) - { - p->prep = calloc(1, sizeof(queue_element)); - p->prep->data = calloc(p->buffersize, p->bits_per_sample / 8); - p->prep->size = p->buffersize * p->bits_per_sample / 8; - p->next = calloc(1, sizeof(queue_element)); - p->next->data = calloc(p->buffersize, p->bits_per_sample / 8); - p->next->size = p->buffersize * p->bits_per_sample / 8; - (*p->recorderBufferQueue)->Enqueue(p->recorderBufferQueue, - p->next->data, p->next->size); - (*p->recorderBufferQueue)->Enqueue(p->recorderBufferQueue, - p->prep->data, p->prep->size); - (*p->recorderRecord)->SetRecordState(p->recorderRecord, - SL_RECORDSTATE_RECORDING); - } - - /* Wait for queue to be filled... */ - if (!Queue_Count(p->queue)) - { - status = WaitForSingleObject(p->queue->event, INFINITE); - - if (status == WAIT_FAILED) - { - WLog_ERR(TAG, "WaitForSingleObject failed with error %"PRIu32"", GetLastError()); - return -1; - } - } - - e = Queue_Dequeue(p->queue); - - if (!e) - { - WLog_ERR(TAG, "[ERROR] got e=NULL from queue"); - return -1; - } - - rc = (e->size < size) ? e->size : size; - assert(size == e->size); - assert(p->buffersize * p->bits_per_sample / 8 == size); - memcpy(buffer, e->data, rc); - free(e->data); - free(e); - return rc; -} - diff --git a/channels/audin/client/opensles/opensl_io.h b/channels/audin/client/opensles/opensl_io.h index 259b55a81..6a54b7cce 100644 --- a/channels/audin/client/opensles/opensl_io.h +++ b/channels/audin/client/opensles/opensl_io.h @@ -33,9 +33,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#include -#include - #include #include @@ -44,51 +41,23 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. extern "C" { #endif -typedef struct -{ - size_t size; - void* data; -} queue_element; +typedef struct opensl_stream OPENSL_STREAM; -typedef struct opensl_stream -{ - // engine interfaces - SLObjectItf engineObject; - SLEngineItf engineEngine; - - // device interfaces - SLDeviceVolumeItf deviceVolume; - - // recorder interfaces - SLObjectItf recorderObject; - SLRecordItf recorderRecord; - SLAndroidSimpleBufferQueueItf recorderBufferQueue; - - unsigned int inchannels; - unsigned int sr; - unsigned int buffersize; - unsigned int bits_per_sample; - - wQueue* queue; - queue_element* prep; - queue_element* next; -} OPENSL_STREAM; +typedef void (*opensl_receive_t)(void* context, const void* data, size_t size); /* Open the audio device with a given sampling rate (sr), input and output channels and IO buffer size in frames. Returns a handle to the OpenSL stream */ -FREERDP_LOCAL OPENSL_STREAM* android_OpenRecDevice(char* name, int sr, +FREERDP_LOCAL OPENSL_STREAM* android_OpenRecDevice(void* context, + opensl_receive_t receive, int sr, int inchannels, int bufferframes, int bits_per_sample); /* Close the audio device */ FREERDP_LOCAL void android_CloseRecDevice(OPENSL_STREAM* p); -/* -Read a buffer from the OpenSL stream *p, of size samples. Returns the number of samples read. -*/ -FREERDP_LOCAL int android_RecIn(OPENSL_STREAM* p, short* buffer, int size); + #ifdef __cplusplus }; #endif From ac1476356481c385558a51ef547928800e385612 Mon Sep 17 00:00:00 2001 From: Armin Novak Date: Thu, 4 Oct 2018 13:53:18 +0200 Subject: [PATCH 2/2] [audin] Abort version exchange if server version is not supported. --- channels/audin/client/audin_main.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/channels/audin/client/audin_main.c b/channels/audin/client/audin_main.c index 5a854b78a..79e0127c2 100644 --- a/channels/audin/client/audin_main.c +++ b/channels/audin/client/audin_main.c @@ -146,6 +146,16 @@ static UINT audin_process_version(AUDIN_PLUGIN* audin, AUDIN_CHANNEL_CALLBACK* c Stream_Read_UINT32(s, ServerVersion); WLog_Print(audin->log, WLOG_DEBUG, "ServerVersion=%"PRIu32", ClientVersion=%"PRIu32, ServerVersion, ClientVersion); + + /* Do not answer server packet, we do not support the channel version. */ + if (ServerVersion != ClientVersion) + { + WLog_Print(audin->log, WLOG_WARN, + "Incompatible channel version server=%"PRIu32", client supports version=%"PRIu32, ServerVersion, + ClientVersion); + return CHANNEL_RC_OK; + } + out = Stream_New(NULL, 5); if (!out) @@ -869,7 +879,7 @@ static UINT audin_set_subsystem(AUDIN_PLUGIN* audin, const char* subsystem) * * @return 0 on success, otherwise a Win32 error code */ -static UINT audin_set_device_name(AUDIN_PLUGIN* audin, char* device_name) +static UINT audin_set_device_name(AUDIN_PLUGIN* audin, const char* device_name) { free(audin->device_name); audin->device_name = _strdup(device_name);