Загрузка данных
#include <gst.h>
#include <stdio.h>
#include <unistd.h>
#include <stdint.h>
#include <stdbool.h>
#include <fcntl.h>
#include "common.h"
#include "gstimx_ai_aecnr.h"
#include "gst_buffers_helpers.h"
#include "imx_ai_aecnr_core.h"
G_DEFINE_TYPE (Gstimx_ai_aecnr, gst_imx_ai_aecnr, GST_TYPE_ELEMENT);
GST_ELEMENT_REGISTER_DEFINE (imx_ai_aecnr, "imx_ai_aecnr", GST_RANK_NONE, GST_TYPE_IMX_AI_AECNR);
enum
{
PROP_0,
PROP_BYPASS, /* bypass AI_AECNR for debug purpose. */
PROP_RX_PADDING, /* Fill Rx with 0 if no data received.*/
PROP_MODEL
};
/* Plugin pads definition. */
static GstStaticPadTemplate sink_remote_pad_template = GST_STATIC_PAD_TEMPLATE ("sink_remote",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("audio/x-raw,channels=1,rate=16000,format=F32LE")
);
static GstStaticPadTemplate sink_mic_pad_template = GST_STATIC_PAD_TEMPLATE ("sink_mic",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("audio/x-raw,channels=1,rate=16000,format=F32LE")
);
static GstStaticPadTemplate src_remote_pad_template = GST_STATIC_PAD_TEMPLATE ("src_remote",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("audio/x-raw,channels=1,rate=16000,format=F32LE")
);
static GstElementClass *parent_class;
/* Input buffer from remote
* - Add it in a queue for processing later with buffer from mic
*/
static GstFlowReturn gst_imx_ai_aecnr_chain_remote (GstPad * pad, GstObject * parent, GstBuffer * buf)
{
Gstimx_ai_aecnr *ai_aecnr = GST_IMX_AI_AECNR (parent);
if (ai_aecnr->bypass) {
/* RxBuf not needed in that case: drop it. */
gst_buffer_unref(buf);
return GST_FLOW_OK;
}
GST_OBJECT_LOCK(ai_aecnr);
buffer_list_append(&ai_aecnr->rx_buf_list, buf, AI_AECNR_SAMPLE_RATE);
GST_OBJECT_UNLOCK(ai_aecnr);
return GST_FLOW_OK;
}
/* Input buffer from microphone
* - retrieve remote buffer from queue
* - generate an empty buffer if nothing available from remote
* - remove occasional glitches because of UAC stack
* - insert both remote and microphone data into their respective circular buffer
* - retrieve data from previous iteration
* - TfLite inference
* - store output tensors for next iteration
* - create output gstreamer buffer from output tensor
*/
static GstFlowReturn gst_imx_ai_aecnr_chain_mic (GstPad * pad, GstObject * parent, GstBuffer * micBuf)
{
Gstimx_ai_aecnr *ai_aecnr = GST_IMX_AI_AECNR (parent);
GstBuffer *buf, *out_buf;
GstMapInfo out_map;
float mic_in[AI_AECNR_IO_SAMPLES];
float rx_in[AI_AECNR_IO_SAMPLES];
float out[AI_AECNR_IO_SAMPLES];
if (!ai_aecnr->ai_aecnr_core) {
return GST_FLOW_ERROR;
}
/* Bypass AEC-ML if option is set */
if (ai_aecnr->bypass) {
/* reset internal buffers. */
GST_OBJECT_LOCK(ai_aecnr);
init_buf_list(&ai_aecnr->rx_buf_list);
init_buf_list(&ai_aecnr->mic_buf_list);
GST_OBJECT_UNLOCK(ai_aecnr);
ai_aecnr->stream_pts = 0;
/* Push directly mic buffer to remote. */
gst_pad_push(ai_aecnr->src_remote_pad, micBuf);
return GST_FLOW_OK;
}
/* Add incoming buffer to input list.
*/
GST_OBJECT_LOCK(ai_aecnr);
buffer_list_append(&ai_aecnr->mic_buf_list, micBuf, AI_AECNR_SAMPLE_RATE);
/* Initialize output timestamp to the first received buffer. */
if (ai_aecnr->stream_pts == 0) {
ai_aecnr->stream_pts = micBuf->pts;
}
/* Enough data in queue to trigger processing. */
while ((ai_aecnr->mic_buf_list.unprocessed_samples >= AI_AECNR_IO_SAMPLES)
&& ((ai_aecnr->rx_buf_list.unprocessed_samples >= AI_AECNR_IO_SAMPLES) || (ai_aecnr->rx_padding))) {
/* If not enough data received from Rx then insert zeros. */
if (ai_aecnr->rx_buf_list.unprocessed_samples < AI_AECNR_IO_SAMPLES) {
buf = gst_buffer_new_allocate(NULL, AI_AECNR_IO_SAMPLES*sizeof(float), NULL);
if (buf) {
gst_buffer_memset(buf, 0, 0, AI_AECNR_IO_SAMPLES*sizeof(float));
GST_BUFFER_PTS (buf) = 0;
GST_BUFFER_DURATION (buf) = SAMPLES_TO_NS(AI_AECNR_IO_SAMPLES, AI_AECNR_SAMPLE_RATE);
buffer_list_append(&ai_aecnr->rx_buf_list, buf, AI_AECNR_SAMPLE_RATE);
}
}
/* Extract data from gst buffers to input buffers */
copy_float_from_gst_buffers(&ai_aecnr->mic_buf_list, mic_in, AI_AECNR_IO_SAMPLES, 1, AI_AECNR_SAMPLE_RATE);
copy_float_from_gst_buffers(&ai_aecnr->rx_buf_list, rx_in, AI_AECNR_IO_SAMPLES, 1, AI_AECNR_SAMPLE_RATE);
GST_OBJECT_UNLOCK(ai_aecnr); /* Release lock now that accesses to buffer lists are done. */
/* Call processing function */
if (imx_ai_aecnr_core_process(ai_aecnr->ai_aecnr_core, mic_in, rx_in, out) != 0) {
GST_ERROR_OBJECT(ai_aecnr, "license expired - disabling imx_ai_aecnr processing\n");
return GST_FLOW_ERROR;
}
/* Create a new output buffer and copy processed data into it. */
out_buf = gst_buffer_new_allocate(NULL, sizeof(float)*AI_AECNR_IO_SAMPLES, NULL);
if (out_buf) {
GST_BUFFER_PTS (out_buf) = ai_aecnr->stream_pts;
ai_aecnr->stream_pts += SAMPLES_TO_NS(AI_AECNR_IO_SAMPLES, AI_AECNR_SAMPLE_RATE);
GST_BUFFER_DURATION (out_buf) = SAMPLES_TO_NS(AI_AECNR_IO_SAMPLES, AI_AECNR_SAMPLE_RATE);
gst_buffer_map (out_buf, &out_map, GST_MAP_WRITE);
memcpy(out_map.data, out, sizeof(float)*AI_AECNR_IO_SAMPLES);
gst_buffer_unmap(out_buf, &out_map);
gst_pad_push(ai_aecnr->src_remote_pad, out_buf);
}
GST_OBJECT_LOCK(ai_aecnr); /* Lock for accessing the buffer list in while condition. */
}
GST_OBJECT_UNLOCK(ai_aecnr);
return GST_FLOW_OK;
}
static gboolean gst_imx_ai_aecnr_sink_event (GstPad * pad, GstObject * parent, GstEvent * event)
{
Gstimx_ai_aecnr *ai_aecnr = GST_IMX_AI_AECNR (parent);
gboolean res = TRUE;
switch (GST_EVENT_TYPE (event)) {
case GST_EVENT_EOS:
/* Consider EOS only from mic pad since remote input is automatically replaced by 0 when no data available. */
if (pad == ai_aecnr->sink_mic_pad ) {
res = gst_pad_push_event (ai_aecnr->src_remote_pad, event);
} else {
gst_event_unref (event);
}
break;
default:
res = gst_pad_push_event (ai_aecnr->src_remote_pad, event);
break;
}
return res;
}
/* Initialization of the plugin */
static void gst_imx_ai_aecnr_init (Gstimx_ai_aecnr * ai_aecnr)
{
ai_aecnr->sink_remote_pad = gst_pad_new_from_static_template (&sink_remote_pad_template, "sink_remote");
gst_pad_set_chain_function (ai_aecnr->sink_remote_pad, GST_DEBUG_FUNCPTR (gst_imx_ai_aecnr_chain_remote));
gst_pad_set_event_function (ai_aecnr->sink_remote_pad, GST_DEBUG_FUNCPTR (gst_imx_ai_aecnr_sink_event));
gst_pad_use_fixed_caps (ai_aecnr->sink_remote_pad);
gst_element_add_pad (GST_ELEMENT (ai_aecnr), ai_aecnr->sink_remote_pad);
ai_aecnr->sink_mic_pad = gst_pad_new_from_static_template (&sink_mic_pad_template, "sink_mic");
gst_pad_set_chain_function (ai_aecnr->sink_mic_pad, GST_DEBUG_FUNCPTR (gst_imx_ai_aecnr_chain_mic));
gst_pad_set_event_function (ai_aecnr->sink_mic_pad, GST_DEBUG_FUNCPTR (gst_imx_ai_aecnr_sink_event));
gst_pad_use_fixed_caps (ai_aecnr->sink_mic_pad);
gst_element_add_pad (GST_ELEMENT (ai_aecnr), ai_aecnr->sink_mic_pad);
ai_aecnr->src_remote_pad = gst_pad_new_from_static_template (&src_remote_pad_template, "src_remote");
gst_pad_use_fixed_caps (ai_aecnr->src_remote_pad);
gst_element_add_pad (GST_ELEMENT (ai_aecnr), ai_aecnr->src_remote_pad);
ai_aecnr->bypass = FALSE;
ai_aecnr->rx_padding = TRUE;
/* by default using small model. */
ai_aecnr->use_small_model = 1;
}
/* Plugin state management.
Used to open and close internal library.
*/
static GstStateChangeReturn gst_imx_ai_aecnr_change_state (GstElement *element, GstStateChange transition)
{
Gstimx_ai_aecnr *ai_aecnr = GST_IMX_AI_AECNR (element);
GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
imx_ai_aecnr_config core_config;
/* plugin starting. */
if (transition == GST_STATE_CHANGE_NULL_TO_READY) {
core_config.use_small_model = ai_aecnr->use_small_model;
core_config.version = NULL;
ai_aecnr->ai_aecnr_core = imx_ai_aecnr_core_open(&core_config);
if (!ai_aecnr->ai_aecnr_core) {
GST_ERROR_OBJECT(ai_aecnr, "Failed to initialize");
return GST_STATE_CHANGE_FAILURE;
}
/* Print version information from the core library */
GST_INFO_OBJECT(ai_aecnr, "plugin version: %s", IMX_VOICE_PLUGINS_VERSION);
if (core_config.version != NULL) {
GST_INFO_OBJECT(ai_aecnr, "%s", core_config.version);
} else {
GST_WARNING_OBJECT(ai_aecnr, "AI AECNR Core Library version information not available");
}
/* Initialize input buffers lists. */
GST_OBJECT_LOCK(ai_aecnr);
init_buf_list(&ai_aecnr->rx_buf_list);
init_buf_list(&ai_aecnr->mic_buf_list);
GST_OBJECT_UNLOCK(ai_aecnr);
ai_aecnr->stream_pts = 0;
}
/* Parent class state management.*/
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
if (ret == GST_STATE_CHANGE_FAILURE)
return ret;
/* Plugin closing. */
if (transition == GST_STATE_CHANGE_READY_TO_NULL) {
imx_ai_aecnr_core_close(ai_aecnr->ai_aecnr_core);
ret = GST_STATE_CHANGE_SUCCESS;
}
return ret;
}
/* Plugin properties management */
static void gst_imx_ai_aecnr_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec)
{
Gstimx_ai_aecnr *ai_aecnr = GST_IMX_AI_AECNR (object);
gchar *model_name = NULL;
switch (prop_id) {
case PROP_BYPASS:
ai_aecnr->bypass = g_value_get_boolean (value);
break;
case PROP_RX_PADDING:
ai_aecnr->rx_padding = g_value_get_boolean (value);
break;
case PROP_MODEL:
model_name = g_value_dup_string (value);
if (g_str_equal(model_name, "small") || (model_name == NULL)) {
GST_INFO_OBJECT(ai_aecnr, "configuring small model\n");
ai_aecnr->use_small_model = true;
} else if (g_str_equal(model_name, "large")) {
GST_INFO_OBJECT(ai_aecnr, "configuring large model\n");
ai_aecnr->use_small_model = false;
} else {
GST_INFO_OBJECT(ai_aecnr, "unknown model identifier %s : using small model by default\n", model_name);
ai_aecnr->use_small_model = true;
}
g_free(model_name);
break;
default:
GST_ERROR_OBJECT(ai_aecnr, "unsupported property ID: %d", prop_id);
break;
}
}
static void gst_imx_ai_aecnr_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec)
{
Gstimx_ai_aecnr *ai_aecnr = GST_IMX_AI_AECNR (object);
switch (prop_id) {
case PROP_BYPASS:
g_value_set_boolean (value, ai_aecnr->bypass);
break;
case PROP_RX_PADDING:
g_value_set_boolean (value, ai_aecnr->rx_padding);
break;
default:
GST_ERROR_OBJECT(ai_aecnr, "unsupported property ID: %d", prop_id);
break;
}
}
/* class initialization */
static void gst_imx_ai_aecnr_class_init (Gstimx_ai_aecnrClass * ai_aecnrclass)
{
GObjectClass *gobject_class = (GObjectClass *)ai_aecnrclass;
GstElementClass *gstelement_class = (GstElementClass *)ai_aecnrclass;
GST_DEBUG_CATEGORY_INIT (imx_ai_aecnr, "imx_ai_aecnr", 0, "imx_ai_aecnr plugin");
gobject_class->set_property = gst_imx_ai_aecnr_set_property;
gobject_class->get_property = gst_imx_ai_aecnr_get_property;
parent_class = (GstElementClass *) g_type_class_peek_parent (ai_aecnrclass);
gstelement_class->change_state = gst_imx_ai_aecnr_change_state;
g_object_class_install_property (gobject_class,
PROP_BYPASS,
g_param_spec_boolean ("bypass", "Bypass", "Bypass AI AECNR processing.",
FALSE, G_PARAM_READWRITE));
g_object_class_install_property (gobject_class,
PROP_RX_PADDING,
g_param_spec_boolean ("rx_padding",
"RxPadding",
"Fill Rx with 0 if no data received.",
TRUE,
G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY));
g_object_class_install_property (gobject_class, PROP_MODEL,
g_param_spec_string ("model",
"model",
"selection of the model [small,large]",
"small",
(GParamFlags)(G_PARAM_WRITABLE | GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS)));
gst_element_class_set_details_simple (gstelement_class,
"imx_ai_aecnr",
"filter/audio/ai_aecnr",
"NXP AI-based acoustic echo canceller and noise suppressor",
IMX_VOICE_PLUGINS_CONTACT);
gst_element_class_add_pad_template (gstelement_class, gst_static_pad_template_get (&src_remote_pad_template));
gst_element_class_add_pad_template (gstelement_class, gst_static_pad_template_get (&sink_remote_pad_template));
gst_element_class_add_pad_template (gstelement_class, gst_static_pad_template_get (&sink_mic_pad_template));
}
/* Plugin creation with platform check */
static gboolean imx_ai_aecnr_init (GstPlugin * imx_ai_aecnr)
{
return GST_ELEMENT_REGISTER (imx_ai_aecnr, imx_ai_aecnr);
}
GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
GST_VERSION_MINOR,
imx_ai_aecnr,
"imx_ai_aecnr",
imx_ai_aecnr_init,
IMX_VOICE_PLUGINS_VERSION,
IMX_VOICE_PLUGINS_LICENSE,
IMX_VOICE_PLUGINS_BINARY_PACKAGE,
IMX_VOICE_PLUGINS_URL)