Загрузка данных
#include "imx_ai_aecnr_core.h"
#include "mediastreamer2/msfilter.h"
#include "mediastreamer2/flowcontrol.h"
#define EC_DUMP
#define USE_CONVERT_NEON
typedef struct ImxAiAecState {
struct imx_ai_aecnr_core *aec_core;
MSBufferizer delayed_ref;
MSFlowControlledBufferizer ref;
MSBufferizer echo;
char *state_str;
int samplerate;
int framesize;
int delay_ms;
int nominal_ref_samples;
bool_t echostarted;
bool_t bypass_mode;
bool_t using_zeroes;
#ifdef EC_DUMP
FILE *echofile;
FILE *reffile;
FILE *cleanfile;
FILE *echofile_float;
FILE *reffile_float;
FILE *cleanfile_float;
#endif
} ImxAiAecState;
#ifdef USE_CONVERT_NEON
static void s16_to_float_neon(const int16_t* src, float* dst) {
asm volatile (
"ld1 {v0.8h}, [%[in]] \n\t"
"sxtl v1.4s, v0.4h \n\t"
"sxtl2 v2.4s, v0.8h \n\t"
"scvtf v1.4s, v1.4s \n\t"
"scvtf v2.4s, v2.4s \n\t"
"st1 {v1.4s, v2.4s}, [%[out]] \n\t"
:
: [in] "r" (src), [out] "r" (dst)
: "v0", "v1", "v2", "memory"
);
}
void float_to_s16_neon(const float* src, int16_t* dst) {
asm volatile (
"ld1 {v1.4s, v2.4s}, [%[in]] \n\t"
"fcvtns v1.4s, v1.4s \n\t"
"fcvtns v2.4s, v2.4s \n\t"
"sqxtn v0.4h, v1.4s \n\t"
"sqxtn2 v0.8h, v2.4s \n\t"
"st1 {v0.8h}, [%[out]] \n\t"
:
: [in] "r" (src), [out] "r" (dst)
: "v0", "v1", "v2", "memory"
);
}
#endif
static void imx_ai_aec_init(MSFilter *f) {
ImxAiAecState *s = ms_new0(ImxAiAecState, 1);
s->samplerate = AI_AECNR_SAMPLE_RATE; // 16000
s->framesize = AI_AECNR_IO_SAMPLES; // 256
ms_bufferizer_init(&s->delayed_ref);
ms_bufferizer_init(&s->echo);
ms_flow_controlled_bufferizer_init(&s->ref, f, s->samplerate, 1);
s->state_str = NULL;
s->delay_ms = 0;
s->echostarted = FALSE;
s->bypass_mode = FALSE;
s->using_zeroes = FALSE;
f->data = s;
#ifdef EC_DUMP
{
char *fname = ms_strdup_printf("/tmp/imx-%p-echo.raw", f);
s->echofile = fopen(fname, "wb");
ms_free(fname);
fname = ms_strdup_printf("/tmp/imx-%p-ref.raw", f);
s->reffile = fopen(fname, "wb");
ms_free(fname);
fname = ms_strdup_printf("/tmp/imx-%p-clean.raw", f);
s->cleanfile = fopen(fname, "wb");
ms_free(fname);
fname = ms_strdup_printf("/tmp/imx-%p-float_echo.raw", f);
s->echofile_float = fopen(fname, "wb");
ms_free(fname);
fname = ms_strdup_printf("/tmp/imx-%p-float_ref.raw", f);
s->reffile_float = fopen(fname, "wb");
ms_free(fname);
fname = ms_strdup_printf("/tmp/imx-%p-float_clean.raw", f);
s->cleanfile_float = fopen(fname, "wb");
ms_free(fname);
ms_message("%s(%d) echofile=%p\n",__FUNCTION__,__LINE__,s->echofile);
ms_message("%s(%d) reffile=%p\n",__FUNCTION__,__LINE__,s->reffile);
ms_message("%s(%d) cleanfile=%p\n",__FUNCTION__,__LINE__,s->cleanfile);
ms_message("%s(%d) echofile_float=%p\n",__FUNCTION__,__LINE__,s->echofile_float);
ms_message("%s(%d) reffile_float=%p\n",__FUNCTION__,__LINE__,s->reffile_float);
ms_message("%s(%d) cleanfile_float=%p\n",__FUNCTION__,__LINE__,s->cleanfile_float);
}
#endif
ms_message("%s(%d) use samplerate=%u framesize=%u\n",__FUNCTION__,__LINE__,s->samplerate,s->framesize);
}
static void imx_ai_aec_preprocess(MSFilter *f) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
imx_ai_aecnr_config config;
int delay_samples;
mblk_t *m;
config.use_small_model = 0;
s->aec_core = imx_ai_aecnr_core_open(&config);
if (s->aec_core == NULL) {
ms_error("imx_ai_aecnr_core_open() failed, bypass mode");
s->bypass_mode = TRUE;
return;
}
s->echostarted = FALSE;
delay_samples = s->delay_ms * s->samplerate / 1000;
s->nominal_ref_samples = delay_samples;
ms_message("IMX AI AECNR initialized: %s, framesize=%i, delay=%i ms\n",
config.version, s->framesize, s->delay_ms);
// flow control for synchronized thread's
ms_flow_controlled_bufferizer_set_samplerate(&s->ref, s->samplerate);
ms_flow_controlled_bufferizer_set_max_size_ms(&s->ref, s->delay_ms);
/* bind begin delay with zero's */
m = allocb(delay_samples * 2, 0);
memset(m->b_wptr, 0, delay_samples * 2);
m->b_wptr += delay_samples * 2;
ms_bufferizer_put(&s->delayed_ref, m);
}
static void configure_flow_controlled_bufferizer(ImxAiAecState *s) {
ms_flow_controlled_bufferizer_set_samplerate(&s->ref, s->samplerate);
ms_flow_controlled_bufferizer_set_max_size_ms(&s->ref, s->delay_ms);
ms_flow_controlled_bufferizer_set_granularity_ms(&s->ref, (s->framesize * 1000) / s->samplerate);
}
static void imx_ai_aec_process(MSFilter *f) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
int nsamples = s->framesize;
int nbytes = s->framesize * sizeof(int16_t);
mblk_t *refm;
int16_t *ref_tmp = (int16_t *)alloca(nbytes);
int16_t *echo_tmp = (int16_t *)alloca(nbytes);
float *ref_float = (float *)alloca(nsamples * sizeof(float));
float *echo_float = (float *)alloca(nsamples * sizeof(float));
float *out_float = (float *)alloca(nsamples * sizeof(float));
if (s->bypass_mode) {
while ((refm = ms_queue_get(f->inputs[0])) != NULL) ms_queue_put(f->outputs[0], refm);
while ((refm = ms_queue_get(f->inputs[1])) != NULL) ms_queue_put(f->outputs[1], refm);
return;
}
// 1. Get ref signal (from remote)
if (f->inputs[0] != NULL) {
if (s->echostarted) {
while ((refm = ms_queue_get(f->inputs[0])) != NULL) {
ms_bufferizer_put(&s->delayed_ref, dupmsg(refm));
ms_flow_controlled_bufferizer_put(&s->ref, refm);
}
} else {
ms_queue_flush(f->inputs[0]);
}
}
// 2. Get mic signal
ms_bufferizer_put_from_queue(&s->echo, f->inputs[1]);
// 3. Main cycle
while (ms_bufferizer_get_avail(&s->echo) >= (size_t)nbytes) {
mblk_t *out_clean = allocb(nbytes, 0);
int avail;
if (!s->echostarted) s->echostarted = TRUE;
// Checking if there is data in the delay buffer
avail = (int)ms_bufferizer_get_avail(&s->delayed_ref);
if (avail < ((s->nominal_ref_samples * 2) + nbytes)) {
// Silence injection if the reference is lagging behind
avail = nbytes;
refm = allocb(nbytes, 0);
memset(refm->b_wptr, 0, nbytes);
refm->b_wptr += nbytes;
ms_bufferizer_put(&s->delayed_ref, refm);
ms_queue_put(f->outputs[0], dupmsg(refm));
if (!s->using_zeroes) {
ms_warning("IMX AEC: Not enough ref samples, using zeroes");
s->using_zeroes = TRUE;
}
} else {
s->using_zeroes = FALSE;
refm = allocb(nbytes, 0);
ms_flow_controlled_bufferizer_read(&s->ref, refm->b_wptr, nbytes);
refm->b_wptr += nbytes;
ms_queue_put(f->outputs[0], refm);
}
// Reading aligned data
ms_bufferizer_read(&s->echo, (uint8_t *)echo_tmp, nbytes);
ms_bufferizer_read(&s->delayed_ref, (uint8_t *)ref_tmp, nbytes);
avail -= nbytes;
#ifdef EC_DUMP
if (s->reffile) fwrite(ref_tmp, nbytes, 1, s->reffile);
if (s->echofile) fwrite(echo_tmp, nbytes, 1, s->echofile);
#endif
#ifdef USE_CONVERT_NEON
for(int i = 0; i < nsamples; i+=8) {
s16_to_float_neon(&ref_tmp[i], &ref_float[i]);
s16_to_float_neon(&echo_tmp[i], &echo_float[i]);
}
#else
//Converts int16_t to float data
for(int i = 0; i < nsamples; i++) {
ref_float[i] = (float)ref_tmp[i] / 32768.0f;
echo_float[i] = (float)echo_tmp[i] / 32768.0f;
}
#endif
#ifdef EC_DUMP
if (s->reffile_float) fwrite(ref_float, sizeof(float) * nsamples, 1, s->reffile_float);
if (s->echofile_float) fwrite(echo_float, sizeof(float) * nsamples, 1, s->echofile_float);
#endif
// AI Processing (AEC + NR)
if (imx_ai_aecnr_core_process(s->aec_core, ref_float, echo_float, out_float) != 0)
{
ms_error("imx_ai_aecnr_core_process failed");
//memcpy(out_clean->b_wptr, echo_tmp, nbytes); // Fallback
for (int i = 0; i < nsamples; i++) out_float[i] = echo_float[i];
}
#ifdef EC_DUMP
if (s->cleanfile_float) fwrite(out_float, sizeof(float) * nsamples, 1, s->cleanfile_float);
#endif
//Converts float data to int16_t
int16_t *out_ptr = (int16_t *)out_clean->b_wptr;
#ifdef USE_CONVERT_NEON
for(int i = 0; i < nsamples; i+=8) {
float_to_s16_neon(&out_float[i], &out_ptr[i]);
}
#else
for(int i = 0; i < nsamples; i++) {
float val = out_float[i] * 32768.0f;
if(val > 32767.0f) val = 32767.0f;
if(val < -32768.0f) val = -32768.0f;
out_ptr[i] = (int16_t)val;
}
#endif
#ifdef EC_DUMP
if (s->cleanfile) fwrite(out_ptr, nbytes, 1, s->cleanfile);
#endif
out_clean->b_wptr += nbytes;
ms_queue_put(f->outputs[1], out_clean);
}
}
static void imx_ai_aec_postprocess(MSFilter *f) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
if (s->aec_core) {
imx_ai_aecnr_core_close(s->aec_core);
s->aec_core = NULL;
}
ms_bufferizer_flush(&s->delayed_ref);
ms_bufferizer_flush(&s->echo);
ms_message("%s(%d)\n",__FUNCTION__,__LINE__);
#ifdef EC_DUMP
if (s->echofile) fclose(s->echofile);
if (s->reffile) fclose(s->reffile);
if (s->cleanfile) fclose(s->cleanfile);
if (s->echofile_float) fclose(s->echofile_float);
if (s->reffile_float) fclose(s->reffile_float);
if (s->cleanfile_float) fclose(s->cleanfile_float);
#endif
}
static void imx_ai_aec_uninit(MSFilter *f) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
ms_message("%s(%d)\n",__FUNCTION__,__LINE__);
if (s->state_str) ms_free(s->state_str);
ms_bufferizer_uninit(&s->delayed_ref);
ms_bufferizer_uninit(&s->echo);
ms_free(s);
}
static int imx_ai_aec_set_sr(MSFilter *f, void *arg) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
ms_message("%s(%d) Try set samplerate=%u\n",__FUNCTION__,__LINE__,*(int *)arg);
#if 0
s->samplerate = *(int *)arg;
configure_flow_controlled_bufferizer(s);
#else
ms_message("%s(%d) Don't support set samplerate, use only %d samplerate\n",__FUNCTION__,__LINE__,AI_AECNR_SAMPLE_RATE);
#endif
return 0;
}
static int imx_ai_aec_set_framesize(MSFilter *f, void *arg) {
#if 0
ImxAiAecState *s = (ImxAiAecState *)f->data;
s->framesize_at_8000 = *(int *)arg;
#else
ms_message("%s(%d) Don't support set framesize, use only %d framesize\n",__FUNCTION__,__LINE__,AI_AECNR_IO_SAMPLES);
#endif
return 0;
}
static int imx_ai_aec_set_delay(MSFilter *f, void *arg) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
s->delay_ms = *(int *)arg;
ms_message("%s(%d) delay_ms=%d\n",__FUNCTION__,__LINE__,s->delay_ms);
configure_flow_controlled_bufferizer(s);
return 0;
}
static int imx_ai_aec_set_tail_length(MSFilter *f, void *arg) {
/* Do nothing because this is not needed by the IMX AI AEC echo canceller. */
return 0;
}
static int imx_ai_aec_set_bypass_mode(MSFilter *f, void *arg) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
ms_message("%s(%d)\n",__FUNCTION__,__LINE__);
s->bypass_mode = *(bool_t *)arg;
ms_message("set EC bypass mode to [%i]", s->bypass_mode);
return 0;
}
static int imx_ai_aec_get_bypass_mode(MSFilter *f, void *arg) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
*(bool_t *)arg = s->bypass_mode;
return 0;
}
static int imx_ai_aec_set_state(MSFilter *f, void *arg) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
s->state_str = ms_strdup((const char *)arg);
return 0;
}
static int imx_ai_aec_get_state(MSFilter *f, void *arg) {
ImxAiAecState *s = (ImxAiAecState *)f->data;
*(char **)arg = s->state_str;
return 0;
}
static MSFilterMethod imx_ai_aec_methods[] = {{MS_FILTER_SET_SAMPLE_RATE, imx_ai_aec_set_sr},
{MS_ECHO_CANCELLER_SET_TAIL_LENGTH, imx_ai_aec_set_tail_length},
{MS_ECHO_CANCELLER_SET_DELAY, imx_ai_aec_set_delay},
{MS_ECHO_CANCELLER_SET_FRAMESIZE, imx_ai_aec_set_framesize},
{MS_ECHO_CANCELLER_SET_BYPASS_MODE, imx_ai_aec_set_bypass_mode},
{MS_ECHO_CANCELLER_GET_BYPASS_MODE, imx_ai_aec_get_bypass_mode},
{MS_ECHO_CANCELLER_GET_STATE_STRING, imx_ai_aec_get_state},
{MS_ECHO_CANCELLER_SET_STATE_STRING, imx_ai_aec_set_state},
{0, 0}};
MSFilterDesc ms_imx_ai_aec_desc = {.id = MS_SPEEX_EC_ID,
.name = "MSIMXAiAec",
.text = N_("Echo canceller using IMX AI AEC library"),
.category = MS_FILTER_OTHER,
.ninputs = 2,
.noutputs = 2,
.init = imx_ai_aec_init,
.preprocess = imx_ai_aec_preprocess,
.process = imx_ai_aec_process,
.postprocess = imx_ai_aec_postprocess,
.uninit = imx_ai_aec_uninit,
.methods = imx_ai_aec_methods};
MS_FILTER_DESC_EXPORT(ms_imx_ai_aec_desc)