Skip to content

Commit

Permalink
Audio processing: Feed each processing step its choice of int or floa…
Browse files Browse the repository at this point in the history 
…t data

Each audio processing step is given a pointer to an AudioBuffer, where
it can read and write int data. This patch adds corresponding
AudioBuffer methods to read and write float data; the buffer will
automatically convert the stored data between int and float as
necessary.

This patch also modifies the echo cancellation step to make use of the
new methods (it was already using floats internally; now it doesn't
have to convert from and to ints anymore).

(The reference data to the ApmTest.Process test had to be modified
slightly; this is because the echo canceller no longer unnecessarily
converts float data to int and then immediately back to float for each
iteration in the loop in EchoCancellationImpl::ProcessCaptureAudio.)

BUG=
R=aluebs@webrtc.org, andrew@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/18399005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@6138 4adac7df-926f-26a2-2b94-8c16560cd09d
  • Loading branch information
kwiberg@webrtc.org committed May 14, 2014
1 parent 3d5cb33 commit 934a265
Show file tree
Hide file tree
Showing 9 changed files with 180 additions and 95 deletions.
Binary file modified data/audio_processing/output_data_float.pb
View file
Binary file not shown.
59 changes: 23 additions & 36 deletions webrtc/modules/audio_processing/aec/aec_core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ extern int webrtc_aec_instance_count;
// "Private" function prototypes.
static void ProcessBlock(AecCore* aec);

static void NonLinearProcessing(AecCore* aec, short* output, short* outputH);
static void NonLinearProcessing(AecCore* aec, float* output, float* outputH);

static void GetHighbandGain(const float* lambda, float* nlpGainHband);

Expand Down Expand Up @@ -160,28 +160,28 @@ int WebRtcAec_CreateAec(AecCore** aecInst) {
return -1;
}

aec->nearFrBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(int16_t));
aec->nearFrBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
if (!aec->nearFrBuf) {
WebRtcAec_FreeAec(aec);
aec = NULL;
return -1;
}

aec->outFrBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(int16_t));
aec->outFrBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
if (!aec->outFrBuf) {
WebRtcAec_FreeAec(aec);
aec = NULL;
return -1;
}

aec->nearFrBufH = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(int16_t));
aec->nearFrBufH = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
if (!aec->nearFrBufH) {
WebRtcAec_FreeAec(aec);
aec = NULL;
return -1;
}

aec->outFrBufH = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(int16_t));
aec->outFrBufH = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
if (!aec->outFrBufH) {
WebRtcAec_FreeAec(aec);
aec = NULL;
Expand Down Expand Up @@ -617,11 +617,11 @@ int WebRtcAec_MoveFarReadPtr(AecCore* aec, int elements) {
}

void WebRtcAec_ProcessFrame(AecCore* aec,
const short* nearend,
const short* nearendH,
const float* nearend,
const float* nearendH,
int knownDelay,
int16_t* out,
int16_t* outH) {
float* out,
float* outH) {
int out_elements = 0;

// For each frame the process is as follows:
Expand Down Expand Up @@ -814,7 +814,7 @@ void WebRtcAec_SetSystemDelay(AecCore* self, int delay) {

static void ProcessBlock(AecCore* aec) {
int i;
float d[PART_LEN], y[PART_LEN], e[PART_LEN], dH[PART_LEN];
float y[PART_LEN], e[PART_LEN];
float scale;

float fft[PART_LEN2];
Expand All @@ -833,30 +833,22 @@ static void ProcessBlock(AecCore* aec) {
const float ramp = 1.0002f;
const float gInitNoise[2] = {0.999f, 0.001f};

int16_t nearend[PART_LEN];
int16_t* nearend_ptr = NULL;
int16_t output[PART_LEN];
int16_t outputH[PART_LEN];
float nearend[PART_LEN];
float* nearend_ptr = NULL;
float output[PART_LEN];
float outputH[PART_LEN];

float* xf_ptr = NULL;

memset(dH, 0, sizeof(dH));
// Concatenate old and new nearend blocks.
if (aec->sampFreq == 32000) {
// Get the upper band first so we can reuse |nearend|.
WebRtc_ReadBuffer(aec->nearFrBufH, (void**)&nearend_ptr, nearend, PART_LEN);
for (i = 0; i < PART_LEN; i++) {
dH[i] = (float)(nearend_ptr[i]);
}
memcpy(aec->dBufH + PART_LEN, dH, sizeof(float) * PART_LEN);
memcpy(aec->dBufH + PART_LEN, nearend_ptr, sizeof(nearend));
}
WebRtc_ReadBuffer(aec->nearFrBuf, (void**)&nearend_ptr, nearend, PART_LEN);
memcpy(aec->dBuf + PART_LEN, nearend_ptr, sizeof(nearend));

// ---------- Ooura fft ----------
// Concatenate old and new nearend blocks.
for (i = 0; i < PART_LEN; i++) {
d[i] = (float)(nearend_ptr[i]);
}
memcpy(aec->dBuf + PART_LEN, d, sizeof(float) * PART_LEN);

#ifdef WEBRTC_AEC_DEBUG_DUMP
{
Expand Down Expand Up @@ -968,7 +960,7 @@ static void ProcessBlock(AecCore* aec) {
}

for (i = 0; i < PART_LEN; i++) {
e[i] = d[i] - y[i];
e[i] = nearend_ptr[i] - y[i];
}

// Error fft
Expand Down Expand Up @@ -1027,7 +1019,7 @@ static void ProcessBlock(AecCore* aec) {
#endif
}

static void NonLinearProcessing(AecCore* aec, short* output, short* outputH) {
static void NonLinearProcessing(AecCore* aec, float* output, float* outputH) {
float efw[2][PART_LEN1], dfw[2][PART_LEN1], xfw[2][PART_LEN1];
complex_t comfortNoiseHband[PART_LEN1];
float fft[PART_LEN2];
Expand Down Expand Up @@ -1321,13 +1313,10 @@ static void NonLinearProcessing(AecCore* aec, short* output, short* outputH) {
fft[i] *= scale; // fft scaling
fft[i] = fft[i] * sqrtHanning[i] + aec->outBuf[i];

// Saturation protection
output[i] = (short)WEBRTC_SPL_SAT(
WEBRTC_SPL_WORD16_MAX, fft[i], WEBRTC_SPL_WORD16_MIN);

fft[PART_LEN + i] *= scale; // fft scaling
aec->outBuf[i] = fft[PART_LEN + i] * sqrtHanning[PART_LEN - i];
}
memcpy(output, fft, sizeof(*output) * PART_LEN);

// For H band
if (aec->sampFreq == 32000) {
Expand All @@ -1351,18 +1340,16 @@ static void NonLinearProcessing(AecCore* aec, short* output, short* outputH) {

// compute gain factor
for (i = 0; i < PART_LEN; i++) {
dtmp = (float)aec->dBufH[i];
dtmp = (float)dtmp * nlpGainHband; // for variable gain
dtmp = aec->dBufH[i];
dtmp = dtmp * nlpGainHband; // for variable gain

// add some comfort noise where Hband is attenuated
if (flagHbandCn == 1) {
fft[i] *= scale; // fft scaling
dtmp += cnScaleHband * fft[i];
}

// Saturation protection
outputH[i] = (short)WEBRTC_SPL_SAT(
WEBRTC_SPL_WORD16_MAX, dtmp, WEBRTC_SPL_WORD16_MIN);
outputH[i] = dtmp;
}
}

Expand Down
8 changes: 4 additions & 4 deletions webrtc/modules/audio_processing/aec/aec_core.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,11 @@ void WebRtcAec_InitAec_mips(void);

void WebRtcAec_BufferFarendPartition(AecCore* aec, const float* farend);
void WebRtcAec_ProcessFrame(AecCore* aec,
const short* nearend,
const short* nearendH,
const float* nearend,
const float* nearendH,
int knownDelay,
int16_t* out,
int16_t* outH);
float* out,
float* outH);

// A helper function to call WebRtc_MoveReadPtr() for all far-end buffers.
// Returns the number of elements moved, and adjusts |system_delay| by the
Expand Down
48 changes: 24 additions & 24 deletions webrtc/modules/audio_processing/aec/echo_cancellation.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,18 +104,18 @@ int webrtc_aec_instance_count = 0;
static void EstBufDelayNormal(aecpc_t* aecInst);
static void EstBufDelayExtended(aecpc_t* aecInst);
static int ProcessNormal(aecpc_t* self,
const int16_t* near,
const int16_t* near_high,
int16_t* out,
int16_t* out_high,
const float* near,
const float* near_high,
float* out,
float* out_high,
int16_t num_samples,
int16_t reported_delay_ms,
int32_t skew);
static void ProcessExtended(aecpc_t* self,
const int16_t* near,
const int16_t* near_high,
int16_t* out,
int16_t* out_high,
const float* near,
const float* near_high,
float* out,
float* out_high,
int16_t num_samples,
int16_t reported_delay_ms,
int32_t skew);
Expand Down Expand Up @@ -372,10 +372,10 @@ int32_t WebRtcAec_BufferFarend(void* aecInst,
}

int32_t WebRtcAec_Process(void* aecInst,
const int16_t* nearend,
const int16_t* nearendH,
int16_t* out,
int16_t* outH,
const float* nearend,
const float* nearendH,
float* out,
float* outH,
int16_t nrOfSamples,
int16_t msInSndCardBuf,
int32_t skew) {
Expand Down Expand Up @@ -632,10 +632,10 @@ AecCore* WebRtcAec_aec_core(void* handle) {
}

static int ProcessNormal(aecpc_t* aecpc,
const int16_t* nearend,
const int16_t* nearendH,
int16_t* out,
int16_t* outH,
const float* nearend,
const float* nearendH,
float* out,
float* outH,
int16_t nrOfSamples,
int16_t msInSndCardBuf,
int32_t skew) {
Expand Down Expand Up @@ -689,10 +689,10 @@ static int ProcessNormal(aecpc_t* aecpc,
if (aecpc->startup_phase) {
// Only needed if they don't already point to the same place.
if (nearend != out) {
memcpy(out, nearend, sizeof(short) * nrOfSamples);
memcpy(out, nearend, sizeof(*out) * nrOfSamples);
}
if (nearendH != outH) {
memcpy(outH, nearendH, sizeof(short) * nrOfSamples);
memcpy(outH, nearendH, sizeof(*outH) * nrOfSamples);
}

// The AEC is in the start up mode
Expand Down Expand Up @@ -789,10 +789,10 @@ static int ProcessNormal(aecpc_t* aecpc,
}

static void ProcessExtended(aecpc_t* self,
const int16_t* near,
const int16_t* near_high,
int16_t* out,
int16_t* out_high,
const float* near,
const float* near_high,
float* out,
float* out_high,
int16_t num_samples,
int16_t reported_delay_ms,
int32_t skew) {
Expand Down Expand Up @@ -823,10 +823,10 @@ static void ProcessExtended(aecpc_t* self,
if (!self->farend_started) {
// Only needed if they don't already point to the same place.
if (near != out) {
memcpy(out, near, sizeof(short) * num_samples);
memcpy(out, near, sizeof(*out) * num_samples);
}
if (near_high != out_high) {
memcpy(out_high, near_high, sizeof(short) * num_samples);
memcpy(out_high, near_high, sizeof(*out_high) * num_samples);
}
return;
}
Expand Down
16 changes: 8 additions & 8 deletions webrtc/modules/audio_processing/aec/include/echo_cancellation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,9 @@ int32_t WebRtcAec_BufferFarend(void* aecInst,
* Inputs Description
* -------------------------------------------------------------------
* void* aecInst Pointer to the AEC instance
* int16_t* nearend In buffer containing one frame of
* float* nearend In buffer containing one frame of
* nearend+echo signal for L band
* int16_t* nearendH In buffer containing one frame of
* float* nearendH In buffer containing one frame of
* nearend+echo signal for H band
* int16_t nrOfSamples Number of samples in nearend buffer
* int16_t msInSndCardBuf Delay estimate for sound card and
Expand All @@ -146,18 +146,18 @@ int32_t WebRtcAec_BufferFarend(void* aecInst,
*
* Outputs Description
* -------------------------------------------------------------------
* int16_t* out Out buffer, one frame of processed nearend
* float* out Out buffer, one frame of processed nearend
* for L band
* int16_t* outH Out buffer, one frame of processed nearend
* float* outH Out buffer, one frame of processed nearend
* for H band
* int32_t return 0: OK
* -1: error
*/
int32_t WebRtcAec_Process(void* aecInst,
const int16_t* nearend,
const int16_t* nearendH,
int16_t* out,
int16_t* outH,
const float* nearend,
const float* nearendH,
float* out,
float* outH,
int16_t nrOfSamples,
int16_t msInSndCardBuf,
int32_t skew);
Expand Down
10 changes: 6 additions & 4 deletions webrtc/modules/audio_processing/aec/system_delay_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,18 @@ class SystemDelayTest : public ::testing::Test {
aecpc_t* self_;
int samples_per_frame_;
// Dummy input/output speech data.
int16_t far_[160];
int16_t near_[160];
int16_t out_[160];
static const int kSamplesPerChunk = 160;
int16_t far_[kSamplesPerChunk];
float near_[kSamplesPerChunk];
float out_[kSamplesPerChunk];
};

SystemDelayTest::SystemDelayTest()
: handle_(NULL), self_(NULL), samples_per_frame_(0) {
// Dummy input data are set with more or less arbitrary non-zero values.
memset(far_, 1, sizeof(far_));
memset(near_, 2, sizeof(near_));
for (int i = 0; i < kSamplesPerChunk; i++)
near_[i] = 514.0;
memset(out_, 0, sizeof(out_));
}

Expand Down
Loading

0 comments on commit 934a265

Please sign in to comment.