From c9f516bbfe929e5fd12eb6881261f50b302f08f4 Mon Sep 17 00:00:00 2001 From: ggfan Date: Thu, 20 Aug 2015 22:08:05 -0700 Subject: [PATCH 1/5] adding fast audio path when creating buffered queue audio player --- native-audio/app/build.gradle | 1 + .../com/example/nativeaudio/NativeAudio.java | 17 +- .../app/src/main/jni/native-audio-jni.c | 168 +++++++++++++++--- 3 files changed, 157 insertions(+), 29 deletions(-) diff --git a/native-audio/app/build.gradle b/native-audio/app/build.gradle index 368422d6..b781049b 100644 --- a/native-audio/app/build.gradle +++ b/native-audio/app/build.gradle @@ -15,6 +15,7 @@ model { android.ndk { moduleName = "native-audio-jni" ldLibs += ["android","OpenSLES", "log"] + CFlags += ['-std=c99'] } android.buildTypes { release { diff --git a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java index 71381237..3b41e0a8 100644 --- a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java +++ b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java @@ -17,7 +17,9 @@ package com.example.nativeaudio; import android.app.Activity; +import android.content.Context; import android.content.res.AssetManager; +import android.media.AudioManager; import android.os.Bundle; //import android.util.Log; import android.view.View; @@ -57,9 +59,18 @@ public class NativeAudio extends Activity { assetManager = getAssets(); // initialize native audio system - createEngine(); - createBufferQueueAudioPlayer(); + + int sampleRate = 0; + int bufSize = 0; + if( android.os.Build.VERSION.SDK_INT > android.os.Build.VERSION_CODES.JELLY_BEAN_MR1) { + AudioManager myAudioMgr = (AudioManager) getSystemService(Context.AUDIO_SERVICE); + String nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE); + sampleRate = Integer.parseInt(nativeParam); + nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER); + bufSize = Integer.parseInt(nativeParam); + } + createBufferQueueAudioPlayer(sampleRate, bufSize); // initialize URI spinner Spinner uriSpinner = (Spinner) findViewById(R.id.uri_spinner); @@ -288,7 +299,7 @@ public class NativeAudio extends Activity { /** Native methods, implemented in jni folder */ public static native void createEngine(); - public static native void createBufferQueueAudioPlayer(); + public static native void createBufferQueueAudioPlayer(int rate, int size); public static native boolean createAssetAudioPlayer(AssetManager assetManager, String filename); // true == PLAYING, false == PAUSED public static native void setPlayingAssetAudioPlayer(boolean isPlaying); diff --git a/native-audio/app/src/main/jni/native-audio-jni.c b/native-audio/app/src/main/jni/native-audio-jni.c index 85439cca..f8e91346 100644 --- a/native-audio/app/src/main/jni/native-audio-jni.c +++ b/native-audio/app/src/main/jni/native-audio-jni.c @@ -62,6 +62,10 @@ static SLAndroidSimpleBufferQueueItf bqPlayerBufferQueue; static SLEffectSendItf bqPlayerEffectSend; static SLMuteSoloItf bqPlayerMuteSolo; static SLVolumeItf bqPlayerVolume; +static SLmilliHertz bqPlayerSampleRate = 0; +static jint bqPlayerBufSize = 0; +static short *resampleBuf = NULL; +static volatile int bqPlayerRecorderBusy = 0; // aux effect on the output mix, used by the buffer queue player static const SLEnvironmentalReverbSettings reverbSettings = @@ -94,7 +98,6 @@ static short sawtoothBuffer[SAWTOOTH_FRAMES]; #define RECORDER_FRAMES (16000 * 5) static short recorderBuffer[RECORDER_FRAMES]; static unsigned recorderSize = 0; -static SLmilliHertz recorderSR; // pointer and size of the next player buffer to enqueue, and number of remaining buffers static short *nextBuffer; @@ -111,6 +114,76 @@ __attribute__((constructor)) static void onDlOpen(void) } } +void releaseResampleBuf(void) { + if( 0 == bqPlayerSampleRate) { + /* + * we are not using fast path, so we were not creating buffers, nothing to do + */ + return; + } + + free(resampleBuf); + resampleBuf = NULL; +} + +/* + * Only support up-sampling + */ +short* createResampledBuf(uint32_t idx, uint32_t srcRate, unsigned *size) { + short *src = NULL; + short *workBuf; + int upSampleRate; + int32_t srcSampleCount =0; + + if(0 == bqPlayerSampleRate) { + return NULL; + } + if(bqPlayerSampleRate % srcRate) { + /* + * simple up-sampling, must be divisible + */ + return NULL; + } + upSampleRate = bqPlayerSampleRate / srcRate; + + switch (idx) { + case 0: + return NULL; + case 1: // HELLO_CLIP + srcSampleCount = sizeof(hello) >> 1; + src = (short*)hello; + break; + case 2: // ANDROID_CLIP + srcSampleCount = sizeof(android) >> 1; + src = (short*) android; + break; + case 3: // SAWTOOTH_CLIP + srcSampleCount = SAWTOOTH_FRAMES; + src = sawtoothBuffer; + break; + case 4: // captured frames + srcSampleCount = recorderSize / sizeof(short); + src = recorderBuffer; + break; + default: + assert(0); + return NULL; + } + + resampleBuf = (short*) malloc((srcSampleCount * upSampleRate) << 1); + if(resampleBuf == NULL) { + return resampleBuf; + } + workBuf = resampleBuf; + for(int sample=0; sample < srcSampleCount; sample++) { + for(int dup = 0; dup < upSampleRate; dup++) { + *workBuf++ = src[sample]; + } + } + + *size = (srcSampleCount * upSampleRate) << 1; // sample format is 16 bit + return resampleBuf; +} // this callback handler is called every time a buffer finishes playing void bqPlayerCallback(SLAndroidSimpleBufferQueueItf bq, void *context) @@ -126,6 +199,9 @@ void bqPlayerCallback(SLAndroidSimpleBufferQueueItf bq, void *context) // which for this code example would indicate a programming error assert(SL_RESULT_SUCCESS == result); (void)result; + } else { + releaseResampleBuf(); + bqPlayerRecorderBusy = 0; } } @@ -141,8 +217,8 @@ void bqRecorderCallback(SLAndroidSimpleBufferQueueItf bq, void *context) result = (*recorderRecord)->SetRecordState(recorderRecord, SL_RECORDSTATE_STOPPED); if (SL_RESULT_SUCCESS == result) { recorderSize = RECORDER_FRAMES * sizeof(short); - recorderSR = SL_SAMPLINGRATE_16; } + bqPlayerRecorderBusy = 0; } @@ -196,28 +272,48 @@ void Java_com_example_nativeaudio_NativeAudio_createEngine(JNIEnv* env, jclass c // create buffer queue audio player void Java_com_example_nativeaudio_NativeAudio_createBufferQueueAudioPlayer(JNIEnv* env, - jclass clazz) + jclass clazz, jint sampleRate, jint bufSize) { SLresult result; + if (sampleRate >= 0 && bufSize >= 0 ) { + bqPlayerSampleRate = sampleRate * 1000; + /* + * device native buffer size is another factor to minimize audio latency, not used in this + * sample: we only play one giant buffer here + */ + bqPlayerBufSize = bufSize; + } // configure audio source SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2}; SLDataFormat_PCM format_pcm = {SL_DATAFORMAT_PCM, 1, SL_SAMPLINGRATE_8, SL_PCMSAMPLEFORMAT_FIXED_16, SL_PCMSAMPLEFORMAT_FIXED_16, SL_SPEAKER_FRONT_CENTER, SL_BYTEORDER_LITTLEENDIAN}; + /* + * Enable Fast Audio when possible: once we set the same rate to be the native, fast audio path + * will be triggered + */ + if(bqPlayerSampleRate) { + format_pcm.samplesPerSec = bqPlayerSampleRate; //sample rate in mili second + } SLDataSource audioSrc = {&loc_bufq, &format_pcm}; // configure audio sink SLDataLocator_OutputMix loc_outmix = {SL_DATALOCATOR_OUTPUTMIX, outputMixObject}; SLDataSink audioSnk = {&loc_outmix, NULL}; - // create audio player - const SLInterfaceID ids[3] = {SL_IID_BUFFERQUEUE, SL_IID_EFFECTSEND, - /*SL_IID_MUTESOLO,*/ SL_IID_VOLUME}; - const SLboolean req[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, - /*SL_BOOLEAN_TRUE,*/ SL_BOOLEAN_TRUE}; + /* + * create audio player: + * fast audio does not support when SL_IID_EFFECTSEND is required, skip it + * for fast audio case + */ + const SLInterfaceID ids[3] = {SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_EFFECTSEND, + /*SL_IID_MUTESOLO,*/}; + const SLboolean req[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, + /*SL_BOOLEAN_TRUE,*/ }; + result = (*engineEngine)->CreateAudioPlayer(engineEngine, &bqPlayerObject, &audioSrc, &audioSnk, - 3, ids, req); + bqPlayerSampleRate? 2 : 3, ids, req); assert(SL_RESULT_SUCCESS == result); (void)result; @@ -243,10 +339,13 @@ void Java_com_example_nativeaudio_NativeAudio_createBufferQueueAudioPlayer(JNIEn (void)result; // get the effect send interface - result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_EFFECTSEND, - &bqPlayerEffectSend); - assert(SL_RESULT_SUCCESS == result); - (void)result; + bqPlayerEffectSend = NULL; + if( 0 == bqPlayerSampleRate) { + result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_EFFECTSEND, + &bqPlayerEffectSend); + assert(SL_RESULT_SUCCESS == result); + (void)result; + } #if 0 // mute/solo is not supported for sources that are known to be mono, as this is // get the mute/solo interface @@ -498,6 +597,12 @@ jboolean Java_com_example_nativeaudio_NativeAudio_enableReverb(JNIEnv* env, jcla return JNI_FALSE; } + if(bqPlayerSampleRate) { + /* + * we are in fast audio, reverb is not supported. + */ + return JNI_FALSE; + } result = (*bqPlayerEffectSend)->EnableEffectSend(bqPlayerEffectSend, outputMixEnvironmentalReverb, (SLboolean) enabled, (SLmillibel) 0); // and even if environmental reverb was present, it might no longer be available @@ -508,7 +613,6 @@ jboolean Java_com_example_nativeaudio_NativeAudio_enableReverb(JNIEnv* env, jcla return JNI_TRUE; } - // select the desired clip and play count, and enqueue the first buffer if idle jboolean Java_com_example_nativeaudio_NativeAudio_selectClip(JNIEnv* env, jclass clazz, jint which, jint count) @@ -519,29 +623,38 @@ jboolean Java_com_example_nativeaudio_NativeAudio_selectClip(JNIEnv* env, jclass nextSize = 0; break; case 1: // CLIP_HELLO - nextBuffer = (short *) hello; - nextSize = sizeof(hello); + nextBuffer = createResampledBuf(1, SL_SAMPLINGRATE_8, &nextSize); + if(!nextBuffer) { + nextBuffer = (short*)hello; + nextSize = sizeof(hello); + } break; case 2: // CLIP_ANDROID - nextBuffer = (short *) android; - nextSize = sizeof(android); + nextBuffer = createResampledBuf(2, SL_SAMPLINGRATE_8, &nextSize); + if(!nextBuffer) { + nextBuffer = (short*)android; + nextSize = sizeof(android); + } break; case 3: // CLIP_SAWTOOTH - nextBuffer = sawtoothBuffer; - nextSize = sizeof(sawtoothBuffer); + nextBuffer = createResampledBuf(3, SL_SAMPLINGRATE_8, &nextSize); + if(!nextBuffer) { + nextBuffer = (short*)sawtoothBuffer; + nextSize = sizeof(sawtoothBuffer); + } break; case 4: // CLIP_PLAYBACK + nextBuffer = createResampledBuf(4, SL_SAMPLINGRATE_16, &nextSize); // we recorded at 16 kHz, but are playing buffers at 8 Khz, so do a primitive down-sample - if (recorderSR == SL_SAMPLINGRATE_16) { + if(!nextBuffer) { unsigned i; for (i = 0; i < recorderSize; i += 2 * sizeof(short)) { recorderBuffer[i >> 2] = recorderBuffer[i >> 1]; } - recorderSR = SL_SAMPLINGRATE_8; recorderSize >>= 1; + nextBuffer = recorderBuffer; + nextSize = recorderSize; } - nextBuffer = recorderBuffer; - nextSize = recorderSize; break; default: nextBuffer = NULL; @@ -555,6 +668,7 @@ jboolean Java_com_example_nativeaudio_NativeAudio_selectClip(JNIEnv* env, jclass SLresult result; result = (*bqPlayerBufferQueue)->Enqueue(bqPlayerBufferQueue, nextBuffer, nextSize); if (SL_RESULT_SUCCESS != result) { + bqPlayerRecorderBusy = 1; return JNI_FALSE; } } @@ -642,7 +756,6 @@ jboolean Java_com_example_nativeaudio_NativeAudio_createAssetAudioPlayer(JNIEnv* return JNI_TRUE; } - // set the playing state for the asset audio player void Java_com_example_nativeaudio_NativeAudio_setPlayingAssetAudioPlayer(JNIEnv* env, jclass clazz, jboolean isPlaying) @@ -661,7 +774,6 @@ void Java_com_example_nativeaudio_NativeAudio_setPlayingAssetAudioPlayer(JNIEnv* } - // create audio recorder jboolean Java_com_example_nativeaudio_NativeAudio_createAudioRecorder(JNIEnv* env, jclass clazz) { @@ -721,6 +833,9 @@ void Java_com_example_nativeaudio_NativeAudio_startRecording(JNIEnv* env, jclass { SLresult result; + if( bqPlayerRecorderBusy) { + return; + } // in case already recording, stop recording and clear buffer queue result = (*recorderRecord)->SetRecordState(recorderRecord, SL_RECORDSTATE_STOPPED); assert(SL_RESULT_SUCCESS == result); @@ -745,6 +860,7 @@ void Java_com_example_nativeaudio_NativeAudio_startRecording(JNIEnv* env, jclass result = (*recorderRecord)->SetRecordState(recorderRecord, SL_RECORDSTATE_RECORDING); assert(SL_RESULT_SUCCESS == result); (void)result; + bqPlayerRecorderBusy = 1; } From 83fed58ef8063580b07b31eb8c9b6e478d9186da Mon Sep 17 00:00:00 2001 From: ggfan Date: Fri, 21 Aug 2015 08:25:57 -0700 Subject: [PATCH 2/5] add comment for fast audio sample rate --- .../src/main/java/com/example/nativeaudio/NativeAudio.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java index 3b41e0a8..10361bd7 100644 --- a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java +++ b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java @@ -63,6 +63,12 @@ public class NativeAudio extends Activity { int sampleRate = 0; int bufSize = 0; + /* + * retrieve fast audio path sample rate and buf size; if we have it, we pass to native + * side to create a player with fast audio enabled [ fast audio == low latency audio ]; + * IF we do not have a fast audio path, we pass 0 for sampleRate, which will force native + * side to pick up the 8Khz sample rate. + */ if( android.os.Build.VERSION.SDK_INT > android.os.Build.VERSION_CODES.JELLY_BEAN_MR1) { AudioManager myAudioMgr = (AudioManager) getSystemService(Context.AUDIO_SERVICE); String nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE); From 7c128c1d40e4d07e45fe790ef59361495c7f8566 Mon Sep 17 00:00:00 2001 From: ggfan Date: Fri, 21 Aug 2015 09:05:16 -0700 Subject: [PATCH 3/5] addressing code review comment --- .../src/main/java/com/example/nativeaudio/NativeAudio.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java index 10361bd7..56e6c519 100644 --- a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java +++ b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java @@ -20,6 +20,7 @@ import android.app.Activity; import android.content.Context; import android.content.res.AssetManager; import android.media.AudioManager; +import android.os.Build; import android.os.Bundle; //import android.util.Log; import android.view.View; @@ -69,7 +70,7 @@ public class NativeAudio extends Activity { * IF we do not have a fast audio path, we pass 0 for sampleRate, which will force native * side to pick up the 8Khz sample rate. */ - if( android.os.Build.VERSION.SDK_INT > android.os.Build.VERSION_CODES.JELLY_BEAN_MR1) { + if(Build.VERSION.SDK_INT > Build.VERSION_CODES.JELLY_BEAN_MR1) { AudioManager myAudioMgr = (AudioManager) getSystemService(Context.AUDIO_SERVICE); String nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE); sampleRate = Integer.parseInt(nativeParam); @@ -305,7 +306,7 @@ public class NativeAudio extends Activity { /** Native methods, implemented in jni folder */ public static native void createEngine(); - public static native void createBufferQueueAudioPlayer(int rate, int size); + public static native void createBufferQueueAudioPlayer(int sampleRate, int samplesPerBuf); public static native boolean createAssetAudioPlayer(AssetManager assetManager, String filename); // true == PLAYING, false == PAUSED public static native void setPlayingAssetAudioPlayer(boolean isPlaying); From d6ca1c7a89103588f075fa7c76cb86f7d3fba612 Mon Sep 17 00:00:00 2001 From: ggfan Date: Fri, 21 Aug 2015 09:13:39 -0700 Subject: [PATCH 4/5] address one more code review comment --- native-audio/app/src/main/jni/native-audio-jni.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native-audio/app/src/main/jni/native-audio-jni.c b/native-audio/app/src/main/jni/native-audio-jni.c index f8e91346..e3162465 100644 --- a/native-audio/app/src/main/jni/native-audio-jni.c +++ b/native-audio/app/src/main/jni/native-audio-jni.c @@ -133,7 +133,7 @@ short* createResampledBuf(uint32_t idx, uint32_t srcRate, unsigned *size) { short *src = NULL; short *workBuf; int upSampleRate; - int32_t srcSampleCount =0; + int32_t srcSampleCount = 0; if(0 == bqPlayerSampleRate) { return NULL; From 5b374ec9cc6d2483ff1ee8f5a777bcc5805a5bb6 Mon Sep 17 00:00:00 2001 From: ggfan Date: Fri, 21 Aug 2015 09:23:15 -0700 Subject: [PATCH 5/5] removing unused andoird.log import --- .../app/src/main/java/com/example/nativeaudio/NativeAudio.java | 1 - 1 file changed, 1 deletion(-) diff --git a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java index 56e6c519..600e97a6 100644 --- a/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java +++ b/native-audio/app/src/main/java/com/example/nativeaudio/NativeAudio.java @@ -22,7 +22,6 @@ import android.content.res.AssetManager; import android.media.AudioManager; import android.os.Build; import android.os.Bundle; -//import android.util.Log; import android.view.View; import android.view.View.OnClickListener; import android.widget.AdapterView;