LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add sample app for VisualQueryDetection connection

Browse Source 
Test: Manual
Bug: 261517532
Change-Id: I5121f86b5ec3469813f5f3d95dfba01d39ce6f1c
This commit is contained in:
Charles Chen
2022-12-07 02:22:08 +00:00
parent 35e7e0cc27
commit d644524452
3 changed files with 307 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
samples/VoiceInteractionService/AndroidManifest.xml
View File

@@ -26,16 +26,19 @@
android:name=".SampleHotwordDetectionService"
android:permission="android.permission.BIND_HOTWORD_DETECTION_SERVICE"
android:isolatedProcess="true"
android:allowSharedIsolatedProcess="true"
android:exported="true">
</service>
<service
android:name=".SampleVisualQueryDetectionService"
android:permission="android.permission.BIND_VISUAL_QUERY_DETECTION_SERVICE"
android:isolatedProcess="true"
android:allowSharedIsolatedProcess="true"
android:exported="true">
</service>
</application>
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.RECORD_BACKGROUND_AUDIO" />
<uses-permission android:name="android.permission.CAPTURE_AUDIO_HOTWORD" />

283
samples/VoiceInteractionService/src/com/example/android/voiceinteractor/SampleVisualQueryDetectionService.java
View File

@@ -16,18 +16,89 @@
package com.example.android.voiceinteractor;
import android.graphics.ImageFormat;
import android.hardware.camera2.CameraAccessException;
import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.CameraManager;
import android.hardware.camera2.CameraMetadata;
import android.hardware.camera2.CaptureRequest;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.Image;
import android.media.ImageReader;
import android.media.MediaRecorder;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.os.SystemClock;
import android.service.voice.VisualQueryDetectionService;
import android.util.Log;
import android.util.Range;
import android.util.Size;
import android.view.Surface;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import java.util.function.IntConsumer;
/**
* Sample VisualQueryDetectionService that captures camera frame and sends back partial query.
*/
public class SampleVisualQueryDetectionService extends VisualQueryDetectionService {
static final String TAG = "SVisualQueryDetectionSrvc";
private final String FAKE_QUERY = "What is the weather today?";
// Service related variables
private Callback mCallback;
// Camera module related variables
// Set this to different values for different modes
private final int CAPTURE_MODE = CameraDevice.TEMPLATE_RECORD;
private String mCameraId;
private CaptureRequest.Builder mCaptureRequestBuilder;
private CameraCaptureSession mCameraCaptureSession;
private CameraDevice mCameraDevice;
private ImageReader mImageReader;
private Handler mCameraBackgroundHandler;
private HandlerThread mCameraBackgroundThread;
// audio module related variables
private static final int AUDIO_SAMPLE_RATE_IN_HZ = 16000;
private static final int AUDIO_CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO;
private static final int AUDIO_FORMAT = AudioFormat.ENCODING_DEFAULT;
private static final int BUFFER_SIZE = AUDIO_SAMPLE_RATE_IN_HZ;
private AudioRecord mAudioRecord;
private Handler mAudioBackgroundHandler;
private HandlerThread mAudioBackgroundThread;
@Override
public void onStartDetection(@NonNull Callback callback) {
Log.i(TAG, "onStartDetection");
mCallback = callback;
startBackgroundThread();
openCamera();
mAudioRecord = new AudioRecord(MediaRecorder.AudioSource.DEFAULT,
AUDIO_SAMPLE_RATE_IN_HZ, AUDIO_CHANNEL_CONFIG, AUDIO_FORMAT, BUFFER_SIZE);
}
@Override
public void onStopDetection() {
Log.i(TAG, "onStopDetection");
mCallback = null;
releaseResources();
stopBackgroundThread();
}
@Override
public void onUpdateState(@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory, long callbackTimeoutMillis,
@@ -37,4 +108,216 @@ public class SampleVisualQueryDetectionService extends VisualQueryDetectionServi
statusCallback.accept(0);
}
}
@Override
public void onDestroy() {
Log.d(TAG, "Destroying visual query detection service");
onStopDetection();
}
/* Main logics of the system */
private void onReceiveImage(ImageReader reader){
Log.i(TAG, "Image received.");
Image image = null;
try {
image = reader.acquireLatestImage();
ByteBuffer buffer = image.getPlanes()[0].getBuffer();
byte[] bytes = new byte[buffer.capacity()];
buffer.get(bytes);
// Camera frame triggers attention
Log.i(TAG, "Image bytes received: " + Arrays.toString(bytes));
mCallback.onAttentionGained();
openMicrophone();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (image != null) {
image.close();
}
}
SystemClock.sleep(2_000); // wait 2 second to turn off attention
closeMicrophone();
mCallback.onAttentionLost();
}
private void onReceiveAudio(){
try {
byte[] bytes = new byte[BUFFER_SIZE];
int result = mAudioRecord.read(bytes, 0, BUFFER_SIZE);
if (result != AudioRecord.ERROR_INVALID_OPERATION) {
// The buffer can be all zeros due to initialization and reading delay
Log.i(TAG, "Audio bytes received: " + Arrays.toString(bytes));
mCallback.onQueryDetected(FAKE_QUERY);
mCallback.onQueryFinished();
}
} catch (Exception e) {
e.printStackTrace();
}
SystemClock.sleep(2_000); //sleep so the buffer is a stable value
}
/* Sample Camera Module */
private void openCamera() {
CameraManager manager = getSystemService(CameraManager.class);
Log.i(TAG, "Attempting to open camera");
try {
mCameraId = manager.getCameraIdList()[0]; //get front facing camera
CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraId);
Size imageSize = characteristics.get(
CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
.getOutputSizes(ImageFormat.JPEG)[0];
initializeImageReader(imageSize.getWidth(), imageSize.getHeight());
manager.openCamera(mCameraId, stateCallback, mCameraBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
Log.i(TAG, "Camera opened.");
}
private void initializeImageReader(int width, int height) {
// Initialize image reader
mImageReader = ImageReader.newInstance(width, height, ImageFormat.JPEG, 2);
ImageReader.OnImageAvailableListener readerListener =
new ImageReader.OnImageAvailableListener() {
@Override
public void onImageAvailable(ImageReader reader) {
onReceiveImage(reader);
}
} ;
mImageReader.setOnImageAvailableListener(readerListener, mCameraBackgroundHandler);
}
private final CameraDevice.StateCallback stateCallback = new CameraDevice.StateCallback() {
@Override
public void onOpened(CameraDevice camera) {
// This is called when the camera is open
Log.i(TAG, "onCameraOpened");
mCameraDevice = camera;
createCameraPreview();
}
@Override
public void onDisconnected(CameraDevice camera) {
mCameraDevice.close();
}
@Override
public void onError(CameraDevice camera, int error) {
mCameraDevice.close();
mCameraDevice = null;
}
};
private void createCameraPreview() {
Range<Integer> fpsRange = new Range<>(1,2);
try {
mCaptureRequestBuilder = mCameraDevice.createCaptureRequest(CAPTURE_MODE);
Surface imageSurface = mImageReader.getSurface();
mCaptureRequestBuilder.addTarget(imageSurface);
mCaptureRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
mCameraDevice.createCaptureSession(List.of(imageSurface),
new CameraCaptureSession.StateCallback() {
@Override
public void onConfigured(
@NonNull CameraCaptureSession cameraCaptureSession) {
//The camera is already closed
if (mCameraDevice == null) {
return;
}
// When the session is ready, we start displaying the preview.
mCameraCaptureSession = cameraCaptureSession;
updatePreview();
Log.i(TAG, "Capture session configured.");
}
@Override
public void onConfigureFailed(
@NonNull CameraCaptureSession cameraCaptureSession) {
//No-op
}
}, null);
} catch (CameraAccessException e) {
e.printStackTrace();
}
Log.i(TAG, "Camera preview created.");
}
private void updatePreview() {
if (null == mCameraDevice) {
Log.e(TAG, "updatePreview error, return");
}
mCaptureRequestBuilder.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO);
try {
if (CAPTURE_MODE == CameraDevice.TEMPLATE_STILL_CAPTURE
|| CAPTURE_MODE == CameraDevice.TEMPLATE_VIDEO_SNAPSHOT) {
mCameraCaptureSession.capture(mCaptureRequestBuilder.build(), null,
mCameraBackgroundHandler);
} else if (CAPTURE_MODE == CameraDevice.TEMPLATE_RECORD
|| CAPTURE_MODE == CameraDevice.TEMPLATE_PREVIEW){
mCameraCaptureSession.setRepeatingRequest(mCaptureRequestBuilder.build(), null,
mCameraBackgroundHandler);
} else {
throw new IllegalStateException("Capture mode not supported.");
}
} catch (Exception e) {
e.printStackTrace();
}
}
/* Sample Microphone Module */
private void openMicrophone() {
mAudioRecord.startRecording();
mAudioBackgroundHandler.post(this::onReceiveAudio);
}
private void closeMicrophone() {
if (mAudioRecord != null) {
mAudioRecord.stop();
}
}
private void releaseResources() {
mCameraId = null;
mCaptureRequestBuilder = null;
// Release mCameraCaptureSession
mCameraCaptureSession.close();
mCameraCaptureSession = null;
// Release mCameraDevice
mCameraDevice.close();
mCameraDevice = null;
// Release mImageReader
mImageReader.close();
mImageReader = null;
// Release mAudioRecord
mAudioRecord.release();
mAudioRecord = null;
}
// Handlers
private void startBackgroundThread() {
mCameraBackgroundThread = new HandlerThread("Camera Background Thread");
mCameraBackgroundThread.start();
mCameraBackgroundHandler = new Handler(mCameraBackgroundThread.getLooper());
mAudioBackgroundThread = new HandlerThread("Audio Background Thread");
mAudioBackgroundThread.start();
mAudioBackgroundHandler = new Handler(mAudioBackgroundThread.getLooper());
}
private void stopBackgroundThread() {
mCameraBackgroundThread.quitSafely();
try {
mCameraBackgroundThread.join();
mCameraBackgroundThread = null;
mCameraBackgroundHandler = null;
} catch (InterruptedException e) {
Log.e(TAG, "Failed to stop camera thread.");
}
mAudioBackgroundThread.quitSafely();
try {
mAudioBackgroundThread.join();
mAudioBackgroundThread = null;
mAudioBackgroundHandler = null;
} catch (InterruptedException e) {
Log.e(TAG, "Failed to stop audio thread.");
}
}
}

27
samples/VoiceInteractionService/src/com/example/android/voiceinteractor/SampleVoiceInteractionService.java
View File

@@ -35,6 +35,7 @@ import android.service.voice.AlwaysOnHotwordDetector.EventPayload;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordDetector.IllegalDetectorStateException;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.SandboxedDetectionServiceBase;
import android.service.voice.VisualQueryDetector;
import android.service.voice.VoiceInteractionService;
import android.util.Log;
@@ -100,10 +101,9 @@ public class SampleVoiceInteractionService extends VoiceInteractionService {
Log.i(TAG, "onReady");
mHotwordDetectorCallback = new Callback();
mVisualQueryDetectorCallback = new VisualQueryDetectorCallback();
mHotwordDetector = createAlwaysOnHotwordDetector(DSP_MODEL_KEYPHRASE, DSP_MODEL_LOCALE, null, null,
mHotwordDetectorCallback);
mVisualQueryDetector = createVisualQueryDetector(null, null,
Executors.newSingleThreadExecutor(), mVisualQueryDetectorCallback);
mHotwordDetector = createAlwaysOnHotwordDetector(DSP_MODEL_KEYPHRASE,
DSP_MODEL_LOCALE, null, null, mHotwordDetectorCallback);
}
@Override
@@ -119,7 +119,7 @@ public class SampleVoiceInteractionService extends VoiceInteractionService {
}
}
static class VisualQueryDetectorCallback implements VisualQueryDetector.Callback {
class VisualQueryDetectorCallback implements VisualQueryDetector.Callback {
@Override
public void onQueryDetected(@NonNull String partialQuery) {
Log.i(TAG, "VQD partial query detected: "+ partialQuery);
@@ -137,12 +137,24 @@ public class SampleVoiceInteractionService extends VoiceInteractionService {
@Override
public void onVisualQueryDetectionServiceInitialized(int status) {
Log.i(TAG, "VQD init: "+ status);
Log.i(TAG, "VQD init: "+ status);
if (status == SandboxedDetectionServiceBase.INITIALIZATION_STATUS_SUCCESS) {
try {
mVisualQueryDetector.startRecognition();
} catch (IllegalDetectorStateException e) {
e.printStackTrace();
}
}
}
@Override
public void onVisualQueryDetectionServiceRestarted() {
Log.i(TAG, "VQD restarted");
try {
mVisualQueryDetector.startRecognition();
} catch (IllegalDetectorStateException e) {
e.printStackTrace();
}
}
@Override
@@ -307,6 +319,9 @@ public class SampleVoiceInteractionService extends VoiceInteractionService {
e.printStackTrace();
}
}
//TODO(b/265535257): Provide two services independent lifecycle.
mVisualQueryDetector = createVisualQueryDetector(null, null,
Executors.newSingleThreadExecutor(), mVisualQueryDetectorCallback);
}
}
}