LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "HotwordDetectionService prototype."

Browse Source
This commit is contained in:
TreeHugger Robot
2022-08-11 23:36:59 +00:00
committed by Android (Google) Code Review
parent 627ff7d85a ae5665592c
commit 4f3f02623f
11 changed files with 897 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
samples/VoiceInteractionService/Android.bp Normal file
View File

@@ -0,0 +1,15 @@
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
android_app {
name: "SampleVoiceInteractor",
srcs: ["**/*.java"],
min_sdk_version: "30",
target_sdk_version: "30",
sdk_version: "system_current",
privileged: true,
static_libs: [
"androidx.annotation_annotation",
],
}

35
samples/VoiceInteractionService/AndroidManifest.xml Executable file
View File

@@ -0,0 +1,35 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.example.android.voiceinteractor">
<application android:label="@string/app_name">
<activity android:name=".MainActivity">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<service
android:name=".SampleVoiceInteractionService"
android:label="@string/app_name"
android:permission="android.permission.BIND_VOICE_INTERACTION">
<intent-filter>
<action android:name="android.service.voice.VoiceInteractionService" />
</intent-filter>
<meta-data
android:name="android.voice_interaction"
android:resource="@xml/voice_interaction" />
</service>
<service
android:name=".SampleHotwordDetectionService"
android:permission="android.permission.BIND_HOTWORD_DETECTION_SERVICE"
android:isolatedProcess="true"
android:exported="true">
</service>
</application>
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.RECORD_BACKGROUND_AUDIO" />
<uses-permission android:name="android.permission.CAPTURE_AUDIO_HOTWORD" />
<uses-permission android:name="android.permission.MANAGE_HOTWORD_DETECTION" />
</manifest>

45
samples/VoiceInteractionService/README.md Normal file
View File

@@ -0,0 +1,45 @@
setup:
1. Set the KEYPHRASE constant in SampleVoiceInteractionService.java to something the device's
default assistant supports.
2. m -j SampleVoiceInteractor
3. adb pull ./system/etc/permissions/privapp-permissions-platform.xml
4. Add:
<privapp-permissions package="com.example.android.voiceinteractor">
<permission name="android.permission.CAPTURE_AUDIO_HOTWORD"/>
</privapp-permissions>
5. adb remount
6. adb push privapp-permissions-platform.xml /system/etc/permissions/privapp-permissions-platform.xml
7. adb shell mkdir /system/priv-app/SampleVoiceInteractor
8. adb push out/target/product/$TARGET_PRODUCT/system/priv-app/SampleVoiceInteractor/SampleVoiceInteractor.apk /system/priv-app/SampleVoiceInteractor/
9. adb reboot
10. Go to the sample app info/settings.
11. Tap on Permissions and grant Mic access.
12. Reboot.
13. Set the sample app as the assistant.
14. Check for this in the logs to make sure it worked:
com.example.android.voiceinteractor I/VIS: onAvailabilityChanged: 2
15. If it didn't, check if the pregrant worked:
adb shell dumpsys package com.example.android.voiceinteractor | grep CAPTURE_AUDIO_HOTWORD
Iterating:
* adb install like usual
* If syncing changes to the system image, either first copy the permissions file into
out/target/product/system/etc/permissions/ or push it again after syncing. Sometimes you might
need to uninstall the app (go to the sample app info/settings -> 3 dots menu -> uninstall
updates).
to test:
1. Say "1,2,Ok Poodle,3,4.."
2. Check the logs for the app and wait till it finishes recording.
3. Either check the logs for the sampled bytes to match, e.g. "sample=[95, 2, 97, ...]" should
appear twice; or open the sample app activity and click the button to play back the recorded
audio.
Tap directRecord to simulate the non-DSP case (must be done after a dsp trigger since it
reuses the previous data).
Debugging:
* Set DEBUG to true in AlwaysOnHotwordDetector
* uncomment LOG_NDEBUG lines at the top in AudioFlinger.cpp, Threads.cpp, Tracks.cpp,
AudioPolicyInterfaceImpl.cpp, AudioPolicyService.cpp
* Use this logcat filter:
com.example.android.voiceinteractor|AlwaysOnHotword|SoundTrigger|RecordingActivityMonitor|soundtrigger|AudioPolicyManager|AudioFlinger|AudioPolicyIntefaceImpl|AudioPolicyService

22
samples/VoiceInteractionService/com.example.android.voiceinteractor.xml Normal file
View File

@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright (C) 2021 The Android Open Source Project
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<permissions>
<privapp-permissions package="com.example.android.voiceinteractor">
<permission name="android.permission.CAPTURE_AUDIO_HOTWORD"/>
</privapp-permissions>
</permissions>

50
samples/VoiceInteractionService/res/layout/main_activity.xml Normal file
View File

@@ -0,0 +1,50 @@
<?xml version="1.0" encoding="utf-8"?><!-- Copyright (C) 2021 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent">
<Button
android:id="@+id/buffer1"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="buffer1" />
<Button
android:id="@+id/buffer2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="buffer2" />
<Button
android:id="@+id/startReco"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="startRecognition" />
<Button
android:id="@+id/directRecord"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="direct record" />
<!-- <ScrollView-->
<!-- android:layout_width="match_parent"-->
<!-- android:layout_height="match_parent">-->
<!-- <TextView-->
<!-- android:layout_width="match_parent"-->
<!-- android:layout_height="wrap_content"-->
<!-- android:text="test" />-->
<!-- </ScrollView>-->
</LinearLayout>

23
samples/VoiceInteractionService/res/values/strings.xml Normal file
View File

@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
/*
**
** Copyright 2019, The Android Open Source Project
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
*/
-->
<resources>
<string name="app_name">Sample Voice Interactor</string>
</resources>

23
samples/VoiceInteractionService/res/xml/voice_interaction.xml Normal file
View File

@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2021 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<voice-interaction-service xmlns:android="http://schemas.android.com/apk/res/android"
android:sessionService=""
android:hotwordDetectionService="com.example.android.voiceinteractor.SampleHotwordDetectionService"
android:recognitionService=""
android:settingsActivity=""
android:supportsAssist="true"
android:supportsLocalInteraction="true" />

54
samples/VoiceInteractionService/src/com/example/android/voiceinteractor/AudioUtils.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.example.android.voiceinteractor;
import android.media.AudioRecord;
import android.util.Log;
import java.util.Arrays;
public class AudioUtils {
private static String TAG = "Hotword-AudioUtils";
static int read(AudioRecord record, int bytesPerSecond, float secondsToRead, byte[] buffer) {
int numBytes = 0;
int nextSecondToSample = 0;
while (true) {
int bytesRead = record.read(buffer, numBytes, numBytes + 1024);
numBytes += bytesRead;
if (bytesRead <= 0) {
Log.i(TAG, "Finished reading, last read()=" + bytesRead);
break;
}
int curSecond = numBytes / bytesPerSecond;
if (curSecond == nextSecondToSample
&& numBytes > (bytesPerSecond * curSecond) + 10) {
Log.i(TAG, "sample=" + Arrays.toString(
Arrays.copyOfRange(
buffer, bytesPerSecond * curSecond,
(bytesPerSecond * curSecond) + 10)));
nextSecondToSample++;
}
if (numBytes * 1.0 / bytesPerSecond >= secondsToRead) {
Log.i(TAG, "recorded enough. stopping.");
break;
}
}
return numBytes;
}
}

212
samples/VoiceInteractionService/src/com/example/android/voiceinteractor/MainActivity.java Normal file
View File

@@ -0,0 +1,212 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.example.android.voiceinteractor;
import static android.media.AudioTrack.PLAYSTATE_PAUSED;
import static android.media.AudioTrack.PLAYSTATE_STOPPED;
import android.app.Activity;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.ServiceConnection;
import android.media.AudioAttributes;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioTrack;
import android.os.Bundle;
import android.os.IBinder;
import android.service.voice.HotwordDetector;
import android.util.Log;
import android.view.View;
import android.widget.Button;
public class MainActivity extends Activity {
private static final String TAG = "VIS";
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.main_activity);
attachClickListener(R.id.buffer1, "1");
attachClickListener(R.id.buffer2, "2");
Button button = (Button) findViewById(R.id.startReco);
button.setOnClickListener(new View.OnClickListener() {
public void onClick(View v) {
if (mService == null) {
Log.e(TAG, "No service");
return;
}
try {
mService.mDetector.startRecognition();
} catch (HotwordDetector.IllegalDetectorStateException e) {
e.printStackTrace();
}
}
});
button = (Button) findViewById(R.id.directRecord);
button.setOnClickListener(v -> {
if (mService == null) {
Log.e(TAG, "No service");
return;
}
mService.mCallback.onDetected(mService.mLastPayload, true);
});
}
private void attachClickListener(int id, String key) {
Button button = (Button) findViewById(id);
button.setOnClickListener(v -> {
if (mService == null) {
Log.e(TAG, "No service");
return;
}
playAudio(mService.mData.getByteArray(key));
});
}
@Override
protected void onStart() {
super.onStart();
// Bind to LocalService
Intent intent = new Intent(this, SampleVoiceInteractionService.class).setAction("local");
bindService(intent, connection, Context.BIND_AUTO_CREATE);
}
SampleVoiceInteractionService mService;
boolean mBound = false;
private final ServiceConnection connection = new ServiceConnection() {
@Override
public void onServiceConnected(ComponentName className,
IBinder service) {
// We've bound to LocalService, cast the IBinder and get LocalService instance
SampleVoiceInteractionService.LocalBinder binder = (SampleVoiceInteractionService.LocalBinder) service;
mService = binder.getService();
mBound = true;
Log.i(TAG, "Connected to local service");
}
@Override
public void onServiceDisconnected(ComponentName arg0) {
mService = null;
mBound = false;
}
};
private void playAudio(byte[] buffer) {
AudioTrack track = new AudioTrack(
new AudioAttributes.Builder()
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.setUsage(AudioAttributes.USAGE_MEDIA)
// .setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD)
.build(),
new AudioFormat.Builder()
.setChannelMask(AudioFormat.CHANNEL_IN_DEFAULT)
.setSampleRate(mService.mAudioFormat.getSampleRate())
.setEncoding(mService.mAudioFormat.getEncoding())
.build(),
// mService.mAudioFormat,
buffer.length,
AudioTrack.MODE_STATIC,
AudioManager.AUDIO_SESSION_ID_GENERATE
);
Log.i(TAG, "track state=" + track.getState());
if (track.getState() == AudioTrack.STATE_UNINITIALIZED) {
return;
}
track.write(buffer, 0, buffer.length);
// track.setNotificationMarkerPosition(track.getP)
track.play();
// TODO: Doesn't work.. fix the releasing.
track.setPlaybackPositionUpdateListener(new AudioTrack.OnPlaybackPositionUpdateListener() {
@Override
public void onMarkerReached(AudioTrack track) {
}
@Override
public void onPeriodicNotification(AudioTrack track) {
if (track.getPlayState() == PLAYSTATE_STOPPED
|| track.getPlayState() == PLAYSTATE_PAUSED) {
Log.i(TAG, "Stopped/paused playback; releasing.");
track.release();
}
}
});
// try {
// Thread.sleep(4000);
// } catch (InterruptedException e) {
// Thread.interrupted();
// throw new RuntimeException(e);
// }
// track.release();
// MediaPlayer player = new MediaPlayer();
// player.setAudioAttributes(
// new AudioAttributes.Builder()
// .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
// .setUsage(AudioAttributes.USAGE_MEDIA)
// .set
// .build());
// player.setDataSource(
// "data:audio/mp3;base64,"
//// new ByteArrayMediaSource(buffer)
// );
// try {
// player.prepare();
// } catch (IOException e) {
// Log.e(TAG, "Failed to play: " + e);
// }
// player.start();
}
// private static class ByteArrayMediaSource extends MediaDataSource {
// final byte[] mData;
//
// public ByteArrayMediaSource(byte[] data) {
// mData = data;
// }
//
// @Override
// public int readAt(long position, byte[] buffer, int offset, int size) throws IOException {
// if (position >= mData.length) {
// return -1; // end of stream
// }
// if (position + size > mData.length) {
// size = (int) (mData.length - position);
// }
//
// System.arraycopy(mData, (int) position, buffer, offset, size);
// return size;
// }
//
// @Override
// public long getSize() throws IOException {
// return 0;
// }
//
// @Override
// public void close() throws IOException {
//
// }
// }
}

155
samples/VoiceInteractionService/src/com/example/android/voiceinteractor/SampleHotwordDetectionService.java Normal file
View File

@@ -0,0 +1,155 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.example.android.voiceinteractor;
import android.media.AudioAttributes;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Handler;
import android.os.Looper;
import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.service.voice.AlwaysOnHotwordDetector;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordRejectedResult;
import android.util.Log;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.function.IntConsumer;
public class SampleHotwordDetectionService extends HotwordDetectionService {
static final String TAG = "SHotwordDetectionSrvc";
// Number of bytes per sample of audio (which is a short).
private static final int BYTES_PER_SAMPLE = 2;
@Override
public void onUpdateState(@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory, long callbackTimeoutMillis,
@Nullable IntConsumer statusCallback) {
Log.i(TAG, "onUpdateState");
if (statusCallback != null) {
statusCallback.accept(0);
}
}
@Override
public void onDetect(
@NonNull AlwaysOnHotwordDetector.EventPayload eventPayload,
long timeoutMillis,
@NonNull Callback callback) {
Log.d(TAG, "onDetect (Hardware trigger)");
int sampleRate = eventPayload.getCaptureAudioFormat().getSampleRate();
int bytesPerSecond = BYTES_PER_SAMPLE * sampleRate;
Integer captureSession = 0;
try {
Method getCaptureSessionMethod = eventPayload.getClass().getMethod("getCaptureSession");
captureSession = (Integer) getCaptureSessionMethod.invoke(eventPayload);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
e.printStackTrace();
}
int sessionId =
// generateSessionId ?
// AudioManager.AUDIO_SESSION_ID_GENERATE :
captureSession;
AudioRecord record = createAudioRecord(eventPayload, bytesPerSecond, sessionId);
if (record.getState() != AudioRecord.STATE_INITIALIZED) {
Log.e(TAG, "Failed to init first AudioRecord.");
callback.onRejected(new HotwordRejectedResult.Builder().build());
return;
}
byte[] buffer = new byte[bytesPerSecond * 10];
record.startRecording();
AudioUtils.read(record, bytesPerSecond, .75f, buffer);
callback.onDetected(
new HotwordDetectedResult.Builder()
.setMediaSyncEvent(
record.shareAudioHistory("com.example.android.voiceinteractor", 0))
.setHotwordPhraseId(getKeyphraseId(eventPayload))
.build());
new Handler(Looper.getMainLooper()).postDelayed(() -> {
Log.i(TAG, "Releasing audio record");
record.stop();
record.release();
}, 5000);
}
private int getKeyphraseId(AlwaysOnHotwordDetector.EventPayload payload) {
return 0;
// if (payload.getKeyphraseRecognitionExtras().isEmpty()) {
// return 0;
// }
// return payload.getKeyphraseRecognitionExtras().get(0).getKeyphraseId();
}
@Override
public void onDetect(@NonNull Callback callback) {
int sampleRate = 16000;
int bytesPerSecond = BYTES_PER_SAMPLE * sampleRate;
AudioRecord record = new AudioRecord.Builder()
.setAudioAttributes(new AudioAttributes.Builder()
.setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD).build())
.setAudioFormat(
new AudioFormat.Builder()
.setChannelMask(AudioFormat.CHANNEL_IN_MONO)
.setEncoding(AudioFormat.ENCODING_DEFAULT)
.setSampleRate(sampleRate)
.build())
.setBufferSizeInBytes(getBufferSizeInBytes(bytesPerSecond, 15))
.setMaxSharedAudioHistoryMillis(AudioRecord.getMaxSharedAudioHistoryMillis())
.build();
if (record.getState() != AudioRecord.STATE_INITIALIZED) {
Log.w(TAG, "Failed to initialize AudioRecord");
record.release();
}
record.startRecording();
byte[] buffer = new byte[bytesPerSecond * 10];
int numBytes = AudioUtils.read(record, bytesPerSecond, .75f, buffer);
}
private static AudioRecord createAudioRecord(AlwaysOnHotwordDetector.EventPayload eventPayload,
int bytesPerSecond,
int sessionId) {
return new AudioRecord.Builder()
.setAudioAttributes(
new AudioAttributes.Builder()
.setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD)
// TODO see what happens if this is too small
.build())
.setAudioFormat(eventPayload.getCaptureAudioFormat())
.setBufferSizeInBytes(getBufferSizeInBytes(bytesPerSecond, 1))
.setSessionId(sessionId)
.setMaxSharedAudioHistoryMillis(AudioRecord.getMaxSharedAudioHistoryMillis())
.build();
}
private static int getBufferSizeInBytes(int bytesPerSecond, float bufferLengthSeconds) {
return (int) (bytesPerSecond * bufferLengthSeconds);
}
}

263
samples/VoiceInteractionService/src/com/example/android/voiceinteractor/SampleVoiceInteractionService.java Normal file
View File

@@ -0,0 +1,263 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.example.android.voiceinteractor;
import static android.service.voice.AlwaysOnHotwordDetector.STATE_HARDWARE_UNAVAILABLE;
import static android.service.voice.AlwaysOnHotwordDetector.STATE_KEYPHRASE_ENROLLED;
import static android.service.voice.AlwaysOnHotwordDetector.STATE_KEYPHRASE_UNENROLLED;
import android.content.ComponentName;
import android.content.Intent;
import android.media.AudioAttributes;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Binder;
import android.os.Bundle;
import android.os.IBinder;
import android.service.voice.AlwaysOnHotwordDetector;
import android.service.voice.AlwaysOnHotwordDetector.EventPayload;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.VoiceInteractionService;
import android.util.Log;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Locale;
public class SampleVoiceInteractionService extends VoiceInteractionService {
private static final String TAG = "VIS";
// Number of bytes per sample of audio (which is a short).
private static final int BYTES_PER_SAMPLE = 2;
public static final String KEYPHRASE = "X Android";
private final IBinder binder = new LocalBinder();
public class LocalBinder extends Binder {
SampleVoiceInteractionService getService() {
// Return this instance of LocalService so clients can call public methods
return SampleVoiceInteractionService.this;
}
}
@Override
public IBinder onBind(Intent intent) {
if ("local".equals(intent.getAction())) {
return binder;
}
return super.onBind(intent);
}
HotwordDetector mDetector;
Callback mCallback;
Bundle mData = new Bundle();
AudioFormat mAudioFormat;
EventPayload mLastPayload;
@Override
public void onReady() {
super.onReady();
Log.i(TAG, "onReady");
mCallback = new Callback();
mDetector = createAlwaysOnHotwordDetector(KEYPHRASE, Locale.US, null, null, mCallback);
}
@Override
public void onShutdown() {
super.onShutdown();
Log.i(TAG, "onShutdown");
}
class Callback extends AlwaysOnHotwordDetector.Callback {
private boolean mAvailable = false;
@Override
public void onAvailabilityChanged(int status) {
Log.i(TAG, "onAvailabilityChanged: " + status);
if (status == STATE_HARDWARE_UNAVAILABLE) {
// adb shell dumpsys package com.example.android.voiceinteractor | grep HOTWO
Log.w(
TAG,
"Hotword hardware unavailable. You may need to pre-grant "
+ "CAPTURE_AUDIO_HOTWORD to this app, grant record audio to the app"
+ "in settings, and/or change the keyphrase "
+ "to one supported by the device's default assistant.");
}
if (status == STATE_KEYPHRASE_UNENROLLED) {
Intent enrollIntent = null;
try {
enrollIntent = ((AlwaysOnHotwordDetector) mDetector).createEnrollIntent();
} catch (HotwordDetector.IllegalDetectorStateException e) {
e.printStackTrace();
}
if (enrollIntent == null) {
Log.w(TAG, "No enroll intent found. Try enrolling the keyphrase using the"
+ " device's default assistant.");
return;
}
ComponentName component = startForegroundService(enrollIntent);
Log.i(TAG, "Start enroll intent: " + component);
}
if (status == STATE_KEYPHRASE_ENROLLED) {
Log.i(TAG, "Keyphrase enrolled; ready to recognize.");
mAvailable = true;
}
}
@Override
public void onRejected(@NonNull HotwordRejectedResult result) {
try {
mDetector.startRecognition();
} catch (HotwordDetector.IllegalDetectorStateException e) {
e.printStackTrace();
}
}
@Override
public void onDetected(@NonNull EventPayload eventPayload) {
onDetected(eventPayload, false);
}
public void onDetected(@NonNull EventPayload eventPayload, boolean generateSessionId) {
Log.i(TAG, "onDetected: " + eventPayload);
Log.i(TAG, "minBufferSize: "
+ AudioRecord.getMinBufferSize(
eventPayload.getCaptureAudioFormat().getSampleRate(),
eventPayload.getCaptureAudioFormat().getChannelMask(),
eventPayload.getCaptureAudioFormat().getEncoding()));
int sampleRate = eventPayload.getCaptureAudioFormat().getSampleRate();
int bytesPerSecond = BYTES_PER_SAMPLE * sampleRate;
// For Non-trusted:
// Integer captureSession = 0;
// try {
// Method getCaptureSessionMethod = eventPayload.getClass().getMethod("getCaptureSession");
// captureSession = (Integer) getCaptureSessionMethod.invoke(eventPayload);
// } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
// e.printStackTrace();
// }
// int sessionId = generateSessionId ?
// AudioManager.AUDIO_SESSION_ID_GENERATE : captureSession;
// AudioRecord record = createAudioRecord(eventPayload, bytesPerSecond, sessionId);
AudioRecord record = createAudioRecord(eventPayload, bytesPerSecond);
if (record.getState() != AudioRecord.STATE_INITIALIZED) {
Log.e(TAG, "Failed to init first AudioRecord.");
try {
mDetector.startRecognition();
} catch (HotwordDetector.IllegalDetectorStateException e) {
e.printStackTrace();
}
return;
}
byte[] buffer = new byte[bytesPerSecond * 6];
record.startRecording();
int numBytes = AudioUtils.read(record, bytesPerSecond, 5, buffer);
// try {
// Thread.sleep(2000);
// } catch (InterruptedException e) {
// Thread.interrupted();
// throw new RuntimeException(e);
// }
record.stop();
record.release();
Log.i(TAG, "numBytes=" + numBytes + " audioSeconds=" + numBytes * 1.0 / bytesPerSecond);
mData.putByteArray("1", buffer);
mAudioFormat = eventPayload.getCaptureAudioFormat();
mLastPayload = eventPayload;
try {
mDetector.startRecognition();
} catch (HotwordDetector.IllegalDetectorStateException e) {
e.printStackTrace();
}
}
@Override
public void onError() {
Log.i(TAG, "onError");
try {
mDetector.startRecognition();
} catch (HotwordDetector.IllegalDetectorStateException e) {
e.printStackTrace();
}
}
@Override
public void onRecognitionPaused() {
Log.i(TAG, "onRecognitionPaused");
}
@Override
public void onRecognitionResumed() {
Log.i(TAG, "onRecognitionResumed");
}
@Override
public void onHotwordDetectionServiceInitialized(int status) {
Log.i(TAG, "onHotwordDetectionServiceInitialized: " + status
+ ". mAvailable=" + mAvailable);
if (mAvailable) {
try {
mDetector.startRecognition();
} catch (HotwordDetector.IllegalDetectorStateException e) {
e.printStackTrace();
}
}
}
}
private static AudioRecord createAudioRecord(EventPayload eventPayload, int bytesPerSecond) {
return new AudioRecord.Builder()
.setAudioAttributes(
new AudioAttributes.Builder()
.setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD)
.build())
.setAudioFormat(eventPayload.getCaptureAudioFormat())
.setBufferSizeInBytes(getBufferSizeInBytes(bytesPerSecond, 2))
.setSharedAudioEvent(eventPayload.getHotwordDetectedResult().getMediaSyncEvent())
.build();
}
private static AudioRecord createAudioRecord(EventPayload eventPayload, int bytesPerSecond,
int sessionId) {
return new AudioRecord(
new AudioAttributes.Builder()
.setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD)
.build(),
eventPayload.getCaptureAudioFormat(),
getBufferSizeInBytes(bytesPerSecond, 2),
sessionId);
}
private static int getBufferSizeInBytes(int bytesPerSecond, float bufferLengthSeconds) {
return (int) (bytesPerSecond * bufferLengthSeconds);
}
}