/*
* Copyright (C) 2011 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.android.talkback;
import android.os.Message;
import android.support.annotation.NonNull;
import android.annotation.SuppressLint;
import android.content.Context;
import android.content.ClipboardManager;
import android.content.ClipData;
import android.content.SharedPreferences;
import android.content.SharedPreferences.OnSharedPreferenceChangeListener;
import android.content.res.Resources;
import android.media.AudioManager;
import android.media.AudioRecordingConfiguration;
import android.media.MediaRecorder.AudioSource;
import android.os.Bundle;
import android.os.Handler;
import android.speech.tts.TextToSpeech.Engine;
import android.support.v4.os.BuildCompat;
import android.text.SpannableStringBuilder;
import android.text.TextUtils;
import android.util.Log;
import android.view.KeyEvent;
import com.android.talkback.controller.FeedbackController;
import com.android.talkback.eventprocessor.EventState;
import com.android.utils.FailoverTextToSpeech;
import com.android.utils.LogUtils;
import com.android.utils.ProximitySensor;
import com.android.utils.SharedPreferencesUtils;
import com.android.utils.StringBuilderUtils;
import com.android.utils.compat.media.AudioSystemCompatUtils;
import com.google.android.marvin.talkback.TalkBackService;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.PriorityQueue;
import java.util.Set;
/**
* Handles text-to-speech.
*/
public class SpeechController implements TalkBackService.KeyEventListener {
/** Prefix for utterance IDs. */
private static final String UTTERANCE_ID_PREFIX = "talkback_";
/** Default stream for speech output. */
public static final int DEFAULT_STREAM = AudioManager.STREAM_MUSIC;
// Queue modes.
public static final int QUEUE_MODE_INTERRUPT = 0;
public static final int QUEUE_MODE_QUEUE = 1;
/**
* Similiar to QUEUE_MODE_QUEUE. The only difference is FeedbackItem in this mode cannot be
* interrupted by another while it is speaking. This includes not being removed from the queue
* unless shutdown is called.
*/
public static final int QUEUE_MODE_UNINTERRUPTIBLE = 2;
public static final int QUEUE_MODE_FLUSH_ALL = 3;
// Speech item status codes.
private static final int STATUS_ERROR = 1;
public static final int STATUS_SPEAKING = 2;
public static final int STATUS_INTERRUPTED = 3;
public static final int STATUS_SPOKEN = 4;
public static final int UTTERANCE_GROUP_DEFAULT = 0;
public static final int UTTERANCE_GROUP_TEXT_SELECTION = 1;
public static final int UTTERANCE_GROUP_SEEK_PROGRESS = 2;
/**
* The number of recently-spoken items to keep in history.
*/
private static final int MAX_HISTORY_ITEMS = 10;
/**
* The delay, in ms, after which a recently-spoken item will be considered for duplicate removal
* in the event that a new feedback item has the flag {@link FeedbackItem#FLAG_SKIP_DUPLICATE}.
* (The delay does not apply to queued items that haven't been spoken yet or to the currently
* speaking item; these items will always be considered.)
*/
private static final int SKIP_DUPLICATES_DELAY = 1000;
/**
* Class defining constants used for describing speech parameters.
*/
public static class SpeechParam {
/** Float parameter for controlling speech volume. Range is {0 ... 2}. */
public static final String VOLUME = Engine.KEY_PARAM_VOLUME;
/** Float parameter for controlling speech rate. Range is {0 ... 2}. */
public static final String RATE = "rate";
/** Float parameter for controlling speech pitch. Range is {0 ... 2}. */
public static final String PITCH = "pitch";
}
/**
* Reusable map used for passing parameters to the TextToSpeech.
*/
private final HashMap<String, String> mSpeechParametersMap = new HashMap<>();
/**
* Priority queue of actions to perform when utterances are completed,
* ordered by ascending utterance index.
*/
private final PriorityQueue<UtteranceCompleteAction> mUtteranceCompleteActions =
new PriorityQueue<>();
/** The list of items to be spoken. */
private final LinkedList<FeedbackItem> mFeedbackQueue = new LinkedList<>();
/** The list of recently-spoken items. */
private final LinkedList<FeedbackItem> mFeedbackHistory = new LinkedList<>();
/** The parent service. */
private final TalkBackService mService;
/** The audio manager, used to query ringer volume. */
private final AudioManager mAudioManager;
/** The feedback controller, used for playing auditory icons and vibration */
private final FeedbackController mFeedbackController;
/** The text-to-speech service, used for speaking. */
private final FailoverTextToSpeech mFailoverTts;
/** Proximity sensor for implementing "shut up" functionality. */
private ProximitySensor mProximitySensor;
/** Listener used for testing. */
private SpeechControllerListener mSpeechListener;
/** An iterator at the fragment currently being processed */
private Iterator<FeedbackFragment> mCurrentFragmentIterator = null;
/** The item current being spoken, or {@code null} if the TTS is idle. */
private FeedbackItem mCurrentFeedbackItem;
/** Whether to use the proximity sensor to silence speech. */
private boolean mSilenceOnProximity;
/** Whether we should request audio focus during speech. */
private boolean mUseAudioFocus;
/** Whether or not the screen is on. */
// This is set by RingerModeAndScreenMonitor and used by SpeechController
// to determine if the ProximitySensor should be on or off.
private boolean mScreenIsOn;
/** The text-to-speech screen overlay. */
private TextToSpeechOverlay mTtsOverlay;
/**
* Whether the speech controller should add utterance callbacks to
* FullScreenReadController
*/
private boolean mInjectFullScreenReadCallbacks;
/** The utterance completed callback for FullScreenReadController */
private UtteranceCompleteRunnable mFullScreenReadNextCallback;
/**
* The next utterance index; each utterance value will be constructed from this
* ever-increasing index.
*/
private int mNextUtteranceIndex = 0;
/** Whether rate and pitch can change. */
private boolean mUseIntonation;
/** The speech rate adjustment (default is 1.0). */
private float mSpeechRate;
/** The speech pitch adjustment (default is 1.0). */
private float mSpeechPitch;
/** The speech volume adjustment (default is 1.0). */
private float mSpeechVolume;
/**
* Whether the controller is currently speaking utterances. Used to check
* consistency of internal speaking state.
*/
private boolean mIsSpeaking;
private VoiceRecognitionChecker mVoiceRecognitionChecker;
/**
* Used to keep track of whether the interrupt TTS key (Ctrl key) is currently pressed.
*/
private boolean mInterruptKeyDown = false;
@SuppressWarnings("FieldCanBeLocal")
private final OnSharedPreferenceChangeListener prefListener;
public SpeechController(TalkBackService context,
FeedbackController feedbackController) {
if (feedbackController == null) throw new IllegalStateException();
mService = context;
mService.addServiceStateListener(new TalkBackService.ServiceStateListener() {
@Override
public void onServiceStateChanged(int newState) {
if (newState == TalkBackService.SERVICE_STATE_ACTIVE) {
setProximitySensorState(true);
} else if (newState == TalkBackService.SERVICE_STATE_SUSPENDED) {
setProximitySensorState(false);
}
}
});
mAudioManager = (AudioManager) mService.getSystemService(Context.AUDIO_SERVICE);
mFailoverTts = new FailoverTextToSpeech(context);
mFailoverTts.addListener(new FailoverTextToSpeech.FailoverTtsListener() {
@Override
public void onTtsInitialized(boolean wasSwitchingEngines) {
SpeechController.this.onTtsInitialized(wasSwitchingEngines);
}
@Override
public void onUtteranceCompleted(String utteranceId, boolean success) {
// Utterances from FailoverTts are considered fragments in SpeechController
SpeechController.this.onFragmentCompleted(utteranceId, success, true /* advance */);
}
});
mFeedbackController = feedbackController;
mInjectFullScreenReadCallbacks = false;
prefListener = new OnSharedPreferenceChangeListener() {
@Override
public void onSharedPreferenceChanged(SharedPreferences prefs, String key) {
if ((key == null) || (prefs == null)) {
return;
}
reloadPreferences(prefs);
}
};
final SharedPreferences prefs = SharedPreferencesUtils.getSharedPreferences(context);
// Handles preference changes that affect speech.
prefs.registerOnSharedPreferenceChangeListener(prefListener);
reloadPreferences(prefs);
mScreenIsOn = true;
mVoiceRecognitionChecker = new VoiceRecognitionChecker();
}
/**
* @return {@code true} if the speech controller is currently speaking.
*/
public boolean isSpeaking() {
return mIsSpeaking;
}
/**
* @return {@code true} if the speech controller has feedback queued up to speak
*/
private boolean isSpeechQueued() {
return !mFeedbackQueue.isEmpty();
}
/* package */ boolean isSpeakingOrSpeechQueued() {
return isSpeaking() || isSpeechQueued();
}
public void setSpeechListener(SpeechControllerListener speechListener) {
mSpeechListener = speechListener;
}
/**
* Sets whether or not the proximity sensor should be used to silence
* speech.
* <p>
* This should be called when the user changes the state of the "silence on
* proximity" preference.
*/
public void setSilenceOnProximity(boolean silenceOnProximity) {
mSilenceOnProximity = silenceOnProximity;
// Propagate the proximity sensor change.
setProximitySensorState(mSilenceOnProximity);
}
/**
* Lets the SpeechController know whether the screen is on.
*/
public void setScreenIsOn(boolean screenIsOn) {
mScreenIsOn = screenIsOn;
// The proximity sensor should always be on when the screen is on so
// that the proximity gesture can be used to silence all apps.
if (mScreenIsOn) {
setProximitySensorState(true);
}
}
/**
* Sets whether the SpeechController should inject utterance completed
* callbacks for advancing continuous reading.
*/
public void setShouldInjectAutoReadingCallbacks(
boolean shouldInject, UtteranceCompleteRunnable nextItemCallback) {
mFullScreenReadNextCallback = (shouldInject) ? nextItemCallback : null;
mInjectFullScreenReadCallbacks = shouldInject;
if (!shouldInject) {
removeUtteranceCompleteAction(nextItemCallback);
}
}
/**
* Forces a reload of the user's preferred TTS engine, if it is available and the current TTS
* engine is not the preferred engine.
* @param quiet suppresses the "Using XYZ engine" message if the TTS engine changes
*/
public void updateTtsEngine(boolean quiet) {
if (quiet) {
EventState.getInstance().addEvent(
EventState.EVENT_SKIP_FEEDBACK_AFTER_QUIET_TTS_CHANGE);
}
mFailoverTts.updateDefaultEngine();
}
/**
* Gets the {@link FailoverTextToSpeech} instance that is serving as a text-to-speech service.
*
* @return The text-to-speech service.
*/
/* package */ FailoverTextToSpeech getFailoverTts() {
return mFailoverTts;
}
/**
* Repeats the last spoken utterance.
*/
public boolean repeatLastUtterance() {
return repeatUtterance(getLastUtterance());
}
/**
* Copies the last phrase spoken by TalkBack to clipboard
*/
public boolean copyLastUtteranceToClipboard(FeedbackItem item) {
if (item == null) {
return false;
}
final ClipboardManager clipboard = (ClipboardManager) mService.getSystemService(
Context.CLIPBOARD_SERVICE);
ClipData clip = ClipData.newPlainText(null, item.getAggregateText());
clipboard.setPrimaryClip(clip);
// Verify that we actually have the utterance on the clipboard
clip = clipboard.getPrimaryClip();
if (clip != null && clip.getItemCount() > 0 && clip.getItemAt(0).getText() != null) {
speak(mService.getString(R.string.template_text_copied,
clip.getItemAt(0).getText().toString()) /* text */,
QUEUE_MODE_INTERRUPT /* queue mode */,
0 /* flags */,
null /* speech params */);
return true;
} else {
return false;
}
}
public FeedbackItem getLastUtterance() {
if (mFeedbackHistory.isEmpty()) {
return null;
}
return mFeedbackHistory.getLast();
}
public boolean repeatUtterance(FeedbackItem item) {
if (item == null) {
return false;
}
item.addFlag(FeedbackItem.FLAG_NO_HISTORY);
speak(item, QUEUE_MODE_FLUSH_ALL, null);
return true;
}
/**
* Spells the last spoken utterance.
*/
public boolean spellLastUtterance() {
if (getLastUtterance() == null) {
return false;
}
CharSequence aggregateText = getLastUtterance().getAggregateText();
return spellUtterance(aggregateText);
}
/**
* Spells the text.
*/
public boolean spellUtterance(CharSequence text) {
if (TextUtils.isEmpty(text)) {
return false;
}
final SpannableStringBuilder builder = new SpannableStringBuilder();
for (int i = 0; i < text.length(); i++) {
final String cleanedChar = SpeechCleanupUtils.getCleanValueFor(
mService, text.charAt(i));
StringBuilderUtils.appendWithSeparator(builder, cleanedChar);
}
speak(builder, null, null, QUEUE_MODE_FLUSH_ALL, UTTERANCE_GROUP_DEFAULT,
FeedbackItem.FLAG_NO_HISTORY, null, null, null);
return true;
}
/**
* Speaks the name of the currently active TTS engine.
*/
private void speakCurrentEngine() {
final CharSequence engineLabel = mFailoverTts.getEngineLabel();
if (TextUtils.isEmpty(engineLabel)) {
return;
}
final String text = mService.getString(R.string.template_current_tts_engine, engineLabel);
speak(text, null, null, QUEUE_MODE_QUEUE, FeedbackItem.FLAG_NO_HISTORY,
UTTERANCE_GROUP_DEFAULT, null, null);
}
/**
* @see #speak(CharSequence, Set, Set, int, int, int, Bundle, Bundle, UtteranceCompleteRunnable)
*/
public void speak(CharSequence text, int queueMode, int flags, Bundle speechParams) {
speak(text, null, null, queueMode, flags, UTTERANCE_GROUP_DEFAULT, speechParams, null);
}
/**
* @see #speak(CharSequence, Set, Set, int, int, int, Bundle, Bundle, UtteranceCompleteRunnable)
*/
public void speak(CharSequence text, Set<Integer> earcons, Set<Integer> haptics, int queueMode,
int flags, int uttranceGroup, Bundle speechParams, Bundle nonSpeechParams) {
speak(text, earcons, haptics, queueMode, flags, uttranceGroup,
speechParams, nonSpeechParams, null);
}
/**
* Cleans up and speaks an <code>utterance</code>. The <code>queueMode</code> determines
* whether the speech will interrupt or wait on queued speech events.
* <p>
* This method does nothing if the text to speak is empty. See
* {@link TextUtils#isEmpty(CharSequence)} for implementation.
* <p>
* See {@link SpeechCleanupUtils#cleanUp} for text clean-up implementation.
*
* @param text The text to speak.
* @param earcons The set of earcon IDs to play.
* @param haptics The set of vibration patterns to play.
* @param queueMode The queue mode to use for speaking. One of:
* <ul>
* <li>{@link #QUEUE_MODE_INTERRUPT} <li>
* {@link #QUEUE_MODE_QUEUE} <li>
* {@link #QUEUE_MODE_UNINTERRUPTIBLE}
* </ul>
* @param flags Bit mask of speaking flags. Use {@code 0} for no flags, or a
* combination of the flags defined in {@link FeedbackItem}
* @param speechParams Speaking parameters. Not all parameters are supported by
* all engines. One of:
* <ul>
* <li>{@link SpeechParam#PITCH}</li>
* <li>{@link SpeechParam#RATE}</li>
* <li>{@link SpeechParam#VOLUME}</li>
* </ul>
* @param nonSpeechParams Non-Speech parameters. Optional, but can include
* {@link Utterance#KEY_METADATA_EARCON_RATE} and
* {@link Utterance#KEY_METADATA_EARCON_VOLUME}
* @param completedAction The action to run after this utterance has been
* spoken.
*/
public void speak(CharSequence text, Set<Integer> earcons, Set<Integer> haptics, int queueMode,
int flags, int utteranceGroup, Bundle speechParams, Bundle nonSpeechParams,
UtteranceCompleteRunnable completedAction) {
if (TextUtils.isEmpty(text) && (earcons == null || earcons.isEmpty()) &&
(haptics == null || haptics.isEmpty())) {
// don't process request with empty feedback
return;
}
final FeedbackItem pendingItem = FeedbackProcessingUtils.generateFeedbackItemFromInput(
mService, text, earcons, haptics, flags, utteranceGroup,
speechParams, nonSpeechParams);
speak(pendingItem, queueMode, completedAction);
}
private void speak(
FeedbackItem item, int queueMode, UtteranceCompleteRunnable completedAction) {
// If this FeedbackItem is flagged as NO_SPEECH, ignore speech and
// immediately process earcons and haptics without disrupting the speech
// queue.
// TODO: Consider refactoring non-speech feedback out of
// this class entirely.
if (item.hasFlag(FeedbackItem.FLAG_NO_SPEECH)) {
for (FeedbackFragment fragment : item.getFragments()) {
playEarconsFromFragment(fragment);
playHapticsFromFragment(fragment);
}
return;
}
if (item.hasFlag(FeedbackItem.FLAG_SKIP_DUPLICATE) && hasItemOnQueueOrSpeaking(item)) {
return;
}
item.setUninterruptible(queueMode == QUEUE_MODE_UNINTERRUPTIBLE);
item.setCompletedAction(completedAction);
boolean currentFeedbackInterrupted = false;
if(shouldClearQueue(item, queueMode)) {
FeedbackItemFilter filter = getFeedbackItemFilter(item, queueMode);
// Call onUtteranceComplete on each queue item to be cleared.
ListIterator<FeedbackItem> iterator = mFeedbackQueue.listIterator(0);
while (iterator.hasNext()) {
FeedbackItem currentItem = iterator.next();
if (filter.accept(currentItem)) {
iterator.remove();
notifyItemInterrupted(currentItem);
}
}
if (mCurrentFeedbackItem != null && filter.accept(mCurrentFeedbackItem)) {
notifyItemInterrupted(mCurrentFeedbackItem);
currentFeedbackInterrupted = true;
}
}
mFeedbackQueue.add(item);
if (mSpeechListener != null) {
mSpeechListener.onUtteranceQueued(item);
}
// If TTS isn't ready, this should be the only item in the queue.
if (!mFailoverTts.isReady()) {
LogUtils.log(this, Log.ERROR, "Attempted to speak before TTS was initialized.");
return;
}
if ((mCurrentFeedbackItem == null) || currentFeedbackInterrupted) {
mCurrentFragmentIterator = null;
speakNextItem();
} else {
LogUtils.log(this, Log.VERBOSE, "Queued speech item, waiting for \"%s\"",
mCurrentFeedbackItem.getUtteranceId());
}
}
private boolean shouldClearQueue(FeedbackItem item, int queueMode) {
// QUEUE_MODE_INTERRUPT and QUEUE_MODE_FLUSH_ALL will clear the queue.
if (queueMode != QUEUE_MODE_QUEUE && queueMode != QUEUE_MODE_UNINTERRUPTIBLE) {
return true;
}
// If there is utterance group different from SpeechController.UTTERANCE_GROUP_DEFAULT
// and flag FeedbackItem.FLAG_CLEAR_QUEUED_UTTERANCES_WITH_SAME_UTTERANCE_GROUP items
// from same UTTERANCE_GRPOUP would be cleared from queue
if (item.getUtteranceGroup() != UTTERANCE_GROUP_DEFAULT &&
item.hasFlag(FeedbackItem.FLAG_CLEAR_QUEUED_UTTERANCES_WITH_SAME_UTTERANCE_GROUP)) {
return true;
}
// If there is utterance group different from SpeechController.UTTERANCE_GROUP_DEFAULT
// and flag FeedbackItem.FLAG_INTERRUPT_CURRENT_UTTERANCE_WITH_SAME_UTTERANCE_GROUP
// currently speaking item would be interrupted if it has the same utterance group
if (item.getUtteranceGroup() != UTTERANCE_GROUP_DEFAULT &&
item.hasFlag(
FeedbackItem.FLAG_INTERRUPT_CURRENT_UTTERANCE_WITH_SAME_UTTERANCE_GROUP)) {
return true;
}
return false;
}
private FeedbackItemFilter getFeedbackItemFilter(FeedbackItem item, int queueMode) {
FeedbackItemFilter filter = new FeedbackItemFilter();
if (queueMode != QUEUE_MODE_QUEUE && queueMode != QUEUE_MODE_UNINTERRUPTIBLE) {
filter.addFeedbackItemPredicate(new FeedbackItemInterruptiblePredicate());
}
if (item.getUtteranceGroup() != UTTERANCE_GROUP_DEFAULT &&
item.hasFlag(FeedbackItem.FLAG_CLEAR_QUEUED_UTTERANCES_WITH_SAME_UTTERANCE_GROUP)) {
FeedbackItemPredicate notCurrentItemPredicate = new FeedbackItemEqualSamplePredicate(
mCurrentFeedbackItem, false);
FeedbackItemPredicate sameUtteranceGroupPredicate =
new FeedbackItemUtteranceGroupPredicate(item.getUtteranceGroup());
FeedbackItemPredicate clearQueuePredicate = new FeedbackItemConjunctionPredicateSet(
notCurrentItemPredicate, sameUtteranceGroupPredicate);
filter.addFeedbackItemPredicate(clearQueuePredicate);
}
if (item.getUtteranceGroup() != UTTERANCE_GROUP_DEFAULT &&
item.hasFlag(
FeedbackItem.FLAG_INTERRUPT_CURRENT_UTTERANCE_WITH_SAME_UTTERANCE_GROUP)) {
FeedbackItemPredicate currentItemPredicate = new FeedbackItemEqualSamplePredicate(
mCurrentFeedbackItem, true);
FeedbackItemPredicate sameUtteranceGroupPredicate =
new FeedbackItemUtteranceGroupPredicate(item.getUtteranceGroup());
FeedbackItemPredicate clearQueuePredicate = new FeedbackItemConjunctionPredicateSet(
currentItemPredicate, sameUtteranceGroupPredicate);
filter.addFeedbackItemPredicate(clearQueuePredicate);
}
return filter;
}
private void notifyItemInterrupted(FeedbackItem item) {
final UtteranceCompleteRunnable queuedItemCompletedAction
= item.getCompletedAction();
if (queuedItemCompletedAction != null) {
queuedItemCompletedAction.run(STATUS_INTERRUPTED);
}
}
private boolean hasItemOnQueueOrSpeaking(FeedbackItem item) {
if (item == null) {
return false;
}
if (feedbackTextEquals(item, mCurrentFeedbackItem)) {
return true;
}
for (FeedbackItem queuedItem : mFeedbackQueue) {
if (feedbackTextEquals(item, queuedItem)) {
return true;
}
}
long currentTime = item.getCreationTime();
for (FeedbackItem recentItem : mFeedbackHistory) {
if (currentTime - recentItem.getCreationTime() < SKIP_DUPLICATES_DELAY) {
if (feedbackTextEquals(item, recentItem)) {
return true;
}
}
}
return false;
}
/**
* Compares feedback fragments based on their text only. Ignores other parameters such as
* earcons and interruptibility.
*/
private boolean feedbackTextEquals(FeedbackItem item1, FeedbackItem item2) {
if (item1 == null || item2 == null) {
return false;
}
List<FeedbackFragment> fragments1 = item1.getFragments();
List<FeedbackFragment> fragments2 = item2.getFragments();
if (fragments1.size() != fragments2.size()) {
return false;
}
int size = fragments1.size();
for (int i = 0; i < size; i++) {
FeedbackFragment fragment1 = fragments1.get(i);
FeedbackFragment fragment2 = fragments2.get(i);
if (fragment1 != null && fragment2 != null
&& !TextUtils.equals(fragment1.getText(), fragment2.getText())) {
return false;
}
if ((fragment1 == null && fragment2 != null)
|| (fragment1 != null && fragment2 == null)) {
return false;
}
}
return true;
}
/**
* Add a new action that will be run when the given utterance index
* completes.
*
* @param index The index of the utterance that should finish before this
* action is executed.
* @param runnable The code to execute.
*/
public void addUtteranceCompleteAction(int index, UtteranceCompleteRunnable runnable) {
final UtteranceCompleteAction action = new UtteranceCompleteAction(index, runnable);
mUtteranceCompleteActions.add(action);
}
/**
* Removes all instances of the specified runnable from the utterance
* complete action list.
*
* @param runnable The runnable to remove.
*/
public void removeUtteranceCompleteAction(UtteranceCompleteRunnable runnable) {
final Iterator<UtteranceCompleteAction> i = mUtteranceCompleteActions.iterator();
while (i.hasNext()) {
final UtteranceCompleteAction action = i.next();
if (action.runnable == runnable) {
i.remove();
}
}
}
/**
* Stops all speech.
*/
public void interrupt() {
// Clear all current and queued utterances.
clearCurrentAndQueuedUtterances();
// Clear and post all remaining completion actions.
clearUtteranceCompletionActions(true);
// Make sure TTS actually stops talking.
mFailoverTts.stopAll();
}
/**
* Stops speech and shuts down this controller.
*/
public void shutdown() {
interrupt();
mFailoverTts.shutdown();
setOverlayEnabled(false);
setProximitySensorState(false);
}
/**
* Returns the next utterance identifier.
*/
public int peekNextUtteranceId() {
return mNextUtteranceIndex;
}
/**
* Returns the next utterance identifier and increments the utterance value.
*/
private int getNextUtteranceId() {
return mNextUtteranceIndex++;
}
/**
* Reloads preferences for this controller.
*
* @param prefs The shared preferences for this service. Pass {@code null}
* to disable the overlay.
*/
private void reloadPreferences(SharedPreferences prefs) {
final Resources res = mService.getResources();
final boolean ttsOverlayEnabled = SharedPreferencesUtils.getBooleanPref(prefs, res,
R.string.pref_tts_overlay_key, R.bool.pref_tts_overlay_default);
setOverlayEnabled(ttsOverlayEnabled);
mUseIntonation = SharedPreferencesUtils.getBooleanPref(prefs, res,
R.string.pref_intonation_key, R.bool.pref_intonation_default);
mSpeechPitch = SharedPreferencesUtils.getFloatFromStringPref(prefs, res,
R.string.pref_speech_pitch_key, R.string.pref_speech_pitch_default);
mSpeechRate = SharedPreferencesUtils.getFloatFromStringPref(prefs, res,
R.string.pref_speech_rate_key, R.string.pref_speech_rate_default);
mUseAudioFocus = SharedPreferencesUtils.getBooleanPref(
prefs, res, R.string.pref_use_audio_focus_key, R.bool.pref_use_audio_focus_default);
// Speech volume is stored as int [0,100] and scaled to float [0,1].
mSpeechVolume = (SharedPreferencesUtils.getIntFromStringPref(prefs, res,
R.string.pref_speech_volume_key, R.string.pref_speech_volume_default) / 100.0f);
if (!mUseAudioFocus) {
mAudioManager.abandonAudioFocus(mAudioFocusListener);
}
}
private void setOverlayEnabled(boolean enabled) {
if (enabled && mTtsOverlay == null) {
mTtsOverlay = new TextToSpeechOverlay(mService);
} else if (!enabled && mTtsOverlay != null) {
mTtsOverlay.hide();
mTtsOverlay = null;
}
}
/**
* Returns {@code true} if speech should be silenced. Does not prevent
* haptic or auditory feedback from occurring. The controller will run
* utterance completion actions immediately for silenced utterances.
* <p>
* Silences speech in the following cases:
* <ul>
* <li>Speech recognition is active and the user is not using a headset
* </ul>
*/
@SuppressWarnings("deprecation")
private boolean shouldSilenceSpeech(FeedbackItem item) {
if (item == null) {
return false;
}
// Unless otherwise flagged, don't speak during speech recognition.
return !item.hasFlag(FeedbackItem.FLAG_DURING_RECO)
&& AudioSystemCompatUtils.isSourceActive(AudioSource.VOICE_RECOGNITION)
&& !mAudioManager.isBluetoothA2dpOn() && !mAudioManager.isWiredHeadsetOn();
}
/**
* Sends the specified item to the text-to-speech engine. Manages internal
* speech controller state.
* <p>
* This method should only be called by {@link #speakNextItem()}.
*
* @param item The item to speak.
*/
@SuppressLint("InlinedApi")
private void speakNextItemInternal(FeedbackItem item) {
final int utteranceIndex = getNextUtteranceId();
final String utteranceId = UTTERANCE_ID_PREFIX + utteranceIndex;
item.setUtteranceId(utteranceId);
final UtteranceCompleteRunnable completedAction = item.getCompletedAction();
if (completedAction != null) {
addUtteranceCompleteAction(utteranceIndex, completedAction);
}
if (mInjectFullScreenReadCallbacks
&& item.hasFlag(FeedbackItem.FLAG_ADVANCE_CONTINUOUS_READING)) {
addUtteranceCompleteAction(utteranceIndex, mFullScreenReadNextCallback);
}
if ((item != null) && !item.hasFlag(FeedbackItem.FLAG_NO_HISTORY)) {
while (mFeedbackHistory.size() >= MAX_HISTORY_ITEMS) {
mFeedbackHistory.removeFirst();
}
mFeedbackHistory.addLast(item);
}
if (mSpeechListener != null) {
mSpeechListener.onUtteranceStarted(item);
}
processNextFragmentInternal();
}
private boolean processNextFragmentInternal() {
if (mCurrentFragmentIterator == null || !mCurrentFragmentIterator.hasNext()) {
return false;
}
FeedbackFragment fragment = mCurrentFragmentIterator.next();
playEarconsFromFragment(fragment);
playHapticsFromFragment(fragment);
// Reuse the global instance of speech parameters.
final HashMap<String, String> params = mSpeechParametersMap;
params.clear();
// Add all custom speech parameters.
final Bundle speechParams = fragment.getSpeechParams();
for (String key : speechParams.keySet()) {
params.put(key, String.valueOf(speechParams.get(key)));
}
// Utterance ID, stream, and volume override item params.
params.put(Engine.KEY_PARAM_UTTERANCE_ID, mCurrentFeedbackItem.getUtteranceId());
params.put(Engine.KEY_PARAM_STREAM, String.valueOf(DEFAULT_STREAM));
params.put(Engine.KEY_PARAM_VOLUME, String.valueOf(mSpeechVolume));
final float pitch =
mSpeechPitch * (mUseIntonation ? parseFloatParam(params, SpeechParam.PITCH, 1) : 1);
final float rate =
mSpeechRate * (mUseIntonation ? parseFloatParam(params, SpeechParam.RATE, 1) : 1);
final CharSequence text;
if (shouldSilenceSpeech(mCurrentFeedbackItem) || TextUtils.isEmpty(fragment.getText())) {
text = null;
} else {
text = fragment.getText();
}
String logText = text == null ? null : text.toString();
LogUtils.log(this, Log.VERBOSE, "Speaking fragment text \"%s\"", logText);
mVoiceRecognitionChecker.onUtteranceStart();
// It's okay if the utterance is empty, the fail-over TTS will
// immediately call the fragment completion listener. This process is
// important for things like continuous reading.
mFailoverTts.speak(text, pitch, rate, params, DEFAULT_STREAM, mSpeechVolume);
if (mTtsOverlay != null) {
mTtsOverlay.speak(text);
}
return true;
}
/**
* Plays all earcons stored in a {@link FeedbackFragment}.
*
* @param fragment The fragment to process
*/
private void playEarconsFromFragment(FeedbackFragment fragment) {
final Bundle nonSpeechParams = fragment.getNonSpeechParams();
final float earconRate = nonSpeechParams.getFloat(Utterance.KEY_METADATA_EARCON_RATE, 1.0f);
final float earconVolume = nonSpeechParams.getFloat(
Utterance.KEY_METADATA_EARCON_VOLUME, 1.0f);
for (int keyResId : fragment.getEarcons()) {
mFeedbackController.playAuditory(keyResId, earconRate, earconVolume);
}
}
/**
* Produces all haptic feedback stored in a {@link FeedbackFragment}.
*
* @param fragment The fragment to process
*/
private void playHapticsFromFragment(FeedbackFragment fragment) {
for (int keyResId : fragment.getHaptics()) {
mFeedbackController.playHaptic(keyResId);
}
}
/**
* @return The utterance ID, or -1 if the ID is invalid.
*/
private static int parseUtteranceId(String utteranceId) {
// Check for bad utterance ID. This should never happen.
if (!utteranceId.startsWith(UTTERANCE_ID_PREFIX)) {
LogUtils.log(SpeechController.class, Log.ERROR, "Bad utterance ID: %s", utteranceId);
return -1;
}
try {
return Integer.parseInt(utteranceId.substring(UTTERANCE_ID_PREFIX.length()));
} catch (NumberFormatException e) {
e.printStackTrace();
return -1;
}
}
/**
* Called when transitioning from an idle state to a speaking state, e.g.
* the queue was empty, there was no current speech, and a speech item was
* added to the queue.
*
* @see #handleSpeechCompleted()
*/
private void handleSpeechStarting() {
// Always enable the proximity sensor when speaking.
setProximitySensorState(true);
boolean useAudioFocus = mUseAudioFocus;
if (BuildCompat.isAtLeastN()) {
List<AudioRecordingConfiguration> recordConfigurations =
mAudioManager.getActiveRecordingConfigurations();
if (recordConfigurations.size() != 0)
useAudioFocus = false;
}
if (useAudioFocus) {
mAudioManager.requestAudioFocus(mAudioFocusListener, AudioManager.STREAM_MUSIC,
AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK);
}
if (mIsSpeaking) {
LogUtils.log(this, Log.ERROR, "Started speech while already speaking!");
}
mIsSpeaking = true;
}
/**
* Called when transitioning from a speaking state to an idle state, e.g.
* all queued utterances have been spoken and the last utterance has
* completed.
*
* @see #handleSpeechStarting()
*/
private void handleSpeechCompleted() {
// If the screen is on, keep the proximity sensor on.
setProximitySensorState(mScreenIsOn);
if (mUseAudioFocus) {
mAudioManager.abandonAudioFocus(mAudioFocusListener);
}
if (!mIsSpeaking) {
LogUtils.log(this, Log.ERROR, "Completed speech while already completed!");
}
mIsSpeaking = false;
}
/**
* Clears the speech queue and completes the current speech item, if any.
*/
private void clearCurrentAndQueuedUtterances() {
mFeedbackQueue.clear();
mCurrentFragmentIterator = null;
if (mCurrentFeedbackItem != null) {
final String utteranceId = mCurrentFeedbackItem.getUtteranceId();
onFragmentCompleted(utteranceId, false /* success */, true /* advance */);
mCurrentFeedbackItem = null;
}
}
/**
* Clears (and optionally posts) all pending completion actions.
*
* @param execute {@code true} to post actions to the handler.
*/
private void clearUtteranceCompletionActions(boolean execute) {
if (!execute) {
mUtteranceCompleteActions.clear();
return;
}
while (!mUtteranceCompleteActions.isEmpty()) {
final UtteranceCompleteRunnable runnable = mUtteranceCompleteActions.poll().runnable;
if (runnable != null) {
mHandler.post(new CompletionRunner(runnable, STATUS_INTERRUPTED));
}
}
// Don't call handleSpeechCompleted(), it will be called by the TTS when
// it stops the current current utterance.
}
/**
* Handles completion of a {@link FeedbackFragment}.
* <p>
*
* @param utteranceId The ID of the {@link FeedbackItem} the fragment belongs to.
* @param success Whether the fragment was spoken successfully.
* @param advance Whether to advance to the next queue item.
*/
private void onFragmentCompleted(String utteranceId, boolean success, boolean advance) {
final int utteranceIndex = SpeechController.parseUtteranceId(utteranceId);
final boolean interrupted = (mCurrentFeedbackItem != null)
&& (!mCurrentFeedbackItem.getUtteranceId().equals(utteranceId));
final int status;
if (interrupted) {
status = STATUS_INTERRUPTED;
} else if (success) {
status = STATUS_SPOKEN;
} else {
status = STATUS_ERROR;
}
// Process the next fragment for this FeedbackItem if applicable.
if ((status != STATUS_SPOKEN) || !processNextFragmentInternal()) {
// If speaking resulted in an error, was ultimately interrupted, or
// there are no additional fragments to speak as part of the current
// FeedbackItem, finish processing of this utterance.
onUtteranceCompleted(utteranceIndex, status, interrupted, advance);
}
}
/**
* Handles the completion of an {@link Utterance}/{@link FeedbackItem}.
*
* @param utteranceIndex The ID of the utterance that has completed.
* @param status One of {@link SpeechController#STATUS_ERROR},
* {@link SpeechController#STATUS_INTERRUPTED}, or
* {@link SpeechController#STATUS_SPOKEN}
* @param interrupted {@code true} if the utterance was interrupted, {@code false} otherwise
* @param advance Whether to advance to the next queue item.
*/
private void onUtteranceCompleted(
int utteranceIndex, int status, boolean interrupted, boolean advance) {
while (!mUtteranceCompleteActions.isEmpty()
&& (mUtteranceCompleteActions.peek().utteranceIndex <= utteranceIndex)) {
final UtteranceCompleteRunnable runnable = mUtteranceCompleteActions.poll().runnable;
if (runnable != null) {
mHandler.post(new CompletionRunner(runnable, status));
}
}
if (mSpeechListener != null) {
mSpeechListener.onUtteranceCompleted(utteranceIndex, status);
}
if (interrupted) {
// We finished an utterance, but we weren't expecting to see a
// completion. This means we interrupted a previous utterance and
// can safely ignore this callback.
LogUtils.log(this, Log.VERBOSE, "Interrupted %d with %s", utteranceIndex,
mCurrentFeedbackItem.getUtteranceId());
return;
}
if (advance && !speakNextItem()) {
handleSpeechCompleted();
}
}
private void onTtsInitialized(boolean wasSwitchingEngines) {
// The previous engine may not have shut down correctly, so make sure to
// clear the "current" speech item.
if (mCurrentFeedbackItem != null) {
onFragmentCompleted(mCurrentFeedbackItem.getUtteranceId(),
false /* success */, false /* advance */);
mCurrentFeedbackItem = null;
}
if (wasSwitchingEngines && !EventState.getInstance().checkAndClearRecentEvent(
EventState.EVENT_SKIP_FEEDBACK_AFTER_QUIET_TTS_CHANGE)) {
speakCurrentEngine();
} else if (!mFeedbackQueue.isEmpty()) {
speakNextItem();
}
}
/**
* Removes and speaks the next {@link FeedbackItem} in the queue,
* interrupting the current utterance if necessary.
*
* @return {@code false} if there are no more queued speech items.
*/
private boolean speakNextItem() {
final FeedbackItem previousItem = mCurrentFeedbackItem;
final FeedbackItem nextItem = (mFeedbackQueue.isEmpty() ? null
: mFeedbackQueue.removeFirst());
mCurrentFeedbackItem = nextItem;
if (nextItem == null) {
LogUtils.log(this, Log.VERBOSE, "No next item, stopping speech queue");
return false;
}
if (previousItem == null) {
handleSpeechStarting();
}
mCurrentFragmentIterator = nextItem.getFragments().iterator();
speakNextItemInternal(nextItem);
return true;
}
/**
* Attempts to parse a float value from a {@link HashMap} of strings.
*
* @param params The map to obtain the value from.
* @param key The key that the value is assigned to.
* @param defaultValue The default value.
* @return The parsed float value, or the default value on failure.
*/
private static float parseFloatParam(
HashMap<String, String> params, String key, float defaultValue) {
final String value = params.get(key);
if (value == null) {
return defaultValue;
}
try {
return Float.parseFloat(value);
} catch (NumberFormatException e) {
e.printStackTrace();
}
return defaultValue;
}
/**
* Enables/disables the proximity sensor. The proximity sensor should be
* disabled when not in use to save battery.
* <p>
* This is a no-op if the user has turned off the "silence on proximity"
* preference.
*
* @param enabled {@code true} if the proximity sensor should be enabled,
* {@code false} otherwise.
*/
// TODO: Rewrite for readability.
private void setProximitySensorState(boolean enabled) {
if (mProximitySensor != null) {
// Should we be using the proximity sensor at all?
if (!mSilenceOnProximity) {
mProximitySensor.stop();
mProximitySensor = null;
return;
}
if (!TalkBackService.isServiceActive()) {
mProximitySensor.stop();
return;
}
} else {
// Do we need to initialize the proximity sensor?
if (enabled && mSilenceOnProximity) {
mProximitySensor = new ProximitySensor(mService);
mProximitySensor.setProximityChangeListener(mProximityChangeListener);
} else {
return;
}
}
// Manage the proximity sensor state.
if (enabled) {
mProximitySensor.start();
} else {
mProximitySensor.stop();
}
}
/**
* Stops the TTS engine when the Ctrl key is tapped without any other keys.
*/
@Override
public boolean onKeyEvent(KeyEvent event) {
int keyCode = event.getKeyCode();
if (event.getAction() == KeyEvent.ACTION_DOWN) {
mInterruptKeyDown = (keyCode == KeyEvent.KEYCODE_CTRL_LEFT ||
keyCode == KeyEvent.KEYCODE_CTRL_RIGHT);
} else if (event.getAction() == KeyEvent.ACTION_UP) {
if (mInterruptKeyDown && (keyCode == KeyEvent.KEYCODE_CTRL_LEFT ||
keyCode == KeyEvent.KEYCODE_CTRL_RIGHT)) {
mInterruptKeyDown = false;
mService.interruptAllFeedback();
}
}
return false;
}
@Override
public boolean processWhenServiceSuspended() {
return false;
}
/**
* Stops the TTS engine when the proximity sensor is close.
*/
private final ProximitySensor.ProximityChangeListener mProximityChangeListener =
new ProximitySensor.ProximityChangeListener() {
@Override
public void onProximityChanged(boolean isClose) {
// Stop feedback if the user is close to the sensor.
if (isClose) {
mService.interruptAllFeedback();
}
}
};
private final Handler mHandler = new Handler();
private final AudioManager.OnAudioFocusChangeListener
mAudioFocusListener = new AudioManager.OnAudioFocusChangeListener() {
@Override
public void onAudioFocusChange(int focusChange) {
LogUtils.log(SpeechController.this, Log.DEBUG, "Saw audio focus change: %d",
focusChange);
}
};
/**
* Listener for speech started and completed.
*/
public interface SpeechControllerListener {
public void onUtteranceQueued(FeedbackItem utterance);
public void onUtteranceStarted(FeedbackItem utterance);
public void onUtteranceCompleted(int utteranceIndex, int status);
}
/**
* An action that should be performed after a particular utterance index
* completes.
*/
private static class UtteranceCompleteAction implements Comparable<UtteranceCompleteAction> {
public UtteranceCompleteAction(int utteranceIndex, UtteranceCompleteRunnable runnable) {
this.utteranceIndex = utteranceIndex;
this.runnable = runnable;
}
/**
* The minimum utterance index that must complete before this action
* should be performed.
*/
public int utteranceIndex;
/** The action to execute. */
public UtteranceCompleteRunnable runnable;
@Override
public int compareTo(@NonNull UtteranceCompleteAction another) {
return (utteranceIndex - another.utteranceIndex);
}
}
/**
* Utility class run an UtteranceCompleteRunnable.
*/
public static class CompletionRunner implements Runnable {
private final UtteranceCompleteRunnable mRunnable;
private final int mStatus;
public CompletionRunner(UtteranceCompleteRunnable runnable, int status) {
mRunnable = runnable;
mStatus = status;
}
@Override
public void run() {
mRunnable.run(mStatus);
}
}
/**
* Interface for a run method with a status.
*/
public interface UtteranceCompleteRunnable {
/**
* @param status The status supplied.
*/
public void run(int status);
}
/**
* Class that responsible for checking whether voice recognition is enabled and interrupt
* utterances that should be silenced when mic is on
*/
private class VoiceRecognitionChecker extends Handler {
private int MESSAGE_ID = 0;
private int NEXT_CHECK_DELAY = 100;
public void onUtteranceStart() {
removeMessages(MESSAGE_ID);
sendEmptyMessageDelayed(MESSAGE_ID, NEXT_CHECK_DELAY);
}
@Override
public void handleMessage(Message msg) {
if (mCurrentFeedbackItem == null || shouldSilenceSpeech(mCurrentFeedbackItem)) {
mFailoverTts.stopFromTalkBack();
removeMessages(MESSAGE_ID);
} else {
sendEmptyMessageDelayed(MESSAGE_ID, NEXT_CHECK_DELAY);
}
}
}
private interface FeedbackItemPredicate {
public boolean accept(FeedbackItem item);
}
private class FeedbackItemDisjunctionPredicateSet implements FeedbackItemPredicate {
private FeedbackItemPredicate mPredicate1;
private FeedbackItemPredicate mPredicate2;
public FeedbackItemDisjunctionPredicateSet(FeedbackItemPredicate predicate1,
FeedbackItemPredicate predicate2) {
mPredicate1 = predicate1;
mPredicate2 = predicate2;
}
@Override
public boolean accept(FeedbackItem item) {
return mPredicate1.accept(item) || mPredicate2.accept(item);
}
}
private class FeedbackItemConjunctionPredicateSet implements FeedbackItemPredicate {
private FeedbackItemPredicate mPredicate1;
private FeedbackItemPredicate mPredicate2;
public FeedbackItemConjunctionPredicateSet(FeedbackItemPredicate predicate1,
FeedbackItemPredicate predicate2) {
mPredicate1 = predicate1;
mPredicate2 = predicate2;
}
@Override
public boolean accept(FeedbackItem item) {
return mPredicate1.accept(item) && mPredicate2.accept(item);
}
}
private class FeedbackItemInterruptiblePredicate implements FeedbackItemPredicate {
public boolean accept(FeedbackItem item) {
if (item == null) {
return false;
}
return item.isInterruptible();
}
}
private class FeedbackItemEqualSamplePredicate implements FeedbackItemPredicate {
private FeedbackItem mSample;
private boolean mEqual;
public FeedbackItemEqualSamplePredicate(FeedbackItem sample, boolean equal) {
mSample = sample;
mEqual = equal;
}
public boolean accept(FeedbackItem item) {
if (mEqual) {
return mSample == item;
}
return mSample != item;
}
}
private class FeedbackItemUtteranceGroupPredicate implements FeedbackItemPredicate {
private int mUtteranceGroup;
public FeedbackItemUtteranceGroupPredicate(int utteranceGroup) {
mUtteranceGroup = utteranceGroup;
}
public boolean accept(FeedbackItem item) {
if (item == null) {
return false;
}
return item.getUtteranceGroup() == mUtteranceGroup;
}
}
private class FeedbackItemFilter {
private FeedbackItemPredicate mPredicate;
public void addFeedbackItemPredicate(FeedbackItemPredicate predicate) {
if (predicate == null) {
return;
}
if (mPredicate == null) {
mPredicate = predicate;
} else {
mPredicate = new FeedbackItemDisjunctionPredicateSet(mPredicate, predicate);
}
}
public boolean accept(FeedbackItem item) {
return mPredicate != null && mPredicate.accept(item);
}
}
}