/*
*
* Copyright (c) Microsoft. All rights reserved.
* Licensed under the MIT license.
*
* Project Oxford: http://ProjectOxford.ai
*
* Project Oxford Mimicker Alarm Github:
* https://github.com/Microsoft/ProjectOxford-Apps-MimickerAlarm
*
* Copyright (c) Microsoft Corporation
* All rights reserved.
*
* MIT License:
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED ""AS IS"", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
*/
package com.microsoft.mimickeralarm.mimics;
import android.content.Context;
import android.content.res.Resources;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.net.Uri;
import android.os.Bundle;
import android.support.annotation.Nullable;
import android.support.v4.app.Fragment;
import android.support.v4.content.ContextCompat;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.LinearLayout;
import android.widget.TextView;
import com.microsoft.mimickeralarm.R;
import com.microsoft.mimickeralarm.mimics.MimicFactory.MimicResultListener;
import com.microsoft.mimickeralarm.ringing.ShareFragment;
import com.microsoft.mimickeralarm.utilities.Loggable;
import com.microsoft.mimickeralarm.utilities.Logger;
import com.microsoft.mimickeralarm.utilities.KeyUtilities;
import com.microsoft.projectoxford.speechrecognition.Confidence;
import com.microsoft.projectoxford.speechrecognition.ISpeechRecognitionServerEvents;
import com.microsoft.projectoxford.speechrecognition.MicrophoneRecognitionClient;
import com.microsoft.projectoxford.speechrecognition.RecognitionResult;
import com.microsoft.projectoxford.speechrecognition.RecognitionStatus;
import com.microsoft.projectoxford.speechrecognition.RecognizedPhrase;
import com.microsoft.projectoxford.speechrecognition.SpeechRecognitionMode;
import com.microsoft.projectoxford.speechrecognition.SpeechRecognitionServiceFactory;
import java.util.Random;
/**
* Implements the UI and logic of the Tongue Twister mimic game
*
* on start randomly selects one of the tongue twisters
* when the user presses the record button, uses the speech SDK binaries to capture and send audio
* to the Project Oxford speech API.
*
* There are two types of results returned. Partial and Final.
* Final result is returned by Project Oxford API. It contains the most likely Speech->Text transcription
* of the provided audio. It is used here to compute the final correctness
*
* Partial results are continuously returned by native code. It is fast but the quality is not as good
* as the final result. It is used here to continuously provide feedback to the user
*
*
* The correctness is computed by a simple Levenshtein distance calculation between the question
* tongue twister and the final result returned by Project Oxford.
*/
public class MimicTongueTwisterFragment extends Fragment
implements ISpeechRecognitionServerEvents,
IMimicImplementation {
private final static int TIMEOUT_MILLISECONDS = 20000;
private final static float DIFFERENCE_SUCCESS_THRESHOLD = 0.3f;
private final static float DIFFERENCE_PERFECT_THRESHOLD = 0.1f;
private static String LOGTAG = "MimicTongueTwisterFragment";
MimicResultListener mCallback;
private MicrophoneRecognitionClient mMicClient = null;
private SpeechRecognitionMode mRecognitionMode;
private String mUnderstoodText = null;
private String mQuestion = null;
private TextView mTextResponse;
private String mSuccessMessage;
private Uri mSharableUri;
private IMimicMediator mStateManager;
@Nullable
@Override
public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
View view = inflater.inflate(R.layout.fragment_tongue_twister_mimic, container, false);
ProgressButton progressButton = (ProgressButton) view.findViewById(R.id.capture_button);
progressButton.setReadyState(ProgressButton.State.ReadyAudio);
mStateManager = new MimicStateManager();
mStateManager.registerCountDownTimer(
(CountDownTimerView) view.findViewById(R.id.countdown_timer), TIMEOUT_MILLISECONDS);
mStateManager.registerStateBanner((MimicStateBanner) view.findViewById(R.id.mimic_state));
mStateManager.registerProgressButton(progressButton, MimicButtonBehavior.AUDIO);
mStateManager.registerMimic(this);
initialize(view);
Logger.init(getContext());
Loggable.UserAction userAction = new Loggable.UserAction(Loggable.Key.ACTION_GAME_TWISTER);
Logger.track(userAction);
return view;
}
@Override
public void onAttach(Context context) {
super.onAttach(context);
mCallback = (MimicResultListener) context;
}
@Override
public void onDetach() {
super.onDetach();
mCallback = null;
}
@Override
public void onStart() {
super.onStart();
mStateManager.start();
}
@Override
public void onStop() {
super.onStop();
mStateManager.stop();
}
@Override
public void onDestroy() {
super.onDestroy();
Logger.flush();
}
@Override
public void onPartialResponseReceived(String s) {
Log.d(LOGTAG, s);
mTextResponse.setText(s);
mUnderstoodText = s;
}
@Override
public void onFinalResponseReceived(RecognitionResult response) {
if (mStateManager.isMimicRunning()) {
boolean isFinalDictationMessage = mRecognitionMode == SpeechRecognitionMode.LongDictation &&
(response.RecognitionStatus == RecognitionStatus.EndOfDictation ||
response.RecognitionStatus == RecognitionStatus.DictationEndSilenceTimeout);
if (mRecognitionMode == SpeechRecognitionMode.ShortPhrase
|| isFinalDictationMessage) {
mMicClient.endMicAndRecognition();
for (RecognizedPhrase res : response.Results) {
Log.d(LOGTAG, String.valueOf(res.Confidence));
Log.d(LOGTAG, String.valueOf(res.DisplayText));
if(res.Confidence == Confidence.Normal) {
mUnderstoodText = res.DisplayText;
}
else if(res.Confidence == Confidence.High) {
mUnderstoodText = res.DisplayText;
break;
}
}
mTextResponse.setText(mUnderstoodText);
verify();
}
}
}
@Override
public void onIntentReceived(final String s) {
Log.d(LOGTAG, s);
}
@Override
public void onError(int errorCode, final String s) {
Loggable.AppError error = new Loggable.AppError(Loggable.Key.APP_ERROR, s);
Logger.track(error);
}
@Override
public void onAudioEvent(boolean recording) {
if (!recording) {
stopCapture();
}
}
@Override
public void initializeCapture() {
mRecognitionMode = SpeechRecognitionMode.ShortPhrase;
try {
//TODO: localize
String language = "en-us";
String subscriptionKey = KeyUtilities.getToken(getActivity(), "speech");
if (mMicClient == null) {
mMicClient = SpeechRecognitionServiceFactory.createMicrophoneClient(getActivity(), mRecognitionMode, language, this, subscriptionKey);
}
}
catch(Exception e){
Logger.trackException(e);
}
}
@Override
public void startCapture() {
mMicClient.startMicAndRecognition();
}
@Override
public void stopCapture() {
if (mMicClient != null) {
mMicClient.endMicAndRecognition();
}
}
@Override
public void onCountDownTimerExpired() {
gameFailure(false);
}
@Override
public void onSucceeded() {
if (mCallback != null) {
mCallback.onMimicSuccess(mSharableUri.getPath());
}
}
@Override
public void onFailed() {
if (mCallback != null) {
mCallback.onMimicFailure();
}
}
@Override
public void onInternalError() {
//TODO: implement
}
protected void gameSuccess(double difference) {
mSuccessMessage = getString(R.string.mimic_success_message);
if (difference <= DIFFERENCE_PERFECT_THRESHOLD) {
mSuccessMessage = getString(R.string.mimic_twister_perfect_message);
}
createSharableBitmap();
mStateManager.onMimicSuccess(mSuccessMessage);
}
protected void gameFailure(boolean allowRetry) {
if (allowRetry) {
String failureMessage = getString(R.string.mimic_failure_message);
mStateManager.onMimicFailureWithRetry(failureMessage);
}
else {
Loggable.UserAction userAction = new Loggable.UserAction(Loggable.Key.ACTION_GAME_TWISTER_TIMEOUT);
userAction.putProp(Loggable.Key.PROP_QUESTION, mQuestion);
Logger.track(userAction);
String failureMessage = getString(R.string.mimic_time_up_message);
mStateManager.onMimicFailure(failureMessage);
}
}
private void initialize(View view) {
mTextResponse = (TextView) view.findViewById(R.id.understood_text);
generateQuestion(view);
}
private void generateQuestion(View view) {
Resources resources = getResources();
String[] questions = resources.getStringArray(R.array.tongue_twisters);
mQuestion = questions[new Random().nextInt(questions.length)];
final TextView instructionTextView = (TextView) view.findViewById(R.id.instruction_text);
instructionTextView.setText(mQuestion);
}
//
// Create bitmap for sharing
//
private void createSharableBitmap() {
Bitmap sharableBitmap = Bitmap.createBitmap(getView().getWidth(), getView().getHeight(), Bitmap.Config.ARGB_8888);
Canvas canvas = new Canvas(sharableBitmap);
canvas.drawColor(ContextCompat.getColor(getContext(), R.color.white));
// Load the view for the sharable. This will be drawn to the bitmap
LayoutInflater inflater = (LayoutInflater) getContext().getSystemService(Context.LAYOUT_INFLATER_SERVICE);
LinearLayout layout = (LinearLayout)inflater.inflate(R.layout.fragment_sharable_tongue_twister, null);
TextView textView = (TextView)layout.findViewById(R.id.twister_sharable_tongue_twister);
textView.setText(mQuestion);
textView = (TextView)layout.findViewById(R.id.twister_sharable_i_said);
textView.setText(mUnderstoodText);
textView = (TextView)layout.findViewById(R.id.mimic_twister_share_success);
textView.setText(mSuccessMessage);
// Perform the layout using the dimension of the bitmap
int widthSpec = View.MeasureSpec.makeMeasureSpec(canvas.getWidth(), View.MeasureSpec.EXACTLY);
int heightSpec = View.MeasureSpec.makeMeasureSpec(canvas.getHeight(), View.MeasureSpec.EXACTLY);
layout.measure(widthSpec, heightSpec);
layout.layout(0, 0, layout.getMeasuredWidth(), layout.getMeasuredHeight());
// Draw the generated view to canvas
layout.draw(canvas);
String title = getString(R.string.app_short_name) + ": " + getString(R.string.mimic_twister_name);
mSharableUri = ShareFragment.saveShareableBitmap(getActivity(), sharableBitmap, title);
}
//https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance
private int levenshteinDistance(CharSequence lhs, CharSequence rhs) {
if (lhs == null && rhs == null) {
return 0;
}
if (lhs == null) {
return rhs.length();
}
if (rhs == null) {
return lhs.length();
}
int len0 = lhs.length() + 1;
int len1 = rhs.length() + 1;
// the array of distances
int[] cost = new int[len0];
int[] newcost = new int[len0];
// initial cost of skipping prefix in String s0
for (int i = 0; i < len0; i++) cost[i] = i;
// dynamically computing the array of distances
// transformation cost for each letter in s1
for (int j = 1; j < len1; j++) {
// initial cost of skipping prefix in String s1
newcost[0] = j;
// transformation cost for each letter in s0
for(int i = 1; i < len0; i++) {
// matching current letters in both strings
int match = (lhs.charAt(i - 1) == rhs.charAt(j - 1)) ? 0 : 1;
// computing cost for each transformation
int cost_replace = cost[i - 1] + match;
int cost_insert = cost[i] + 1;
int cost_delete = newcost[i - 1] + 1;
// keep minimum cost
newcost[i] = Math.min(Math.min(cost_insert, cost_delete), cost_replace);
}
// swap cost/newcost arrays
int[] swap = cost; cost = newcost; newcost = swap;
}
// the distance is the cost for transforming all letters in both strings
return cost[len0 - 1];
}
private void verify() {
if (mUnderstoodText == null) {
gameFailure(true);
return;
}
double difference = (double)levenshteinDistance(mUnderstoodText, mQuestion) / (double)mQuestion.length();
Loggable.UserAction userAction = new Loggable.UserAction(Loggable.Key.ACTION_GAME_TWISTER_SUCCESS);
userAction.putProp(Loggable.Key.PROP_QUESTION, mQuestion);
userAction.putProp(Loggable.Key.PROP_DIFF, difference);
if (difference <= DIFFERENCE_SUCCESS_THRESHOLD) {
Logger.track(userAction);
gameSuccess(difference);
}
else {
userAction.Name = Loggable.Key.ACTION_GAME_TWISTER_FAIL;
Logger.track(userAction);
gameFailure(true);
}
}
}