package com.limelight; import com.limelight.binding.PlatformBinding; import com.limelight.binding.input.ControllerHandler; import com.limelight.binding.input.KeyboardTranslator; import com.limelight.binding.input.capture.InputCaptureManager; import com.limelight.binding.input.capture.InputCaptureProvider; import com.limelight.binding.input.TouchContext; import com.limelight.binding.input.driver.UsbDriverService; import com.limelight.binding.input.evdev.EvdevListener; import com.limelight.binding.input.virtual_controller.VirtualController; import com.limelight.binding.video.EnhancedDecoderRenderer; import com.limelight.binding.video.MediaCodecDecoderRenderer; import com.limelight.binding.video.MediaCodecHelper; import com.limelight.nvstream.NvConnection; import com.limelight.nvstream.NvConnectionListener; import com.limelight.nvstream.StreamConfiguration; import com.limelight.nvstream.av.video.VideoDecoderRenderer; import com.limelight.nvstream.http.NvApp; import com.limelight.nvstream.input.KeyboardPacket; import com.limelight.nvstream.input.MouseButtonPacket; import com.limelight.preferences.PreferenceConfiguration; import com.limelight.ui.GameGestures; import com.limelight.ui.StreamView; import com.limelight.utils.Dialog; import com.limelight.utils.ShortcutHelper; import com.limelight.utils.SpinnerDialog; import com.limelight.utils.UiHelper; import android.annotation.SuppressLint; import android.app.Activity; import android.app.Service; import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.ServiceConnection; import android.content.res.Configuration; import android.graphics.Point; import android.hardware.input.InputManager; import android.media.AudioManager; import android.net.ConnectivityManager; import android.net.wifi.WifiManager; import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.os.IBinder; import android.os.SystemClock; import android.view.Display; import android.view.InputDevice; import android.view.KeyEvent; import android.view.MotionEvent; import android.view.SurfaceHolder; import android.view.View; import android.view.View.OnGenericMotionListener; import android.view.View.OnSystemUiVisibilityChangeListener; import android.view.View.OnTouchListener; import android.view.Window; import android.view.WindowManager; import android.widget.FrameLayout; import android.view.inputmethod.InputMethodManager; import android.widget.Toast; import java.util.Locale; public class Game extends Activity implements SurfaceHolder.Callback, OnGenericMotionListener, OnTouchListener, NvConnectionListener, EvdevListener, OnSystemUiVisibilityChangeListener, GameGestures { private int lastMouseX = Integer.MIN_VALUE; private int lastMouseY = Integer.MIN_VALUE; private int lastButtonState = 0; // Only 2 touches are supported private final TouchContext[] touchContextMap = new TouchContext[2]; private long threeFingerDownTime = 0; private static final int REFERENCE_HORIZ_RES = 1280; private static final int REFERENCE_VERT_RES = 720; private static final int THREE_FINGER_TAP_THRESHOLD = 300; private ControllerHandler controllerHandler; private VirtualController virtualController; private KeyboardTranslator keybTranslator; private PreferenceConfiguration prefConfig; private NvConnection conn; private SpinnerDialog spinner; private boolean displayedFailureDialog = false; private boolean connecting = false; private boolean connected = false; private InputCaptureProvider inputCaptureProvider; private int modifierFlags = 0; private boolean grabbedInput = true; private boolean grabComboDown = false; private StreamView streamView; private ShortcutHelper shortcutHelper; private EnhancedDecoderRenderer decoderRenderer; private WifiManager.WifiLock wifiLock; private int drFlags = 0; private boolean connectedToUsbDriverService = false; private ServiceConnection usbDriverServiceConnection = new ServiceConnection() { @Override public void onServiceConnected(ComponentName componentName, IBinder iBinder) { UsbDriverService.UsbDriverBinder binder = (UsbDriverService.UsbDriverBinder) iBinder; binder.setListener(controllerHandler); connectedToUsbDriverService = true; } @Override public void onServiceDisconnected(ComponentName componentName) { connectedToUsbDriverService = false; } }; public static final String EXTRA_HOST = "Host"; public static final String EXTRA_APP_NAME = "AppName"; public static final String EXTRA_APP_ID = "AppId"; public static final String EXTRA_UNIQUEID = "UniqueId"; public static final String EXTRA_STREAMING_REMOTE = "Remote"; public static final String EXTRA_PC_UUID = "UUID"; public static final String EXTRA_PC_NAME = "PcName"; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); shortcutHelper = new ShortcutHelper(this); UiHelper.setLocale(this); // We don't want a title bar requestWindowFeature(Window.FEATURE_NO_TITLE); // Full-screen and don't let the display go off getWindow().addFlags( WindowManager.LayoutParams.FLAG_FULLSCREEN | WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); // If we're going to use immersive mode, we want to have // the entire screen if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.KITKAT) { getWindow().getDecorView().setSystemUiVisibility( View.SYSTEM_UI_FLAG_LAYOUT_STABLE | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN); getWindow().addFlags(WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN); } // Listen for UI visibility events getWindow().getDecorView().setOnSystemUiVisibilityChangeListener(this); // Change volume button behavior setVolumeControlStream(AudioManager.STREAM_MUSIC); // Inflate the content setContentView(R.layout.activity_game); // Start the spinner spinner = SpinnerDialog.displayDialog(this, getResources().getString(R.string.conn_establishing_title), getResources().getString(R.string.conn_establishing_msg), true); // Read the stream preferences prefConfig = PreferenceConfiguration.readPreferences(this); if (prefConfig.stretchVideo) { drFlags |= VideoDecoderRenderer.FLAG_FILL_SCREEN; } // Listen for events on the game surface streamView = (StreamView) findViewById(R.id.surfaceView); streamView.setOnGenericMotionListener(this); streamView.setOnTouchListener(this); // Warn the user if they're on a metered connection checkDataConnection(); // Make sure Wi-Fi is fully powered up WifiManager wifiMgr = (WifiManager) getApplicationContext().getSystemService(Context.WIFI_SERVICE); wifiLock = wifiMgr.createWifiLock(WifiManager.WIFI_MODE_FULL_HIGH_PERF, "Limelight"); wifiLock.setReferenceCounted(false); wifiLock.acquire(); String host = Game.this.getIntent().getStringExtra(EXTRA_HOST); String appName = Game.this.getIntent().getStringExtra(EXTRA_APP_NAME); int appId = Game.this.getIntent().getIntExtra(EXTRA_APP_ID, StreamConfiguration.INVALID_APP_ID); String uniqueId = Game.this.getIntent().getStringExtra(EXTRA_UNIQUEID); boolean remote = Game.this.getIntent().getBooleanExtra(EXTRA_STREAMING_REMOTE, false); String uuid = Game.this.getIntent().getStringExtra(EXTRA_PC_UUID); String pcName = Game.this.getIntent().getStringExtra(EXTRA_PC_NAME); if (appId == StreamConfiguration.INVALID_APP_ID) { finish(); return; } // Add a launcher shortcut for this PC (forced, since this is user interaction) shortcutHelper.createAppViewShortcut(uuid, pcName, uuid, true); shortcutHelper.reportShortcutUsed(uuid); // Initialize the MediaCodec helper before creating the decoder MediaCodecHelper.initializeWithContext(this); decoderRenderer = new MediaCodecDecoderRenderer(prefConfig.videoFormat); // Display a message to the user if H.265 was forced on but we still didn't find a decoder if (prefConfig.videoFormat == PreferenceConfiguration.FORCE_H265_ON && !decoderRenderer.isHevcSupported()) { Toast.makeText(this, "No H.265 decoder found.\nFalling back to H.264.", Toast.LENGTH_LONG).show(); } if (!decoderRenderer.isAvcSupported()) { if (spinner != null) { spinner.dismiss(); spinner = null; } // If we can't find an AVC decoder, we can't proceed Dialog.displayDialog(this, getResources().getString(R.string.conn_error_title), "This device or ROM doesn't support hardware accelerated H.264 playback.", true); return; } StreamConfiguration config = new StreamConfiguration.Builder() .setResolution(prefConfig.width, prefConfig.height) .setRefreshRate(prefConfig.fps) .setApp(new NvApp(appName, appId)) .setBitrate(prefConfig.bitrate * 1000) .setEnableSops(prefConfig.enableSops) .enableAdaptiveResolution((decoderRenderer.getCapabilities() & VideoDecoderRenderer.CAPABILITY_ADAPTIVE_RESOLUTION) != 0) .enableLocalAudioPlayback(prefConfig.playHostAudio) .setMaxPacketSize(remote ? 1024 : 1292) .setRemote(remote) .setHevcSupported(decoderRenderer.isHevcSupported()) .setAudioConfiguration(prefConfig.enable51Surround ? StreamConfiguration.AUDIO_CONFIGURATION_5_1 : StreamConfiguration.AUDIO_CONFIGURATION_STEREO) .build(); // Initialize the connection conn = new NvConnection(host, uniqueId, Game.this, config, PlatformBinding.getCryptoProvider(this)); keybTranslator = new KeyboardTranslator(conn); controllerHandler = new ControllerHandler(this, conn, this, prefConfig.multiController, prefConfig.deadzonePercentage); InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE); inputManager.registerInputDeviceListener(controllerHandler, null); // Set to the optimal mode for streaming prepareDisplayForRendering(); // Initialize touch contexts for (int i = 0; i < touchContextMap.length; i++) { touchContextMap[i] = new TouchContext(conn, i, REFERENCE_HORIZ_RES, REFERENCE_VERT_RES, streamView); } // Use sustained performance mode on N+ to ensure consistent // CPU availability if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { getWindow().setSustainedPerformanceMode(true); } inputCaptureProvider = InputCaptureManager.getInputCaptureProvider(this, this); if (prefConfig.onscreenController) { // create virtual onscreen controller virtualController = new VirtualController(conn, (FrameLayout)findViewById(R.id.surfaceView).getParent(), this); virtualController.refreshLayout(); } if (prefConfig.usbDriver) { // Start the USB driver bindService(new Intent(this, UsbDriverService.class), usbDriverServiceConnection, Service.BIND_AUTO_CREATE); } // The connection will be started when the surface gets created streamView.getHolder().addCallback(this); } private void prepareDisplayForRendering() { Display display = getWindowManager().getDefaultDisplay(); WindowManager.LayoutParams windowLayoutParams = getWindow().getAttributes(); // On M, we can explicitly set the optimal display mode if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { Display.Mode bestMode = display.getMode(); for (Display.Mode candidate : display.getSupportedModes()) { boolean refreshRateOk = candidate.getRefreshRate() >= bestMode.getRefreshRate() && candidate.getRefreshRate() < 63; boolean resolutionOk = candidate.getPhysicalWidth() >= bestMode.getPhysicalWidth() && candidate.getPhysicalHeight() >= bestMode.getPhysicalHeight() && candidate.getPhysicalWidth() <= 4096; LimeLog.info("Examining display mode: "+candidate.getPhysicalWidth()+"x"+ candidate.getPhysicalHeight()+"x"+candidate.getRefreshRate()); // On non-4K streams, we force the resolution to never change if (prefConfig.width < 3840) { if (display.getMode().getPhysicalWidth() != candidate.getPhysicalWidth() || display.getMode().getPhysicalHeight() != candidate.getPhysicalHeight()) { continue; } } // Make sure the refresh rate doesn't regress if (!refreshRateOk) { continue; } // Make sure the resolution doesn't regress if (!resolutionOk) { continue; } bestMode = candidate; } LimeLog.info("Selected display mode: "+bestMode.getPhysicalWidth()+"x"+ bestMode.getPhysicalHeight()+"x"+bestMode.getRefreshRate()); windowLayoutParams.preferredDisplayModeId = bestMode.getModeId(); } // On L, we can at least tell the OS that we want 60 Hz else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { float bestRefreshRate = display.getRefreshRate(); for (float candidate : display.getSupportedRefreshRates()) { if (candidate > bestRefreshRate && candidate < 63) { LimeLog.info("Examining refresh rate: "+candidate); bestRefreshRate = candidate; } } LimeLog.info("Selected refresh rate: "+bestRefreshRate); windowLayoutParams.preferredRefreshRate = bestRefreshRate; } // Apply the display mode change getWindow().setAttributes(windowLayoutParams); // From 4.4 to 5.1 we can't ask for a 4K display mode, so we'll // need to hint the OS to provide one. boolean aspectRatioMatch = false; if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT && Build.VERSION.SDK_INT <= Build.VERSION_CODES.LOLLIPOP_MR1) { // On KitKat and later (where we can use the whole screen via immersive mode), we'll // calculate whether we need to scale by aspect ratio or not. If not, we'll use // setFixedSize so we can handle 4K properly. The only known devices that have // >= 4K screens have exactly 4K screens, so we'll be able to hit this good path // on these devices. On Marshmallow, we can start changing to 4K manually but no // 4K devices run 6.0 at the moment. Point screenSize = new Point(0, 0); display.getSize(screenSize); double screenAspectRatio = ((double)screenSize.y) / screenSize.x; double streamAspectRatio = ((double)prefConfig.height) / prefConfig.width; if (Math.abs(screenAspectRatio - streamAspectRatio) < 0.001) { LimeLog.info("Stream has compatible aspect ratio with output display"); aspectRatioMatch = true; } } if (prefConfig.stretchVideo || aspectRatioMatch) { // Set the surface to the size of the video streamView.getHolder().setFixedSize(prefConfig.width, prefConfig.height); } else { // Set the surface to scale based on the aspect ratio of the stream streamView.setDesiredAspectRatio((double)prefConfig.width / (double)prefConfig.height); } } private void checkDataConnection() { ConnectivityManager mgr = (ConnectivityManager) getSystemService(Context.CONNECTIVITY_SERVICE); if (mgr.isActiveNetworkMetered()) { displayTransientMessage(getResources().getString(R.string.conn_metered)); } } @SuppressLint("InlinedApi") private final Runnable hideSystemUi = new Runnable() { @Override public void run() { // Use immersive mode on 4.4+ or standard low profile on previous builds if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.KITKAT) { Game.this.getWindow().getDecorView().setSystemUiVisibility( View.SYSTEM_UI_FLAG_LAYOUT_STABLE | View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_FULLSCREEN | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY); } else { Game.this.getWindow().getDecorView().setSystemUiVisibility( View.SYSTEM_UI_FLAG_FULLSCREEN | View.SYSTEM_UI_FLAG_LOW_PROFILE); } } }; private void hideSystemUi(int delay) { Handler h = getWindow().getDecorView().getHandler(); if (h != null) { h.removeCallbacks(hideSystemUi); h.postDelayed(hideSystemUi, delay); } } @Override protected void onStop() { super.onStop(); SpinnerDialog.closeDialogs(this); Dialog.closeDialogs(); if (controllerHandler != null) { InputManager inputManager = (InputManager) getSystemService(Context.INPUT_SERVICE); inputManager.unregisterInputDeviceListener(controllerHandler); } wifiLock.release(); if (connectedToUsbDriverService) { // Unbind from the discovery service unbindService(usbDriverServiceConnection); } if (conn != null) { VideoDecoderRenderer.VideoFormat videoFormat = conn.getActiveVideoFormat(); displayedFailureDialog = true; stopConnection(); int averageEndToEndLat = decoderRenderer.getAverageEndToEndLatency(); int averageDecoderLat = decoderRenderer.getAverageDecoderLatency(); String message = null; if (averageEndToEndLat > 0) { message = getResources().getString(R.string.conn_client_latency)+" "+averageEndToEndLat+" ms"; if (averageDecoderLat > 0) { message += " ("+getResources().getString(R.string.conn_client_latency_hw)+" "+averageDecoderLat+" ms)"; } } else if (averageDecoderLat > 0) { message = getResources().getString(R.string.conn_hardware_latency)+" "+averageDecoderLat+" ms"; } // Add the video codec to the post-stream toast if (message != null && videoFormat != VideoDecoderRenderer.VideoFormat.Unknown) { if (videoFormat == VideoDecoderRenderer.VideoFormat.H265) { message += " [H.265]"; } else { message += " [H.264]"; } } if (message != null) { Toast.makeText(this, message, Toast.LENGTH_LONG).show(); } } finish(); } private final Runnable toggleGrab = new Runnable() { @Override public void run() { if (grabbedInput) { inputCaptureProvider.disableCapture(); } else { inputCaptureProvider.enableCapture(); } grabbedInput = !grabbedInput; } }; // Returns true if the key stroke was consumed private boolean handleSpecialKeys(short translatedKey, boolean down) { int modifierMask = 0; // Mask off the high byte translatedKey &= 0xff; if (translatedKey == KeyboardTranslator.VK_CONTROL) { modifierMask = KeyboardPacket.MODIFIER_CTRL; } else if (translatedKey == KeyboardTranslator.VK_SHIFT) { modifierMask = KeyboardPacket.MODIFIER_SHIFT; } else if (translatedKey == KeyboardTranslator.VK_ALT) { modifierMask = KeyboardPacket.MODIFIER_ALT; } if (down) { this.modifierFlags |= modifierMask; } else { this.modifierFlags &= ~modifierMask; } // Check if Ctrl+Shift+Z is pressed if (translatedKey == KeyboardTranslator.VK_Z && (modifierFlags & (KeyboardPacket.MODIFIER_CTRL | KeyboardPacket.MODIFIER_SHIFT)) == (KeyboardPacket.MODIFIER_CTRL | KeyboardPacket.MODIFIER_SHIFT)) { if (down) { // Now that we've pressed the magic combo // we'll wait for one of the keys to come up grabComboDown = true; } else { // Toggle the grab if Z comes up Handler h = getWindow().getDecorView().getHandler(); if (h != null) { h.postDelayed(toggleGrab, 250); } grabComboDown = false; } return true; } // Toggle the grab if control or shift comes up else if (grabComboDown) { Handler h = getWindow().getDecorView().getHandler(); if (h != null) { h.postDelayed(toggleGrab, 250); } grabComboDown = false; return true; } // Not a special combo return false; } private static byte getModifierState(KeyEvent event) { byte modifier = 0; if (event.isShiftPressed()) { modifier |= KeyboardPacket.MODIFIER_SHIFT; } if (event.isCtrlPressed()) { modifier |= KeyboardPacket.MODIFIER_CTRL; } if (event.isAltPressed()) { modifier |= KeyboardPacket.MODIFIER_ALT; } return modifier; } private byte getModifierState() { return (byte) modifierFlags; } @Override public boolean onKeyDown(int keyCode, KeyEvent event) { // Pass-through virtual navigation keys if ((event.getFlags() & KeyEvent.FLAG_VIRTUAL_HARD_KEY) != 0) { return super.onKeyDown(keyCode, event); } boolean handled = false; boolean detectedGamepad = event.getDevice() == null ? false : ((event.getDevice().getSources() & InputDevice.SOURCE_JOYSTICK) != 0 || (event.getDevice().getSources() & InputDevice.SOURCE_GAMEPAD) != 0); if (detectedGamepad || (event.getDevice() == null || event.getDevice().getKeyboardType() != InputDevice.KEYBOARD_TYPE_ALPHABETIC )) { // Always try the controller handler first, unless it's an alphanumeric keyboard device. // Otherwise, controller handler will eat keyboard d-pad events. handled = controllerHandler.handleButtonDown(event); } if (!handled) { // Try the keyboard handler short translated = keybTranslator.translate(event.getKeyCode()); if (translated == 0) { return super.onKeyDown(keyCode, event); } // Let this method take duplicate key down events if (handleSpecialKeys(translated, true)) { return true; } // Pass through keyboard input if we're not grabbing if (!grabbedInput) { return super.onKeyDown(keyCode, event); } keybTranslator.sendKeyDown(translated, getModifierState(event)); } return true; } @Override public boolean onKeyUp(int keyCode, KeyEvent event) { // Pass-through virtual navigation keys if ((event.getFlags() & KeyEvent.FLAG_VIRTUAL_HARD_KEY) != 0) { return super.onKeyUp(keyCode, event); } boolean handled = false; boolean detectedGamepad = event.getDevice() == null ? false : ((event.getDevice().getSources() & InputDevice.SOURCE_JOYSTICK) != 0 || (event.getDevice().getSources() & InputDevice.SOURCE_GAMEPAD) != 0); if (detectedGamepad || (event.getDevice() == null || event.getDevice().getKeyboardType() != InputDevice.KEYBOARD_TYPE_ALPHABETIC )) { // Always try the controller handler first, unless it's an alphanumeric keyboard device. // Otherwise, controller handler will eat keyboard d-pad events. handled = controllerHandler.handleButtonUp(event); } if (!handled) { // Try the keyboard handler short translated = keybTranslator.translate(event.getKeyCode()); if (translated == 0) { return super.onKeyUp(keyCode, event); } if (handleSpecialKeys(translated, false)) { return true; } // Pass through keyboard input if we're not grabbing if (!grabbedInput) { return super.onKeyUp(keyCode, event); } keybTranslator.sendKeyUp(translated, getModifierState(event)); } return true; } private TouchContext getTouchContext(int actionIndex) { if (actionIndex < touchContextMap.length) { return touchContextMap[actionIndex]; } else { return null; } } @Override public void showKeyboard() { LimeLog.info("Showing keyboard overlay"); InputMethodManager inputManager = (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE); inputManager.toggleSoftInput(InputMethodManager.SHOW_FORCED, InputMethodManager.HIDE_IMPLICIT_ONLY); } // Returns true if the event was consumed private boolean handleMotionEvent(MotionEvent event) { // Pass through keyboard input if we're not grabbing if (!grabbedInput) { return false; } if ((event.getSource() & InputDevice.SOURCE_CLASS_JOYSTICK) != 0) { if (controllerHandler.handleMotionEvent(event)) { return true; } } else if ((event.getSource() & InputDevice.SOURCE_CLASS_POINTER) != 0) { // This case is for mice if (event.getSource() == InputDevice.SOURCE_MOUSE || (event.getPointerCount() >= 1 && event.getToolType(0) == MotionEvent.TOOL_TYPE_MOUSE)) { int changedButtons = event.getButtonState() ^ lastButtonState; if (event.getActionMasked() == MotionEvent.ACTION_SCROLL) { // Send the vertical scroll packet byte vScrollClicks = (byte) event.getAxisValue(MotionEvent.AXIS_VSCROLL); conn.sendMouseScroll(vScrollClicks); } if ((changedButtons & MotionEvent.BUTTON_PRIMARY) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_PRIMARY) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_LEFT); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_LEFT); } } if ((changedButtons & MotionEvent.BUTTON_SECONDARY) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_SECONDARY) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_RIGHT); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_RIGHT); } } if ((changedButtons & MotionEvent.BUTTON_TERTIARY) != 0) { if ((event.getButtonState() & MotionEvent.BUTTON_TERTIARY) != 0) { conn.sendMouseButtonDown(MouseButtonPacket.BUTTON_MIDDLE); } else { conn.sendMouseButtonUp(MouseButtonPacket.BUTTON_MIDDLE); } } // Get relative axis values if we can if (inputCaptureProvider.eventHasRelativeMouseAxes(event)) { // Send the deltas straight from the motion event conn.sendMouseMove((short) inputCaptureProvider.getRelativeAxisX(event), (short) inputCaptureProvider.getRelativeAxisY(event)); // We have to also update the position Android thinks the cursor is at // in order to avoid jumping when we stop moving or click. lastMouseX = (int)event.getX(); lastMouseY = (int)event.getY(); } else { // First process the history for (int i = 0; i < event.getHistorySize(); i++) { updateMousePosition((int)event.getHistoricalX(i), (int)event.getHistoricalY(i)); } // Now process the current values updateMousePosition((int)event.getX(), (int)event.getY()); } lastButtonState = event.getButtonState(); } // This case is for touch-based input devices else { if (virtualController != null && virtualController.getControllerMode() == VirtualController.ControllerMode.Configuration) { // Ignore presses when the virtual controller is in configuration mode return true; } int actionIndex = event.getActionIndex(); int eventX = (int)event.getX(actionIndex); int eventY = (int)event.getY(actionIndex); // Special handling for 3 finger gesture if (event.getActionMasked() == MotionEvent.ACTION_POINTER_DOWN && event.getPointerCount() == 3) { // Three fingers down threeFingerDownTime = SystemClock.uptimeMillis(); // Cancel the first and second touches to avoid // erroneous events for (TouchContext aTouchContext : touchContextMap) { aTouchContext.cancelTouch(); } return true; } TouchContext context = getTouchContext(actionIndex); if (context == null) { return false; } switch (event.getActionMasked()) { case MotionEvent.ACTION_POINTER_DOWN: case MotionEvent.ACTION_DOWN: context.touchDownEvent(eventX, eventY); break; case MotionEvent.ACTION_POINTER_UP: case MotionEvent.ACTION_UP: if (event.getPointerCount() == 1) { // All fingers up if (SystemClock.uptimeMillis() - threeFingerDownTime < THREE_FINGER_TAP_THRESHOLD) { // This is a 3 finger tap to bring up the keyboard showKeyboard(); return true; } } context.touchUpEvent(eventX, eventY); if (actionIndex == 0 && event.getPointerCount() > 1 && !context.isCancelled()) { // The original secondary touch now becomes primary context.touchDownEvent((int)event.getX(1), (int)event.getY(1)); } break; case MotionEvent.ACTION_MOVE: // ACTION_MOVE is special because it always has actionIndex == 0 // We'll call the move handlers for all indexes manually // First process the historical events for (int i = 0; i < event.getHistorySize(); i++) { for (TouchContext aTouchContextMap : touchContextMap) { if (aTouchContextMap.getActionIndex() < event.getPointerCount()) { aTouchContextMap.touchMoveEvent( (int)event.getHistoricalX(aTouchContextMap.getActionIndex(), i), (int)event.getHistoricalY(aTouchContextMap.getActionIndex(), i)); } } } // Now process the current values for (TouchContext aTouchContextMap : touchContextMap) { if (aTouchContextMap.getActionIndex() < event.getPointerCount()) { aTouchContextMap.touchMoveEvent( (int)event.getX(aTouchContextMap.getActionIndex()), (int)event.getY(aTouchContextMap.getActionIndex())); } } break; default: return false; } } // Handled a known source return true; } // Unknown class return false; } @Override public boolean onTouchEvent(MotionEvent event) { return handleMotionEvent(event) || super.onTouchEvent(event); } @Override public boolean onGenericMotionEvent(MotionEvent event) { return handleMotionEvent(event) || super.onGenericMotionEvent(event); } private void updateMousePosition(int eventX, int eventY) { // Send a mouse move if we already have a mouse location // and the mouse coordinates change if (lastMouseX != Integer.MIN_VALUE && lastMouseY != Integer.MIN_VALUE && !(lastMouseX == eventX && lastMouseY == eventY)) { int deltaX = eventX - lastMouseX; int deltaY = eventY - lastMouseY; // Scale the deltas if the device resolution is different // than the stream resolution deltaX = (int)Math.round((double)deltaX * (REFERENCE_HORIZ_RES / (double)streamView.getWidth())); deltaY = (int)Math.round((double)deltaY * (REFERENCE_VERT_RES / (double)streamView.getHeight())); conn.sendMouseMove((short)deltaX, (short)deltaY); } // Update pointer location for delta calculation next time lastMouseX = eventX; lastMouseY = eventY; } @Override public boolean onGenericMotion(View v, MotionEvent event) { return handleMotionEvent(event); } @SuppressLint("ClickableViewAccessibility") @Override public boolean onTouch(View v, MotionEvent event) { return handleMotionEvent(event); } @Override public void stageStarting(Stage stage) { if (spinner != null) { spinner.setMessage(getResources().getString(R.string.conn_starting)+" "+stage.getName()); } } @Override public void stageComplete(Stage stage) { } private void stopConnection() { if (connecting || connected) { connecting = connected = false; conn.stop(); } // Enable cursor visibility again inputCaptureProvider.disableCapture(); // Destroy the capture provider inputCaptureProvider.destroy(); } @Override public void stageFailed(Stage stage) { if (spinner != null) { spinner.dismiss(); spinner = null; } if (!displayedFailureDialog) { displayedFailureDialog = true; stopConnection(); Dialog.displayDialog(this, getResources().getString(R.string.conn_error_title), getResources().getString(R.string.conn_error_msg)+" "+stage.getName(), true); } } @Override public void connectionTerminated(Exception e) { if (!displayedFailureDialog) { displayedFailureDialog = true; e.printStackTrace(); stopConnection(); Dialog.displayDialog(this, getResources().getString(R.string.conn_terminated_title), getResources().getString(R.string.conn_terminated_msg), true); } } @Override public void connectionStarted() { if (spinner != null) { spinner.dismiss(); spinner = null; } connecting = false; connected = true; runOnUiThread(new Runnable() { @Override public void run() { // Hide the mouse cursor now. Doing it before // dismissing the spinner seems to be undone // when the spinner gets displayed. inputCaptureProvider.enableCapture(); } }); hideSystemUi(1000); } @Override public void displayMessage(final String message) { runOnUiThread(new Runnable() { @Override public void run() { Toast.makeText(Game.this, message, Toast.LENGTH_LONG).show(); } }); } @Override public void displayTransientMessage(final String message) { if (!prefConfig.disableWarnings) { runOnUiThread(new Runnable() { @Override public void run() { Toast.makeText(Game.this, message, Toast.LENGTH_LONG).show(); } }); } } @Override public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { } @Override public void surfaceCreated(SurfaceHolder holder) { if (!connected && !connecting) { connecting = true; conn.start(PlatformBinding.getDeviceName(), holder, drFlags, PlatformBinding.getAudioRenderer(), decoderRenderer); } } @Override public void surfaceDestroyed(SurfaceHolder holder) { if (connected) { // HACK: Android is supposed to let you return from this function // before throwing a fit if you access the surface again. Unfortunately, // MediaCodec often tries to access the destroyed surface and triggers // an IllegalStateException. To workaround this, we will invoke // the DecoderRenderer's stop function ourselves, so it will hopefully // happen early enough to not trigger the bug decoderRenderer.stop(); stopConnection(); } } @Override public void mouseMove(int deltaX, int deltaY) { conn.sendMouseMove((short) deltaX, (short) deltaY); } @Override public void mouseButtonEvent(int buttonId, boolean down) { byte buttonIndex; switch (buttonId) { case EvdevListener.BUTTON_LEFT: buttonIndex = MouseButtonPacket.BUTTON_LEFT; break; case EvdevListener.BUTTON_MIDDLE: buttonIndex = MouseButtonPacket.BUTTON_MIDDLE; break; case EvdevListener.BUTTON_RIGHT: buttonIndex = MouseButtonPacket.BUTTON_RIGHT; break; default: LimeLog.warning("Unhandled button: "+buttonId); return; } if (down) { conn.sendMouseButtonDown(buttonIndex); } else { conn.sendMouseButtonUp(buttonIndex); } } @Override public void mouseScroll(byte amount) { conn.sendMouseScroll(amount); } @Override public void keyboardEvent(boolean buttonDown, short keyCode) { short keyMap = keybTranslator.translate(keyCode); if (keyMap != 0) { if (handleSpecialKeys(keyMap, buttonDown)) { return; } if (buttonDown) { keybTranslator.sendKeyDown(keyMap, getModifierState()); } else { keybTranslator.sendKeyUp(keyMap, getModifierState()); } } } @Override public void onSystemUiVisibilityChange(int visibility) { // Don't do anything if we're not connected if (!connected) { return; } // This flag is set for all devices if ((visibility & View.SYSTEM_UI_FLAG_FULLSCREEN) == 0) { hideSystemUi(2000); } // This flag is only set on 4.4+ else if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.KITKAT && (visibility & View.SYSTEM_UI_FLAG_HIDE_NAVIGATION) == 0) { hideSystemUi(2000); } // This flag is only set before 4.4+ else if (android.os.Build.VERSION.SDK_INT < android.os.Build.VERSION_CODES.KITKAT && (visibility & View.SYSTEM_UI_FLAG_LOW_PROFILE) == 0) { hideSystemUi(2000); } } }