OpenGL filter prototype (does not work)

Power on the device on start only if scrcpy is mirroring the main
display.

server/src/main/java/com/genymobile/scrcpy/control/Controller.java

@@ -166,7 +166,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
 
     private void control() throws IOException {
         // on start, power on the device
-        if (powerOn && displayId != Device.DISPLAY_ID_NONE && !Device.isScreenOn()) {
+        if (powerOn && displayId == 0 && !Device.isScreenOn()) {
             Device.pressReleaseKeycode(KeyEvent.KEYCODE_POWER, displayId, Device.INJECT_MODE_ASYNC);
 
             // dirty hack
@@ -272,16 +272,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
                 break;
             case ControlMessage.TYPE_SET_DISPLAY_POWER:
                 if (supportsInputEvents && displayId != Device.DISPLAY_ID_NONE) {
-                    boolean on = msg.getOn();
-                    boolean setDisplayPowerOk = Device.setDisplayPower(displayId, on);
-                    if (setDisplayPowerOk) {
-                        keepDisplayPowerOff = !on;
-                        Ln.i("Device display turned " + (on ? "on" : "off"));
-                        if (cleanUp != null) {
-                            boolean mustRestoreOnExit = !on;
-                            cleanUp.setRestoreDisplayPower(mustRestoreOnExit);
-                        }
-                    }
+                    setDisplayPower(msg.getOn());
                 }
                 break;
             case ControlMessage.TYPE_ROTATE_DEVICE:
@@ -677,4 +668,16 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
             return data;
         }
     }
+
+    private void setDisplayPower(boolean on) {
+        boolean setDisplayPowerOk = Device.setDisplayPower(displayId, on);
+        if (setDisplayPowerOk) {
+            keepDisplayPowerOff = !on;
+            Ln.i("Device display turned " + (on ? "on" : "off"));
+            if (cleanUp != null) {
+                boolean mustRestoreOnExit = !on;
+                cleanUp.setRestoreDisplayPower(mustRestoreOnExit);
+            }
+        }
+    }
 }

server/src/main/java/com/genymobile/scrcpy/device/Size.java

@@ -52,6 +52,6 @@ public final class Size {
 
     @Override
     public String toString() {
-        return "Size{" + "width=" + width + ", height=" + height + '}';
+        return "Size{" + width + 'x' + height + '}';
     }
 }

server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java

@@ -7,12 +7,15 @@ import com.genymobile.scrcpy.device.DisplayInfo;
 import com.genymobile.scrcpy.device.Size;
 import com.genymobile.scrcpy.util.Ln;
 import com.genymobile.scrcpy.util.LogUtils;
+import com.genymobile.scrcpy.wrappers.DisplayManager;
 import com.genymobile.scrcpy.wrappers.ServiceManager;
 import com.genymobile.scrcpy.wrappers.SurfaceControl;
 
 import android.graphics.Rect;
 import android.hardware.display.VirtualDisplay;
 import android.os.Build;
+import android.os.Handler;
+import android.os.HandlerThread;
 import android.os.IBinder;
 import android.view.IDisplayFoldListener;
 import android.view.IRotationWatcher;
@@ -29,9 +32,19 @@ public class ScreenCapture extends SurfaceCapture {
     private DisplayInfo displayInfo;
     private ScreenInfo screenInfo;
 
+    // Source display size (before resizing/crop) for the current session
+    private Size sessionDisplaySize;
+
     private IBinder display;
     private VirtualDisplay virtualDisplay;
 
+    private DisplayManager.DisplayListenerHandle displayListenerHandle;
+    private HandlerThread handlerThread;
+
+    // On Android 14, the DisplayListener may be broken (it never sends events). This is fixed in recent Android 14 upgrades, but we can't really
+    // detect it directly, so register a RotationWatcher and a DisplayFoldListener as a fallback, until we receive the first event from
+    // DisplayListener (which proves that it works).
+    private boolean displayListenerWorks; // only accessed from the display listener thread
     private IRotationWatcher rotationWatcher;
     private IDisplayFoldListener displayFoldListener;
 
@@ -45,39 +58,57 @@ public class ScreenCapture extends SurfaceCapture {
 
     @Override
     public void init() {
-        if (displayId == 0) {
-            rotationWatcher = new IRotationWatcher.Stub() {
-                @Override
-                public void onRotationChanged(int rotation) {
-                    requestReset();
-                }
-            };
-            ServiceManager.getWindowManager().registerRotationWatcher(rotationWatcher, displayId);
+        if (Build.VERSION.SDK_INT == AndroidVersions.API_34_ANDROID_14) {
+            registerDisplayListenerFallbacks();
         }
 
-        if (Build.VERSION.SDK_INT >= AndroidVersions.API_29_ANDROID_10) {
-            displayFoldListener = new IDisplayFoldListener.Stub() {
-
-                private boolean first = true;
-
-                @Override
-                public void onDisplayFoldChanged(int displayId, boolean folded) {
-                    if (first) {
-                        // An event is posted on registration to signal the initial state. Ignore it to avoid restarting encoding.
-                        first = false;
-                        return;
-                    }
-
-                    if (ScreenCapture.this.displayId != displayId) {
-                        // Ignore events related to other display ids
-                        return;
-                    }
-
-                    requestReset();
+        handlerThread = new HandlerThread("DisplayListener");
+        handlerThread.start();
+        Handler handler = new Handler(handlerThread.getLooper());
+        displayListenerHandle = ServiceManager.getDisplayManager().registerDisplayListener(displayId -> {
+            if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                Ln.v("ScreenCapture: onDisplayChanged(" + displayId + ")");
+            }
+            if (Build.VERSION.SDK_INT == AndroidVersions.API_34_ANDROID_14) {
+                if (!displayListenerWorks) {
+                    // On the first display listener event, we know it works, we can unregister the fallbacks
+                    displayListenerWorks = true;
+                    unregisterDisplayListenerFallbacks();
                 }
-            };
-            ServiceManager.getWindowManager().registerDisplayFoldListener(displayFoldListener);
-        }
+            }
+            if (this.displayId == displayId) {
+                DisplayInfo di = ServiceManager.getDisplayManager().getDisplayInfo(displayId);
+                if (di == null) {
+                    Ln.w("DisplayInfo for " + displayId + " cannot be retrieved");
+                    // We can't compare with the current size, so reset unconditionally
+                    if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                        Ln.v("ScreenCapture: requestReset(): " + getSessionDisplaySize() + " -> (unknown)");
+                    }
+                    setSessionDisplaySize(null);
+                    requestReset();
+                } else {
+                    Size size = di.getSize();
+
+                    // The field is hidden on purpose, to read it with synchronization
+                    @SuppressWarnings("checkstyle:HiddenField")
+                    Size sessionDisplaySize = getSessionDisplaySize(); // synchronized
+
+                    // .equals() also works if sessionDisplaySize == null
+                    if (!size.equals(sessionDisplaySize)) {
+                        // Reset only if the size is different
+                        if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                            Ln.v("ScreenCapture: requestReset(): " + sessionDisplaySize + " -> " + size);
+                        }
+                        // Set the new size immediately, so that a future onDisplayChanged() event called before the asynchronous prepare()
+                        // considers that the current size is the requested size (to avoid a duplicate requestReset())
+                        setSessionDisplaySize(size);
+                        requestReset();
+                    } else if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                        Ln.v("ScreenCapture: Size not changed (" + size + "): do not requestReset()");
+                    }
+                }
+            }
+        }, handler);
     }
 
     @Override
@@ -92,6 +123,7 @@ public class ScreenCapture extends SurfaceCapture {
             Ln.w("Display doesn't have FLAG_SUPPORTS_PROTECTED_BUFFERS flag, mirroring can be restricted");
         }
 
+        setSessionDisplaySize(displayInfo.getSize());
         screenInfo = ScreenInfo.computeScreenInfo(displayInfo.getRotation(), displayInfo.getSize(), crop, maxSize, lockVideoOrientation);
     }
 
@@ -146,12 +178,19 @@ public class ScreenCapture extends SurfaceCapture {
 
     @Override
     public void release() {
-        if (rotationWatcher != null) {
-            ServiceManager.getWindowManager().unregisterRotationWatcher(rotationWatcher);
+        if (Build.VERSION.SDK_INT == AndroidVersions.API_34_ANDROID_14) {
+            unregisterDisplayListenerFallbacks();
         }
-        if (Build.VERSION.SDK_INT >= AndroidVersions.API_29_ANDROID_10) {
-            ServiceManager.getWindowManager().unregisterDisplayFoldListener(displayFoldListener);
+
+        handlerThread.quitSafely();
+        handlerThread = null;
+
+        // displayListenerHandle may be null if registration failed
+        if (displayListenerHandle != null) {
+            ServiceManager.getDisplayManager().unregisterDisplayListener(displayListenerHandle);
+            displayListenerHandle = null;
         }
+
         if (display != null) {
             SurfaceControl.destroyDisplay(display);
             display = null;
@@ -191,4 +230,65 @@ public class ScreenCapture extends SurfaceCapture {
             SurfaceControl.closeTransaction();
         }
     }
+
+    private synchronized Size getSessionDisplaySize() {
+        return sessionDisplaySize;
+    }
+
+    private synchronized void setSessionDisplaySize(Size sessionDisplaySize) {
+        this.sessionDisplaySize = sessionDisplaySize;
+    }
+
+    private void registerDisplayListenerFallbacks() {
+        rotationWatcher = new IRotationWatcher.Stub() {
+            @Override
+            public void onRotationChanged(int rotation) {
+                if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                    Ln.v("ScreenCapture: onRotationChanged(" + rotation + ")");
+                }
+                requestReset();
+            }
+        };
+        ServiceManager.getWindowManager().registerRotationWatcher(rotationWatcher, displayId);
+
+        // Build.VERSION.SDK_INT >= AndroidVersions.API_29_ANDROID_10 (but implied by == API_34_ANDROID 14)
+        displayFoldListener = new IDisplayFoldListener.Stub() {
+
+            private boolean first = true;
+
+            @Override
+            public void onDisplayFoldChanged(int displayId, boolean folded) {
+                if (first) {
+                    // An event is posted on registration to signal the initial state. Ignore it to avoid restarting encoding.
+                    first = false;
+                    return;
+                }
+
+                if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                    Ln.v("ScreenCapture: onDisplayFoldChanged(" + displayId + ", " + folded + ")");
+                }
+
+                if (ScreenCapture.this.displayId != displayId) {
+                    // Ignore events related to other display ids
+                    return;
+                }
+                requestReset();
+            }
+        };
+        ServiceManager.getWindowManager().registerDisplayFoldListener(displayFoldListener);
+    }
+
+    private void unregisterDisplayListenerFallbacks() {
+        synchronized (this) {
+            if (rotationWatcher != null) {
+                ServiceManager.getWindowManager().unregisterRotationWatcher(rotationWatcher);
+                rotationWatcher = null;
+            }
+            if (displayFoldListener != null) {
+                // Build.VERSION.SDK_INT >= AndroidVersions.API_29_ANDROID_10 (but implied by == API_34_ANDROID 14)
+                ServiceManager.getWindowManager().unregisterDisplayFoldListener(displayFoldListener);
+                displayFoldListener = null;
+            }
+        }
+    }
 }

server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java

@@ -72,6 +72,7 @@ public class SurfaceEncoder implements AsyncProcessor {
             boolean headerWritten = false;
 
             do {
+                capture.consumeReset(); // If a capture reset was requested, it is implicitly fulfilled
                 capture.prepare();
                 Size size = capture.getSize();
                 if (!headerWritten) {
@@ -87,6 +88,9 @@ public class SurfaceEncoder implements AsyncProcessor {
                     mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
                     surface = mediaCodec.createInputSurface();
 
+                    VideoFilter filter = new VideoFilter(surface);
+                    surface = filter.getInputSurface();
+
                     capture.start(surface);
 
                     mediaCodec.start();
@@ -94,6 +98,7 @@ public class SurfaceEncoder implements AsyncProcessor {
                     alive = encode(mediaCodec, streamer);
                     // do not call stop() on exception, it would trigger an IllegalStateException
                     mediaCodec.stop();
+                    filter.release();
                 } catch (IllegalStateException | IllegalArgumentException e) {
                     Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
                     if (!prepareRetry(size)) {

server/src/main/java/com/genymobile/scrcpy/video/VideoFilter.java

@@ -0,0 +1,116 @@
+package com.genymobile.scrcpy.video;
+
+import com.genymobile.scrcpy.util.Ln;
+
+import android.graphics.SurfaceTexture;
+import android.opengl.EGL14;
+import android.opengl.EGLConfig;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLSurface;
+import android.opengl.GLES11Ext;
+import android.opengl.GLES20;
+import android.view.Surface;
+
+public class VideoFilter {
+    private EGLDisplay eglDisplay;
+    private EGLContext eglContext;
+    private EGLSurface eglSurface;
+    private SurfaceTexture surfaceTexture;
+    private Surface inputSurface;
+    private int textureId;
+
+    public VideoFilter(Surface outputSurface) {
+        eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
+        if (eglDisplay == EGL14.EGL_NO_DISPLAY) {
+            throw new RuntimeException("Unable to get EGL14 display");
+        }
+
+        int[] version = new int[2];
+        if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) {
+            throw new RuntimeException("Unable to initialize EGL14");
+        }
+
+        int[] attribList = {
+                EGL14.EGL_RED_SIZE, 8,
+                EGL14.EGL_GREEN_SIZE, 8,
+                EGL14.EGL_BLUE_SIZE, 8,
+                EGL14.EGL_ALPHA_SIZE, 8,
+                EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
+                EGL14.EGL_NONE
+        };
+        EGLConfig[] configs = new EGLConfig[1];
+        int[] numConfigs = new int[1];
+        EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, configs.length, numConfigs, 0);
+        if (numConfigs[0] <= 0) {
+            throw new RuntimeException("Unable to find ES2 EGL config");
+        }
+        EGLConfig eglConfig = configs[0];
+
+        int[] contextAttribList = {
+                EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
+                EGL14.EGL_NONE
+        };
+        eglContext = EGL14.eglCreateContext(eglDisplay, eglConfig, EGL14.EGL_NO_CONTEXT, contextAttribList, 0);
+        if (eglContext == null) {
+            throw new RuntimeException("Failed to create EGL context");
+        }
+
+        int[] surfaceAttribList = {
+                EGL14.EGL_NONE
+        };
+        eglSurface = EGL14.eglCreateWindowSurface(eglDisplay, eglConfig, outputSurface, surfaceAttribList, 0);
+        if (eglSurface == null) {
+            throw new RuntimeException("Failed to create EGL window surface");
+        }
+
+        if (!EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) {
+            throw new RuntimeException("Failed to make EGL context current");
+        }
+
+        int[] textures = new int[1];
+        GLES20.glGenTextures(1, textures, 0);
+        textureId = textures[0];
+
+        surfaceTexture = new SurfaceTexture(textureId);
+        inputSurface = new Surface(surfaceTexture);
+
+        surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() {
+            @Override
+            public void onFrameAvailable(SurfaceTexture surfaceTexture) {
+                // XXX This should be called when the VirtualDisplay has rendered a new frame
+                Ln.i("==== render");
+                render();
+            }
+        });
+    }
+
+    public Surface getInputSurface() {
+        return inputSurface;
+    }
+
+    public void render() {
+        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId);
+
+        // For now, just paint with a color
+        GLES20.glClearColor(0.0f, 0.5f, 0.5f, 1.0f);
+        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+
+        GLES20.glViewport(0, 0, 1920, 1080);
+
+        EGL14.eglSwapBuffers(eglDisplay, eglSurface);
+    }
+
+    public void release() {
+        if (eglDisplay != EGL14.EGL_NO_DISPLAY) {
+            EGL14.eglDestroySurface(eglDisplay, eglSurface);
+            EGL14.eglDestroyContext(eglDisplay, eglContext);
+            EGL14.eglTerminate(eglDisplay);
+        }
+        eglDisplay = EGL14.EGL_NO_DISPLAY;
+        eglContext = EGL14.EGL_NO_CONTEXT;
+        eglSurface = EGL14.EGL_NO_SURFACE;
+        surfaceTexture.release();
+        inputSurface.release();
+    }
+}

server/src/main/java/com/genymobile/scrcpy/wrappers/DisplayManager.java

@@ -11,17 +11,40 @@ import android.annotation.SuppressLint;
 import android.annotation.TargetApi;
 import android.content.Context;
 import android.hardware.display.VirtualDisplay;
+import android.os.Handler;
 import android.view.Display;
 import android.view.Surface;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 @SuppressLint("PrivateApi,DiscouragedPrivateApi")
 public final class DisplayManager {
+
+    // android.hardware.display.DisplayManager.EVENT_FLAG_DISPLAY_CHANGED
+    public static final long EVENT_FLAG_DISPLAY_CHANGED = 1L << 2;
+
+    public interface DisplayListener {
+        /**
+         * Called whenever the properties of a logical {@link android.view.Display},
+         * such as size and density, have changed.
+         *
+         * @param displayId The id of the logical display that changed.
+         */
+        void onDisplayChanged(int displayId);
+    }
+
+    public static final class DisplayListenerHandle {
+        private final Object displayListenerProxy;
+        private DisplayListenerHandle(Object displayListenerProxy) {
+            this.displayListenerProxy = displayListenerProxy;
+        }
+    }
+
     private final Object manager; // instance of hidden class android.hardware.display.DisplayManagerGlobal
     private Method createVirtualDisplayMethod;
     private Method requestDisplayPowerMethod;
@@ -158,4 +181,50 @@ public final class DisplayManager {
             return false;
         }
     }
+
+    public DisplayListenerHandle registerDisplayListener(DisplayListener listener, Handler handler) {
+        try {
+            Class<?> displayListenerClass = Class.forName("android.hardware.display.DisplayManager$DisplayListener");
+            Object displayListenerProxy = Proxy.newProxyInstance(
+                    ClassLoader.getSystemClassLoader(),
+                    new Class[] {displayListenerClass},
+                    (proxy, method, args) -> {
+                        if ("onDisplayChanged".equals(method.getName())) {
+                            listener.onDisplayChanged((int) args[0]);
+                        }
+                        return null;
+                    });
+            try {
+                manager.getClass()
+                        .getMethod("registerDisplayListener", displayListenerClass, Handler.class, long.class, String.class)
+                        .invoke(manager, displayListenerProxy, handler, EVENT_FLAG_DISPLAY_CHANGED, FakeContext.PACKAGE_NAME);
+            } catch (NoSuchMethodException e) {
+                try {
+                    manager.getClass()
+                            .getMethod("registerDisplayListener", displayListenerClass, Handler.class, long.class)
+                            .invoke(manager, displayListenerProxy, handler, EVENT_FLAG_DISPLAY_CHANGED);
+                } catch (NoSuchMethodException e2) {
+                    manager.getClass()
+                            .getMethod("registerDisplayListener", displayListenerClass, Handler.class)
+                            .invoke(manager, displayListenerProxy, handler);
+                }
+            }
+
+            return new DisplayListenerHandle(displayListenerProxy);
+        } catch (Exception e) {
+            // Rotation and screen size won't be updated, not a fatal error
+            Ln.e("Could not register display listener", e);
+        }
+
+        return null;
+    }
+
+    public void unregisterDisplayListener(DisplayListenerHandle listener) {
+        try {
+            Class<?> displayListenerClass = Class.forName("android.hardware.display.DisplayManager$DisplayListener");
+            manager.getClass().getMethod("unregisterDisplayListener", displayListenerClass).invoke(manager, listener.displayListenerProxy);
+        } catch (Exception e) {
+            Ln.e("Could not unregister display listener", e);
+        }
+    }
 }