From 2a04858a2271fdb44a0eeaeeadaedfc966eab48d Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 20 Oct 2024 16:51:32 +0200 Subject: [PATCH] Add on-device OpenGL video filter architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce several key components to perform OpenGL filters: - OpenGLRunner: a tool for running a filter to be rendered to a Surface from an OpenGL-dedicated thread - OpenGLFilter: a simple OpenGL filter API - AffineOpenGLFilter: a generic OpenGL implementation to apply any 2D affine transform - AffineMatrix: an affine transform matrix, with helpers to build matrices from semantic transformations (rotate, scale, translate…) PR #5455 --- server/build_without_gradle.sh | 1 + .../java/com/genymobile/scrcpy/Server.java | 4 + .../scrcpy/opengl/AffineOpenGLFilter.java | 135 +++++++ .../com/genymobile/scrcpy/opengl/GLUtils.java | 124 ++++++ .../scrcpy/opengl/OpenGLException.java | 13 + .../scrcpy/opengl/OpenGLFilter.java | 21 + .../scrcpy/opengl/OpenGLRunner.java | 246 ++++++++++++ .../genymobile/scrcpy/util/AffineMatrix.java | 368 ++++++++++++++++++ 8 files changed, 912 insertions(+) create mode 100644 server/src/main/java/com/genymobile/scrcpy/opengl/AffineOpenGLFilter.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/opengl/GLUtils.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLException.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLFilter.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java create mode 100644 server/src/main/java/com/genymobile/scrcpy/util/AffineMatrix.java diff --git a/server/build_without_gradle.sh b/server/build_without_gradle.sh index e0fc3a95..206aa604 100755 --- a/server/build_without_gradle.sh +++ b/server/build_without_gradle.sh @@ -60,6 +60,7 @@ SRC=( \ com/genymobile/scrcpy/audio/*.java \ com/genymobile/scrcpy/control/*.java \ com/genymobile/scrcpy/device/*.java \ + com/genymobile/scrcpy/opengl/*.java \ com/genymobile/scrcpy/util/*.java \ com/genymobile/scrcpy/video/*.java \ com/genymobile/scrcpy/wrappers/*.java \ diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index dae73b64..ca53d861 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -14,6 +14,7 @@ import com.genymobile.scrcpy.device.DesktopConnection; import com.genymobile.scrcpy.device.Device; import com.genymobile.scrcpy.device.NewDisplay; import com.genymobile.scrcpy.device.Streamer; +import com.genymobile.scrcpy.opengl.OpenGLRunner; import com.genymobile.scrcpy.util.Ln; import com.genymobile.scrcpy.util.LogUtils; import com.genymobile.scrcpy.video.CameraCapture; @@ -191,6 +192,8 @@ public final class Server { asyncProcessor.stop(); } + OpenGLRunner.quit(); // quit the OpenGL thread, if any + connection.shutdown(); try { @@ -200,6 +203,7 @@ public final class Server { for (AsyncProcessor asyncProcessor : asyncProcessors) { asyncProcessor.join(); } + OpenGLRunner.join(); } catch (InterruptedException e) { // ignore } diff --git a/server/src/main/java/com/genymobile/scrcpy/opengl/AffineOpenGLFilter.java b/server/src/main/java/com/genymobile/scrcpy/opengl/AffineOpenGLFilter.java new file mode 100644 index 00000000..7608a574 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/opengl/AffineOpenGLFilter.java @@ -0,0 +1,135 @@ +package com.genymobile.scrcpy.opengl; + +import com.genymobile.scrcpy.util.AffineMatrix; + +import android.opengl.GLES11Ext; +import android.opengl.GLES20; + +import java.nio.FloatBuffer; + +public class AffineOpenGLFilter implements OpenGLFilter { + + private int program; + private FloatBuffer vertexBuffer; + private FloatBuffer texCoordsBuffer; + private final float[] userMatrix; + + private int vertexPosLoc; + private int texCoordsInLoc; + + private int texLoc; + private int texMatrixLoc; + private int userMatrixLoc; + + public AffineOpenGLFilter(AffineMatrix transform) { + userMatrix = transform.to4x4(); + } + + @Override + public void init() throws OpenGLException { + // @formatter:off + String vertexShaderCode = "#version 100\n" + + "attribute vec4 vertex_pos;\n" + + "attribute vec4 tex_coords_in;\n" + + "varying vec2 tex_coords;\n" + + "uniform mat4 tex_matrix;\n" + + "uniform mat4 user_matrix;\n" + + "void main() {\n" + + " gl_Position = vertex_pos;\n" + + " tex_coords = (tex_matrix * user_matrix * tex_coords_in).xy;\n" + + "}"; + + // @formatter:off + String fragmentShaderCode = "#version 100\n" + + "#extension GL_OES_EGL_image_external : require\n" + + "precision highp float;\n" + + "uniform samplerExternalOES tex;\n" + + "varying vec2 tex_coords;\n" + + "void main() {\n" + + " if (tex_coords.x >= 0.0 && tex_coords.x <= 1.0\n" + + " && tex_coords.y >= 0.0 && tex_coords.y <= 1.0) {\n" + + " gl_FragColor = texture2D(tex, tex_coords);\n" + + " } else {\n" + + " gl_FragColor = vec4(0.0);\n" + + " }\n" + + "}"; + + program = GLUtils.createProgram(vertexShaderCode, fragmentShaderCode); + if (program == 0) { + throw new OpenGLException("Cannot create OpenGL program"); + } + + float[] vertices = { + -1, -1, // Bottom-left + 1, -1, // Bottom-right + -1, 1, // Top-left + 1, 1, // Top-right + }; + + float[] texCoords = { + 0, 0, // Bottom-left + 1, 0, // Bottom-right + 0, 1, // Top-left + 1, 1, // Top-right + }; + + // OpenGL will fill the 3rd and 4th coordinates of the vec4 automatically with 0.0 and 1.0 respectively + vertexBuffer = GLUtils.createFloatBuffer(vertices); + texCoordsBuffer = GLUtils.createFloatBuffer(texCoords); + + vertexPosLoc = GLES20.glGetAttribLocation(program, "vertex_pos"); + assert vertexPosLoc != -1; + + texCoordsInLoc = GLES20.glGetAttribLocation(program, "tex_coords_in"); + assert texCoordsInLoc != -1; + + texLoc = GLES20.glGetUniformLocation(program, "tex"); + assert texLoc != -1; + + texMatrixLoc = GLES20.glGetUniformLocation(program, "tex_matrix"); + assert texMatrixLoc != -1; + + userMatrixLoc = GLES20.glGetUniformLocation(program, "user_matrix"); + assert userMatrixLoc != -1; + } + + @Override + public void draw(int textureId, float[] texMatrix) { + GLES20.glUseProgram(program); + GLUtils.checkGlError(); + + GLES20.glEnableVertexAttribArray(vertexPosLoc); + GLUtils.checkGlError(); + GLES20.glEnableVertexAttribArray(texCoordsInLoc); + GLUtils.checkGlError(); + + GLES20.glVertexAttribPointer(vertexPosLoc, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer); + GLUtils.checkGlError(); + GLES20.glVertexAttribPointer(texCoordsInLoc, 2, GLES20.GL_FLOAT, false, 0, texCoordsBuffer); + GLUtils.checkGlError(); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLUtils.checkGlError(); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); + GLUtils.checkGlError(); + GLES20.glUniform1i(texLoc, 0); + GLUtils.checkGlError(); + + GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, texMatrix, 0); + GLUtils.checkGlError(); + + GLES20.glUniformMatrix4fv(userMatrixLoc, 1, false, userMatrix, 0); + GLUtils.checkGlError(); + + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GLUtils.checkGlError(); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GLUtils.checkGlError(); + } + + @Override + public void release() { + GLES20.glDeleteProgram(program); + GLUtils.checkGlError(); + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/opengl/GLUtils.java b/server/src/main/java/com/genymobile/scrcpy/opengl/GLUtils.java new file mode 100644 index 00000000..72a3f400 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/opengl/GLUtils.java @@ -0,0 +1,124 @@ +package com.genymobile.scrcpy.opengl; + +import com.genymobile.scrcpy.BuildConfig; +import com.genymobile.scrcpy.util.Ln; + +import android.opengl.GLES20; +import android.opengl.GLU; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; + +public final class GLUtils { + + private static final boolean DEBUG = BuildConfig.DEBUG; + + private GLUtils() { + // not instantiable + } + + public static int createProgram(String vertexSource, String fragmentSource) { + int vertexShader = createShader(GLES20.GL_VERTEX_SHADER, vertexSource); + if (vertexShader == 0) { + return 0; + } + + int fragmentShader = createShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); + if (fragmentShader == 0) { + GLES20.glDeleteShader(vertexShader); + return 0; + } + + int program = GLES20.glCreateProgram(); + if (program == 0) { + GLES20.glDeleteShader(fragmentShader); + GLES20.glDeleteShader(vertexShader); + return 0; + } + + GLES20.glAttachShader(program, vertexShader); + checkGlError(); + GLES20.glAttachShader(program, fragmentShader); + checkGlError(); + GLES20.glLinkProgram(program); + checkGlError(); + + int[] linkStatus = new int[1]; + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); + if (linkStatus[0] == 0) { + Ln.e("Could not link program: " + GLES20.glGetProgramInfoLog(program)); + GLES20.glDeleteProgram(program); + GLES20.glDeleteShader(fragmentShader); + GLES20.glDeleteShader(vertexShader); + return 0; + } + + return program; + } + + public static int createShader(int type, String source) { + int shader = GLES20.glCreateShader(type); + if (shader == 0) { + Ln.e(getGlErrorMessage("Could not create shader")); + return 0; + } + + GLES20.glShaderSource(shader, source); + GLES20.glCompileShader(shader); + + int[] compileStatus = new int[1]; + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compileStatus, 0); + if (compileStatus[0] == 0) { + Ln.e("Could not compile " + getShaderTypeString(type) + ": " + GLES20.glGetShaderInfoLog(shader)); + GLES20.glDeleteShader(shader); + return 0; + } + + return shader; + } + + private static String getShaderTypeString(int type) { + switch (type) { + case GLES20.GL_VERTEX_SHADER: + return "vertex shader"; + case GLES20.GL_FRAGMENT_SHADER: + return "fragment shader"; + default: + return "shader"; + } + } + + /** + * Throws a runtime exception if {@link GLES20#glGetError()} returns an error (useful for debugging). + */ + public static void checkGlError() { + if (DEBUG) { + int error = GLES20.glGetError(); + if (error != GLES20.GL_NO_ERROR) { + throw new RuntimeException(toErrorString(error)); + } + } + } + + public static String getGlErrorMessage(String userError) { + int glError = GLES20.glGetError(); + if (glError == GLES20.GL_NO_ERROR) { + return userError; + } + + return userError + " (" + toErrorString(glError) + ")"; + } + + private static String toErrorString(int glError) { + String errorString = GLU.gluErrorString(glError); + return "glError 0x" + Integer.toHexString(glError) + " " + errorString; + } + + public static FloatBuffer createFloatBuffer(float[] values) { + FloatBuffer fb = ByteBuffer.allocateDirect(values.length * 4).order(ByteOrder.nativeOrder()).asFloatBuffer(); + fb.put(values); + fb.position(0); + return fb; + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLException.java b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLException.java new file mode 100644 index 00000000..cbc9539b --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLException.java @@ -0,0 +1,13 @@ +package com.genymobile.scrcpy.opengl; + +import java.io.IOException; + +public class OpenGLException extends IOException { + public OpenGLException(String message) { + super(message); + } + + public OpenGLException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLFilter.java b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLFilter.java new file mode 100644 index 00000000..6f27777e --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLFilter.java @@ -0,0 +1,21 @@ +package com.genymobile.scrcpy.opengl; + +public interface OpenGLFilter { + + /** + * Initialize the OpenGL filter (typically compile the shaders and create the program). + * + * @throws OpenGLException if an initialization error occurs + */ + void init() throws OpenGLException; + + /** + * Render a frame (call for each frame). + */ + void draw(int textureId, float[] texMatrix); + + /** + * Release resources. + */ + void release(); +} diff --git a/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java new file mode 100644 index 00000000..a3f9335c --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/opengl/OpenGLRunner.java @@ -0,0 +1,246 @@ +package com.genymobile.scrcpy.opengl; + +import com.genymobile.scrcpy.device.Size; + +import android.graphics.SurfaceTexture; +import android.opengl.EGL14; +import android.opengl.EGLConfig; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLExt; +import android.opengl.EGLSurface; +import android.opengl.GLES11Ext; +import android.opengl.GLES20; +import android.os.Handler; +import android.os.HandlerThread; +import android.view.Surface; + +import java.util.concurrent.Semaphore; + +public final class OpenGLRunner { + + private static HandlerThread handlerThread; + private static Handler handler; + private static boolean quit; + + private EGLDisplay eglDisplay; + private EGLContext eglContext; + private EGLSurface eglSurface; + + private final OpenGLFilter filter; + + private SurfaceTexture surfaceTexture; + private Surface inputSurface; + private int textureId; + + private boolean stopped; + + public OpenGLRunner(OpenGLFilter filter) { + this.filter = filter; + } + + public static synchronized void initOnce() { + if (handlerThread == null) { + if (quit) { + throw new IllegalStateException("Could not init OpenGLRunner after it is quit"); + } + handlerThread = new HandlerThread("OpenGLRunner"); + handlerThread.start(); + handler = new Handler(handlerThread.getLooper()); + } + } + + public static void quit() { + HandlerThread thread; + synchronized (OpenGLRunner.class) { + thread = handlerThread; + quit = true; + } + if (thread != null) { + thread.quitSafely(); + } + } + + public static void join() throws InterruptedException { + HandlerThread thread; + synchronized (OpenGLRunner.class) { + thread = handlerThread; + } + if (thread != null) { + thread.join(); + } + } + + public Surface start(Size inputSize, Size outputSize, Surface outputSurface) throws OpenGLException { + initOnce(); + + // Simulate CompletableFuture, but working for all Android versions + final Semaphore sem = new Semaphore(0); + Throwable[] throwableRef = new Throwable[1]; + + // The whole OpenGL execution must be performed on a Handler, so that SurfaceTexture.setOnFrameAvailableListener() works correctly. + // See + handler.post(() -> { + try { + run(inputSize, outputSize, outputSurface); + } catch (Throwable throwable) { + throwableRef[0] = throwable; + } finally { + sem.release(); + } + }); + + try { + sem.acquire(); + } catch (InterruptedException e) { + // Behave as if this method call was synchronous + Thread.currentThread().interrupt(); + } + + Throwable throwable = throwableRef[0]; + if (throwable != null) { + if (throwable instanceof OpenGLException) { + throw (OpenGLException) throwable; + } + throw new OpenGLException("Asynchronous OpenGL runner init failed", throwable); + } + + // Synchronization is ok: inputSurface is written before sem.release() and read after sem.acquire() + return inputSurface; + } + + private void run(Size inputSize, Size outputSize, Surface outputSurface) throws OpenGLException { + eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY); + if (eglDisplay == EGL14.EGL_NO_DISPLAY) { + throw new OpenGLException("Unable to get EGL14 display"); + } + + int[] version = new int[2]; + if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { + throw new OpenGLException("Unable to initialize EGL14"); + } + + // @formatter:off + int[] attribList = { + EGL14.EGL_RED_SIZE, 8, + EGL14.EGL_GREEN_SIZE, 8, + EGL14.EGL_BLUE_SIZE, 8, + EGL14.EGL_ALPHA_SIZE, 8, + EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, + EGL14.EGL_NONE + }; + + EGLConfig[] configs = new EGLConfig[1]; + int[] numConfigs = new int[1]; + EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, configs.length, numConfigs, 0); + if (numConfigs[0] <= 0) { + EGL14.eglTerminate(eglDisplay); + throw new OpenGLException("Unable to find ES2 EGL config"); + } + EGLConfig eglConfig = configs[0]; + + // @formatter:off + int[] contextAttribList = { + EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, + EGL14.EGL_NONE + }; + eglContext = EGL14.eglCreateContext(eglDisplay, eglConfig, EGL14.EGL_NO_CONTEXT, contextAttribList, 0); + if (eglContext == null) { + EGL14.eglTerminate(eglDisplay); + throw new OpenGLException("Failed to create EGL context"); + } + + int[] surfaceAttribList = { + EGL14.EGL_NONE + }; + eglSurface = EGL14.eglCreateWindowSurface(eglDisplay, eglConfig, outputSurface, surfaceAttribList, 0); + if (eglSurface == null) { + EGL14.eglDestroyContext(eglDisplay, eglContext); + EGL14.eglTerminate(eglDisplay); + throw new OpenGLException("Failed to create EGL window surface"); + } + + if (!EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) { + EGL14.eglDestroySurface(eglDisplay, eglSurface); + EGL14.eglDestroyContext(eglDisplay, eglContext); + EGL14.eglTerminate(eglDisplay); + throw new OpenGLException("Failed to make EGL context current"); + } + + int[] textures = new int[1]; + GLES20.glGenTextures(1, textures, 0); + GLUtils.checkGlError(); + textureId = textures[0]; + + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLUtils.checkGlError(); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLUtils.checkGlError(); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLUtils.checkGlError(); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLUtils.checkGlError(); + + surfaceTexture = new SurfaceTexture(textureId); + surfaceTexture.setDefaultBufferSize(inputSize.getWidth(), inputSize.getHeight()); + inputSurface = new Surface(surfaceTexture); + + filter.init(); + + surfaceTexture.setOnFrameAvailableListener(surfaceTexture -> { + if (stopped) { + // Make sure to never render after resources have been released + return; + } + + render(outputSize); + }, handler); + } + + private void render(Size outputSize) { + GLES20.glViewport(0, 0, outputSize.getWidth(), outputSize.getHeight()); + GLUtils.checkGlError(); + + surfaceTexture.updateTexImage(); + float[] matrix = new float[16]; + surfaceTexture.getTransformMatrix(matrix); + + filter.draw(textureId, matrix); + + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTexture.getTimestamp()); + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + } + + public void stopAndRelease() { + final Semaphore sem = new Semaphore(0); + + handler.post(() -> { + stopped = true; + surfaceTexture.setOnFrameAvailableListener(null, handler); + + filter.release(); + + int[] textures = {textureId}; + GLES20.glDeleteTextures(1, textures, 0); + GLUtils.checkGlError(); + + EGL14.eglDestroySurface(eglDisplay, eglSurface); + EGL14.eglDestroyContext(eglDisplay, eglContext); + EGL14.eglTerminate(eglDisplay); + eglDisplay = EGL14.EGL_NO_DISPLAY; + eglContext = EGL14.EGL_NO_CONTEXT; + eglSurface = EGL14.EGL_NO_SURFACE; + surfaceTexture.release(); + inputSurface.release(); + + sem.release(); + }); + + try { + sem.acquire(); + } catch (InterruptedException e) { + // Behave as if this method call was synchronous + Thread.currentThread().interrupt(); + } + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/util/AffineMatrix.java b/server/src/main/java/com/genymobile/scrcpy/util/AffineMatrix.java new file mode 100644 index 00000000..0db74af6 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/util/AffineMatrix.java @@ -0,0 +1,368 @@ +package com.genymobile.scrcpy.util; + +import com.genymobile.scrcpy.device.Point; +import com.genymobile.scrcpy.device.Size; + +/** + * Represents a 2D affine transform (a 3x3 matrix): + * + * 
+ *     / a c e \
+ *     | b d f |
+ *     \ 0 0 1 /
+ *
+ *

+ * Or, a 4x4 matrix if we add a z axis: + * + * 

+ *     / a c 0 e \
+ *     | b d 0 f |
+ *     | 0 0 1 0 |
+ *     \ 0 0 0 1 /
+ *
+ */ +public class AffineMatrix { + + private final double a, b, c, d, e, f; + + /** + * The identity matrix. + */ + public static final AffineMatrix IDENTITY = new AffineMatrix(1, 0, 0, 1, 0, 0); + + /** + * Create a new matrix: + * + * 
+     *     / a c e \
+     *     | b d f |
+     *     \ 0 0 1 /
+     *
+ */ + public AffineMatrix(double a, double b, double c, double d, double e, double f) { + this.a = a; + this.b = b; + this.c = c; + this.d = d; + this.e = e; + this.f = f; + } + + @Override + public String toString() { + return "[" + a + ", " + c + ", " + e + "; " + b + ", " + d + ", " + f + "]"; + } + + /** + * Return a matrix which converts from Normalized Device Coordinates to pixels. + * + * @param size the target size + * @return the transform matrix + */ + public static AffineMatrix ndcFromPixels(Size size) { + double w = size.getWidth(); + double h = size.getHeight(); + return new AffineMatrix(1 / w, 0, 0, -1 / h, 0, 1); + } + + /** + * Return a matrix which converts from pixels to Normalized Device Coordinates. + * + * @param size the source size + * @return the transform matrix + */ + public static AffineMatrix ndcToPixels(Size size) { + double w = size.getWidth(); + double h = size.getHeight(); + return new AffineMatrix(w, 0, 0, -h, 0, h); + } + + /** + * Apply the transform to a point ({@code this} should be a matrix converted to pixels coordinates via {@link #ndcToPixels(Size)}). + * + * @param point the source point + * @return the converted point + */ + public Point apply(Point point) { + int x = point.getX(); + int y = point.getY(); + int xx = (int) (a * x + c * y + e); + int yy = (int) (b * x + d * y + f); + return new Point(xx, yy); + } + + /** + * Compute this * rhs. + * + * @param rhs the matrix to multiply + * @return the product + */ + public AffineMatrix multiply(AffineMatrix rhs) { + if (rhs == null) { + // For convenience + return this; + } + + double aa = this.a * rhs.a + this.c * rhs.b; + double bb = this.b * rhs.a + this.d * rhs.b; + double cc = this.a * rhs.c + this.c * rhs.d; + double dd = this.b * rhs.c + this.d * rhs.d; + double ee = this.a * rhs.e + this.c * rhs.f + this.e; + double ff = this.b * rhs.e + this.d * rhs.f + this.f; + return new AffineMatrix(aa, bb, cc, dd, ee, ff); + } + + /** + * Multiply all matrices from left to right, ignoring any {@code null} matrix (for convenience). + * + * @param matrices the matrices + * @return the product + */ + public static AffineMatrix multiplyAll(AffineMatrix... matrices) { + AffineMatrix result = null; + for (AffineMatrix matrix : matrices) { + if (result == null) { + result = matrix; + } else { + result = result.multiply(matrix); + } + } + return result; + } + + /** + * Invert the matrix. + * + * @return the inverse matrix (or {@code null} if not invertible). + */ + public AffineMatrix invert() { + // The 3x3 matrix M can be decomposed into M = M1 * M2: + // M1 M2 + // / 1 0 e \ / a c 0 \ + // | 0 1 f | * | b d 0 | + // \ 0 0 1 / \ 0 0 1 / + // + // The inverse of an invertible 2x2 matrix is given by this formula: + // + // / A B \⁻¹ 1 / D -B \ + // \ C D / = ----- \ -C A / + // AD-BC + // + // Let B=c and C=b (to apply the general formula with the same letters). + // + // M⁻¹ = (M1 * M2)⁻¹ = M2⁻¹ * M1⁻¹ + // + // M2⁻¹ M1⁻¹ + // /----------------\ + // 1 / d -B 0 \ / 1 0 -e \ + // = ----- | -C a 0 | * | 0 1 -f | + // ad-BC \ 0 0 1 / \ 0 0 1 / + // + // With the original letters: + // + // 1 / d -c 0 \ / 1 0 -e \ + // M⁻¹ = ----- | -b a 0 | * | 0 1 -f | + // ad-cb \ 0 0 1 / \ 0 0 1 / + // + // 1 / d -c cf-de \ + // = ----- | -b a be-af | + // ad-cb \ 0 0 1 / + + double det = a * d - c * b; + if (det == 0) { + // Not invertible + return null; + } + + double aa = d / det; + double bb = -b / det; + double cc = -c / det; + double dd = a / det; + double ee = (c * f - d * e) / det; + double ff = (b * e - a * f) / det; + + return new AffineMatrix(aa, bb, cc, dd, ee, ff); + } + + /** + * Return this transform applied from the center (0.5, 0.5). + * + * @return the resulting matrix + */ + public AffineMatrix fromCenter() { + return translate(0.5, 0.5).multiply(this).multiply(translate(-0.5, -0.5)); + } + + /** + * Return this transform with the specified aspect ratio. + * + * @param ar the aspect ratio + * @return the resulting matrix + */ + public AffineMatrix withAspectRatio(double ar) { + return scale(1 / ar, 1).multiply(this).multiply(scale(ar, 1)); + } + + /** + * Return this transform with the specified aspect ratio. + * + * @param size the size describing the aspect ratio + * @return the transform + */ + public AffineMatrix withAspectRatio(Size size) { + double ar = (double) size.getWidth() / size.getHeight(); + return withAspectRatio(ar); + } + + /** + * Return a translation matrix. + * + * @param x the horizontal translation + * @param y the vertical translation + * @return the matrix + */ + public static AffineMatrix translate(double x, double y) { + return new AffineMatrix(1, 0, 0, 1, x, y); + } + + /** + * Return a scaling matrix. + * + * @param x the horizontal scaling + * @param y the vertical scaling + * @return the matrix + */ + public static AffineMatrix scale(double x, double y) { + return new AffineMatrix(x, 0, 0, y, 0, 0); + } + + /** + * Return a scaling matrix. + * + * @param from the source size + * @param to the destination size + * @return the matrix + */ + public static AffineMatrix scale(Size from, Size to) { + double scaleX = (double) to.getWidth() / from.getWidth(); + double scaleY = (double) to.getHeight() / from.getHeight(); + return scale(scaleX, scaleY); + } + + /** + * Return a matrix applying a "reframing" (cropping a rectangle). + *

+ * (x, y) is the bottom-left corner, (w, h) is the size of the rectangle. + * + * @param x horizontal coordinate (increasing to the right) + * @param y vertical coordinate (increasing upwards) + * @param w width + * @param h height + * @return the matrix + */ + public static AffineMatrix reframe(double x, double y, double w, double h) { + if (w == 0 || h == 0) { + throw new IllegalArgumentException("Cannot reframe to an empty area: " + w + "x" + h); + } + return scale(1 / w, 1 / h).multiply(translate(-x, -y)); + } + + /** + * Return an orthogonal rotation matrix. + * + * @param ccwRotation the counter-clockwise rotation + * @return the matrix + */ + public static AffineMatrix rotateOrtho(int ccwRotation) { + switch (ccwRotation) { + case 0: + return IDENTITY; + case 1: + // 90° counter-clockwise + return new AffineMatrix(0, 1, -1, 0, 1, 0); + case 2: + // 180° + return new AffineMatrix(-1, 0, 0, -1, 1, 1); + case 3: + // 90° clockwise + return new AffineMatrix(0, -1, 1, 0, 0, 1); + default: + throw new IllegalArgumentException("Invalid rotation: " + ccwRotation); + } + } + + /** + * Return an horizontal flip matrix. + * + * @return the matrix + */ + public static AffineMatrix hflip() { + return new AffineMatrix(-1, 0, 0, 1, 1, 0); + } + + /** + * Return a vertical flip matrix. + * + * @return the matrix + */ + public static AffineMatrix vflip() { + return new AffineMatrix(1, 0, 0, -1, 0, 1); + } + + /** + * Return a rotation matrix. + * + * @param ccwDegrees the angle, in degrees (counter-clockwise) + * @return the matrix + */ + public static AffineMatrix rotate(double ccwDegrees) { + double radians = Math.toRadians(ccwDegrees); + double cos = Math.cos(radians); + double sin = Math.sin(radians); + return new AffineMatrix(cos, sin, -sin, cos, 0, 0); + } + + /** + * Export this affine transform to a 4x4 column-major order matrix. + * + * @param matrix output 4x4 matrix + */ + public void to4x4(float[] matrix) { + // matrix is a 4x4 matrix in column-major order + + // Column 0 + matrix[0] = (float) a; + matrix[1] = (float) b; + matrix[2] = 0; + matrix[3] = 0; + + // Column 1 + matrix[4] = (float) c; + matrix[5] = (float) d; + matrix[6] = 0; + matrix[7] = 0; + + // Column 2 + matrix[8] = 0; + matrix[9] = 0; + matrix[10] = 1; + matrix[11] = 0; + + // Column 3 + matrix[12] = (float) e; + matrix[13] = (float) f; + matrix[14] = 0; + matrix[15] = 1; + } + + /** + * Export this affine transform to a 4x4 column-major order matrix. + * + * @return 4x4 matrix + */ + public float[] to4x4() { + float[] matrix = new float[16]; + to4x4(matrix); + return matrix; + } +}