Always request fullscreen and maximized state

The window state may be changed by the window manager without scrcpy
being aware of it (for example the fullscreen mode on macOS).

To avoid problems, request the window state to SDL when needed.

app/src/input_manager.c

@@ -7,6 +7,33 @@
 #include "util/lock.h"
 #include "util/log.h"
 
+// Convert window coordinates (as provided by SDL_GetMouseState() to renderer
+// coordinates (as provided in SDL mouse events)
+//
+// See my question:
+// <https://stackoverflow.com/questions/49111054/how-to-get-mouse-position-on-mouse-wheel-event>
+static void
+convert_to_renderer_coordinates(SDL_Renderer *renderer, int *x, int *y) {
+    SDL_Rect viewport;
+    float scale_x, scale_y;
+    SDL_RenderGetViewport(renderer, &viewport);
+    SDL_RenderGetScale(renderer, &scale_x, &scale_y);
+    *x = (int) (*x / scale_x) - viewport.x;
+    *y = (int) (*y / scale_y) - viewport.y;
+}
+
+static struct point
+get_mouse_point(struct screen *screen) {
+    int x;
+    int y;
+    SDL_GetMouseState(&x, &y);
+    convert_to_renderer_coordinates(screen->renderer, &x, &y);
+    return (struct point) {
+        .x = x,
+        .y = y,
+    };
+}
+
 static const int ACTION_DOWN = 1;
 static const int ACTION_UP = 1 << 1;
 
@@ -460,17 +487,11 @@ convert_touch(const SDL_TouchFingerEvent *from, struct screen *screen,
 
     to->inject_touch_event.pointer_id = from->fingerId;
     to->inject_touch_event.position.screen_size = screen->frame_size;
-
-    int ww;
-    int wh;
-    SDL_GL_GetDrawableSize(screen->window, &ww, &wh);
-
     // SDL touch event coordinates are normalized in the range [0; 1]
-    int32_t x = from->x * ww;
-    int32_t y = from->y * wh;
+    float x = from->x * screen->content_size.width;
+    float y = from->y * screen->content_size.height;
     to->inject_touch_event.position.point =
         screen_convert_to_frame_coords(screen, x, y);
-
     to->inject_touch_event.pressure = from->pressure;
     to->inject_touch_event.buttons = 0;
     return true;
@@ -487,6 +508,13 @@ input_manager_process_touch(struct input_manager *im,
     }
 }
 
+static bool
+is_outside_device_screen(struct input_manager *im, int x, int y)
+{
+    return x < 0 || x >= im->screen->content_size.width ||
+           y < 0 || y >= im->screen->content_size.height;
+}
+
 static bool
 convert_mouse_button(const SDL_MouseButtonEvent *from, struct screen *screen,
                      struct control_msg *to) {
@@ -524,15 +552,10 @@ input_manager_process_mouse_button(struct input_manager *im,
             action_home(im->controller, ACTION_DOWN | ACTION_UP);
             return;
         }
-
         // double-click on black borders resize to fit the device screen
         if (event->button == SDL_BUTTON_LEFT && event->clicks == 2) {
-            int32_t x = event->x;
-            int32_t y = event->y;
-            screen_hidpi_scale_coords(im->screen, &x, &y);
-            SDL_Rect *r = &im->screen->rect;
-            bool outside = x < r->x || x >= r->x + r->w
-                        || y < r->y || y >= r->y + r->h;
+            bool outside =
+                is_outside_device_screen(im, event->x, event->y);
             if (outside) {
                 screen_resize_to_fit(im->screen);
                 return;
@@ -556,15 +579,9 @@ input_manager_process_mouse_button(struct input_manager *im,
 static bool
 convert_mouse_wheel(const SDL_MouseWheelEvent *from, struct screen *screen,
                     struct control_msg *to) {
-
-    // mouse_x and mouse_y are expressed in pixels relative to the window
-    int mouse_x;
-    int mouse_y;
-    SDL_GetMouseState(&mouse_x, &mouse_y);
-
     struct position position = {
         .screen_size = screen->frame_size,
-        .point = screen_convert_to_frame_coords(screen, mouse_x, mouse_y),
+        .point = get_mouse_point(screen),
     };
 
     to->type = CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT;

app/src/screen.c

@@ -15,6 +15,18 @@
 
 #define DISPLAY_MARGINS 96
 
+static bool
+is_maximized(const struct screen *screen) {
+    uint32_t flags = SDL_GetWindowFlags(screen->window);
+    return !!(flags & SDL_WINDOW_MAXIMIZED);
+}
+
+static bool
+is_fullscreen(const struct screen *screen) {
+    uint32_t flags = SDL_GetWindowFlags(screen->window);
+    return !!(flags & (SDL_WINDOW_FULLSCREEN | SDL_WINDOW_FULLSCREEN_DESKTOP));
+}
+
 static inline struct size
 get_rotated_size(struct size size, int rotation) {
     struct size rotated_size;
@@ -44,7 +56,7 @@ get_window_size(SDL_Window *window) {
 // get the windowed window size
 static struct size
 get_windowed_window_size(const struct screen *screen) {
-    if (screen->fullscreen || screen->maximized) {
+    if (is_fullscreen(screen) || is_maximized(screen)) {
         return screen->windowed_window_size;
     }
     return get_window_size(screen->window);
@@ -53,7 +65,7 @@ get_windowed_window_size(const struct screen *screen) {
 // apply the windowed window size if fullscreen and maximized are disabled
 static void
 apply_windowed_size(struct screen *screen) {
-    if (!screen->fullscreen && !screen->maximized) {
+    if (!is_fullscreen(screen) && !is_maximized(screen)) {
         SDL_SetWindowSize(screen->window, screen->windowed_window_size.width,
                                           screen->windowed_window_size.height);
     }
@@ -87,16 +99,6 @@ get_preferred_display_bounds(struct size *bounds) {
     return true;
 }
 
-static bool
-is_optimal_size(struct size current_size, struct size content_size) {
-    // The size is optimal if we can recompute one dimension of the current
-    // size from the other
-    return current_size.height == current_size.width * content_size.height
-                                                     / content_size.width
-        || current_size.width == current_size.height * content_size.width
-                                                     / content_size.height;
-}
-
 // return the optimal size of the window, with the following constraints:
 //  - it attempts to keep at least one dimension of the current_size (i.e. it
 //    crops the black borders)
@@ -109,36 +111,40 @@ get_optimal_size(struct size current_size, struct size content_size) {
         return current_size;
     }
 
-    struct size window_size;
-
     struct size display_size;
+    // 32 bits because we need to multiply two 16 bits values
+    uint32_t w;
+    uint32_t h;
+
     if (!get_preferred_display_bounds(&display_size)) {
         // could not get display bounds, do not constraint the size
-        window_size.width = current_size.width;
-        window_size.height = current_size.height;
+        w = current_size.width;
+        h = current_size.height;
     } else {
-        window_size.width = MIN(current_size.width, display_size.width);
-        window_size.height = MIN(current_size.height, display_size.height);
+        w = MIN(current_size.width, display_size.width);
+        h = MIN(current_size.height, display_size.height);
     }
 
-    if (is_optimal_size(window_size, content_size)) {
-        return window_size;
+    if (h == w * content_size.height / content_size.width
+     || w == h * content_size.width / content_size.height) {
+        // The size is already optimal, if we ignore rounding errors due to
+        // integer window dimensions
+        return (struct size) {w, h};
     }
 
-    bool keep_width = content_size.width * window_size.height
-                    > content_size.height * window_size.width;
+    bool keep_width = content_size.width * h > content_size.height * w;
     if (keep_width) {
         // remove black borders on top and bottom
-        window_size.height = content_size.height * window_size.width
-                           / content_size.width;
+        h = content_size.height * w / content_size.width;
     } else {
         // remove black borders on left and right (or none at all if it already
         // fits)
-        window_size.width = content_size.width * window_size.height
-                          / content_size.height;
+        w = content_size.width * h / content_size.height;
     }
 
-    return window_size;
+    // w and h must fit into 16 bits
+    assert(w < 0x10000 && h < 0x10000);
+    return (struct size) {w, h};
 }
 
 // same as get_optimal_size(), but read the current size from the window
@@ -175,45 +181,6 @@ get_initial_optimal_size(struct size content_size, uint16_t req_width,
     return window_size;
 }
 
-static void
-update_content_rect(struct screen *screen) {
-    int dw;
-    int dh;
-    SDL_GL_GetDrawableSize(screen->window, &dw, &dh);
-
-    struct size content_size = screen->content_size;
-    // The drawable size is the window size * the HiDPI scale
-    struct size drawable_size = {dw, dh};
-
-    SDL_Rect *rect = &screen->rect;
-
-    if (is_optimal_size(drawable_size, content_size)) {
-        rect->x = 0;
-        rect->y = 0;
-        rect->w = drawable_size.width;
-        rect->h = drawable_size.height;
-        return;
-    }
-
-    bool keep_width = content_size.width * drawable_size.height
-                    > content_size.height * drawable_size.width;
-    if (keep_width) {
-        rect->x = 0;
-        rect->w = drawable_size.width;
-        rect->h = drawable_size.width * content_size.height
-                                      / content_size.width;
-        rect->y = (drawable_size.height - rect->h) / 2;
-    } else {
-        rect->y = 0;
-        rect->h = drawable_size.height;
-        rect->w = drawable_size.height * content_size.width
-                                       / content_size.height;
-        rect->x = (drawable_size.width - rect->w) / 2;
-    }
-
-    LOGI("== update_content_rect: %d,%d %dx%d", rect->x, rect->y, rect->w, rect->h);
-}
-
 void
 screen_init(struct screen *screen) {
     *screen = (struct screen) SCREEN_INITIALIZER;
@@ -303,6 +270,13 @@ screen_init_rendering(struct screen *screen, const char *window_title,
     const char *renderer_name = r ? NULL : renderer_info.name;
     LOGI("Renderer: %s", renderer_name ? renderer_name : "(unknown)");
 
+    if (SDL_RenderSetLogicalSize(screen->renderer, content_size.width,
+                                 content_size.height)) {
+        LOGE("Could not set renderer logical size: %s", SDL_GetError());
+        screen_destroy(screen);
+        return false;
+    }
+
     // starts with "opengl"
     screen->use_opengl = renderer_name && !strncmp(renderer_name, "opengl", 6);
     if (screen->use_opengl) {
@@ -346,9 +320,6 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         return false;
     }
 
-    LOGI("# init");
-    update_content_rect(screen);
-
     screen->windowed_window_size = window_size;
 
     return true;
@@ -383,6 +354,13 @@ screen_set_rotation(struct screen *screen, unsigned rotation) {
     struct size new_content_size =
         get_rotated_size(screen->frame_size, rotation);
 
+    if (SDL_RenderSetLogicalSize(screen->renderer,
+                                 new_content_size.width,
+                                 new_content_size.height)) {
+        LOGE("Could not set renderer logical size: %s", SDL_GetError());
+        return;
+    }
+
     struct size windowed_size = get_windowed_window_size(screen);
     struct size target_size = {
         .width = (uint32_t) windowed_size.width * new_content_size.width
@@ -397,7 +375,6 @@ screen_set_rotation(struct screen *screen, unsigned rotation) {
     screen->rotation = rotation;
     LOGI("Display rotation set to %u", rotation);
 
-    update_content_rect(screen);
     screen_render(screen);
 }
 
@@ -406,11 +383,18 @@ static bool
 prepare_for_frame(struct screen *screen, struct size new_frame_size) {
     if (screen->frame_size.width != new_frame_size.width
             || screen->frame_size.height != new_frame_size.height) {
+        struct size new_content_size =
+            get_rotated_size(new_frame_size, screen->rotation);
+        if (SDL_RenderSetLogicalSize(screen->renderer,
+                                     new_content_size.width,
+                                     new_content_size.height)) {
+            LOGE("Could not set renderer logical size: %s", SDL_GetError());
+            return false;
+        }
+
         // frame dimension changed, destroy texture
         SDL_DestroyTexture(screen->texture);
 
-        struct size new_content_size =
-            get_rotated_size(new_frame_size, screen->rotation);
         struct size content_size = screen->content_size;
         struct size windowed_size = get_windowed_window_size(screen);
         struct size target_size = {
@@ -424,8 +408,6 @@ prepare_for_frame(struct screen *screen, struct size new_frame_size) {
 
         screen->frame_size = new_frame_size;
         screen->content_size = new_content_size;
-        LOGI("# prepare_for_frame");
-        update_content_rect(screen);
 
         LOGI("New texture: %" PRIu16 "x%" PRIu16,
                      screen->frame_size.width, screen->frame_size.height);
@@ -475,7 +457,7 @@ void
 screen_render(struct screen *screen) {
     SDL_RenderClear(screen->renderer);
     if (screen->rotation == 0) {
-        SDL_RenderCopy(screen->renderer, screen->texture, NULL, &screen->rect);
+        SDL_RenderCopy(screen->renderer, screen->texture, NULL, NULL);
     } else {
         // rotation in RenderCopyEx() is clockwise, while screen->rotation is
         // counterclockwise (to be consistent with --lock-video-orientation)
@@ -485,14 +467,12 @@ screen_render(struct screen *screen) {
         SDL_Rect *dstrect = NULL;
         SDL_Rect rect;
         if (screen->rotation & 1) {
-            rect.x = screen->rect.x + (screen->rect.w - screen->rect.h) / 2;
-            rect.y = screen->rect.y + (screen->rect.h - screen->rect.w) / 2;
-            rect.w = screen->rect.h;
-            rect.h = screen->rect.w;
+            struct size size = screen->content_size;
+            rect.x = (size.width - size.height) / 2;
+            rect.y = (size.height - size.width) / 2;
+            rect.w = size.height;
+            rect.h = size.width;
             dstrect = &rect;
-        } else {
-            assert(screen->rotation == 2);
-            dstrect = &screen->rect;
         }
 
         SDL_RenderCopyEx(screen->renderer, screen->texture, NULL, dstrect,
@@ -503,31 +483,27 @@ screen_render(struct screen *screen) {
 
 void
 screen_switch_fullscreen(struct screen *screen) {
-    uint32_t new_mode = screen->fullscreen ? 0 : SDL_WINDOW_FULLSCREEN_DESKTOP;
+    bool was_fullscreen = is_fullscreen(screen);
+    uint32_t new_mode = was_fullscreen ? 0 : SDL_WINDOW_FULLSCREEN_DESKTOP;
     if (SDL_SetWindowFullscreen(screen->window, new_mode)) {
         LOGW("Could not switch fullscreen mode: %s", SDL_GetError());
         return;
     }
 
-    screen->fullscreen = !screen->fullscreen;
     apply_windowed_size(screen);
 
-    LOGD("Switched to %s mode", screen->fullscreen ? "fullscreen" : "windowed");
-    LOGI("# screen_switch_fullscreen");
-    update_content_rect(screen);
+    LOGD("Switched to %s mode", was_fullscreen ? "windowed" : "fullscreen");
     screen_render(screen);
 }
 
 void
 screen_resize_to_fit(struct screen *screen) {
-    LOGI("resize_to_fit (fullscreen=%d)", screen->fullscreen);
-    if (screen->fullscreen) {
+    if (is_fullscreen(screen)) {
         return;
     }
 
-    if (screen->maximized) {
+    if (is_maximized(screen)) {
         SDL_RestoreWindow(screen->window);
-        screen->maximized = false;
     }
 
     struct size optimal_size =
@@ -539,13 +515,12 @@ screen_resize_to_fit(struct screen *screen) {
 
 void
 screen_resize_to_pixel_perfect(struct screen *screen) {
-    if (screen->fullscreen) {
+    if (is_fullscreen(screen)) {
         return;
     }
 
-    if (screen->maximized) {
+    if (is_maximized(screen)) {
         SDL_RestoreWindow(screen->window);
-        screen->maximized = false;
     }
 
     struct size content_size = screen->content_size;
@@ -559,12 +534,10 @@ screen_handle_window_event(struct screen *screen,
                            const SDL_WindowEvent *event) {
     switch (event->event) {
         case SDL_WINDOWEVENT_EXPOSED:
-            LOGI("SDL_WINDOW_EVENT_EXPOSED");
-            update_content_rect(screen);
             screen_render(screen);
             break;
         case SDL_WINDOWEVENT_SIZE_CHANGED:
-            if (!screen->fullscreen && !screen->maximized) {
+            if (!is_fullscreen(screen) && !is_maximized(screen)) {
                 // Backup the previous size: if we receive the MAXIMIZED event,
                 // then the new size must be ignored (it's the maximized size).
                 // We could not rely on the window flags due to race conditions
@@ -576,8 +549,6 @@ screen_handle_window_event(struct screen *screen,
                 // window is maximized or fullscreen is enabled.
                 screen->windowed_window_size = get_window_size(screen->window);
             }
-            LOGI("SDL_WINDOW_EVENT_SIZE_CHANGED");
-            update_content_rect(screen);
             screen_render(screen);
             break;
         case SDL_WINDOWEVENT_MAXIMIZED:
@@ -591,10 +562,8 @@ screen_handle_window_event(struct screen *screen,
             screen->windowed_window_size_backup.width = 0;
             screen->windowed_window_size_backup.height = 0;
 #endif
-            screen->maximized = true;
             break;
         case SDL_WINDOWEVENT_RESTORED:
-            screen->maximized = false;
             apply_windowed_size(screen);
             break;
     }
@@ -605,23 +574,8 @@ screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y) {
     unsigned rotation = screen->rotation;
     assert(rotation < 4);
 
-    int old_x = x;
-    int old_y = y;
-
     int32_t w = screen->content_size.width;
     int32_t h = screen->content_size.height;
-
-    screen_hidpi_scale_coords(screen, &x, &y);
-
-    x = (int64_t) (x - screen->rect.x) * w / screen->rect.w;
-    y = (int64_t) (y - screen->rect.y) * h / screen->rect.h;
-
-    LOGI("content=%dx%d, rect={%d, %d, %dx%d}: (%d, %d) -> (%d, %d)",
-         (int) w, (int) h,
-         screen->rect.x, screen->rect.y, screen->rect.w, screen->rect.h,
-         old_x, old_y, (int) x, (int) y);
-
-    // rotate
     struct point result;
     switch (rotation) {
         case 0:
@@ -644,15 +598,3 @@ screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y) {
     }
     return result;
 }
-
-void
-screen_hidpi_scale_coords(struct screen *screen, int32_t *x, int32_t *y) {
-    // take the HiDPI scaling (dw/ww and dh/wh) into account
-    int ww, wh, dw, dh;
-    SDL_GetWindowSize(screen->window, &ww, &wh);
-    SDL_GL_GetDrawableSize(screen->window, &dw, &dh);
-
-    // scale for HiDPI (64 bits for intermediate multiplications)
-    *x = (int64_t) *x * dw / ww;
-    *y = (int64_t) *y * dh / wh;
-}

app/src/screen.h

@@ -28,11 +28,7 @@ struct screen {
     struct size windowed_window_size_backup;
     // client rotation: 0, 1, 2 or 3 (x90 degrees counterclockwise)
     unsigned rotation;
-    // rectangle of the content (excluding black borders)
-    struct SDL_Rect rect;
     bool has_frame;
-    bool fullscreen;
-    bool maximized;
     bool no_window;
     bool mipmaps;
 };
@@ -60,15 +56,7 @@ struct screen {
         .height = 0, \
     }, \
     .rotation = 0, \
-    .rect = { \
-        .x = 0, \
-        .y = 0, \
-        .w = 0, \
-        .h = 0, \
-    }, \
     .has_frame = false, \
-    .fullscreen = false, \
-    .maximized = false, \
     .no_window = false, \
     .mipmaps = false, \
 }
@@ -127,11 +115,4 @@ screen_handle_window_event(struct screen *screen, const SDL_WindowEvent *event);
 struct point
 screen_convert_to_frame_coords(struct screen *screen, int32_t x, int32_t y);
 
-// Convert coordinates from window to drawable.
-// Events are expressed in window coordinates, but content is expressed in
-// drawable coordinates. They are the same if HiDPI scaling is 1, but differ
-// otherwise.
-void
-screen_hidpi_scale_coords(struct screen *screen, int32_t *x, int32_t *y);
-
 #endif

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -215,7 +215,7 @@ public class Controller {
 
         MotionEvent event = MotionEvent
                 .obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, 0, 1f, 1f, DEVICE_ID_VIRTUAL, 0,
-                        InputDevice.SOURCE_MOUSE, 0);
+                        InputDevice.SOURCE_TOUCHSCREEN, 0);
         return injectEvent(event);
     }