wip resizer

The video buffer held 3 frames:
 - the producer frame (for decoding)
 - the pending frame (to exchange between the producer and consumer)
 - the consumer frame (for rendering)

It worked well, but it prevented video buffers to be chained, because
the consumer frame of the first video buffer must be the same as the
producer frame of the second video buffer.

To solve this problem, make the decoder and the screen handle their own
frames, and keep only the pending frame in the video_buffer.

This paves the way to support asynchronous swscale.

app/meson.build

@@ -15,6 +15,7 @@ src = [
     'src/opengl.c',
     'src/receiver.c',
     'src/recorder.c',
+    'src/resizer.c',
     'src/scrcpy.c',
     'src/screen.c',
     'src/server.c',

app/scrcpy.1

@@ -167,6 +167,13 @@ Set the initial display rotation. Possibles values are 0, 1, 2 and 3. Each incre
 .BI "\-s, \-\-serial " number
 The device serial number. Mandatory only if several devices are connected to adb.
 
+.TP
+.BI "\-\-scale\-filter filter
+Supported filters are "none" and "trilinear".
+
+Trilinear filtering is only available if the renderer is OpenGL 3.0+ or OpenGL
+ES 2.0+.
+
 .TP
 .BI "\-\-shortcut\-mod " key[+...]][,...]
 Specify the modifiers to use for scrcpy shortcuts. Possible keys are "lctrl", "rctrl", "lalt", "ralt", "lsuper" and "rsuper".

app/src/cli.c

@@ -106,11 +106,6 @@ scrcpy_print_usage(const char *arg0) {
         "    --no-key-repeat\n"
         "        Do not forward repeated key events when a key is held down.\n"
         "\n"
-        "    --no-mipmaps\n"
-        "        If the renderer is OpenGL 3.0+ or OpenGL ES 2.0+, then\n"
-        "        mipmaps are automatically generated to improve downscaling\n"
-        "        quality. This option disables the generation of mipmaps.\n"
-        "\n"
         "    -p, --port port[:port]\n"
         "        Set the TCP port (range) used by the client to listen.\n"
         "        Default is " STR(DEFAULT_LOCAL_PORT_RANGE_FIRST) ":"
@@ -158,6 +153,11 @@ scrcpy_print_usage(const char *arg0) {
         "        The device serial number. Mandatory only if several devices\n"
         "        are connected to adb.\n"
         "\n"
+        "    --scale-filter filter\n"
+        "        Supported filters are \"none\" and \"trilinear\".\n"
+        "        Trilinear filtering is only available if the renderer is\n"
+        "        OpenGL 3.0+ or OpenGL ES 2.0+.\n"
+        "\n"
         "    --shortcut-mod key[+...]][,...]\n"
         "        Specify the modifiers to use for scrcpy shortcuts.\n"
         "        Possible keys are \"lctrl\", \"rctrl\", \"lalt\", \"ralt\",\n"
@@ -640,6 +640,21 @@ guess_record_format(const char *filename) {
     return 0;
 }
 
+static bool
+parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) {
+    if (!strcmp(optarg, "none")) {
+        *filter = SC_SCALE_FILTER_NONE;
+        return true;
+    }
+    if (!strcmp(optarg, "trilinear")) {
+        *filter = SC_SCALE_FILTER_TRILINEAR;
+        return true;
+    }
+    LOGE("Unsupported scale filter: %s "
+         "(expected \"none\" or \"trilinear\")", optarg);
+    return false;
+}
+
 #define OPT_RENDER_EXPIRED_FRAMES  1000
 #define OPT_WINDOW_TITLE           1001
 #define OPT_PUSH_TARGET            1002
@@ -666,6 +681,7 @@ guess_record_format(const char *filename) {
 #define OPT_FORWARD_ALL_CLICKS     1023
 #define OPT_LEGACY_PASTE           1024
 #define OPT_ENCODER_NAME           1025
+#define OPT_SCALE_FILTER           1026
 
 bool
 scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
@@ -702,6 +718,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
         {"render-expired-frames",  no_argument,       NULL,
                                                   OPT_RENDER_EXPIRED_FRAMES},
         {"rotation",               required_argument, NULL, OPT_ROTATION},
+        {"scale-filter",           required_argument, NULL, OPT_SCALE_FILTER},
         {"serial",                 required_argument, NULL, 's'},
         {"shortcut-mod",           required_argument, NULL, OPT_SHORTCUT_MOD},
         {"show-touches",           no_argument,       NULL, 't'},
@@ -856,7 +873,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
                 opts->render_driver = optarg;
                 break;
             case OPT_NO_MIPMAPS:
-                opts->mipmaps = false;
+                LOGW("Deprecated option --no-mipmaps. "
+                     "Use --scale-filter=none instead.");
+                opts->scale_filter = SC_SCALE_FILTER_NONE;
                 break;
             case OPT_NO_KEY_REPEAT:
                 opts->forward_key_repeat = false;
@@ -884,6 +903,11 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
             case OPT_LEGACY_PASTE:
                 opts->legacy_paste = true;
                 break;
+            case OPT_SCALE_FILTER:
+                if (!parse_scale_filter(optarg, &opts->scale_filter)) {
+                    return false;
+                }
+                break;
             default:
                 // getopt prints the error message on stderr
                 return false;

app/src/decoder.c

@@ -11,22 +11,6 @@
 #include "util/buffer_util.h"
 #include "util/log.h"
 
-// set the decoded frame as ready for rendering, and notify
-static void
-push_frame(struct decoder *decoder) {
-    bool previous_frame_skipped;
-    video_buffer_offer_decoded_frame(decoder->video_buffer,
-                                     &previous_frame_skipped);
-    if (previous_frame_skipped) {
-        // the previous EVENT_NEW_FRAME will consume this frame
-        return;
-    }
-    static SDL_Event new_frame_event = {
-        .type = EVENT_NEW_FRAME,
-    };
-    SDL_PushEvent(&new_frame_event);
-}
-
 void
 decoder_init(struct decoder *decoder, struct video_buffer *vb) {
     decoder->video_buffer = vb;
@@ -46,11 +30,20 @@ decoder_open(struct decoder *decoder, const AVCodec *codec) {
         return false;
     }
 
+    decoder->frame = av_frame_alloc();
+    if (!decoder->frame) {
+        LOGE("Could not create decoder frame");
+        avcodec_close(decoder->codec_ctx);
+        avcodec_free_context(&decoder->codec_ctx);
+        return false;
+    }
+
     return true;
 }
 
 void
 decoder_close(struct decoder *decoder) {
+    av_frame_free(&decoder->frame);
     avcodec_close(decoder->codec_ctx);
     avcodec_free_context(&decoder->codec_ctx);
 }
@@ -65,11 +58,11 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) {
         LOGE("Could not send video packet: %d", ret);
         return false;
     }
-    ret = avcodec_receive_frame(decoder->codec_ctx,
-                                decoder->video_buffer->decoding_frame);
+    ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame);
     if (!ret) {
         // a frame was received
-        push_frame(decoder);
+        video_buffer_producer_offer_frame(decoder->video_buffer,
+                                          &decoder->frame);
     } else if (ret != AVERROR(EAGAIN)) {
         LOGE("Could not receive video frame: %d", ret);
         return false;
@@ -77,7 +70,7 @@ decoder_push(struct decoder *decoder, const AVPacket *packet) {
 #else
     int got_picture;
     int len = avcodec_decode_video2(decoder->codec_ctx,
-                                    decoder->video_buffer->decoding_frame,
+                                    decoder->frame,
                                     &got_picture,
                                     packet);
     if (len < 0) {

app/src/decoder.h

@@ -10,7 +10,9 @@ struct video_buffer;
 
 struct decoder {
     struct video_buffer *video_buffer;
+
     AVCodecContext *codec_ctx;
+    AVFrame *frame;
 };
 
 void

app/src/input_manager.c

@@ -480,9 +480,7 @@ input_manager_process_key(struct input_manager *im,
                 return;
             case SDLK_i:
                 if (!shift && !repeat && down) {
-                    struct fps_counter *fps_counter =
-                        im->video_buffer->fps_counter;
-                    switch_fps_counter_state(fps_counter);
+                    switch_fps_counter_state(im->fps_counter);
                 }
                 return;
             case SDLK_n:

app/src/input_manager.h

@@ -16,6 +16,7 @@
 struct input_manager {
     struct controller *controller;
     struct video_buffer *video_buffer;
+    struct fps_counter *fps_counter;
     struct screen *screen;
 
     // SDL reports repeated events as a boolean, but Android expects the actual

app/src/resizer.c

@@ -0,0 +1,132 @@
+#include "resizer.h"
+
+#include <assert.h>
+
+#include "util/log.h"
+
+bool
+sc_resizer_init(struct sc_resizer *resizer, struct video_buffer *vb_in,
+                struct video_buffer *vb_out, struct size size) {
+    bool ok = sc_mutex_init(&resizer->mutex);
+    if (!ok) {
+        return false;
+    }
+
+    ok = sc_cond_init(&resizer->req_cond);
+    if (!ok) {
+        sc_mutex_destroy(&resizer->mutex);
+        return false;
+    }
+
+    resizer->resized_frame = av_frame_alloc();
+    if (!resizer->resized_frame) {
+        sc_cond_destroy(&resizer->req_cond);
+        sc_mutex_destroy(&resizer->mutex);
+        return false;
+    }
+
+    resizer->vb_in = vb_in;
+    resizer->vb_out = vb_out;
+    resizer->size = size;
+
+    resizer->input_frame = NULL;
+    resizer->has_request = false;
+    resizer->has_new_frame = false;
+    resizer->interrupted = false;
+
+    return true;
+}
+
+void
+sc_resizer_destroy(struct sc_resizer *resizer) {
+    av_frame_free(&resizer->resized_frame);
+    sc_cond_destroy(&resizer->req_cond);
+    sc_mutex_destroy(&resizer->mutex);
+}
+
+static bool
+sc_resizer_swscale(struct sc_resizer *resizer) {
+    assert(!resizer->resized_frame->buf[0]); // The frame must be "empty"
+
+    return false;
+}
+
+static int
+run_resizer(void *data) {
+    struct sc_resizer *resizer = data;
+
+    sc_mutex_lock(&resizer->mutex);
+    while (!resizer->interrupted) {
+        while (!resizer->interrupted && !resizer->has_request) {
+            sc_cond_wait(&resizer->req_cond, &resizer->mutex);
+        }
+
+        if (resizer->has_new_frame) {
+            resizer->input_frame =
+                video_buffer_consumer_take_frame(resizer->vb_in);
+
+            resizer->has_new_frame = false;
+        }
+
+        resizer->has_request = false;
+        sc_mutex_unlock(&resizer->mutex);
+
+        // Do the actual work without mutex
+        sc_resizer_swscale(resizer);
+
+        video_buffer_producer_offer_frame(resizer->vb_out,
+                                          &resizer->resized_frame);
+
+        sc_mutex_lock(&resizer->mutex);
+    }
+    sc_mutex_unlock(&resizer->mutex);
+
+    return 0;
+}
+
+bool
+sc_resizer_start(struct sc_resizer *resizer) {
+    LOGD("Starting resizer thread");
+
+    bool ok = sc_thread_create(&resizer->thread, run_resizer, "resizer",
+                               resizer);
+    if (!ok) {
+        LOGE("Could not start resizer thread");
+        return false;
+    }
+
+    return true;
+}
+
+void
+sc_resizer_stop(struct sc_resizer *resizer) {
+    sc_mutex_lock(&resizer->mutex);
+    resizer->interrupted = true;
+    sc_cond_signal(&resizer->req_cond);
+    sc_mutex_unlock(&resizer->mutex);
+
+    video_buffer_interrupt(resizer->vb_out);
+}
+
+void
+sc_resizer_join(struct sc_resizer *resizer) {
+    sc_thread_join(&resizer->thread, NULL);
+}
+
+void
+sc_resizer_process_new_frame(struct sc_resizer *resizer) {
+    sc_mutex_lock(&resizer->mutex);
+    resizer->has_request = true;
+    resizer->has_new_frame = true;
+    sc_cond_signal(&resizer->req_cond);
+    sc_mutex_unlock(&resizer->mutex);
+}
+
+void
+sc_resizer_process_new_size(struct sc_resizer *resizer, struct size size) {
+    sc_mutex_lock(&resizer->mutex);
+    resizer->size = size;
+    resizer->has_request = true;
+    sc_cond_signal(&resizer->req_cond);
+    sc_mutex_unlock(&resizer->mutex);
+}

app/src/resizer.h

@@ -0,0 +1,53 @@
+#ifndef SC_RESIZER
+#define SC_RESIZER
+
+#include "common.h"
+
+#include <stdbool.h>
+#include <libavformat/avformat.h>
+
+#include "coords.h"
+#include "util/thread.h"
+#include "video_buffer.h"
+
+struct sc_resizer {
+    struct video_buffer *vb_in;
+    struct video_buffer *vb_out;
+    struct size size;
+
+    // valid until the next call to video_buffer_consumer_take_frame(vb_in)
+    const AVFrame *input_frame;
+    AVFrame *resized_frame;
+
+    sc_thread thread;
+    sc_mutex mutex;
+    sc_cond req_cond;
+
+    bool has_request;
+    bool has_new_frame;
+    bool interrupted;
+};
+
+bool
+sc_resizer_init(struct sc_resizer *resizer, struct video_buffer *vb_in,
+                struct video_buffer *vb_out, struct size initial_size);
+
+void
+sc_resizer_destroy(struct sc_resizer *resizer);
+
+bool
+sc_resizer_start(struct sc_resizer *resizer);
+
+void
+sc_resizer_stop(struct sc_resizer *resizer);
+
+void
+sc_resizer_join(struct sc_resizer *resizer);
+
+void
+sc_resizer_process_new_frame(struct sc_resizer *resizer);
+
+void
+sc_resizer_process_new_size(struct sc_resizer *resizer, struct size size);
+
+#endif

app/src/scrcpy.c

@@ -42,6 +42,7 @@ static struct file_handler file_handler;
 static struct input_manager input_manager = {
     .controller = &controller,
     .video_buffer = &video_buffer,
+    .fps_counter = &fps_counter,
     .screen = &screen,
     .repeat = 0,
 
@@ -332,8 +333,7 @@ scrcpy(const struct scrcpy_options *options) {
         }
         fps_counter_initialized = true;
 
-        if (!video_buffer_init(&video_buffer, &fps_counter,
-                               options->render_expired_frames)) {
+        if (!video_buffer_init(&video_buffer, options->render_expired_frames)) {
             goto end;
         }
         video_buffer_initialized = true;
@@ -366,13 +366,6 @@ scrcpy(const struct scrcpy_options *options) {
 
     stream_init(&stream, server.video_socket, dec, rec);
 
-    // now we consumed the header values, the socket receives the video stream
-    // start the stream
-    if (!stream_start(&stream)) {
-        goto end;
-    }
-    stream_started = true;
-
     if (options->display) {
         if (options->control) {
             if (!controller_init(&controller, server.control_socket)) {
@@ -389,14 +382,21 @@ scrcpy(const struct scrcpy_options *options) {
         const char *window_title =
             options->window_title ? options->window_title : device_name;
 
-        screen_init(&screen, &video_buffer);
+        struct screen_params screen_params = {
+            .window_title = window_title,
+            .frame_size = frame_size,
+            .always_on_top = options->always_on_top,
+            .window_x = options->window_x,
+            .window_y = options->window_y,
+            .window_width = options->window_width,
+            .window_height = options->window_height,
+            .window_borderless = options->window_borderless,
+            .rotation = options->rotation,
+            .scale_filter = options->scale_filter,
+        };
 
-        if (!screen_init_rendering(&screen, window_title, frame_size,
-                                   options->always_on_top, options->window_x,
-                                   options->window_y, options->window_width,
-                                   options->window_height,
-                                   options->window_borderless,
-                                   options->rotation, options->mipmaps)) {
+        if (!screen_init(&screen, &video_buffer, &fps_counter,
+                         &screen_params)) {
             goto end;
         }
 
@@ -415,6 +415,13 @@ scrcpy(const struct scrcpy_options *options) {
         }
     }
 
+    // now we consumed the header values, the socket receives the video stream
+    // start the stream
+    if (!stream_start(&stream)) {
+        goto end;
+    }
+    stream_started = true;
+
     input_manager_init(&input_manager, options);
 
     ret = event_loop(options);

app/src/scrcpy.h

@@ -41,6 +41,11 @@ struct sc_port_range {
     uint16_t last;
 };
 
+enum sc_scale_filter {
+    SC_SCALE_FILTER_NONE,
+    SC_SCALE_FILTER_TRILINEAR, // mipmaps
+};
+
 #define SC_WINDOW_POSITION_UNDEFINED (-0x8000)
 
 struct scrcpy_options {
@@ -56,6 +61,7 @@ struct scrcpy_options {
     enum sc_record_format record_format;
     struct sc_port_range port_range;
     struct sc_shortcut_mods shortcut_mods;
+    enum sc_scale_filter scale_filter;
     uint16_t max_size;
     uint32_t bit_rate;
     uint16_t max_fps;
@@ -75,7 +81,6 @@ struct scrcpy_options {
     bool render_expired_frames;
     bool prefer_text;
     bool window_borderless;
-    bool mipmaps;
     bool stay_awake;
     bool force_adb_forward;
     bool disable_screensaver;
@@ -103,6 +108,7 @@ struct scrcpy_options {
         .data = {SC_MOD_LALT, SC_MOD_LSUPER}, \
         .count = 2, \
     }, \
+    .scale_filter = SC_SCALE_FILTER_TRILINEAR, \
     .max_size = 0, \
     .bit_rate = DEFAULT_BIT_RATE, \
     .max_fps = 0, \
@@ -122,7 +128,6 @@ struct scrcpy_options {
     .render_expired_frames = false, \
     .prefer_text = false, \
     .window_borderless = false, \
-    .mipmaps = true, \
     .stay_awake = false, \
     .force_adb_forward = false, \
     .disable_screensaver = false, \

app/src/screen.c

@@ -6,7 +6,6 @@
 
 #include "events.h"
 #include "icon.xpm"
-#include "scrcpy.h"
 #include "tiny_xpm.h"
 #include "video_buffer.h"
 #include "util/log.h"
@@ -191,10 +190,25 @@ screen_update_content_rect(struct screen *screen) {
     }
 }
 
-void
-screen_init(struct screen *screen, struct video_buffer *vb) {
-    *screen = (struct screen) SCREEN_INITIALIZER;
-    screen->vb = vb;
+static void
+on_frame_available(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+    (void) userdata;
+
+    static SDL_Event new_frame_event = {
+        .type = EVENT_NEW_FRAME,
+    };
+
+    // Post the event on the UI thread
+    SDL_PushEvent(&new_frame_event);
+}
+
+static void
+on_frame_skipped(struct video_buffer *vb, void *userdata) {
+    (void) vb;
+
+    struct screen *screen = userdata;
+    fps_counter_add_skipped_frame(screen->fps_counter);
 }
 
 static inline SDL_Texture *
@@ -225,26 +239,40 @@ create_texture(struct screen *screen) {
 }
 
 bool
-screen_init_rendering(struct screen *screen, const char *window_title,
-                      struct size frame_size, bool always_on_top,
-                      int16_t window_x, int16_t window_y, uint16_t window_width,
-                      uint16_t window_height, bool window_borderless,
-                      uint8_t rotation, bool mipmaps) {
-    screen->frame_size = frame_size;
-    screen->rotation = rotation;
-    if (rotation) {
-        LOGI("Initial display rotation set to %u", rotation);
+screen_init(struct screen *screen, struct video_buffer *vb,
+            struct fps_counter *fps_counter,
+            const struct screen_params *params) {
+    screen->vb = vb;
+    screen->fps_counter = fps_counter;
+
+    screen->resize_pending = false;
+    screen->has_frame = false;
+    screen->fullscreen = false;
+    screen->maximized = false;
+
+    static const struct video_buffer_callbacks cbs = {
+        .on_frame_available = on_frame_available,
+        .on_frame_skipped = on_frame_skipped,
+    };
+    video_buffer_set_consumer_callbacks(vb, &cbs, screen);
+
+    screen->frame_size = params->frame_size;
+    screen->rotation = params->rotation;
+    if (screen->rotation) {
+        LOGI("Initial display rotation set to %u", screen->rotation);
     }
-    struct size content_size = get_rotated_size(frame_size, screen->rotation);
+    struct size content_size =
+        get_rotated_size(screen->frame_size, screen->rotation);
     screen->content_size = content_size;
 
-    struct size window_size =
-        get_initial_optimal_size(content_size, window_width, window_height);
+    struct size window_size = get_initial_optimal_size(content_size,
+                                                       params->window_width,
+                                                       params->window_height);
     uint32_t window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_RESIZABLE;
 #ifdef HIDPI_SUPPORT
     window_flags |= SDL_WINDOW_ALLOW_HIGHDPI;
 #endif
-    if (always_on_top) {
+    if (params->always_on_top) {
 #ifdef SCRCPY_SDL_HAS_WINDOW_ALWAYS_ON_TOP
         window_flags |= SDL_WINDOW_ALWAYS_ON_TOP;
 #else
@@ -252,15 +280,15 @@ screen_init_rendering(struct screen *screen, const char *window_title,
              "(compile with SDL >= 2.0.5 to enable it)");
 #endif
     }
-    if (window_borderless) {
+    if (params->window_borderless) {
         window_flags |= SDL_WINDOW_BORDERLESS;
     }
 
-    int x = window_x != SC_WINDOW_POSITION_UNDEFINED
-          ? window_x : (int) SDL_WINDOWPOS_UNDEFINED;
-    int y = window_y != SC_WINDOW_POSITION_UNDEFINED
-          ? window_y : (int) SDL_WINDOWPOS_UNDEFINED;
-    screen->window = SDL_CreateWindow(window_title, x, y,
+    int x = params->window_x != SC_WINDOW_POSITION_UNDEFINED
+          ? params->window_x : (int) SDL_WINDOWPOS_UNDEFINED;
+    int y = params->window_y != SC_WINDOW_POSITION_UNDEFINED
+          ? params->window_y : (int) SDL_WINDOWPOS_UNDEFINED;
+    screen->window = SDL_CreateWindow(params->window_title, x, y,
                                       window_size.width, window_size.height,
                                       window_flags);
     if (!screen->window) {
@@ -281,8 +309,11 @@ screen_init_rendering(struct screen *screen, const char *window_title,
     const char *renderer_name = r ? NULL : renderer_info.name;
     LOGI("Renderer: %s", renderer_name ? renderer_name : "(unknown)");
 
+    screen->mipmaps = false;
+
     // starts with "opengl"
     bool use_opengl = renderer_name && !strncmp(renderer_name, "opengl", 6);
+    bool mipmaps = params->scale_filter == SC_SCALE_FILTER_TRILINEAR;
     if (use_opengl) {
         struct sc_opengl *gl = &screen->gl;
         sc_opengl_init(gl);
@@ -315,8 +346,8 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         LOGW("Could not load icon");
     }
 
-    LOGI("Initial texture: %" PRIu16 "x%" PRIu16, frame_size.width,
-                                                  frame_size.height);
+    LOGI("Initial texture: %" PRIu16 "x%" PRIu16, params->frame_size.width,
+                                                  params->frame_size.height);
     screen->texture = create_texture(screen);
     if (!screen->texture) {
         LOGC("Could not create texture: %s", SDL_GetError());
@@ -325,6 +356,15 @@ screen_init_rendering(struct screen *screen, const char *window_title,
         return false;
     }
 
+    screen->frame = av_frame_alloc();
+    if (!screen->frame) {
+        LOGC("Could not create screen frame");
+        SDL_DestroyTexture(screen->texture);
+        SDL_DestroyRenderer(screen->renderer);
+        SDL_DestroyWindow(screen->window);
+        return false;
+    }
+
     // Reset the window size to trigger a SIZE_CHANGED event, to workaround
     // HiDPI issues with some SDL renderers when several displays having
     // different HiDPI scaling are connected
@@ -342,6 +382,7 @@ screen_show_window(struct screen *screen) {
 
 void
 screen_destroy(struct screen *screen) {
+    av_frame_free(&screen->frame);
     SDL_DestroyTexture(screen->texture);
     SDL_DestroyRenderer(screen->renderer);
     SDL_DestroyWindow(screen->window);
@@ -449,7 +490,11 @@ update_texture(struct screen *screen, const AVFrame *frame) {
 
 static bool
 screen_update_frame(struct screen *screen) {
-    const AVFrame *frame = video_buffer_take_rendering_frame(screen->vb);
+    video_buffer_consumer_take_frame(screen->vb, &screen->frame);
+    AVFrame *frame = screen->frame;
+
+    fps_counter_add_rendered_frame(screen->fps_counter);
+
     struct size new_frame_size = {frame->width, frame->height};
     if (!prepare_for_frame(screen, new_frame_size)) {
         return false;

app/src/screen.h

@@ -9,11 +9,14 @@
 
 #include "coords.h"
 #include "opengl.h"
+#include "scrcpy.h"
 
 struct video_buffer;
 
 struct screen {
     struct video_buffer *vb;
+    struct fps_counter *fps_counter;
+
     SDL_Window *window;
     SDL_Renderer *renderer;
     SDL_Texture *texture;
@@ -34,52 +37,31 @@ struct screen {
     bool fullscreen;
     bool maximized;
     bool mipmaps;
+
+    AVFrame *frame;
 };
 
-#define SCREEN_INITIALIZER { \
-    .vb = NULL, \
-    .window = NULL, \
-    .renderer = NULL, \
-    .texture = NULL, \
-    .gl = {0}, \
-    .frame_size = { \
-        .width = 0, \
-        .height = 0, \
-    }, \
-    .content_size = { \
-        .width = 0, \
-        .height = 0, \
-    }, \
-    .resize_pending = false, \
-    .windowed_content_size = { \
-        .width = 0, \
-        .height = 0, \
-    }, \
-    .rotation = 0, \
-    .rect = { \
-        .x = 0, \
-        .y = 0, \
-        .w = 0, \
-        .h = 0, \
-    }, \
-    .has_frame = false, \
-    .fullscreen = false, \
-    .maximized = false, \
-    .mipmaps = false, \
-}
+struct screen_params {
+    const char *window_title;
+    struct size frame_size;
+    bool always_on_top;
 
-// initialize default values
-void
-screen_init(struct screen *screen, struct video_buffer *vb);
+    int16_t window_x;
+    int16_t window_y;
+    uint16_t window_width;  // accepts SC_WINDOW_POSITION_UNDEFINED
+    uint16_t window_height; // accepts SC_WINDOW_POSITION_UNDEFINED
+
+    bool window_borderless;
+
+    uint8_t rotation;
+    enum sc_scale_filter scale_filter;
+};
 
 // initialize screen, create window, renderer and texture (window is hidden)
-// window_x and window_y accept SC_WINDOW_POSITION_UNDEFINED
 bool
-screen_init_rendering(struct screen *screen, const char *window_title,
-                      struct size frame_size, bool always_on_top,
-                      int16_t window_x, int16_t window_y, uint16_t window_width,
-                      uint16_t window_height, bool window_borderless,
-                      uint8_t rotation, bool mipmaps);
+screen_init(struct screen *screen, struct video_buffer *vb,
+            struct fps_counter *fps_counter,
+            const struct screen_params *params);
 
 // show the window
 void

app/src/video_buffer.c

@@ -7,129 +7,118 @@
 #include "util/log.h"
 
 bool
-video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
-                  bool render_expired_frames) {
-    vb->fps_counter = fps_counter;
-
-    vb->decoding_frame = av_frame_alloc();
-    if (!vb->decoding_frame) {
-        goto error_0;
-    }
-
+video_buffer_init(struct video_buffer *vb, bool wait_consumer) {
     vb->pending_frame = av_frame_alloc();
     if (!vb->pending_frame) {
-        goto error_1;
-    }
-
-    vb->rendering_frame = av_frame_alloc();
-    if (!vb->rendering_frame) {
-        goto error_2;
+        goto error_0;
     }
 
     bool ok = sc_mutex_init(&vb->mutex);
     if (!ok) {
-        goto error_3;
+        goto error_1;
     }
 
-    vb->render_expired_frames = render_expired_frames;
-    if (render_expired_frames) {
+    vb->wait_consumer = wait_consumer;
+    if (wait_consumer) {
         ok = sc_cond_init(&vb->pending_frame_consumed_cond);
         if (!ok) {
             sc_mutex_destroy(&vb->mutex);
-            goto error_2;
+            goto error_1;
         }
-        // interrupted is not used if expired frames are not rendered
-        // since offering a frame will never block
+        // interrupted is not used if wait_consumer is disabled since offering
+        // a frame will never block
         vb->interrupted = false;
     }
 
-    // there is initially no rendering frame, so consider it has already been
-    // consumed
+    // there is initially no frame, so consider it has already been consumed
     vb->pending_frame_consumed = true;
 
+    // The callbacks must be set by the consumer via
+    // video_buffer_set_consumer_callbacks()
+    vb->cbs = NULL;
+
     return true;
 
-error_3:
-    av_frame_free(&vb->rendering_frame);
-error_2:
-    av_frame_free(&vb->pending_frame);
 error_1:
-    av_frame_free(&vb->decoding_frame);
+    av_frame_free(&vb->pending_frame);
 error_0:
     return false;
 }
 
 void
 video_buffer_destroy(struct video_buffer *vb) {
-    if (vb->render_expired_frames) {
+    if (vb->wait_consumer) {
         sc_cond_destroy(&vb->pending_frame_consumed_cond);
     }
     sc_mutex_destroy(&vb->mutex);
-    av_frame_free(&vb->rendering_frame);
     av_frame_free(&vb->pending_frame);
-    av_frame_free(&vb->decoding_frame);
 }
 
-static void
-video_buffer_swap_decoding_frame(struct video_buffer *vb) {
-    sc_mutex_assert(&vb->mutex);
-    AVFrame *tmp = vb->decoding_frame;
-    vb->decoding_frame = vb->pending_frame;
-    vb->pending_frame = tmp;
-}
-
-static void
-video_buffer_swap_rendering_frame(struct video_buffer *vb) {
-    sc_mutex_assert(&vb->mutex);
-    AVFrame *tmp = vb->rendering_frame;
-    vb->rendering_frame = vb->pending_frame;
-    vb->pending_frame = tmp;
+static inline void
+swap_frames(AVFrame **lhs, AVFrame **rhs) {
+    AVFrame *tmp = *lhs;
+    *lhs = *rhs;
+    *rhs = tmp;
 }
 
 void
-video_buffer_offer_decoded_frame(struct video_buffer *vb,
-                                 bool *previous_frame_skipped) {
+video_buffer_set_consumer_callbacks(struct video_buffer *vb,
+                                    const struct video_buffer_callbacks *cbs,
+                                    void *cbs_userdata) {
+    assert(!vb->cbs); // must be set only once
+    assert(cbs);
+    assert(cbs->on_frame_available);
+    vb->cbs = cbs;
+    vb->cbs_userdata = cbs_userdata;
+}
+
+void
+video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe) {
+    assert(vb->cbs);
+
     sc_mutex_lock(&vb->mutex);
-    if (vb->render_expired_frames) {
+    if (vb->wait_consumer) {
         // wait for the current (expired) frame to be consumed
         while (!vb->pending_frame_consumed && !vb->interrupted) {
             sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex);
         }
-    } else if (!vb->pending_frame_consumed) {
-        fps_counter_add_skipped_frame(vb->fps_counter);
     }
 
-    video_buffer_swap_decoding_frame(vb);
+    av_frame_unref(vb->pending_frame);
+    swap_frames(pframe, &vb->pending_frame);
 
-    *previous_frame_skipped = !vb->pending_frame_consumed;
+    bool skipped = !vb->pending_frame_consumed;
     vb->pending_frame_consumed = false;
 
     sc_mutex_unlock(&vb->mutex);
+
+    if (skipped) {
+        if (vb->cbs->on_frame_skipped)
+            vb->cbs->on_frame_skipped(vb, vb->cbs_userdata);
+    } else {
+        vb->cbs->on_frame_available(vb, vb->cbs_userdata);
+    }
 }
 
-const AVFrame *
-video_buffer_take_rendering_frame(struct video_buffer *vb) {
+void
+video_buffer_consumer_take_frame(struct video_buffer *vb, AVFrame **pframe) {
     sc_mutex_lock(&vb->mutex);
     assert(!vb->pending_frame_consumed);
     vb->pending_frame_consumed = true;
 
-    fps_counter_add_rendered_frame(vb->fps_counter);
+    swap_frames(pframe, &vb->pending_frame);
+    av_frame_unref(vb->pending_frame);
 
-    video_buffer_swap_rendering_frame(vb);
-
-    if (vb->render_expired_frames) {
+    if (vb->wait_consumer) {
         // unblock video_buffer_offer_decoded_frame()
         sc_cond_signal(&vb->pending_frame_consumed_cond);
     }
     sc_mutex_unlock(&vb->mutex);
-
-    // rendering_frame is only written from this thread, no need to lock
-    return vb->rendering_frame;
 }
 
 void
 video_buffer_interrupt(struct video_buffer *vb) {
-    if (vb->render_expired_frames) {
+    if (vb->wait_consumer) {
         sc_mutex_lock(&vb->mutex);
         vb->interrupted = true;
         sc_mutex_unlock(&vb->mutex);

app/src/video_buffer.h

@@ -13,53 +13,66 @@ typedef struct AVFrame AVFrame;
 
 /**
  * There are 3 frames in memory:
- *  - one frame is held by the decoder (decoding_frame)
- *  - one frame is held by the renderer (rendering_frame)
- *  - one frame is shared between the decoder and the renderer (pending_frame)
+ *  - one frame is held by the producer (producer_frame)
+ *  - one frame is held by the consumer (consumer_frame)
+ *  - one frame is shared between the producer and the consumer (pending_frame)
  *
- * The decoder decodes a packet into the decoding_frame (it may takes time).
+ * The producer generates a frame into the producer_frame (it may takes time).
  *
- * Once the frame is decoded, it calls video_buffer_offer_decoded_frame(),
- * which swaps the decoding and pending frames.
+ * Once the frame is produced, it calls video_buffer_producer_offer_frame(),
+ * which swaps the producer and pending frames.
  *
- * When the renderer is notified that a new frame is available, it calls
- * video_buffer_take_rendering_frame() to retrieve it, which swaps the pending
- * and rendering frames. The frame is valid until the next call, without
- * blocking the decoder.
+ * When the consumer is notified that a new frame is available, it calls
+ * video_buffer_consumer_take_frame() to retrieve it, which swaps the pending
+ * and consumer frames. The frame is valid until the next call, without
+ * blocking the producer.
  */
 
 struct video_buffer {
-    AVFrame *decoding_frame;
     AVFrame *pending_frame;
-    AVFrame *rendering_frame;
 
     sc_mutex mutex;
-    bool render_expired_frames;
+    bool wait_consumer; // never overwrite a pending frame if it is not consumed
     bool interrupted;
 
     sc_cond pending_frame_consumed_cond;
     bool pending_frame_consumed;
 
-    struct fps_counter *fps_counter;
+    const struct video_buffer_callbacks *cbs;
+    void *cbs_userdata;
+};
+
+struct video_buffer_callbacks {
+    // Called when a new frame can be consumed by
+    // video_buffer_consumer_take_frame(vb)
+    // This callback is mandatory (it must not be NULL).
+    void (*on_frame_available)(struct video_buffer *vb, void *userdata);
+
+    // Called when a pending frame has been overwritten by the producer
+    // This callback is optional (it may be NULL).
+    void (*on_frame_skipped)(struct video_buffer *vb, void *userdata);
 };
 
 bool
-video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
-                  bool render_expired_frames);
+video_buffer_init(struct video_buffer *vb, bool wait_consumer);
 
 void
 video_buffer_destroy(struct video_buffer *vb);
 
-// set the decoded frame as ready for rendering
-// the output flag is set to report whether the previous frame has been skipped
 void
-video_buffer_offer_decoded_frame(struct video_buffer *vb,
-                                 bool *previous_frame_skipped);
+video_buffer_set_consumer_callbacks(struct video_buffer *vb,
+                                    const struct video_buffer_callbacks *cbs,
+                                    void *cbs_userdata);
 
-// mark the rendering frame as consumed and return it
-// the frame is valid until the next call to this function
-const AVFrame *
-video_buffer_take_rendering_frame(struct video_buffer *vb);
+// set the producer frame as ready for consuming
+// the produced frame is exchanged with an unused allocated frame
+void
+video_buffer_producer_offer_frame(struct video_buffer *vb, AVFrame **pframe);
+
+// mark the consumer frame as consumed and exchange it with an unused allocated
+// frame
+void
+video_buffer_consumer_take_frame(struct video_buffer *vb, AVFrame **pframe);
 
 // wake up and avoid any blocking call
 void