changes and cleanup to process_enc_rfx

2 files changed, 67 insertions, 51 deletions

diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c
index ae6a1b35..e6cfec27 100644
--- a/xrdp/xrdp_encoder.c
+++ b/xrdp/xrdp_encoder.c
@@ -39,6 +39,8 @@
   } \
   while (0)
 
+#define XRDP_SURCMD_PREFIX_BYTES 256
+
 /*****************************************************************************/
 static int
 process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
@@ -129,6 +131,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
     self->xrdp_encoder_event_processed = g_create_wait_obj(buf);
     g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_term", pid);
     self->xrdp_encoder_term = g_create_wait_obj(buf);
+    self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15;
 
     /* create thread to process messages */
     tc_thread_create(proc_enc_msg, self);
@@ -312,6 +315,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
     tbus event_processed;
     struct rfx_tile *tiles;
     struct rfx_rect *rfxrects;
+    int alloc_bytes;
 
     LLOGLN(10, ("process_enc_rfx:"));
     LLOGLN(10, ("process_enc_rfx: num_crects %d num_drects %d",
@@ -320,64 +324,75 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
     mutex = self->mutex;
     event_processed = self->xrdp_encoder_event_processed;
 
-    if ((enc->num_crects > 4096) || (enc->num_drects > 4096))
-    {
-        return 0;
-    }
+    error = 1;
+    out_data = NULL;
+    out_data_bytes = 0;
 
-    out_data_bytes = 16 * 1024 * 1024;
-    index = 256 + sizeof(struct rfx_tile) * 4096 +
-                  sizeof(struct rfx_rect) * 4096;
-    out_data = (char *) g_malloc(out_data_bytes + index, 0);
-    if (out_data == 0)
+    if ((enc->num_crects > 0) && (enc->num_crects <= UINT16_MAX) &&
+        (enc->num_drects > 0) && (enc->num_drects <= UINT16_MAX))
     {
-        return 0;
-    }
-    tiles = (struct rfx_tile *) (out_data + out_data_bytes + 256);
-    rfxrects = (struct rfx_rect *) (tiles + 4096);
+        alloc_bytes = XRDP_SURCMD_PREFIX_BYTES;
+        alloc_bytes += self->max_compressed_bytes;
+        alloc_bytes += sizeof(struct rfx_tile) * enc->num_crects +
+                       sizeof(struct rfx_rect) * enc->num_drects;
+        out_data = g_new(char, alloc_bytes);
+        if (out_data != NULL)
+        {
+            tiles = (struct rfx_tile *)
+                    (out_data + XRDP_SURCMD_PREFIX_BYTES +
+                     self->max_compressed_bytes);
+            rfxrects = (struct rfx_rect *) (tiles + enc->num_crects);
 
-    count = enc->num_crects;
-    for (index = 0; index < count; index++)
-    {
-        x = enc->crects[index * 4 + 0];
-        y = enc->crects[index * 4 + 1];
-        cx = enc->crects[index * 4 + 2];
-        cy = enc->crects[index * 4 + 3];
-        LLOGLN(10, ("process_enc_rfx:"));
-        tiles[index].x = x;
-        tiles[index].y = y;
-        tiles[index].cx = cx;
-        tiles[index].cy = cy;
-        LLOGLN(10, ("x %d y %d cx %d cy %d", x, y, cx, cy));
-        tiles[index].quant_y = 0;
-        tiles[index].quant_cb = 0;
-        tiles[index].quant_cr = 0;
-    }
+            count = enc->num_crects;
+            for (index = 0; index < count; index++)
+            {
+                x = enc->crects[index * 4 + 0];
+                y = enc->crects[index * 4 + 1];
+                cx = enc->crects[index * 4 + 2];
+                cy = enc->crects[index * 4 + 3];
+                tiles[index].x = x;
+                tiles[index].y = y;
+                tiles[index].cx = cx;
+                tiles[index].cy = cy;
+                tiles[index].quant_y = 0;
+                tiles[index].quant_cb = 0;
+                tiles[index].quant_cr = 0;
+            }
 
-    count = enc->num_drects;
-    for (index = 0; index < count; index++)
-    {
-        x = enc->drects[index * 4 + 0];
-        y = enc->drects[index * 4 + 1];
-        cx = enc->drects[index * 4 + 2];
-        cy = enc->drects[index * 4 + 3];
-        LLOGLN(10, ("process_enc_rfx:"));
-        rfxrects[index].x = x;
-        rfxrects[index].y = y;
-        rfxrects[index].cx = cx;
-        rfxrects[index].cy = cy;
+            count = enc->num_drects;
+            for (index = 0; index < count; index++)
+            {
+                x = enc->drects[index * 4 + 0];
+                y = enc->drects[index * 4 + 1];
+                cx = enc->drects[index * 4 + 2];
+                cy = enc->drects[index * 4 + 3];
+                rfxrects[index].x = x;
+                rfxrects[index].y = y;
+                rfxrects[index].cx = cx;
+                rfxrects[index].cy = cy;
+            }
+
+            out_data_bytes = self->max_compressed_bytes;
+            error = rfxcodec_encode(self->codec_handle,
+                                    out_data + XRDP_SURCMD_PREFIX_BYTES,
+                                    &out_data_bytes, enc->data,
+                                    enc->width, enc->height, enc->width * 4,
+                                    rfxrects, enc->num_drects,
+                                    tiles, enc->num_crects, 0, 0);
+        }
     }
 
-    error = rfxcodec_encode(self->codec_handle, out_data + 256, &out_data_bytes,
-                            enc->data, enc->width, enc->height, enc->width * 4,
-                            rfxrects, enc->num_drects,
-                            tiles, enc->num_crects, 0, 0);
     LLOGLN(10, ("process_enc_rfx: rfxcodec_encode rv %d", error));
-
-    enc_done = (XRDP_ENC_DATA_DONE *)
-               g_malloc(sizeof(XRDP_ENC_DATA_DONE), 1);
-    enc_done->comp_bytes = out_data_bytes;
-    enc_done->pad_bytes = 256;
+    /* only if enc_done->comp_bytes is not zero is something sent
+       to the client but you must always send something back even
+       on error so Xorg can get ack */
+    enc_done = g_new0(XRDP_ENC_DATA_DONE, 1);
+    if (enc_done == NULL)
+    {
+        return 1;
+    }
+    enc_done->comp_bytes = error == 0 ? out_data_bytes : 0;
+    enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES;
     enc_done->comp_pad_data = out_data;
     enc_done->enc = enc;
     enc_done->last = 1;
diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h
index f138d749..4c57471c 100644
--- a/xrdp/xrdp_encoder.h
+++ b/xrdp/xrdp_encoder.h
@@ -14,6 +14,7 @@ struct xrdp_encoder
     int in_codec_mode;
     int codec_id;
     int codec_quality;
+    int max_compressed_bytes;
     tbus xrdp_encoder_event_to_proc;
     tbus xrdp_encoder_event_processed;
     tbus xrdp_encoder_term;