websockets: Add UTF-8 encoding support.

This is not completely standard UTF-8 encoding. Only code points 0-255
are encoded and never encoded to more than two octets. Since '\x00' is
a WebSockets framing character, it's easier for all parties to encode
zero as '\xc4\x80', i.e. 194+128, i.e. UTF-8 256.

This means that a random stream will be slightly more than 50% larger
using this encoding scheme. But it's easy CPU-wise for client and
server to decode/encode. This is especially important for clients
written in languages that have weak bitops, like Javascript (i.e. the
noVNC client).

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

1 files changed, 20 insertions, 8 deletions

diff --git a/libvncserver/rfbserver.c b/libvncserver/rfbserver.c
index 491a438..25204cd 100644
--- a/libvncserver/rfbserver.c
+++ b/libvncserver/rfbserver.c
@@ -1840,16 +1840,28 @@ rfbProcessClientNormalMessage(rfbClientPtr cl)
     char encBuf2[64];
 
 #ifdef LIBVNCSERVER_WITH_WEBSOCKETS
-    if (cl->webSockets && cl->webSocketsBase64) {
-        /* With Base64 encoding we need at least 4 bytes */
+    if (cl->webSockets) {
         n = recv(cl->sock, encBuf, 4, MSG_PEEK);
-        if ((n > 0) && (n < 4)) {
-            if (encBuf[0] == '\xff') {
-                /* Make sure we don't miss a client disconnect on an end frame
-                 * marker */
-                n = read(cl->sock, encBuf, 1);
+        if (cl->webSocketsBase64) {
+            /* With Base64 encoding we need at least 4 bytes */
+            if ((n > 0) && (n < 4)) {
+                if (encBuf[0] == '\xff') {
+                    /* Make sure we don't miss a client disconnect on an end frame
+                    * marker */
+                    n = read(cl->sock, encBuf, 1);
+                }
+                return;
+            }
+        } else {
+            /* With UTF-8 encoding we need at least 3 bytes (framing + 1) */
+            if ((n == 1) || (n == 2)) {
+                if (encBuf[0] == '\xff') {
+                    /* Make sure we don't miss a client disconnect on an end frame
+                    * marker */
+                    n = read(cl->sock, encBuf, 1);
+                }
+                return;
             }
-            return;
         }
     }
 #endif