websockets: Add encryption support

[jes: moved out GnuTLS and OpenSSL support, added a dummy support, to
separate changes better, and to keep things compiling]

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

1 files changed, 47 insertions, 8 deletions

diff --git a/libvncserver/rfbserver.c b/libvncserver/rfbserver.c
index 25204cd..c623329 100644
--- a/libvncserver/rfbserver.c
+++ b/libvncserver/rfbserver.c
@@ -73,6 +73,10 @@
 /* strftime() */
 #include <time.h>
 
+#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
+#include "rfbssl.h"
+#endif
+
 #ifdef __MINGW32__
 static int compat_mkdir(const char *path, int mode)
 {
@@ -360,7 +364,6 @@ rfbNewTCPOrUDPClient(rfbScreenInfoPtr rfbScreen,
 
 #ifdef LIBVNCSERVER_WITH_WEBSOCKETS
       cl->webSockets       = FALSE;
-      cl->webSocketsSSL    = FALSE;
       cl->webSocketsBase64 = FALSE;
       cl->dblen= 0;
       cl->carrylen = 0;
@@ -1841,16 +1844,50 @@ rfbProcessClientNormalMessage(rfbClientPtr cl)
 
 #ifdef LIBVNCSERVER_WITH_WEBSOCKETS
     if (cl->webSockets) {
-        n = recv(cl->sock, encBuf, 4, MSG_PEEK);
-        if (cl->webSocketsBase64) {
+	if (cl->sslctx)
+	    n = rfbssl_peek(cl, encBuf, 4);
+	else
+	    n = recv(cl->sock, encBuf, 4, MSG_PEEK);
+
+	if (n <= 0) {
+	    if (n != 0)
+		rfbLogPerror("rfbProcessClientNormalMessage: peek");
+	    rfbCloseClient(cl);
+	    return;
+	}
+
+	if (cl->webSocketsBase64) {
             /* With Base64 encoding we need at least 4 bytes */
             if ((n > 0) && (n < 4)) {
                 if (encBuf[0] == '\xff') {
+		    int doclose = 0;
                     /* Make sure we don't miss a client disconnect on an end frame
-                    * marker */
-                    n = read(cl->sock, encBuf, 1);
+                     * marker. Because we use a peek buffer in some cases it is not
+		     * applicable to wait for more data per select(). */
+		    switch (n) {
+			case 3:
+			    if (encBuf[1] == '\xff' && encBuf[2] == '\x00')
+				doclose = 1;
+			    break;
+			case 2:
+			    if (encBuf[1] == '\x00')
+				doclose = 1;
+			    break;
+			default:
+			    ;
+		    }
+
+		    if (cl->sslctx)
+			n = rfbssl_read(cl, encBuf, n);
+		    else
+			n = read(cl->sock, encBuf, n);
+
+		    if (doclose) {
+			rfbErr("rfbProcessClientNormalMessage: websocket close frame received\n");
+			rfbCloseClient(cl);
+		    }
+		    return;
                 }
-                return;
             }
         } else {
             /* With UTF-8 encoding we need at least 3 bytes (framing + 1) */
@@ -1858,9 +1895,11 @@ rfbProcessClientNormalMessage(rfbClientPtr cl)
                 if (encBuf[0] == '\xff') {
                     /* Make sure we don't miss a client disconnect on an end frame
                     * marker */
-                    n = read(cl->sock, encBuf, 1);
+		    if (cl->sslctx)
+			    n = rfbssl_read(cl, encBuf, 1);
+		    else
+			    n = read(cl->sock, encBuf, 1);
                 }
-                return;
             }
         }
     }