implement ZRLE decoding

7 files changed, 437 insertions, 6 deletions

diff --git a/TODO b/TODO
index e74dc41..49bdd71 100644
--- a/TODO
+++ b/TODO
@@ -6,9 +6,7 @@ VisualNaCro testing
 test IRIX -overlay (x11vnc)
 java vncviewer doesn't do colour cursors?
 MinGW32 doesn't do fcntl on sockets; use setsockopt instead...
-make libvncclient threadsafe (static zlib buffer -> rfbClient*)
 make corre work again (libvncclient or libvncserver?)
-implement zrle in libvncclient
 implement "-record" in libvncclient
 libtoolize
 come up with an alpha cursor patch for Windows vncviewer
diff --git a/libvncclient/rfbproto.c b/libvncclient/rfbproto.c
index dd5d961..0e8ccfe 100644
--- a/libvncclient/rfbproto.c
+++ b/libvncclient/rfbproto.c
@@ -164,6 +164,12 @@ static void JpegTermSource(j_decompress_ptr cinfo);
 static void JpegSetSrcManager(j_decompress_ptr cinfo, uint8_t *compressedData,
                               int compressedLen);
 #endif
+static rfbBool HandleZRLE8(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleZRLE16(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleZRLE24(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleZRLE24Up(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleZRLE24Down(rfbClient* client, int rx, int ry, int rw, int rh);
+static rfbBool HandleZRLE32(rfbClient* client, int rx, int ry, int rw, int rh);
 #endif
 
 /*
@@ -497,6 +503,7 @@ SetFormatAndEncodings(rfbClient* client)
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingHextile);
 #ifdef LIBVNCSERVER_HAVE_LIBZ
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZlib);
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZRLE);
 #endif
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingCoRRE);
     encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRRE);
@@ -879,6 +886,40 @@ HandleRFBServerMessage(rfbClient* client)
 	break;
       }
 #endif
+      case rfbEncodingZRLE:
+      {
+	switch (client->format.bitsPerPixel) {
+	case 8:
+	  if (!HandleZRLE8(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	    return FALSE;
+	  break;
+	case 16:
+	  if (!HandleZRLE16(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	    return FALSE;
+	  break;
+	case 32:
+	{
+	  uint32_t maxColor=(client->format.redMax<<client->format.redShift)|
+		(client->format.greenMax<<client->format.greenShift)|
+		(client->format.blueMax<<client->format.blueShift);
+	  if ((client->format.bigEndian && (maxColor&0xff)==0) ||
+	      (!client->format.bigEndian && (maxColor&0xff000000)==0)) {
+	    if (!HandleZRLE24(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	      return FALSE;
+	  } else if (!client->format.bigEndian && (maxColor&0xff)==0) {
+	    if (!HandleZRLE24Up(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	      return FALSE;
+	  } else if (client->format.bigEndian && (maxColor&0xff000000)==0) {
+	    if (!HandleZRLE24Down(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	      return FALSE;
+	  } else if (!HandleZRLE32(client, rect.r.x,rect.r.y,rect.r.w,rect.r.h))
+	    return FALSE;
+	  break;
+	}
+	}
+	break;
+     }
+
 #endif
 
       default:
@@ -972,6 +1013,7 @@ HandleRFBServerMessage(rfbClient* client)
 #include "hextile.c"
 #include "zlib.c"
 #include "tight.c"
+#include "zrle.c"
 #undef BPP
 #define BPP 16
 #include "rre.c"
@@ -979,6 +1021,7 @@ HandleRFBServerMessage(rfbClient* client)
 #include "hextile.c"
 #include "zlib.c"
 #include "tight.c"
+#include "zrle.c"
 #undef BPP
 #define BPP 32
 #include "rre.c"
@@ -986,6 +1029,15 @@ HandleRFBServerMessage(rfbClient* client)
 #include "hextile.c"
 #include "zlib.c"
 #include "tight.c"
+#include "zrle.c"
+#define REALBPP 24
+#include "zrle.c"
+#define REALBPP 24
+#define UNCOMP 8
+#include "zrle.c"
+#define REALBPP 24
+#define UNCOMP -8
+#include "zrle.c"
 #undef BPP
 
 
diff --git a/libvncclient/tight.c b/libvncclient/tight.c
index 348c423..48a27e9 100644
--- a/libvncclient/tight.c
+++ b/libvncclient/tight.c
@@ -679,5 +679,7 @@ JpegSetSrcManager(j_decompress_ptr cinfo,
 
 #endif
 
+#undef CARDBPP
+
 #endif
 
diff --git a/libvncclient/vncviewer.c b/libvncclient/vncviewer.c
index 12c801b..607a2bb 100644
--- a/libvncclient/vncviewer.c
+++ b/libvncclient/vncviewer.c
@@ -87,7 +87,7 @@ static rfbBool MallocFrameBuffer(rfbClient* client) {
 static void initAppData(AppData* data) {
 	data->shareDesktop=TRUE;
 	data->viewOnly=FALSE;
-	data->encodingsString="tight hextile zlib corre rre raw";
+	data->encodingsString="tight zrle hextile zlib corre rre raw";
 	data->useBGR233=FALSE;
 	data->nColours=0;
 	data->forceOwnCmap=FALSE;
diff --git a/libvncclient/zlib.c b/libvncclient/zlib.c
index 2a32d1c..89db504 100644
--- a/libvncclient/zlib.c
+++ b/libvncclient/zlib.c
@@ -30,7 +30,7 @@
  */
 
 #define HandleZlibBPP CONCAT2E(HandleZlib,BPP)
-#define CARDBPP CONCAT2E(uint,BPP,_t)
+#define CARDBPP CONCAT3E(uint,BPP,_t)
 
 static rfbBool
 HandleZlibBPP (rfbClient* client, int rx, int ry, int rw, int rh)
diff --git a/libvncclient/zrle.c b/libvncclient/zrle.c
new file mode 100644
index 0000000..53ae6f1
--- /dev/null
+++ b/libvncclient/zrle.c
@@ -0,0 +1,380 @@
+/*
+ *  Copyright (C) 2005 Johannes E. Schindelin.  All Rights Reserved.
+ *
+ *  This is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This software is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this software; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
+ *  USA.
+ */
+
+#ifdef LIBVNCSERVER_HAVE_LIBZ
+
+/*
+ * zrle.c - handle zrle encoding.
+ *
+ * This file shouldn't be compiled directly.  It is included multiple times by
+ * rfbproto.c, each time with a different definition of the macro BPP.  For
+ * each value of BPP, this file defines a function which handles an zrle
+ * encoded rectangle with BPP bits per pixel.
+ */
+
+#ifndef REALBPP
+#define REALBPP BPP
+#endif
+
+#if !defined(UNCOMP) || UNCOMP==0
+#define HandleZRLE CONCAT2E(HandleZRLE,REALBPP)
+#define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP)
+#elif UNCOMP>0
+#define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down)
+#define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down)
+#else
+#define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up)
+#define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up)
+#endif
+#define CARDBPP CONCAT3E(uint,BPP,_t)
+#define CARDREALBPP CONCAT3E(uint,REALBPP,_t)
+
+static int HandleZRLETile(rfbClient* client,
+	uint8_t* buffer,size_t buffer_length,
+	int x,int y,int w,int h);
+
+static rfbBool
+HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh)
+{
+	rfbZRLEHeader header;
+	int remaining;
+	int inflateResult;
+	int toRead;
+
+	/* First make sure we have a large enough raw buffer to hold the
+	 * decompressed data.  In practice, with a fixed REALBPP, fixed frame
+	 * buffer size and the first update containing the entire frame
+	 * buffer, this buffer allocation should only happen once, on the
+	 * first update.
+	 */
+	if ( client->raw_buffer_size < (( rw * rh ) * ( REALBPP / 8 ))) {
+
+		if ( client->raw_buffer != NULL ) {
+
+			free( client->raw_buffer );
+
+		}
+
+		client->raw_buffer_size = (( rw * rh ) * ( REALBPP / 8 ));
+		client->raw_buffer = (char*) malloc( client->raw_buffer_size );
+
+	}
+
+	if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader))
+		return FALSE;
+
+	remaining = rfbClientSwap32IfLE(header.length);
+
+	/* Need to initialize the decompressor state. */
+	client->decompStream.next_in   = ( Bytef * )client->buffer;
+	client->decompStream.avail_in  = 0;
+	client->decompStream.next_out  = ( Bytef * )client->raw_buffer;
+	client->decompStream.avail_out = client->raw_buffer_size;
+	client->decompStream.data_type = Z_BINARY;
+
+	/* Initialize the decompression stream structures on the first invocation. */
+	if ( client->decompStreamInited == FALSE ) {
+
+		inflateResult = inflateInit( &client->decompStream );
+
+		if ( inflateResult != Z_OK ) {
+			rfbClientLog(
+					"inflateInit returned error: %d, msg: %s\n",
+					inflateResult,
+					client->decompStream.msg);
+			return FALSE;
+		}
+
+		client->decompStreamInited = TRUE;
+
+	}
+
+	inflateResult = Z_OK;
+
+	/* Process buffer full of data until no more to process, or
+	 * some type of inflater error, or Z_STREAM_END.
+	 */
+	while (( remaining > 0 ) &&
+			( inflateResult == Z_OK )) {
+
+		if ( remaining > RFB_BUFFER_SIZE ) {
+			toRead = RFB_BUFFER_SIZE;
+		}
+		else {
+			toRead = remaining;
+		}
+
+		/* Fill the buffer, obtaining data from the server. */
+		if (!ReadFromRFBServer(client, client->buffer,toRead))
+			return FALSE;
+
+		client->decompStream.next_in  = ( Bytef * )client->buffer;
+		client->decompStream.avail_in = toRead;
+
+		/* Need to uncompress buffer full. */
+		inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH );
+
+		/* We never supply a dictionary for compression. */
+		if ( inflateResult == Z_NEED_DICT ) {
+			rfbClientLog("zlib inflate needs a dictionary!\n");
+			return FALSE;
+		}
+		if ( inflateResult < 0 ) {
+			rfbClientLog(
+					"zlib inflate returned error: %d, msg: %s\n",
+					inflateResult,
+					client->decompStream.msg);
+			return FALSE;
+		}
+
+		/* Result buffer allocated to be at least large enough.  We should
+		 * never run out of space!
+		 */
+		if (( client->decompStream.avail_in > 0 ) &&
+				( client->decompStream.avail_out <= 0 )) {
+			rfbClientLog("zlib inflate ran out of space!\n");
+			return FALSE;
+		}
+
+		remaining -= toRead;
+
+	} /* while ( remaining > 0 ) */
+
+	if ( inflateResult == Z_OK ) {
+		void* buf=client->raw_buffer;
+		int i,j;
+
+		remaining = client->raw_buffer_size-client->decompStream.avail_out;
+
+		for(j=0; j<rh; j+=rfbZRLETileHeight)
+			for(i=0; i<rw; i+=rfbZRLETileWidth) {
+				int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth;
+				int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight;
+				int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight);
+
+				if(result<0) {
+					rfbClientLog("ZRLE decoding failed (%d)\n",result);
+return TRUE;
+					return FALSE;
+				}
+
+				buf+=result;
+				remaining-=result;
+			}
+	}
+	else {
+
+		rfbClientLog(
+				"zlib inflate returned error: %d, msg: %s\n",
+				inflateResult,
+				client->decompStream.msg);
+		return FALSE;
+
+	}
+
+	return TRUE;
+}
+
+#if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0
+#if UNCOMP>0
+#define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP)
+#else
+#define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP)))
+#endif
+#else
+#define UncompressCPixel(pointer) (*(CARDBPP*)pointer)
+#endif
+
+static int HandleZRLETile(rfbClient* client,
+		uint8_t* buffer,size_t buffer_length,
+		int x,int y,int w,int h) {
+	uint8_t* buffer_copy = buffer;
+	uint8_t* buffer_end = buffer+buffer_length;
+	uint8_t type;
+
+	if(buffer_length<1)
+		return -2;
+
+	type = *buffer;
+	buffer++;
+	switch(type) {
+		case 0: /* raw */
+		{
+#if REALBPP!=BPP
+			int i,j,j2;
+
+			if(1+w*h*REALBPP/8>buffer_length) {
+				rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h);
+				return -3;
+			}
+
+			for(j=y*client->width; j<(y+h)*client->width; j+=client->width)
+				for(i=x; i<x+w; i++,buffer+=REALBPP/8)
+					((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer);
+#else
+			CopyRectangle(client, buffer, x, y, w, h);
+#endif
+			break;
+		}
+		case 1: /* solid */
+		{
+			CARDBPP color = UncompressCPixel(buffer);
+
+			if(1+REALBPP/8>buffer_length)
+				return -4;
+				
+			FillRectangle(client, x, y, w, h, color);
+
+			break;
+		}
+		case 2 ... 127: /* packed Palette */
+		{
+			CARDBPP palette[16];
+			int i,j,shift,
+				bpp=(type>4?(type>16?8:4):(type>2?2:1)),
+				mask=(1<<bpp)-1,
+				divider=(8/bpp);
+
+			if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length)
+				return -5;
+
+			/* read palette */
+			for(i=0; i<type; i++,buffer+=REALBPP/8)
+				palette[i] = UncompressCPixel(buffer);
+
+			/* read palettized pixels */
+			for(j=y*client->width; j<(y+h)*client->width; j+=client->width) {
+				for(i=x,shift=8-bpp; i<x+w; i++) {
+					((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask];
+					shift-=bpp;
+					if(shift<0) {
+						shift=8-bpp;
+						buffer++;
+					}
+				}
+				if(shift<8-bpp)
+					buffer++;
+			}
+
+			break;
+		}
+		/* case 17 ... 127: not used, but valid */
+		case 128: /* plain RLE */
+		{
+			int i=0,j=0;
+			while(j<h) {
+				int color,length;
+				/* read color */
+				if(buffer+REALBPP/8+1>buffer_end)
+					return -7;
+				color = UncompressCPixel(buffer);
+				buffer+=REALBPP/8;
+				/* read run length */
+				length=1;
+				while(*buffer==0xff) {
+					if(buffer+1>=buffer_end)
+						return -8;
+					length+=*buffer;
+					buffer++;
+				}
+				length+=*buffer;
+				buffer++;
+				while(j<h && length>0) {
+					((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
+					length--;
+					i++;
+					if(i>=w) {
+						i=0;
+						j++;
+					}
+				}
+				if(length>0)
+					rfbClientLog("Warning: possible ZRLE corruption\n");
+			}
+
+			break;
+		}
+		case 129: /* unused */
+		{
+			return -8;
+		}
+		case 130 ... 255: /* palette RLE */
+		{
+			CARDBPP palette[128];
+			int i,j;
+
+			if(2+(type-128)*REALBPP/8>buffer_length)
+				return -9;
+
+			/* read palette */
+			for(i=0; i<type-128; i++,buffer+=REALBPP/8)
+				palette[i] = UncompressCPixel(buffer);
+			/* read palettized pixels */
+			i=j=0;
+			while(j<h) {
+				int color,length;
+				/* read color */
+				if(buffer>=buffer_end)
+					return -10;
+				color = palette[(*buffer)&0x7f];
+				length=1;
+				if(*buffer&0x80) {
+					if(buffer+1>=buffer_end)
+						return -11;
+					buffer++;
+					/* read run length */
+					while(*buffer==0xff) {
+						if(buffer+1>=buffer_end)
+							return -8;
+						length+=*buffer;
+						buffer++;
+					}
+					length+=*buffer;
+				}
+				buffer++;
+				while(j<h && length>0) {
+					((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
+					length--;
+					i++;
+					if(i>=w) {
+						i=0;
+						j++;
+					}
+				}
+				if(length>0)
+					rfbClientLog("Warning: possible ZRLE corruption\n");
+			}
+
+			break;
+		}
+	}
+
+	return buffer-buffer_copy;	
+}
+
+#undef CARDBPP
+#undef CARDREALBPP
+#undef HandleZRLE
+#undef HandleZRLETile
+#undef UncompressCPixel
+#undef REALBPP
+#undef UNCOMP
+
+#endif
+
diff --git a/test/encodingstest.c b/test/encodingstest.c
index 821b2ae..c2f8135 100644
--- a/test/encodingstest.c
+++ b/test/encodingstest.c
@@ -25,8 +25,7 @@ static encoding_t testEncodings[]={
 #ifdef LIBVNCSERVER_HAVE_LIBZ
 	{ rfbEncodingZlib, "zlib" },
 	{ rfbEncodingZlibHex, "zlibhex" },
-	/* TODO: implement ZRLE decoding */
-	/* { rfbEncodingZRLE, "zrle" }, */
+	{ rfbEncodingZRLE, "zrle" },
 #ifdef LIBVNCSERVER_HAVE_LIBJPEG
 	{ rfbEncodingTight, "tight" },
 #endif