summaryrefslogtreecommitdiffstats
path: root/libvncserver/rfbssl_openssl.c
blob: cbd68658a037780cc2f689b7def40bb80bf31aef (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
/*
 * rfbssl_openssl.c - Secure socket funtions (openssl version)
 */

/*
 *  Copyright (C) 2011 Gernot Tenchio
 *
 *  This is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This software is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this software; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
 *  USA.
 */

#include "rfbssl.h"
#include <openssl/ssl.h>
#include <openssl/err.h>

struct rfbssl_ctx {
    SSL_CTX *ssl_ctx;
    SSL     *ssl;
};

static void rfbssl_error(void)
{
    char buf[1024];
    unsigned long e = ERR_get_error();
    rfbErr("%s (%ld)\n", ERR_error_string(e, buf), e);
}

int rfbssl_init(rfbClientPtr cl)
{
    char *keyfile;
    int r, ret = -1;
    struct rfbssl_ctx *ctx;

    SSL_library_init();
    SSL_load_error_strings();

    if (cl->screen->sslkeyfile && *cl->screen->sslkeyfile) {
      keyfile = cl->screen->sslkeyfile;
    } else {
      keyfile = cl->screen->sslcertfile;
    }

    if (NULL == (ctx = malloc(sizeof(struct rfbssl_ctx)))) {
	rfbErr("OOM\n");
    } else if (!cl->screen->sslcertfile || !cl->screen->sslcertfile[0]) {
	rfbErr("SSL connection but no cert specified\n");
    } else if (NULL == (ctx->ssl_ctx = SSL_CTX_new(TLSv1_server_method()))) {
	rfbssl_error();
    } else if (SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, keyfile, SSL_FILETYPE_PEM) <= 0) {
	rfbErr("Unable to load private key file %s\n", keyfile);
    } else if (SSL_CTX_use_certificate_file(ctx->ssl_ctx, cl->screen->sslcertfile, SSL_FILETYPE_PEM) <= 0) {
	rfbErr("Unable to load certificate file %s\n", cl->screen->sslcertfile);
    } else if (NULL == (ctx->ssl = SSL_new(ctx->ssl_ctx))) {
	rfbErr("SSL_new failed\n");
	rfbssl_error();
    } else if (!(SSL_set_fd(ctx->ssl, cl->sock))) {
	rfbErr("SSL_set_fd failed\n");
	rfbssl_error();
    } else {
	while ((r = SSL_accept(ctx->ssl)) < 0) {
	    if (SSL_get_error(ctx->ssl, r) != SSL_ERROR_WANT_READ)
		break;
	}
	if (r < 0) {
	    rfbErr("SSL_accept failed %d\n", SSL_get_error(ctx->ssl, r));
	} else {
	    cl->sslctx = (rfbSslCtx *)ctx;
	    ret = 0;
	}
    }
    return ret;
}

int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize)
{
    int ret;
    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;

    while ((ret = SSL_write(ctx->ssl, buf, bufsize)) <= 0) {
	if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_WRITE)
	    break;
    }
    return ret;
}

int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize)
{
    int ret;
    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;

    while ((ret = SSL_peek(ctx->ssl, buf, bufsize)) <= 0) {
	if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_READ)
	    break;
    }
    return ret;
}

int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize)
{
    int ret;
    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;

    while ((ret = SSL_read(ctx->ssl, buf, bufsize)) <= 0) {
	if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_READ)
	    break;
    }
    return ret;
}

int rfbssl_pending(rfbClientPtr cl)
{
    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
    return SSL_pending(ctx->ssl);
}

void rfbssl_destroy(rfbClientPtr cl)
{
    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
    if (ctx->ssl)
	SSL_free(ctx->ssl);
    if (ctx->ssl_ctx)
	SSL_CTX_free(ctx->ssl_ctx);
}