diff --git a/vnc/Makefile b/vnc/Makefile index 9e70862f..b94673e2 100644 --- a/vnc/Makefile +++ b/vnc/Makefile @@ -1,5 +1,5 @@ -VNCOBJ = ../common/os_calls.o vnc.o d3des.o vncauth.o +VNCOBJ = ../common/os_calls.o vnc.o ../common/d3des.o vncauth.o CFLAGS = -Wall -O2 -I../common LDFLAGS = -shared LIBS = -ldl diff --git a/vnc/d3des.c b/vnc/d3des.c deleted file mode 100644 index 21362ad7..00000000 --- a/vnc/d3des.c +++ /dev/null @@ -1,442 +0,0 @@ -/* - * This is D3DES (V5.09) by Richard Outerbridge with the double and - * triple-length support removed for use in VNC. Also the bytebit[] array - * has been reversed so that the most significant bit in each byte of the - * key is ignored, not the least significant. - * - * These changes are: - * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved. - * - * This software is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - */ - -/* D3DES (V5.09) - - * - * A portable, public domain, version of the Data Encryption Standard. - * - * Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge. - * Thanks to: Dan Hoey for his excellent Initial and Inverse permutation - * code; Jim Gillogly & Phil Karn for the DES key schedule code; Dennis - * Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau, - * for humouring me on. - * - * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge. - * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992. - */ - -#include "d3des.h" - -static void scrunch(unsigned char *, unsigned long *); -static void unscrun(unsigned long *, unsigned char *); -static void desfunc(unsigned long *, unsigned long *); -static void cookey(unsigned long *); - -static unsigned long KnL[32] = { 0L }; -/* -static unsigned long KnR[32] = { 0L }; -static unsigned long Kn3[32] = { 0L }; -static unsigned char Df_Key[24] = { - 0x01,0x23,0x45,0x67,0x89,0xab,0xcd,0xef, - 0xfe,0xdc,0xba,0x98,0x76,0x54,0x32,0x10, - 0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 }; -*/ - -static unsigned short bytebit[8] = { - 01, 02, 04, 010, 020, 040, 0100, 0200 }; - -static unsigned long bigbyte[24] = { - 0x800000L, 0x400000L, 0x200000L, 0x100000L, - 0x80000L, 0x40000L, 0x20000L, 0x10000L, - 0x8000L, 0x4000L, 0x2000L, 0x1000L, - 0x800L, 0x400L, 0x200L, 0x100L, - 0x80L, 0x40L, 0x20L, 0x10L, - 0x8L, 0x4L, 0x2L, 0x1L }; - -/* Use the key schedule specified in the Standard (ANSI X3.92-1981). */ - -static unsigned char pc1[56] = { - 56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17, - 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35, - 62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, - 13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3 }; - -static unsigned char totrot[16] = { - 1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 }; - -static unsigned char pc2[48] = { - 13, 16, 10, 23, 0, 4, 2, 27, 14, 5, 20, 9, - 22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1, - 40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47, - 43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 }; - -void rfbDesKey(key, edf) /* Thanks to James Gillogly & Phil Karn! */ -unsigned char *key; -int edf; -{ - register int i, j, l, m, n; - unsigned char pc1m[56], pcr[56]; - unsigned long kn[32]; - - for ( j = 0; j < 56; j++ ) { - l = pc1[j]; - m = l & 07; - pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0; - } - for( i = 0; i < 16; i++ ) { - if( edf == DE1 ) m = (15 - i) << 1; - else m = i << 1; - n = m + 1; - kn[m] = kn[n] = 0L; - for( j = 0; j < 28; j++ ) { - l = j + totrot[i]; - if( l < 28 ) pcr[j] = pc1m[l]; - else pcr[j] = pc1m[l - 28]; - } - for( j = 28; j < 56; j++ ) { - l = j + totrot[i]; - if( l < 56 ) pcr[j] = pc1m[l]; - else pcr[j] = pc1m[l - 28]; - } - for( j = 0; j < 24; j++ ) { - if( pcr[pc2[j]] ) kn[m] |= bigbyte[j]; - if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j]; - } - } - cookey(kn); - return; - } - -static void cookey(raw1) -register unsigned long *raw1; -{ - register unsigned long *cook, *raw0; - unsigned long dough[32]; - register int i; - - cook = dough; - for( i = 0; i < 16; i++, raw1++ ) { - raw0 = raw1++; - *cook = (*raw0 & 0x00fc0000L) << 6; - *cook |= (*raw0 & 0x00000fc0L) << 10; - *cook |= (*raw1 & 0x00fc0000L) >> 10; - *cook++ |= (*raw1 & 0x00000fc0L) >> 6; - *cook = (*raw0 & 0x0003f000L) << 12; - *cook |= (*raw0 & 0x0000003fL) << 16; - *cook |= (*raw1 & 0x0003f000L) >> 4; - *cook++ |= (*raw1 & 0x0000003fL); - } - rfbUseKey(dough); - return; - } - -void rfbCPKey(into) -register unsigned long *into; -{ - register unsigned long *from, *endp; - - from = KnL, endp = &KnL[32]; - while( from < endp ) *into++ = *from++; - return; - } - -void rfbUseKey(from) -register unsigned long *from; -{ - register unsigned long *to, *endp; - - to = KnL, endp = &KnL[32]; - while( to < endp ) *to++ = *from++; - return; - } - -void rfbDes(inblock, outblock) -unsigned char *inblock, *outblock; -{ - unsigned long work[2]; - - scrunch(inblock, work); - desfunc(work, KnL); - unscrun(work, outblock); - return; - } - -static void scrunch(outof, into) -register unsigned char *outof; -register unsigned long *into; -{ - *into = (*outof++ & 0xffL) << 24; - *into |= (*outof++ & 0xffL) << 16; - *into |= (*outof++ & 0xffL) << 8; - *into++ |= (*outof++ & 0xffL); - *into = (*outof++ & 0xffL) << 24; - *into |= (*outof++ & 0xffL) << 16; - *into |= (*outof++ & 0xffL) << 8; - *into |= (*outof & 0xffL); - return; - } - -static void unscrun(outof, into) -register unsigned long *outof; -register unsigned char *into; -{ - *into++ = (unsigned char)((*outof >> 24) & 0xffL); - *into++ = (unsigned char)((*outof >> 16) & 0xffL); - *into++ = (unsigned char)((*outof >> 8) & 0xffL); - *into++ = (unsigned char)( *outof++ & 0xffL); - *into++ = (unsigned char)((*outof >> 24) & 0xffL); - *into++ = (unsigned char)((*outof >> 16) & 0xffL); - *into++ = (unsigned char)((*outof >> 8) & 0xffL); - *into = (unsigned char)( *outof & 0xffL); - return; - } - -static unsigned long SP1[64] = { - 0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L, - 0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L, - 0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L, - 0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L, - 0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L, - 0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L, - 0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L, - 0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L, - 0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L, - 0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L, - 0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L, - 0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L, - 0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L, - 0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L, - 0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L, - 0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L }; - -static unsigned long SP2[64] = { - 0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L, - 0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L, - 0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L, - 0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L, - 0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L, - 0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L, - 0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L, - 0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L, - 0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L, - 0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L, - 0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L, - 0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L, - 0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L, - 0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L, - 0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L, - 0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L }; - -static unsigned long SP3[64] = { - 0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L, - 0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L, - 0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L, - 0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L, - 0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L, - 0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L, - 0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L, - 0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L, - 0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L, - 0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L, - 0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L, - 0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L, - 0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L, - 0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L, - 0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L, - 0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L }; - -static unsigned long SP4[64] = { - 0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L, - 0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L, - 0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L, - 0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L, - 0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L, - 0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L, - 0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L, - 0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L, - 0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L, - 0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L, - 0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L, - 0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L, - 0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L, - 0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L, - 0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L, - 0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L }; - -static unsigned long SP5[64] = { - 0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L, - 0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L, - 0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L, - 0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L, - 0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L, - 0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L, - 0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L, - 0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L, - 0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L, - 0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L, - 0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L, - 0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L, - 0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L, - 0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L, - 0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L, - 0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L }; - -static unsigned long SP6[64] = { - 0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L, - 0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L, - 0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L, - 0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L, - 0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L, - 0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L, - 0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L, - 0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L, - 0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L, - 0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L, - 0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L, - 0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L, - 0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L, - 0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L, - 0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L, - 0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L }; - -static unsigned long SP7[64] = { - 0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L, - 0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L, - 0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L, - 0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L, - 0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L, - 0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L, - 0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L, - 0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L, - 0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L, - 0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L, - 0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L, - 0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L, - 0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L, - 0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L, - 0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L, - 0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L }; - -static unsigned long SP8[64] = { - 0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L, - 0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L, - 0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L, - 0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L, - 0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L, - 0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L, - 0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L, - 0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L, - 0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L, - 0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L, - 0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L, - 0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L, - 0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L, - 0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L, - 0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L, - 0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L }; - -static void desfunc(block, keys) -register unsigned long *block, *keys; -{ - register unsigned long fval, work, right, leftt; - register int round; - - leftt = block[0]; - right = block[1]; - work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL; - right ^= work; - leftt ^= (work << 4); - work = ((leftt >> 16) ^ right) & 0x0000ffffL; - right ^= work; - leftt ^= (work << 16); - work = ((right >> 2) ^ leftt) & 0x33333333L; - leftt ^= work; - right ^= (work << 2); - work = ((right >> 8) ^ leftt) & 0x00ff00ffL; - leftt ^= work; - right ^= (work << 8); - right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL; - work = (leftt ^ right) & 0xaaaaaaaaL; - leftt ^= work; - right ^= work; - leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL; - - for( round = 0; round < 8; round++ ) { - work = (right << 28) | (right >> 4); - work ^= *keys++; - fval = SP7[ work & 0x3fL]; - fval |= SP5[(work >> 8) & 0x3fL]; - fval |= SP3[(work >> 16) & 0x3fL]; - fval |= SP1[(work >> 24) & 0x3fL]; - work = right ^ *keys++; - fval |= SP8[ work & 0x3fL]; - fval |= SP6[(work >> 8) & 0x3fL]; - fval |= SP4[(work >> 16) & 0x3fL]; - fval |= SP2[(work >> 24) & 0x3fL]; - leftt ^= fval; - work = (leftt << 28) | (leftt >> 4); - work ^= *keys++; - fval = SP7[ work & 0x3fL]; - fval |= SP5[(work >> 8) & 0x3fL]; - fval |= SP3[(work >> 16) & 0x3fL]; - fval |= SP1[(work >> 24) & 0x3fL]; - work = leftt ^ *keys++; - fval |= SP8[ work & 0x3fL]; - fval |= SP6[(work >> 8) & 0x3fL]; - fval |= SP4[(work >> 16) & 0x3fL]; - fval |= SP2[(work >> 24) & 0x3fL]; - right ^= fval; - } - - right = (right << 31) | (right >> 1); - work = (leftt ^ right) & 0xaaaaaaaaL; - leftt ^= work; - right ^= work; - leftt = (leftt << 31) | (leftt >> 1); - work = ((leftt >> 8) ^ right) & 0x00ff00ffL; - right ^= work; - leftt ^= (work << 8); - work = ((leftt >> 2) ^ right) & 0x33333333L; - right ^= work; - leftt ^= (work << 2); - work = ((right >> 16) ^ leftt) & 0x0000ffffL; - leftt ^= work; - right ^= (work << 16); - work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL; - leftt ^= work; - right ^= (work << 4); - *block++ = right; - *block = leftt; - return; - } - -/* Validation sets: - * - * Single-length key, single-length plaintext - - * Key : 0123 4567 89ab cdef - * Plain : 0123 4567 89ab cde7 - * Cipher : c957 4425 6a5e d31d - * - * Double-length key, single-length plaintext - - * Key : 0123 4567 89ab cdef fedc ba98 7654 3210 - * Plain : 0123 4567 89ab cde7 - * Cipher : 7f1d 0a77 826b 8aff - * - * Double-length key, double-length plaintext - - * Key : 0123 4567 89ab cdef fedc ba98 7654 3210 - * Plain : 0123 4567 89ab cdef 0123 4567 89ab cdff - * Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7 - * - * Triple-length key, single-length plaintext - - * Key : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567 - * Plain : 0123 4567 89ab cde7 - * Cipher : de0b 7c06 ae5e 0ed5 - * - * Triple-length key, double-length plaintext - - * Key : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567 - * Plain : 0123 4567 89ab cdef 0123 4567 89ab cdff - * Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5 - * - * d3des V5.0a rwo 9208.07 18:44 Graven Imagery - **********************************************************************/ diff --git a/vnc/d3des.h b/vnc/d3des.h deleted file mode 100644 index e3761ca2..00000000 --- a/vnc/d3des.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef D3DES_H -#define D3DES_H - -/* - * This is D3DES (V5.09) by Richard Outerbridge with the double and - * triple-length support removed for use in VNC. - * - * These changes are: - * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved. - * - * This software is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - */ - -/* d3des.h - - * - * Headers and defines for d3des.c - * Graven Imagery, 1992. - * - * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge - * (GEnie : OUTER; CIS : [71755,204]) - */ - -#define EN0 0 /* MODE == encrypt */ -#define DE1 1 /* MODE == decrypt */ - -extern void rfbDesKey(unsigned char *, int); -/* hexkey[8] MODE - * Sets the internal key register according to the hexadecimal - * key contained in the 8 bytes of hexkey, according to the DES, - * for encryption or decryption according to MODE. - */ - -extern void rfbUseKey(unsigned long *); -/* cookedkey[32] - * Loads the internal key register with the data in cookedkey. - */ - -extern void rfbCPKey(unsigned long *); -/* cookedkey[32] - * Copies the contents of the internal key register into the storage - * located at &cookedkey[0]. - */ - -extern void rfbDes(unsigned char *, unsigned char *); -/* from[8] to[8] - * Encrypts/Decrypts (according to the key currently loaded in the - * internal key register) one block of eight bytes at address 'from' - * into the block at address 'to'. They can be the same. - */ - -/* d3des.h V5.09 rwo 9208.04 15:06 Graven Imagery - ********************************************************************/ - -#endif diff --git a/vnc/vnc.c b/vnc/vnc.c index 94c39ae8..63eda712 100644 --- a/vnc/vnc.c +++ b/vnc/vnc.c @@ -14,7 +14,7 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. xrdp: A Remote Desktop Protocol server. - Copyright (C) Jay Sorg 2004 + Copyright (C) Jay Sorg 2004-2005 libvnc @@ -83,6 +83,8 @@ int lib_mod_event(struct vnc* v, int msg, int param1, int param2) out_uint32_be(s, key); if (g_tcp_force_send(v->sck, s->data, 8) != 0) { + g_tcp_close(v->sck); + v->sck = 0; free_stream(s); return 1; } @@ -111,6 +113,8 @@ int lib_mod_event(struct vnc* v, int msg, int param1, int param2) out_uint16_be(s, param2); if (g_tcp_force_send(v->sck, s->data, 6) != 0) { + g_tcp_close(v->sck); + v->sck = 0; free_stream(s); return 1; } @@ -299,7 +303,12 @@ int lib_framebuffer_update(struct vnc* v) free_stream(s); return 1; } - v->server_paint_rect(v, x, y, cx, cy, data); + if (v->server_paint_rect(v, x, y, cx, cy, data) != 0) + { + g_free(data); + free_stream(s); + return 1; + } } else if (encoding == 1) /* copy rect */ { @@ -312,7 +321,12 @@ int lib_framebuffer_update(struct vnc* v) } in_uint16_be(s, srcx); in_uint16_be(s, srcy); - v->server_screen_blt(v, x, y, cx, cy, srcx, srcy); + if (v->server_screen_blt(v, x, y, cx, cy, srcx, srcy) != 0) + { + g_free(data); + free_stream(s); + return 1; + } } else if (encoding == 0xffffff11) /* cursor */ { @@ -343,7 +357,12 @@ int lib_framebuffer_update(struct vnc* v) } } } - v->server_set_cursor(v, x, y, cursor_data, cursor_mask); + if (v->server_set_cursor(v, x, y, cursor_data, cursor_mask) != 0) + { + g_free(data); + free_stream(s); + return 1; + } } else { @@ -446,12 +465,16 @@ int lib_mod_signal(struct vnc* v) if (g_tcp_force_recv(v->sck, &type, 1) != 0) { + g_tcp_close(v->sck); + v->sck = 0; return 1; } if (type == 0) /* framebuffer update */ { if (lib_framebuffer_update(v) != 0) { + g_tcp_close(v->sck); + v->sck = 0; return 1; } } @@ -459,6 +482,8 @@ int lib_mod_signal(struct vnc* v) { if (lib_palette_update(v) != 0) { + g_tcp_close(v->sck); + v->sck = 0; return 1; } } @@ -466,6 +491,8 @@ int lib_mod_signal(struct vnc* v) { if (lib_clip_data(v) != 0) { + g_tcp_close(v->sck); + v->sck = 0; return 1; } } @@ -497,6 +524,7 @@ int lib_mod_start(struct vnc* v, int w, int h, int bpp) 4 - server and client bpp do not match 5 - no more available X desktops when spawning a new session 6 - no ip + 7 - sck closed */ int lib_mod_connect(struct vnc* v) { @@ -544,7 +572,12 @@ int lib_mod_connect(struct vnc* v) s_pop_layer(s, channel_hdr); out_uint32_be(s, 0); // version out_uint32_be(s, s->end - s->data); // size - g_tcp_force_send(sck, s->data, s->end - s->data); + if (g_tcp_force_send(sck, s->data, s->end - s->data) != 0) + { + g_tcp_close(sck); + free_stream(s); + return 7; + } init_stream(s, 8192); if (g_tcp_force_recv(sck, s->data, 8) == 0) { @@ -567,6 +600,18 @@ int lib_mod_connect(struct vnc* v) } } } + else + { + g_tcp_close(sck); + free_stream(s); + return 7; + } + } + else + { + g_tcp_close(sck); + free_stream(s); + return 7; } } g_tcp_close(sck); @@ -590,11 +635,32 @@ int lib_mod_connect(struct vnc* v) g_tcp_set_non_blocking(v->sck); /* protocal version */ init_stream(s, 8192); - g_tcp_force_recv(v->sck, s->data, 12); - g_tcp_force_send(v->sck, "RFB 003.003\n", 12); + if (g_tcp_force_recv(v->sck, s->data, 12) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } + if (g_tcp_force_send(v->sck, "RFB 003.003\n", 12) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } /* sec type */ init_stream(s, 8192); - g_tcp_force_recv(v->sck, s->data, 4); + if (g_tcp_force_recv(v->sck, s->data, 4) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } in_uint32_be(s, i); if (i == 1) /* none */ { @@ -603,9 +669,23 @@ int lib_mod_connect(struct vnc* v) else if (i == 2) /* dec the password and the server random */ { init_stream(s, 8192); - g_tcp_force_recv(v->sck, s->data, 16); + if (g_tcp_force_recv(v->sck, s->data, 16) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } rfbEncryptBytes((unsigned char*)s->data, v->password); - g_tcp_force_send(v->sck, s->data, 16); + if (g_tcp_force_send(v->sck, s->data, 16) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } } else { @@ -616,7 +696,14 @@ int lib_mod_connect(struct vnc* v) { /* sec result */ init_stream(s, 8192); - g_tcp_force_recv(v->sck, s->data, 4); + if (g_tcp_force_recv(v->sck, s->data, 4) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } in_uint32_be(s, i); if (i != 0) { @@ -627,15 +714,43 @@ int lib_mod_connect(struct vnc* v) { init_stream(s, 8192); s->data[0] = 1; - g_tcp_force_send(v->sck, s->data, 1); /* share flag */ - g_tcp_force_recv(v->sck, s->data, 4); /* server init */ + if (g_tcp_force_send(v->sck, s->data, 1) != 0) /* share flag */ + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } + if (g_tcp_force_recv(v->sck, s->data, 4) != 0) /* server init */ + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } in_uint16_be(s, v->mod_width); in_uint16_be(s, v->mod_height); init_stream(pixel_format, 8192); - g_tcp_force_recv(v->sck, pixel_format->data, 16); + if (g_tcp_force_recv(v->sck, pixel_format->data, 16) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } in_uint8(pixel_format, v->mod_bpp); init_stream(s, 8192); - g_tcp_force_recv(v->sck, s->data, 4); /* name len */ + if (g_tcp_force_recv(v->sck, s->data, 4) != 0) /* name len */ + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } in_uint32_be(s, i); if (i > 255 || i < 0) { @@ -643,7 +758,14 @@ int lib_mod_connect(struct vnc* v) } else { - g_tcp_force_recv(v->sck, v->mod_name, i); + if (g_tcp_force_recv(v->sck, v->mod_name, i) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } v->mod_name[i] = 0; } /* should be connected */ @@ -672,7 +794,14 @@ int lib_mod_connect(struct vnc* v) out_uint8s(pixel_format, 3); /* pad */ } out_uint8a(s, pixel_format->data, 16); - g_tcp_force_send(v->sck, s->data, 20); + if (g_tcp_force_send(v->sck, s->data, 20) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } /* SetEncodings */ init_stream(s, 8192); out_uint8(s, 2); @@ -681,7 +810,14 @@ int lib_mod_connect(struct vnc* v) out_uint32_be(s, 0); /* raw */ out_uint32_be(s, 1); /* copy rect */ out_uint32_be(s, 0xffffff11); /* cursor */ - g_tcp_force_send(v->sck, s->data, 4 + 3 * 4); + if (g_tcp_force_send(v->sck, s->data, 4 + 3 * 4) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } /* FrambufferUpdateRequest */ init_stream(s, 8192); out_uint8(s, 3); @@ -690,7 +826,14 @@ int lib_mod_connect(struct vnc* v) out_uint16_be(s, 0); out_uint16_be(s, v->mod_width); out_uint16_be(s, v->mod_height); - g_tcp_force_send(v->sck, s->data, 10); + if (g_tcp_force_send(v->sck, s->data, 10) != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + free_stream(s); + free_stream(pixel_format); + return 7; + } } if (error == 0) { @@ -709,6 +852,11 @@ int lib_mod_connect(struct vnc* v) v->server_set_cursor(v, 0, 0, cursor_data, cursor_mask); free_stream(s); free_stream(pixel_format); + if (error != 0) + { + g_tcp_close(v->sck); + v->sck = 0; + } return error; } @@ -726,7 +874,13 @@ int lib_mod_invalidate(struct vnc* v, int x, int y, int cx, int cy) out_uint16_be(s, y); out_uint16_be(s, cx); out_uint16_be(s, cy); - g_tcp_force_send(v->sck, s->data, 10); + if (g_tcp_force_send(v->sck, s->data, 10) != 0) + { + free_stream(s); + g_tcp_close(v->sck); + v->sck = 0; + return 1; + } free_stream(s); return 0; }