From 24ff3f256dc9227a51d17d50f0aeffa7b1027b4d Mon Sep 17 00:00:00 2001 From: Petr Sumbera Date: Tue, 21 Jul 2015 09:20:57 -0700 Subject: [PATCH 1/2] This array is so big that it eats whole stack for the thread and makes the process dump core on some systems (e.g. Solaris). --- server/shadow/shadow_capture.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/shadow/shadow_capture.c b/server/shadow/shadow_capture.c index 7aa321e67..f7035bb81 100644 --- a/server/shadow/shadow_capture.c +++ b/server/shadow/shadow_capture.c @@ -80,6 +80,9 @@ int shadow_capture_align_clip_rect(RECTANGLE_16* rect, RECTANGLE_16* clip) return 1; } +/* this is too big to consume whole stack on some systems */ +static BOOL grid[1024][1024]; + int shadow_capture_compare(BYTE* pData1, int nStep1, int nWidth, int nHeight, BYTE* pData2, int nStep2, RECTANGLE_16* rect) { BOOL equal; @@ -91,7 +94,6 @@ int shadow_capture_compare(BYTE* pData1, int nStep1, int nWidth, int nHeight, BY BYTE *p1, *p2; BOOL rows[1024]; BOOL cols[1024]; - BOOL grid[1024][1024]; allEqual = TRUE; ZeroMemory(rect, sizeof(RECTANGLE_16)); From 903e484fedd7d148cb06e3142156d617e3a0b353 Mon Sep 17 00:00:00 2001 From: Petr Sumbera Date: Wed, 22 Jul 2015 08:51:09 -0700 Subject: [PATCH 2/2] Removes unused (big big) array. Adds ifdef for debug stuff. --- server/shadow/shadow_capture.c | 62 +++++++++++++++++----------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/server/shadow/shadow_capture.c b/server/shadow/shadow_capture.c index f7035bb81..42163f834 100644 --- a/server/shadow/shadow_capture.c +++ b/server/shadow/shadow_capture.c @@ -80,9 +80,6 @@ int shadow_capture_align_clip_rect(RECTANGLE_16* rect, RECTANGLE_16* clip) return 1; } -/* this is too big to consume whole stack on some systems */ -static BOOL grid[1024][1024]; - int shadow_capture_compare(BYTE* pData1, int nStep1, int nWidth, int nHeight, BYTE* pData2, int nStep2, RECTANGLE_16* rect) { BOOL equal; @@ -93,13 +90,16 @@ int shadow_capture_compare(BYTE* pData1, int nStep1, int nWidth, int nHeight, BY int l, t, r, b; BYTE *p1, *p2; BOOL rows[1024]; +#ifdef WITH_DEBUG_SHADOW_CAPTURE BOOL cols[1024]; +#endif allEqual = TRUE; ZeroMemory(rect, sizeof(RECTANGLE_16)); FillMemory(rows, sizeof(rows), 0xFF); +#ifdef WITH_DEBUG_SHADOW_CAPTURE FillMemory(cols, sizeof(cols), 0xFF); - FillMemory(grid, sizeof(grid), 0xFF); +#endif nrow = (nHeight + 15) / 16; ncol = (nWidth + 15) / 16; @@ -143,9 +143,10 @@ int shadow_capture_compare(BYTE* pData1, int nStep1, int nWidth, int nHeight, BY if (!equal) { - grid[ty][tx] = FALSE; rows[ty] = FALSE; +#ifdef WITH_DEBUG_SHADOW_CAPTURE cols[tx] = FALSE; +#endif if (l > tx) l = tx; @@ -181,40 +182,39 @@ int shadow_capture_compare(BYTE* pData1, int nStep1, int nWidth, int nHeight, BY if (rect->bottom > nHeight) rect->bottom = nHeight; - if (0) +#ifdef WITH_DEBUG_SHADOW_CAPTURE + char *col_str = calloc(ncol + 1, sizeof(char)); + if (!col_str) { - char *col_str = calloc(ncol + 1, sizeof(char)); - if (!col_str) - { - WLog_ERR(TAG, "calloc failed!"); - return 1; - } + WLog_ERR(TAG, "calloc failed!"); + return 1; + } - for (tx = 0; tx < ncol; tx++) - sprintf(&col_str[tx], "-"); - WLog_INFO(TAG, "%s", col_str); + for (tx = 0; tx < ncol; tx++) + sprintf(&col_str[tx], "-"); + WLog_INFO(TAG, "%s", col_str); + for (tx = 0; tx < ncol; tx++) + sprintf(&col_str[tx], "%c", cols[tx] ? 'O' : 'X'); + WLog_INFO(TAG, "%s", col_str); + + for (tx = 0; tx < ncol; tx++) + sprintf(&col_str[tx], "-"); + WLog_INFO(TAG, "%s", col_str); + + for (ty = 0; ty < nrow; ty++) + { for (tx = 0; tx < ncol; tx++) sprintf(&col_str[tx], "%c", cols[tx] ? 'O' : 'X'); WLog_INFO(TAG, "%s", col_str); - - for (tx = 0; tx < ncol; tx++) - sprintf(&col_str[tx], "-"); - WLog_INFO(TAG, "%s", col_str); - - for (ty = 0; ty < nrow; ty++) - { - for (tx = 0; tx < ncol; tx++) - sprintf(&col_str[tx], "%c", cols[tx] ? 'O' : 'X'); - WLog_INFO(TAG, "%s", col_str); - WLog_INFO(TAG, "|%s|", rows[ty] ? "O" : "X"); - } - - WLog_INFO(TAG, "left: %d top: %d right: %d bottom: %d ncol: %d nrow: %d", - l, t, r, b, ncol, nrow); - free(col_str); + WLog_INFO(TAG, "|%s|", rows[ty] ? "O" : "X"); } + WLog_INFO(TAG, "left: %d top: %d right: %d bottom: %d ncol: %d nrow: %d", + l, t, r, b, ncol, nrow); + free(col_str); +#endif + return 1; }