fixed space checking and not understood buffer overrun

src/Makefile

 CC = gcc
 #CFLAGS = -g -pg -lc -O2 -Wall -fprofile-arcs -ftest-coverage -DPROFILE
 #CFLAGS = -g -pg -lc -O2 -Wall -fprofile-arcs -ftest-coverage -DUNROLL_BLOCK_8
-CFLAGS = -g -O3 -Wall -DUNROLL_BLOCK_8
+CFLAGS = -g -O0 -Wall -DUNROLL_BLOCK_8
 
 OBJS = aee.o aed.o sz_compat.o
 

src/aed.c

@@ -12,7 +12,7 @@
 #define MIN(a, b) (((a) < (b))? (a): (b))
 
 #define SAFE (strm->avail_in >= state->in_blklen        \
-              && strm->avail_out >= strm->block_size)
+              && strm->avail_out >= state->out_blklen)
 
 #define ROS 5
 
@@ -28,6 +28,7 @@ typedef struct internal_state {
     int mode;          /* current mode of FSM */
     int in_blklen;     /* length of uncompressed input block
                           should be the longest possible block */
+    int out_blklen;    /* length of output block in bytes */
     int n, i;          /* counter for samples */
     int64_t *block;    /* block buffer for split-sample options */
     int se;            /* set if second extension option is selected */
@@ -263,6 +264,7 @@ int ae_decode_init(ae_streamp strm)
     if (strm->bit_per_sample > 16)
     {
         state->id_len = 5;
+        state->out_blklen = strm->block_size * 4;
         if (strm->flags & AE_DATA_MSB)
             state->put_sample = put_msb_32;
         else
@@ -271,6 +273,7 @@ int ae_decode_init(ae_streamp strm)
     else if (strm->bit_per_sample > 8)
     {
         state->id_len = 4;
+        state->out_blklen = strm->block_size * 2;
         if (strm->flags & AE_DATA_MSB)
             state->put_sample = put_msb_16;
         else
@@ -279,6 +282,7 @@ int ae_decode_init(ae_streamp strm)
     else
     {
         state->id_len = 3;
+        state->out_blklen = strm->block_size;
         state->put_sample = put_8;
 
     }

src/aee.c

@@ -182,7 +182,7 @@ int ae_encode_init(ae_streamp strm)
     }
 
     blklen = (strm->block_size * strm->bit_per_sample
-              + state->id_len) / 8 + 2;
+              + state->id_len) / 8 + 16;
 
     state->block_out = (uint8_t *)malloc(blklen);
     if (state->block_out == NULL)

src/sz_compat.c

@@ -21,10 +21,11 @@ int SZ_BufftoBuffCompress(void *dest, size_t *destLen, const void *source, size_
     if ((status = ae_encode(&strm, AE_FLUSH)) != AE_OK)
         return status;
 
+    *destLen = strm.total_out;
+
     if ((status = ae_encode_end(&strm)) != AE_OK)
         return status;
 
-    *destLen = strm.total_out;
     return SZ_OK;
 }
 
@@ -48,9 +49,10 @@ int SZ_BufftoBuffDecompress(void *dest, size_t *destLen, const void *source, siz
     if ((status = ae_decode(&strm, AE_FLUSH)) != AE_OK)
         return status;
 
+    *destLen = strm.total_out;
+
     if ((status = ae_decode_end(&strm)) != AE_OK)
         return status;
 
-    *destLen = strm.total_out;
     return SZ_OK;
 }