Commit 7b90370f authored by Mathis Rosenhauer's avatar Mathis Rosenhauer

imitate szlib for gribex

parent 053ffc57
CC = gcc CC = gcc
#CFLAGS = -g -Wall -DPROFILE
CFLAGS = -g -Wall CFLAGS = -g -Wall
OBJS = aee.o aed.o OBJS = aee.o aed.o sz_compat.o
.PHONY : all clean test .PHONY : all clean test test_sz
all: libae.a test all: libae.a
test_encode: test_encode.o libae.a test_encode: test_encode.o libae.a
$(CC) $(CFLAGS) -o test_encode test_encode.o -L. -lae $(CC) $(CFLAGS) -o test_encode test_encode.o -L. -lae
...@@ -13,6 +14,9 @@ test_encode: test_encode.o libae.a ...@@ -13,6 +14,9 @@ test_encode: test_encode.o libae.a
test_decode: test_decode.o libae.a test_decode: test_decode.o libae.a
$(CC) $(CFLAGS) -o test_decode test_decode.o -L. -lae $(CC) $(CFLAGS) -o test_decode test_decode.o -L. -lae
test_szcomp: test_szcomp.o libae.a
$(CC) $(CFLAGS) -o test_szcomp test_szcomp.o -L. -lae
libae.a: $(OBJS) libae.a: $(OBJS)
$(AR) $(ARFLAGS) $@ $(OBJS) $(AR) $(ARFLAGS) $@ $(OBJS)
-@ ($(RANLIB) $@ || true) >/dev/null 2>&1 -@ ($(RANLIB) $@ || true) >/dev/null 2>&1
...@@ -20,9 +24,15 @@ libae.a: $(OBJS) ...@@ -20,9 +24,15 @@ libae.a: $(OBJS)
aed.o: libae.h aed.o: libae.h
aee.o: libae.h aee.o: libae.h
install: libae.a
ln -sfT ../src/szlib.h ../include/szlib.h
ln -sfT ../src/libae.h ../include/libae.h
ln -sfT ../src/libae.a ../lib/libsz.a
clean: clean:
rm -f $(OBJS) test_encode.o test_decode.o \ rm -f $(OBJS) test_encode.o test_decode.o \
test_encode test_decode libae.a \ test_encode test_decode libae.a \
test_szcomp test_szcomp.o \
../data/test.ae ../data/test ../data/test.ae ../data/test
test: test_encode test_decode test: test_encode test_decode
...@@ -32,3 +42,8 @@ test: test_encode test_decode ...@@ -32,3 +42,8 @@ test: test_encode test_decode
./test_encode 99 99 < ../data/example_data > ../data/test.ae ./test_encode 99 99 < ../data/example_data > ../data/test.ae
./test_decode 101 101 < ../data/test.ae > ../data/test ./test_decode 101 101 < ../data/test.ae > ../data/test
diff ../data/test ../data/example_data diff ../data/test ../data/example_data
test_sz: test_szcomp
./test_szcomp 65536 < ../data/example_data_16 > ../data/test
diff ../data/test ../data/example_data_16
...@@ -19,7 +19,7 @@ typedef struct internal_state { ...@@ -19,7 +19,7 @@ typedef struct internal_state {
int id_len; /* bit length of code option identification key */ int id_len; /* bit length of code option identification key */
int *id_table; /* table maps IDs to states */ int *id_table; /* table maps IDs to states */
void (*put_sample)(ae_streamp, int64_t); void (*put_sample)(ae_streamp, int64_t);
size_t ref_int; /* reference sample is every ref_int samples */ int ref_int; /* reference sample is every ref_int samples */
int64_t last_out; /* previous output for post-processing */ int64_t last_out; /* previous output for post-processing */
int64_t xmin; /* minimum integer for post-processing */ int64_t xmin; /* minimum integer for post-processing */
int64_t xmax; /* maximum integer for post-processing */ int64_t xmax; /* maximum integer for post-processing */
...@@ -34,6 +34,7 @@ typedef struct internal_state { ...@@ -34,6 +34,7 @@ typedef struct internal_state {
int fs; /* last fundamental sequence in accumulator */ int fs; /* last fundamental sequence in accumulator */
int ref; /* 1 if current block has reference sample */ int ref; /* 1 if current block has reference sample */
int pp; /* 1 if postprocessor has to be used */ int pp; /* 1 if postprocessor has to be used */
size_t samples_out;
} decode_state; } decode_state;
/* decoding table for the second-extension option */ /* decoding table for the second-extension option */
...@@ -64,20 +65,48 @@ enum ...@@ -64,20 +65,48 @@ enum
M_UNCOMP_COPY, M_UNCOMP_COPY,
}; };
#define PUTF(type) static void put_##type(ae_streamp strm, int64_t data) \ static void put_msb_32(ae_streamp strm, int64_t data)
{ \ {
strm->avail_out--; \ *strm->next_out++ = data >> 24;
strm->total_out++; \ *strm->next_out++ = data >> 16;
*(type##_t *)strm->next_out = data; \ *strm->next_out++ = data >> 8;
strm->next_out += sizeof(type##_t); \ *strm->next_out++ = data;
} strm->avail_out -= 4;
strm->total_out += 4;
}
static void put_msb_16(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data >> 8;
*strm->next_out++ = data;
strm->avail_out -= 2;
strm->total_out += 2;
}
static void put_lsb_32(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data;
*strm->next_out++ = data >> 8;
*strm->next_out++ = data >> 16;
*strm->next_out++ = data >> 24;
strm->avail_out -= 4;
strm->total_out += 4;
}
static void put_lsb_16(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data;
*strm->next_out++ = data >> 8;
strm->avail_out -= 2;
strm->total_out += 2;
}
PUTF(uint8) static void put_8(ae_streamp strm, int64_t data)
PUTF(int8) {
PUTF(uint16) *strm->next_out++ = data;
PUTF(int16) strm->avail_out--;
PUTF(uint32) strm->total_out++;
PUTF(int32) }
static inline void u_put(ae_streamp strm, int64_t sample) static inline void u_put(ae_streamp strm, int64_t sample)
{ {
...@@ -85,7 +114,7 @@ static inline void u_put(ae_streamp strm, int64_t sample) ...@@ -85,7 +114,7 @@ static inline void u_put(ae_streamp strm, int64_t sample)
decode_state *state; decode_state *state;
state = strm->state; state = strm->state;
if (state->pp && (strm->total_out % state->ref_int != 0)) if (state->pp && (state->samples_out % state->ref_int != 0))
{ {
d = sample; d = sample;
x = state->last_out; x = state->last_out;
...@@ -115,6 +144,7 @@ static inline void u_put(ae_streamp strm, int64_t sample) ...@@ -115,6 +144,7 @@ static inline void u_put(ae_streamp strm, int64_t sample)
} }
state->last_out = sample; state->last_out = sample;
state->put_sample(strm, sample); state->put_sample(strm, sample);
state->samples_out++;
} }
static inline int64_t u_get(ae_streamp strm, unsigned int n) static inline int64_t u_get(ae_streamp strm, unsigned int n)
...@@ -243,26 +273,24 @@ int ae_decode_init(ae_streamp strm) ...@@ -243,26 +273,24 @@ int ae_decode_init(ae_streamp strm)
if (strm->bit_per_sample > 16) if (strm->bit_per_sample > 16)
{ {
state->id_len = 5; state->id_len = 5;
if (strm->flags & AE_DATA_SIGNED) if (strm->flags & AE_DATA_MSB)
state->put_sample = put_int32; state->put_sample = put_msb_32;
else else
state->put_sample = put_uint32; state->put_sample = put_lsb_32;
} }
else if (strm->bit_per_sample > 8) else if (strm->bit_per_sample > 8)
{ {
state->id_len = 4; state->id_len = 4;
if (strm->flags & AE_DATA_SIGNED) if (strm->flags & AE_DATA_MSB)
state->put_sample = put_int16; state->put_sample = put_msb_16;
else else
state->put_sample = put_uint16; state->put_sample = put_lsb_16;
} }
else else
{ {
state->id_len = 3; state->id_len = 3;
if (strm->flags & AE_DATA_SIGNED) state->put_sample = put_8;
state->put_sample = put_int8;
else
state->put_sample = put_uint8;
} }
if (strm->flags & AE_DATA_SIGNED) if (strm->flags & AE_DATA_SIGNED)
...@@ -301,6 +329,7 @@ int ae_decode_init(ae_streamp strm) ...@@ -301,6 +329,7 @@ int ae_decode_init(ae_streamp strm)
strm->total_in = 0; strm->total_in = 0;
strm->total_out = 0; strm->total_out = 0;
state->samples_out = 0;
state->bitp = 0; state->bitp = 0;
state->pp = strm->flags & AE_DATA_PREPROCESS; state->pp = strm->flags & AE_DATA_PREPROCESS;
state->mode = M_ID; state->mode = M_ID;
...@@ -365,7 +394,7 @@ int ae_decode(ae_streamp strm, int flush) ...@@ -365,7 +394,7 @@ int ae_decode(ae_streamp strm, int flush)
of the states are called. Inspired by zlib. of the states are called. Inspired by zlib.
*/ */
size_t zero_blocks; int zero_blocks;
int64_t gamma, beta, ms, delta1; int64_t gamma, beta, ms, delta1;
int k; int k;
decode_state *state; decode_state *state;
...@@ -378,7 +407,7 @@ int ae_decode(ae_streamp strm, int flush) ...@@ -378,7 +407,7 @@ int ae_decode(ae_streamp strm, int flush)
{ {
case M_ID: case M_ID:
if (state->pp if (state->pp
&& (strm->total_out / strm->block_size) % strm->segment_size == 0) && (state->samples_out / strm->block_size) % strm->segment_size == 0)
state->ref = 1; state->ref = 1;
else else
state->ref = 0; state->ref = 0;
...@@ -461,7 +490,7 @@ int ae_decode(ae_streamp strm, int flush) ...@@ -461,7 +490,7 @@ int ae_decode(ae_streamp strm, int flush)
if (zero_blocks == ROS) if (zero_blocks == ROS)
{ {
zero_blocks = strm->segment_size - ( zero_blocks = strm->segment_size - (
(strm->total_out / strm->block_size) (state->samples_out / strm->block_size)
% strm->segment_size); % strm->segment_size);
} }
......
...@@ -38,6 +38,7 @@ typedef struct internal_state { ...@@ -38,6 +38,7 @@ typedef struct internal_state {
int64_t *block_in; /* input block buffer */ int64_t *block_in; /* input block buffer */
uint8_t *block_out; /* output block buffer */ uint8_t *block_out; /* output block buffer */
uint8_t *bp_out; /* pointer to current output */ uint8_t *bp_out; /* pointer to current output */
size_t total_blocks;
int bitp; /* bit pointer to the next unused bit in accumulator */ int bitp; /* bit pointer to the next unused bit in accumulator */
int block_deferred; /* there is a block in the input buffer int block_deferred; /* there is a block in the input buffer
but we first have to emit a zero block */ but we first have to emit a zero block */
...@@ -47,25 +48,69 @@ typedef struct internal_state { ...@@ -47,25 +48,69 @@ typedef struct internal_state {
int zero_ref; /* current zero block has a reference sample */ int zero_ref; /* current zero block has a reference sample */
int64_t zero_ref_sample;/* reference sample of zero block */ int64_t zero_ref_sample;/* reference sample of zero block */
int zero_blocks; /* number of contiguous zero blocks */ int zero_blocks; /* number of contiguous zero blocks */
#ifdef PROFILE
int *prof;
#endif
} encode_state; } encode_state;
static int64_t get_lsb_32(ae_streamp strm)
{
int64_t data;
#define GETF(type) static int64_t get_##type(ae_streamp strm) \ data = *strm->next_in++;
{ \ data |= *strm->next_in++ << 8;
int64_t data; \ data |= *strm->next_in++ << 16;
strm->avail_in--; \ data |= *strm->next_in++ << 24;
strm->total_in++; \
data = *(type##_t *)strm->next_in; \ strm->avail_in -= 4;
strm->next_in += sizeof(type##_t); \ strm->total_in += 4;
return data; \ return data;
} }
static int64_t get_lsb_16(ae_streamp strm)
{
int64_t data;
data = *strm->next_in++;
data |= *strm->next_in++ << 8;
strm->avail_in -= 2;
strm->total_in += 2;
return data;
}
static int64_t get_msb_32(ae_streamp strm)
{
int64_t data;
GETF(uint8) data = *strm->next_in++ << 24;
GETF(int8) data |= *strm->next_in++ << 16;
GETF(uint16) data |= *strm->next_in++ << 8;
GETF(int16) data |= *strm->next_in++;
GETF(uint32)
GETF(int32) strm->avail_in -= 4;
strm->total_in += 4;
return data;
}
static int64_t get_msb_16(ae_streamp strm)
{
int64_t data;
data = *strm->next_in++ << 8;
data |= *strm->next_in++;
strm->avail_in -= 2;
strm->total_in += 2;
return data;
}
static int64_t get_8(ae_streamp strm)
{
strm->avail_in--;
strm->total_in++;
return *strm->next_in++;
}
int ae_encode_init(ae_streamp strm) int ae_encode_init(ae_streamp strm)
{ {
...@@ -89,26 +134,23 @@ int ae_encode_init(ae_streamp strm) ...@@ -89,26 +134,23 @@ int ae_encode_init(ae_streamp strm)
if (strm->bit_per_sample > 16) if (strm->bit_per_sample > 16)
{ {
state->id_len = 5; state->id_len = 5;
if (strm->flags & AE_DATA_SIGNED) if (strm->flags & AE_DATA_MSB)
state->get_sample = get_int32; state->get_sample = get_msb_32;
else else
state->get_sample = get_uint32; state->get_sample = get_lsb_32;
} }
else if (strm->bit_per_sample > 8) else if (strm->bit_per_sample > 8)
{ {
state->id_len = 4; state->id_len = 4;
if (strm->flags & AE_DATA_SIGNED) if (strm->flags & AE_DATA_MSB)
state->get_sample = get_int16; state->get_sample = get_msb_16;
else else
state->get_sample = get_uint16; state->get_sample = get_lsb_16;
} }
else else
{ {
state->id_len = 3; state->id_len = 3;
if (strm->flags & AE_DATA_SIGNED) state->get_sample = get_8;
state->get_sample = get_int8;
else
state->get_sample = get_uint8;
} }
if (strm->flags & AE_DATA_SIGNED) if (strm->flags & AE_DATA_SIGNED)
...@@ -122,6 +164,15 @@ int ae_encode_init(ae_streamp strm) ...@@ -122,6 +164,15 @@ int ae_encode_init(ae_streamp strm)
state->xmax = (1ULL << strm->bit_per_sample) - 1; state->xmax = (1ULL << strm->bit_per_sample) - 1;
} }
#ifdef PROFILE
state->prof = (int *)malloc((strm->bit_per_sample + 2) * sizeof(int));
if (state->prof == NULL)
{
return AE_MEM_ERROR;
}
memset(state->prof, 0, (strm->bit_per_sample + 2) * sizeof(int));
#endif
state->block_in = (int64_t *)malloc(strm->block_size * sizeof(int64_t)); state->block_in = (int64_t *)malloc(strm->block_size * sizeof(int64_t));
if (state->block_in == NULL) if (state->block_in == NULL)
{ {
...@@ -144,6 +195,7 @@ int ae_encode_init(ae_streamp strm) ...@@ -144,6 +195,7 @@ int ae_encode_init(ae_streamp strm)
state->mode = M_NEW_BLOCK; state->mode = M_NEW_BLOCK;
state->total_blocks = 0;
state->block_deferred = 0; state->block_deferred = 0;
state->zero_ref = 0; state->zero_ref = 0;
state->ref = 0; state->ref = 0;
...@@ -155,7 +207,7 @@ static inline void emit(encode_state *state, int64_t data, int bits) ...@@ -155,7 +207,7 @@ static inline void emit(encode_state *state, int64_t data, int bits)
{ {
while(bits) while(bits)
{ {
data &= ((1UL << bits) - 1); data &= ((1ULL << bits) - 1);
if (bits <= state->bitp) if (bits <= state->bitp)
{ {
data <<= state->bitp - bits; data <<= state->bitp - bits;
...@@ -179,6 +231,31 @@ static inline void emitfs(encode_state *state, int fs) ...@@ -179,6 +231,31 @@ static inline void emitfs(encode_state *state, int fs)
emit(state, 1, 1); emit(state, 1, 1);
} }
#ifdef PROFILE
static inline void profile_print(ae_streamp strm)
{
int i, total;
encode_state *state;
state = strm->state;
fprintf(stderr, "Blocks encoded by each coding option\n");
fprintf(stderr, "Zero blocks: %i\n", state->prof[0]);
total = state->prof[0];
fprintf(stderr, "Second Ext.: %i\n", state->prof[strm->bit_per_sample+1]);
total += state->prof[strm->bit_per_sample+1];
fprintf(stderr, "FS: %i\n", state->prof[1]);
total += state->prof[1];
for (i = 2; i < strm->bit_per_sample - 1; i++)
{
fprintf(stderr, "k = %02i: %i\n", i-1, state->prof[i]);
total += state->prof[i];
}
fprintf(stderr, "Uncompressed: %i\n", state->prof[strm->bit_per_sample]);
total += state->prof[strm->bit_per_sample];
fprintf(stderr, "Total blocks: %i\n", total);
}
#endif
int ae_encode(ae_streamp strm, int flush) int ae_encode(ae_streamp strm, int flush)
{ {
/** /**
...@@ -186,9 +263,9 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -186,9 +263,9 @@ int ae_encode(ae_streamp strm, int flush)
encoder. encoder.
*/ */
int i, j, zb; int i, j, k, zb;
int k_len[strm->bit_per_sample - 2]; int64_t k_len[strm->bit_per_sample - 2];
int k, k_min, se_len, blk_head; int64_t k_len_min, se_len, blk_head;
int64_t d; int64_t d;
int64_t theta, Delta; int64_t theta, Delta;
...@@ -229,7 +306,8 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -229,7 +306,8 @@ int ae_encode(ae_streamp strm, int flush)
{ {
/* pad block with zeros if we have /* pad block with zeros if we have
a partial block */ a partial block */
state->block_in[state->i] = 0; state->block_in[state->i] = state->block_in[state->i - 1];
fprintf(stderr, "padding %lx\n", state->block_in[state->i]);
} }
else else
{ {
...@@ -241,9 +319,16 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -241,9 +319,16 @@ int ae_encode(ae_streamp strm, int flush)
*strm->next_out++ = *state->bp_out; *strm->next_out++ = *state->bp_out;
strm->avail_out--; strm->avail_out--;
strm->total_out++; strm->total_out++;
#ifdef PROFILE
profile_print(strm);
#endif
goto req_buffer;
} }
} }
goto req_buffer; else
{
goto req_buffer;
}
} }
else else
{ {
...@@ -258,7 +343,7 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -258,7 +343,7 @@ int ae_encode(ae_streamp strm, int flush)
/* If this is the first block in a segment /* If this is the first block in a segment
then we need to insert a reference sample. then we need to insert a reference sample.
*/ */
if((strm->total_in / strm->block_size) % strm->segment_size == 1) if(state->total_blocks % strm->segment_size == 0)
{ {
state->ref = 1; state->ref = 1;
state->last_in = state->block_in[0]; state->last_in = state->block_in[0];
...@@ -272,8 +357,7 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -272,8 +357,7 @@ int ae_encode(ae_streamp strm, int flush)
{ {
theta = MIN(state->last_in - state->xmin, theta = MIN(state->last_in - state->xmin,
state->xmax - state->last_in); state->xmax - state->last_in);
Delta = (long long)state->block_in[i] - (long long)state->last_in; Delta = state->block_in[i] - state->last_in;
if (0 <= Delta && Delta <= theta) if (0 <= Delta && Delta <= theta)
d = 2 * Delta; d = 2 * Delta;
else if (-theta <= Delta && Delta < 0) else if (-theta <= Delta && Delta < 0)
...@@ -285,10 +369,10 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -285,10 +369,10 @@ int ae_encode(ae_streamp strm, int flush)
state->block_in[i] = d; state->block_in[i] = d;
} }
} }
state->total_blocks++;
state->mode = M_CHECK_ZERO_BLOCK; state->mode = M_CHECK_ZERO_BLOCK;
case M_CHECK_ZERO_BLOCK: case M_CHECK_ZERO_BLOCK:
/* Check zero block */
zb = 1; zb = 1;
for (i = state->ref; i < strm->block_size && zb; i++) for (i = state->ref; i < strm->block_size && zb; i++)
if (state->block_in[i] != 0) zb = 0; if (state->block_in[i] != 0) zb = 0;
...@@ -309,6 +393,9 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -309,6 +393,9 @@ int ae_encode(ae_streamp strm, int flush)
{ {
if (state->zero_blocks > ROS) if (state->zero_blocks > ROS)
state->zero_blocks = ROS; state->zero_blocks = ROS;
#ifdef PROFILE
state->prof[0] += state->zero_blocks;
#endif
state->mode = M_ENCODE_ZERO; state->mode = M_ENCODE_ZERO;
break; break;
} }
...@@ -317,6 +404,9 @@ int ae_encode(ae_streamp strm, int flush) ...@@ -317,6 +404,9 @@ int ae_encode(ae_streamp strm, int flush)
} }
else if (state->zero_blocks) else if (state->zero_blocks)
{ {
#ifdef PROFILE
<