Skip to content

GitLab

Commit 7b90370f authored by Mathis Rosenhauer's avatar Mathis Rosenhauer
Browse files

imitate szlib for gribex

parent 053ffc57
Hide whitespace changes
Inline Side-by-side
Showing with 405 additions and 89 deletions
src/Makefile
View file @ 7b90370f
CC = gcc
#CFLAGS = -g -Wall -DPROFILE
CFLAGS = -g -Wall
OBJS = aee.o aed.o
OBJS = aee.o aed.o sz_compat.o
.PHONY : all clean test
.PHONY : all clean test test_sz
all: libae.a test
all: libae.a
test_encode: test_encode.o libae.a
$(CC) $(CFLAGS) -o test_encode test_encode.o -L. -lae
......@@ -13,6 +14,9 @@ test_encode: test_encode.o libae.a
test_decode: test_decode.o libae.a
$(CC) $(CFLAGS) -o test_decode test_decode.o -L. -lae
test_szcomp: test_szcomp.o libae.a
$(CC) $(CFLAGS) -o test_szcomp test_szcomp.o -L. -lae
libae.a: $(OBJS)
$(AR) $(ARFLAGS) $@ $(OBJS)
-@ ($(RANLIB) $@ || true) >/dev/null 2>&1
......@@ -20,9 +24,15 @@ libae.a: $(OBJS)
aed.o: libae.h
aee.o: libae.h
install: libae.a
ln -sfT ../src/szlib.h ../include/szlib.h
ln -sfT ../src/libae.h ../include/libae.h
ln -sfT ../src/libae.a ../lib/libsz.a
clean:
rm -f $(OBJS) test_encode.o test_decode.o \
test_encode test_decode libae.a \
test_szcomp test_szcomp.o \
../data/test.ae ../data/test
test: test_encode test_decode
......@@ -32,3 +42,8 @@ test: test_encode test_decode
./test_encode 99 99 < ../data/example_data > ../data/test.ae
./test_decode 101 101 < ../data/test.ae > ../data/test
diff ../data/test ../data/example_data
test_sz: test_szcomp
./test_szcomp 65536 < ../data/example_data_16 > ../data/test
diff ../data/test ../data/example_data_16
src/aed.c
View file @ 7b90370f
......@@ -19,7 +19,7 @@ typedef struct internal_state {
int id_len; /* bit length of code option identification key */
int *id_table; /* table maps IDs to states */
void (*put_sample)(ae_streamp, int64_t);
size_t ref_int; /* reference sample is every ref_int samples */
int ref_int; /* reference sample is every ref_int samples */
int64_t last_out; /* previous output for post-processing */
int64_t xmin; /* minimum integer for post-processing */
int64_t xmax; /* maximum integer for post-processing */
......@@ -34,6 +34,7 @@ typedef struct internal_state {
int fs; /* last fundamental sequence in accumulator */
int ref; /* 1 if current block has reference sample */
int pp; /* 1 if postprocessor has to be used */
size_t samples_out;
} decode_state;
/* decoding table for the second-extension option */
......@@ -64,20 +65,48 @@ enum
M_UNCOMP_COPY,
};
#define PUTF(type) static void put_##type(ae_streamp strm, int64_t data) \
{ \
strm->avail_out--; \
strm->total_out++; \
*(type##_t *)strm->next_out = data; \
strm->next_out += sizeof(type##_t); \
}
static void put_msb_32(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data >> 24;
*strm->next_out++ = data >> 16;
*strm->next_out++ = data >> 8;
*strm->next_out++ = data;
strm->avail_out -= 4;
strm->total_out += 4;
}
static void put_msb_16(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data >> 8;
*strm->next_out++ = data;
strm->avail_out -= 2;
strm->total_out += 2;
}
static void put_lsb_32(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data;
*strm->next_out++ = data >> 8;
*strm->next_out++ = data >> 16;
*strm->next_out++ = data >> 24;
strm->avail_out -= 4;
strm->total_out += 4;
}
static void put_lsb_16(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data;
*strm->next_out++ = data >> 8;
strm->avail_out -= 2;
strm->total_out += 2;
}
PUTF(uint8)
PUTF(int8)
PUTF(uint16)
PUTF(int16)
PUTF(uint32)
PUTF(int32)
static void put_8(ae_streamp strm, int64_t data)
{
*strm->next_out++ = data;
strm->avail_out--;
strm->total_out++;
}
static inline void u_put(ae_streamp strm, int64_t sample)
{
......@@ -85,7 +114,7 @@ static inline void u_put(ae_streamp strm, int64_t sample)
decode_state *state;
state = strm->state;
if (state->pp && (strm->total_out % state->ref_int != 0))
if (state->pp && (state->samples_out % state->ref_int != 0))
{
d = sample;
x = state->last_out;
......@@ -115,6 +144,7 @@ static inline void u_put(ae_streamp strm, int64_t sample)
}
state->last_out = sample;
state->put_sample(strm, sample);
state->samples_out++;
}
static inline int64_t u_get(ae_streamp strm, unsigned int n)
......@@ -243,26 +273,24 @@ int ae_decode_init(ae_streamp strm)
if (strm->bit_per_sample > 16)
{
state->id_len = 5;
if (strm->flags & AE_DATA_SIGNED)
state->put_sample = put_int32;
if (strm->flags & AE_DATA_MSB)
state->put_sample = put_msb_32;
else
state->put_sample = put_uint32;
state->put_sample = put_lsb_32;
}
else if (strm->bit_per_sample > 8)
{
state->id_len = 4;
if (strm->flags & AE_DATA_SIGNED)
state->put_sample = put_int16;
if (strm->flags & AE_DATA_MSB)
state->put_sample = put_msb_16;
else
state->put_sample = put_uint16;
state->put_sample = put_lsb_16;
}
else
{
state->id_len = 3;
if (strm->flags & AE_DATA_SIGNED)
state->put_sample = put_int8;
else
state->put_sample = put_uint8;
state->put_sample = put_8;
}
if (strm->flags & AE_DATA_SIGNED)
......@@ -301,6 +329,7 @@ int ae_decode_init(ae_streamp strm)
strm->total_in = 0;
strm->total_out = 0;
state->samples_out = 0;
state->bitp = 0;
state->pp = strm->flags & AE_DATA_PREPROCESS;
state->mode = M_ID;
......@@ -365,7 +394,7 @@ int ae_decode(ae_streamp strm, int flush)
of the states are called. Inspired by zlib.
*/
size_t zero_blocks;
int zero_blocks;
int64_t gamma, beta, ms, delta1;
int k;
decode_state *state;
......@@ -378,7 +407,7 @@ int ae_decode(ae_streamp strm, int flush)
{
case M_ID:
if (state->pp
&& (strm->total_out / strm->block_size) % strm->segment_size == 0)
&& (state->samples_out / strm->block_size) % strm->segment_size == 0)
state->ref = 1;
else
state->ref = 0;
......@@ -461,7 +490,7 @@ int ae_decode(ae_streamp strm, int flush)
if (zero_blocks == ROS)
{
zero_blocks = strm->segment_size - (
(strm->total_out / strm->block_size)
(state->samples_out / strm->block_size)
% strm->segment_size);
}
......
src/aee.c
View file @ 7b90370f
......@@ -38,6 +38,7 @@ typedef struct internal_state {
int64_t *block_in; /* input block buffer */
uint8_t *block_out; /* output block buffer */
uint8_t *bp_out; /* pointer to current output */
size_t total_blocks;
int bitp; /* bit pointer to the next unused bit in accumulator */
int block_deferred; /* there is a block in the input buffer
but we first have to emit a zero block */
......@@ -47,25 +48,69 @@ typedef struct internal_state {
int zero_ref; /* current zero block has a reference sample */
int64_t zero_ref_sample;/* reference sample of zero block */
int zero_blocks; /* number of contiguous zero blocks */
#ifdef PROFILE
int *prof;
#endif
} encode_state;
static int64_t get_lsb_32(ae_streamp strm)
{
int64_t data;
#define GETF(type) static int64_t get_##type(ae_streamp strm) \
{ \
int64_t data; \
strm->avail_in--; \
strm->total_in++; \
data = *(type##_t *)strm->next_in; \
strm->next_in += sizeof(type##_t); \
return data; \
}
data = *strm->next_in++;
data |= *strm->next_in++ << 8;
data |= *strm->next_in++ << 16;
data |= *strm->next_in++ << 24;
strm->avail_in -= 4;
strm->total_in += 4;
return data;
}
static int64_t get_lsb_16(ae_streamp strm)
{
int64_t data;
data = *strm->next_in++;
data |= *strm->next_in++ << 8;
strm->avail_in -= 2;
strm->total_in += 2;
return data;
}
static int64_t get_msb_32(ae_streamp strm)
{
int64_t data;
GETF(uint8)
GETF(int8)
GETF(uint16)
GETF(int16)
GETF(uint32)
GETF(int32)
data = *strm->next_in++ << 24;
data |= *strm->next_in++ << 16;
data |= *strm->next_in++ << 8;
data |= *strm->next_in++;
strm->avail_in -= 4;
strm->total_in += 4;
return data;
}
static int64_t get_msb_16(ae_streamp strm)
{
int64_t data;
data = *strm->next_in++ << 8;
data |= *strm->next_in++;
strm->avail_in -= 2;
strm->total_in += 2;
return data;
}
static int64_t get_8(ae_streamp strm)
{
strm->avail_in--;
strm->total_in++;
return *strm->next_in++;
}
int ae_encode_init(ae_streamp strm)
{
......@@ -89,26 +134,23 @@ int ae_encode_init(ae_streamp strm)
if (strm->bit_per_sample > 16)
{
state->id_len = 5;
if (strm->flags & AE_DATA_SIGNED)
state->get_sample = get_int32;
if (strm->flags & AE_DATA_MSB)
state->get_sample = get_msb_32;
else
state->get_sample = get_uint32;
state->get_sample = get_lsb_32;
}
else if (strm->bit_per_sample > 8)
{
state->id_len = 4;
if (strm->flags & AE_DATA_SIGNED)
state->get_sample = get_int16;
if (strm->flags & AE_DATA_MSB)
state->get_sample = get_msb_16;
else
state->get_sample = get_uint16;
state->get_sample = get_lsb_16;
}
else
{
state->id_len = 3;
if (strm->flags & AE_DATA_SIGNED)
state->get_sample = get_int8;
else
state->get_sample = get_uint8;
state->get_sample = get_8;
}
if (strm->flags & AE_DATA_SIGNED)
......@@ -122,6 +164,15 @@ int ae_encode_init(ae_streamp strm)
state->xmax = (1ULL << strm->bit_per_sample) - 1;
}
#ifdef PROFILE
state->prof = (int *)malloc((strm->bit_per_sample + 2) * sizeof(int));
if (state->prof == NULL)
{
return AE_MEM_ERROR;
}
memset(state->prof, 0, (strm->bit_per_sample + 2) * sizeof(int));
#endif
state->block_in = (int64_t *)malloc(strm->block_size * sizeof(int64_t));
if (state->block_in == NULL)
{
......@@ -144,6 +195,7 @@ int ae_encode_init(ae_streamp strm)
state->mode = M_NEW_BLOCK;
state->total_blocks = 0;
state->block_deferred = 0;
state->zero_ref = 0;
state->ref = 0;
......@@ -155,7 +207,7 @@ static inline void emit(encode_state *state, int64_t data, int bits)
{
while(bits)
{
data &= ((1UL << bits) - 1);
data &= ((1ULL << bits) - 1);
if (bits <= state->bitp)
{
data <<= state->bitp - bits;
......@@ -179,6 +231,31 @@ static inline void emitfs(encode_state *state, int fs)
emit(state, 1, 1);
}
#ifdef PROFILE
static inline void profile_print(ae_streamp strm)
{
int i, total;
encode_state *state;
state = strm->state;
fprintf(stderr, "Blocks encoded by each coding option\n");
fprintf(stderr, "Zero blocks: %i\n", state->prof[0]);
total = state->prof[0];
fprintf(stderr, "Second Ext.: %i\n", state->prof[strm->bit_per_sample+1]);
total += state->prof[strm->bit_per_sample+1];
fprintf(stderr, "FS: %i\n", state->prof[1]);
total += state->prof[1];
for (i = 2; i < strm->bit_per_sample - 1; i++)
{
fprintf(stderr, "k = %02i: %i\n", i-1, state->prof[i]);
total += state->prof[i];
}
fprintf(stderr, "Uncompressed: %i\n", state->prof[strm->bit_per_sample]);
total += state->prof[strm->bit_per_sample];
fprintf(stderr, "Total blocks: %i\n", total);
}
#endif
int ae_encode(ae_streamp strm, int flush)
{
/**
......@@ -186,9 +263,9 @@ int ae_encode(ae_streamp strm, int flush)
encoder.
*/
int i, j, zb;
int k_len[strm->bit_per_sample - 2];
int k, k_min, se_len, blk_head;
int i, j, k, zb;
int64_t k_len[strm->bit_per_sample - 2];
int64_t k_len_min, se_len, blk_head;
int64_t d;
int64_t theta, Delta;
......@@ -229,7 +306,8 @@ int ae_encode(ae_streamp strm, int flush)
{
/* pad block with zeros if we have
a partial block */
state->block_in[state->i] = 0;
state->block_in[state->i] = state->block_in[state->i - 1];
fprintf(stderr, "padding %lx\n", state->block_in[state->i]);
}
else
{
......@@ -241,9 +319,16 @@ int ae_encode(ae_streamp strm, int flush)
*strm->next_out++ = *state->bp_out;
strm->avail_out--;
strm->total_out++;
#ifdef PROFILE
profile_print(strm);
#endif
goto req_buffer;
}
}
goto req_buffer;
else
{
goto req_buffer;
}
}
else
{
......@@ -258,7 +343,7 @@ int ae_encode(ae_streamp strm, int flush)
/* If this is the first block in a segment
then we need to insert a reference sample.
*/
if((strm->total_in / strm->block_size) % strm->segment_size == 1)
if(state->total_blocks % strm->segment_size == 0)
{
state->ref = 1;
state->last_in = state->block_in[0];
......@@ -272,8 +357,7 @@ int ae_encode(ae_streamp strm, int flush)
{
theta = MIN(state->last_in - state->xmin,
state->xmax - state->last_in);
Delta = (long long)state->block_in[i] - (long long)state->last_in;
Delta = state->block_in[i] - state->last_in;
if (0 <= Delta && Delta <= theta)
d = 2 * Delta;
else if (-theta <= Delta && Delta < 0)
......@@ -285,10 +369,10 @@ int ae_encode(ae_streamp strm, int flush)
state->block_in[i] = d;
}
}
state->total_blocks++;
state->mode = M_CHECK_ZERO_BLOCK;
case M_CHECK_ZERO_BLOCK:
/* Check zero block */
zb = 1;
for (i = state->ref; i < strm->block_size && zb; i++)
if (state->block_in[i] != 0) zb = 0;
......@@ -309,6 +393,9 @@ int ae_encode(ae_streamp strm, int flush)
{
if (state->zero_blocks > ROS)
state->zero_blocks = ROS;
#ifdef PROFILE
state->prof[0] += state->zero_blocks;
#endif
state->mode = M_ENCODE_ZERO;
break;
}
......@@ -317,6 +404,9 @@ int ae_encode(ae_streamp strm, int flush)
}
else if (state->zero_blocks)
{
#ifdef PROFILE
state->prof[0] += state->zero_blocks;
#endif
state->mode = M_ENCODE_ZERO;
/* The current block isn't zero but we have to
emit a previous zero block first. The
......@@ -329,11 +419,15 @@ int ae_encode(ae_streamp strm, int flush)
state->mode = M_SELECT_CODE_OPTION;
case M_SELECT_CODE_OPTION:
/* Encoded block always starts with ID and possibly
a reference sample. */
blk_head = state->id_len;
/* If zero block isn't an option then count length of
sample splitting options */
/* Encoded block always starts with ID and possibly a
reference sample. */
if (state->ref)
blk_head += strm->bit_per_sample;
blk_head = state->id_len + strm->bit_per_sample;
else
blk_head = state->id_len;
for (j = 0; j < strm->bit_per_sample - 2; j++)
k_len[j] = blk_head;
......@@ -344,43 +438,56 @@ int ae_encode(ae_streamp strm, int flush)
k_len[j] += (state->block_in[i] >> j) + 1 + j;
/* Baseline is the size of an uncompressed block */
k_min = state->id_len + strm->block_size * strm->bit_per_sample;
k_len_min = state->id_len + strm->block_size * strm->bit_per_sample;
k = strm->bit_per_sample;
/* See if splitting option is better */
for (j = 0; j < strm->bit_per_sample - 2; j++)
{
if (k_len[j] < k_min)
if (k_len[j] < k_len_min)
{
k = j;
k_min = k_len[j];
k_len_min = k_len[j];
}
}
/* Count bits for 2nd extension */
se_len = blk_head + 1;
for (i = 0; i < strm->block_size && k_min > se_len; i+= 2)
for (i = 0; i < strm->block_size && k_len_min > se_len; i+= 2)
{
d = state->block_in[i] + state->block_in[i + 1];
se_len += d * (d + 1) / 2 + state->block_in[i + 1];
/* we have to worry about overflow here */
if (d > k_len_min)
se_len = d;
else
se_len += d * (d + 1) / 2 + state->block_in[i + 1];
}
/* Decide which option to use */
if (k_min <= se_len)
if (k_len_min <= se_len)
{
if (k == strm->bit_per_sample)
{
#ifdef PROFILE
state->prof[k]++;
#endif
state->mode = M_ENCODE_UNCOMP;
break;
}
else
{
#ifdef PROFILE
state->prof[k + 1]++;
#endif
state->mode = M_ENCODE_SPLIT;
}
}
else
{
#ifdef PROFILE
state->prof[strm->bit_per_sample + 1]++;
#endif
state->mode = M_ENCODE_SE;
break;
}
......@@ -403,6 +510,9 @@ int ae_encode(ae_streamp strm, int flush)
{
/* pad last byte with 1 bits */
emit(state, 0xff, state->bitp);
#ifdef PROFILE
profile_print(strm);
#endif
}
state->i = 0;
state->mode = M_FLUSH_BLOCK_LOOP;
......@@ -411,7 +521,12 @@ int ae_encode(ae_streamp strm, int flush)
while(state->block_out + state->i < state->bp_out)
{
if (strm->avail_out == 0)
{
#ifdef PROFILE
profile_print(strm);
#endif
goto req_buffer;
}
*strm->next_out++ = state->block_out[state->i];
strm->avail_out--;
......@@ -430,7 +545,7 @@ int ae_encode(ae_streamp strm, int flush)
break;
case M_ENCODE_SE:
emit(state, 1, state->id_len + 1);
emit(state, 1, state->id_len + 1);
if (state->ref)
emit(state, state->block_in[0], strm->bit_per_sample);
......
src/libae.h
View file @ 7b90370f
......@@ -28,9 +28,11 @@ typedef struct _ae_stream
typedef ae_stream *ae_streamp;
/* Coder flags */
#define AE_DATA_UNSIGNED 0
#define AE_DATA_SIGNED 1
#define AE_DATA_PREPROCESS 2 /* Set if preprocessor should be used */
#define AE_DATA_UNSIGNED 0
#define AE_DATA_SIGNED 1
#define AE_DATA_LSB 8
#define AE_DATA_MSB 16
#define AE_DATA_PREPROCESS 32 /* Set if preprocessor should be used */
/* Return codes of library functions */
......
src/sz_compat.c 0 → 100644
View file @ 7b90370f
#include <stddef.h>
#include "szlib.h"
int SZ_BufftoBuffCompress(void *dest, size_t *destLen, const void *source, size_t sourceLen, SZ_com_t *param)
{
int status;
ae_stream strm;
strm.bit_per_sample = param->bits_per_pixel;
strm.block_size = param->pixels_per_block;
strm.segment_size = param->pixels_per_scanline / param->pixels_per_block;
strm.flags = param->options_mask;
strm.avail_in = sourceLen;
strm.avail_out = *destLen;
strm.next_out = dest;
strm.next_in = source;
if ((status = ae_encode_init(&strm)) != AE_OK)