Skip to content

GitLab

Commit 053ffc57 authored by Mathis Rosenhauer's avatar Mathis Rosenhauer
Browse files
Options

support all data sizes and signedness

parent 92bd99e7
Hide whitespace changes
Inline Side-by-side
Showing with 187 additions and 87 deletions
src/Makefile
View file @ 053ffc57
......@@ -29,3 +29,6 @@ test: test_encode test_decode
./test_encode 1 1 < ../data/example_data > ../data/test.ae
./test_decode 1 1 < ../data/test.ae > ../data/test
diff ../data/test ../data/example_data
./test_encode 99 99 < ../data/example_data > ../data/test.ae
./test_decode 101 101 < ../data/test.ae > ../data/test
diff ../data/test ../data/example_data
src/aed.c
View file @ 053ffc57
......@@ -15,29 +15,29 @@
#define ROS 5
typedef struct internal_state {
const uint8_t *next_in;
uint32_t *next_out;
int id; /* option ID */
uint32_t id_len; /* bit length of code option identification key */
int id_len; /* bit length of code option identification key */
int *id_table; /* table maps IDs to states */
void (*put_sample)(ae_streamp, int64_t);
size_t ref_int; /* reference sample is every ref_int samples */
uint32_t last_out; /* previous output for post-processing */
int64_t last_out; /* previous output for post-processing */
int64_t xmin; /* minimum integer for post-processing */
int64_t xmax; /* maximum integer for post-processing */
int mode; /* current mode of FSM */
size_t in_blklen; /* length of uncompressed input block
int in_blklen; /* length of uncompressed input block
should be the longest possible block */
size_t n, i; /* counter for samples */
uint32_t *block; /* block buffer for split-sample options */
int n, i; /* counter for samples */
int64_t *block; /* block buffer for split-sample options */
int se; /* set if second extension option is selected */
uint64_t acc; /* accumulator for currently used bit sequence */
uint8_t bitp; /* bit pointer to the next unused bit in accumulator */
uint32_t fs; /* last fundamental sequence in accumulator */
int bitp; /* bit pointer to the next unused bit in accumulator */
int fs; /* last fundamental sequence in accumulator */
int ref; /* 1 if current block has reference sample */
int pp; /* 1 if postprocessor has to be used */
} decode_state;
/* decoding table for the second-extension option */
static const uint32_t second_extension[36][2] = {
static const int second_extension[36][2] = {
{0, 0},
{1, 1}, {1, 1},
{2, 3}, {2, 3}, {2, 3},
......@@ -64,13 +64,28 @@ enum
M_UNCOMP_COPY,
};
static inline void u_put(ae_streamp strm, uint32_t sample)
#define PUTF(type) static void put_##type(ae_streamp strm, int64_t data) \
{ \
strm->avail_out--; \
strm->total_out++; \
*(type##_t *)strm->next_out = data; \
strm->next_out += sizeof(type##_t); \
}
PUTF(uint8)
PUTF(int8)
PUTF(uint16)
PUTF(int16)
PUTF(uint32)
PUTF(int32)
static inline void u_put(ae_streamp strm, int64_t sample)
{
int64_t x, d, th, D;
decode_state *state;
state = strm->state;
if (strm->pp && (strm->total_out % state->ref_int != 0))
if (state->pp && (strm->total_out % state->ref_int != 0))
{
d = sample;
x = state->last_out;
......@@ -98,12 +113,11 @@ static inline void u_put(ae_streamp strm, uint32_t sample)
}
sample = x + D;
}
*state->next_out++ = state->last_out = sample;
strm->avail_out--;
strm->total_out++;
state->last_out = sample;
state->put_sample(strm, sample);
}
static inline uint32_t u_get(ae_streamp strm, unsigned int n)
static inline int64_t u_get(ae_streamp strm, unsigned int n)
{
/**
Unsafe get n bit from input stream
......@@ -118,14 +132,14 @@ static inline uint32_t u_get(ae_streamp strm, unsigned int n)
{
strm->avail_in--;
strm->total_in++;
state->acc = (state->acc << 8) + *state->next_in++;
state->acc = (state->acc << 8) | *strm->next_in++;
state->bitp += 8;
}
state->bitp -= n;
return (state->acc >> state->bitp) & ((1ULL << n) - 1);
}
static inline uint32_t u_get_fs(ae_streamp strm)
static inline int u_get_fs(ae_streamp strm)
{
/**
Interpret a Fundamental Sequence from the input buffer.
......@@ -134,7 +148,7 @@ static inline uint32_t u_get_fs(ae_streamp strm)
1 is encountered. TODO: faster version.
*/
uint32_t fs = 0;
int fs = 0;
while(u_get(strm, 1) == 0)
fs++;
......@@ -178,11 +192,11 @@ static inline void fast_zero(ae_streamp strm)
static inline void fast_se(ae_streamp strm)
{
int i;
uint32_t gamma, beta, ms, delta1;
int64_t gamma, beta, ms, delta1;
i = strm->state->ref;
while (i < strm->bit_per_sample)
while (i < strm->block_size)
{
gamma = u_get_fs(strm);
beta = second_extension[gamma][0];
......@@ -226,12 +240,41 @@ int ae_decode_init(ae_streamp strm)
}
strm->state = state;
if (16 < strm->bit_per_sample)
if (strm->bit_per_sample > 16)
{
state->id_len = 5;
else if (8 < strm->bit_per_sample)
if (strm->flags & AE_DATA_SIGNED)
state->put_sample = put_int32;
else
state->put_sample = put_uint32;
}
else if (strm->bit_per_sample > 8)
{
state->id_len = 4;
if (strm->flags & AE_DATA_SIGNED)
state->put_sample = put_int16;
else
state->put_sample = put_uint16;
}
else
{
state->id_len = 3;
if (strm->flags & AE_DATA_SIGNED)
state->put_sample = put_int8;
else
state->put_sample = put_uint8;
}
if (strm->flags & AE_DATA_SIGNED)
{
state->xmin = -(1ULL << (strm->bit_per_sample - 1));
state->xmax = (1ULL << (strm->bit_per_sample - 1)) - 1;
}
else
{
state->xmin = 0;
state->xmax = (1ULL << strm->bit_per_sample) - 1;
}
state->ref_int = strm->block_size * strm->segment_size;
state->in_blklen = (strm->block_size * strm->bit_per_sample
......@@ -250,17 +293,16 @@ int ae_decode_init(ae_streamp strm)
}
state->id_table[modi - 1] = M_UNCOMP;
state->block = (uint32_t *)malloc(strm->block_size * sizeof(uint32_t));
state->block = (int64_t *)malloc(strm->block_size * sizeof(int64_t));
if (state->block == NULL)
{
return AE_MEM_ERROR;
}
strm->total_in = 0;
strm->total_out = 0;
state->xmin = 0;
state->xmax = (1ULL << strm->bit_per_sample) - 1;
state->bitp = 0;
state->pp = strm->flags & AE_DATA_PREPROCESS;
state->mode = M_ID;
return AE_OK;
}
......@@ -273,7 +315,7 @@ int ae_decode_init(ae_streamp strm)
strm->avail_in--; \
strm->total_in++; \
state->acc <<= 8; \
state->acc |= (uint64_t)(*state->next_in++); \
state->acc |= *strm->next_in++; \
state->bitp += 8; \
} \
} while (0)
......@@ -324,20 +366,18 @@ int ae_decode(ae_streamp strm, int flush)
*/
size_t zero_blocks;
uint32_t gamma, beta, ms, delta1;
int64_t gamma, beta, ms, delta1;
int k;
decode_state *state;
state = strm->state;
state->next_in = strm->next_in;
state->next_out = strm->next_out;
for (;;)
{
switch(state->mode)
{
case M_ID:
if (strm->pp
if (state->pp
&& (strm->total_out / strm->block_size) % strm->segment_size == 0)
state->ref = 1;
else
......@@ -460,7 +500,7 @@ int ae_decode(ae_streamp strm, int flush)
state->i = state->ref;
case M_SE_DECODE:
while(state->i < strm->bit_per_sample)
while(state->i < strm->block_size)
{
ASKFS();
gamma = GETFS();
......@@ -507,7 +547,5 @@ int ae_decode(ae_streamp strm, int flush)
}
req_buffer:
strm->next_in = state->next_in;
strm->next_out = state->next_out;
return AE_OK;
}
src/aee.c
View file @ 053ffc57
......@@ -28,26 +28,45 @@ enum
};
typedef struct internal_state {
const uint32_t *next_in;
uint8_t *next_out;
uint32_t id_len; /* bit length of code option identification key */
uint32_t last_in; /* previous input for preprocessing */
int id_len; /* bit length of code option identification key */
int64_t last_in; /* previous input for preprocessing */
int64_t (*get_sample)(ae_streamp);
int64_t xmin; /* minimum integer for preprocessing */
int64_t xmax; /* maximum integer for preprocessing */
int mode; /* current mode of FSM */
size_t i; /* counter for samples */
uint32_t *block_in; /* input block buffer */
int i; /* counter for samples */
int64_t *block_in; /* input block buffer */
uint8_t *block_out; /* output block buffer */
uint8_t *bp_out; /* pointer to current output */
uint8_t bitp; /* bit pointer to the next unused bit in accumulator */
uint8_t block_deferred; /* there is a block in the input buffer
int bitp; /* bit pointer to the next unused bit in accumulator */
int block_deferred; /* there is a block in the input buffer
but we first have to emit a zero block */
uint8_t ref; /* current buffer has a reference sample */
uint8_t zero_ref; /* current zero block has a reference sample */
uint32_t zero_ref_sample; /* reference sample of zero block */
size_t zero_blocks; /* number of contiguous zero blocks */
int ref; /* length of reference sample in current block
i.e. 0 or 1 depending on whether the block has
a reference sample or not */
int zero_ref; /* current zero block has a reference sample */
int64_t zero_ref_sample;/* reference sample of zero block */
int zero_blocks; /* number of contiguous zero blocks */
} encode_state;
#define GETF(type) static int64_t get_##type(ae_streamp strm) \
{ \
int64_t data; \
strm->avail_in--; \
strm->total_in++; \
data = *(type##_t *)strm->next_in; \
strm->next_in += sizeof(type##_t); \
return data; \
}
GETF(uint8)
GETF(int8)
GETF(uint16)
GETF(int16)
GETF(uint32)
GETF(int32)
int ae_encode_init(ae_streamp strm)
{
int blklen;
......@@ -67,14 +86,43 @@ int ae_encode_init(ae_streamp strm)
}
strm->state = state;
if (16 < strm->bit_per_sample)
if (strm->bit_per_sample > 16)
{
state->id_len = 5;
else if (8 < strm->bit_per_sample)
if (strm->flags & AE_DATA_SIGNED)
state->get_sample = get_int32;
else
state->get_sample = get_uint32;
}
else if (strm->bit_per_sample > 8)
{
state->id_len = 4;
if (strm->flags & AE_DATA_SIGNED)
state->get_sample = get_int16;
else
state->get_sample = get_uint16;
}
else
{
state->id_len = 3;
if (strm->flags & AE_DATA_SIGNED)
state->get_sample = get_int8;
else
state->get_sample = get_uint8;
}
if (strm->flags & AE_DATA_SIGNED)
{
state->xmin = -(1ULL << (strm->bit_per_sample - 1));
state->xmax = (1ULL << (strm->bit_per_sample - 1)) - 1;
}
else
{
state->xmin = 0;
state->xmax = (1ULL << strm->bit_per_sample) - 1;
}
state->block_in = (uint32_t *)malloc(strm->block_size * sizeof(uint32_t));
state->block_in = (int64_t *)malloc(strm->block_size * sizeof(int64_t));
if (state->block_in == NULL)
{
return AE_MEM_ERROR;
......@@ -93,8 +141,6 @@ int ae_encode_init(ae_streamp strm)
strm->total_in = 0;
strm->total_out = 0;
state->xmin = 0;
state->xmax = (1ULL << strm->bit_per_sample) - 1;
state->mode = M_NEW_BLOCK;
......@@ -105,7 +151,7 @@ int ae_encode_init(ae_streamp strm)
return AE_OK;
}
static inline void emit(encode_state *state, uint32_t data, int bits)
static inline void emit(encode_state *state, int64_t data, int bits)
{
while(bits)
{
......@@ -127,7 +173,7 @@ static inline void emit(encode_state *state, uint32_t data, int bits)
}
}
static inline void emitfs(encode_state *state, uint32_t fs)
static inline void emitfs(encode_state *state, int fs)
{
emit(state, 0, fs);
emit(state, 1, 1);
......@@ -143,14 +189,12 @@ int ae_encode(ae_streamp strm, int flush)
int i, j, zb;
int k_len[strm->bit_per_sample - 2];
int k, k_min, se_len, blk_head;
uint32_t d;
int64_t d;
int64_t theta, Delta;
encode_state *state;
state = strm->state;
state->next_in = strm->next_in;
state->next_out = strm->next_out;
for (;;)
{
......@@ -194,7 +238,7 @@ int ae_encode(ae_streamp strm, int flush)
all input there is.
*/
emit(state, 0xff, state->bitp);
*state->next_out++ = *state->bp_out;
*strm->next_out++ = *state->bp_out;
strm->avail_out--;
strm->total_out++;
}
......@@ -203,15 +247,13 @@ int ae_encode(ae_streamp strm, int flush)
}
else
{
state->block_in[state->i] = *state->next_in++;
strm->avail_in--;
strm->total_in++;
state->block_in[state->i] = state->get_sample(strm);
}
}
while (++state->i < strm->block_size);
/* preprocess block if needed */
if (strm->pp)
if (strm->flags & AE_DATA_PREPROCESS)
{
/* If this is the first block in a segment
then we need to insert a reference sample.
......@@ -371,7 +413,7 @@ int ae_encode(ae_streamp strm, int flush)
if (strm->avail_out == 0)
goto req_buffer;
*state->next_out++ = state->block_out[state->i];
*strm->next_out++ = state->block_out[state->i];
strm->avail_out--;
strm->total_out++;
state->i++;
......@@ -388,7 +430,7 @@ int ae_encode(ae_streamp strm, int flush)
break;
case M_ENCODE_SE:
emit(state, 1, state->id_len + 1);
emit(state, 1, state->id_len + 1);
if (state->ref)
emit(state, state->block_in[0], strm->bit_per_sample);
......@@ -418,7 +460,5 @@ int ae_encode(ae_streamp strm, int flush)
}
req_buffer:
strm->next_in = state->next_in;
strm->next_out = state->next_out;
return AE_OK;
}
src/libae.h
View file @ 053ffc57
......@@ -7,25 +7,33 @@ struct internal_state;
typedef struct _ae_stream
{
const void *next_in;
size_t avail_in; /* number of bytes available at next_in */
size_t total_in; /* total number of input bytes read so far */
void *next_out;
size_t avail_out; /* remaining free space at next_out */
size_t total_out; /* total number of bytes output so far */
uint32_t bit_per_sample; /* resolution in bits per sample (n = 1,..., 32) */
uint32_t block_size; /* block size in samples (J = 8 or 16) */
uint32_t segment_size; /* set of blocks between consecutive reference
samples */
uint8_t pp; /* pre/post-processor used? */
const uint8_t *next_in;
size_t avail_in; /* number of bytes available at
* next_in */
size_t total_in; /* total number of input bytes read so
* far */
uint8_t *next_out;
size_t avail_out; /* remaining free space at next_out */
size_t total_out; /* total number of bytes output so far */
uint32_t bit_per_sample; /* resolution in bits per sample (n =
* 1,..., 32) */
uint32_t block_size; /* block size in samples (J = 8 or 16) */
uint32_t segment_size; /* set of blocks between consecutive
* reference samples */
uint32_t flags;
struct internal_state *state;
} ae_stream;
typedef ae_stream *ae_streamp;
/* Coder flags */
#define AE_DATA_UNSIGNED 0
#define AE_DATA_SIGNED 1
#define AE_DATA_PREPROCESS 2 /* Set if preprocessor should be used */
/* Return codes of library functions */
#define AE_OK 0
#define AE_STREAM_END 1
#define AE_ERRNO (-1)
......@@ -33,8 +41,19 @@ typedef ae_stream *ae_streamp;
#define AE_DATA_ERROR (-3)
#define AE_MEM_ERROR (-4)
#define AE_NO_FLUSH 0
#define AE_FLUSH 1
/* Options for flushing */
#define AE_NO_FLUSH 0 /* Do not enforce output flushing. More
* input may be provided with later
* calls. So far only relevant for
* encoding. */
#define AE_FLUSH 1 /* Flush output and end encoding. The last
* call to ae_encode() must set AE_FLUSH to
* drain all output.
*
* It is not possible to continue encoding
* of the same stream after it has been
* flushed because the last byte may be
* padded with fill bits. */
int ae_decode_init(ae_streamp strm);
int ae_decode(ae_streamp strm, int flush);
......
src/test_decode.c
View file @ 053ffc57
......@@ -31,10 +31,10 @@ int main(int argc, char *argv[])
if (in == NULL || out == NULL)
return 1;
strm.bit_per_sample = 8;
strm.bit_per_sample = 17;
strm.block_size = 8;
strm.segment_size = 2;
strm.pp = 1;
strm.flags = AE_DATA_SIGNED | AE_DATA_PREPROCESS;
if (ae_decode_init(&strm) != AE_OK)
return 1;
......@@ -42,7 +42,7 @@ int main(int argc, char *argv[])
total_out = 0;
strm.avail_in = 0;
strm.avail_out = chunk_out;
strm.next_out = out;
strm.next_out = (uint8_t *)out;
input_avail = 1;
output_avail = 1;
......@@ -75,7 +75,7 @@ int main(int argc, char *argv[])
}
total_out = strm.total_out;
output_avail = 1;
strm.next_out = out;
strm.next_out = (uint8_t *)out;
strm.avail_out = chunk_out;
}
else
......
src/test_encode.c
View file @ 053ffc57
......@@ -32,10 +32,10 @@ int main(int argc, char *argv[])
if (in == NULL || out == NULL)
return 1;
strm.bit_per_sample = 8;
strm.bit_per_sample = 17;
strm.block_size = 8;
strm.segment_size = 2;
strm.pp = 1;
strm.flags = AE_DATA_SIGNED | AE_DATA_PREPROCESS;
if (ae_encode_init(&strm) != AE_OK)
return 1;
......@@ -57,7 +57,7 @@ int main(int argc, char *argv[])
in[i++] = c;
strm.avail_in = i;
strm.next_in = in;
strm.next_in = (uint8_t *)in;
if (c == EOF)
input_avail = 0;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment