From a7a692773f8fb3bed1f7b8774b3ec9ed98761232 Mon Sep 17 00:00:00 2001
From: Eugen Betke <eugen.betke@ecmwf.int>
Date: Tue, 9 May 2023 06:58:58 +0000
Subject: [PATCH] RSI block access
---
.gitignore | 2 +
README.md | 103 +++++++
include/libaec.h.in | 12 +-
src/CMakeLists.txt | 3 +-
src/Makefile.am | 4 +-
src/decode.c | 109 +++++++-
src/decode.h | 6 +-
src/encode.c | 46 +++-
src/encode.h | 3 +
src/vector.c | 68 +++++
src/vector.h | 18 ++
tests/CMakeLists.txt | 7 +
tests/Makefile.am | 7 +-
tests/check_rsi_block_access.c | 488 +++++++++++++++++++++++++++++++++
tests/check_seeking.c | 4 +-
15 files changed, 864 insertions(+), 16 deletions(-)
create mode 100644 src/vector.c
create mode 100644 src/vector.h
create mode 100644 tests/check_rsi_block_access.c
diff --git a/.gitignore b/.gitignore
index d2f2392..8e42ef9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,5 @@ autom4te.cache
build*
m4/*
lib/*
+*snalyzerinfo
+*analyzerinfo
diff --git a/README.md b/README.md
index 782310e..46effa3 100644
--- a/README.md
+++ b/README.md
@@ -225,6 +225,109 @@ zero blocks. The output data must therefore be truncated to the
correct length. This can also be achieved by providing an output
buffer of just the correct length.
+### Decoding data ranges
+
+The Libaec library has functionality that allows individual data areas to be decoded without having to decode the entire file.
+This allows efficient access to the data.
+
+This is possible because AEC-encoded data consists of independent blocks.
+It is, therefore, possible to decode individual blocks if their offsets are known.
+The Libaec library can capture the offsets when encoding or decoding the data and make them available to the user.
+
+The following example shows how to obtain the offsets.
+
+```c
+#include <libaec.h>
+
+...
+ struct aec_stream strm;
+ int32_t *source;
+ unsigned char *dest;
+ size_t count_offsets;
+ size_t *offsets;
+
+ strm.bits_per_sample = 32;
+ strm.block_size = 16;
+ strm.rsi = 128;
+ strm.flags = AEC_DATA_SIGNED | AEC_DATA_PREPROCESS;
+ strm.next_in = (unsigned char *)source;
+ strm.avail_in = source_length * sizeof(int32_t);
+ strm.next_out = dest;
+ strm.avail_out = dest_length;
+ if (aec_encode_init(&strm) != AEC_OK)
+ return 1;
+ /* Enable RSI offsets */
+ if (aec_encode_enable_offsets(&strm) != AEC_OK)
+ return 1;
+ if (aec_encode(&strm, AEC_FLUSH) != AEC_OK)
+ return 1;
+ /* Count RSI offsets */
+ if (aec_encode_count_offsets(&strm, &count_offsets) != AEC_OK)
+ return 1;
+ offsets = malloc(count_offsets * sizeof(*offsets));
+ /* Get RSI offsets */
+ if (aec_encode_get_offsets(&strm, offsets, offsets_count) != AEC_OK)
+ return 1;
+
+ aec_encode_end(&strm);
+ free(offsets);
+...
+```
+
+The offsets can then be used to decode ranges of data.
+The procedure is similar to the previous section, but use aec_decode_range() instead of aec_decode() and pass the offsets and the range as parameters.
+The decoded ranges are written to the buffer as a stream.
+When decoding the ranges into the individual buffers, set strm.total_out to zero.
+
+```c
+#include <libaec.h>
+
+...
+ struct aec_stream strm;
+ unsigned char *source;
+ int32_t *dest;
+ /* Suppose we got the offsets from the previous step */
+ size_t count_offsets;
+ size_t *offsets;
+
+ strm.bits_per_sample = 32;
+ strm.block_size = 16;
+ strm.rsi = 128;
+ strm.flags = AEC_DATA_SIGNED | AEC_DATA_PREPROCESS;
+ strm.next_in = (unsigned char *)source;
+ strm.avail_in = source_length * sizeof(int32_t);
+ strm.next_out = dest;
+ strm.avail_out = dest_length;
+
+ if (aec_decode_init(&strm))
+ return 1;
+
+ /* Decode data as stream of ranges*/
+ if (aec_decode_range(&strm, offsets, count_offsets, 12, 16);
+ return 1;
+ if (aec_decode_range(&strm, offsets, count_offsets, 244, 255);
+ return 1;
+
+ /* Decode data ranges to individual buffers */
+ strm.avail_out = 12
+ unsigned char buf_a[strm.avail_out];
+ strm.next_out = buf_a;
+ strm.total_out = 0;
+ if (aec_decode_range(&strm, offsets, count_offsets, 12, strm.avail_out);
+ return 1;
+
+ strm.avail_out = 255;
+ unsigned char buf_b[strm.avail_out];
+ strm.next_out = buf_b;
+ strm.total_out = 0;
+ if (aec_decode_range(&strm, offsets, count_offsets, 244, strm.avail_out);
+ return 1;
+
+ aec_decode_end(&strm);
+...
+```
+
+
## References
[Lossless Data Compression. Recommendation for Space Data System
diff --git a/include/libaec.h.in b/include/libaec.h.in
index 85b3de3..147f9b4 100644
--- a/include/libaec.h.in
+++ b/include/libaec.h.in
@@ -126,6 +126,7 @@ struct aec_stream {
#define AEC_STREAM_ERROR (-2)
#define AEC_DATA_ERROR (-3)
#define AEC_MEM_ERROR (-4)
+#define AEC_RSI_OFFSETS_ERROR (-5)
/************************/
/* Options for flushing */
@@ -148,11 +149,19 @@ struct aec_stream {
/* Streaming encoding and decoding functions */
/*********************************************/
LIBAEC_DLL_EXPORTED int aec_encode_init(struct aec_stream *strm);
+LIBAEC_DLL_EXPORTED int aec_encode_enable_offsets(struct aec_stream *strm);
+LIBAEC_DLL_EXPORTED int aec_encode_count_offsets(struct aec_stream *strm, size_t *rsi_offsets_count);
+LIBAEC_DLL_EXPORTED int aec_encode_get_offsets(struct aec_stream *strm, size_t *rsi_offsets, size_t rsi_offsets_count);
+LIBAEC_DLL_EXPORTED int aec_buffer_seek(struct aec_stream *strm, size_t offset);
LIBAEC_DLL_EXPORTED int aec_encode(struct aec_stream *strm, int flush);
LIBAEC_DLL_EXPORTED int aec_encode_end(struct aec_stream *strm);
LIBAEC_DLL_EXPORTED int aec_decode_init(struct aec_stream *strm);
+LIBAEC_DLL_EXPORTED int aec_decode_enable_offsets(struct aec_stream *strm);
+LIBAEC_DLL_EXPORTED int aec_decode_count_offsets(struct aec_stream *strm, size_t *rsi_offsets_count);
+LIBAEC_DLL_EXPORTED int aec_decode_get_offsets(struct aec_stream *strm, size_t *rsi_offsets, size_t rsi_offsets_count);
LIBAEC_DLL_EXPORTED int aec_decode(struct aec_stream *strm, int flush);
+LIBAEC_DLL_EXPORTED int aec_decode_range(struct aec_stream *strm, const size_t *rsi_offsets, size_t rsi_offsets_count, size_t pos, size_t size);
LIBAEC_DLL_EXPORTED int aec_decode_end(struct aec_stream *strm);
/***************************************************************/
@@ -160,9 +169,6 @@ LIBAEC_DLL_EXPORTED int aec_decode_end(struct aec_stream *strm);
/***************************************************************/
LIBAEC_DLL_EXPORTED int aec_buffer_encode(struct aec_stream *strm);
LIBAEC_DLL_EXPORTED int aec_buffer_decode(struct aec_stream *strm);
-LIBAEC_DLL_EXPORTED int aec_buffer_seek(struct aec_stream *strm,
- size_t byte_offset,
- unsigned char bit_offset);
#ifdef __cplusplus
}
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 6b9783e..b434426 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -2,7 +2,8 @@
add_library(aec OBJECT
encode.c
encode_accessors.c
- decode.c)
+ decode.c
+ vector.c)
target_include_directories(aec
PUBLIC
diff --git a/src/Makefile.am b/src/Makefile.am
index 823dfd2..47ce42c 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -2,8 +2,8 @@ AM_CFLAGS = $(CFLAG_VISIBILITY)
AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include \
-DBUILDING_LIBAEC
lib_LTLIBRARIES = libaec.la libsz.la
-libaec_la_SOURCES = encode.c encode_accessors.c decode.c \
-encode.h encode_accessors.h decode.h
+libaec_la_SOURCES = encode.c encode_accessors.c decode.c vector.c\
+encode.h encode_accessors.h decode.h vector.h
libaec_la_LDFLAGS = -version-info 0:12:0 -no-undefined
libsz_la_SOURCES = sz_compat.c
diff --git a/src/decode.c b/src/decode.c
index efb08c2..5b68e2a 100644
--- a/src/decode.c
+++ b/src/decode.c
@@ -389,6 +389,12 @@ static inline int m_id(struct aec_stream *strm)
static int m_next_cds(struct aec_stream *strm)
{
struct internal_state *state = strm->state;
+
+ if ((state->offsets != NULL) && (state->rsi_size == RSI_USED_SIZE(state)))
+ vector_push_back(
+ state->offsets,
+ strm->total_in * 8 - (strm->avail_in * 8 + state->bitp));
+
if (state->rsi_size == RSI_USED_SIZE(state)) {
state->flush_output(strm);
state->flush_start = state->rsi_buffer;
@@ -766,6 +772,8 @@ int aec_decode_init(struct aec_stream *strm)
state->bitp = 0;
state->fs = 0;
state->mode = m_id;
+ state->offsets = NULL;
+
return AEC_OK;
}
@@ -808,10 +816,13 @@ int aec_decode(struct aec_stream *strm, int flush)
int aec_decode_end(struct aec_stream *strm)
{
struct internal_state *state = strm->state;
+ if (state->offsets != NULL)
+ vector_destroy(state->offsets);
free(state->id_table);
free(state->rsi_buffer);
free(state);
+
return AEC_OK;
}
@@ -826,14 +837,12 @@ int aec_buffer_decode(struct aec_stream *strm)
return status;
}
-int aec_buffer_seek(struct aec_stream *strm,
- size_t byte_offset,
- unsigned char bit_offset)
+int aec_buffer_seek(struct aec_stream *strm, size_t offset)
{
struct internal_state *state = strm->state;
- if (bit_offset > 7)
- return AEC_CONF_ERROR;
+ size_t byte_offset = offset / 8;
+ unsigned char bit_offset = offset % 8;
if (strm->avail_in < byte_offset)
return AEC_MEM_ERROR;
@@ -852,3 +861,93 @@ int aec_buffer_seek(struct aec_stream *strm,
}
return AEC_OK;
}
+
+int aec_decode_range(struct aec_stream *strm, const size_t *rsi_offsets, size_t rsi_offsets_count, size_t pos, size_t size)
+{
+ struct internal_state *state = strm->state;
+ int status;
+ size_t rsi_size;
+ size_t rsi_n;
+ unsigned char *out_tmp;
+ struct aec_stream strm_tmp = *strm;
+
+ if (state->pp) {
+ state->ref = 1;
+ state->encoded_block_size = strm->block_size - 1;
+ } else {
+ state->ref = 0;
+ state->encoded_block_size = strm->block_size;
+ }
+
+ state->rsip = state->rsi_buffer;
+ state->flush_start = state->rsi_buffer;
+ state->bitp = 0;
+ state->fs = 0;
+ state->mode = m_id;
+
+ rsi_size = strm->rsi * strm->block_size * state->bytes_per_sample;
+ rsi_n = pos / rsi_size;
+ if (rsi_n >= rsi_offsets_count)
+ return AEC_DATA_ERROR;
+
+ /* resize and align to bytes_per_sample */
+ strm_tmp.total_out = 0;
+ strm_tmp.avail_out = size + pos % rsi_size + 1;
+ strm_tmp.avail_out += state->bytes_per_sample - strm_tmp.avail_out % state->bytes_per_sample;
+ if ((out_tmp = malloc(strm_tmp.avail_out)) == NULL)
+ return AEC_MEM_ERROR;
+ strm_tmp.next_out = out_tmp;
+
+ if ((status = aec_buffer_seek(&strm_tmp, rsi_offsets[rsi_n])) != AEC_OK)
+ return status;
+
+ if ((status = aec_decode(&strm_tmp, AEC_FLUSH)) != 0)
+ return status;
+
+ memcpy(strm->next_out, out_tmp + (pos - rsi_n * rsi_size), size);
+
+ strm->next_out += size;
+ strm->avail_out -= size;
+ strm->total_out += size;
+ free(out_tmp);
+
+ return AEC_OK;
+}
+
+int aec_decode_count_offsets(struct aec_stream *strm, size_t *count)
+{
+ struct internal_state *state = strm->state;
+ if (state->offsets == NULL) {
+ *count = 0;
+ return AEC_RSI_OFFSETS_ERROR;
+ } else {
+ *count = vector_size(state->offsets);
+ }
+ return AEC_OK;
+}
+
+int aec_decode_get_offsets(struct aec_stream *strm, size_t *offsets,
+ size_t offsets_count)
+{
+ struct internal_state *state = strm->state;
+ if (state->offsets == NULL) {
+ return AEC_RSI_OFFSETS_ERROR;
+ }
+ if (offsets_count < vector_size(state->offsets)) {
+ return AEC_MEM_ERROR;
+ }
+ memcpy(offsets, vector_data(state->offsets),
+ vector_size(state->offsets) * sizeof(size_t));
+ return AEC_OK;
+}
+
+int aec_decode_enable_offsets(struct aec_stream *strm)
+{
+ struct internal_state *state = strm->state;
+ if (state->offsets != NULL) {
+ return AEC_RSI_OFFSETS_ERROR;
+ }
+ state->offsets = vector_create();
+ vector_push_back(state->offsets, 0);
+ return AEC_OK;
+}
diff --git a/src/decode.h b/src/decode.h
index 0aa40d7..a033ad4 100644
--- a/src/decode.h
+++ b/src/decode.h
@@ -42,6 +42,7 @@
#include "config.h"
#include <stdint.h>
#include <stddef.h>
+#include "vector.h"
#define M_CONTINUE 1
#define M_EXIT 0
@@ -120,6 +121,9 @@ struct internal_state {
/* table for decoding second extension option */
int se_table[2 * (SE_TABLE_SIZE + 1)];
-} decode_state;
+
+ /* RSI table */
+ struct vector_t *offsets;
+};
#endif /* DECODE_H */
diff --git a/src/encode.c b/src/encode.c
index bd83a14..b87a364 100644
--- a/src/encode.c
+++ b/src/encode.c
@@ -411,6 +411,7 @@ static uint32_t assess_se_option(struct aec_stream *strm)
if (len > state->uncomp_len)
return UINT32_MAX;
}
+
return (uint32_t)len;
}
@@ -709,8 +710,10 @@ static int m_get_block(struct aec_stream *strm)
state->blocks_avail = strm->rsi - 1;
state->block = state->data_pp;
state->blocks_dispensed = 1;
-
if (strm->avail_in >= state->rsi_len) {
+ if (state->offsets != NULL)
+ vector_push_back(state->offsets, (strm->total_out - strm->avail_out) * 8 + (8 - state->bits));
+
state->get_rsi(strm);
if (strm->flags & AEC_DATA_PREPROCESS)
state->preprocess(strm);
@@ -883,6 +886,7 @@ int aec_encode_init(struct aec_stream *strm)
state->bits = 8;
state->mode = m_get_block;
+ struct vector_t *offsets = NULL;
return AEC_OK;
}
@@ -921,10 +925,50 @@ int aec_encode_end(struct aec_stream *strm)
int status = AEC_OK;
if (state->flush == AEC_FLUSH && state->flushed == 0)
status = AEC_STREAM_ERROR;
+ if (state->offsets != NULL) {
+ vector_destroy(state->offsets);
+ state->offsets = NULL;
+ }
cleanup(strm);
return status;
}
+int aec_encode_count_offsets(struct aec_stream *strm, size_t *count)
+{
+ struct internal_state *state = strm->state;
+ if (state->offsets == NULL) {
+ *count = 0;
+ return AEC_RSI_OFFSETS_ERROR;
+ } else {
+ *count = vector_size(state->offsets);
+ }
+ return AEC_OK;
+}
+
+int aec_encode_get_offsets(struct aec_stream *strm, size_t *offsets, size_t offsets_count)
+{
+ struct internal_state *state = strm->state;
+ if (state->offsets == NULL) {
+ return AEC_RSI_OFFSETS_ERROR;
+ }
+ if (offsets_count < vector_size(state->offsets)) {
+ return AEC_MEM_ERROR;
+ }
+ memcpy(offsets, vector_data(state->offsets), offsets_count * sizeof(size_t));
+ return AEC_OK;
+}
+
+int aec_encode_enable_offsets(struct aec_stream *strm)
+{
+ struct internal_state *state = strm->state;
+
+ if (state->offsets != NULL)
+ return AEC_RSI_OFFSETS_ERROR;
+
+ state->offsets = vector_create();
+ return AEC_OK;
+}
+
int aec_buffer_encode(struct aec_stream *strm)
{
int status = aec_encode_init(strm);
diff --git a/src/encode.h b/src/encode.h
index d8dd18c..05c5862 100644
--- a/src/encode.h
+++ b/src/encode.h
@@ -39,6 +39,7 @@
#ifndef ENCODE_H
#define ENCODE_H 1
+#include "vector.h"
#include "config.h"
#include <stdint.h>
@@ -140,6 +141,8 @@ struct internal_state {
/* length of uncompressed CDS */
uint32_t uncomp_len;
+
+ struct vector_t *offsets;
};
#endif /* ENCODE_H */
diff --git a/src/vector.c b/src/vector.c
new file mode 100644
index 0000000..2cb96fd
--- /dev/null
+++ b/src/vector.c
@@ -0,0 +1,68 @@
+#include "vector.h"
+#include <stdio.h>
+
+#define VECTOR_INITIAL_CAPACITY 128
+#define VECTOR_GROWTH_FACTOR 2
+
+#define VECTOR_FATAL_ERROR(...) \
+ { \
+ fprintf(stderr, "Fatal error in %s at line %d: Exiting", __FILE__, __LINE__); \
+ exit(1); \
+ }
+
+struct vector_t* vector_create()
+{
+ struct vector_t *vec = malloc(sizeof(struct vector_t));
+ if (vec == NULL)
+ VECTOR_FATAL_ERROR("Failed to allocate memory for vector\n");
+
+ vec->capacity = VECTOR_INITIAL_CAPACITY;
+ vec->size = 0;
+ vec->values = malloc(sizeof(*vec->values) * vec->capacity);
+ if (vec->values == NULL)
+ VECTOR_FATAL_ERROR("Failed to allocate memory for vector values\n");
+ return vec;
+}
+
+size_t vector_size(struct vector_t* vec)
+{
+ return vec->size;
+}
+
+void vector_destroy(struct vector_t *vec)
+{
+ free(vec->values);
+ free(vec);
+}
+
+int vector_equal(struct vector_t *vec1, struct vector_t *vec2)
+{
+ if (vec1->size != vec2->size)
+ return 0;
+ for (size_t i = 0; i < vec1->size; i++)
+ if (vec1->values[i] != vec2->values[i])
+ return 0;
+ return 1;
+}
+
+size_t vector_at(struct vector_t *vector, size_t idx)
+{
+ return vector->values[idx];
+}
+
+void vector_push_back(struct vector_t *vec, size_t value)
+{
+ if (vec->size == vec->capacity) {
+ vec->capacity *= VECTOR_GROWTH_FACTOR;
+ vec->values = realloc(vec->values, sizeof(*vec->values) * vec->capacity);
+ if (vec->values == NULL)
+ VECTOR_FATAL_ERROR("Failed to reallocate memory for vector values\n");
+ }
+ vec->values[vec->size] = value;
+ vec->size++;
+}
+
+size_t* vector_data(struct vector_t *vec)
+{
+ return vec->values;
+}
diff --git a/src/vector.h b/src/vector.h
new file mode 100644
index 0000000..747661e
--- /dev/null
+++ b/src/vector.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <libaec.h>
+#include <stdlib.h>
+
+struct vector_t {
+ size_t size;
+ size_t capacity;
+ size_t *values;
+};
+
+struct vector_t* vector_create();
+void vector_destroy(struct vector_t* vec);
+size_t vector_size(struct vector_t* vec);
+void vector_push_back(struct vector_t *vec, size_t offset);
+size_t vector_at(struct vector_t *vector, size_t idx);
+int vector_equal(struct vector_t *vec1, struct vector_t *vec2);
+size_t* vector_data(struct vector_t *vec);
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 77f4918..ce0272e 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,8 +1,10 @@
add_library(check_aec STATIC check_aec.c)
target_link_libraries(check_aec PUBLIC aec)
+
add_executable(check_code_options check_code_options.c)
target_link_libraries(check_code_options PUBLIC check_aec aec)
add_test(NAME check_code_options COMMAND check_code_options)
+
add_executable(check_buffer_sizes check_buffer_sizes.c)
target_link_libraries(check_buffer_sizes PUBLIC check_aec aec)
add_test(NAME check_buffer_sizes COMMAND check_buffer_sizes)
@@ -12,11 +14,16 @@ add_test(NAME check_seeking COMMAND check_seeking)
add_executable(check_long_fs check_long_fs.c)
target_link_libraries(check_long_fs PUBLIC check_aec aec)
add_test(NAME check_long_fs COMMAND check_long_fs)
+
add_executable(check_szcomp check_szcomp.c)
target_link_libraries(check_szcomp PUBLIC check_aec sz)
add_test(NAME check_szcomp
COMMAND check_szcomp ${PROJECT_SOURCE_DIR}/data/121B2TestData/ExtendedParameters/sar32bit.dat)
+add_executable(check_rsi_block_access check_rsi_block_access.c)
+target_link_libraries(check_rsi_block_access PUBLIC check_aec aec)
+add_test(NAME check_rsi_block_access COMMAND check_rsi_block_access)
+
if(UNIX)
add_test(
NAME sampledata.sh
diff --git a/tests/Makefile.am b/tests/Makefile.am
index f2bfd4d..3ca67c2 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,13 +1,13 @@
AUTOMAKE_OPTIONS = color-tests
AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include
TESTS = check_code_options check_buffer_sizes check_long_fs \
-check_seeking szcomp.sh sampledata.sh
+check_seeking check_rsi_block_access szcomp.sh sampledata.sh
TEST_EXTENSIONS = .sh
CLEANFILES = test.dat test.rz
check_LTLIBRARIES = libcheck_aec.la
libcheck_aec_la_SOURCES = check_aec.c check_aec.h
check_PROGRAMS = check_code_options check_buffer_sizes check_long_fs \
-check_szcomp check_seeking
+check_szcomp check_seeking check_rsi_block_access
check_code_options_SOURCES = check_code_options.c check_aec.h \
$(top_builddir)/include/libaec.h
@@ -21,6 +21,9 @@ $(top_builddir)/include/libaec.h
check_seeking_SOURCES = check_seeking.c check_aec.h \
$(top_builddir)/include/libaec.h
+check_rsi_block_access_SOURCES = check_rsi_block_access.c check_aec.h \
+$(top_builddir)/include/libaec.h $(top_builddir)/src/decode.h
+
check_szcomp_SOURCES = check_szcomp.c $(top_srcdir)/include/szlib.h
LDADD = libcheck_aec.la $(top_builddir)/src/libaec.la
diff --git a/tests/check_rsi_block_access.c b/tests/check_rsi_block_access.c
new file mode 100644
index 0000000..b6afea7
--- /dev/null
+++ b/tests/check_rsi_block_access.c
@@ -0,0 +1,488 @@
+#include "check_aec.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <assert.h>
+
+struct aec_context
+{
+ size_t nvalues;
+ int flags;
+ int rsi;
+ int block_size;
+ int bits_per_sample;
+ int bytes_per_sample;
+ unsigned char * obuf;
+ unsigned char * ebuf;
+ unsigned char * dbuf;
+ size_t obuf_len;
+ size_t ebuf_len;
+ size_t dbuf_len;
+ size_t ebuf_total;
+};
+
+typedef void (*data_generator_t)(struct aec_context *ctx);
+
+static int get_input_bytes(struct aec_context *ctx)
+{
+ if (ctx->flags & AEC_DATA_3BYTE) {
+ fprintf(stderr, "AES_DATA_3BYTE is not supported\n");
+ exit(1);
+ }
+ if (ctx->bits_per_sample < 1 || ctx->bits_per_sample > 32) {
+ fprintf(stderr, "Invalid bits_per_sample: %d\n", ctx->bits_per_sample);
+ exit(1);
+ }
+ int nbytes = (ctx->bits_per_sample + 7) / 8;
+ if (nbytes == 3) nbytes = 4;
+ return nbytes;
+}
+
+static void data_generator_zero(struct aec_context *ctx)
+{
+ size_t nbytes = ctx->bytes_per_sample;
+ if (ctx->obuf_len % nbytes) {
+ fprintf(stderr, "Invalid buffer_size: %lu\n", ctx->obuf_len);
+ exit(1);
+ }
+
+ size_t nvalues = ctx->obuf_len / nbytes;
+
+ for (size_t i = 0; i < nvalues; i++) {
+ size_t value = 0;
+ unsigned char *value_p = (unsigned char*) &value;
+ for (size_t j = 0; j < nbytes; j++) {
+ if (ctx->flags & AEC_DATA_MSB)
+ ctx->obuf[i * nbytes + j] = value_p[nbytes - j - 1];
+ else
+ ctx->obuf[i * nbytes + j] = value_p[j];
+ }
+ }
+}
+
+static void data_generator_random(struct aec_context *ctx)
+{
+ size_t nbytes = ctx->bytes_per_sample;
+ if (ctx->obuf_len % nbytes) {
+ fprintf(stderr, "Invalid buffer_size: %lu\n", ctx->obuf_len);
+ exit(1);
+ }
+
+ size_t nvalues = ctx->obuf_len / nbytes;
+ size_t mask = (1 << (ctx->bits_per_sample - 1))-1;
+
+ for (size_t i = 0; i < nvalues; i++) {
+ size_t value = rand() & mask;
+ unsigned char *value_p = (unsigned char*) &value;
+
+ for (size_t j = 0; j < nbytes; j++) {
+ if (ctx->flags & AEC_DATA_MSB) {
+ ctx->obuf[i * nbytes + j] = value_p[nbytes - j - 1];
+ }
+ else {
+ ctx->obuf[i * nbytes + j] = value_p[j];
+ }
+ }
+ }
+}
+
+static void data_generator_incr(struct aec_context *ctx)
+{
+ size_t nbytes = ctx->bytes_per_sample;
+ if (ctx->obuf_len % nbytes) {
+ fprintf(stderr, "Invalid buffer_size: %lu\n", ctx->obuf_len);
+ exit(1);
+ }
+
+ size_t nvalues = ctx->obuf_len / nbytes;
+ size_t max_value = 1 << (ctx->bits_per_sample - 1);
+
+ for (size_t i = 0; i < nvalues; i++) {
+ size_t value = i % max_value;
+ unsigned char *value_p = (unsigned char*) &value;
+ for (size_t j = 0; j < nbytes; j++) {
+ if (ctx->flags & AEC_DATA_MSB) {
+ ctx->obuf[i * nbytes + j] = value_p[nbytes - j - 1];
+ }
+ else {
+ ctx->obuf[i * nbytes + j] = value_p[j];
+ }
+ }
+ }
+}
+
+static void ctx_init(struct aec_context *ctx)
+{
+ ctx->nvalues = 0;
+ ctx->flags = 0;
+ ctx->rsi = 0;
+ ctx->block_size = 0;
+ ctx->bits_per_sample = 0;
+ ctx->obuf = NULL;
+ ctx->ebuf = NULL;
+ ctx->dbuf = NULL;
+ ctx->obuf_len = 0;
+ ctx->ebuf_len = 0;
+ ctx->dbuf_len = 0;
+ ctx->ebuf_total = 0;
+}
+
+#define PREPARE_ENCODE(strm_e, ctx, flags) \
+{ \
+ (strm_e)->flags = flags; \
+ (strm_e)->rsi = (ctx)->rsi; \
+ (strm_e)->block_size = (ctx)->block_size; \
+ (strm_e)->bits_per_sample = (ctx)->bits_per_sample; \
+ (strm_e)->next_in = (ctx)->obuf; \
+ (strm_e)->avail_in = (ctx)->obuf_len; \
+ (strm_e)->next_out = (ctx)->ebuf; \
+ (strm_e)->avail_out = (ctx)->ebuf_len; \
+ int status = 0; \
+ if ((status = aec_buffer_encode((strm_e))) != 0) { \
+ return status; \
+ } \
+ (ctx)->ebuf_total = (strm_e)->total_out; \
+ \
+ struct aec_stream strm_d; \
+ strm_d = (*strm_e); \
+ strm_d.next_in = (ctx)->ebuf; \
+ strm_d.avail_in = (ctx)->ebuf_total; \
+ strm_d.next_out = (ctx)->dbuf; \
+ strm_d.avail_out = (ctx)->dbuf_len; \
+ if ((status = aec_buffer_decode((&strm_d))) != 0) { \
+ return status; \
+ } \
+}
+
+#define PREPARE_ENCODE_WITH_OFFSETS(strm_eo, ctx, flags, offsets_ptr, offsets_count_ptr) \
+{ \
+ (strm_eo)->flags = flags; \
+ (strm_eo)->rsi = (ctx)->rsi; \
+ (strm_eo)->block_size = (ctx)->block_size; \
+ (strm_eo)->bits_per_sample = (ctx)->bits_per_sample; \
+ (strm_eo)->next_in = (ctx)->obuf; \
+ (strm_eo)->avail_in = (ctx)->obuf_len; \
+ (strm_eo)->next_out = (ctx)->ebuf; \
+ (strm_eo)->avail_out = (ctx)->ebuf_len; \
+ int status = 0; \
+ if ((status = aec_encode_init((strm_eo))) != AEC_OK) { \
+ return(status); \
+ } \
+ aec_encode_enable_offsets((strm_eo)); \
+ if ((status = aec_encode((strm_eo), AEC_FLUSH)) != 0) { \
+ return(status); \
+ } \
+ aec_encode_count_offsets((strm_eo), (offsets_count_ptr)); \
+ (offsets_ptr) = (size_t*) malloc(sizeof(*(offsets_ptr)) * *(offsets_count_ptr)); \
+ if ((status = aec_encode_get_offsets((strm_eo), (offsets_ptr), *(offsets_count_ptr)))) { \
+ return(status); \
+ } \
+ aec_encode_end((strm_eo)); \
+ ctx->ebuf_total = (strm_eo)->total_out; \
+}
+
+#define PREPARE_DECODE_WITH_OFFSETS(strm_do, ctx, flags, offsets_ptr, offsets_count_ptr) \
+{ \
+ (strm_do)->flags = (ctx)->flags; \
+ (strm_do)->rsi = (ctx)->rsi; \
+ (strm_do)->block_size = (ctx)->block_size; \
+ (strm_do)->bits_per_sample = (ctx)->bits_per_sample; \
+ (strm_do)->next_in = (ctx)->ebuf; \
+ (strm_do)->avail_in = (ctx)->ebuf_total; \
+ (strm_do)->next_out = (ctx)->dbuf; \
+ (strm_do)->avail_out = (ctx)->dbuf_len; \
+ if ((status = aec_decode_init((strm_do))) != AEC_OK) { \
+ return status; \
+ } \
+ if ((status = aec_decode_enable_offsets((strm_do))) != AEC_OK) { \
+ return status; \
+ }; \
+ if ((status = aec_decode((strm_do), AEC_FLUSH)) != AEC_OK) { \
+ return status; \
+ } \
+ if ((status = aec_decode_count_offsets((strm_do), (offsets_count_ptr))) != AEC_OK) { \
+ return status; \
+ } \
+ (offsets_ptr) = (size_t*) malloc(sizeof(*(offsets_ptr)) * *(offsets_count_ptr)); \
+ if ((status = aec_decode_get_offsets((strm_do), (offsets_ptr), *(offsets_count_ptr))) != AEC_OK) { \
+ return status; \
+ }; \
+ aec_decode_end((strm_do)); \
+ for (size_t i = 0; i < (strm_do)->total_out; ++i) { \
+ if ((ctx)->dbuf[i] != (ctx)->obuf[i]) { \
+ return 104; \
+ } \
+ } \
+}
+
+static int aec_rsi_at(struct aec_stream *strm, const size_t *offsets,
+ size_t offsets_count, size_t idx)
+{
+ if (offsets == NULL || idx >= offsets_count) return AEC_RSI_OFFSETS_ERROR;
+
+ int status = 0;
+ size_t rsi_offset = offsets[idx];
+
+ if ((status = aec_decode_init(strm)) != AEC_OK)
+ return status;
+ if ((status = aec_buffer_seek(strm, rsi_offset)) != AEC_OK)
+ return status;
+ if ((status = aec_decode(strm, AEC_FLUSH)) != AEC_OK)
+ return status;
+ aec_decode_end(strm);
+
+ return AEC_OK;
+}
+
+static int test_rsi_at(struct aec_context *ctx)
+{
+ int status = AEC_OK;
+ int flags = ctx->flags;
+ unsigned short *obuf = (unsigned short*) ctx->obuf;
+
+ struct aec_stream strm_encode;
+ PREPARE_ENCODE(&strm_encode, ctx, flags);
+
+ struct aec_stream strm_decode;
+ size_t *offsets;
+ size_t offsets_count;
+ PREPARE_DECODE_WITH_OFFSETS(&strm_decode, ctx, flags, offsets, &offsets_count);
+
+ size_t rsi_len = ctx->rsi * ctx->block_size * ctx->bytes_per_sample;
+ unsigned char *rsi_buf = malloc(rsi_len);
+ if (rsi_buf == NULL) {
+ fprintf(stderr, "ERROR: Failed to allocate rsi buffer\n");
+ exit(1);
+ }
+
+ for (int i = 0; i < offsets_count; ++i) {
+ struct aec_stream strm_at;
+ strm_at.flags = flags;
+ strm_at.rsi = ctx->rsi;
+ strm_at.block_size = ctx->block_size;
+ strm_at.bits_per_sample = ctx->bits_per_sample;
+ strm_at.next_in = ctx->ebuf;
+ strm_at.avail_in = ctx->ebuf_total;
+ strm_at.next_out = rsi_buf;
+ strm_at.avail_out = ctx->dbuf_len - i * rsi_len > rsi_len ? rsi_len : ctx->dbuf_len % rsi_len;
+
+ if ((status = aec_rsi_at(&strm_at, offsets, offsets_count, i)) != AEC_OK) {
+ return status;
+ }
+ for (int j = 0; j < strm_at.total_out; j++) {
+ if (j == ctx->rsi * ctx->block_size * ctx->bytes_per_sample + j > ctx->obuf_len) {
+ break;
+ }
+ if (rsi_buf[j] != ctx->obuf[i * ctx->block_size * ctx->rsi * ctx->bytes_per_sample + j]) {
+ return 101;
+ }
+ }
+ }
+
+ free(offsets);
+ free(rsi_buf);
+ return status;
+}
+
+int test_read(struct aec_context *ctx)
+{
+ int status = AEC_OK;
+ int flags = ctx->flags;
+
+ struct aec_stream strm_encode;
+ PREPARE_ENCODE(&strm_encode, ctx, flags);
+
+ struct aec_stream strm_decode;
+ size_t *offsets = NULL;
+ size_t offsets_size = 0;
+ PREPARE_DECODE_WITH_OFFSETS(&strm_decode, ctx, flags, offsets, &offsets_size);
+
+ size_t rsi_len = ctx->rsi * ctx->block_size * ctx->bytes_per_sample;
+ unsigned rsi_n = ctx->obuf_len / (ctx->rsi * ctx->block_size); // Number of full rsi blocks
+ unsigned rsi_r = ctx->obuf_len % (ctx->rsi * ctx->block_size); // Remainder
+
+ // Edge case: Imposible to get wanted number of slices
+ size_t wanted_num_slices = 3;
+ if (wanted_num_slices > ctx->obuf_len) {
+ wanted_num_slices = ctx->obuf_len;
+ }
+
+ // Optimize the size of the last slice
+ // Make sure that the last slice is not too small
+ size_t slice_size = (ctx->obuf_len % ((ctx->obuf_len / wanted_num_slices) * wanted_num_slices)) == 0 ? ctx->obuf_len / wanted_num_slices : ctx->obuf_len / wanted_num_slices + 1;
+
+ size_t num_slices = ctx->obuf_len / slice_size;
+ size_t remainder = ctx->obuf_len % slice_size;
+
+ size_t slice_offsets[num_slices + 1];
+ size_t slice_sizes[num_slices + 1];
+
+ for (size_t i = 0; i < num_slices; ++i) {
+ slice_offsets[i] = slice_size * i;
+ slice_sizes[i] = slice_size;
+ }
+ if (remainder > 0) {
+ slice_offsets[num_slices] = slice_size * num_slices;
+ slice_sizes[num_slices] = remainder;
+ ++num_slices;
+ }
+
+ struct aec_stream strm_read;
+ strm_read.flags = ctx->flags;
+ strm_read.rsi = ctx->rsi;
+ strm_read.block_size = ctx->block_size;
+ strm_read.bits_per_sample = ctx->bits_per_sample;
+ strm_read.next_in = ctx->ebuf;
+ strm_read.avail_in = strm_encode.total_out;
+ strm_read.next_out = ctx->dbuf;
+ strm_read.avail_out = ctx->dbuf_len;
+
+ if ((status = aec_decode_init(&strm_read)) != AEC_OK)
+ return status;
+ struct internal_state *state = strm_read.state;
+
+ // Test 1: Stream data
+ for (size_t i = 0; i < num_slices; ++i) {
+ if ((status = aec_decode_range(&strm_read, offsets, offsets_size, slice_offsets[i], slice_sizes[i])) != AEC_OK) {
+ return status;
+ }
+ }
+
+ for (size_t i = 0; i < ctx->obuf_len; ++i) {
+ if (ctx->obuf[i] != ctx->dbuf[i]) {
+ fprintf(stderr, "Index: %zu Size: %zu strm_read.total_out: %zu\n", i, ctx->obuf_len, strm_read.total_out);
+ fprintf(stderr, "Expected: %u Got: %u\n", ctx->obuf[i], ctx->dbuf[i]);
+ assert(0);
+ return 102;
+ }
+ }
+
+ // Test 2: Read slices
+ for (size_t i = 0; i < num_slices; ++i) {
+ struct internal_state *state = strm_read.state;
+ size_t buf_size = slice_sizes[i];;
+ unsigned char *buf = malloc(buf_size);
+ if (buf == NULL) {
+ fprintf(stderr, "Error: malloc failed\n");
+ return 1;
+
+ }
+ strm_read.next_out = buf;
+ strm_read.avail_out = buf_size;
+ strm_read.total_out = 0;
+ if ((status = aec_decode_range(&strm_read, offsets, offsets_size, slice_offsets[i], buf_size)) != AEC_OK) {
+ return status;
+ }
+ for (size_t j = 0; j < buf_size; ++j) {
+ if (ctx->obuf[slice_offsets[i] + j] != buf[j]) {
+ return 103;
+ }
+ }
+ free(buf);
+ }
+
+ aec_decode_end(&strm_read);
+
+ free(offsets);
+ return status;
+}
+
+
+int test_offsets(struct aec_context *ctx)
+{
+ int status = AEC_OK;
+ int flags = ctx->flags;
+
+ struct aec_stream strm1;
+ size_t *encode_offsets_ptr;
+ size_t encode_offsets_size;
+ PREPARE_ENCODE_WITH_OFFSETS(&strm1, ctx, flags, encode_offsets_ptr, &encode_offsets_size);
+
+ struct aec_stream strm2;
+ size_t *decode_offsets_ptr;
+ size_t decode_offsets_size;
+ PREPARE_DECODE_WITH_OFFSETS(&strm2, ctx, flags, decode_offsets_ptr, &decode_offsets_size);
+ size_t size = decode_offsets_size > 10 ? 10 : decode_offsets_size;
+
+ for (size_t i = 0; i < encode_offsets_size; ++i) {
+ if (encode_offsets_ptr[i] != decode_offsets_ptr[i]) {
+ fprintf(stderr, "Error: encode_offsets_ptr[%zu] = %zu, decode_offsets_ptr[%zu] = %zu\n", i, encode_offsets_ptr[i], i, decode_offsets_ptr[i]);
+ return 103;
+ }
+ }
+
+ free(decode_offsets_ptr);
+ free(encode_offsets_ptr);
+ return status;
+}
+
+int main(void)
+{
+ int status = AEC_OK;
+ size_t ns[] = {1, 255, 256, 255*10, 256*10, 67000};
+ size_t rsis[] = {1, 2, 255, 256, 512, 1024, 4095, 4096};
+ size_t bss[] = {8, 16, 32, 64};
+ size_t bpss[] = {1, 7, 8, 9, 15, 16, 17, 23, 24, 25, 31, 32};
+ data_generator_t data_generators[] = {data_generator_zero, data_generator_random, data_generator_incr};
+
+ for (size_t n_i = 0; n_i < sizeof(ns) / sizeof(ns[0]); ++n_i) {
+ for (size_t rsi_i = 0; rsi_i < sizeof(rsis) / sizeof(rsis[0]); ++rsi_i) {
+ for (size_t bs_i = 0; bs_i < sizeof(bss) / sizeof(bss[0]); ++bs_i) {
+ for (size_t bps_i = 0; bps_i < sizeof(bpss) / sizeof(bpss[0]); ++bps_i) {
+ struct aec_context ctx;
+ ctx.nvalues = ns[n_i];
+ ctx.flags = AEC_DATA_PREPROCESS;
+ ctx.rsi = rsis[rsi_i];
+ ctx.block_size = bss[bs_i];
+ ctx.bits_per_sample = bpss[bps_i];
+ ctx.bytes_per_sample = get_input_bytes(&ctx);
+ size_t input_size = ctx.nvalues * ctx.bytes_per_sample;
+ ctx.obuf_len = input_size;
+ ctx.ebuf_len = input_size * 67 / 64 + 256;
+ ctx.dbuf_len = input_size;
+ ctx.obuf = malloc(ctx.obuf_len);
+ ctx.ebuf = malloc(ctx.ebuf_len);
+ ctx.dbuf = malloc(ctx.dbuf_len);
+ if (ctx.obuf == NULL || ctx.ebuf == NULL || ctx.dbuf == NULL) {
+ fprintf(stderr, "Error: Failed allocating memory\n");
+ exit(1);
+ }
+
+ for (size_t i = 0; i < sizeof(data_generators) / sizeof(data_generators[0]); ++i) {
+ data_generators[i](&ctx);
+
+ status = test_rsi_at(&ctx);
+ fprintf(stderr,
+ "Testing test_rsi_at() "
+ "nvalues=%zu, rsi=%zu, block_size=%zu, bits_per_sample=%zu ... %s\n",
+ ns[n_i], rsis[rsi_i], bss[bs_i], bpss[bps_i], status == AEC_OK ? CHECK_PASS : CHECK_FAIL);
+ if (status != AEC_OK)
+ return status;
+
+ status = test_read(&ctx);
+ fprintf(stderr,
+ "Testing test_read() "
+ "nvalues=%zu, rsi=%zu, block_size=%zu, bits_per_sample=%zu ... %s\n",
+ ns[n_i], rsis[rsi_i], bss[bs_i], bpss[bps_i], status == AEC_OK ? CHECK_PASS : CHECK_FAIL);
+ if (status != AEC_OK)
+ return status;
+
+ status = test_offsets(&ctx);
+ fprintf(stderr,
+ "Testing test_offsets() "
+ "nvalues=%zu, rsi=%zu, block_size=%zu, bits_per_sample=%zu ... %s\n",
+ ns[n_i], rsis[rsi_i], bss[bs_i], bpss[bps_i], status == AEC_OK ? CHECK_PASS : CHECK_FAIL);
+ if (status != AEC_OK)
+ return status;
+ }
+ free(ctx.obuf);
+ free(ctx.ebuf);
+ free(ctx.dbuf);
+ }
+ }
+ }
+ }
+ return status;
+}
diff --git a/tests/check_seeking.c b/tests/check_seeking.c
index 2cb9d83..1fca2f9 100644
--- a/tests/check_seeking.c
+++ b/tests/check_seeking.c
@@ -1,10 +1,12 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+
#include "check_aec.h"
#define BUF_SIZE 1024 * 3
+
void shift_cdata(struct test_state *state, unsigned char *cbuf_unshifted,
int byte_offset, int bit_offset)
{
@@ -113,7 +115,7 @@ int encode_decode_large_seek(struct test_state *state)
printf("Init failed.\n");
return 99;
}
- status = aec_buffer_seek(strm, byte_offset, bit_offset);
+ status = aec_buffer_seek(strm, byte_offset * 8 + bit_offset);
if (status != AEC_OK) {
printf("Seeking failed.\n");
return 99;
--
GitLab