From 477a055c211b8c7bfd1593e60c33b3aab7e4aa38 Mon Sep 17 00:00:00 2001
From: Mathis Rosenhauer <rosenhauer@dkrz.de>
Date: Thu, 30 Aug 2018 16:04:30 +0200
Subject: [PATCH] Avoid undefined xmax for bits_per_sample==1
Thanks to Kurt Schwehr for finding this
---
src/decode.c | 4 ++--
src/encode.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/decode.c b/src/decode.c
index 5be506c..309a835 100644
--- a/src/decode.c
+++ b/src/decode.c
@@ -739,11 +739,11 @@ int aec_decode_init(struct aec_stream *strm)
}
if (strm->flags & AEC_DATA_SIGNED) {
- state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1);
+ state->xmax = (INT64_C(1) << (strm->bits_per_sample - 1)) - 1;
state->xmin = ~state->xmax;
} else {
state->xmin = 0;
- state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample);
+ state->xmax = (UINT64_C(1) << strm->bits_per_sample) - 1;
}
state->in_blklen = (strm->block_size * strm->bits_per_sample
diff --git a/src/encode.c b/src/encode.c
index db5c009..63dc6f4 100644
--- a/src/encode.c
+++ b/src/encode.c
@@ -849,12 +849,12 @@ int aec_encode_init(struct aec_stream *strm)
state->rsi_len = strm->rsi * strm->block_size * state->bytes_per_sample;
if (strm->flags & AEC_DATA_SIGNED) {
- state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1);
+ state->xmax = (INT64_C(1) << (strm->bits_per_sample - 1)) - 1;
state->xmin = ~state->xmax;
state->preprocess = preprocess_signed;
} else {
+ state->xmax = (UINT64_C(1) << strm->bits_per_sample) - 1;
state->xmin = 0;
- state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample);
state->preprocess = preprocess_unsigned;
}
--
GitLab