Commit 477a055c authored by Mathis Rosenhauer's avatar Mathis Rosenhauer

Avoid undefined xmax for bits_per_sample==1

Thanks to Kurt Schwehr for finding this
parent b60f0293
...@@ -739,11 +739,11 @@ int aec_decode_init(struct aec_stream *strm) ...@@ -739,11 +739,11 @@ int aec_decode_init(struct aec_stream *strm)
} }
if (strm->flags & AEC_DATA_SIGNED) { if (strm->flags & AEC_DATA_SIGNED) {
state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1); state->xmax = (INT64_C(1) << (strm->bits_per_sample - 1)) - 1;
state->xmin = ~state->xmax; state->xmin = ~state->xmax;
} else { } else {
state->xmin = 0; state->xmin = 0;
state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample); state->xmax = (UINT64_C(1) << strm->bits_per_sample) - 1;
} }
state->in_blklen = (strm->block_size * strm->bits_per_sample state->in_blklen = (strm->block_size * strm->bits_per_sample
......
...@@ -849,12 +849,12 @@ int aec_encode_init(struct aec_stream *strm) ...@@ -849,12 +849,12 @@ int aec_encode_init(struct aec_stream *strm)
state->rsi_len = strm->rsi * strm->block_size * state->bytes_per_sample; state->rsi_len = strm->rsi * strm->block_size * state->bytes_per_sample;
if (strm->flags & AEC_DATA_SIGNED) { if (strm->flags & AEC_DATA_SIGNED) {
state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample + 1); state->xmax = (INT64_C(1) << (strm->bits_per_sample - 1)) - 1;
state->xmin = ~state->xmax; state->xmin = ~state->xmax;
state->preprocess = preprocess_signed; state->preprocess = preprocess_signed;
} else { } else {
state->xmax = (UINT64_C(1) << strm->bits_per_sample) - 1;
state->xmin = 0; state->xmin = 0;
state->xmax = UINT32_MAX >> (32 - strm->bits_per_sample);
state->preprocess = preprocess_unsigned; state->preprocess = preprocess_unsigned;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment