From 6f7ec60c7e7f05f5ab0b1cf6b7b0945e44afcd4b Mon Sep 17 00:00:00 2001 From: Erik de Castro Lopo Date: Wed, 17 Jul 2013 18:38:45 +1000 Subject: [PATCH] stream_encoder.c : Fix an arithmetic overflow in the RICE2 partitioner. For a specific 24 bit WAV file provided by Leigh Dyer http://lists.xiph.org/pipermail/flac-dev/2013-July/004284.html encoding with compression level 7 was generating a file a couple of orders of magintude larger than the original. Debugging showed that variable abs_residual_partition_sum (a FLAC__uint32) in function precompute_partition_info_sums_() was suffering from an arithmetic overflowing on some 24 bit input files although this value overflowing did not always cause larger output files. Since the value abs_residual_partition_sum is eventually stored in an array of FLAC__uint64, it makes sense to make abs_residual_partition_sum a FLAC__uint64 anyway. Debugging this problem was made easier by use of the Clang compiler's -fsanitize=integer option. --- src/libFLAC/stream_encoder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libFLAC/stream_encoder.c b/src/libFLAC/stream_encoder.c index 290d038c..4045189f 100644 --- a/src/libFLAC/stream_encoder.c +++ b/src/libFLAC/stream_encoder.c @@ -3784,7 +3784,7 @@ void precompute_partition_info_sums_( /* slightly pessimistic but still catches all common cases */ /* WATCHOUT: "+ bps" is an assumption that the average residual magnitude will not be more than "bps" bits */ if(FLAC__bitmath_ilog2(default_partition_samples) + bps < 32) { - FLAC__uint32 abs_residual_partition_sum; + FLAC__uint64 abs_residual_partition_sum; for(partition = residual_sample = 0; partition < partitions; partition++) { end += default_partition_samples;