summaryrefslogtreecommitdiff
path: root/libguile/numbers.c
diff options
context:
space:
mode:
authorMark H Weaver <mhw@netris.org>2018-10-14 05:29:52 -0400
committerAndy Wingo <wingo@pobox.com>2019-05-23 16:09:45 +0200
commit00973cbd2efe192b86f4e2363fccae5b75b09b3f (patch)
tree83f40356a45416f80089d3aa48692161a4eca88e /libguile/numbers.c
parente6100f64bbab1b2bac085c0ea5250163197df4a9 (diff)
downloadguile-00973cbd2efe192b86f4e2363fccae5b75b09b3f.tar.gz
In 'ash' and 'round-ash', handle right shift count of LONG_MIN.
Fixes <https://bugs.gnu.org/21901>. Reported by Zefram <zefram@fysh.org>. * libguile/numbers.c: Add another top-level 'verify' to ensure that LONG_MIN is not a fixnum. (scm_ash, scm_round_ash): Ensure that when the shift count is LONG_MIN, it is not handled via the normal code path, to avoid signed overflow when the shift count is negated. * test-suite/tests/numbers.test: Add tests.
Diffstat (limited to 'libguile/numbers.c')
-rw-r--r--libguile/numbers.c13
1 files changed, 11 insertions, 2 deletions
diff --git a/libguile/numbers.c b/libguile/numbers.c
index 21dcdb62c..a11ed3a8c 100644
--- a/libguile/numbers.c
+++ b/libguile/numbers.c
@@ -5053,6 +5053,11 @@ round_right_shift_exact_integer (SCM n, long count)
assert (0);
}
+/* 'scm_ash' and 'scm_round_ash' assume that fixnums fit within a long,
+ and moreover that they can be negated without overflow. */
+verify (SCM_MOST_NEGATIVE_FIXNUM >= LONG_MIN + 1
+ && SCM_MOST_POSITIVE_FIXNUM <= LONG_MAX);
+
SCM_DEFINE (scm_ash, "ash", 2, 0, 0,
(SCM n, SCM count),
"Return @math{floor(@var{n} * 2^@var{count})}.\n"
@@ -5078,7 +5083,9 @@ SCM_DEFINE (scm_ash, "ash", 2, 0, 0,
if (SCM_I_INUMP (count)) /* fast path, not strictly needed */
bits_to_shift = SCM_I_INUM (count);
- else if (scm_is_signed_integer (count, LONG_MIN, LONG_MAX))
+ else if (scm_is_signed_integer (count, LONG_MIN + 1, LONG_MAX))
+ /* We exclude LONG_MIN to ensure that 'bits_to_shift' can be
+ negated without overflowing. */
bits_to_shift = scm_to_long (count);
else if (scm_is_false (scm_positive_p (scm_sum (scm_integer_length (n),
count))))
@@ -5130,7 +5137,9 @@ SCM_DEFINE (scm_round_ash, "round-ash", 2, 0, 0,
if (SCM_I_INUMP (count)) /* fast path, not strictly needed */
bits_to_shift = SCM_I_INUM (count);
- else if (scm_is_signed_integer (count, LONG_MIN, LONG_MAX))
+ else if (scm_is_signed_integer (count, LONG_MIN + 1, LONG_MAX))
+ /* We exclude LONG_MIN to ensure that 'bits_to_shift' can be
+ negated without overflowing. */
bits_to_shift = scm_to_long (count);
else if (scm_is_true (scm_negative_p (scm_sum (scm_integer_length (n),
count)))