From 11b5c567124a61d8e8249a0fbcce47f2688573c6 Mon Sep 17 00:00:00 2001
From: Kaz Kylheku <kaz@kylheku.com>
Date: Sat, 25 May 2019 22:03:54 -0700
Subject: bugfix: c_num won't convert most negative value.

This bug causes a problem particularly in FFI. THe conversion
of an integer to the FFI int type begins by conversion via
c_num to the cnum type, which is at least as wide as int.
Alas, the INT_MIN value (e.g. #x-80000000 on 32 bits) will
not convert!

Fixing this has a ripple effect. We alter the INT_PTR_MIN
constant to include this value. This causes the derived
NUM_MIN to also include the extra negative value, extending
the fixnum range by one value. Everything seems to be okay.

* configure: Decrease value of INT_PTR_MIN and
DOUBLE_INTPTR_MIN constants by one to include the most
negative two's complement value.

* ffi.c (make_ffi_type_enum): We must not subtract 1 from
INT_PTR_MIN here any more.

* mpi.c (mp_in_range): If the bignum is negative, then extend
the range check by one value, so that we don't reject the
most negative two's complement value.
---
 configure | 4 ++--
 ffi.c     | 2 +-
 mpi/mpi.c | 5 +++--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/configure b/configure
index 8ee672b4..064ea3ee 100755
--- a/configure
+++ b/configure
@@ -1256,12 +1256,12 @@ printf "typedef $intptr int_ptr_t;\n" >> config.h
 printf "typedef unsigned $intptr uint_ptr_t;\n" >> config.h
 intptr_max_expr="((((convert(int_ptr_t, 1) << $((SIZEOF_PTR * SIZEOF_BYTE - 2))) - 1) << 1) + 1)"
 printf "#define INT_PTR_MAX %s\n" "$intptr_max_expr" >> config.h
-printf "#define INT_PTR_MIN (-INT_PTR_MAX)\n" >> config.h
+printf "#define INT_PTR_MIN (-INT_PTR_MAX-1)\n" >> config.h
 printf "#define UINT_PTR_MAX (convert(uint_ptr_t, -1))\n" >> config.h
 double_intptr_max_expr="((((convert(double_intptr_t, 1) << $((2 * SIZEOF_PTR * SIZEOF_BYTE - 2))) - 1) << 1) + 1)"
 printf "#define SIZEOF_DOUBLE_INTPTR (2*SIZEOF_PTR)\n" >> config.h
 printf "#define DOUBLE_INTPTR_MAX %s\n" "$double_intptr_max_expr" >> config.h
-printf "#define DOUBLE_INTPTR_MIN (-DOUBLE_INTPTR_MAX)\n" >> config.h
+printf "#define DOUBLE_INTPTR_MIN (-DOUBLE_INTPTR_MAX-1)\n" >> config.h
 printf "#define DOUBLE_UINTPTR_MAX (convert(double_uintptr_t, -1))\n" >> config.h
 
 if [ -n "$longlong" ] && [ $SIZEOF_LONGLONG_T -eq $(( 2 * SIZEOF_PTR )) ]
diff --git a/ffi.c b/ffi.c
index 7dd5b9e5..61551228 100644
--- a/ffi.c
+++ b/ffi.c
@@ -3018,7 +3018,7 @@ static val make_ffi_type_enum(val syntax, val enums,
   val num_sym = make_hash(nil, nil, nil);
   val obj = cobj(coerce(mem_t *, tft), ffi_type_s, &ffi_type_enum_ops);
   cnum lowest = INT_PTR_MAX;
-  cnum highest = INT_PTR_MIN - 1;
+  cnum highest = INT_PTR_MIN;
   cnum cur = -1;
   ucnum count = 0;
   val iter;
diff --git a/mpi/mpi.c b/mpi/mpi.c
index 9ece8047..5befd2b3 100644
--- a/mpi/mpi.c
+++ b/mpi/mpi.c
@@ -463,8 +463,9 @@ int mp_in_range(mp_int *mp, uint_ptr_t lim, int unsig)
 {
   const unsigned ptrnd = (SIZEOF_PTR + MP_DIGIT_BIT - 1) / MP_DIGIT_BIT;
   mp_size nd = USED(mp);
+  int neg = ISNEG(mp);
 
-  if (unsig && ISNEG(mp))
+  if (unsig && neg)
     return 0;
 
   if (nd < ptrnd)
@@ -476,7 +477,7 @@ int mp_in_range(mp_int *mp, uint_ptr_t lim, int unsig)
   {
     mp_digit top = DIGITS(mp)[ptrnd - 1];
     lim >>= ((ptrnd - 1) * MP_DIGIT_BIT);
-    return top <= lim;
+    return (top - neg) <= lim;
   }
 }
 
-- 
cgit v1.2.3