merge default into releaserelease-pypy2.7-v7.2.0rc0

9 files changed, 81 insertions, 42 deletions

diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py
index 1866516031..308211e196 100644
--- a/extra_tests/cffi_tests/cffi1/test_re_python.py
+++ b/extra_tests/cffi_tests/cffi1/test_re_python.py
@@ -66,7 +66,7 @@ def setup_module(mod):
     int add43(int, ...);
     int globalvar42;
     const int globalconst42;
-    const char *const globalconsthello = "hello";
+    const char *const globalconsthello;
     int no_such_function(int);
     int no_such_globalvar;
     struct foo_s;
diff --git a/extra_tests/test_semlock.py b/extra_tests/test_semlock.py
index 712d550028..1b61331756 100644
--- a/extra_tests/test_semlock.py
+++ b/extra_tests/test_semlock.py
@@ -1,5 +1,6 @@
 from _multiprocessing import SemLock
 from threading import Thread
+import thread
 import time
 
 
@@ -17,10 +18,19 @@ def test_notify_all():
             lock.release()
 
     threads = [Thread(target=f, args=(i,)) for i in range(N_THREADS)]
+    n_started = N_THREADS
     with lock:
         for t in threads:
-            t.start()
+            try:
+                t.start()
+            except thread.error:
+                # too many threads for this system
+                t.started = False
+                n_started -= 1
+            else:
+                t.started = True
         time.sleep(0.1)
     for t in threads:
-        t.join()
-    assert len(results) == N_THREADS
+        if t.started:
+            t.join()
+    assert len(results) == n_started
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
index 679ae057a2..e7956a79cf 100644
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -403,7 +403,7 @@ class CTypesBackend(object):
                         source = _cast_source_to_int(source)
                     return cls(bool(source))
                 def __int__(self):
-                    return self._value
+                    return int(self._value)
 
             if kind == 'char':
                 @classmethod
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
index b36da31ef9..bb97cc769e 100644
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -313,6 +313,10 @@ def detect_custom_layout(w_ctype, sflags, cdef_value, compiler_value,
                     cdef_value, compiler_value, w_ctype.name)
         w_ctype._custom_field_pos = True
 
+def roundup_bytes(bytes, bit):
+    assert bit == (bit & 7)
+    return bytes + (bit > 0)
+
 @unwrap_spec(w_ctype=ctypeobj.W_CType, totalsize=int, totalalignment=int,
              sflags=int, pack=int)
 def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
@@ -334,8 +338,9 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
 
     is_union = isinstance(w_ctype, ctypestruct.W_CTypeUnion)
     alignment = 1
-    boffset = 0         # this number is in *bits*, not bytes!
-    boffsetmax = 0      # the maximum value of boffset, in bits too
+    byteoffset = 0     # the real value is 'byteoffset+bitoffset*8', which

+    bitoffset = 0      #     counts the offset in bits

+    byteoffsetmax = 0  # the maximum value of byteoffset-rounded-up-to-byte

     prev_bitfield_size = 0
     prev_bitfield_free = 0
     fields_w = space.listview(w_fields)
@@ -380,7 +385,7 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
                 with_var_array = True
         #
         if is_union:
-            boffset = 0         # reset each field at offset 0
+            byteoffset = bitoffset = 0        # reset each field at offset 0
         #
         # update the total alignment requirement, but skip it if the
         # field is an anonymous bitfield or if SF_PACKED
@@ -410,19 +415,24 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
             else:
                 bs_flag = ctypestruct.W_CField.BS_REGULAR
 
-            # align this field to its own 'falign' by inserting padding
-            boffsetorg = (boffset + falignorg*8-1) & ~(falignorg*8-1)
-            boffset = (boffset + falign*8-1) & ~(falign*8-1)
-            if boffsetorg != boffset:
+            # align this field to its own 'falign' by inserting padding.
+            # first, pad to the next byte,

+            # then pad to 'falign' or 'falignorg' bytes
+            byteoffset = roundup_bytes(byteoffset, bitoffset)

+            bitoffset = 0

+            byteoffsetorg = (byteoffset + falignorg-1) & ~(falignorg-1)

+            byteoffset = (byteoffset + falign-1) & ~(falign-1)

+
+            if byteoffsetorg != byteoffset:
                 with_packed_change = True
 
             if foffset >= 0:
                 # a forced field position: ignore the offset just computed,
                 # except to know if we must set 'custom_field_pos'
-                detect_custom_layout(w_ctype, sflags, boffset // 8, foffset,
+                detect_custom_layout(w_ctype, sflags, byteoffset, foffset,
                                      "wrong offset for field '",
                                      fname, "'")
-                boffset = foffset * 8
+                byteoffset = foffset
 
             if (fname == '' and
                     isinstance(ftype, ctypestruct.W_CTypeStructOrUnion)):
@@ -432,7 +442,7 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8, fflags)
+                    fld = srcfld.make_shifted(byteoffset, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -442,13 +452,13 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                fld = ctypestruct.W_CField(ftype, byteoffset, bs_flag, -1,
                                            fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
             if ftype.size >= 0:
-                boffset += ftype.size * 8
+                byteoffset += ftype.size
             prev_bitfield_size = 0
 
         else:
@@ -474,7 +484,7 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
             # compute the starting position of the theoretical field
             # that covers a complete 'ftype', inside of which we will
             # locate the real bitfield
-            field_offset_bytes = boffset // 8
+            field_offset_bytes = byteoffset
             field_offset_bytes &= ~(falign - 1)
 
             if fbitsize == 0:
@@ -484,11 +494,13 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
                                 w_ctype.name, fname)
                 if (sflags & SF_MSVC_BITFIELDS) == 0:
                     # GCC's notion of "ftype :0;"
-                    # pad boffset to a value aligned for "ftype"
-                    if boffset > field_offset_bytes * 8:
+                    # pad byteoffset to a value aligned for "ftype"
+                    if (roundup_bytes(byteoffset, bitoffset) >
+                                                    field_offset_bytes):
                         field_offset_bytes += falign
-                        assert boffset < field_offset_bytes * 8
-                    boffset = field_offset_bytes * 8
+                        assert byteoffset < field_offset_bytes
+                    byteoffset = field_offset_bytes
+                    bitoffset = 0
                 else:
                     # MSVC's notion of "ftype :0;
                     # Mostly ignored.  It seems they only serve as
@@ -503,7 +515,8 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
 
                     # Can the field start at the offset given by 'boffset'?  It
                     # can if it would entirely fit into an aligned ftype field.
-                    bits_already_occupied = boffset - (field_offset_bytes * 8)
+                    bits_already_occupied = (
+                        (byteoffset-field_offset_bytes) * 8 + bitoffset)
 
                     if bits_already_occupied + fbitsize > 8 * ftype.size:
                         # it would not fit, we need to start at the next
@@ -516,13 +529,16 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
                                         "the previous field",
                                         w_ctype.name, fname)
                         field_offset_bytes += falign
-                        assert boffset < field_offset_bytes * 8
-                        boffset = field_offset_bytes * 8
+                        assert byteoffset < field_offset_bytes
+                        byteoffset = field_offset_bytes
+                        bitoffset = 0
                         bitshift = 0
                     else:
                         bitshift = bits_already_occupied
                         assert bitshift >= 0
-                    boffset += fbitsize
+                    bitoffset += fbitsize
+                    byteoffset += (bitoffset >> 3)
+                    bitoffset &= 7
 
                 else:
                     # MSVC's algorithm
@@ -537,14 +553,17 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
                         bitshift = 8 * prev_bitfield_size - prev_bitfield_free
                     else:
                         # no: start a new full field
-                        boffset = (boffset + falign*8-1) & ~(falign*8-1)
-                        boffset += ftype.size * 8
+                        byteoffset = roundup_bytes(byteoffset, bitoffset)
+                        bitoffset = 0
+                        # align
+                        byteoffset = (byteoffset + falign-1) & ~(falign-1)
+                        byteoffset += ftype.size
                         bitshift = 0
                         prev_bitfield_size = ftype.size
                         prev_bitfield_free = 8 * prev_bitfield_size
                     #
                     prev_bitfield_free -= fbitsize
-                    field_offset_bytes = boffset / 8 - ftype.size
+                    field_offset_bytes = byteoffset - ftype.size
 
                 if sflags & SF_GCC_BIG_ENDIAN:
                     bitshift = 8 * ftype.size - fbitsize- bitshift
@@ -555,14 +574,13 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
                     fields_list.append(fld)
                     fields_dict[fname] = fld
 
-        if boffset > boffsetmax:
-            boffsetmax = boffset
+        if roundup_bytes(byteoffset, bitoffset) > byteoffsetmax:
+            byteoffsetmax = roundup_bytes(byteoffset, bitoffset)
 
     # Like C, if the size of this structure would be zero, we compute it
     # as 1 instead.  But for ctypes support, we allow the manually-
     # specified totalsize to be zero in this case.
-    boffsetmax = (boffsetmax + 7) // 8      # bits -> bytes
-    alignedsize = (boffsetmax + alignment - 1) & ~(alignment - 1)
+    alignedsize = (byteoffsetmax + alignment - 1) & ~(alignment - 1)
     alignedsize = alignedsize or 1
 
     if totalsize < 0:
@@ -570,10 +588,10 @@ def complete_struct_or_union(space, w_ctype, w_fields, w_ignored=None,
     else:
         detect_custom_layout(w_ctype, sflags, alignedsize, totalsize,
                              "wrong total size")
-        if totalsize < boffsetmax:
+        if totalsize < byteoffsetmax:
             raise oefmt(space.w_TypeError,
                 "%s cannot be of size %d: there are fields at least up to %d",
-                w_ctype.name, totalsize, boffsetmax)
+                w_ctype.name, totalsize, byteoffsetmax)
     if totalalignment < 0:
         totalalignment = alignment
     else:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
index e32586a75d..e82e5e3efd 100644
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -4432,3 +4432,11 @@ def test_cannot_call_null_function_pointer():
     f = cast(BFunc, 0)
     with pytest.raises(RuntimeError):
         f(40, 2)
+
+def test_huge_structure():
+    BChar = new_primitive_type("char")
+    BArray = new_array_type(new_pointer_type(BChar), sys.maxsize)
+    assert sizeof(BArray) == sys.maxsize
+    BStruct = new_struct_type("struct foo")
+    complete_struct_or_union(BStruct, [('a1', BArray, -1)])
+    assert sizeof(BStruct) == sys.maxsize
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
index 542f33003f..318a32a05b 100644
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -425,11 +425,9 @@ class Test__ffi(BaseTestPyPyC):
         setarrayitem_raw(i153, 0, i106, descr=...)
         p156 = getfield_gc_r(p48, descr=...)
         i158 = getfield_raw_i(..., descr=...)
-        i160 = int_sub(i158, 16)
-        setfield_raw(#, i160, descr=...)
         setfield_gc(p48, p49, descr=...)
         setfield_gc(p134, ConstPtr(null), descr=...)
-        i160 = int_lt(i160, 0)
-        guard_false(i160, descr=...)
+        i159 = int_lt(i158, 0)
+        guard_false(i159, descr=...)
         jump(..., descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
index 1e20bdc81b..ca6543cd91 100644
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import sys
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py
index 4b9c2a8b30..3e4fe93a00 100755
--- a/pypy/tool/import_cffi.py
+++ b/pypy/tool/import_cffi.py
@@ -20,6 +20,10 @@ def mangle(lines, ext):
     else:
         raise AssertionError(ext)
 
+def fixeol(s):
+    s = s.replace('\r\n', '\n')
+    return s
+
 def main(cffi_dir):
     cffi_dir = py.path.local(cffi_dir)
     rootdir = py.path.local(__file__).join('..', '..', '..')
@@ -29,13 +33,13 @@ def main(cffi_dir):
     test_dest.ensure(dir=1)
     for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) +
               list(cffi_dir.join('cffi').visit(fil='*.h'))):
-        cffi_dest.join('..', p.relto(cffi_dir)).write(p.read())
+        cffi_dest.join('..', p.relto(cffi_dir)).write_binary(fixeol(p.read()))
     for p in (list(cffi_dir.join('testing').visit(fil='*.py')) +
               list(cffi_dir.join('testing').visit(fil='*.h')) +
               list(cffi_dir.join('testing').visit(fil='*.c'))):
         path = test_dest.join(p.relto(cffi_dir.join('testing')))
         path.join('..').ensure(dir=1)
-        path.write(''.join(mangle(p.readlines(), p.ext)))
+        path.write_binary(fixeol(''.join(mangle(p.readlines(), p.ext))))
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
index 63f9b854d6..7d957db5f7 100755
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -3,7 +3,7 @@ pmaj=2  # python main version: 2 or 3
 pmin=7  # python minor version
 exe=pypy3 # pypy3 or pypy
 maj=7
-min=1
+min=2
 rev=0