merge py3.6 into releaserelease-pypy3.6-v7.3.0rc3

13 files changed, 226 insertions, 43 deletions

diff --git a/.hgtags b/.hgtags
index f844d9d537..b3cf7a47b5 100644
--- a/.hgtags
+++ b/.hgtags
@@ -59,3 +59,5 @@ de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
 5da45ced70e515f94686be0df47c59abd1348ebc release-pypy3.6-v7.2.0
 e6471221abc16f4584a07fbfeece7ebcaeb7fc38 release-pypy2.7-v7.3.0rc1
 533398cfd64e5146a07c4824e90a1b629c8b6523 release-pypy3.6-v7.3.0rc1
+285307a0f5a77ffa46781b5c54c52eb1c385081d release-pypy2.7-v7.3.0rc2
+008914050baeedb6d3ca30fe26ef43b78bb63841 release-pypy3.6-v7.3.0rc2
diff --git a/extra_tests/test_datetime.py b/extra_tests/test_datetime.py
index 2fe26f194e..1cd8c783a6 100644
--- a/extra_tests/test_datetime.py
+++ b/extra_tests/test_datetime.py
@@ -350,3 +350,31 @@ def test_subclass_datetime():
     d2 = d.replace(hour=7)
     assert type(d2) is MyDatetime
     assert d2 == datetime.datetime(2016, 4, 5, 7, 2, 3)
+
+def test_normalize_pair():
+    normalize = datetime._normalize_pair
+
+    assert normalize(1, 59, 60) == (1, 59)
+    assert normalize(1, 60, 60) == (2, 0)
+    assert normalize(1, 95, 60) == (2, 35)
+
+def test_normalize_date():
+    normalize = datetime._normalize_date
+
+    # Huge year is caught correctly
+    with pytest.raises(OverflowError):
+        normalize(1000 * 1000, 1, 1)
+    # Normal dates should be unchanged
+    assert normalize(3000, 1, 1) == (3000, 1, 1)
+    # Month overflows year boundary
+    assert normalize(2001, 24, 1) == (2002, 12, 1)
+    # Day overflows month boundary
+    assert normalize(2001, 14, 31) == (2002, 3, 3)
+    # Leap years? :S
+    assert normalize(2001, 1, 61) == (2001, 3, 2)
+    assert normalize(2000, 1, 61) == (2000, 3, 1)
+
+def test_normalize_datetime():
+    normalize = datetime._normalize_datetime
+    abnormal = (2002, 13, 35, 30, 95, 75, 1000001)
+    assert normalize(*abnormal) == (2003, 2, 5, 7, 36, 16, 1)
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
index b3a4a52d16..87523131f0 100644
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -1416,9 +1416,13 @@ class datetime(date):
             self.__setstate(year, month)
             self._hashcode = -1
             return self
-        year, month, day = _check_date_fields(year, month, day)
-        hour, minute, second, microsecond, fold = _check_time_fields(
-            hour, minute, second, microsecond, fold)
+        elif isinstance(year, tuple) and len(year) == 7:
+            # Internal operation - numbers guaranteed to be valid
+            year, month, day, hour, minute, second, microsecond = year
+        else:
+            year, month, day = _check_date_fields(year, month, day)
+            hour, minute, second, microsecond, fold = _check_time_fields(
+                hour, minute, second, microsecond, fold)
         _check_tzinfo_arg(tzinfo)
         self = dateinterop.__new__(cls)
         self._year = int(year)
@@ -1890,20 +1894,18 @@ class datetime(date):
         "Add a datetime and a timedelta."
         if not isinstance(other, timedelta):
             return NotImplemented
-        delta = timedelta(self.toordinal(),
-                          hours=self._hour,
-                          minutes=self._minute,
-                          seconds=self._second,
-                          microseconds=self._microsecond)
-        delta += other
-        hour, rem = divmod(delta.seconds, 3600)
-        minute, second = divmod(rem, 60)
-        if 0 < delta.days <= _MAXORDINAL:
-            return datetime.combine(date.fromordinal(delta.days),
-                                    time(hour, minute, second,
-                                         delta.microseconds,
-                                         tzinfo=self._tzinfo))
-        raise OverflowError("result out of range")
+
+        result = _normalize_datetime(
+            self._year,
+            self._month,
+            self._day + other.days,
+            self._hour,
+            self._minute,
+            self._second + other.seconds,
+            self._microsecond + other.microseconds,
+        )
+
+        return datetime(result, tzinfo=self._tzinfo)
 
     __radd__ = __add__
 
@@ -2000,6 +2002,65 @@ datetime.max = datetime(9999, 12, 31, 23, 59, 59, 999999)
 datetime.resolution = timedelta(microseconds=1)
 
 
+def _normalize_pair(hi, lo, factor):
+    if not 0 <= lo <= factor-1:
+        inc, lo = divmod(lo, factor)
+        hi += inc
+    return hi, lo
+
+
+def _normalize_datetime(y, m, d, hh, mm, ss, us):
+    # Normalize all the inputs, and store the normalized values.
+    ss, us = _normalize_pair(ss, us, 1000000)
+    mm, ss = _normalize_pair(mm, ss, 60)
+    hh, mm = _normalize_pair(hh, mm, 60)
+    d, hh = _normalize_pair(d, hh, 24)
+    y, m, d = _normalize_date(y, m, d)
+    return y, m, d, hh, mm, ss, us
+
+
+def _normalize_date(year, month, day):
+    # That was easy.  Now it gets muddy:  the proper range for day
+    # can't be determined without knowing the correct month and year,
+    # but if day is, e.g., plus or minus a million, the current month
+    # and year values make no sense (and may also be out of bounds
+    # themselves).
+    # Saying 12 months == 1 year should be non-controversial.
+    if not 1 <= month <= 12:
+        year, month = _normalize_pair(year, month-1, 12)
+        month += 1
+        assert 1 <= month <= 12
+
+    # Now only day can be out of bounds (year may also be out of bounds
+    # for a datetime object, but we don't care about that here).
+    # If day is out of bounds, what to do is arguable, but at least the
+    # method here is principled and explainable.
+    dim = _days_in_month(year, month)
+    if not 1 <= day <= dim:
+        # Move day-1 days from the first of the month.  First try to
+        # get off cheap if we're only one day out of range (adjustments
+        # for timezone alone can't be worse than that).
+        if day == 0:    # move back a day
+            month -= 1
+            if month > 0:
+                day = _days_in_month(year, month)
+            else:
+                year, month, day = year-1, 12, 31
+        elif day == dim + 1:    # move forward a day
+            month += 1
+            day = 1
+            if month > 12:
+                month = 1
+                year += 1
+        else:
+            ordinal = _ymd2ord(year, month, 1) + (day - 1)
+            year, month, day = _ord2ymd(ordinal)
+
+    if not MINYEAR <= year <= MAXYEAR:
+        raise OverflowError("date value out of range")
+    return year, month, day
+
+
 def _isoweek1monday(year):
     # Helper to calculate the day number of the Monday starting week 1
     # XXX This could be done more efficiently
diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py
index 028c42b1f9..3130766f1c 100644
--- a/lib-python/3/subprocess.py
+++ b/lib-python/3/subprocess.py
@@ -1657,3 +1657,9 @@ def _pypy_install_libs_after_virtualenv(target_executable):
             src_library = os.path.join(src_dir, libname)
             if os.path.exists(src_library):
                 caller.f_globals['copyfile'](src_library, dest_library)
+        src_lib = os.path.join(src_dir, '../lib')
+        if os.path.exists(src_lib):
+            # portable build
+            import shutil
+            shutil.copytree(src_lib, os.path.join(dest_dir, '../lib'))
+
diff --git a/lib-python/3/venv/__init__.py b/lib-python/3/venv/__init__.py
index 2bda178de4..e32e0d6147 100644
--- a/lib-python/3/venv/__init__.py
+++ b/lib-python/3/venv/__init__.py
@@ -233,6 +233,16 @@ class EnvBuilder:
                     copier(src_library, dest_library)
                     if not os.path.islink(dest_library):
                         os.chmod(dest_library, 0o755)
+            libsrc = os.path.join(context.python_dir, '..', 'lib')
+            if os.path.exists(libsrc):
+                # PyPy: also copy lib/*.so* for portable builds
+                libdst = os.path.join(context.env_dir, 'lib')
+                if not os.path.exists(libdst):
+                    os.mkdir(libdst)
+                for f in os.listdir(libsrc):
+                    src = os.path.join(libsrc, f)
+                    dst = os.path.join(libdst, f)
+                    copier(src, dst)
             #
         else:
             subdir = 'DLLs'
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
index bcdea19661..76d989c75b 100644
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -266,7 +266,7 @@ def test():
     reader.ps1 = "c**> "
     reader.ps2 = "c/*> "
     reader.ps3 = "c|*> "
-    reader.ps4 = "c\*> "
+    reader.ps4 = r"c\*> "
     while reader.readline():
         pass
 
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
index bc971cf1a6..5ae180845e 100644
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -648,7 +648,7 @@ def test():
     reader.ps1 = "**> "
     reader.ps2 = "/*> "
     reader.ps3 = "|*> "
-    reader.ps4 = "\*> "
+    reader.ps4 = r"\*> "
     while reader.readline():
         pass
 
diff --git a/pypy/doc/release-v7.3.0.rst b/pypy/doc/release-v7.3.0.rst
index da6905b100..cd84ce46b7 100644
--- a/pypy/doc/release-v7.3.0.rst
+++ b/pypy/doc/release-v7.3.0.rst
@@ -138,6 +138,10 @@ Changes shared across versions
 * Overflow in RPython when converting ``2<<32`` into a ``Signed`` on 32-bit
   platforms rather than automatically using a ``SignedLongLong``, require an
   explicit ``r_int64()`` call instead
+* Fix multithread contention when creating an object in cffi (PyPy only)
+* Copy lib/* shared objects in portable builds when creating virtual
+  environments with virtualenv and venv
+* Potential fix in rare-case JIT optimizer (`issue 3128`_)
 
 C-API (cpyext) and c-extensions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -226,6 +230,7 @@ Python 3.6 C-API
 .. _`issue 3117`: https://bitbucket.com/pypy/pypy/issues/3117
 .. _`issue 3119`: https://bitbucket.com/pypy/pypy/issues/3119
 .. _`issue 3120`: https://bitbucket.com/pypy/pypy/issues/3120
+.. _`issue 3128`: https://bitbucket.com/pypy/pypy/issues/3120
 
 .. _13312: https://bugs.python.org/issue13312
 .. _13617: https://bugs.python.org/issue13617
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
index bab6efe5e5..240a6df15c 100644
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -83,6 +83,8 @@ class RealizeCache:
         self.space = space
         self.all_primitives = [None] * cffi_opcode._NUM_PRIM
         self.file_struct = None
+        self.lock = None
+        self.lock_owner = 0
         self.rec_level = 0
 
     def get_file_struct(self):
@@ -90,6 +92,33 @@ class RealizeCache:
             self.file_struct = ctypestruct.W_CTypeStruct(self.space, "FILE")
         return self.file_struct
 
+    def __enter__(self):
+        # This is a simple recursive lock implementation
+        if self.space.config.objspace.usemodules.thread:
+            from rpython.rlib import rthread
+            #
+            tid = rthread.get_ident()
+            if tid != self.lock_owner:
+                if self.lock is None:
+                    self.lock = self.space.allocate_lock()
+                self.lock.acquire(True)
+                assert self.lock_owner == 0
+                assert self.rec_level == 0
+                self.lock_owner = tid
+        self.rec_level += 1
+
+    def __exit__(self, *args):
+        assert self.rec_level > 0
+        self.rec_level -= 1
+        if self.space.config.objspace.usemodules.thread:
+            from rpython.rlib import rthread
+            #
+            tid = rthread.get_ident()
+            assert tid == self.lock_owner
+            if self.rec_level == 0:
+                self.lock_owner = 0
+                self.lock.release()
+
 
 def get_primitive_type(ffi, num):
     space = ffi.space
@@ -408,21 +437,25 @@ def realize_c_type_or_func(ffi, opcodes, index):
         return ffi.cached_types[index]
 
     realize_cache = ffi.space.fromcache(RealizeCache)
-    if realize_cache.rec_level >= 1000:
-        raise oefmt(ffi.space.w_RuntimeError,
-            "type-building recursion too deep or infinite.  "
-            "This is known to occur e.g. in ``struct s { void(*callable)"
-            "(struct s); }''.  Please report if you get this error and "
-            "really need support for your case.")
-    realize_cache.rec_level += 1
-    try:
+    with realize_cache:
+        #
+        # check again cached_types, which might have been filled while
+        # we were waiting for the recursive lock
+        if from_ffi and ffi.cached_types[index] is not None:
+            return ffi.cached_types[index]
+
+        if realize_cache.rec_level > 1000:
+            raise oefmt(ffi.space.w_RuntimeError,
+                "type-building recursion too deep or infinite.  "
+                "This is known to occur e.g. in ``struct s { void(*callable)"
+                "(struct s); }''.  Please report if you get this error and "
+                "really need support for your case.")
         x = realize_c_type_or_func_now(ffi, op, opcodes, index)
-    finally:
-        realize_cache.rec_level -= 1
 
-    if from_ffi:
-        assert ffi.cached_types[index] is None or ffi.cached_types[index] is x
-        ffi.cached_types[index] = x
+        if from_ffi:
+            old = ffi.cached_types[index]
+            assert old is None or old is x
+            ffi.cached_types[index] = x
 
     return x
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
index 898eddc5fd..04cfc5bb99 100644
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -610,19 +610,14 @@ class AppTestSocket:
 
     def test_recvmsg_issue2649(self):
         import _socket as socket
-        listener = socket.socket(family=socket.AF_INET6, type=socket.SOCK_DGRAM)
+        listener = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
         listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        listener.bind(('::1', 1234))
+        listener.bind(('127.0.0.1', 1234))
 
-        s = socket.socket(family=socket.AF_INET6, type=socket.SOCK_DGRAM)
-        IPV6_RECVERR = 25
-        s.setsockopt(socket.IPPROTO_IPV6, IPV6_RECVERR, 1)
-
-        s.sendto(b'x', ('::1', 1234))
-        try:
+        s = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
+        s.sendto(b'x', ('127.0.0.1', 1234))
+        with raises(BlockingIOError):
             queue = s.recvmsg(1024, 1024, socket.MSG_ERRQUEUE)
-        except BlockingIOError as e:
-            assert True
 
     def test_buffer(self):
         # Test that send/sendall/sendto accept a buffer as arg
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
index d9313793b8..2ad13248bc 100644
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -683,7 +683,14 @@ class Optimizer(Optimization):
                 elif constvalue == 1:
                     opnum = rop.GUARD_TRUE
                 else:
-                    raise AssertionError("uh?")
+                    # Issue #3128: there might be rare cases where strange
+                    # code is produced.  That issue hits the assert from
+                    # OptUnroll.inline_short_preamble's send_extra_operation().
+                    # Better just disable this optimization than crash with
+                    # an AssertionError here.  Note also that such code might
+                    # trigger an InvalidLoop to be raised later---so we must
+                    # not crash here.
+                    return op
                 newop = self.replace_op_with(op, opnum, [op.getarg(0)], descr)
                 return newop
         return op
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
index ce411bf79e..c2816d7421 100644
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -698,6 +698,15 @@ class TestOptimizeOpt(BaseTestWithUnroll):
         """
         self.optimize_loop(ops, expected, preamble)
 
+    def test_guard_value_on_boolean_but_not_zero_or_one(self):
+        ops = """
+        [i]
+        i1 = int_lt(i, 3)
+        guard_value(i1, -1) [i]
+        jump(i)
+        """
+        py.test.raises(InvalidLoop, self.optimize_loop, ops, ops, ops)
+
     def test_int_is_true_of_bool(self):
         ops = """
         [i0, i1]
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
index cab7484bf3..4ae9441bb3 100644
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1146,6 +1146,33 @@ class TestStandalone(StandaloneTests):
         out = cbuilder.cmdexec('')
         assert out.strip() == 'ok'
 
+    def test_int_manipulation(self):
+        # Distilled from micronumpy.descriptor._compute_hash
+        # which, for some version of gcc8 compiler produced
+        # out1 == out2
+        from rpython.rlib.rarithmetic import intmask
+        
+        def entry_point(argv):
+            if len(argv) < 4:
+                print 'need 3 arguments, not %s' % str(argv)
+                return -1
+            flags = 0
+            x = 0x345678
+            y = 0x345678
+            s = str(argv[1])[0]
+            y = intmask((1000003 * y) ^ ord(s))
+            y = intmask((1000003 * y) ^ ord(str(argv[2])[0]))
+            y = (1000003 * y)
+            y = intmask(y ^ flags)
+            y = intmask((1000003 * y) ^ int(argv[3]))
+            print y
+            return 0
+
+        t, cbuilder = self.compile(entry_point)
+        out1 = cbuilder.cmdexec(args=['i', '>', '64'])
+        out2 = cbuilder.cmdexec(args=['f', '>', '64'])
+        assert out1 != out2
+
 
 class TestThread(object):
     gcrootfinder = 'shadowstack'