[GR-20576] Fix Python 3 stdlib unittest runner

PullRequest: graalpython/777

Author: timfel

ci.jsonnet

@@ -1 +1 @@
-{ overlay: "9c01bf051c60a83731b2cabd5b06c50e31624741" }
+{ overlay: "c73a52e014ae17658cb055f67f58408f590b99ff" }

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -129,6 +129,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
                 case "-O":
                 case "-OO":
                 case "-R":
+                case "-d":
                     break;
                 case "-q":
                     quietFlag = true;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 
@@ -132,6 +133,14 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
 
     @CompilationFinal(dimensions = 1) private static final Object[] CONTEXT_INSENSITIVE_SINGLETONS = new Object[]{PNone.NONE, PNone.NO_VALUE, PEllipsis.INSTANCE, PNotImplemented.NOT_IMPLEMENTED};
 
+    /**
+     * Named semaphores are shared between all processes in a system, and they persist until the
+     * system is shut down, unless explicitly removed. We interpret this as meaning they all exist
+     * globally per language instance, that is, they are shared between different Contexts in the
+     * same engine.
+     */
+    public final ConcurrentHashMap<String, Semaphore> namedSemaphores = new ConcurrentHashMap<>();
+
     /*
      * We need to store this here, because the check is on the language and can come from a thread
      * that has no context, but we enable or disable threads with a context option. So we store this

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -158,6 +158,7 @@
 import com.oracle.graal.python.builtins.objects.superobject.SuperBuiltins;
 import com.oracle.graal.python.builtins.objects.thread.LockBuiltins;
 import com.oracle.graal.python.builtins.objects.thread.RLockBuiltins;
+import com.oracle.graal.python.builtins.objects.thread.SemLockBuiltins;
 import com.oracle.graal.python.builtins.objects.thread.ThreadBuiltins;
 import com.oracle.graal.python.builtins.objects.traceback.TracebackBuiltins;
 import com.oracle.graal.python.builtins.objects.tuple.TupleBuiltins;
@@ -373,6 +374,7 @@ private static final PythonBuiltins[] initializeBuiltins() {
                         new LZMACompressorBuiltins(),
                         new LZMADecompressorBuiltins(),
                         new MultiprocessingModuleBuiltins(),
+                        new SemLockBuiltins(),
                         new GraalPythonModuleBuiltins()));
         if (!TruffleOptions.AOT) {
             ServiceLoader<PythonBuiltins> providers = ServiceLoader.load(PythonBuiltins.class, Python3Core.class.getClassLoader());

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -54,7 +54,9 @@ public abstract class PythonBuiltins {
 
     /**
      * Initialize everything that is truly independent of commandline arguments and that can be
-     * initialized and frozen into an SVM image.
+     * initialized and frozen into an SVM image. When in a subclass, any modifications to
+     * {@link #builtinConstants} or such should be made before calling
+     * {@code super.initialize(core)}.
      */
     public void initialize(PythonCore core) {
         if (builtinFunctions.size() > 0) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MultiprocessingModuleBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -41,18 +41,25 @@
 package com.oracle.graal.python.builtins.modules;
 
 import java.util.List;
+import java.util.concurrent.Semaphore;
 
+import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.Builtin;
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.thread.PSemLock;
 import com.oracle.graal.python.builtins.objects.type.LazyPythonClass;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
+import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.nodes.util.CastToJavaIntNode;
+import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
 import com.oracle.graal.python.runtime.PythonCore;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
+import com.oracle.truffle.api.dsl.CachedLanguage;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.dsl.Specialization;
@@ -76,21 +83,69 @@ public void initialize(PythonCore core) {
     abstract static class ConstructSemLockNode extends PythonBuiltinNode {
         @Specialization
         PSemLock construct(LazyPythonClass cls, Object kindObj, Object valueObj, Object maxvalueObj, Object nameObj, Object unlinkObj,
+                        @Cached CastToJavaStringNode castNameNode,
                         @Cached CastToJavaIntNode castKindToIntNode,
                         @Cached CastToJavaIntNode castValueToIntNode,
                         @Cached CastToJavaIntNode castMaxvalueToIntNode,
-                        @Cached CastToJavaIntNode castUnlinkToIntNode) {
+                        @Cached CastToJavaIntNode castUnlinkToIntNode,
+                        @CachedLanguage PythonLanguage lang) {
             int kind = castKindToIntNode.execute(kindObj);
             if (kind != PSemLock.RECURSIVE_MUTEX && kind != PSemLock.SEMAPHORE) {
                 throw raise(PythonBuiltinClassType.ValueError, "unrecognized kind");
             }
             int value = castValueToIntNode.execute(valueObj);
-            int maxvalue = castMaxvalueToIntNode.execute(maxvalueObj);
+            castMaxvalueToIntNode.execute(maxvalueObj); // executed for the side-effect, but ignored
+                                                        // on posix
+            Semaphore semaphore = newSemaphore(value);
             int unlink = castUnlinkToIntNode.execute(unlinkObj);
-            if (unlink != 0) {
-                throw raise(PythonBuiltinClassType.SystemError, "semaphore unlinking is not yet implemented");
+            String name = castNameNode.execute(nameObj);
+            if (unlink == 0) {
+                // CPython creates a named semaphore, and if unlink != 0 unlinks
+                // it directly so it cannot be access by other processes. We
+                // have to explicitly link it, so we do that here if we
+                // must. CPython always uses O_CREAT | O_EXCL for creating named
+                // semaphores, so a conflict raises.
+                if (semaphoreExists(lang, name)) {
+                    throw raise(PythonBuiltinClassType.FileExistsError, "Semaphore name taken: '%s'", name);
+                } else {
+                    semaphorePut(lang, semaphore, name);
+                }
             }
-            return factory().createSemLock(cls, kind, value, maxvalue, nameObj);
+            return factory().createSemLock(cls, name, kind, semaphore);
+        }
+
+        @TruffleBoundary
+        private static Object semaphorePut(PythonLanguage lang, Semaphore semaphore, String name) {
+            return lang.namedSemaphores.put(name, semaphore);
+        }
+
+        @TruffleBoundary
+        private static boolean semaphoreExists(PythonLanguage lang, String name) {
+            return lang.namedSemaphores.containsKey(name);
+        }
+
+        @TruffleBoundary
+        private static Semaphore newSemaphore(int value) {
+            return new Semaphore(value);
+        }
+    }
+
+    @GenerateNodeFactory
+    @Builtin(name = "sem_unlink", parameterNames = {"name"})
+    abstract static class SemUnlink extends PythonUnaryBuiltinNode {
+        @Specialization
+        PNone doit(String name,
+                        @CachedLanguage PythonLanguage lang) {
+            Semaphore prev = semaphoreRemove(name, lang);
+            if (prev == null) {
+                throw raise(PythonBuiltinClassType.FileNotFoundError, "No such file or directory: 'semaphores:/%s'", name);
+            }
+            return PNone.NONE;
+        }
+
+        @TruffleBoundary
+        private static Semaphore semaphoreRemove(String name, PythonLanguage lang) {
+            return lang.namedSemaphores.remove(name);
         }
     }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/thread/PSemLock.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -52,14 +52,15 @@ public final class PSemLock extends AbstractPythonLock {
 
     private final Semaphore semaphore;
     private final int kind;
+    private final String name;
 
     private int lastThreadID = -1;
     private int count;
 
-    @TruffleBoundary
-    public PSemLock(LazyPythonClass cls, int kind, int value) {
+    public PSemLock(LazyPythonClass cls, String name, int kind, Semaphore sharedSemaphore) {
         super(cls);
-        semaphore = new Semaphore(value);
+        this.name = name;
+        this.semaphore = sharedSemaphore;
         this.kind = kind;
     }
 
@@ -129,4 +130,8 @@ public boolean isMine() {
     public int getKind() {
         return kind;
     }
+
+    public String getName() {
+        return name;
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/thread/SemLockBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -55,6 +55,8 @@
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
+import com.oracle.graal.python.runtime.PythonCore;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
@@ -69,6 +71,12 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
         return SemLockBuiltinsFactory.getFactories();
     }
 
+    @Override
+    public void initialize(PythonCore core) {
+        builtinConstants.put("SEM_VALUE_MAX", Integer.MAX_VALUE);
+        super.initialize(core);
+    }
+
     @Builtin(name = "_count", minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     abstract static class CountNode extends PythonUnaryBuiltinNode {
@@ -96,6 +104,43 @@ int getValue(PSemLock self) {
         }
     }
 
+    @Builtin(name = "handle", minNumOfPositionalArgs = 1, isGetter = true)
+    @GenerateNodeFactory
+    abstract static class GetHandleNode extends PythonUnaryBuiltinNode {
+        @Specialization
+        @TruffleBoundary
+        int getHandle(PSemLock self) {
+            return self.hashCode();
+        }
+    }
+
+    @Builtin(name = "name", minNumOfPositionalArgs = 1, isGetter = true)
+    @GenerateNodeFactory
+    abstract static class GetNameNode extends PythonUnaryBuiltinNode {
+        @Specialization
+        Object getName(@SuppressWarnings("unused") PSemLock self) {
+            return PNone.NONE;
+        }
+    }
+
+    @Builtin(name = "maxvalue", minNumOfPositionalArgs = 1, isGetter = true)
+    @GenerateNodeFactory
+    abstract static class GetMaxValue extends PythonUnaryBuiltinNode {
+        @Specialization
+        Object getMax(@SuppressWarnings("unused") PSemLock self) {
+            return Integer.MAX_VALUE;
+        }
+    }
+
+    @Builtin(name = "kind", minNumOfPositionalArgs = 1, isGetter = true)
+    @GenerateNodeFactory
+    abstract static class GetKindNode extends PythonUnaryBuiltinNode {
+        @Specialization
+        int getKind(PSemLock self) {
+            return self.getKind();
+        }
+    }
+
     @Builtin(name = "acquire", minNumOfPositionalArgs = 1, parameterNames = {"self", "blocking", "timeout"})
     @GenerateNodeFactory
     abstract static class AcquireNode extends PythonTernaryBuiltinNode {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PythonObjectFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -31,6 +31,7 @@
 import java.nio.channels.SeekableByteChannel;
 import java.nio.file.DirectoryStream;
 import java.util.Map;
+import java.util.concurrent.Semaphore;
 
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
@@ -823,8 +824,8 @@ public PThread createPythonThread(LazyPythonClass cls, Thread thread) {
         return trace(new PThread(cls, thread));
     }
 
-    public PSemLock createSemLock(LazyPythonClass cls, int kind, int value, @SuppressWarnings("unused") int maxvalue, @SuppressWarnings("unused") Object nameObj) {
-        return trace(new PSemLock(cls, kind, value));
+    public PSemLock createSemLock(LazyPythonClass cls, String name, int kind, Semaphore sharedSemaphore) {
+        return trace(new PSemLock(cls, name, kind, sharedSemaphore));
     }
 
     public PScandirIterator createScandirIterator(LazyPythonClass cls, String path, DirectoryStream<TruffleFile> next) {

graalpython/lib-python/3/multiprocessing/context.py

@@ -311,6 +311,9 @@ def _check_available(self):
         'forkserver': ForkServerContext(),
     }
     _default_context = DefaultContext(_concrete_contexts['fork'])
+    # BEGIN TRUFFLE PATCH
+    _default_context = DefaultContext(_concrete_contexts['spawn'])
+    # END TRUFFLE PATCH
 
 else:
 

mx.graalpython/copyrights/oracle.copyright.hash

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0

mx.graalpython/copyrights/oracle.copyright.star

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0

mx.graalpython/mx_graalpython.py

@@ -215,7 +215,7 @@ def nativeclean(args):
 
 def python3_unittests(args):
     """run the cPython stdlib unittests"""
-    python(["graalpython/com.oracle.graal.python.test/src/python_unittests.py", "-v"] + args)
+    mx.run([sys.executable, "graalpython/com.oracle.graal.python.test/src/python_unittests.py", "-v"] + args)
 
 
 def retag_unittests(args):

mx.graalpython/suite.py

@@ -44,15 +44,15 @@
             },
             {
                 "name": "sulong",
-                "version": "48aa9f2523f7ecfcd09b748c3c8696efeddd2291",
+                "version": "f0d558f462231679629b663fc62323b1c70d4ea9",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},
                 ]
             },
             {
                 "name": "regex",
-                "version": "48aa9f2523f7ecfcd09b748c3c8696efeddd2291",
+                "version": "f0d558f462231679629b663fc62323b1c70d4ea9",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},