diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/module/SubprocessTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/module/SubprocessTests.java new file mode 100644 index 0000000000..334fed6913 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/module/SubprocessTests.java @@ -0,0 +1,61 @@ +package com.oracle.graal.python.test.module; + + +import org.junit.Test; + +import static com.oracle.graal.python.test.PythonTests.assertPrints; +import static com.oracle.graal.python.test.PythonTests.runScript; + +public class SubprocessTests { + + @Test + public void importSmokeTest(){ + String code = "import subprocess"; + + assertPrints("",code); + } + + + @Test + public void lsSmokeTestRun(){ + String code = "import subprocess\n" + + "n = subprocess.run([\"ls\",\"-a\"]) \n"+ + "print(n)"; + //assertPrints("",code); + + runScript(new String[0],code,System.out,System.err); + } + + @Test + public void lsTestCall(){ + String code = "import subprocess\n" + + "n = subprocess.call([\"ls\",\"-a\"]) \n"+ + "print(n)"; + + //assertPrints("0\n",code); + runScript(new String[0],code,System.out,System.err); + } + + + + + @Test + public void pipeSmokeTest(){ + String code = "import os\n" + + "os.pipe()\n"; + + runScript(new String[0],code,System.out,System.err); + + } + + + @Test + public void envParamSmokeTest(){ + String code = "import subprocess\n" + + "subprocess.run([\"ls\",\"-a\"], env={'test':'test'})"; + runScript(new String[0],code,System.out,System.err); + } + + + +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java index 14eb5cb729..517695db45 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java @@ -200,7 +200,7 @@ protected void removeFile(int fd) { filePaths.set(fd, null); } - private static int nextFreeFd() { + protected static int nextFreeFd() { for (int i = 0; i < filePaths.size(); i++) { String openPath = filePaths.get(i); if (openPath == null) { @@ -1024,15 +1024,6 @@ private int getLength(PTuple times) { } } - @Builtin(name = "waitpid", fixedNumOfPositionalArgs = 2) - @GenerateNodeFactory - abstract static class WaitpidNode extends PythonBinaryBuiltinNode { - @SuppressWarnings("unused") - @Specialization - PTuple waitpid(int pid, int options) { - throw raise(NotImplementedError, "waitpid"); - } - } // FIXME: this is not nearly ready, just good enough for now @Builtin(name = "system", fixedNumOfPositionalArgs = 1) @@ -1200,4 +1191,264 @@ PBytes urandom(int size) { return factory().createBytes(range); } } + + + + @Builtin(name="WIFSIGNALED", fixedNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class WifsignaledNode extends PythonBuiltinNode{ + + /** + * Note: Implementation taken from glibc + * @param status + * @return + */ + @Specialization + @TruffleBoundary + boolean wifsignaled(int status){ + return ((char) (((status)&0x7f)+1)>>1) >1; + } + } + + @Builtin(name="WTERMSIG", fixedNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class WtermsigNode extends PythonBuiltinNode{ + + /** + * Note: Implementation taken from glibc + * @param status + * @return + */ + @Specialization + @TruffleBoundary + boolean wtermsig(int status){ + return (status & 0x7f) != 0; + } + } + + @Builtin(name="WIFEXITED", fixedNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class WifexitedNode extends PythonBuiltinNode{ + /** + * Note: Implementation taken from glibc + * @param status + * @return + */ + @Specialization + @TruffleBoundary + boolean wifexited(int status){ + return ((status) & 0x7f ) == 0; + } + } + + + @Builtin(name="WEXITSTATUS", fixedNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class WexitstatusNode extends PythonBuiltinNode{ + + /** + * Note: Implementation taken from glibc + * @param status + * @return + */ + @Specialization + @TruffleBoundary + boolean wexitstatus(int status){ + return ((status & 0xFF00) >> 8) != 0; + } + } + + + + @Builtin(name="WIFSTOPPED", fixedNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class WifstoppedNode extends PythonBuiltinNode{ + /** + * Note: Implementation taken from glibc + * @param status + * @return + */ + @Specialization + @TruffleBoundary + boolean wifstopped(int status){ + return (status & 0xff)== 0x7f; + } + } + + @Builtin(name="WSTOPSIG", fixedNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class WstopsigNode extends PythonBuiltinNode{ + /** + * Note: Implementation taken from glibc + * @param status + * @return + */ + @Specialization + @TruffleBoundary + boolean wstopsig(int status){ + return ((status & 0xFF00) >> 8) != 0; + } + } + + @Builtin(name="waitpid", fixedNumOfPositionalArgs = 2) + @GenerateNodeFactory + abstract static class WaitpidNode extends PythonBuiltinNode{ + @Specialization + @TruffleBoundary + PTuple waitpid(long pid, int options){ + + //TODO check what is really needed + if(pid < -1){ + //wait for any chlid process whose process group id is equal to the value of pid + throw raise(PythonErrorType.NotImplementedError, "Not implemented for pid <= 1"); + }else if(pid == -1){ + //wait for any child process + throw raise(PythonErrorType.NotImplementedError, "Not implemented for pid == -1"); + }else if(pid == 0){ + //wait for any child process whose process group id is equal to the value of the calling process + throw raise(PythonErrorType.NotImplementedError, "Not implemented for pid == 0"); + }else{ + //pid > 0 wait for the child process with this pid + + //TODO consider possible options: WNOHANG, WUNTRACED and WCONTINUED + // seems like only WNOHANG is needed? + + Process p = PosixSubprocessModuleBuiltins.getProcessByPid(pid); + int status = -1; + try { + status = p.waitFor(); + + //TODO: Make sure that here is the right place + PosixSubprocessModuleBuiltins.removeProcessByPid(pid); + + } catch (InterruptedException e) { + //TODO: Error handling + e.printStackTrace(); + throw raise(PythonErrorType.NotImplementedError, "Error handling for waitFor not implemented"); + } + + + + //TODO encode status return the right way + return factory().createTuple(new Object[]{ + PosixSubprocessModuleBuiltins.getPidOfProcess(p), status}); + } + } + } + + /** + * taken from glibc: + * + * + * Bits in the third argument to 'waitpid' + * WNOHANG: Dont block waiting + */ + @Builtin(name="WNOHANG", fixedNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class WnohangNode extends PythonBuiltinNode{ + @Specialization + @TruffleBoundary + int wnohang(){ + return 1; + } + } + + @SuppressWarnings("MagicConstant") + @Builtin(name="pipe", fixedNumOfPositionalArgs = 0) + @GenerateNodeFactory + abstract static class PipeNode extends PythonFileNode{ + + /*TODO find good (already existing?) implementation + Looks like there is no good implementation in java + for in-memory files that implement SeekableByteChannel? + */ + private class SeekableByteBuffer implements SeekableByteChannel{ + + ByteBuffer buffer; + + private long position; + private boolean open; + public SeekableByteBuffer(ByteBuffer buffer){ + this.buffer = buffer; + open = true; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + if(!isOpen()){ + return -1; + } + + int i = 0; + dst.reset(); + + while(dst.remaining() > 0 && buffer.remaining() > 0){ + byte b = buffer.get(); + dst.put(b); + i++; + } + + return i; + } + + @Override + public int write(ByteBuffer src) throws IOException { + + int before = buffer.position(); + buffer.put(src); + return buffer.position()-before; + } + + @Override + public long position() throws IOException { + return buffer.position(); + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + //seeking not allowed in pipes... + throw new UnsupportedOperationException(); + } + + @Override + public long size() throws IOException { + return buffer.position(); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() throws IOException { + open = false; + } + } + + @Specialization + @TruffleBoundary + Object pipe(){ + + ByteBuffer buffer = ByteBuffer.allocate(1024); + SeekableByteBuffer p = new SeekableByteBuffer(buffer); + + int a = nextFreeFd(); + files.set(a,p); + filePaths.set(a,"pipe"); + + int b = nextFreeFd(); + files.set(b,p); + filePaths.set(b,"pipe"); + + return factory().createTuple(new Object[]{a,b}); + } + + } + } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java index a466aab6f0..4e753a40de 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java @@ -40,18 +40,244 @@ */ package com.oracle.graal.python.builtins.modules; +import java.io.IOException; +import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import com.oracle.graal.python.builtins.Builtin; import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltins; -import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.builtins.objects.list.PList; +import com.oracle.graal.python.builtins.objects.tuple.PTuple; +import com.oracle.graal.python.nodes.function.PythonBuiltinNode; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Fallback; +import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.NodeFactory; +import com.oracle.truffle.api.dsl.Specialization; + +import static com.oracle.graal.python.runtime.exception.PythonErrorType.NotImplementedError; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError; @CoreFunctions(defineModule = "_posixsubprocess") public class PosixSubprocessModuleBuiltins extends PythonBuiltins { + private static ArrayList childProcesses = new ArrayList(); + @Override - protected List> getNodeFactories() { - return new ArrayList<>(); + protected List> getNodeFactories() { + return PosixSubprocessModuleBuiltinsFactory.getFactories(); + } + + /** Since getting the PID of a process in Java is only availble since Java 9 + * + * https://stackoverflow.com/a/33171840 + */ + + public static long getPidOfProcess(Process p) { + long pid = -1; + + try { + if (p.getClass().getName().equals("java.lang.UNIXProcess")) { + Field f = p.getClass().getDeclaredField("pid"); + f.setAccessible(true); + pid = f.getLong(p); + f.setAccessible(false); + } + } catch (Exception e) { + pid = -1; + } + return pid; + } + + + public static Process getProcessByPid(long pid) { + + for (Process p : childProcesses) { + if (getPidOfProcess(p) == pid) { + return p; + } + } + return null; + } + + + public static boolean removeProcessByPid(long pid) { + int i = 0; + for (Process p : childProcesses) { + if (getPidOfProcess(p) == pid) { + childProcesses.remove(i); + return true; + } + i++; + } + return false; + } + + + @Builtin(name = "fork_exec", fixedNumOfPositionalArgs = 17, keywordArguments = { + "args", "executable_list", "close_fds", "fds_to_keep", "cwd", "env", "p2cread", "p2cwrite", "c2pread", "c2pwrite", + "erread", "errwrite", "errpipe_read", "errpipe_write", "restore_signals", "call_setsid", "preexec"}) + @GenerateNodeFactory + abstract static class PythonForkExecNode extends PythonBuiltinNode { + //TODO check if fds are always Integers, still possible to have long type but in this value range? + + /** + * @param fdsToKeep + * @return Returns true if there is a problem with the fds + */ + private boolean sanityCheckPythonFdSequence(PList fdsToKeep) { + + long prevFd = -1; + + //TODO iterator? + for (int seqIdx = 0; seqIdx < fdsToKeep.getSequenceStorage().length(); seqIdx++) { + Object pyFd = fdsToKeep.getSequenceStorage().getItemNormalized(seqIdx); + + if (!(pyFd instanceof Long) && !(pyFd instanceof Integer)) { + return true; + } + + long iterFd = pyFd instanceof Long ? (Long) pyFd : (Integer) pyFd; + + //TODO compare Integer.MAX_VALUE to the native implementation + if (iterFd < 0 || iterFd < prevFd || iterFd > Integer.MAX_VALUE) { + /*negative, overflow, unsorted, too big for fd*/ + return true; + } + + prevFd = iterFd; + } + + return false; + } + + private int getMaxFd() { + //TODO which value should it be? + + return 255; //<- legacy variable + } + + private boolean isFdInSortedFdSequence(Integer fd, PTuple fdSequence) { + + return + Arrays.binarySearch(fdSequence.getArray(), fd, new Comparator() { + @Override + public int compare(Object x, Object y) { + return ((Integer) x).compareTo((Integer) y); + } + }) >= 0; + + } + + @Specialization + @CompilerDirectives.TruffleBoundary + Object forkExecGeneral(PList args, PTuple executable_list, boolean close_fds, PTuple fds_to_keep, Object cwd, + Object env, int p2cread, int p2cwrite, int c2pread, int c2pwrite, int erread, int errwrite, + int errpipe_read, int errpipe_write, boolean restore_signals, boolean call_setsid, + Object preexec) { + + //TODO see cpython documentation for fds_to_keep + System.out.println("fork_exec: start"); + + //TODO check PyArg_ParseTuple function + + if (close_fds && errpipe_write < 3) { + throw raise(ValueError, "errpipe_write must be >= 3"); + } + + /* + TODO adopt for differnt types of fds_to_keep + if (sanityCheckPythonFdSequence(fds_to_keep)) { + throw raise(ValueError, "bad value(s) in fds_to_keep"); + }*/ + + //TODO check why to disable gc when preexec is present + + String[] jArgs = new String[args.getSequenceStorage().length()]; + + //TODO better would be iterator! + for (int i = 0; i < jArgs.length; i++) { + Object currItem = args.getSequenceStorage().getItemNormalized(i); + if (currItem instanceof String) { + jArgs[i] = (String) args.getSequenceStorage().getItemNormalized(i); + } else { + //TODO not handled in standard? (check parse function for string) + throw raise(ValueError, "value was not of type string"); + } + } + + ProcessBuilder processBuilder = new ProcessBuilder(jArgs); + + if (env != null && (env instanceof PList) ) { + PList envPList = (PList) env; + //TODO iterator? Possible 0(n²) loop! + for (int i = 0; i < envPList.getSequenceStorage().length(); i++) { + //TODO is of type pbytes, bad (what other option?) to go over toString() representation! + String str = envPList.getSequenceStorage().getItemNormalized(i).toString(); + str = str.substring(2,str.length()-1); + String[] splitted = str.split("="); + processBuilder.environment().put(splitted[0], splitted[1]); + } + } + + + assert p2cread == -1 && p2cwrite == -1 || p2cread != -1 && p2cwrite != -1; + + if(p2cread==-1 && p2cwrite == -1){ + processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT); + }else{ + //TODO: pipe input + } + + assert c2pread == -1 && c2pwrite == -1 || c2pread != -1 && c2pwrite != -1; + + if(c2pread == -1 && c2pwrite == -1){ + processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); + }else{ + // TODO: pipe output + } + + //TODO set other parameters + + + + Process process = null; + try { + process = processBuilder.start(); + + + } catch (IOException e) { + e.printStackTrace(); + //TODO right error handling + + return -1; + } + + //TODO error handling + long pid = getPidOfProcess(process); + + childProcesses.add(process); + + System.out.println("fork_exec: end"); + + return pid; + } + + + @Fallback + Object forkExecGeneralFallback(Object args, Object executable_list, Object close_fds, Object fds_to_keep, Object cwd, + Object env, Object p2cread, Object p2cwrite, Object c2pread, Object c2pwrite, Object erread, Object errwrite, + Object errpipe_read, Object errpipe_write, Object restore_signals, Object call_setsid, + Object preexec) { + + throw raise(NotImplementedError,"No implementation for this specialization!"); + } + } + + + }