[GR-13552] Initial support for module mmap.

PullRequest: graalpython/390

Author: fangerer

graalpython/benchmarks/src/micro/mmap-anonymous.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import mmap
+
+# create 64 MB anonymous mmap
+size = 64*1024
+mm = mmap.mmap(-1, size)
+data = b"Hello World"
+ndata = len(data)
+niter = size // ndata
+
+
+def fill():
+    # sequential access
+    mm.seek(0)
+    for i in range(niter):
+        mm.write(data)
+
+
+def measure(num):
+    for i in range(num):
+        fill()
+    print(mm[0:ndata])
+
+
+def __benchmark__(num=100):
+    measure(num)
+

graalpython/benchmarks/src/micro/mmap-file.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import mmap
+import tempfile
+
+# create temporary file
+data = b"Hello World"
+ndata = len(data)
+
+
+def fill(mm, size):
+    # sequential access
+    mm.seek(0)
+    for i in range(size // ndata):
+        mm.write(data)
+
+
+def measure(num):
+    tmp_path = tempfile.mkstemp()
+    with open(tmp_path[1], "wb") as f:
+        f.write(b'\x00' * (64 * 1024))
+
+    with open(tmp_path[1], "r+b") as f:
+        mm = mmap.mmap(f.fileno(), 0)
+        size = mm.size()
+        for i in range(num):
+            fill(mm, size)
+        print(mm[0:ndata])
+
+
+def __benchmark__(num=100):
+    measure(num)
+

graalpython/com.oracle.graal.python.cext/modules/_mmap.c

@@ -0,0 +1,92 @@
+/* Copyright (c) 2019, Oracle and/or its affiliates.
+ * Copyright (C) 1996-2017 Python Software Foundation
+ *
+ * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+ */
+
+#include "../src/capi.h"
+
+typedef enum
+{
+    ACCESS_DEFAULT,
+    ACCESS_READ,
+    ACCESS_WRITE,
+    ACCESS_COPY
+} access_mode;
+
+typedef struct {
+    PyObject_HEAD
+    char *      data;
+    Py_ssize_t  size;
+    Py_ssize_t  pos;    /* relative to offset */
+#ifdef MS_WINDOWS
+    long long offset;
+#else
+    off_t       offset;
+#endif
+    int     exports;
+
+#ifdef MS_WINDOWS
+    HANDLE      map_handle;
+    HANDLE      file_handle;
+    char *      tagname;
+#endif
+
+#ifdef UNIX
+    int fd;
+#endif
+
+    PyObject *weakreflist;
+    access_mode access;
+} mmap_object;
+
+
+POLYGLOT_DECLARE_TYPE(mmap_object);
+static PyTypeObject mmap_object_type = PY_TRUFFLE_TYPE("mmap.mmap", NULL, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, sizeof(mmap_object));
+
+int mmap_getbuffer(PyObject *self, Py_buffer *view, int flags) {
+	// TODO(fa) readonly flag
+    return PyBuffer_FillInfo(view, (PyObject*)self, ((mmap_object *)self)->data, PyObject_Size((PyObject *)self), 0, flags);
+}
+
+static PyObject* mmap_init_bufferprotocol(PyObject* self, PyObject* mmap_type) {
+	assert(PyType_Check(mmap_type));
+    initialize_type_structure(&mmap_object_type, mmap_type, polyglot_mmap_object_typeid());
+
+	polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java(mmap_type), (getbufferproc) mmap_getbuffer, (releasebufferproc) NULL);
+	return Py_None;
+}
+
+static struct PyMethodDef module_functions[] = {
+    {"init_bufferprotocol", mmap_init_bufferprotocol, METH_O, NULL},
+    {NULL,       NULL}          /* sentinel */
+};
+
+static struct PyModuleDef mmapmodule = {
+    PyModuleDef_HEAD_INIT,
+    "_mmap",
+    NULL,
+    -1,
+    module_functions,
+    NULL,
+    NULL,
+    NULL,
+    NULL
+};
+
+PyMODINIT_FUNC
+PyInit__mmap(void)
+{
+    PyObject *dict, *module;
+
+    module = PyModule_Create(&mmapmodule);
+    if (module == NULL) {
+        return NULL;
+    }
+    dict = PyModule_GetDict(module);
+    if (!dict) {
+        return NULL;
+    }
+
+    return module;
+}

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_mmap.py

@@ -0,0 +1,87 @@
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import sys
+import mmap
+import tempfile
+from . import CPyExtTestCase, CPyExtFunction, CPyExtFunctionOutVars, unhandled_error_compare, GRAALPYTHON
+__dir__ = __file__.rpartition("/")[0]
+
+
+def create_and_map_file():
+    tmp = tempfile.mktemp(prefix="pymmap_")
+    with open(tmp, "w") as f:
+        f.write("hello, world")
+    f = open(tmp, "r")
+    return mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+    
+
+class TestPyMmap(CPyExtTestCase):
+    def compile_module(self, name):
+        type(self).mro()[1].__dict__["test_%s" % name].create_module(name)
+        super(TestPyMmap, self).compile_module(name)
+
+
+    test_buffer = CPyExtFunction(
+        lambda args: b"hello, world",
+        lambda: (
+            (create_and_map_file(),),
+        ),
+        code="""
+        static PyObject* get_mmap_buf(PyObject* mmapObj) {
+            Py_buffer buf;
+            Py_ssize_t len, i;
+            char* data = NULL;
+            if (PyObject_GetBuffer(mmapObj, &buf, PyBUF_SIMPLE)) {
+                return NULL;
+            }
+            len = buf.len;
+            data = (char*) malloc(sizeof(char)*len);
+            for (i=0; i < buf.len; i++) {
+                data[i] = ((char *) buf.buf)[i];
+            }
+            return PyBytes_FromStringAndSize(data, len); 
+        }
+        """,
+        resultspec="O",
+        argspec='O',
+        arguments=["PyObject* mmapObj"],
+        callfunction="get_mmap_buf",
+        cmpfunc=unhandled_error_compare
+    )