Skip to content

Commit bcc8b7d

Browse files
timfeltimfel
committed
freeze lib-graalpython
1 parent 5e999e2 commit bcc8b7d

File tree

4 files changed

+151
-39
lines changed

4 files changed

+151
-39
lines changed

‎graalpython/com.oracle.graal.python.frozen/freeze_modules.py

Lines changed: 71 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,30 +16,30 @@
1616
import textwrap
1717
import shutil
1818

19-
FROZEN_ONLY = os.path.join(os.path.dirname(__file__), 'flag.py')
19+
FROZEN_ONLY = os.path.join(os.path.dirname(__file__), "flag.py")
2020

2121
# These are modules that get frozen.
2222
TESTS_SECTION = "Test module"
2323
FROZEN = [
2424
# See parse_frozen_spec() for the format.
2525
# In cases where the frozenid is duplicated, the first one is re-used.
2626
(
27-
'import system',
27+
"import system",
2828
[
2929
# These frozen modules are necessary for bootstrapping the import
3030
# system.
31-
'importlib._bootstrap : _frozen_importlib',
32-
'importlib._bootstrap_external : _frozen_importlib_external',
31+
"importlib._bootstrap : _frozen_importlib",
32+
"importlib._bootstrap_external : _frozen_importlib_external",
3333
# CPython freezes zipimport, but we have it entirely intrinsified
3434
# 'zipimport',
35-
]
35+
],
3636
),
3737
(
3838
"stdlib - startup, without site (python -S)",
3939
[
4040
"abc",
4141
"codecs",
42-
'<encodings.*>',
42+
"<encodings.*>",
4343
"io",
4444
],
4545
),
@@ -79,13 +79,13 @@
7979
TESTS_SECTION,
8080
[
8181
"__hello__",
82-
'__hello__ : __hello_alias__',
83-
'__hello__ : <__phello_alias__>',
84-
'__hello__ : __phello_alias__.spam',
85-
'__hello__ : <__phello__>',
86-
'__hello__ : __phello__.spam',
82+
"__hello__ : __hello_alias__",
83+
"__hello__ : <__phello_alias__>",
84+
"__hello__ : __phello_alias__.spam",
85+
"__hello__ : <__phello__>",
86+
"__hello__ : __phello__.spam",
8787
# '<__phello__.**.*>',
88-
f'frozen_only : __hello_only__ = {FROZEN_ONLY}',
88+
f"frozen_only : __hello_only__ = {FROZEN_ONLY}",
8989
],
9090
),
9191
]
@@ -95,6 +95,42 @@
9595
"zipimport",
9696
}
9797

98+
# add graalpython modules and core files
99+
def add_graalpython_core():
100+
lib_graalpython = os.path.join(os.path.dirname(__file__), "..", "lib-graalpython")
101+
l = []
102+
for name in [
103+
"modules/_sysconfigdata",
104+
"modules/ginstall",
105+
]:
106+
modname = os.path.basename(name)
107+
modpath = os.path.join(lib_graalpython, f"{name}.py")
108+
l.append(f"{modname} : {modname} = {modpath}")
109+
for name in [
110+
"__graalpython__",
111+
"_sre",
112+
"_struct",
113+
"_sysconfig",
114+
"_weakref",
115+
"builtins",
116+
"bytearray",
117+
"ctypes",
118+
"function",
119+
"java",
120+
"pip_hook",
121+
"type",
122+
"unicodedata",
123+
"zipimport",
124+
]:
125+
modname = f"graalpython.{os.path.basename(name)}"
126+
modpath = os.path.join(lib_graalpython, f"{name}.py")
127+
l.append(f"{modname} : {modname} = {modpath}")
128+
FROZEN.append(("graalpython-lib", l))
129+
130+
131+
add_graalpython_core()
132+
133+
98134
#######################################
99135
# specs
100136

@@ -110,7 +146,12 @@ def parse_frozen_specs(stdlib_path, output_path):
110146
try:
111147
source = seen[frozenid]
112148
except KeyError:
113-
source = FrozenSource.from_id(stdlib_path=stdlib_path, output_path=output_path, frozenid=frozenid, pyfile=pyfile)
149+
source = FrozenSource.from_id(
150+
stdlib_path=stdlib_path,
151+
output_path=output_path,
152+
frozenid=frozenid,
153+
pyfile=pyfile,
154+
)
114155
seen[frozenid] = source
115156
else:
116157
assert not pyfile or pyfile == source.pyfile, item
@@ -488,6 +529,7 @@ def lower_camel_case(str):
488529
489530
public final class FrozenModules {"""
490531

532+
491533
def freeze_module(src):
492534
with open(src.pyfile, "r") as src_file, open(src.binaryfile, "wb") as binary_file:
493535
code_obj = compile(src_file.read(), src.id, "exec")
@@ -497,9 +539,10 @@ def freeze_module(src):
497539
def write_frozen_modules_map(out_file, modules):
498540
out_file.write(" private static final class Map {\n")
499541
for module in modules:
500-
if (not module.isalias or
501-
not module.orig or
502-
not any(module.orig == m.orig for m in modules if m != module)
542+
if (
543+
not module.isalias
544+
or not module.orig
545+
or not any(module.orig == m.orig for m in modules if m != module)
503546
):
504547
ispkg = "true" if module.ispkg else "false"
505548
out_file.write(
@@ -514,10 +557,12 @@ def write_frozen_lookup(out_file, modules):
514557
for module in modules:
515558
if module.source and (module.source.ispkg != module.ispkg):
516559
out_file.write(f' case "{module.name}":\n')
517-
out_file.write(f' return Map.{module.symbol}.asPackage({"true" if module.ispkg else "false"});\n')
560+
out_file.write(
561+
f' return Map.{module.symbol}.asPackage({"true" if module.ispkg else "false"});\n'
562+
)
518563
else:
519564
out_file.write(f' case "{module.name}":\n')
520-
out_file.write(f' return Map.{module.symbol};\n')
565+
out_file.write(f" return Map.{module.symbol};\n")
521566
out_file.write(" default:\n")
522567
out_file.write(" return null;\n")
523568
out_file.write(" }\n")
@@ -550,6 +595,7 @@ def write_frozen_module_file(file, modules):
550595
print(f"{file} modified, rebuild needed!")
551596
sys.exit(1)
552597

598+
553599
def add_tabs(str, number):
554600
lines = str.splitlines()
555601
tabbed_lines = []
@@ -562,10 +608,11 @@ def add_tabs(str, number):
562608

563609
def main(args):
564610
from argparse import ArgumentParser
611+
565612
parser = ArgumentParser()
566-
parser.add_argument('--python-lib', required=True)
567-
parser.add_argument('--binary-dir', required=True)
568-
parser.add_argument('--sources-dir', required=True)
613+
parser.add_argument("--python-lib", required=True)
614+
parser.add_argument("--binary-dir", required=True)
615+
parser.add_argument("--sources-dir", required=True)
569616
parsed_args = parser.parse_args(args)
570617

571618
# create module specs
@@ -579,7 +626,9 @@ def main(args):
579626
freeze_module(src)
580627

581628
# write frozen modules class used for storing frozen modules byte code arrays
582-
write_frozen_module_file(os.path.join(parsed_args.sources_dir, "FrozenModules.java"), modules)
629+
write_frozen_module_file(
630+
os.path.join(parsed_args.sources_dir, "FrozenModules.java"), modules
631+
)
583632

584633

585634
if __name__ == "__main__":

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,7 @@
320320
import com.oracle.graal.python.builtins.objects.type.TypeBuiltins;
321321
import com.oracle.graal.python.builtins.objects.type.TypeNodes.GetNameNode;
322322
import com.oracle.graal.python.builtins.objects.zipimporter.ZipImporterBuiltins;
323+
import com.oracle.graal.python.lib.PyDictSetItem;
323324
import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs;
324325
import com.oracle.graal.python.lib.PyObjectLookupAttr;
325326
import com.oracle.graal.python.nodes.BuiltinNames;
@@ -842,22 +843,30 @@ private void initializeJavaCore() {
842843
}
843844

844845
private void initializeImportlib() {
845-
PythonModule bootstrap = (PythonModule) ImpModuleBuiltins.importFrozenModuleObject(this, "_frozen_importlib", false);
846+
String importlibName = "_frozen_importlib";
847+
PythonModule bootstrap = ImpModuleBuiltins.importFrozenModuleObject(this, importlibName, false);
846848

847849
PyObjectCallMethodObjArgs callNode = PyObjectCallMethodObjArgs.getUncached();
848850
WriteAttributeToDynamicObjectNode writeNode = WriteAttributeToDynamicObjectNode.getUncached();
849851
ReadAttributeFromDynamicObjectNode readNode = ReadAttributeFromDynamicObjectNode.getUncached();
852+
PyDictSetItem setItem = PyDictSetItem.getUncached();
853+
854+
// first, a workaround since postInitialize hasn't run yet for the _weakref module aliases
855+
writeNode.execute(lookupBuiltinModule("_weakref"), "ref", lookupType(PythonBuiltinClassType.PReferenceType));
850856

851857
if (bootstrap == null) {
852858
// true when the frozen module is not available
853859
PythonModule bootstrapExternal = createModule("importlib._bootstrap_external");
854860
writeNode.execute(bootstrapExternal, __PACKAGE__, "importlib");
855-
addBuiltinModule("_frozen_importlib_external", bootstrapExternal);
861+
setItem.execute(null, sysModules, "_frozen_importlib_external", bootstrapExternal);
856862
bootstrap = createModule("importlib._bootstrap");
857863
writeNode.execute(bootstrap, __PACKAGE__, "importlib");
858-
addBuiltinModule("_frozen_importlib", bootstrap);
864+
setItem.execute(null, sysModules, importlibName, bootstrap);
859865
loadFile("importlib/_bootstrap_external", getContext().getStdlibHome(), bootstrapExternal);
860866
loadFile("importlib/_bootstrap", getContext().getStdlibHome(), bootstrap);
867+
} else {
868+
setItem.execute(null, sysModules, importlibName, bootstrap);
869+
LOGGER.log(Level.FINE, () -> "import '_frozen_importlib' # <frozen>");
861870
}
862871

863872
callNode.execute(null, bootstrap, "_install", getSysModule(), lookupBuiltinModule("_imp"));
@@ -1128,6 +1137,10 @@ private void loadFile(String s, String prefix) {
11281137
}
11291138

11301139
private void loadFile(String s, String prefix, PythonModule mod) {
1140+
if (ImpModuleBuiltins.importFrozenModuleObject(this, "graalpython." + s, false, mod) != null) {
1141+
LOGGER.log(Level.FINE, () -> "import '" + s + "' # <frozen>");
1142+
return;
1143+
}
11311144
Supplier<CallTarget> getCode = () -> {
11321145
Source source = getInternalSource(s, prefix);
11331146
return PythonUtils.getOrCreateCallTarget((RootNode) getParser().parse(ParserMode.File, 0, this, source, null, null));

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,7 @@ protected ArgumentClinicProvider getArgumentClinic() {
616616
}
617617

618618
@Specialization
619-
Object run(String name) {
619+
PythonModule run(String name) {
620620
return importFrozenModuleObject(getCore(), name, true);
621621
}
622622
}
@@ -626,7 +626,17 @@ Object run(String name) {
626626
* imported module, null, or raises a Python exception.
627627
*/
628628
@TruffleBoundary
629-
public static Object importFrozenModuleObject(Python3Core core, String name, boolean doRaise) {
629+
public static PythonModule importFrozenModuleObject(Python3Core core, String name, boolean doRaise) {
630+
return importFrozenModuleObject(core, name, doRaise, null);
631+
}
632+
633+
/**
634+
* @see importFrozenModuleObject
635+
*
636+
* Uses {@code globals} if given as the globals for execution.
637+
*/
638+
@TruffleBoundary
639+
public static PythonModule importFrozenModuleObject(Python3Core core, String name, boolean doRaise, PythonModule globals) {
630640
FrozenResult result = findFrozen(name);
631641
FrozenStatus status = result.status;
632642
FrozenInfo info = result.info;
@@ -639,14 +649,14 @@ public static Object importFrozenModuleObject(Python3Core core, String name, boo
639649
default:
640650
if (doRaise) {
641651
raiseFrozenError(status, name, PRaiseNode.getUncached());
642-
} else {
652+
} else if (status != FROZEN_OKAY) {
643653
return null;
644654
}
645655
}
646656

647657
PCode code = (PCode) MarshalModuleBuiltins.Marshal.load(info.data, info.size);
648658

649-
PythonModule module = core.factory().createPythonModule(name);
659+
PythonModule module = globals == null ? core.factory().createPythonModule(name) : globals;
650660

651661
if (info.isPackage) {
652662
/* Set __path__ to the empty list */
@@ -656,18 +666,10 @@ public static Object importFrozenModuleObject(Python3Core core, String name, boo
656666
RootCallTarget callTarget = CodeNodes.GetCodeCallTargetNode.getUncached().execute(code);
657667
GenericInvokeNode.getUncached().execute(callTarget, PArguments.withGlobals(module));
658668

659-
PythonModule importedModule = core.lookupBuiltinModule(name);
660-
661-
if (importedModule == null) {
662-
throw PRaiseNode.getUncached().raise(ImportError, ErrorMessages.MODULE_NOT_FOUND, name);
663-
}
664-
665-
/* Set __origname__ (consumed in FrozenImporter._setup_module()). */
666669
Object origName = info.origName == null ? PNone.NONE : info.origName;
670+
WriteAttributeToDynamicObjectNode.getUncached().execute(module, "__origname__", origName);
667671

668-
WriteAttributeToDynamicObjectNode.getUncached().execute(importedModule, "__origname__", origName);
669-
670-
return importedModule;
672+
return module;
671673
}
672674

673675
/*

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,22 @@ private static final class Map {
199199
private static final PythonFrozenModule STAT = new PythonFrozenModule("STAT", "stat", false);
200200
private static final PythonFrozenModule __HELLO__ = new PythonFrozenModule("__HELLO__", "__hello__", false);
201201
private static final PythonFrozenModule FROZEN_ONLY = new PythonFrozenModule("FROZEN_ONLY", "frozen_only", false);
202+
private static final PythonFrozenModule _SYSCONFIGDATA = new PythonFrozenModule("_SYSCONFIGDATA", "_sysconfigdata", false);
203+
private static final PythonFrozenModule GINSTALL = new PythonFrozenModule("GINSTALL", "ginstall", false);
204+
private static final PythonFrozenModule GRAALPYTHON___GRAALPYTHON__ = new PythonFrozenModule("GRAALPYTHON___GRAALPYTHON__", "graalpython.__graalpython__", false);
205+
private static final PythonFrozenModule GRAALPYTHON__SRE = new PythonFrozenModule("GRAALPYTHON__SRE", "graalpython._sre", false);
206+
private static final PythonFrozenModule GRAALPYTHON__STRUCT = new PythonFrozenModule("GRAALPYTHON__STRUCT", "graalpython._struct", false);
207+
private static final PythonFrozenModule GRAALPYTHON__SYSCONFIG = new PythonFrozenModule("GRAALPYTHON__SYSCONFIG", "graalpython._sysconfig", false);
208+
private static final PythonFrozenModule GRAALPYTHON__WEAKREF = new PythonFrozenModule("GRAALPYTHON__WEAKREF", "graalpython._weakref", false);
209+
private static final PythonFrozenModule GRAALPYTHON_BUILTINS = new PythonFrozenModule("GRAALPYTHON_BUILTINS", "graalpython.builtins", false);
210+
private static final PythonFrozenModule GRAALPYTHON_BYTEARRAY = new PythonFrozenModule("GRAALPYTHON_BYTEARRAY", "graalpython.bytearray", false);
211+
private static final PythonFrozenModule GRAALPYTHON_CTYPES = new PythonFrozenModule("GRAALPYTHON_CTYPES", "graalpython.ctypes", false);
212+
private static final PythonFrozenModule GRAALPYTHON_FUNCTION = new PythonFrozenModule("GRAALPYTHON_FUNCTION", "graalpython.function", false);
213+
private static final PythonFrozenModule GRAALPYTHON_JAVA = new PythonFrozenModule("GRAALPYTHON_JAVA", "graalpython.java", false);
214+
private static final PythonFrozenModule GRAALPYTHON_PIP_HOOK = new PythonFrozenModule("GRAALPYTHON_PIP_HOOK", "graalpython.pip_hook", false);
215+
private static final PythonFrozenModule GRAALPYTHON_TYPE = new PythonFrozenModule("GRAALPYTHON_TYPE", "graalpython.type", false);
216+
private static final PythonFrozenModule GRAALPYTHON_UNICODEDATA = new PythonFrozenModule("GRAALPYTHON_UNICODEDATA", "graalpython.unicodedata", false);
217+
private static final PythonFrozenModule GRAALPYTHON_ZIPIMPORT = new PythonFrozenModule("GRAALPYTHON_ZIPIMPORT", "graalpython.zipimport", false);
202218
}
203219

204220
public static final PythonFrozenModule lookup(String name) {
@@ -531,6 +547,38 @@ public static final PythonFrozenModule lookup(String name) {
531547
return Map.__HELLO__;
532548
case "__hello_only__":
533549
return Map.FROZEN_ONLY;
550+
case "_sysconfigdata":
551+
return Map._SYSCONFIGDATA;
552+
case "ginstall":
553+
return Map.GINSTALL;
554+
case "graalpython.__graalpython__":
555+
return Map.GRAALPYTHON___GRAALPYTHON__;
556+
case "graalpython._sre":
557+
return Map.GRAALPYTHON__SRE;
558+
case "graalpython._struct":
559+
return Map.GRAALPYTHON__STRUCT;
560+
case "graalpython._sysconfig":
561+
return Map.GRAALPYTHON__SYSCONFIG;
562+
case "graalpython._weakref":
563+
return Map.GRAALPYTHON__WEAKREF;
564+
case "graalpython.builtins":
565+
return Map.GRAALPYTHON_BUILTINS;
566+
case "graalpython.bytearray":
567+
return Map.GRAALPYTHON_BYTEARRAY;
568+
case "graalpython.ctypes":
569+
return Map.GRAALPYTHON_CTYPES;
570+
case "graalpython.function":
571+
return Map.GRAALPYTHON_FUNCTION;
572+
case "graalpython.java":
573+
return Map.GRAALPYTHON_JAVA;
574+
case "graalpython.pip_hook":
575+
return Map.GRAALPYTHON_PIP_HOOK;
576+
case "graalpython.type":
577+
return Map.GRAALPYTHON_TYPE;
578+
case "graalpython.unicodedata":
579+
return Map.GRAALPYTHON_UNICODEDATA;
580+
case "graalpython.zipimport":
581+
return Map.GRAALPYTHON_ZIPIMPORT;
534582
default:
535583
return null;
536584
}

0 commit comments

Comments
 (0)