Skip to content

Commit d20cd27

Browse files
ppislppisl
committed
[GR-20595] Implement ReadVariableTag in Python.
PullRequest: graalpython/783
2 parents 61e559c + c980b5f commit d20cd27

File tree

171 files changed

+2768
-2017
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

171 files changed

+2768
-2017
lines changed

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/interop/ReadAndWriteVariableTagsTests.java

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
/*
2+
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.test.interop;
42+
43+
import com.oracle.graal.python.nodes.PNode;
44+
import com.oracle.graal.python.test.PythonTests;
45+
import com.oracle.truffle.api.frame.VirtualFrame;
46+
import com.oracle.truffle.api.instrumentation.EventContext;
47+
import com.oracle.truffle.api.instrumentation.ExecutionEventListener;
48+
import com.oracle.truffle.api.instrumentation.InstrumentableNode;
49+
import com.oracle.truffle.api.instrumentation.SourceSectionFilter;
50+
import com.oracle.truffle.api.instrumentation.StandardTags;
51+
import com.oracle.truffle.api.instrumentation.TruffleInstrument;
52+
import com.oracle.truffle.api.interop.InteropLibrary;
53+
import com.oracle.truffle.api.library.LibraryFactory;
54+
import com.oracle.truffle.api.nodes.Node;
55+
import java.util.HashMap;
56+
import org.graalvm.polyglot.Context;
57+
import org.graalvm.polyglot.Engine;
58+
import org.graalvm.polyglot.Instrument;
59+
import org.graalvm.polyglot.Source;
60+
import static org.junit.Assert.assertTrue;
61+
import com.oracle.graal.python.nodes.instrumentation.NodeObjectDescriptor;
62+
import com.oracle.truffle.api.interop.UnknownIdentifierException;
63+
import com.oracle.truffle.api.interop.UnsupportedMessageException;
64+
import java.util.HashSet;
65+
import java.util.Set;
66+
import org.junit.Test;
67+
68+
public class ReadAndWriteVariableTagsTests extends PythonTests {
69+
70+
static final InteropLibrary INTEROP = LibraryFactory.resolve(InteropLibrary.class).getUncached();
71+
72+
private static Context newContext(Engine engine) {
73+
return Context.newBuilder().allowExperimentalOptions(true).allowAllAccess(true).engine(engine).build();
74+
}
75+
76+
@Test
77+
public void testInModule() throws Exception {
78+
String code = "a = 0\n" +
79+
"b = a\n" +
80+
"c = a + a + b";
81+
HashMap<String, Set<InstrumentableNode>> nodes = checkReadVariableTagAndWriteVariableTag(code);
82+
// checkWriteNodeTag(nodes, "a", 0, 5);
83+
checkReadNodeTag(nodes, "a", 10, 11);
84+
checkReadNodeTag(nodes, "a", 16, 17);
85+
checkReadNodeTag(nodes, "a", 20, 21);
86+
checkReadNodeTag(nodes, "b", 24, 25);
87+
}
88+
89+
@Test
90+
public void testFunction01() throws Exception {
91+
String code = "def fn(arg):\n" +
92+
" arg = arg + 1\n" +
93+
" return arg\n" +
94+
"fn(1)";
95+
HashMap<String, Set<InstrumentableNode>> nodes = checkReadVariableTagAndWriteVariableTag(code);
96+
checkReadNodeTag(nodes, "arg", 21, 24);
97+
checkReadNodeTag(nodes, "arg", 38, 41);
98+
checkReadNodeTag(nodes, "fn", 42, 44);
99+
}
100+
101+
@Test
102+
public void testFunction02() throws Exception {
103+
String code = "c = 2\n" + "def foo(a, b): \n" +
104+
" return a + b + c\n" +
105+
"foo(1, 2)";
106+
HashMap<String, Set<InstrumentableNode>> nodes = checkReadVariableTagAndWriteVariableTag(code);
107+
checkReadNodeTag(nodes, "a", 31, 32);
108+
checkReadNodeTag(nodes, "b", 35, 36);
109+
checkReadNodeTag(nodes, "c", 39, 40);
110+
checkReadNodeTag(nodes, "foo", 41, 44);
111+
}
112+
113+
@Test
114+
public void testGenerator01() throws Exception {
115+
String code = "g = (x*x for x in range(10))\n" +
116+
"for i in g:\n" +
117+
" i";
118+
HashMap<String, Set<InstrumentableNode>> nodes = checkReadVariableTagAndWriteVariableTag(code);
119+
checkReadNodeTag(nodes, "g", 38, 39);
120+
checkReadNodeTag(nodes, "x", 7, 8);
121+
checkReadNodeTag(nodes, "x", 5, 6);
122+
checkReadNodeTag(nodes, "range", 18, 23);
123+
checkReadNodeTag(nodes, "i", 43, 44);
124+
}
125+
126+
@Test
127+
public void testGenerator02() throws Exception {
128+
String code = "def fn(arg):\n" +
129+
" g = (x*x for x in range(arg))\n" +
130+
" return g\n" +
131+
"for i in fn(10):\n" +
132+
" i";
133+
HashMap<String, Set<InstrumentableNode>> nodes = checkReadVariableTagAndWriteVariableTag(code);
134+
checkReadNodeTag(nodes, "arg", 39, 42);
135+
checkReadNodeTag(nodes, "g", 54, 55);
136+
checkReadNodeTag(nodes, "fn", 65, 67);
137+
checkReadNodeTag(nodes, "x", 22, 23);
138+
checkReadNodeTag(nodes, "x", 20, 21);
139+
checkReadNodeTag(nodes, "range", 33, 38);
140+
checkReadNodeTag(nodes, "i", 75, 76);
141+
}
142+
143+
@Test
144+
public void testGenerator03() throws Exception {
145+
String code = "def fn(item1, item2, item3):\n" +
146+
" yield item1\n" +
147+
" yield item2\n" +
148+
" yield item3\n" +
149+
"g = fn(1, 2, 3)\n" +
150+
"for i in g:\n" +
151+
" i\n";
152+
HashMap<String, Set<InstrumentableNode>> nodes = checkReadVariableTagAndWriteVariableTag(code);
153+
checkReadNodeTag(nodes, "item1", 37, 42);
154+
checkReadNodeTag(nodes, "item2", 51, 56);
155+
checkReadNodeTag(nodes, "item3", 65, 70);
156+
checkReadNodeTag(nodes, "g", 96, 97);
157+
checkReadNodeTag(nodes, "fn", 75, 77);
158+
checkReadNodeTag(nodes, "i", 101, 102);
159+
}
160+
161+
@Test
162+
public void testListComp01() throws Exception {
163+
String code = "y = 1\n" +
164+
"[x**y for x in range(20) if x*y % 3]\n";
165+
HashMap<String, Set<InstrumentableNode>> nodes = checkReadVariableTagAndWriteVariableTag(code);
166+
checkReadNodeTag(nodes, "x", 7, 8);
167+
checkReadNodeTag(nodes, "x", 34, 35);
168+
checkReadNodeTag(nodes, "range", 21, 26);
169+
checkReadNodeTag(nodes, "y", 10, 11);
170+
checkReadNodeTag(nodes, "y", 36, 37);
171+
}
172+
173+
private void printNodes(HashMap<String, Set<InstrumentableNode>> nodesMap) {
174+
for (String name : nodesMap.keySet()) {
175+
for (InstrumentableNode node : nodesMap.get(name)) {
176+
StringBuilder sb = new StringBuilder();
177+
sb.append("check");
178+
if (node.hasTag(StandardTags.ReadVariableTag.class)) {
179+
sb.append("ReadNodeTag(nodes, \"");
180+
} else {
181+
sb.append("WriteNodeTag(nodes, \"");
182+
}
183+
sb.append(name);
184+
sb.append("\", ").append(((PNode) node).getSourceSection().getCharIndex());
185+
sb.append(", ").append(((PNode) node).getSourceSection().getCharEndIndex());
186+
sb.append(");");
187+
System.out.println(sb.toString());
188+
}
189+
}
190+
}
191+
192+
private void checkWriteNodeTag(HashMap<String, Set<InstrumentableNode>> nodesMap, String name, int startOffset, int endOffset) {
193+
Set<InstrumentableNode> nodes = nodesMap.get(name);
194+
assertTrue("Any node with name '" + name + "' was found.", nodes != null);
195+
for (InstrumentableNode node : nodes) {
196+
if (node.hasTag(StandardTags.WriteVariableTag.class) && ((PNode) node).getSourceSection().getCharIndex() == startOffset &&
197+
((PNode) node).getSourceSection().getCharEndIndex() == endOffset) {
198+
return;
199+
}
200+
}
201+
assertTrue("Any write node with name '" + name + "' and source section[" + startOffset + ", " + endOffset + "] was found.", false);
202+
}
203+
204+
private void checkReadNodeTag(HashMap<String, Set<InstrumentableNode>> nodesMap, String name, int startOffset, int endOffset) {
205+
Set<InstrumentableNode> nodes = nodesMap.get(name);
206+
assertTrue("Any node with name '" + name + "' was found.", nodes != null);
207+
for (InstrumentableNode node : nodes) {
208+
if (node.hasTag(StandardTags.ReadVariableTag.class) && ((PNode) node).getSourceSection().getCharIndex() == startOffset &&
209+
((PNode) node).getSourceSection().getCharEndIndex() == endOffset) {
210+
return;
211+
}
212+
}
213+
assertTrue("Any read node with name '" + name + "' and source section[" + startOffset + ", " + endOffset + "] was found.", false);
214+
}
215+
216+
private HashMap<String, Set<InstrumentableNode>> checkReadVariableTagAndWriteVariableTag(String code) throws Exception {
217+
Engine engine = Engine.newBuilder().build();
218+
Context newContext = newContext(engine);
219+
newContext.initialize("python");
220+
Instrument envInstr = newContext.getEngine().getInstruments().get("TestPythonInstrumentRW");
221+
222+
Source source = Source.newBuilder("python", code, "testing").build();
223+
TruffleInstrument.Env env = envInstr.lookup(TestEnvProvider.class).env;
224+
SourceSectionFilter filter = SourceSectionFilter.newBuilder().lineIn(1, source.getLineCount()).tagIs(StandardTags.ReadVariableTag.class, StandardTags.WriteVariableTag.class).build();
225+
HashMap<String, Set<InstrumentableNode>> nodesMap = new HashMap<>();
226+
env.getInstrumenter().attachExecutionEventListener(filter, new ReadVariableTagAndWriteVariableTagListener(nodesMap));
227+
228+
newContext.eval(source);
229+
return nodesMap;
230+
}
231+
232+
private static class TestEnvProvider {
233+
234+
TruffleInstrument.Env env;
235+
236+
TestEnvProvider(TruffleInstrument.Env env) {
237+
this.env = env;
238+
}
239+
}
240+
241+
@TruffleInstrument.Registration(id = "TestPythonInstrumentRW", services = TestEnvProvider.class)
242+
public static class TestPythonInstrument extends TruffleInstrument {
243+
244+
@Override
245+
protected void onCreate(final TruffleInstrument.Env env) {
246+
env.registerService(new TestEnvProvider(env));
247+
}
248+
}
249+
250+
private static class ReadVariableTagAndWriteVariableTagListener implements ExecutionEventListener {
251+
252+
// map of the variable name -> list of nodes
253+
private final HashMap<String, Set<InstrumentableNode>> result;
254+
255+
public ReadVariableTagAndWriteVariableTagListener(HashMap<String, Set<InstrumentableNode>> resultMap) {
256+
this.result = resultMap;
257+
}
258+
259+
@Override
260+
public void onEnter(EventContext context, VirtualFrame frame) {
261+
Node node = context.getInstrumentedNode();
262+
InstrumentableNode inode = (InstrumentableNode) node;
263+
264+
NodeObjectDescriptor descr = (NodeObjectDescriptor) inode.getNodeObject();
265+
String name = null;
266+
try {
267+
if (inode.hasTag(StandardTags.WriteVariableTag.class)) {
268+
name = (String) INTEROP.readMember(descr, StandardTags.WriteVariableTag.NAME);
269+
} else if (inode.hasTag(StandardTags.ReadVariableTag.class)) {
270+
name = (String) INTEROP.readMember(descr, StandardTags.ReadVariableTag.NAME);
271+
}
272+
} catch (UnsupportedMessageException | UnknownIdentifierException ex) {
273+
274+
}
275+
assertTrue("Attribute name was not found", name != null);
276+
Set<InstrumentableNode> nodes = result.get(name);
277+
if (nodes == null) {
278+
nodes = new HashSet<>();
279+
result.put(name, nodes);
280+
}
281+
nodes.add(inode);
282+
}
283+
284+
@Override
285+
public void onReturnValue(EventContext context, VirtualFrame frame, Object result) {
286+
}
287+
288+
@Override
289+
public void onReturnExceptional(EventContext context, VirtualFrame frame, Throwable exception) {
290+
}
291+
}
292+
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ClassDefTests.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -145,6 +145,15 @@ public void classDef12() throws Exception {
145145
" t = get_nested_class()\n");
146146
}
147147

148+
@Test
149+
public void classDef13() throws Exception {
150+
checkScopeAndTree("class Point:\n" +
151+
" x = 0\n" +
152+
" y = 0\n" +
153+
"p = Point()\n" +
154+
"a = p.x + p.y\n");
155+
}
156+
148157
@Test
149158
public void decorator01() throws Exception {
150159
checkScopeAndTree("@class_decorator\n" +

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/FuncDefTests.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -183,6 +183,14 @@ public void functionDef20() throws Exception {
183183
checkScopeAndTree("def __init__(self, max_size=0, *, ctx, pending_work_items): pass");
184184
}
185185

186+
@Test
187+
public void functionDef21() throws Exception {
188+
checkScopeAndTree("c = 2\n" +
189+
"def foo(a, b): \n" +
190+
" return a + b + c\n" +
191+
"foo(1,2)\n");
192+
}
193+
186194
@Test
187195
public void decorator01() throws Exception {
188196
checkScopeAndTree();

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/AssignmentTests/assignment09.tast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ ModuleRootNode Name: <module 'assignment09'> SourceSection: [0,9]`f().b = 1`
1313
Attribute: b
1414
PythonCallNodeGen SourceSection: [0,3]`f()`
1515
CallNodeGen SourceSection: None
16-
ReadNameNodeGen SourceSection: None
16+
ReadNameNodeGen SourceSection: [0,1]`f`
1717
Identifier: f
1818
IntegerLiteralNode SourceSection: [8,9]`1`
1919
Value: 1

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/AwaitAndAsyncTests/await01.tast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ ModuleRootNode Name: <module 'await01'> SourceSection: [0,28]`async def f():↵
3737
ExpressionStatementNode SourceSection: [16,28]`await smth()`
3838
PythonCallNodeGen SourceSection: [16,28]`await smth()`
3939
CallNodeGen SourceSection: None
40-
ReadGlobalOrBuiltinNodeGen SourceSection: None
40+
ReadGlobalOrBuiltinNodeGen SourceSection: [22,26]`smth`
4141
Identifier: smth
4242
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
4343
Return Expresssion: ReadLocalVariableNode SourceSection: None

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/AwaitAndAsyncTests/await02.tast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ ModuleRootNode Name: <module 'await02'> SourceSection: [0,34]`async def f():↵
4040
Frame: [0,foo,Illegal]
4141
PythonCallNodeGen SourceSection: [22,34]`await smth()`
4242
CallNodeGen SourceSection: None
43-
ReadGlobalOrBuiltinNodeGen SourceSection: None
43+
ReadGlobalOrBuiltinNodeGen SourceSection: [28,32]`smth`
4444
Identifier: smth
4545
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
4646
Return Expresssion: ReadLocalVariableNode SourceSection: None

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/AwaitAndAsyncTests/await03.tast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ ModuleRootNode Name: <module 'await03'> SourceSection: [0,39]`async def f():↵
3737
DestructuringAssignmentNodeGen SourceSection: [16,39]`foo, bar = await smt...`
3838
PythonCallNodeGen SourceSection: [27,39]`await smth()`
3939
CallNodeGen SourceSection: None
40-
ReadGlobalOrBuiltinNodeGen SourceSection: None
40+
ReadGlobalOrBuiltinNodeGen SourceSection: [33,37]`smth`
4141
Identifier: smth
4242
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
4343
WriteLocalVariableNodeGen SourceSection: None

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/AwaitAndAsyncTests/await04.tast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ ModuleRootNode Name: <module 'await04'> SourceSection: [0,30]`async def f():↵
3737
ExpressionStatementNode SourceSection: [17,29]`await smth()`
3838
PythonCallNodeGen SourceSection: [17,29]`await smth()`
3939
CallNodeGen SourceSection: None
40-
ReadGlobalOrBuiltinNodeGen SourceSection: None
40+
ReadGlobalOrBuiltinNodeGen SourceSection: [23,27]`smth`
4141
Identifier: smth
4242
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
4343
Return Expresssion: ReadLocalVariableNode SourceSection: None

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/AwaitAndAsyncTests/await05.tast

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ ModuleRootNode Name: <module 'await05'> SourceSection: [0,35]`async def f():↵
3939
CallNodeGen SourceSection: None
4040
PythonCallNodeGen SourceSection: [21,33]`await smth()`
4141
CallNodeGen SourceSection: None
42-
ReadGlobalOrBuiltinNodeGen SourceSection: None
42+
ReadGlobalOrBuiltinNodeGen SourceSection: [27,31]`smth`
4343
Identifier: smth
4444
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
45-
ReadGlobalOrBuiltinNodeGen SourceSection: None
45+
ReadGlobalOrBuiltinNodeGen SourceSection: [16,19]`foo`
4646
Identifier: foo
4747
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
4848
Return Expresssion: ReadLocalVariableNode SourceSection: None

0 commit comments

Comments
 (0)