Skip to content

Commit 79240d1

Browse files
timfeltimfel
committed
make the recorded patterns and retagging in general more robust
1 parent 70f506d commit 79240d1

File tree

1 file changed

+84
-41
lines changed

1 file changed

+84
-41
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

Lines changed: 84 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -99,16 +99,21 @@ def fun():
9999
import re
100100

101101
executable = sys.executable.split(" ") # HACK: our sys.executable on Java is a cmdline
102-
re_success = re.compile("^(test\S+)[^\r\n]* \.\.\. ok$", re.MULTILINE)
102+
re_success = re.compile("(test\S+) \(([^\s]+)\) \.\.\. ok$", re.MULTILINE)
103103
kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True, "check": False}
104104

105105
glob_pattern = os.path.join(os.path.dirname(test.__file__), "test_*.py")
106106
retag = False
107+
maxrepeats = 2
107108
for arg in sys.argv[1:]:
108109
if arg == "--retag":
109110
retag = True
111+
elif arg.startswith("--maxrepeats="):
112+
maxrepeats = int(arg.partition("=")[2])
113+
elif arg == "--help":
114+
print(sys.argv[0] + " [--retag] [--maxrepeats=n] [glob]")
110115
else:
111-
glob_pattern = os.path.join(os.path.dirname(test.__file__), sys.argv[1])
116+
glob_pattern = os.path.join(os.path.dirname(test.__file__), arg)
112117

113118
p = subprocess.run(["/usr/bin/which", "timeout"], **kwargs)
114119
if p.returncode != 0:
@@ -118,43 +123,81 @@ def fun():
118123

119124
testfiles = glob.glob(glob_pattern)
120125
for idx, testfile in enumerate(testfiles):
121-
testfile_stem = os.path.splitext(os.path.basename(testfile))[0]
122-
testmod = "test." + testfile_stem
123-
cmd = [timeout, "-s", "9", "60"] + executable + ["-S", "-m"]
124-
tagfile = os.path.join(TAGS_DIR, testfile_stem + ".txt")
125-
test_selectors = working_selectors(tagfile)
126-
127-
if test_selectors is None and not retag:
128-
# there's no tagfile for this, so it's not working at all (or has
129-
# not been tried)
130-
continue
131-
132-
print("[%d/%d] Testing %s" %(idx + 1, len(testfiles), testmod))
133-
cmd += ["unittest", "-v"]
134-
for selector in test_selectors:
135-
cmd += ["-k", selector]
136-
cmd.append(testfile)
137-
138-
print(" ".join(cmd))
139-
p = subprocess.run(cmd, **kwargs)
140-
print("*stdout*")
141-
print(p.stdout)
142-
print("*stderr*")
143-
print(p.stderr)
144-
145-
if p.returncode == 0 and not os.path.exists(tagfile):
146-
# if we're re-tagging a test without tags, all passed
147-
with open(tagfile, "w") as f:
148-
pass
149-
else:
150-
passing_tests = []
151-
for m in re_success.findall(p.stdout):
152-
passing_tests.append(m)
153-
for m in re_success.findall(p.stderr):
154-
passing_tests.append(m)
155-
with open(tagfile, "w") as f:
156-
for passing_test in passing_tests:
157-
f.write(passing_test)
158-
f.write("\n")
159-
if not passing_tests:
126+
for repeat in range(maxrepeats):
127+
# we always do this multiple times, because sometimes the tagging
128+
# doesn't quite work e.g. we create a tags file that'll still fail
129+
# when we use it. Thus, when we run this multiple times, we'll just
130+
# use the tags and if it fails in the last run, we assume something
131+
# sad is happening and delete the tags file to skip the tests
132+
# entirely
133+
testfile_stem = os.path.splitext(os.path.basename(testfile))[0]
134+
testmod = "test." + testfile_stem
135+
cmd = [timeout, "-s", "9", "60"] + executable + ["-S", "-m"]
136+
tagfile = os.path.join(TAGS_DIR, testfile_stem + ".txt")
137+
test_selectors = working_selectors(tagfile)
138+
139+
if test_selectors is None:
140+
if retag:
141+
test_selectors = []
142+
else:
143+
# there's no tagfile for this, so it's not working at all
144+
# (or has not been tried).
145+
continue
146+
147+
print("[%d/%d, Try %d] Testing %s" %(idx + 1, len(testfiles), repeat + 1, testmod))
148+
cmd += ["unittest", "-v"]
149+
for selector in test_selectors:
150+
cmd += ["-k", selector]
151+
cmd.append(testfile)
152+
153+
print(" ".join(cmd))
154+
p = subprocess.run(cmd, **kwargs)
155+
print("*stdout*")
156+
print(p.stdout)
157+
print("*stderr*")
158+
print(p.stderr)
159+
160+
if p.returncode == 0 and not os.path.exists(tagfile):
161+
# if we're re-tagging a test without tags, all passed
162+
with open(tagfile, "w") as f:
163+
pass
164+
elif p.returncode == 0:
165+
# we ran the tagged tests and they were fine
166+
break
167+
elif repeat < maxrepeats:
168+
# we failed the first run, create a tag file with the passing
169+
# tests (if any)
170+
passing_tests = []
171+
172+
try:
173+
imported_test_module = __import__(testmod)
174+
except:
175+
imported_test_module = None
176+
177+
def get_pass_name(funcname, classname):
178+
if imported_test_module:
179+
classname = "".join(classname.rpartition(testmod)[1:])
180+
clazz = imported_test_module
181+
path_to_class = classname.split(".")[1:]
182+
for part in path_to_class:
183+
clazz = getattr(clazz, part)
184+
return getattr(clazz, funcname).__qualname__
185+
else:
186+
return funcname
187+
188+
# n.b.: we add a '*' in the front, so that unittests doesn't add
189+
# its own asterisks, because now this is already a pattern
190+
for funcname,classname in re_success.findall(p.stdout):
191+
passing_tests.append("*" + get_pass_name(funcname, classname))
192+
for funcname,classname in re_success.findall(p.stderr):
193+
passing_tests.append("*" + get_pass_name(funcname, classname))
194+
195+
with open(tagfile, "w") as f:
196+
for passing_test in passing_tests:
197+
f.write(passing_test)
198+
f.write("\n")
199+
if not passing_tests:
200+
os.unlink(tagfile)
201+
else:
202+
# we tried a second time and failed, so our tags don't work for some reason
160203
os.unlink(tagfile)

0 commit comments

Comments
 (0)