|
| 1 | +# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. |
| 2 | +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 3 | +# |
| 4 | +# The Universal Permissive License (UPL), Version 1.0 |
| 5 | +# |
| 6 | +# Subject to the condition set forth below, permission is hereby granted to any |
| 7 | +# person obtaining a copy of this software, associated documentation and/or |
| 8 | +# data (collectively the "Software"), free of charge and under any and all |
| 9 | +# copyright rights in the Software, and any and all patent rights owned or |
| 10 | +# freely licensable by each licensor hereunder covering either (i) the |
| 11 | +# unmodified Software as contributed to or provided by such licensor, or (ii) |
| 12 | +# the Larger Works (as defined below), to deal in both |
| 13 | +# |
| 14 | +# (a) the Software, and |
| 15 | +# |
| 16 | +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if |
| 17 | +# one is included with the Software each a "Larger Work" to which the Software |
| 18 | +# is contributed by such licensors), |
| 19 | +# |
| 20 | +# without restriction, including without limitation the rights to copy, create |
| 21 | +# derivative works of, display, perform, and distribute the Software and make, |
| 22 | +# use, sell, offer for sale, import, export, have made, and have sold the |
| 23 | +# Software and the Larger Work(s), and to sublicense the foregoing rights on |
| 24 | +# either these or other terms. |
| 25 | +# |
| 26 | +# This license is subject to the following condition: |
| 27 | +# |
| 28 | +# The above copyright notice and either this complete permission notice or at a |
| 29 | +# minimum a reference to the UPL must be included in all copies or substantial |
| 30 | +# portions of the Software. |
| 31 | +# |
| 32 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 33 | +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 34 | +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 35 | +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 36 | +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 37 | +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 38 | +# SOFTWARE. |
| 39 | + |
| 40 | +import glob |
| 41 | +import os |
| 42 | +import subprocess |
| 43 | +import sys |
| 44 | +import test |
| 45 | + |
| 46 | + |
| 47 | +if os.environ.get("ENABLE_CPYTHON_TAGGED_UNITTESTS") == "true" or os.environ.get("CI") is None: |
| 48 | + # On the CI, I'd like to explicitly enable these, so we can run them in a |
| 49 | + # separate job easily. But it's not important for local execution |
| 50 | + TAGS_DIR = os.path.join(os.path.dirname(__file__), "unittest_tags") |
| 51 | +else: |
| 52 | + TAGS_DIR = "null" |
| 53 | + |
| 54 | + |
| 55 | +def working_selectors(tagfile): |
| 56 | + if os.path.exists(tagfile): |
| 57 | + with open(tagfile) as f: |
| 58 | + return [line.strip() for line in f if line] |
| 59 | + else: |
| 60 | + return None |
| 61 | + |
| 62 | + |
| 63 | +def working_tests(): |
| 64 | + working_tests = [] |
| 65 | + glob_pattern = os.path.join(TAGS_DIR, "*.txt") |
| 66 | + for arg in sys.argv: |
| 67 | + if arg.startswith("--tagfile="): |
| 68 | + glob_pattern = os.path.join(TAGS_DIR, arg.partition("=")[2]) |
| 69 | + sys.argv.remove(arg) |
| 70 | + break |
| 71 | + for tagfile in glob.glob(glob_pattern): |
| 72 | + test = os.path.splitext(os.path.basename(tagfile))[0] |
| 73 | + working_tests.append((test, working_selectors(tagfile))) |
| 74 | + return working_tests |
| 75 | + |
| 76 | + |
| 77 | +WORKING_TESTS = working_tests() |
| 78 | +for idx, working_test in enumerate(WORKING_TESTS): |
| 79 | + def make_test_func(working_test): |
| 80 | + def fun(): |
| 81 | + cmd = [sys.executable, "-S", "-m", "unittest"] |
| 82 | + for testpattern in working_test[1]: |
| 83 | + cmd.extend(["-k", testpattern]) |
| 84 | + testmod = working_test[0].rpartition(".")[2] |
| 85 | + print() |
| 86 | + cmd.append(os.path.join(os.path.dirname(test.__file__), "%s.py" % testmod)) |
| 87 | + subprocess.check_call(cmd) |
| 88 | + |
| 89 | + fun.__name__ = "%s[%d/%d]" % (working_test[0], idx + 1, len(WORKING_TESTS)) |
| 90 | + return fun |
| 91 | + |
| 92 | + test_f = make_test_func(working_test) |
| 93 | + globals()[test_f.__name__] = test_f |
| 94 | + del test_f |
| 95 | + |
| 96 | + |
| 97 | +if __name__ == "__main__": |
| 98 | + # find working tests |
| 99 | + import re |
| 100 | + |
| 101 | + executable = sys.executable.split(" ") # HACK: our sys.executable on Java is a cmdline |
| 102 | + re_success = re.compile("(test\S+) \(([^\s]+)\) \.\.\. ok$", re.MULTILINE) |
| 103 | + re_failure = re.compile("(test\S+) \(([^\s]+)\) \.\.\. (?:ERROR|FAIL)$", re.MULTILINE) |
| 104 | + kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True, "check": False} |
| 105 | + |
| 106 | + glob_pattern = os.path.join(os.path.dirname(test.__file__), "test_*.py") |
| 107 | + retag = False |
| 108 | + maxrepeats = 2 |
| 109 | + for arg in sys.argv[1:]: |
| 110 | + if arg == "--retag": |
| 111 | + retag = True |
| 112 | + elif arg.startswith("--maxrepeats="): |
| 113 | + maxrepeats = int(arg.partition("=")[2]) |
| 114 | + elif arg == "--help": |
| 115 | + print(sys.argv[0] + " [--retag] [--maxrepeats=n] [glob]") |
| 116 | + else: |
| 117 | + glob_pattern = os.path.join(os.path.dirname(test.__file__), arg) |
| 118 | + |
| 119 | + p = subprocess.run(["/usr/bin/which", "timeout"], **kwargs) |
| 120 | + if p.returncode != 0: |
| 121 | + print("Cannot find the 'timeout' GNU tool. Do you have coreutils installed?") |
| 122 | + sys.exit(1) |
| 123 | + timeout = p.stdout.strip() |
| 124 | + |
| 125 | + testfiles = glob.glob(glob_pattern) |
| 126 | + for idx, testfile in enumerate(testfiles): |
| 127 | + for repeat in range(maxrepeats): |
| 128 | + # we always do this multiple times, because sometimes the tagging |
| 129 | + # doesn't quite work e.g. we create a tags file that'll still fail |
| 130 | + # when we use it. Thus, when we run this multiple times, we'll just |
| 131 | + # use the tags and if it fails in the last run, we assume something |
| 132 | + # sad is happening and delete the tags file to skip the tests |
| 133 | + # entirely |
| 134 | + testfile_stem = os.path.splitext(os.path.basename(testfile))[0] |
| 135 | + testmod = "test." + testfile_stem |
| 136 | + cmd = [timeout, "-s", "9", "60"] + executable + ["-S", "-m"] |
| 137 | + tagfile = os.path.join(TAGS_DIR, testfile_stem + ".txt") |
| 138 | + if retag: |
| 139 | + test_selectors = [] |
| 140 | + else: |
| 141 | + test_selectors = working_selectors(tagfile) |
| 142 | + |
| 143 | + if test_selectors is None: |
| 144 | + # there's no tagfile for this, so it's not working at all (or |
| 145 | + # shouldn't be tried). |
| 146 | + continue |
| 147 | + |
| 148 | + print("[%d/%d, Try %d] Testing %s" %(idx + 1, len(testfiles), repeat + 1, testmod)) |
| 149 | + cmd += ["unittest", "-v"] |
| 150 | + for selector in test_selectors: |
| 151 | + cmd += ["-k", selector] |
| 152 | + cmd.append(testfile) |
| 153 | + |
| 154 | + print(" ".join(cmd)) |
| 155 | + p = subprocess.run(cmd, **kwargs) |
| 156 | + print("*stdout*") |
| 157 | + print(p.stdout) |
| 158 | + print("*stderr*") |
| 159 | + print(p.stderr) |
| 160 | + |
| 161 | + if p.returncode == 0 and not os.path.exists(tagfile): |
| 162 | + # if we're re-tagging a test without tags, all passed |
| 163 | + with open(tagfile, "w") as f: |
| 164 | + pass |
| 165 | + break |
| 166 | + elif p.returncode == 0: |
| 167 | + # we ran the tagged tests and they were fine |
| 168 | + break |
| 169 | + elif repeat < maxrepeats: |
| 170 | + # we failed the first run, create a tag file with the passing |
| 171 | + # tests (if any) |
| 172 | + passing_tests = [] |
| 173 | + failed_tests = [] |
| 174 | + |
| 175 | + def get_pass_name(funcname, classname): |
| 176 | + try: |
| 177 | + imported_test_module = __import__(testmod) |
| 178 | + except: |
| 179 | + imported_test_module = None |
| 180 | + else: |
| 181 | + # try hard to get a most specific pattern |
| 182 | + classname = "".join(classname.rpartition(testmod)[1:]) |
| 183 | + clazz = imported_test_module |
| 184 | + path_to_class = classname.split(".")[1:] |
| 185 | + for part in path_to_class: |
| 186 | + clazz = getattr(clazz, part, None) |
| 187 | + if clazz: |
| 188 | + func = getattr(clazz, funcname, None) |
| 189 | + if func: |
| 190 | + return func.__qualname__ |
| 191 | + return funcname |
| 192 | + |
| 193 | + # n.b.: we add a '*' in the front, so that unittests doesn't add |
| 194 | + # its own asterisks, because now this is already a pattern |
| 195 | + |
| 196 | + for funcname,classname in re_failure.findall(p.stdout): |
| 197 | + failed_tests.append("*" + get_pass_name(funcname, classname)) |
| 198 | + for funcname,classname in re_failure.findall(p.stderr): |
| 199 | + failed_tests.append("*" + get_pass_name(funcname, classname)) |
| 200 | + |
| 201 | + for funcname,classname in re_success.findall(p.stdout): |
| 202 | + passing_tests.append("*" + get_pass_name(funcname, classname)) |
| 203 | + for funcname,classname in re_success.findall(p.stderr): |
| 204 | + passing_tests.append("*" + get_pass_name(funcname, classname)) |
| 205 | + |
| 206 | + # n.b.: unittests uses the __qualname__ of the function as |
| 207 | + # pattern, which we're trying to do as well. however, sometimes |
| 208 | + # the same function is shared in multiple test classes, and |
| 209 | + # fails in some. so we always subtract the failed patterns from |
| 210 | + # the passed patterns |
| 211 | + passing_only_patterns = set(passing_tests) - set(failed_tests) |
| 212 | + with open(tagfile, "w") as f: |
| 213 | + for passing_test in passing_only_patterns: |
| 214 | + f.write(passing_test) |
| 215 | + f.write("\n") |
| 216 | + if not passing_only_patterns: |
| 217 | + os.unlink(tagfile) |
| 218 | + else: |
| 219 | + # we tried the last time and failed, so our tags don't work for |
| 220 | + # some reason |
| 221 | + os.unlink(tagfile) |
0 commit comments