feat(AST parser): use v2 parser architecture (GoogleCloudPlatform#5266)

Author: Ace Nassri

.kokoro/docker/Dockerfile

@@ -143,8 +143,8 @@ RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/ge
   # https://github.com/docker-library/python/pull/100
     && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ]
 
-# Ensure Pip for 3.8.0
-RUN python3.8 /tmp/get-pip.py
+# Ensure Pip for python3
+RUN python3 /tmp/get-pip.py
 RUN rm /tmp/get-pip.py
 
 # Install "virtualenv", since the vast majority of users of this image

.kokoro/tests/run_single_test.sh

@@ -50,20 +50,36 @@ if [[ "${INJECT_REGION_TAGS:-}" == "true" ]]; then
     export XUNIT_PATH="$PWD/sponge_log.xml"
     export XUNIT_TMP_PATH="$(mktemp)"
 
+    # We use `python3.8` because it is the version pip3 installs to.
+    export PYTHON_CMD="python3"
+
     if [[ -f "$XUNIT_PATH" ]]; then
         echo "=== Injecting region tags into XUnit output ==="
         echo "Processing XUnit output file: $XUNIT_PATH (saving output to $XUNIT_TMP_PATH)"
 
-	# We use `python3` because it will work even if we remove old
-	# python versions from the docker image.
-	echo "Calling python3 ${PARSER_PATH} inject-snippet-mapping --output_file ${XUNIT_TMP_PATH} ${PWD}"
+        TARGET_DIR=$PWD
+
+        # First, we generate a Python-specific "polyglot_snippet_data.json" file
+        # (cd'ing into script's working dir is necessary for imports to work)
+        echo "Calling $PYTHON_CMD ${PYTHON_PARSER_PATH} ${TARGET_DIR}"
+        pushd $(dirname $PYTHON_PARSER_PATH)
+        $PYTHON_CMD "$PYTHON_PARSER_PATH" "$TARGET_DIR"
+        popd
+
+        # Then, we pass the "polyglot_snippet_data.json" file and the XUnit output
+        # to the "polyglot" parser.
+        #   This outputs the XUnit input with the snippet-test map added
+        #   Again, we use `python3` for pip3 support.
+        echo "Calling: cat $XUNIT_PATH | $PYTHON_CMD ${POLYGLOT_PARSER_PATH} inject-snippet-mapping --output_file ${XUNIT_TMP_PATH} ${TARGET_DIR}"
+        pushd $(dirname $POLYGLOT_PARSER_PATH)
         cat "$XUNIT_PATH" | \
-	    python3 "$PARSER_PATH" inject-snippet-mapping --output_file "$XUNIT_TMP_PATH" "$PWD"
+        $PYTHON_CMD "$POLYGLOT_PARSER_PATH" --output_file "$XUNIT_TMP_PATH" inject-snippet-mapping "$TARGET_DIR"
         if [[ $? -eq 0 ]] && [[ -s "$XUNIT_PATH" ]]; then
             mv $XUNIT_TMP_PATH $XUNIT_PATH
         else
             echo "Region tag injection FAILED; XUnit file not modified."
         fi
+        popd
     else
         echo "No XUnit output file found!"
     fi

.kokoro/tests/run_tests.sh

@@ -130,18 +130,21 @@ ROOT=$(pwd)
 if [[ "${INJECT_REGION_TAGS:-}" == "true" ]]; then
     echo "=== Setting up DRIFT region tag injector ==="
     # install PyYaml (used by the DRIFT region tag parsing system)
-    echo "--- Installing PyYaml ---"
-    python3 -m pip install --user pyyaml
+    echo "--- Installing pip packages ---"
+    python3 -m pip install --user pyyaml frozendict recordclass
 
     # Use ${HOME} because trampoline will automatically clean up this
     # directory.
     export REGION_TAG_PARSER_DIR="${HOME}/region-tag-parser"
-    export PARSER_PATH="${REGION_TAG_PARSER_DIR}/wizard-py/cli.py"
+    export POLYGLOT_PARSER_PATH="${REGION_TAG_PARSER_DIR}/xunit-autolabeler-v2/cli_bootstrap.py"
+    export PYTHON_PARSER_PATH="${REGION_TAG_PARSER_DIR}/xunit-autolabeler-v2/ast_parser/python_bootstrap.py"
 
-    if [[ ! -f $PARSER_PATH ]]; then
+    if [[ ! -f $POLYGLOT_PARSER_PATH ]]; then
         echo "--- Fetching injection script from HEAD (via GitHub) ---"
         git clone https://github.com/GoogleCloudPlatform/repo-automation-playground "$REGION_TAG_PARSER_DIR" --single-branch
-        chmod +x $PARSER_PATH
+
+        chmod +x $PYTHON_PARSER_PATH
+        chmod +x $POLYGLOT_PARSER_PATH
     fi
     echo "=== Region tag injector setup complete ==="
 fi