api-bench

#!/usr/bin/env bash

# Copyright (C) 2020 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set -eo pipefail

API_BENCH_EXECUTABLE=$(basename "$0")

pushd "$(dirname "$0")" &> /dev/null
API_BENCH_DIR=$PWD
popd &> /dev/null

CURRENT_BENCHMARK_ID=0
BENCHMARKS=()
BUILD_DIRECTORIES=()
REFERENCES=("X")
CURRENT_API=()
UPCOMING_API=()
ARCHS=${ARCHS:-$(uname -m)}
CONFIGURATION=${CONFIGURATION:-Release}

shouldBuild=true
runReferences=true
verbose=false
iterations=5

log() {
    if [ "$verbose" == true ]; then
        echo "$*"
    fi
}

printUsage() {
    cat <<EOF
usage: $API_BENCH_EXECUTABLE [-h|--help] [--no-build] [--no-reference(s)] [-v|--verbose] [--iterations <n=$iterations>] <build_directory1> [<build_directory2> ...]
EOF
    exit 0
}

collectFlags() {
    while [ $# -gt 0 ]; do
        case "$1" in
            -h|--help)
                printUsage
                ;;
            --no-build)
                shouldBuild=false
                ;;
            --no-reference|--no-references)
                runReferences=false
                ;;
            -v|--verbose)
                verbose=true
                ;;
            --iterations)
                shift
                iterations=$1
                ;;
            *)
                BUILD_DIRECTORIES=("$@")
                break 2
                ;;
        esac
        shift
  done

  if [ ${#BUILD_DIRECTORIES[@]} -lt 1 ]; then
      printUsage
      exit 1
  fi

  for i in "${!BUILD_DIRECTORIES[@]}"; do
      local buildDirectory
      buildDirectory=${BUILD_DIRECTORIES[$i]}

      if ! [ -d "$buildDirectory" ]; then
          echo "No such file or directory: $buildDirectory"
          exit 1
      fi

      pushd "$buildDirectory" &> /dev/null
      BUILD_DIRECTORIES[$i]=$PWD
      popd &> /dev/null
  done
}

collectBenchmarkDir() {
    local benchmarkVar=$1
    local benchmarkDir=$2
    benchmarks=("$API_BENCH_DIR"/"$benchmarkDir"/*)
    for benchmark in "${benchmarks[@]}"; do
        local benchmarkID=$CURRENT_BENCHMARK_ID
        CURRENT_BENCHMARK_ID=$((CURRENT_BENCHMARK_ID+1))
        pushd "$benchmark" &> /dev/null
        BENCHMARKS[$benchmarkID]=$PWD
        eval "${benchmarkVar}[\$benchmarkID]=$benchmarkID"
        popd &> /dev/null
    done
}

collectBenchmarks() {
    if [ "$runReferences" == true ]; then
        collectBenchmarkDir "REFERENCES" References
    fi
    collectBenchmarkDir "CURRENT_API" CurrentAPI
    collectBenchmarkDir "UPCOMING_API" UpcomingAPI
}

forEachBenchmark() {
    local functor
    functor="$1"
    declare "$functor"
    for benchmarkID in "${!BENCHMARKS[@]}"; do
        local benchmark
        benchmark=${BENCHMARKS[$benchmarkID]}
        pushd "$benchmark" &> /dev/null
        BENCHMARK_ID=$benchmarkID
        BENCHMARK_NAME=$(basename "$benchmark")
        "$functor"
        popd &> /dev/null
    done
}

buildBenchmark() {
    project=$BENCHMARK_NAME.xcodeproj

    if ! [ -e "$project" ]; then
        return
    fi

    log "Building $BENCHMARK_NAME... (FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY')"
    xcodebuild -project "$project" -target "$BENCHMARK_NAME" -configuration "$CONFIGURATION" build "FRAMEWORK_SEARCH_PATHS='$BUILD_DIRECTORY'" "ARCHS='$ARCHS'"
}

build() {
    BUILD_DIRECTORY=$1
    forEachBenchmark "buildBenchmark"
}

runJSBenchmark() {
    "$DYLD_FRAMEWORK_PATH/jsc" "$BENCHMARK_NAME.js"
}

runNativeBenchmark() {
    pushd "build/$CONFIGURATION" &> /dev/null
    "./$BENCHMARK_NAME"
    local exitCode=$?
    if [ $exitCode -ne 0 ]; then
        echo "Benchmark exited with an error: $PWD/$BENCHMARK_NAME exited with $exitCode" 1>&2
    fi
    popd &> /dev/null
    return $exitCode
}

runBenchmarkIteration() {
    if [ -e "build" ]; then
        runNativeBenchmark
    else
        runJSBenchmark
    fi
}

runBenchmark() {
    log "Running $BENCHMARK_NAME..."

    local result
    result=$(runBenchmarkIteration)
    if [ $? -ne 0 ]; then
        exit 1
    fi
    RESULTS[$((BENCHMARK_ID * iterations + ITERATION))]=$result
    log "Finished in ${result}ms"
}

computeAverage() {
    local benchmarkID benchmark sum avg
    benchmarkID=$1
    benchmark=${BENCHMARKS[$benchmarkID]}
    sum=0
    for iteration in $(seq 0 1 $((iterations - 1))); do
        local result
        result=${RESULTS[$((benchmarkID * iterations + iteration))]}
        sum=$((sum + result))
    done
    bc <<< "$sum.0 / $iterations.0"
}

printAverage() {
    local benchmarkID benchmark name avg
    benchmarkID=$1
    avg=$2
    benchmark=${BENCHMARKS[$benchmarkID]}
    name=$(basename "$benchmark")
    echo "$name: ${avg}ms"
}

printBenchmark() {
    printAverage "$@" "$(computeAverage "$@")"
}

printResults() {
    local buildDirectory=$1

    echo "Results for $buildDirectory ($iterations iterations)"
    echo "================================================================================"

    if [ "$runReferences" == true ]; then
        echo ""
        echo "References:"
        echo "----------------------------------------"
        for benchmarkID in "${REFERENCES[@]}"; do
            printBenchmark "$benchmarkID"
        done
    fi

    local score count
    score=0
    count=0
    
    printCategory() {
        local category=$1

        echo ""
        echo "$category:"
        echo "----------------------------------------"

        for benchmarkID in $(eval 'echo ${'"$category"'[@]}'); do
            avg=$(computeAverage "$benchmarkID")
            printAverage "$benchmarkID" "$avg"
            count=$((count + 1))
            score=$(echo "$score" "$avg" | awk '{print ($1 + log($2)) }')
        done
    }
    printCategory "CURRENT_API"
    printCategory "UPCOMING_API"
    score=$(echo "$score" "$count" | awk '{ print(5000.0 / exp($1 / $2)) }')

    echo ""
    echo "Score: $score"
}

run() {
    local buildDirectory
    buildDirectory=$1

    export DYLD_FRAMEWORK_PATH=$buildDirectory
    for iteration in $(seq 0 1 $((iterations-1))); do
        ITERATION=$iteration
        forEachBenchmark "runBenchmark"
    done

    printResults "$buildDirectory"
}


main() {
    collectFlags "$@"
    collectBenchmarks
    for buildDirectory in "${BUILD_DIRECTORIES[@]}"; do
        if [ "$shouldBuild" == true ]; then
            build "$buildDirectory"
        fi

        run "$buildDirectory"
    done
}

main "$@"