diff --git a/.gitignore b/.gitignore index 076d7a3..dacce08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .mypy_cache/ __pycache__/ test.csv +failed.txt Vagrantfile .vagrant *.log diff --git a/Dockerfile b/Dockerfile index 97715a0..7566bd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM python:3.10 WORKDIR /app COPY requirements.txt /app/ RUN pip install -r requirements.txt diff --git a/gen_report.py b/gen_report.py index 69b6d19..5ab71e8 100755 --- a/gen_report.py +++ b/gen_report.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -from glob import glob import json +from glob import glob + import bitmath @@ -8,7 +9,11 @@ def read_requirements(): reqs = {} with open('runs/requirements.txt') as f: for line in f.readlines(): - pkg, version = line.strip().split('==') + if ' @ ' in line: + pkg = line.strip().split(' @ ')[0] + version = line.strip().split('@')[-1][:7] + else: + pkg, version = line.strip().split('==') reqs[pkg.lower()] = (pkg, version) return reqs diff --git a/pgbouncer-userlist.txt b/pgbouncer-userlist.txt index b8b19c5..9396827 100644 --- a/pgbouncer-userlist.txt +++ b/pgbouncer-userlist.txt @@ -1,2 +1,2 @@ -"test" "md505a671c66aefea124cc08b76ea6d30bb" -"postgres" "md53175bce1d3201d16594cebf9d7eb3f9d" +"test" "test" +"postgres" "postgres" diff --git a/requirements.txt b/requirements.txt index d05be97..ac9f651 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ aiopg psycopg2-binary gevent psycogreen -meinheld +git+https://github.com/idot/meinheld.git@2bfe452#egg=meinheld falcon sanic bottle diff --git a/run-docker.sh b/run-docker.sh index e24a63e..b5abfa7 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -4,8 +4,9 @@ [[ -z "$NUM_CONNECTIONS" ]] && NUM_CONNECTIONS=50000 [[ -z "$NUM_WORKERS_SYNC" ]] && NUM_WORKERS_SYNC=19 [[ -z "$NUM_WORKERS_ASYNC" ]] && NUM_WORKERS_ASYNC=6 -[[ -z "$NUM_DB_SESSIONS" ]] && NUM_DB_SESSIONS=100 +[[ -z "$NUM_DB_SESSIONS" ]] && export NUM_DB_SESSIONS=100 [[ -z "$DB_SLEEP" ]] && DB_SLEEP=0.02 +[[ -z "$WARMUP_SECONDS" ]] && WARMUP_SECONDS=10 ALL_TESTS=$(cat tests.txt | shuf) @@ -18,22 +19,57 @@ if [[ "$(docker-compose ps -q)" != "" ]]; then ALREADY_UP=1 fi +# Uncomment to aid debugging: +# set -x + if [[ "$ALREADY_UP" == "" ]]; then if [[ ! -f data.csv ]]; then - ./gen_test_data.py + ./gen_test_data.py > data.csv fi NUM_DB_SESSIONS=$NUM_DB_SESSIONS docker-compose up -d sleep 2 - docker-compose run --rm -e PGPASSWORD=test dbpool psql -h perf-dbpool -U test < schema.sql + docker-compose run --rm -T -e PGPASSWORD=test dbpool psql -h perf-dbpool -U test < schema.sql docker-compose run --rm -e PGPASSWORD=test dbpool psql -h perf-dbpool -U test -c "COPY test FROM '/tmp/data.csv' DELIMITER ',' CSV HEADER;" fi docker build -t perf-app . mkdir -p runs rm -f runs/* +rm -f failed.txt docker run --rm perf-app pip freeze > runs/requirements.txt +function finalize_test() { + set +e + kill $MONITOR_PID + docker rm -f perf-app + set -e + DB_CONN=$(docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -P pager=off -c "show servers;" pgbouncer | wc -l) + DB_CONN=$((DB_CONN-4)) + docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "kill test;" pgbouncer + docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "resume test;" pgbouncer + echo $DB_CONN > runs/$test-x$PWPWORKERS.db +} + +function finalize_end() { + if [[ "$ALREADY_UP" == "" ]]; then + docker-compose down -v + fi + + ./gen_report.py > test.csv + cat test.csv +} + +function interrupt_test() { + finalize_test + finalize_end +} + +trap interrupt_test INT + +LEN=$(c() { echo $#; }; c $ALL_TESTS) +COUNTER=0 for test in $ALL_TESTS; do + COUNTER=$((COUNTER+1)) if [[ -z "$NUM_WORKERS" ]]; then PWPWORKERS=$NUM_WORKERS_ASYNC if [[ "$test" == *gunicorn* || "$test" == *uwsgi* ]]; then @@ -46,23 +82,37 @@ for test in $ALL_TESTS; do else PWPWORKERS=$NUM_WORKERS fi - echo Running $test x$PWPWORKERS + echo Running $COUNTER/$LEN $test x$PWPWORKERS docker run --rm -d --name perf-app --network container:perf-server -e PWPWORKERS=$PWPWORKERS -e DB_SLEEP=$DB_SLEEP perf-app ./serve-$test.sh - sleep 2 - ./monitor-app.sh $test-x$PWPWORKERS & - MONITOR_PID=$! - $(docker run --rm --name perf-test --network container:perf-server jordi/ab -c$NUM_CLIENTS -n$NUM_CONNECTIONS http://localhost:8000/test | python ab2json.py > runs/$test-x$PWPWORKERS.json) - kill $MONITOR_PID - docker rm -f perf-app - DB_CONN=$(docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -P pager=off -c "show servers;" pgbouncer | wc -l) - DB_CONN=$((DB_CONN-4)) - docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "kill test;" pgbouncer - docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "resume test;" pgbouncer - echo $DB_CONN > runs/$test-x$PWPWORKERS.db + SKIP_TEST= + WARMUP=0 + echo # Print a blank line + while true; do + HTTP_STATUS=$(curl --silent -o /dev/null -w "%{http_code}" http://localhost:8000/test) + CURL_STATUS=$? + echo -n -e "\r\033[1A\033[0K" # Clear previous line + echo -n "Warming up (curl exit code $CURL_STATUS, HTTP status $HTTP_STATUS)" + WARMUP=$((WARMUP+1)) + printf '%*s\n' $WARMUP | tr ' ' '.' + if [[ $CURL_STATUS == 0 ]]; then + if [[ $HTTP_STATUS == "200" ]]; then + break + fi + fi + if [[ $WARMUP -gt $WARMUP_SECONDS ]]; then + echo "App failed to start, skipping test" + SKIP_TEST=1 + echo $test >> failed.txt + break + fi + sleep 1 + done + if [[ "$SKIP_TEST" == "" ]]; then + ./monitor-app.sh $test-x$PWPWORKERS & + MONITOR_PID=$! + $(docker run --rm --name perf-test --network container:perf-server --platform linux/amd64 jordi/ab -c$NUM_CLIENTS -n$NUM_CONNECTIONS http://localhost:8000/test | python3 ab2json.py > runs/$test-x$PWPWORKERS.json) + fi + finalize_test done -if [[ "$ALREADY_UP" == "" ]]; then - docker-compose down -v -fi - -./gen_report.py +finalize_end