From 1ad8a045ca2a512cf396f25decf7916af12bb815 Mon Sep 17 00:00:00 2001 From: Kiran Jonnalagadda Date: Tue, 6 Sep 2022 02:38:49 +0530 Subject: [PATCH 1/3] Updates for 2022 * Postgres no longer uses MD5 auth * Meinheld has been abandoned and must be installed from an open PR * uvicorn-aioflask is also failing and must be skipped * run-docker.sh now has graceful shutdown on interrupt, and recovery from test failure --- .gitignore | 1 + gen_report.py | 9 ++++- pgbouncer-userlist.txt | 4 +- requirements.txt | 2 +- run-docker.sh | 90 ++++++++++++++++++++++++++++++++---------- 5 files changed, 81 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 076d7a3..dacce08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .mypy_cache/ __pycache__/ test.csv +failed.txt Vagrantfile .vagrant *.log diff --git a/gen_report.py b/gen_report.py index 69b6d19..36c1cdf 100755 --- a/gen_report.py +++ b/gen_report.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -from glob import glob import json +from glob import glob + import bitmath @@ -8,7 +9,11 @@ def read_requirements(): reqs = {} with open('runs/requirements.txt') as f: for line in f.readlines(): - pkg, version = line.strip().split('==') + if ' @ ' in line: + pkg = line.strip().split(' @ ')[0] + version = line.strip().split('@')[-1] + else: + pkg, version = line.strip().split('==') reqs[pkg.lower()] = (pkg, version) return reqs diff --git a/pgbouncer-userlist.txt b/pgbouncer-userlist.txt index b8b19c5..9396827 100644 --- a/pgbouncer-userlist.txt +++ b/pgbouncer-userlist.txt @@ -1,2 +1,2 @@ -"test" "md505a671c66aefea124cc08b76ea6d30bb" -"postgres" "md53175bce1d3201d16594cebf9d7eb3f9d" +"test" "test" +"postgres" "postgres" diff --git a/requirements.txt b/requirements.txt index d05be97..ac9f651 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ aiopg psycopg2-binary gevent psycogreen -meinheld +git+https://github.com/idot/meinheld.git@2bfe452#egg=meinheld falcon sanic bottle diff --git a/run-docker.sh b/run-docker.sh index e24a63e..6af03a5 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -4,8 +4,9 @@ [[ -z "$NUM_CONNECTIONS" ]] && NUM_CONNECTIONS=50000 [[ -z "$NUM_WORKERS_SYNC" ]] && NUM_WORKERS_SYNC=19 [[ -z "$NUM_WORKERS_ASYNC" ]] && NUM_WORKERS_ASYNC=6 -[[ -z "$NUM_DB_SESSIONS" ]] && NUM_DB_SESSIONS=100 +[[ -z "$NUM_DB_SESSIONS" ]] && export NUM_DB_SESSIONS=100 [[ -z "$DB_SLEEP" ]] && DB_SLEEP=0.02 +[[ -z "$WARMUP_SECONDS" ]] && WARMUP_SECONDS=10 ALL_TESTS=$(cat tests.txt | shuf) @@ -18,22 +19,57 @@ if [[ "$(docker-compose ps -q)" != "" ]]; then ALREADY_UP=1 fi +# Uncomment to aid debugging: +# set -x + if [[ "$ALREADY_UP" == "" ]]; then if [[ ! -f data.csv ]]; then - ./gen_test_data.py + ./gen_test_data.py > data.csv fi NUM_DB_SESSIONS=$NUM_DB_SESSIONS docker-compose up -d sleep 2 - docker-compose run --rm -e PGPASSWORD=test dbpool psql -h perf-dbpool -U test < schema.sql + docker-compose run --rm -T -e PGPASSWORD=test dbpool psql -h perf-dbpool -U test < schema.sql docker-compose run --rm -e PGPASSWORD=test dbpool psql -h perf-dbpool -U test -c "COPY test FROM '/tmp/data.csv' DELIMITER ',' CSV HEADER;" fi docker build -t perf-app . mkdir -p runs rm -f runs/* +rm -f failed.txt docker run --rm perf-app pip freeze > runs/requirements.txt +function finalize_test() { + set +e + kill $MONITOR_PID + docker rm -f perf-app + set -e + DB_CONN=$(docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -P pager=off -c "show servers;" pgbouncer | wc -l) + DB_CONN=$((DB_CONN-4)) + docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "kill test;" pgbouncer + docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "resume test;" pgbouncer + echo $DB_CONN > runs/$test-x$PWPWORKERS.db +} + +function finalize_end() { + if [[ "$ALREADY_UP" == "" ]]; then + docker-compose down -v + fi + + ./gen_report.py > test.csv + cat test.csv +} + +function interrupt_test() { + finalize_test + finalize_end +} + +trap interrupt_test INT + +LEN=$(c() { echo $#; }; c $ALL_TESTS) +COUNTER=0 for test in $ALL_TESTS; do + COUNTER=$((COUNTER+1)) if [[ -z "$NUM_WORKERS" ]]; then PWPWORKERS=$NUM_WORKERS_ASYNC if [[ "$test" == *gunicorn* || "$test" == *uwsgi* ]]; then @@ -46,23 +82,37 @@ for test in $ALL_TESTS; do else PWPWORKERS=$NUM_WORKERS fi - echo Running $test x$PWPWORKERS + echo Running $COUNTER/$LEN $test x$PWPWORKERS docker run --rm -d --name perf-app --network container:perf-server -e PWPWORKERS=$PWPWORKERS -e DB_SLEEP=$DB_SLEEP perf-app ./serve-$test.sh - sleep 2 - ./monitor-app.sh $test-x$PWPWORKERS & - MONITOR_PID=$! - $(docker run --rm --name perf-test --network container:perf-server jordi/ab -c$NUM_CLIENTS -n$NUM_CONNECTIONS http://localhost:8000/test | python ab2json.py > runs/$test-x$PWPWORKERS.json) - kill $MONITOR_PID - docker rm -f perf-app - DB_CONN=$(docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -P pager=off -c "show servers;" pgbouncer | wc -l) - DB_CONN=$((DB_CONN-4)) - docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "kill test;" pgbouncer - docker-compose run --rm -e PGPASSWORD=postgres db psql -h perf-dbpool -U postgres -c "resume test;" pgbouncer - echo $DB_CONN > runs/$test-x$PWPWORKERS.db + SKIP_TEST= + WARMUP=0 + echo # Print a blank line + while true; do + HTTP_STATUS=$(curl --silent -o /dev/null -w "%{http_code}" http://localhost:8000/test) + CURL_STATUS=$? + if [[ $CURL_STATUS == 0 ]]; then + if [[ $HTTP_STATUS == "200" ]]; then + break + fi + fi + echo -n -e "\r\033[1A\033[0K" # Clear previous line + echo -n "Warming up (curl exit code $CURL_STATUS, HTTP code $HTTP_STATUS)" + WARMUP=$((WARMUP+1)) + printf '%*s\n' $WARMUP | tr ' ' '.' + sleep 1 + if [[ $WARMUP -ge $WARMUP_SECONDS ]]; then + echo "App failed to start, skipping test" + SKIP_TEST=1 + echo $test >> failed.txt + break + fi + done + if [[ "$SKIP_TEST" == "" ]]; then + ./monitor-app.sh $test-x$PWPWORKERS & + MONITOR_PID=$! + $(docker run --rm --name perf-test --network container:perf-server --platform linux/amd64 jordi/ab -c$NUM_CLIENTS -n$NUM_CONNECTIONS http://localhost:8000/test | python3 ab2json.py > runs/$test-x$PWPWORKERS.json) + fi + finalize_test done -if [[ "$ALREADY_UP" == "" ]]; then - docker-compose down -v -fi - -./gen_report.py +finalize_end From f90cf3a1e18d3999930786446276f7bec2bcc23a Mon Sep 17 00:00:00 2001 From: Kiran Jonnalagadda Date: Tue, 6 Sep 2022 02:53:59 +0530 Subject: [PATCH 2/3] Report curl and HTTP status after warmup Also report git commit by id instead of full hash --- gen_report.py | 2 +- run-docker.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/gen_report.py b/gen_report.py index 36c1cdf..5ab71e8 100755 --- a/gen_report.py +++ b/gen_report.py @@ -11,7 +11,7 @@ def read_requirements(): for line in f.readlines(): if ' @ ' in line: pkg = line.strip().split(' @ ')[0] - version = line.strip().split('@')[-1] + version = line.strip().split('@')[-1][:7] else: pkg, version = line.strip().split('==') reqs[pkg.lower()] = (pkg, version) diff --git a/run-docker.sh b/run-docker.sh index 6af03a5..b5abfa7 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -90,22 +90,22 @@ for test in $ALL_TESTS; do while true; do HTTP_STATUS=$(curl --silent -o /dev/null -w "%{http_code}" http://localhost:8000/test) CURL_STATUS=$? + echo -n -e "\r\033[1A\033[0K" # Clear previous line + echo -n "Warming up (curl exit code $CURL_STATUS, HTTP status $HTTP_STATUS)" + WARMUP=$((WARMUP+1)) + printf '%*s\n' $WARMUP | tr ' ' '.' if [[ $CURL_STATUS == 0 ]]; then if [[ $HTTP_STATUS == "200" ]]; then break fi fi - echo -n -e "\r\033[1A\033[0K" # Clear previous line - echo -n "Warming up (curl exit code $CURL_STATUS, HTTP code $HTTP_STATUS)" - WARMUP=$((WARMUP+1)) - printf '%*s\n' $WARMUP | tr ' ' '.' - sleep 1 - if [[ $WARMUP -ge $WARMUP_SECONDS ]]; then + if [[ $WARMUP -gt $WARMUP_SECONDS ]]; then echo "App failed to start, skipping test" SKIP_TEST=1 echo $test >> failed.txt break fi + sleep 1 done if [[ "$SKIP_TEST" == "" ]]; then ./monitor-app.sh $test-x$PWPWORKERS & From 3e705781c4c40efc5d87277375d33ca26e5dadeb Mon Sep 17 00:00:00 2001 From: Kiran Jonnalagadda Date: Tue, 6 Sep 2022 03:07:25 +0530 Subject: [PATCH 3/3] Use Python 3.10 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 97715a0..7566bd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM python:3.10 WORKDIR /app COPY requirements.txt /app/ RUN pip install -r requirements.txt