Skip to content
This repository was archived by the owner on Jul 6, 2021. It is now read-only.

Commit 5d862d1

Browse files
anatolystansleranatolystansler
committed
Merge branch 'improve-dockerfile' into 'master'
fix: dockerfile, pghrep build See merge request postgres-ai/postgres-checkup!446
2 parents 1c722a6 + c5886dd commit 5d862d1

File tree

4 files changed

+112
-144
lines changed

4 files changed

+112
-144
lines changed

‎.gitlab-ci.yml

+73-71
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,75 @@ workflow:
1111
- when: always
1212

1313
variables:
14-
POSTGRES_DB: dbname # set database
15-
POSTGRES_USER: test_user # set username
14+
POSTGRES_DB: dbname # Set database.
15+
POSTGRES_USER: test_user # Set username.
1616
POSTGRES_PASSWORD: ""
1717
POSTGRES_HOST_AUTH_METHOD: "trust"
1818

19+
test-general:
20+
stage: test
21+
before_script:
22+
- apt-get update
23+
- apt-get install -y jq curl wget git s3cmd sudo golang-1.9-go git
24+
- echo "export PATH=\$PATH:/usr/lib/go-1.9/bin" >> ~/.profile
25+
- source ~/.profile
26+
- cd ./pghrep && make install main && cd ..
27+
- sudo .ci/prepare_cluster.sh "11"
28+
- ps ax | grep postgres
29+
- psql -U postgres -p 5432 -c 'show data_directory;'
30+
- psql -U postgres -p 5433 -c 'show data_directory;'
31+
- psql -U postgres -p 5434 -c 'show data_directory;'
32+
# Configure ssh keys
33+
- mkdir -p ~/.ssh && echo "${PRIVATE_KEY}" > ~/.ssh/id_rsa && echo "${PUBLIC_KEY}" > ~/.ssh/authorized_keys && echo "${PUBLIC_KEY}" > ~/.ssh/id_rsa.pub && chmod 644 ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub && chmod 600 ~/.ssh/id_rsa
34+
- echo "${PRIVATE_KEY}" > ~/.ssh/gitlab_key && chmod 600 ~/.ssh/gitlab_key
35+
- echo "Host gitlab.com" > ~/.ssh/config && echo " Preferredauthentications publickey" >> ~/.ssh/config && echo " IdentityFile ~/.ssh/gitlab_key" >> ~/.ssh/config
36+
- apt-get -y install openssh-server
37+
- service ssh restart
38+
39+
script:
40+
- bash -version
41+
- psql -d dbname -U test_user -c "SELECT version();"
42+
- psql -p 5433 -d dbname -U test_user -c "SELECT version();"
43+
- psql -p 5434 -d dbname -U test_user -c "SELECT version();"
44+
- psql -d dbname -U test_user -f .ci/test_db_dump.sql
45+
- .ci/prepare_test_db.sh
46+
- vacuumdb -U test_user dbname --analyze
47+
- export ARTIFACTS_PATH=$(pwd)/artifacts && echo $ARTIFACTS_PATH
48+
# First run only for K00X reports
49+
- ./checkup collect -h postgres.test1.node --pg-port 5433 --username test_user --project test --dbname dbname -e 1 --file resources/checks/K000_query_analysis.sh > >(tee -a std.log) 2> >(tee -a err.log >&2)
50+
- ./checkup collect -h postgres.test2.node --pg-port 5434 --username test_user --project test --dbname dbname -e 1 --file resources/checks/K000_query_analysis.sh > >(tee -a std.log) 2> >(tee -a err.log >&2)
51+
- ./checkup collect -h postgres.test3.node --username test_user --project test --dbname dbname -e 1 --file resources/checks/K000_query_analysis.sh > >(tee -a std.log) 2> >(tee -a err.log >&2)
52+
# Last run of checkup
53+
- ./checkup collect -h postgres.test1.node --pg-port 5433 --username test_user --project test --dbname dbname -e 1 > >(tee -a std.log) 2> >(tee -a err.log >&2)
54+
- ./checkup collect -h postgres.test2.node --pg-port 5434 --username test_user --project test --dbname dbname -e 1 > >(tee -a std.log) 2> >(tee -a err.log >&2)
55+
- ./checkup collect -h postgres.test3.node --username test_user --project test --dbname dbname -e 1 > >(tee -a std.log) 2> >(tee -a err.log >&2)
56+
- ./checkup process --project test
57+
# Check results
58+
- cat err.log
59+
- grep -v "To compare results, 2 runs are needed. Please run './checkup collect' once again for this epoch." err.log > err1.log || true
60+
- grep -v "this is not a real error. Just run check again." err1.log > err.log || true
61+
- grep -v "Failed to create bus connection" err.log > err1.log || true
62+
- cat err1.log
63+
- export TEST_RESULT=$(stat --format="%s" err1.log) && echo $TEST_RESULT
64+
- ([[ $TEST_RESULT -ne 0 ]]) && exit 1
65+
- (git config --global user.name "postgres-ai" && git config --global user.email "[email protected]" && git config --global push.default simple) || true
66+
- eval $(ssh-agent -s) || true
67+
- ssh-add ~/.ssh/gitlab_key || true
68+
- ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts || true
69+
- ssh -T [email protected] || true
70+
- (cd ~/ && git clone [email protected]:postgres-ai-team/postgres-checkup-tests.git) || true
71+
- (mkdir -p ~/postgres-checkup-tests/$CI_COMMIT_REF_NAME && [[ "$CI_COMMIT_REF_NAME" != "master" ]] && rm -Rf ~/postgres-checkup-tests/$CI_COMMIT_REF_NAME/*) || true
72+
- (cp -Rf $ARTIFACTS_PATH/test/* ~/postgres-checkup-tests/$CI_COMMIT_REF_NAME/) || true
73+
- (cd ~/postgres-checkup-tests && git add ./$CI_COMMIT_REF_NAME/* && git commit -m "$CI_COMMIT_REF_NAME artifacts" && git push) || true
74+
- exit $TEST_RESULT
75+
76+
artifacts:
77+
paths:
78+
- ./artifacts/test
79+
- ./std.log
80+
- ./err.log
81+
expire_in: 1 week
82+
1983
.prepare:
2084
stage: test
2185
before_script:
@@ -29,6 +93,7 @@ variables:
2993
- psql --version
3094
- echo "export PATH=\$PATH:/usr/lib/go-1.9/bin" >> ~/.profile
3195
- source ~/.profile
96+
- cd ./pghrep && make install main && cd ..
3297

3398
.test-check:
3499
stage: test
@@ -86,69 +151,6 @@ variables:
86151
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 213
87152
- echo "Check small indexes in large db mode passed"
88153

89-
test-general:
90-
stage: test
91-
before_script:
92-
- apt-get update
93-
- apt-get install -y jq curl wget git s3cmd sudo golang-1.9-go git
94-
- echo "export PATH=\$PATH:/usr/lib/go-1.9/bin" >> ~/.profile
95-
- source ~/.profile
96-
- sudo .ci/prepare_cluster.sh "11"
97-
- ps ax | grep postgres
98-
- psql -U postgres -p 5432 -c 'show data_directory;'
99-
- psql -U postgres -p 5433 -c 'show data_directory;'
100-
- psql -U postgres -p 5434 -c 'show data_directory;'
101-
# Configure ssh keys
102-
- mkdir -p ~/.ssh && echo "${PRIVATE_KEY}" > ~/.ssh/id_rsa && echo "${PUBLIC_KEY}" > ~/.ssh/authorized_keys && echo "${PUBLIC_KEY}" > ~/.ssh/id_rsa.pub && chmod 644 ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub && chmod 600 ~/.ssh/id_rsa
103-
- echo "${PRIVATE_KEY}" > ~/.ssh/gitlab_key && chmod 600 ~/.ssh/gitlab_key
104-
- echo "Host gitlab.com" > ~/.ssh/config && echo " Preferredauthentications publickey" >> ~/.ssh/config && echo " IdentityFile ~/.ssh/gitlab_key" >> ~/.ssh/config
105-
- apt-get -y install openssh-server
106-
- service ssh restart
107-
108-
script:
109-
- bash -version
110-
- psql -d dbname -U test_user -c "SELECT version();"
111-
- psql -p 5433 -d dbname -U test_user -c "SELECT version();"
112-
- psql -p 5434 -d dbname -U test_user -c "SELECT version();"
113-
- psql -d dbname -U test_user -f .ci/test_db_dump.sql
114-
- .ci/prepare_test_db.sh
115-
- vacuumdb -U test_user dbname --analyze
116-
- export ARTIFACTS_PATH=$(pwd)/artifacts && echo $ARTIFACTS_PATH
117-
# First run only for K00X reports
118-
- ./checkup collect -h postgres.test1.node --pg-port 5433 --username test_user --project test --dbname dbname -e 1 --file resources/checks/K000_query_analysis.sh > >(tee -a std.log) 2> >(tee -a err.log >&2)
119-
- ./checkup collect -h postgres.test2.node --pg-port 5434 --username test_user --project test --dbname dbname -e 1 --file resources/checks/K000_query_analysis.sh > >(tee -a std.log) 2> >(tee -a err.log >&2)
120-
- ./checkup collect -h postgres.test3.node --username test_user --project test --dbname dbname -e 1 --file resources/checks/K000_query_analysis.sh > >(tee -a std.log) 2> >(tee -a err.log >&2)
121-
# Last run of checkup
122-
- ./checkup collect -h postgres.test1.node --pg-port 5433 --username test_user --project test --dbname dbname -e 1 > >(tee -a std.log) 2> >(tee -a err.log >&2)
123-
- ./checkup collect -h postgres.test2.node --pg-port 5434 --username test_user --project test --dbname dbname -e 1 > >(tee -a std.log) 2> >(tee -a err.log >&2)
124-
- ./checkup collect -h postgres.test3.node --username test_user --project test --dbname dbname -e 1 > >(tee -a std.log) 2> >(tee -a err.log >&2)
125-
- ./checkup process --project test
126-
# Check results
127-
- cat err.log
128-
- grep -v "To compare results, 2 runs are needed. Please run './checkup collect' once again for this epoch." err.log > err1.log || true
129-
- grep -v "this is not a real error. Just run check again." err1.log > err.log || true
130-
- grep -v "Failed to create bus connection" err.log > err1.log || true
131-
- cat err1.log
132-
- export TEST_RESULT=$(stat --format="%s" err1.log) && echo $TEST_RESULT
133-
- ([[ $TEST_RESULT -ne 0 ]]) && exit 1
134-
- (git config --global user.name "postgres-ai" && git config --global user.email "[email protected]" && git config --global push.default simple) || true
135-
- eval $(ssh-agent -s) || true
136-
- ssh-add ~/.ssh/gitlab_key || true
137-
- ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts || true
138-
- ssh -T [email protected] || true
139-
- (cd ~/ && git clone [email protected]:postgres-ai-team/postgres-checkup-tests.git) || true
140-
- (mkdir -p ~/postgres-checkup-tests/$CI_COMMIT_REF_NAME && [[ "$CI_COMMIT_REF_NAME" != "master" ]] && rm -Rf ~/postgres-checkup-tests/$CI_COMMIT_REF_NAME/*) || true
141-
- (cp -Rf $ARTIFACTS_PATH/test/* ~/postgres-checkup-tests/$CI_COMMIT_REF_NAME/) || true
142-
- (cd ~/postgres-checkup-tests && git add ./$CI_COMMIT_REF_NAME/* && git commit -m "$CI_COMMIT_REF_NAME artifacts" && git push) || true
143-
- exit $TEST_RESULT
144-
145-
artifacts:
146-
paths:
147-
- ./artifacts/test
148-
- ./std.log
149-
- ./err.log
150-
expire_in: 1 week
151-
152154
test-pghrep:
153155
stage: test
154156
extends: ".prepare"
@@ -162,7 +164,7 @@ test-check-9.6:
162164
- postgres:9.6
163165

164166
test-check-10:
165-
extends: ".test-check"
167+
extends: ".test-check"
166168
stage: "test"
167169
services:
168170
- postgres:10
@@ -226,9 +228,9 @@ test-check-cli:
226228

227229
.only_var_template: &only_feature
228230
rules:
229-
- if: $CI_COMMIT_REF_SLUG != "master" #$CI_COMMIT_REF_SLUG &&
231+
- if: $CI_COMMIT_REF_SLUG && $CI_COMMIT_REF_SLUG != "master"
230232

231-
build-image-feature-server:
233+
build-image-feature:
232234
<<: *build_image_definition
233235
<<: *only_feature
234236
variables:
@@ -238,14 +240,14 @@ build-image-feature-server:
238240
DOCKER_NAME: "registry.gitlab.com/postgres-ai/postgres-checkup"
239241
TAGS: "${DOCKER_NAME}:${CI_COMMIT_REF_SLUG}"
240242

241-
build-image-master-server:
243+
build-image-master:
242244
<<: *build_image_definition
243245
<<: *only_master
244246
variables:
245247
DOCKER_NAME: "registry.gitlab.com/postgres-ai/postgres-checkup"
246248
TAGS: "${DOCKER_NAME}:master,${DOCKER_NAME}:master-${CI_COMMIT_SHORT_SHA}"
247249

248-
build-image-latest-server:
250+
build-image-latest:
249251
<<: *build_image_definition
250252
<<: *only_tag_release
251253
variables:
@@ -255,7 +257,7 @@ build-image-latest-server:
255257
DOCKER_NAME: "postgresai/postgres-checkup"
256258
TAGS: "${DOCKER_NAME}:latest,${DOCKER_NAME}:${CI_COMMIT_TAG}"
257259

258-
build-image-rc-server:
260+
build-image-rc:
259261
<<: *build_image_definition
260262
<<: *only_tag_rc
261263
variables:

‎Dockerfile

+14-14
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
FROM alpine:3.9
1+
FROM golang:1.14-alpine as build
2+
COPY ./pghrep /go/pghrep
3+
RUN apk add --update --no-cache make git
4+
RUN cd /go/pghrep && make install main
25

6+
FROM alpine:3.11 as production
37
RUN apk add --update --no-cache \
4-
bash \
5-
openssh-client \
6-
postgresql-client \
7-
coreutils \
8-
jq \
9-
curl \
10-
go \
11-
gawk \
12-
sed \
13-
make \
14-
build-base \
15-
git
16-
8+
bash \
9+
openssh-client \
10+
postgresql-client \
11+
jq \
12+
curl \
13+
gawk \
14+
sed
15+
WORKDIR ./checkup
16+
COPY --from=build /go/pghrep/bin/pghrep ./pghrep/bin/
1717
COPY . .

‎README.md

+19-11
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,14 @@ git clone https://gitlab.com/postgres-ai/postgres-checkup.git
148148
cd postgres-checkup
149149
```
150150

151+
#### 3. Build pghrep
152+
153+
```bash
154+
cd ./pghrep
155+
make install main
156+
cd ..
157+
```
158+
151159
## Example of Use
152160

153161
Let's make a report for a project named `prod1`. Assume that we have two servers,
@@ -257,16 +265,16 @@ There is an option to run postgres-checkup in a Docker container:
257265
```bash
258266
docker run --rm \
259267
--name postgres-checkup \
260-
-e PGPASSWORD="postgres" \
261-
-v `pwd`/artifacts:/artifacts \
262-
registry.gitlab.com/postgres-ai/postgres-checkup:latest \
268+
--env PGPASSWORD="postgres" \
269+
--volume `pwd`/artifacts:/artifacts \
270+
postgresai/postgres-checkup:latest \
263271
./checkup \
264-
-h hostname \
265-
-p 5432 \
272+
--hostname hostname \
273+
--port 5432 \
266274
--username postgres \
267275
--dbname postgres \
268276
--project c \
269-
-e "$(date +'%Y%m%d')001"
277+
--epoch "$(date +'%Y%m%d')001"
270278
```
271279

272280
In this case some checks (those requiring SSH connection) will be skipped.
@@ -282,15 +290,15 @@ on Windows, but should work well on Linux and MacOS machines):
282290
```bash
283291
docker run --rm \
284292
--name postgres-checkup \
285-
-v "$(pwd)/artifacts:/artifacts" \
286-
-v "$(echo ~)/.ssh/id_rsa:/root/.ssh/id_rsa:ro" \
287-
registry.gitlab.com/postgres-ai/postgres-checkup:latest \
293+
--volume "$(pwd)/artifacts:/artifacts" \
294+
--volume "$(echo ~)/.ssh/id_rsa:/root/.ssh/id_rsa:ro" \
295+
postgresai/postgres-checkup:latest \
288296
./checkup \
289-
-h sshusername@hostname \
297+
--hostname sshusername@hostname \
290298
--username my_postgres_user \
291299
--dbname my_postgres_database \
292300
--project docker_test_with_ssh \
293-
-e "$(date +'%Y%m%d')001"
301+
--epoch "$(date +'%Y%m%d')001"
294302
```
295303

296304
If you try to check the local instance of postgres on your host from a container,

‎checkup

+6-48
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ FULL_REPORT_FNAME="0_Full_report.md"
1414
: ${DEBUG:=false} # print debug output
1515
SCRIPT_NAME=$(basename $0)
1616
SCRIPT_DIR=$(dirname $0)
17+
PGHREP_BIN="${SCRIPT_DIR}/pghrep/bin/pghrep"
1718
SAFE_IFS="$IFS"
1819
ALL_ARGS="$@"
1920
OPTIONS_ERROR_EXIT="false"
@@ -793,25 +794,6 @@ is_large_database() {
793794
fi
794795
}
795796

796-
#######################################
797-
# Build pghrep - a .md report generator
798-
# Globals:
799-
# PGHREP_BIN
800-
# Arguments:
801-
# None
802-
# Returns:
803-
# (text) stdout/stderr
804-
#######################################
805-
build_pghrep() {
806-
dbg "building pghrep..."
807-
local save_pwd="${PWD}"
808-
cd "${SCRIPT_DIR}/pghrep"
809-
make install main >/dev/null
810-
cd "${save_pwd}"
811-
export PGHREP_BIN="${SCRIPT_DIR}/pghrep/bin/pghrep"
812-
dbg "done building pghrep"
813-
}
814-
815797
#######################################
816798
# Check binary dependencies
817799
# Globals:
@@ -868,34 +850,6 @@ check_bin_deps() {
868850
return 1
869851
fi
870852

871-
# golang
872-
if ! $(which go >/dev/null 2>&1); then
873-
err "Can't find the 'go' executable. Please install it:"
874-
if [[ "${KERNEL_NAME}" = "Darwin" ]]; then
875-
err "${OS_NAME}: 'brew install golang'"
876-
elif [[ "${KERNEL_NAME}" = "Linux" ]]; then
877-
err "Debian/Ubuntu GNU/${OS_NAME}: 'sudo apt-get install golang'"
878-
err "RHEL/CentOS GNU/${OS_NAME}: 'sudo yum install golang'"
879-
fi
880-
return 1
881-
fi
882-
# go version check (>= 1.8)
883-
re="go([0-9]+.[0-9]+).+"
884-
local go_version_full=$(go version)
885-
local version="" # short form (e.g. '1.11')
886-
local go_version_num="" # get only first two numbers devided by '.'
887-
if [[ "${go_version_full}" =~ $re ]]; then
888-
version="${BASH_REMATCH[1]}"
889-
local major=${version%.*}
890-
local minor=${version#*.}
891-
go_version_num="${major}${minor}"
892-
if [[ $(( go_version_num % 1000 )) -lt "18" ]]; then
893-
err "Unsupported golang version '${go_version_full}'"
894-
err "Please install golang version >= '1.8'"
895-
return 1
896-
fi
897-
fi
898-
899853
# jq
900854
if ! $(which jq >/dev/null 2>&1); then
901855
err "Can't find the 'jq' executable. Please install it:"
@@ -1496,7 +1450,11 @@ main() {
14961450
mode="$1"
14971451
fi
14981452

1499-
build_pghrep
1453+
if [[ ! -f "${PGHREP_BIN}" ]]; then
1454+
err "Cannot load '${PGHREP_BIN}'. The binary needs to be built."
1455+
exit 2
1456+
fi
1457+
15001458
read_possible_args
15011459
process_cli_args "$@"
15021460
validate_args

0 commit comments

Comments
 (0)