Skip to content
This repository was archived by the owner on Jul 6, 2021. It is now read-only.

Commit c480173

Browse files
dmiusdmius
committed
H002: build list of redundant indexes fixed, generation json data for indexes fixed
1 parent 87c4cfa commit c480173

File tree

10 files changed

+279
-252
lines changed

10 files changed

+279
-252
lines changed

‎.ci/test_db_dump.sql

+11
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@ create index concurrently i_unused on t_with_unused_index(i);
1919
create table t_with_redundant_index as select i from generate_series(1, 1000000) _(i);
2020
create index concurrently i_redundant_1 on t_with_redundant_index(i);
2121
create index concurrently i_redundant_2 on t_with_redundant_index(i);
22+
23+
-- redundant for functional and uniq keys
24+
create table ctnr as select id, id as pnum, id as m_id, id as type_id, id as b, id as todel from generate_series(1, 1000000) _(id);
25+
alter table ctnr add primary key (id);
26+
alter table ctnr add constraint ctnr_uk01 unique (pnum, m_id);
27+
create index ctnr_idx01 on ctnr using btree(pnum);
28+
create index ctnr_idx02 on ctnr using btree(type_id);
29+
create index ctnr_idx03 on ctnr using btree(b);
30+
create index ctnr_idx04 on ctnr using btree(b) where pnum > 0;
31+
create index ctnr_idx06 on ctnr using btree(todel);
32+
2233
-- redundant for uniq, primary keys
2334
create table t_with_redundant_idx as select id, id as f1, id as f2, id as f3, id as f4 from generate_series(1, 1000000) _(id);
2435
alter table t_with_redundant_idx add primary key (id);

‎.gitlab-ci.yml

+16-14
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,19 @@ variables:
3737
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H003_non_indexed_fks.sh
3838
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 302
3939
- echo "H003 passed"
40-
- echo "Test H002 redundant indexes"
40+
- echo "Test H004 redundant indexes"
4141
- psql -h postgres -d dbname -U test_user -f .ci/test_db_dump.sql
42-
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
43-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_id"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 201
44-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 202
45-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1_uniq"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 203
46-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_pkey"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 204
47-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 205
48-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 206
49-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
50-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
42+
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H004_redundant_indexes.sh
43+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_id"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 201
44+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 202
45+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1_uniq"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 203
46+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_pkey"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 204
47+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 205
48+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 206
49+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
50+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
51+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.ctnr_idx01"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 209
52+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.ctnr_idx04"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 210
5153
- echo "H002 passed"
5254
- ./checkup -c .ci/test.yml --file ./resources/checks/F005_index_bloat.sh
5355
- ./checkup -c .ci/test.yml --file ./resources/checks/L001_table_sizes.sh
@@ -62,16 +64,16 @@ variables:
6264
- .ci/prepare_test_db.sh postgres
6365
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
6466
## unused
65-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 209
67+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 210
6668
## redundant
67-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 210
69+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 211
6870
- echo "Check small indexes in small db mode passed"
6971
- .ci/prepare_large_db.sh postgres
7072
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
7173
## unused
72-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 211
74+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 212
7375
## redundant
74-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 212
76+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 213
7577
- echo "Check small indexes in large db mode passed"
7678

7779
test-general:

‎HELP.md

+7-1
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ The list of broken indexes (invalid state) to be removed or reindexed.
214214

215215
### H002 Unused and Redundant Indexes
216216

217-
Shows the list of never used, rarely used and redundant indexes.
217+
Shows the list of never used and rarely used indexes.
218218
Helps to understand how much space they occupy.
219219

220220
> Insights:
@@ -227,6 +227,12 @@ Helps to understand how much space they occupy.
227227

228228
Checks if all foreign keys have indexes in referencing tables.
229229

230+
### H004 Redundant Indexes
231+
232+
Shows the list of redundant indexes.
233+
Helps to understand how much space they occupy.
234+
235+
230236
# K. SQL Query Analysis
231237

232238
This is important group of reports providing deep SQL query analysis, based on pg_stat_statements and (optional) pg_stat_kcache.

‎README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -364,8 +364,9 @@ Docker support implemented by [Ivan Muratov](https://gitlab.com/binakot).
364364
## H. Index Analysis
365365

366366
- [x] H001 Invalid indexes #192, #51
367-
- [x] H002 Unused and redundant indexes #51, #180, #170, #168, #322
367+
- [x] H002 Unused indexes #51, #180, #170, #168, #322
368368
- [x] H003 Non-indexed foreign keys #52, #142, #173
369+
- [x] H004 Redundant indexes
369370

370371
## J. Capacity Planning
371372

‎checkup

+3-2
Original file line numberDiff line numberDiff line change
@@ -875,14 +875,15 @@ check_bin_deps() {
875875
glue_md_reports() {
876876
# final report path and name
877877
local out_fname="${MD_REPORTS_DIR}/${FULL_REPORT_FNAME}"
878-
local database=$(jq '.last_check.database' ${PROJECT_DIR}/nodes.json)
878+
local epoch=$(jq -r '.last_check.epoch' ${PROJECT_DIR}/nodes.json)
879+
local database=$(jq -r '.last_check.database' ${PROJECT_DIR}/nodes.json)
879880

880881
# do not re-generate full report if '--file' is given
881882
[[ "${FILE}" != "None" ]] && return 0
882883

883884
# make header
884885
echo "# PostgreSQL Checkup. Project: '${PROJECT}'. Database: '${database}'" > "${out_fname}"
885-
echo "## Epoch number: '${EPOCH}'" >> "${out_fname}"
886+
echo "## Epoch number: '${epoch}'" >> "${out_fname}"
886887
echo "NOTICE: while most reports describe the “current database”, some of them may contain cluster-wide information describing all databases in the cluster." >> "${out_fname}"
887888
echo >> "${out_fname}"
888889
echo "Last modified at: " $(date +'%Y-%m-%d %H:%M:%S %z') >> "${out_fname}"

‎pghrep/templates/H002.tpl

+5-44
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
# {{ .checkId }} Unused and Redundant Indexes #
1+
# {{ .checkId }} Unused Indexes #
22
## Observations ##
33
Data collected: {{ DtFormat .timestamptz }}
44
Current database: {{ .database }}
55

6-
{{- if (index .results .reorderedHosts.master)}}
6+
{{ if (index .results .reorderedHosts.master)}}
77
{{- if (index (index .results .reorderedHosts.master) "data") }}
88
Stats reset: {{ (index (index (index .results .reorderedHosts.master) "data") "database_stat").stats_age }} ago ({{ DtFormat (index (index (index .results .reorderedHosts.master) "data") "database_stat").stats_reset }})
9-
{{- if le (Int (index (index (index .results .reorderedHosts.master) "data") "database_stat").days) 30 }}
9+
{{ if le (Int (index (index (index .results .reorderedHosts.master) "data") "database_stat").days) 30 }}
1010
:warning: Statistics age is less than 30 days. Make decisions on index cleanup with caution!
1111
{{- end }}
1212
{{ if gt (Int (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes")) 0 }}NOTICE: only indexes larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes") 0 }} are analyzed. {{end}}
@@ -33,7 +33,7 @@ Nothing found.
3333

3434
### Rarely Used Indexes ###
3535
{{ if (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes") }}
36-
{{ if ge (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) 1 }}.{{ end }}
36+
{{ if ge (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) 1 }}.{{ end }}
3737

3838
|\#| Table | Index | {{.reorderedHosts.master}} usage {{ range $skey, $host := .reorderedHosts.replicas }}| {{ $host }} usage {{ end }}| ▼ Index size | Table size | Comment | Supports FK|
3939
|--|-------|-------|-----{{ range $skey, $host := .reorderedHosts.replicas }}|--------{{ end }}|-----|-----|----|-----|
@@ -53,27 +53,6 @@ Nothing found.
5353
Nothing found.
5454
{{ end }}{{/* rarely used indexes found */}}
5555

56-
### Redundant Indexes ###
57-
{{ if (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes") }}
58-
{{ if ge (len (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes")) 1 }}.{{ end }}
59-
60-
|\#| Table | Index | Redundant to |{{.reorderedHosts.master}} usage {{ range $skey, $host := .reorderedHosts.replicas }}| {{ $host }} usage {{ end }}| ▼ Index size | Table size | Supports FK |
61-
|--|-------|-------|--------------|--{{ range $skey, $host := .reorderedHosts.replicas }}|--------{{ end }}|-----|-----|-----|
62-
{{ if (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes_total") }}| |=====TOTAL=====|||{{ range $skey, $host := .reorderedHosts.replicas }}|{{ end }}|{{ ByteFormat ((index (index (index .results .reorderedHosts.master) "data") "redundant_indexes_total").index_size_bytes_sum) 2 }}|{{ ByteFormat ((index (index (index .results .reorderedHosts.master) "data") "redundant_indexes_total").table_size_bytes_sum) 2 }}||{{ end }}
63-
{{ range $i, $key := (index (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes") "_keys") }}
64-
{{- if lt $i $.LISTLIMIT -}}
65-
{{- $value:=(index (index (index (index $.results $.reorderedHosts.master) "data") "redundant_indexes") $key) -}}
66-
|{{- $value.num}}|`{{ $value.formated_relation_name}}`|`{{- $value.formated_index_name}}`|
67-
{{- $rinexes := Split $value.reason ", " -}}{{ range $r, $rto:= $rinexes }}`{{$rto}}`<br/>{{end}}|{{- RawIntFormat $value.idx_scan }}{{ range $skey, $host := $.reorderedHosts.replicas }}|{{ if (index $.results $host) }}{{ if (index (index $.results $host) "data") }}{{ if (index (index (index $.results $host) "data") "never_used_indexes") }}{{ if (index (index (index (index $.results $host) "data") "never_used_indexes") $key) }}{{ RawIntFormat ((index (index (index (index $.results $host) "data") "redundant_indexes") $key).idx_scan) }}{{end}}{{ end }}{{ end }}{{ end }}{{end}}|
68-
{{- ByteFormat $value.index_size_bytes 2}}|
69-
{{- ByteFormat $value.table_size_bytes 2}}|
70-
{{- if $value.supports_fk }}Yes{{end}}|
71-
{{/* if limit list */}}{{ end -}}
72-
{{ end }}{{/* range */}}
73-
{{else}}
74-
Nothing found.
75-
{{ end }}{{/* redundant indexes found */}}
76-
7756
{{- else -}}{{/* end if master*/}}
7857
Nothing found
7958
{{end}}{{/* end if master*/}}
@@ -83,23 +62,5 @@ Nothing found
8362

8463
## Conclusions ##
8564

86-
8765
## Recommendations ##
88-
{{ if (index .results .reorderedHosts.master) }}
89-
{{ if (index (index .results .reorderedHosts.master) "data") }}
90-
{{ if (index (index (index .results .reorderedHosts.master) "data") "do")}}
91-
#### "DO" database migration code ####
92-
```
93-
{{ range $i, $drop_code := (index (index (index .results .reorderedHosts.master) "data") "do") }}{{ $drop_code }}
94-
{{ end }}
95-
```
96-
{{end}}
97-
{{ if (index (index (index .results .reorderedHosts.master) "data") "undo") }}
98-
#### "UNDO" database migration code ####
99-
```
100-
{{ range $i, $revert_code := (index (index (index .results .reorderedHosts.master) "data") "undo") }}{{ $revert_code }}
101-
{{ end }}
102-
```
103-
{{end}}
104-
{{ end }}{{/* data found */}}
105-
{{ end }}{{/* master */}}
66+

0 commit comments

Comments
 (0)