Merge branch 'master' into 'dmius-hot-a006-fix'

# Conflicts:
#   pghrep/templates/A006.tpl

Author: dmius

.ci/h003_step2.sql

@@ -2,3 +2,5 @@ create table t1 as select id::int8 from generate_series(0, 1000000) _(id);
 alter table t1 add primary key (id);
 create table t2 as select id, (random() * 1000000)::int8 as t1_id from generate_series(1, 1000000) _(id);
 alter table t2 add constraint fk_t2_t1 foreign key (t1_id) references t1(id);
+vacuum analyze t1;
+vacuum analyze t2;

.ci/h003.sql renamed to .ci/h003_step_1.sql

@@ -0,0 +1,2 @@
+create index on t2 using btree (t1_id);
+vacuum analyze t2;

.ci/h003_step_2.sql

@@ -31,19 +31,26 @@ create index concurrently t_with_redundant_idx_f2 on t_with_redundant_idx(f2);
 create index concurrently t_with_redundant_idx_f3_1 on t_with_redundant_idx(f3);
 create index concurrently t_with_redundant_idx_f3_2 on t_with_redundant_idx(f3);
 
+drop table if exists public.t_with_redundant_ref_idx;
+create table public.t_with_redundant_ref_idx as select i from generate_series(1, 1000000) _(i);
+create index t_with_redundant_ref_idx_1 on public.t_with_redundant_ref_idx using btree (i);
+create index t_with_redundant_ref_idx_2 on public.t_with_redundant_ref_idx using btree (i);
+create index t_with_redundant_ref_idx_3 on public.t_with_redundant_ref_idx using btree (i);
+
+create schema exp_redundant;
+drop table if exists exp_redundant.t_with_redundant_ref_idx;
+create table exp_redundant.t_with_redundant_ref_idx as select i from generate_series(1, 1000000) _(i);
+create index t_with_redundant_ref_idx_1 on exp_redundant.t_with_redundant_ref_idx using btree (i);
+create index t_with_redundant_ref_idx_2 on exp_redundant.t_with_redundant_ref_idx using btree (i);
+create index t_with_redundant_ref_idx_3 on exp_redundant.t_with_redundant_ref_idx using btree (i);
+
 -- H001 invalid indexes
 create schema test_schema;
 create table test_schema.t_with_invalid_index as select i from generate_series(1, 1000000) _(i);
 set statement_timeout to '20ms';
 create index concurrently i_invalid on test_schema.t_with_invalid_index(i);
 set statement_timeout to 0;
 
--- H003 non indexed fks 
-create table t_fk_1 as select id::int8 from generate_series(0, 1000000) _(id);
-alter table t_fk_1 add primary key (id);
-create table t_fk_2 as select id, (random() * 1000000)::int8 as t1_id from generate_series(1, 1000000) _(id);
-alter table t_fk_2 add constraint fk_t2_t1 foreign key (t1_id) references t_fk_1(id);
-
 -- Bloat level
 create table bloated as select i from generate_series(1, 100000) _(i); 
 create index i_bloated on bloated(i); 

.ci/test_db_dump.sql

@@ -25,10 +25,10 @@ before_script:
   script:
     - psql -h postgres -d dbname -U username -c "SELECT version();"
     - echo "Test H003 Non indexed FKs"
-    - psql -h postgres -d dbname -U username -f .ci/h003.sql
+    - psql -h postgres -d dbname -U username -f .ci/h003_step_1.sql
     - ./checkup -h postgres --username username --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
     - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data') && ([[ "$result" == "[]" ]] || [[ "$result" == "null" ]]) && exit 301
-    - psql -h postgres -d dbname -U username -f .ci/h003_step2.sql
+    - psql -h postgres -d dbname -U username -f .ci/h003_step_2.sql
     - rm -Rf ./artifacts/
     - ./checkup -h postgres --username username --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
     - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && (! [[ "$result" == "[]" ]]) && exit 302
@@ -40,6 +40,10 @@ before_script:
     - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 202
     - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1_uniq"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 203
     - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_pkey"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 204
+    - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 205
+    - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 206
+    - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
+    - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
     - echo "H002 Passed"
 
 test-general:

.gitlab-ci.yml

@@ -352,7 +352,7 @@ validate_args() {
   # fill default (not given) psql connection related variables
   [[ "${PGPORT}" = "None" ]] && export PGPORT=5432
   [[ "${DBNAME}" = "None" ]] && export DBNAME=postgres
-  [[ "${STIMEOUT}" = "None" ]] && export STIMEOUT=15 # statement timeout
+  [[ "${STIMEOUT}" = "None" ]] && export STIMEOUT=30 # statement timeout
   [[ "${USERNAME}" = "None" ]] && export USERNAME=""
 
   # custom UNIX domain socket directory for PostgreSQL

checkup

@@ -38,7 +38,12 @@ test:
 	go test ./src/fmtutils/
 
 vet:
-	go vet ./... 
+	# Command go vet examine all go files in a single scope, which isn't suitable for plugins.
+	# We will run it separately for plugins and main sources.
+	@for f in $(basename $(patsubst plugins/%.go,%,$(PLUGINS_SRC))); do \
+		go vet ./plugins/$$f.go || exit 1 ; \
+	done
+	go vet ./src/...
 
 fmt:
 	go fmt $$(go list ./... | grep -v /vendor/)

pghrep/Makefile

@@ -1,7 +1,9 @@
+Pghrep used by `checkup` during database analysis to create markdown reports from JSON reports.
+
 To build report generator use:
 
 `cd /pghrep`
-make all`
+`make all`
 
 To generate report use:
 

pghrep/README.md

@@ -31,7 +31,7 @@ with data as (
     join pg_namespace as ns on ns.oid = tbl.relnamespace
     join pg_stats as s on s.schemaname = ns.nspname and s.tablename = tbl.relname and not s.inherited and s.attname = att.attname
     left join pg_class as toast on tbl.reltoastrelid = toast.oid
-    where att.attnum > 0 and not att.attisdropped and s.schemaname not in ('pg_catalog', 'information_schema')
+    where att.attnum > 0 and not att.attisdropped and s.schemaname not in ('pg_catalog', 'information_schema') and tbl.relpages > 10
     group by 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, tbl.relhasoids
     order by 2, 3
   ), step2 as (

resources/checks/F004_heap_bloat.sh

@@ -9,6 +9,21 @@ with data as (
     from pg_class pc
     join pg_namespace pn on pn.oid = pc.relnamespace
     where reloptions::text ~ 'autovacuum'
+  ), step0 as (
+      select
+        tbl.oid tblid, nspname, tbl.relname AS tblname, idx.relname AS idxname, idx.reltuples, idx.relpages, idx.relam,
+        indrelid, indexrelid, regexp_split_to_table(indkey::text, ' ')::smallint AS attnum, --indkey::smallint[] AS attnum,
+        coalesce(substring(array_to_string(idx.reloptions, ' ') from 'fillfactor=([0-9]+)')::smallint, 90) as fillfactor
+      from pg_index
+      join pg_class idx on idx.oid = pg_index.indexrelid
+      join pg_class tbl on tbl.oid = pg_index.indrelid
+      join pg_namespace on pg_namespace.oid = idx.relnamespace
+      join pg_am a ON idx.relam = a.oid
+      where a.amname = 'btree'
+        AND pg_index.indisvalid
+        AND tbl.relkind = 'r'
+        AND idx.relpages > 10
+        AND pg_namespace.nspname NOT IN ('pg_catalog', 'information_schema')
   ), step1 as (
     select
       i.tblid,
@@ -36,17 +51,7 @@ with data as (
       sum((1 - coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) as nulldatawidth,
       max(case when a.atttypid = 'pg_catalog.name'::regtype then 1 else 0 end) > 0 as is_na
     from pg_attribute as a
-    join (
-      select
-        tbl.oid tblid, nspname, tbl.relname AS tblname, idx.relname AS idxname, idx.reltuples, idx.relpages, idx.relam,
-        indrelid, indexrelid, indkey::smallint[] AS attnum,
-        coalesce(substring(array_to_string(idx.reloptions, ' ') from 'fillfactor=([0-9]+)')::smallint, 90) as fillfactor
-      from pg_index
-      join pg_class idx on idx.oid = pg_index.indexrelid
-      join pg_class tbl on tbl.oid = pg_index.indrelid
-      join pg_namespace on pg_namespace.oid = idx.relnamespace
-      where pg_index.indisvalid AND tbl.relkind = 'r' AND idx.relpages > 0
-    ) as i on a.attrelid = i.indexrelid
+    join step0 as i on a.attrelid = i.indexrelid AND a.attnum = i.attnum
     join pg_stats as s on
       s.schemaname = i.nspname
       and (

resources/checks/F005_index_bloat.sh

@@ -1,57 +1,61 @@
 ${CHECK_HOST_CMD} "${_PSQL} -f -" <<SQL
 with fk_indexes as (
   select
-    schemaname as schema_name,
-    (indexrelid::regclass)::text as index_name,
-    (relid::regclass)::text as table_name,
+    n.nspname as schema_name,
+    ci.relname as index_name,
+    cr.relname as table_name,
     (confrelid::regclass)::text as fk_table_ref,
     array_to_string(indclass, ', ') as opclasses
-  from
-    pg_stat_user_indexes
-  join pg_index using (indexrelid)
-  left join pg_constraint
-    on array_to_string(indkey, ',') = array_to_string(conkey, ',')
-      and schemaname = (connamespace::regnamespace)::text
-      and conrelid = relid
-      and contype = 'f'
-  where idx_scan = 0
-     and indisunique is false
-     and conkey is not null --conkey is not null then true else false end as is_fk_idx
+  from pg_index i
+  join pg_class ci on ci.oid = i.indexrelid and ci.relkind = 'i'
+  join pg_class cr on cr.oid = i.indrelid and cr.relkind = 'r'
+  join pg_namespace n on n.oid = ci.relnamespace
+  join pg_constraint cn on cn.conrelid = cr.oid
+  left join pg_stat_user_indexes si on si.indexrelid = i.indexrelid
+  where
+     contype = 'f'
+     and i.indisunique is false
+     and conkey is not null
+     and ci.relpages > 100
+     and si.idx_scan < 10
 ), table_scans as (
   select relid,
       tables.idx_scan + tables.seq_scan as all_scans,
       ( tables.n_tup_ins + tables.n_tup_upd + tables.n_tup_del ) as writes,
     pg_relation_size(relid) as table_size
       from pg_stat_user_tables as tables
+      join pg_class c on c.oid = relid
+      where c.relpages > 100
 ), all_writes as (
   select sum(writes) as total_writes
   from table_scans
 ), indexes as (
   select
-    idx_stat.relid,
-    idx_stat.indexrelid,
-    idx_stat.schemaname as schema_name,
-    idx_stat.relname as table_name,
-    idx_stat.indexrelname as index_name,
-    quote_ident(idx_stat.schemaname) as formated_schema_name,
-    quote_ident(idx_stat.indexrelname) as formated_index_name,
-    quote_ident(idx_stat.relname) as formated_table_name,
-    coalesce(nullif(quote_ident(idx_stat.schemaname), 'public') || '.', '') || quote_ident(idx_stat.relname) as formated_relation_name,
-    idx_stat.idx_scan,
-    pg_relation_size(idx_stat.indexrelid) as index_bytes,
-    indexdef ~* 'using btree' as idx_is_btree,
-    pg_get_indexdef(pg_index.indexrelid) as index_def,
-    array_to_string(pg_index.indclass, ', ') as opclasses
-  from pg_stat_user_indexes as idx_stat
-      join pg_index
-          using (indexrelid)
-      join pg_indexes as indexes
-          on idx_stat.schemaname = indexes.schemaname
-              and idx_stat.relname = indexes.tablename
-              and idx_stat.indexrelname = indexes.indexname
+    i.indrelid,
+    i.indexrelid,
+    n.nspname as schema_name,
+    cr.relname as table_name,
+    ci.relname as index_name,
+    quote_ident(n.nspname) as formated_schema_name,
+    quote_ident(ci.relname) as formated_index_name,
+    quote_ident(cr.relname) as formated_table_name,
+    coalesce(nullif(quote_ident(n.nspname), 'public') || '.', '') || quote_ident(cr.relname) as formated_relation_name,
+    si.idx_scan,
+    pg_relation_size(i.indexrelid) as index_bytes,
+    ci.relpages,
+    (case when a.amname = 'btree' then true else false end) as idx_is_btree,
+    pg_get_indexdef(i.indexrelid) as index_def,
+    array_to_string(i.indclass, ', ') as opclasses
+  from pg_index i
+     join pg_class ci on ci.oid = i.indexrelid and ci.relkind = 'i'
+     join pg_class cr on cr.oid = i.indrelid and cr.relkind = 'r'
+     join pg_namespace n on n.oid = ci.relnamespace
+     join pg_am a ON ci.relam = a.oid
+     left join pg_stat_user_indexes si on si.indexrelid = i.indexrelid
   where
-    pg_index.indisunique = false
-    and pg_index.indisvalid = true
+    i.indisunique = false
+    and i.indisvalid = true
+    and ci.relpages > 100
 ), index_ratios as (
   select
     i.indexrelid as index_id,
@@ -67,6 +71,7 @@ with fk_indexes as (
       as scans_per_write,
     index_bytes as index_size_bytes,
     table_size as table_size_bytes,
+    i.relpages,
     idx_is_btree,
     index_def,
     formated_index_name,
@@ -78,9 +83,9 @@ with fk_indexes as (
   from indexes i
   left join fk_indexes fi on
     fi.fk_table_ref = i.table_name
+    and fi.schema_name = i.schema_name 
     and fi.opclasses like (i.opclasses || '%')
-  join table_scans
-  using (relid)
+  join table_scans ts on ts.relid = i.indrelid
 ),
 -- Never used indexes
 never_used_indexes as (
@@ -163,8 +168,9 @@ index_data as (
     *,
     indkey::text as columns,
     array_to_string(indclass, ', ') as opclasses
-  from pg_index
-  where indisvalid = true
+  from pg_index i
+  join pg_class ci on ci.oid = i.indexrelid and ci.relkind = 'i'
+  where indisvalid = true and ci.relpages > 100
 ), redundant_indexes as (
   select
     i2.indexrelid as index_id,
@@ -174,13 +180,14 @@ index_data as (
     irel.relname AS index_name,
     am1.amname as access_method,
     (i1.indexrelid::regclass)::text as reason,
+    i1.indexrelid as reason_index_id,
     pg_get_indexdef(i1.indexrelid) main_index_def,
     pg_size_pretty(pg_relation_size(i1.indexrelid)) main_index_size,
     pg_get_indexdef(i2.indexrelid) index_def,
     pg_relation_size(i2.indexrelid) index_size_bytes,
     s.idx_scan as index_usage,
     quote_ident(tnsp.nspname) as formated_schema_name,
-    quote_ident(irel.relname) as formated_index_name,
+    coalesce(nullif(quote_ident(tnsp.nspname), 'public') || '.', '') || quote_ident(irel.relname) as formated_index_name,
     quote_ident(trel.relname) AS formated_table_name,
     coalesce(nullif(quote_ident(tnsp.nspname), 'public') || '.', '') || quote_ident(trel.relname) as formated_relation_name,
     i2.opclasses
@@ -218,6 +225,28 @@ index_data as (
   left join fk_indexes fi on
     fi.fk_table_ref = ri.table_name
     and fi.opclasses like (ri.opclasses || '%')
+),
+-- Cut recursive links
+redundant_indexes_tmp_num as (
+  select row_number() over () num, rig.*
+  from redundant_indexes_fk rig
+), redundant_indexes_tmp_links as (
+    select
+     ri1.*,
+     ri2.num as r_num
+    from redundant_indexes_tmp_num ri1
+    left join redundant_indexes_tmp_num ri2 on ri2.reason_index_id = ri1.index_id and ri1.reason_index_id = ri2.index_id
+), redundant_indexes_tmp_cut as (
+    select
+     *
+    from redundant_indexes_tmp_links
+    where num < r_num or r_num is null
+), redundant_indexes_cut_grouped as (
+  select
+    distinct(num),
+    *
+  from redundant_indexes_tmp_cut
+  order by index_size_bytes desc
 ), redundant_indexes_grouped as (
   select
     index_id,
@@ -237,7 +266,7 @@ index_data as (
     formated_table_name,
     formated_relation_name,
     supports_fk
-  from redundant_indexes_fk
+  from redundant_indexes_cut_grouped
   group by
     index_id,
     table_size_bytes,
@@ -258,8 +287,7 @@ index_data as (
   select row_number() over () num, rig.*
   from redundant_indexes_grouped rig
   limit ${ROWS_LIMIT}
-),
-redundant_indexes_json as (
+), redundant_indexes_json as (
   select
     json_object_agg(rin.schema_name || '.' || rin.index_name, rin) as json
   from redundant_indexes_num rin