Query optimization refs #343

Author: Oleg

.ci/h003_step2.sql

@@ -2,3 +2,5 @@ create table t1 as select id::int8 from generate_series(0, 1000000) _(id);
 alter table t1 add primary key (id);
 create table t2 as select id, (random() * 1000000)::int8 as t1_id from generate_series(1, 1000000) _(id);
 alter table t2 add constraint fk_t2_t1 foreign key (t1_id) references t1(id);
+vacuum analyze t1;
+vacuum analyze t2;

.ci/h003.sql renamed to .ci/h003_step_1.sql

@@ -0,0 +1,2 @@
+create index on t2 using btree (t1_id);
+vacuum analyze t2;

.ci/h003_step_2.sql

@@ -38,12 +38,6 @@ set statement_timeout to '20ms';
 create index concurrently i_invalid on test_schema.t_with_invalid_index(i);
 set statement_timeout to 0;
 
--- H003 non indexed fks 
-create table t_fk_1 as select id::int8 from generate_series(0, 1000000) _(id);
-alter table t_fk_1 add primary key (id);
-create table t_fk_2 as select id, (random() * 1000000)::int8 as t1_id from generate_series(1, 1000000) _(id);
-alter table t_fk_2 add constraint fk_t2_t1 foreign key (t1_id) references t_fk_1(id);
-
 -- Bloat level
 create table bloated as select i from generate_series(1, 100000) _(i); 
 create index i_bloated on bloated(i); 

.ci/test_db_dump.sql

@@ -25,10 +25,10 @@ before_script:
   script:
     - psql -h postgres -d dbname -U username -c "SELECT version();"
     - echo "Test H003 Non indexed FKs"
-    - psql -h postgres -d dbname -U username -f .ci/h003.sql
+    - psql -h postgres -d dbname -U username -f .ci/h003_step_1.sql
     - ./checkup -h postgres --username username --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
     - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data') && ([[ "$result" == "[]" ]] || [[ "$result" == "null" ]]) && exit 301
-    - psql -h postgres -d dbname -U username -f .ci/h003_step2.sql
+    - psql -h postgres -d dbname -U username -f .ci/h003_step_2.sql
     - rm -Rf ./artifacts/
     - ./checkup -h postgres --username username --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
     - data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && (! [[ "$result" == "[]" ]]) && exit 302

.gitlab-ci.yml

@@ -352,7 +352,7 @@ validate_args() {
   # fill default (not given) psql connection related variables
   [[ "${PGPORT}" = "None" ]] && export PGPORT=5432
   [[ "${DBNAME}" = "None" ]] && export DBNAME=postgres
-  [[ "${STIMEOUT}" = "None" ]] && export STIMEOUT=15 # statement timeout
+  [[ "${STIMEOUT}" = "None" ]] && export STIMEOUT=30 # statement timeout
   [[ "${USERNAME}" = "None" ]] && export USERNAME=""
 
   # custom UNIX domain socket directory for PostgreSQL

checkup

@@ -31,7 +31,7 @@ with data as (
     join pg_namespace as ns on ns.oid = tbl.relnamespace
     join pg_stats as s on s.schemaname = ns.nspname and s.tablename = tbl.relname and not s.inherited and s.attname = att.attname
     left join pg_class as toast on tbl.reltoastrelid = toast.oid
-    where att.attnum > 0 and not att.attisdropped and s.schemaname not in ('pg_catalog', 'information_schema')
+    where att.attnum > 0 and not att.attisdropped and s.schemaname not in ('pg_catalog', 'information_schema') and tbl.relpages > 10
     group by 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, tbl.relhasoids
     order by 2, 3
   ), step2 as (

resources/checks/F004_heap_bloat.sh

@@ -9,6 +9,21 @@ with data as (
     from pg_class pc
     join pg_namespace pn on pn.oid = pc.relnamespace
     where reloptions::text ~ 'autovacuum'
+  ), step0 as (
+      select
+        tbl.oid tblid, nspname, tbl.relname AS tblname, idx.relname AS idxname, idx.reltuples, idx.relpages, idx.relam,
+        indrelid, indexrelid, regexp_split_to_table(indkey::text, ' ')::smallint AS attnum, --indkey::smallint[] AS attnum,
+        coalesce(substring(array_to_string(idx.reloptions, ' ') from 'fillfactor=([0-9]+)')::smallint, 90) as fillfactor
+      from pg_index
+      join pg_class idx on idx.oid = pg_index.indexrelid
+      join pg_class tbl on tbl.oid = pg_index.indrelid
+      join pg_namespace on pg_namespace.oid = idx.relnamespace
+      join pg_am a ON idx.relam = a.oid
+      where a.amname = 'btree'
+        AND pg_index.indisvalid
+        AND tbl.relkind = 'r'
+        AND idx.relpages > 10
+        AND pg_namespace.nspname NOT IN ('pg_catalog', 'information_schema')
   ), step1 as (
     select
       i.tblid,
@@ -36,17 +51,7 @@ with data as (
       sum((1 - coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) as nulldatawidth,
       max(case when a.atttypid = 'pg_catalog.name'::regtype then 1 else 0 end) > 0 as is_na
     from pg_attribute as a
-    join (
-      select
-        tbl.oid tblid, nspname, tbl.relname AS tblname, idx.relname AS idxname, idx.reltuples, idx.relpages, idx.relam,
-        indrelid, indexrelid, indkey::smallint[] AS attnum,
-        coalesce(substring(array_to_string(idx.reloptions, ' ') from 'fillfactor=([0-9]+)')::smallint, 90) as fillfactor
-      from pg_index
-      join pg_class idx on idx.oid = pg_index.indexrelid
-      join pg_class tbl on tbl.oid = pg_index.indrelid
-      join pg_namespace on pg_namespace.oid = idx.relnamespace
-      where pg_index.indisvalid AND tbl.relkind = 'r' AND idx.relpages > 0
-    ) as i on a.attrelid = i.indexrelid
+    join step0 as i on a.attrelid = i.indexrelid AND a.attnum = i.attnum
     join pg_stats as s on
       s.schemaname = i.nspname
       and (

resources/checks/F005_index_bloat.sh

@@ -1,57 +1,61 @@
 ${CHECK_HOST_CMD} "${_PSQL} -f -" <<SQL
 with fk_indexes as (
   select
-    schemaname as schema_name,
-    (indexrelid::regclass)::text as index_name,
-    (relid::regclass)::text as table_name,
+    n.nspname as schema_name,
+    ci.relname as index_name,
+    cr.relname as table_name,
     (confrelid::regclass)::text as fk_table_ref,
     array_to_string(indclass, ', ') as opclasses
-  from
-    pg_stat_user_indexes
-  join pg_index using (indexrelid)
-  left join pg_constraint
-    on array_to_string(indkey, ',') = array_to_string(conkey, ',')
-      and schemaname = (connamespace::regnamespace)::text
-      and conrelid = relid
-      and contype = 'f'
-  where idx_scan = 0
-     and indisunique is false
-     and conkey is not null --conkey is not null then true else false end as is_fk_idx
+  from pg_index i
+  join pg_class ci on ci.oid = i.indexrelid and ci.relkind = 'i'
+  join pg_class cr on cr.oid = i.indrelid and cr.relkind = 'r'
+  join pg_namespace n on n.oid = ci.relnamespace
+  join pg_constraint cn on cn.conrelid = cr.oid
+  left join pg_stat_user_indexes si on si.indexrelid = i.indexrelid
+  where
+     contype = 'f'
+     and i.indisunique is false
+     and conkey is not null
+     and ci.relpages > 100
+     and si.idx_scan < 10
 ), table_scans as (
   select relid,
       tables.idx_scan + tables.seq_scan as all_scans,
       ( tables.n_tup_ins + tables.n_tup_upd + tables.n_tup_del ) as writes,
     pg_relation_size(relid) as table_size
       from pg_stat_user_tables as tables
+      join pg_class c on c.oid = relid
+      where c.relpages > 100
 ), all_writes as (
   select sum(writes) as total_writes
   from table_scans
 ), indexes as (
   select
-    idx_stat.relid,
-    idx_stat.indexrelid,
-    idx_stat.schemaname as schema_name,
-    idx_stat.relname as table_name,
-    idx_stat.indexrelname as index_name,
-    quote_ident(idx_stat.schemaname) as formated_schema_name,
-    quote_ident(idx_stat.indexrelname) as formated_index_name,
-    quote_ident(idx_stat.relname) as formated_table_name,
-    coalesce(nullif(quote_ident(idx_stat.schemaname), 'public') || '.', '') || quote_ident(idx_stat.relname) as formated_relation_name,
-    idx_stat.idx_scan,
-    pg_relation_size(idx_stat.indexrelid) as index_bytes,
-    indexdef ~* 'using btree' as idx_is_btree,
-    pg_get_indexdef(pg_index.indexrelid) as index_def,
-    array_to_string(pg_index.indclass, ', ') as opclasses
-  from pg_stat_user_indexes as idx_stat
-      join pg_index
-          using (indexrelid)
-      join pg_indexes as indexes
-          on idx_stat.schemaname = indexes.schemaname
-              and idx_stat.relname = indexes.tablename
-              and idx_stat.indexrelname = indexes.indexname
+    i.indrelid,
+    i.indexrelid,
+    n.nspname as schema_name,
+    cr.relname as table_name,
+    ci.relname as index_name,
+    quote_ident(n.nspname) as formated_schema_name,
+    quote_ident(ci.relname) as formated_index_name,
+    quote_ident(cr.relname) as formated_table_name,
+    coalesce(nullif(quote_ident(n.nspname), 'public') || '.', '') || quote_ident(cr.relname) as formated_relation_name,
+    si.idx_scan,
+    pg_relation_size(i.indexrelid) as index_bytes,
+    ci.relpages,
+    (case when a.amname = 'btree' then true else false end) as idx_is_btree,
+    pg_get_indexdef(i.indexrelid) as index_def,
+    array_to_string(i.indclass, ', ') as opclasses
+  from pg_index i
+     join pg_class ci on ci.oid = i.indexrelid and ci.relkind = 'i'
+     join pg_class cr on cr.oid = i.indrelid and cr.relkind = 'r'
+     join pg_namespace n on n.oid = ci.relnamespace
+     join pg_am a ON ci.relam = a.oid
+     left join pg_stat_user_indexes si on si.indexrelid = i.indexrelid
   where
-    pg_index.indisunique = false
-    and pg_index.indisvalid = true
+    i.indisunique = false
+    and i.indisvalid = true
+    and ci.relpages > 100
 ), index_ratios as (
   select
     i.indexrelid as index_id,
@@ -67,6 +71,7 @@ with fk_indexes as (
       as scans_per_write,
     index_bytes as index_size_bytes,
     table_size as table_size_bytes,
+    i.relpages,
     idx_is_btree,
     index_def,
     formated_index_name,
@@ -78,9 +83,9 @@ with fk_indexes as (
   from indexes i
   left join fk_indexes fi on
     fi.fk_table_ref = i.table_name
+    and fi.schema_name = i.schema_name 
     and fi.opclasses like (i.opclasses || '%')
-  join table_scans
-  using (relid)
+  join table_scans ts on ts.relid = i.indrelid
 ),
 -- Never used indexes
 never_used_indexes as (
@@ -163,8 +168,9 @@ index_data as (
     *,
     indkey::text as columns,
     array_to_string(indclass, ', ') as opclasses
-  from pg_index
-  where indisvalid = true
+  from pg_index i
+  join pg_class ci on ci.oid = i.indexrelid and ci.relkind = 'i'
+  where indisvalid = true and ci.relpages > 100
 ), redundant_indexes as (
   select
     i2.indexrelid as index_id,

resources/checks/H002_unused_indexes.sh

@@ -15,13 +15,14 @@ with data as (
       nspname,
       fk_actions_update.action as update_action,
       fk_actions_delete.action as delete_action,
-      conkey as key_cols
+      conkey as key_cols,
+      pg_class.relpages
     from pg_constraint
     join pg_class on conrelid = pg_class.oid
     join pg_namespace on pg_class.relnamespace = pg_namespace.oid
     join fk_actions as fk_actions_update on confupdtype = fk_actions_update.code
     join fk_actions as fk_actions_delete on confdeltype = fk_actions_delete.code
-    where contype = 'f'
+    where contype = 'f' -- and pg_class.relpages > 10
   ), fk_attributes as (
     select fkoid, conrelid, attname, attnum
     from fk_list
@@ -37,36 +38,32 @@ with data as (
       pg_class.relname as indexname,
       indrelid,
       indkey,
-      indpred is not null as has_predicate,
-      pg_get_indexdef(indexrelid) as indexdef
+      indpred is not null as has_predicate
     from pg_index
     join pg_class on indexrelid = pg_class.oid
-    where indisvalid
+    where indisvalid and pg_class.relkind = 'i' and pg_class.relpages > 10
   ), fk_index_match as (
     select
       fk_list.*,
       indexid,
       indexname,
       indkey::int[] as indexatts,
       has_predicate,
-      indexdef,
       array_length(key_cols, 1) as fk_colcount,
       array_length(indkey,1) as index_colcount,
-      round(pg_relation_size(conrelid)/(1024^2)::numeric) as table_mb,
+      relpages,
       cols_list
     from fk_list
     join fk_cols_list using (fkoid)
     left join index_list on
       conrelid = indrelid
       and (indkey::int2[])[0:(array_length(key_cols,1) -1)] operator(pg_catalog.@>) key_cols
-
   ), fk_perfect_match as (
     select fkoid
     from fk_index_match
     where
       (index_colcount - 1) <= fk_colcount
       and not has_predicate
-      and indexdef like '%USING btree%'
   ), fk_index_check as (
     select 'no index' as issue, *, 1 as issue_sort
     from fk_index_match
@@ -82,14 +79,15 @@ with data as (
       fkoid,
       tabstats.relname as parent_name,
       (n_tup_ins + n_tup_upd + n_tup_del + n_tup_hot_upd) as parent_writes,
-      round(pg_relation_size(parentid)/(1024^2)::numeric) as parent_mb
+      fk_list.relpages
     from pg_stat_user_tables as tabstats
     join fk_list on relid = parentid
   ), fk_table_stats as (
     select
       fkoid,
       (n_tup_ins + n_tup_upd + n_tup_del + n_tup_hot_upd) as writes,
-      seq_scan as table_scans
+      seq_scan as table_scans,
+      relpages
     from pg_stat_user_tables as tabstats
     join fk_list on relid = conrelid
   )
@@ -98,29 +96,41 @@ with data as (
     relname as table_name,
     conname as fk_name,
     issue,
-    table_mb,
+    conrelid,
     writes,
     table_scans,
     parent_name,
-    parent_mb,
+    parentid,
     parent_writes,
     cols_list,
-    indexdef
+    indexid
   from fk_index_check
   join parent_table_stats using (fkoid)
   join fk_table_stats using (fkoid)
   where
-    table_mb > 9
+    fk_table_stats.relpages > 100
     and (
       writes > 1000
       or parent_writes > 1000
-      or parent_mb > 10
+       or parent_table_stats.relpages > 100
     )
-  order by issue_sort, table_mb desc, table_name, fk_name
+  order by issue_sort, fk_table_stats.relpages desc, table_name, fk_name
 ),
 num_data as (
-  select row_number() over () num, data.* from data limit ${ROWS_LIMIT}
+  select row_number() over () num,
+    schema_name,
+    table_name,
+    fk_name,
+    issue,
+    round(pg_relation_size(conrelid)/(1024^2)::numeric) as table_mb,
+    writes,
+    table_scans,
+    parent_name,
+    round(pg_relation_size(parentid)/(1024^2)::numeric) as parent_mb,
+    parent_writes,
+    cols_list,
+    pg_get_indexdef(indexid) as indexdef
+  from data limit ${ROWS_LIMIT}
 )
 select json_object_agg(num_data.num, num_data) from num_data
 SQL
-

resources/checks/H003_non_indexed_fks.sh

@@ -14,6 +14,8 @@ with data as (
   from pg_class c
   left join pg_namespace n on n.oid = c.relnamespace
   where relkind = 'r' and nspname <> 'pg_catalog'
+  order by c.relpages desc
+  limit ${ROWS_LIMIT}
 ), data2 as (
   select
     null::oid as oid,
@@ -83,7 +85,6 @@ with data as (
     ts.*
   from tables ts
   where "Table" <> '=====TOTAL====='
-  limit ${ROWS_LIMIT}
 ), together as (
   select * from total
   union all