Skip to content

Commit 9b41773

Browse files
michail-nikolaevCommitfest Bot
michail-nikolaev
authored and
Commitfest Bot
committed
Add stress tests for concurrent index builds
Introduce stress tests for concurrent index operations: - test concurrent inserts/updates during CREATE/REINDEX INDEX CONCURRENTLY - cover various index types (btree, gin, gist, brin, hash, spgist) - test unique and non-unique indexes - test with expressions and predicates - test both parallel and non-parallel operations These tests verify the behavior of the following commits.
1 parent 75e3e65 commit 9b41773

File tree

2 files changed

+224
-0
lines changed

2 files changed

+224
-0
lines changed

‎src/bin/pg_amcheck/meson.build‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ tests += {
2828
't/003_check.pl',
2929
't/004_verify_heapam.pl',
3030
't/005_opclass_damage.pl',
31+
't/006_cic.pl',
3132
],
3233
},
3334
}

‎src/bin/pg_amcheck/t/006_cic.pl‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
# Copyright (c) 2024, PostgreSQL Global Development Group
2+
3+
# Test REINDEX CONCURRENTLY with concurrent modifications and HOT updates
4+
use strict;
5+
use warnings FATAL => 'all';
6+
7+
use PostgreSQL::Test::Cluster;
8+
use PostgreSQL::Test::Utils;
9+
use Test::More;
10+
11+
Test::More->builder->todo_start('filesystem bug')
12+
if PostgreSQL::Test::Utils::has_wal_read_bug;
13+
14+
my ($node, $result);
15+
16+
#
17+
# Test set-up
18+
#
19+
$node = PostgreSQL::Test::Cluster->new('RC_test');
20+
$node->init;
21+
$node->append_conf('postgresql.conf',
22+
'lock_timeout = ' . (1000 * $PostgreSQL::Test::Utils::timeout_default));
23+
$node->append_conf('postgresql.conf', 'fsync = off');
24+
$node->start;
25+
$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
26+
$node->safe_psql('postgres', q(CREATE TABLE tbl(i int primary key,
27+
c1 money default 0, c2 money default 0,
28+
c3 money default 0, updated_at timestamp,
29+
ia int4[], p point)));
30+
$node->safe_psql('postgres', q(CREATE INDEX CONCURRENTLY idx ON tbl(i, updated_at);));
31+
# create sequence
32+
$node->safe_psql('postgres', q(CREATE UNLOGGED SEQUENCE in_row_rebuild START 1 INCREMENT 1;));
33+
$node->safe_psql('postgres', q(SELECT nextval('in_row_rebuild');));
34+
35+
# Create helper functions for predicate tests
36+
$node->safe_psql('postgres', q(
37+
CREATE FUNCTION predicate_stable() RETURNS bool IMMUTABLE
38+
LANGUAGE plpgsql AS $$
39+
BEGIN
40+
EXECUTE 'SELECT txid_current()';
41+
RETURN true;
42+
END; $$;
43+
));
44+
45+
$node->safe_psql('postgres', q(
46+
CREATE FUNCTION predicate_const(integer) RETURNS bool IMMUTABLE
47+
LANGUAGE plpgsql AS $$
48+
BEGIN
49+
RETURN MOD($1, 2) = 0;
50+
END; $$;
51+
));
52+
53+
# Run CIC/RIC in different options concurrently with upserts
54+
$node->pgbench(
55+
'--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
56+
0,
57+
[qr{actually processed}],
58+
[qr{^$}],
59+
'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY',
60+
{
61+
'concurrent_ops' => q(
62+
SET debug_parallel_query = off; -- this is because predicate_stable implementation
63+
SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
64+
\if :gotlock
65+
SELECT nextval('in_row_rebuild') AS last_value \gset
66+
\set variant random(0, 5)
67+
\set parallels random(0, 4)
68+
\if :last_value < 3
69+
ALTER TABLE tbl SET (parallel_workers=:parallels);
70+
\if :variant = 0
71+
CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at);
72+
\elif :variant = 1
73+
CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at) WHERE predicate_stable();
74+
\elif :variant = 2
75+
CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at) WHERE MOD(i, 2) = 0;
76+
\elif :variant = 3
77+
CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at) WHERE predicate_const(i);
78+
\elif :variant = 4
79+
CREATE INDEX CONCURRENTLY new_idx ON tbl(predicate_const(i));
80+
\elif :variant = 5
81+
CREATE INDEX CONCURRENTLY new_idx ON tbl(i, predicate_const(i), updated_at) WHERE predicate_const(i);
82+
\endif
83+
\sleep 10 ms
84+
SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
85+
REINDEX INDEX CONCURRENTLY new_idx;
86+
\sleep 10 ms
87+
SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
88+
DROP INDEX CONCURRENTLY new_idx;
89+
\endif
90+
SELECT pg_advisory_unlock(42);
91+
\else
92+
\set num random(1000, 100000)
93+
BEGIN;
94+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
95+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
96+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
97+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
98+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
99+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
100+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
101+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
102+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
103+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
104+
SELECT setval('in_row_rebuild', 1);
105+
COMMIT;
106+
\endif
107+
)
108+
});
109+
110+
$node->safe_psql('postgres', q(TRUNCATE TABLE tbl;));
111+
112+
# Run CIC/RIC for unique index concurrently with upserts
113+
$node->pgbench(
114+
'--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
115+
0,
116+
[qr{actually processed}],
117+
[qr{^$}],
118+
'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY for unique BTREE',
119+
{
120+
'concurrent_ops_unique_idx' => q(
121+
SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
122+
\if :gotlock
123+
SELECT nextval('in_row_rebuild') AS last_value \gset
124+
\set parallels random(0, 4)
125+
\if :last_value < 3
126+
ALTER TABLE tbl SET (parallel_workers=:parallels);
127+
CREATE UNIQUE INDEX CONCURRENTLY new_idx ON tbl(i);
128+
\sleep 10 ms
129+
SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
130+
REINDEX INDEX CONCURRENTLY new_idx;
131+
\sleep 10 ms
132+
SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
133+
DROP INDEX CONCURRENTLY new_idx;
134+
\endif
135+
SELECT pg_advisory_unlock(42);
136+
\else
137+
\set num random(1, power(10, random(1, 5)))
138+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
139+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
140+
SELECT setval('in_row_rebuild', 1);
141+
\endif
142+
)
143+
});
144+
145+
$node->safe_psql('postgres', q(TRUNCATE TABLE tbl;));
146+
147+
# Run CIC/RIC for GIN with upserts
148+
$node->pgbench(
149+
'--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
150+
0,
151+
[qr{actually processed}],
152+
[qr{^$}],
153+
'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY for GIN/GIST/BRIN/HASH/SPGIST',
154+
{
155+
'concurrent_ops_gin_idx' => q(
156+
SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
157+
\if :gotlock
158+
SELECT nextval('in_row_rebuild') AS last_value \gset
159+
\set parallels random(0, 4)
160+
\if :last_value < 3
161+
ALTER TABLE tbl SET (parallel_workers=:parallels);
162+
CREATE INDEX CONCURRENTLY new_idx ON tbl USING GIN (ia);
163+
\sleep 10 ms
164+
SELECT gin_index_check('new_idx');
165+
REINDEX INDEX CONCURRENTLY new_idx;
166+
\sleep 10 ms
167+
SELECT gin_index_check('new_idx');
168+
DROP INDEX CONCURRENTLY new_idx;
169+
\endif
170+
SELECT pg_advisory_unlock(42);
171+
\else
172+
\set num random(1, power(10, random(1, 5)))
173+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
174+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
175+
SELECT setval('in_row_rebuild', 1);
176+
\endif
177+
)
178+
});
179+
180+
$node->safe_psql('postgres', q(TRUNCATE TABLE tbl;));
181+
182+
# Run CIC/RIC for GIST/BRIN/HASH/SPGIST index concurrently with upserts
183+
$node->pgbench(
184+
'--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
185+
0,
186+
[qr{actually processed}],
187+
[qr{^$}],
188+
'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY for GIN/GIST/BRIN/HASH/SPGIST',
189+
{
190+
'concurrent_ops_other_idx' => q(
191+
SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
192+
\if :gotlock
193+
SELECT nextval('in_row_rebuild') AS last_value \gset
194+
\set parallels random(0, 4)
195+
\if :last_value < 3
196+
ALTER TABLE tbl SET (parallel_workers=:parallels);
197+
\set variant random(0, 3)
198+
\if :variant = 0
199+
CREATE INDEX CONCURRENTLY new_idx ON tbl USING GIST (p);
200+
\elif :variant = 1
201+
CREATE INDEX CONCURRENTLY new_idx ON tbl USING BRIN (updated_at);
202+
\elif :variant = 2
203+
CREATE INDEX CONCURRENTLY new_idx ON tbl USING HASH (updated_at);
204+
\elif :variant = 3
205+
CREATE INDEX CONCURRENTLY new_idx ON tbl USING SPGIST (p);
206+
\endif
207+
\sleep 10 ms
208+
REINDEX INDEX CONCURRENTLY new_idx;
209+
\sleep 10 ms
210+
DROP INDEX CONCURRENTLY new_idx;
211+
\endif
212+
SELECT pg_advisory_unlock(42);
213+
\else
214+
\set num random(1, power(10, random(1, 5)))
215+
INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
216+
ON CONFLICT(i) DO UPDATE SET updated_at = now();
217+
SELECT setval('in_row_rebuild', 1);
218+
\endif
219+
)
220+
});
221+
222+
$node->stop;
223+
done_testing();

0 commit comments

Comments
 (0)