--- /dev/null
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+# Test CREATE INDEX CONCURRENTLY with concurrent modifications
+use strict;
+use warnings;
+
+use Config;
+use PostgresNode;
+use TestLib;
+
+use Test::More tests => 4;
+
+my ($node, $result);
+
+#
+# Test set-up
+#
+$node = PostgresNode->new('CIC_test');
+$node->init;
+$node->append_conf('postgresql.conf', 'lock_timeout = 180000');
+$node->start;
+$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
+$node->safe_psql('postgres', q(CREATE TABLE tbl(i int)));
+$node->safe_psql('postgres', q(CREATE INDEX idx ON tbl(i)));
+
+#
+# Stress CIC with pgbench
+#
+
+# Run background pgbench with CIC. We cannot mix-in this script into single
+# pgbench: CIC will deadlock with itself occasionally.
+my $pgbench_out = '';
+my $pgbench_timer = IPC::Run::timeout(180);
+my $pgbench_h = $node->background_pgbench(
+ '--no-vacuum --client=1 --transactions=200',
+ {
+ '002_pgbench_concurrent_cic' => q(
+ DROP INDEX CONCURRENTLY idx;
+ CREATE INDEX CONCURRENTLY idx ON tbl(i);
+ SELECT bt_index_check('idx',true);
+ )
+ },
+ \$pgbench_out,
+ $pgbench_timer);
+
+# Run pgbench.
+$node->pgbench(
+ '--no-vacuum --client=5 --transactions=200',
+ 0,
+ [qr{actually processed}],
+ [qr{^$}],
+ 'concurrent INSERTs',
+ {
+ '002_pgbench_concurrent_transaction' => q(
+ BEGIN;
+ INSERT INTO tbl VALUES(0);
+ COMMIT;
+ ),
+ '002_pgbench_concurrent_transaction_savepoints' => q(
+ BEGIN;
+ SAVEPOINT s1;
+ INSERT INTO tbl VALUES(0);
+ COMMIT;
+ )
+ });
+
+$pgbench_h->pump_nb;
+$pgbench_h->finish();
+$result =
+ ($Config{osname} eq "MSWin32")
+ ? ($pgbench_h->full_results)[0]
+ : $pgbench_h->result(0);
+is($result, 0, "pgbench with CIC works");
+
+# done
+$node->stop;
+done_testing();
int i;
if (msg->rc.relId == InvalidOid)
- RelationCacheInvalidate();
+ RelationCacheInvalidate(false);
else
RelationCacheInvalidateEntry(msg->rc.relId);
*/
void
InvalidateSystemCaches(void)
+{
+ InvalidateSystemCachesExtended(false);
+}
+
+void
+InvalidateSystemCachesExtended(bool debug_discard)
{
int i;
InvalidateCatalogSnapshot();
ResetCatalogCaches();
- RelationCacheInvalidate(); /* gets smgr and relmap too */
+ RelationCacheInvalidate(debug_discard); /* gets smgr and relmap too */
for (i = 0; i < syscache_callback_count; i++)
{
if (recursion_depth < debug_discard_caches)
{
recursion_depth++;
- InvalidateSystemCaches();
+ InvalidateSystemCachesExtended(true);
recursion_depth--;
}
}
*/
static long relcacheInvalsReceived = 0L;
+/*
+ * in_progress_list is a stack of ongoing RelationBuildDesc() calls. CREATE
+ * INDEX CONCURRENTLY makes catalog changes under ShareUpdateExclusiveLock.
+ * It critically relies on each backend absorbing those changes no later than
+ * next transaction start. Hence, RelationBuildDesc() loops until it finishes
+ * without accepting a relevant invalidation. (Most invalidation consumers
+ * don't do this.)
+ */
+typedef struct inprogressent
+{
+ Oid reloid; /* OID of relation being built */
+ bool invalidated; /* whether an invalidation arrived for it */
+} InProgressEnt;
+
+static InProgressEnt *in_progress_list;
+static int in_progress_list_len;
+static int in_progress_list_maxlen;
+
/*
* eoxact_list[] stores the OIDs of relations that (might) need AtEOXact
* cleanup work. This list intentionally has limited size; if it overflows,
static Relation
RelationBuildDesc(Oid targetRelId, bool insertIt)
{
+ int in_progress_offset;
Relation relation;
Oid relid;
HeapTuple pg_class_tuple;
}
#endif
+ /* Register to catch invalidation messages */
+ if (in_progress_list_len >= in_progress_list_maxlen)
+ {
+ int allocsize;
+
+ allocsize = in_progress_list_maxlen * 2;
+ in_progress_list = repalloc(in_progress_list,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+ }
+ in_progress_offset = in_progress_list_len++;
+ in_progress_list[in_progress_offset].reloid = targetRelId;
+retry:
+ in_progress_list[in_progress_offset].invalidated = false;
+
/*
* find the tuple in pg_class corresponding to the given relation id
*/
MemoryContextDelete(tmpcxt);
}
#endif
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
return NULL;
}
*/
heap_freetuple(pg_class_tuple);
+ /*
+ * If an invalidation arrived mid-build, start over. Between here and the
+ * end of this function, don't add code that does or reasonably could read
+ * system catalogs. That range must be free from invalidation processing
+ * for the !insertIt case. For the insertIt case, RelationCacheInsert()
+ * will enroll this relation in ordinary relcache invalidation processing,
+ */
+ if (in_progress_list[in_progress_offset].invalidated)
+ {
+ RelationDestroyRelation(relation, false);
+ goto retry;
+ }
+ Assert(in_progress_offset + 1 == in_progress_list_len);
+ in_progress_list_len--;
+
/*
* Insert newly created relation into relcache hash table, if requested.
*
/* Build temporary entry, but don't link it into hashtable */
newrel = RelationBuildDesc(save_relid, false);
+
+ /*
+ * Between here and the end of the swap, don't add code that does or
+ * reasonably could read system catalogs. That range must be free
+ * from invalidation processing. See RelationBuildDesc() manipulation
+ * of in_progress_list.
+ */
+
if (newrel == NULL)
{
/*
relcacheInvalsReceived++;
RelationFlushRelation(relation);
}
+ else
+ {
+ int i;
+
+ for (i = 0; i < in_progress_list_len; i++)
+ if (in_progress_list[i].reloid == relationId)
+ in_progress_list[i].invalidated = true;
+ }
}
/*
* and rebuild those with positive reference counts. Also reset the smgr
* relation cache and re-read relation mapping data.
*
- * This is currently used only to recover from SI message buffer overflow,
- * so we do not touch relations having new-in-transaction relfilenodes; they
- * cannot be targets of cross-backend SI updates (and our own updates now go
- * through a separate linked list that isn't limited by the SI message
- * buffer size).
+ * Apart from debug_discard_caches, this is currently used only to recover
+ * from SI message buffer overflow, so we do not touch relations having
+ * new-in-transaction relfilenodes; they cannot be targets of cross-backend
+ * SI updates (and our own updates now go through a separate linked list
+ * that isn't limited by the SI message buffer size).
*
* We do this in two phases: the first pass deletes deletable items, and
* the second one rebuilds the rebuildable items. This is essential for
* second pass processes nailed-in-cache items before other nondeletable
* items. This should ensure that system catalogs are up to date before
* we attempt to use them to reload information about other open relations.
+ *
+ * After those two phases of work having immediate effects, we normally
+ * signal any RelationBuildDesc() on the stack to start over. However, we
+ * don't do this if called as part of debug_discard_caches. Otherwise,
+ * RelationBuildDesc() would become an infinite loop.
*/
void
-RelationCacheInvalidate(void)
+RelationCacheInvalidate(bool debug_discard)
{
HASH_SEQ_STATUS status;
RelIdCacheEnt *idhentry;
List *rebuildFirstList = NIL;
List *rebuildList = NIL;
ListCell *l;
+ int i;
/*
* Reload relation mapping data before starting to reconstruct cache.
RelationClearRelation(relation, true);
}
list_free(rebuildList);
+
+ if (!debug_discard)
+ /* Any RelationBuildDesc() on the stack must start over. */
+ for (i = 0; i < in_progress_list_len; i++)
+ in_progress_list[i].invalidated = true;
}
/*
RelIdCacheEnt *idhentry;
int i;
+ /*
+ * Forget in_progress_list. This is relevant when we're aborting due to
+ * an error during RelationBuildDesc().
+ */
+ Assert(in_progress_list_len == 0 || !isCommit);
+ in_progress_list_len = 0;
+
/*
* Unless the eoxact_list[] overflowed, we only need to examine the rels
* listed in it. Otherwise fall back on a hash_seq_search scan.
RelIdCacheEnt *idhentry;
int i;
+ /*
+ * Forget in_progress_list. This is relevant when we're aborting due to
+ * an error during RelationBuildDesc(). We don't commit subtransactions
+ * during RelationBuildDesc().
+ */
+ Assert(in_progress_list_len == 0 || !isCommit);
+ in_progress_list_len = 0;
+
/*
* Unless the eoxact_list[] overflowed, we only need to examine the rels
* listed in it. Otherwise fall back on a hash_seq_search scan. Same
RelationCacheInitialize(void)
{
HASHCTL ctl;
+ int allocsize;
/*
* make sure cache memory context exists
RelationIdCache = hash_create("Relcache by OID", INITRELCACHESIZE,
&ctl, HASH_ELEM | HASH_BLOBS);
+ /*
+ * reserve enough in_progress_list slots for many cases
+ */
+ allocsize = 4;
+ in_progress_list =
+ MemoryContextAlloc(CacheMemoryContext,
+ allocsize * sizeof(*in_progress_list));
+ in_progress_list_maxlen = allocsize;
+
/*
* relation mapper needs to be initialized too
*/
$node->init;
$node->start;
-# invoke pgbench, with parameters:
-# $opts: options as a string to be split on spaces
-# $stat: expected exit status
-# $out: reference to a regexp list that must match stdout
-# $err: reference to a regexp list that must match stderr
-# $name: name of test for error messages
-# $files: reference to filename/contents dictionary
-# @args: further raw options or arguments
-sub pgbench
-{
- local $Test::Builder::Level = $Test::Builder::Level + 1;
-
- my ($opts, $stat, $out, $err, $name, $files, @args) = @_;
- my @cmd = ('pgbench', split /\s+/, $opts);
- my @filenames = ();
- if (defined $files)
- {
-
- # note: files are ordered for determinism
- for my $fn (sort keys %$files)
- {
- my $filename = $node->basedir . '/' . $fn;
- push @cmd, '-f', $filename;
-
- # cleanup file weight
- $filename =~ s/\@\d+$//;
-
- #push @filenames, $filename;
- # filenames are expected to be unique on a test
- if (-e $filename)
- {
- ok(0, "$filename must not already exist");
- unlink $filename or die "cannot unlink $filename: $!";
- }
- append_to_file($filename, $$files{$fn});
- }
- }
-
- push @cmd, @args;
-
- $node->command_checks_all(\@cmd, $stat, $out, $err, $name);
-
- # cleanup?
- #unlink @filenames or die "cannot unlink files (@filenames): $!";
-
- return;
-}
-
# tablespace for testing, because partitioned tables cannot use pg_default
# explicitly and we want to test that table creation with tablespace works
# for partitioned tables.
# Test concurrent OID generation via pg_enum_oid_index. This indirectly
# exercises LWLock and spinlock concurrency.
my $labels = join ',', map { "'l$_'" } 1 .. 1000;
-pgbench(
+$node->pgbench(
'--no-vacuum --client=5 --protocol=prepared --transactions=25',
0,
[qr{processed: 125/125}],
});
# Trigger various connection errors
-pgbench(
+$node->pgbench(
'no-such-database',
1,
[qr{^$}],
],
'no such database');
-pgbench(
+$node->pgbench(
'-S -t 1', 1, [],
[qr{Perhaps you need to do initialization}],
'run without init');
# Initialize pgbench tables scale 1
-pgbench(
+$node->pgbench(
'-i', 0,
[qr{^$}],
[
'pgbench scale 1 initialization',);
# Again, with all possible options
-pgbench(
+$node->pgbench(
'--initialize --init-steps=dtpvg --scale=1 --unlogged-tables --fillfactor=98 --foreign-keys --quiet --tablespace=regress_pgbench_tap_1_ts --index-tablespace=regress_pgbench_tap_1_ts --partitions=2 --partition-method=hash',
0,
[qr{^$}i],
'pgbench scale 1 initialization');
# Test interaction of --init-steps with legacy step-selection options
-pgbench(
+$node->pgbench(
'--initialize --init-steps=dtpvGvv --no-vacuum --foreign-keys --unlogged-tables --partitions=3',
0,
[qr{^$}],
'pgbench --init-steps');
# Run all builtin scripts, for a few transactions each
-pgbench(
+$node->pgbench(
'--transactions=5 -Dfoo=bla --client=2 --protocol=simple --builtin=t'
. ' --connect -n -v -n',
0,
[qr{^$}],
'pgbench tpcb-like');
-pgbench(
+$node->pgbench(
'--transactions=20 --client=5 -M extended --builtin=si -C --no-vacuum -s 1',
0,
[
[qr{scale option ignored}],
'pgbench simple update');
-pgbench(
+$node->pgbench(
'-t 100 -c 7 -M prepared -b se --debug',
0,
[
}
# run custom scripts
-pgbench(
+$node->pgbench(
"-t 100 -c 1 -j $nthreads -M prepared -n",
0,
[
}
});
-pgbench(
+$node->pgbench(
'-n -t 10 -c 1 -M simple',
0,
[
}
});
-pgbench(
+$node->pgbench(
'-n -t 10 -c 2 -M extended',
0,
[
. "log_parameter_max_length = 0\n"
. "log_parameter_max_length_on_error = 0");
$node->reload;
-pgbench(
+$node->pgbench(
'-n -t1 -c1 -M prepared',
2,
[],
"log_parameter_max_length = -1\n"
. "log_parameter_max_length_on_error = 64");
$node->reload;
-pgbench(
+$node->pgbench(
'-n -t1 -c1 -M prepared',
2,
[],
SELECT 1 / (random() / 2)::int, :one::int, :two::int;
}
});
-pgbench(
+$node->pgbench(
'-n -t1 -c1 -M prepared',
2,
[],
. "log_parameter_max_length = 7\n"
. "log_parameter_max_length_on_error = -1");
$node->reload;
-pgbench(
+$node->pgbench(
'-n -t1 -c1 -M prepared',
2,
[],
$node->append_conf('postgresql.conf', "log_min_duration_statement = 0");
$node->reload;
-pgbench(
+$node->pgbench(
'-n -t1 -c1 -M prepared',
2,
[],
$log = undef;
# Check that bad parameters are reported during typinput phase of BIND
-pgbench(
+$node->pgbench(
'-n -t1 -c1 -M prepared',
2,
[],
# test expressions
# command 1..3 and 23 depend on random seed which is used to call srandom.
-pgbench(
+$node->pgbench(
'--random-seed=5432 -t 1 -Dfoo=-10.1 -Dbla=false -Di=+3 -Dn=null -Dt=t -Df=of -Dd=1.0',
0,
[ qr{type: .*/001_pgbench_expressions}, qr{processed: 1/1} ],
my $seed = int(rand(1000000000));
for my $i (1, 2)
{
- pgbench(
+ $node->pgbench(
"--random-seed=$seed -t 1",
0,
[qr{processed: 1/1}],
$node->safe_psql('postgres', 'DROP TABLE seeded_random;');
# backslash commands
-pgbench(
+$node->pgbench(
'-t 1', 0,
[
qr{type: .*/001_pgbench_backslash_commands},
});
# working \gset
-pgbench(
+$node->pgbench(
'-t 1', 0,
[ qr{type: .*/001_pgbench_gset}, qr{processed: 1/1} ],
[
}
});
# \gset cannot accept more than one row, causing command to fail.
-pgbench(
+$node->pgbench(
'-t 1', 2,
[ qr{type: .*/001_pgbench_gset_two_rows}, qr{processed: 0/1} ],
[qr{expected one row, got 2\b}],
# working \aset
# Valid cases.
-pgbench(
+$node->pgbench(
'-t 1', 0,
[ qr{type: .*/001_pgbench_aset}, qr{processed: 1/1} ],
[ qr{command=3.: int 8\b}, qr{command=4.: int 7\b} ],
}
});
# Empty result set with \aset, causing command to fail.
-pgbench(
+$node->pgbench(
'-t 1', 2,
[ qr{type: .*/001_pgbench_aset_empty}, qr{processed: 0/1} ],
[
});
# Working \startpipeline
-pgbench(
+$node->pgbench(
'-t 1 -n -M extended',
0,
[ qr{type: .*/001_pgbench_pipeline}, qr{actually processed: 1/1} ],
});
# Working \startpipeline in prepared query mode
-pgbench(
+$node->pgbench(
'-t 1 -n -M prepared',
0,
[ qr{type: .*/001_pgbench_pipeline_prep}, qr{actually processed: 1/1} ],
});
# Try \startpipeline twice
-pgbench(
+$node->pgbench(
'-t 1 -n -M extended',
2,
[],
});
# Try to end a pipeline that hasn't started
-pgbench(
+$node->pgbench(
'-t 1 -n -M extended',
2,
[],
});
# Try \gset in pipeline mode
-pgbench(
+$node->pgbench(
'-t 1 -n -M extended',
2,
[],
$status != 0 or die "invalid expected status for test \"$name\"";
my $n = '001_pgbench_error_' . $name;
$n =~ s/ /_/g;
- pgbench(
+ $node->pgbench(
'-n -t 1 -Dfoo=bla -Dnull=null -Dtrue=true -Done=1 -Dzero=0.0 -Dbadtrue=trueXXX'
. ' -Dmaxint=9223372036854775807 -Dminint=-9223372036854775808'
. ($no_prepare ? '' : ' -M prepared'),
}
# throttling
-pgbench(
+$node->pgbench(
'-t 100 -S --rate=100000 --latency-limit=1000000 -c 2 -n -r',
0,
[ qr{processed: 200/200}, qr{builtin: select only} ],
[qr{^$}],
'pgbench throttling');
-pgbench(
+$node->pgbench(
# given the expected rate and the 2 ms tx duration, at most one is executed
'-t 10 --rate=100000 --latency-limit=1 -n -r',
my $bdir = $node->basedir;
# Run with sampling rate, 2 clients with 50 transactions each.
-pgbench(
+$node->pgbench(
"-n -S -t 50 -c 2 --log --sampling-rate=0.5", 0,
[ qr{select only}, qr{processed: 100/100} ], [qr{^$}],
'pgbench logs', undef,
qr{^[01] \d{1,2} \d+ \d \d+ \d+$});
# Run with different read-only option pattern, 1 client with 10 transactions.
-pgbench(
+$node->pgbench(
"-n -b select-only -t 10 -l", 0,
[ qr{select only}, qr{processed: 10/10} ], [qr{^$}],
'pgbench logs contents', undef,
extern void CallSyscacheCallbacks(int cacheid, uint32 hashvalue);
extern void InvalidateSystemCaches(void);
+extern void InvalidateSystemCachesExtended(bool debug_discard);
extern void LogLogicalInvalidations(void);
#endif /* INVAL_H */
extern void RelationCacheInvalidateEntry(Oid relationId);
-extern void RelationCacheInvalidate(void);
+extern void RelationCacheInvalidate(bool debug_discard);
extern void RelationCloseSmgrByOid(Oid relationId);
return $harness;
}
+# Common sub of pgbench-invoking interfaces. Makes any requested script files
+# and returns pgbench command-line options causing use of those files.
+sub _pgbench_make_files
+{
+ my ($self, $files) = @_;
+ my @file_opts;
+
+ if (defined $files)
+ {
+
+ # note: files are ordered for determinism
+ for my $fn (sort keys %$files)
+ {
+ my $filename = $self->basedir . '/' . $fn;
+ push @file_opts, '-f', $filename;
+
+ # cleanup file weight
+ $filename =~ s/\@\d+$//;
+
+ #push @filenames, $filename;
+ # filenames are expected to be unique on a test
+ if (-e $filename)
+ {
+ ok(0, "$filename must not already exist");
+ unlink $filename or die "cannot unlink $filename: $!";
+ }
+ TestLib::append_to_file($filename, $$files{$fn});
+ }
+ }
+
+ return @file_opts;
+}
+
+=pod
+
+=item $node->pgbench($opts, $stat, $out, $err, $name, $files, @args)
+
+Invoke B<pgbench>, with parameters and files.
+
+=over
+
+=item $opts
+
+Options as a string to be split on spaces.
+
+=item $stat
+
+Expected exit status.
+
+=item $out
+
+Reference to a regexp list that must match stdout.
+
+=item $err
+
+Reference to a regexp list that must match stderr.
+
+=item $name
+
+Name of test for error messages.
+
+=item $files
+
+Reference to filename/contents dictionary.
+
+=item @args
+
+Further raw options or arguments.
+
+=back
+
+=cut
+
+sub pgbench
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+ my ($self, $opts, $stat, $out, $err, $name, $files, @args) = @_;
+ my @cmd = (
+ 'pgbench',
+ split(/\s+/, $opts),
+ $self->_pgbench_make_files($files), @args);
+
+ $self->command_checks_all(\@cmd, $stat, $out, $err, $name);
+}
+
+=pod
+
+=item $node->background_pgbench($opts, $files, \$stdout, $timer) => harness
+
+Invoke B<pgbench> and return an IPC::Run harness object. The process's stdin
+is empty, and its stdout and stderr go to the $stdout scalar reference. This
+allows the caller to act on other parts of the system while B<pgbench> is
+running. Errors from B<pgbench> are the caller's problem.
+
+The specified timer object is attached to the harness, as well. It's caller's
+responsibility to select the timeout length, and to restart the timer after
+each command if the timeout is per-command.
+
+Be sure to "finish" the harness when done with it.
+
+=over
+
+=item $opts
+
+Options as a string to be split on spaces.
+
+=item $files
+
+Reference to filename/contents dictionary.
+
+=back
+
+=cut
+
+sub background_pgbench
+{
+ my ($self, $opts, $files, $stdout, $timer) = @_;
+
+ my @cmd =
+ ('pgbench', split(/\s+/, $opts), $self->_pgbench_make_files($files));
+
+ local %ENV = $self->_get_env();
+
+ my $stdin = "";
+ # IPC::Run would otherwise append to existing contents:
+ $$stdout = "" if ref($stdout);
+
+ my $harness = IPC::Run::start \@cmd, '<', \$stdin, '>', $stdout, '2>&1',
+ $timer;
+
+ return $harness;
+}
+
=pod
=item $node->connect_ok($connstr, $test_name, %params)
ImportForeignSchemaType
ImportForeignSchema_function
ImportQual
+InProgressEnt
IncludeWal
InclusionOpaque
IncrementVarSublevelsUp_context
projects / postgresql.git / commitdiff