mirrors/postgres
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix 004_subscription.pl to allow its usage in --link mode.

Browse Source 
The test was failing when executed in --link mode and the reason was that
we were using the old cluster from a previously successful upgrade test.
Re-arrange the tests so that the successful test case is at the end.

Reported-by: Justin Pryzby, Peter Eisentraut
Author: Kuroda Hayato
Reviewed-by: Vignesh C, Amit Kapila
Discussion: https://postgr.es/m/ZcvZipRoi_kopIpb@pryzbyj2023
Discussion: https://postgr.es/m/25f7f4bf-9e75-4453-b666-7818000cefe6@eisentraut.org
This commit is contained in:
Amit Kapila 2024-02-19 10:36:05 +05:30
parent e77a1c58e3
commit f17529b710
1 changed files with 192 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

376
src/bin/pg_upgrade/t/004_subscription.pl
View File

@ -1,10 +1,12 @@
# Copyright (c) 2023-2024, PostgreSQL Global Development Group # Copyright (c) 2023-2024, PostgreSQL Global Development Group
# Test for pg_upgrade of logical subscription # Test for pg_upgrade of logical subscription. Note that after the successful
# upgrade test, we can't use the old cluster to prevent failing in --link mode.
use strict; use strict;
use warnings FATAL => 'all'; use warnings FATAL => 'all';
use File::Find qw(find); use File::Find qw(find);
use File::Path qw(rmtree);
use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils; use PostgreSQL::Test::Utils;
@ -34,37 +36,164 @@ my $newbindir = $new_sub->config_data('--bindir');
# in it, like delete_old_cluster.{sh,bat}. # in it, like delete_old_cluster.{sh,bat}.
chdir ${PostgreSQL::Test::Utils::tmp_check}; chdir ${PostgreSQL::Test::Utils::tmp_check};
# Initial setup # Remember a connection string for the publisher node. It would be used
# several times.
my $connstr = $publisher->connstr . ' dbname=postgres';
# ------------------------------------------------------
# Check that pg_upgrade fails when max_replication_slots configured in the new
# cluster is less than the number of subscriptions in the old cluster.
# ------------------------------------------------------
# It is sufficient to use disabled subscription to test upgrade failure.
$publisher->safe_psql('postgres', "CREATE PUBLICATION regress_pub1");
$old_sub->safe_psql('postgres',
"CREATE SUBSCRIPTION regress_sub1 CONNECTION '$connstr' PUBLICATION regress_pub1 WITH (enabled = false)"
);
$old_sub->stop;
$new_sub->append_conf('postgresql.conf', "max_replication_slots = 0");
# pg_upgrade will fail because the new cluster has insufficient
# max_replication_slots.
command_checks_all(
[
'pg_upgrade', '--no-sync', '-d', $old_sub->data_dir,
'-D', $new_sub->data_dir, '-b', $oldbindir,
'-B', $newbindir, '-s', $new_sub->host,
'-p', $old_sub->port, '-P', $new_sub->port,
$mode, '--check',
],
1,
[
qr/max_replication_slots \(0\) must be greater than or equal to the number of subscriptions \(1\) on the old cluster/
],
[qr//],
'run of pg_upgrade where the new cluster has insufficient max_replication_slots'
);
# Reset max_replication_slots
$new_sub->append_conf('postgresql.conf', "max_replication_slots = 10");
# Cleanup
$publisher->safe_psql('postgres', "DROP PUBLICATION regress_pub1");
$old_sub->start;
$old_sub->safe_psql('postgres', "DROP SUBSCRIPTION regress_sub1;");
# ------------------------------------------------------
# Check that pg_upgrade refuses to run if:
# a) there's a subscription with tables in a state other than 'r' (ready) or
# 'i' (init) and/or
# b) the subscription has no replication origin.
# ------------------------------------------------------
$publisher->safe_psql( $publisher->safe_psql(
'postgres', qq[ 'postgres', qq[
CREATE TABLE tab_upgraded1(id int); CREATE TABLE tab_primary_key(id serial PRIMARY KEY);
CREATE TABLE tab_upgraded2(id int); INSERT INTO tab_primary_key values(1);
CREATE PUBLICATION regress_pub2 FOR TABLE tab_primary_key;
]);
# Insert the same value that is already present in publisher to the primary key
# column of subscriber so that the table sync will fail.
$old_sub->safe_psql(
'postgres', qq[
CREATE TABLE tab_primary_key(id serial PRIMARY KEY);
INSERT INTO tab_primary_key values(1);
CREATE SUBSCRIPTION regress_sub2 CONNECTION '$connstr' PUBLICATION regress_pub2;
]);
# Table will be in 'd' (data is being copied) state as table sync will fail
# because of primary key constraint error.
my $started_query =
"SELECT count(1) = 1 FROM pg_subscription_rel WHERE srsubstate = 'd'";
$old_sub->poll_query_until('postgres', $started_query)
or die
"Timed out while waiting for the table state to become 'd' (datasync)";
# Setup another logical replication and drop the subscription's replication
# origin.
$publisher->safe_psql('postgres', "CREATE PUBLICATION regress_pub3");
$old_sub->safe_psql('postgres',
"CREATE SUBSCRIPTION regress_sub3 CONNECTION '$connstr' PUBLICATION regress_pub3 WITH (enabled = false)"
);
my $sub_oid = $old_sub->safe_psql('postgres',
"SELECT oid FROM pg_subscription WHERE subname = 'regress_sub3'");
my $reporigin = 'pg_' . qq($sub_oid);
$old_sub->safe_psql('postgres',
"SELECT pg_replication_origin_drop('$reporigin')");
$old_sub->stop;
command_fails(
[
'pg_upgrade', '--no-sync', '-d', $old_sub->data_dir,
'-D', $new_sub->data_dir, '-b', $oldbindir,
'-B', $newbindir, '-s', $new_sub->host,
'-p', $old_sub->port, '-P', $new_sub->port,
$mode, '--check',
],
'run of pg_upgrade --check for old instance with relation in \'d\' datasync(invalid) state and missing replication origin'
);
# Verify the reason why the subscriber cannot be upgraded
my $sub_relstate_filename;
# Find a txt file that contains a list of tables that cannot be upgraded. We
# cannot predict the file's path because the output directory contains a
# milliseconds timestamp. File::Find::find must be used.
find(
sub {
if ($File::Find::name =~ m/subs_invalid\.txt/)
{
$sub_relstate_filename = $File::Find::name;
}
},
$new_sub->data_dir . "/pg_upgrade_output.d");
# Check the file content which should have tab_primary_key table in an invalid
# state.
like(
slurp_file($sub_relstate_filename),
qr/The table sync state \"d\" is not allowed for database:\"postgres\" subscription:\"regress_sub2\" schema:\"public\" relation:\"tab_primary_key\"/m,
'the previous test failed due to subscription table in invalid state');
# Check the file content which should have regress_sub3 subscription.
like(
slurp_file($sub_relstate_filename),
qr/The replication origin is missing for database:\"postgres\" subscription:\"regress_sub3\"/m,
'the previous test failed due to missing replication origin');
# Cleanup
$old_sub->start;
$publisher->safe_psql(
'postgres', qq[
DROP PUBLICATION regress_pub2;
DROP PUBLICATION regress_pub3;
DROP TABLE tab_primary_key;
]); ]);
$old_sub->safe_psql( $old_sub->safe_psql(
'postgres', qq[ 'postgres', qq[
CREATE TABLE tab_upgraded1(id int); DROP SUBSCRIPTION regress_sub2;
CREATE TABLE tab_upgraded2(id int); DROP SUBSCRIPTION regress_sub3;
DROP TABLE tab_primary_key;
]); ]);
rmtree($new_sub->data_dir . "/pg_upgrade_output.d");
# Setup logical replication
my $connstr = $publisher->connstr . ' dbname=postgres';
# Setup an enabled subscription to verify that the running status and failover
# option are retained after the upgrade.
$publisher->safe_psql('postgres', "CREATE PUBLICATION regress_pub1");
$old_sub->safe_psql('postgres',
"CREATE SUBSCRIPTION regress_sub1 CONNECTION '$connstr' PUBLICATION regress_pub1 WITH (failover = true)"
);
$old_sub->wait_for_subscription_sync($publisher, 'regress_sub1');
# Verify that the upgrade should be successful with tables in 'ready'/'init' # Verify that the upgrade should be successful with tables in 'ready'/'init'
# state along with retaining the replication origin's remote lsn, and # state along with retaining the replication origin's remote lsn, subscription's
# subscription's running status. # running status, and failover option.
$publisher->safe_psql('postgres', $publisher->safe_psql(
"CREATE PUBLICATION regress_pub2 FOR TABLE tab_upgraded1"); 'postgres', qq[
$old_sub->safe_psql('postgres', CREATE TABLE tab_upgraded1(id int);
"CREATE SUBSCRIPTION regress_sub2 CONNECTION '$connstr' PUBLICATION regress_pub2" CREATE PUBLICATION regress_pub4 FOR TABLE tab_upgraded1;
); ]);
$old_sub->safe_psql(
'postgres', qq[
CREATE TABLE tab_upgraded1(id int);
CREATE SUBSCRIPTION regress_sub4 CONNECTION '$connstr' PUBLICATION regress_pub4 WITH (failover = true);
]);
# Wait till the table tab_upgraded1 reaches 'ready' state # Wait till the table tab_upgraded1 reaches 'ready' state
my $synced_query = my $synced_query =
"SELECT count(1) = 1 FROM pg_subscription_rel WHERE srsubstate = 'r'"; "SELECT count(1) = 1 FROM pg_subscription_rel WHERE srsubstate = 'r'";
@ -73,19 +202,26 @@ $old_sub->poll_query_until('postgres', $synced_query)
$publisher->safe_psql('postgres', $publisher->safe_psql('postgres',
"INSERT INTO tab_upgraded1 VALUES (generate_series(1,50))"); "INSERT INTO tab_upgraded1 VALUES (generate_series(1,50))");
$publisher->wait_for_catchup('regress_sub2'); $publisher->wait_for_catchup('regress_sub4');
# Change configuration to prepare a subscription table in init state # Change configuration to prepare a subscription table in init state
$old_sub->append_conf('postgresql.conf', $old_sub->append_conf('postgresql.conf',
"max_logical_replication_workers = 0"); "max_logical_replication_workers = 0");
$old_sub->restart; $old_sub->restart;
$publisher->safe_psql('postgres', # Setup another logical replication
"ALTER PUBLICATION regress_pub2 ADD TABLE tab_upgraded2"); $publisher->safe_psql(
$old_sub->safe_psql('postgres', 'postgres', qq[
"ALTER SUBSCRIPTION regress_sub2 REFRESH PUBLICATION"); CREATE TABLE tab_upgraded2(id int);
CREATE PUBLICATION regress_pub5 FOR TABLE tab_upgraded2;
]);
$old_sub->safe_psql(
'postgres', qq[
CREATE TABLE tab_upgraded2(id int);
CREATE SUBSCRIPTION regress_sub5 CONNECTION '$connstr' PUBLICATION regress_pub5;
]);
# The table tab_upgraded2 will be in init state as the subscriber # The table tab_upgraded2 will be in the init state as the subscriber's
# configuration for max_logical_replication_workers is set to 0. # configuration for max_logical_replication_workers is set to 0.
my $result = $old_sub->safe_psql('postgres', my $result = $old_sub->safe_psql('postgres',
"SELECT count(1) = 1 FROM pg_subscription_rel WHERE srsubstate = 'i'"); "SELECT count(1) = 1 FROM pg_subscription_rel WHERE srsubstate = 'i'");
@ -93,10 +229,10 @@ is($result, qq(t), "Check that the table is in init state");
# Get the replication origin's remote_lsn of the old subscriber # Get the replication origin's remote_lsn of the old subscriber
my $remote_lsn = $old_sub->safe_psql('postgres', my $remote_lsn = $old_sub->safe_psql('postgres',
"SELECT remote_lsn FROM pg_replication_origin_status os, pg_subscription s WHERE os.external_id = 'pg_' || s.oid AND s.subname = 'regress_sub2'" "SELECT remote_lsn FROM pg_replication_origin_status os, pg_subscription s WHERE os.external_id = 'pg_' || s.oid AND s.subname = 'regress_sub4'"
); );
# Have the subscription in disabled state before upgrade # Have the subscription in disabled state before upgrade
$old_sub->safe_psql('postgres', "ALTER SUBSCRIPTION regress_sub2 DISABLE"); $old_sub->safe_psql('postgres', "ALTER SUBSCRIPTION regress_sub5 DISABLE");
my $tab_upgraded1_oid = $old_sub->safe_psql('postgres', my $tab_upgraded1_oid = $old_sub->safe_psql('postgres',
"SELECT oid FROM pg_class WHERE relname = 'tab_upgraded1'"); "SELECT oid FROM pg_class WHERE relname = 'tab_upgraded1'");
@ -105,6 +241,11 @@ my $tab_upgraded2_oid = $old_sub->safe_psql('postgres',
$old_sub->stop; $old_sub->stop;
# Change configuration so that initial table sync sync does not get started
# automatically
$new_sub->append_conf('postgresql.conf',
"max_logical_replication_workers = 0");
# ------------------------------------------------------ # ------------------------------------------------------
# Check that pg_upgrade is successful when all tables are in ready or in # Check that pg_upgrade is successful when all tables are in ready or in
# init state (tab_upgraded1 table is in ready state and tab_upgraded2 table is # init state (tab_upgraded1 table is in ready state and tab_upgraded2 table is
@ -138,21 +279,19 @@ $publisher->safe_psql(
$new_sub->start; $new_sub->start;
# The subscription's running status and failover option should be preserved # The subscription's running status and failover option should be preserved
# in the upgraded instance. So regress_sub1 should still have subenabled and # in the upgraded instance. So regress_sub4 should still have subenabled and
# subfailover set to true, while regress_sub2 should have both set to false. # subfailover set to true, while regress_sub5 should have both set to false.
$result = $result = $new_sub->safe_psql('postgres',
$new_sub->safe_psql('postgres', "SELECT subname, subenabled, subfailover FROM pg_subscription ORDER BY subname"
"SELECT subname, subenabled, subfailover FROM pg_subscription ORDER BY subname"); );
is( $result, qq(regress_sub1|t|t is( $result, qq(regress_sub4|t|t
regress_sub2|f|f), regress_sub5|f|f),
"check that the subscription's running status and failover are preserved"); "check that the subscription's running status and failover are preserved"
);
my $sub_oid = $new_sub->safe_psql('postgres',
"SELECT oid FROM pg_subscription WHERE subname = 'regress_sub2'");
# Subscription relations should be preserved # Subscription relations should be preserved
$result = $new_sub->safe_psql('postgres', $result = $new_sub->safe_psql('postgres',
"SELECT srrelid, srsubstate FROM pg_subscription_rel WHERE srsubid = $sub_oid ORDER BY srrelid" "SELECT srrelid, srsubstate FROM pg_subscription_rel ORDER BY srrelid"
); );
is( $result, qq($tab_upgraded1_oid|r is( $result, qq($tab_upgraded1_oid|r
$tab_upgraded2_oid|i), $tab_upgraded2_oid|i),
@ -160,16 +299,20 @@ $tab_upgraded2_oid|i),
); );
# The replication origin's remote_lsn should be preserved # The replication origin's remote_lsn should be preserved
$sub_oid = $new_sub->safe_psql('postgres',
"SELECT oid FROM pg_subscription WHERE subname = 'regress_sub4'");
$result = $new_sub->safe_psql('postgres', $result = $new_sub->safe_psql('postgres',
"SELECT remote_lsn FROM pg_replication_origin_status WHERE external_id = 'pg_' || $sub_oid" "SELECT remote_lsn FROM pg_replication_origin_status WHERE external_id = 'pg_' || $sub_oid"
); );
is($result, qq($remote_lsn), "remote_lsn should have been preserved"); is($result, qq($remote_lsn), "remote_lsn should have been preserved");
# Enable the subscription # Resume the initial sync and wait until all tables of subscription
$new_sub->safe_psql('postgres', "ALTER SUBSCRIPTION regress_sub2 ENABLE"); # 'regress_sub5' are synchronized
$new_sub->append_conf('postgresql.conf',
# Wait until all tables of subscription 'regress_sub2' are synchronized "max_logical_replication_workers = 10");
$new_sub->wait_for_subscription_sync($publisher, 'regress_sub2'); $new_sub->restart;
$new_sub->safe_psql('postgres', "ALTER SUBSCRIPTION regress_sub5 ENABLE");
$new_sub->wait_for_subscription_sync($publisher, 'regress_sub5');
# Rows on tab_upgraded1 and tab_upgraded2 should have been replicated # Rows on tab_upgraded1 and tab_upgraded2 should have been replicated
$result = $result =
@ -181,139 +324,4 @@ is($result, qq(1),
"check the data is synced after enabling the subscription for the table that was in init state" "check the data is synced after enabling the subscription for the table that was in init state"
); );
# cleanup
$new_sub->stop;
$old_sub->append_conf('postgresql.conf',
"max_logical_replication_workers = 4");
$old_sub->start;
$old_sub->safe_psql(
'postgres', qq[
ALTER SUBSCRIPTION regress_sub1 DISABLE;
ALTER SUBSCRIPTION regress_sub1 SET (slot_name = none);
DROP SUBSCRIPTION regress_sub1;
]);
$old_sub->stop;
# ------------------------------------------------------
# Check that pg_upgrade fails when max_replication_slots configured in the new
# cluster is less than the number of subscriptions in the old cluster.
# ------------------------------------------------------
my $new_sub1 = PostgreSQL::Test::Cluster->new('new_sub1');
$new_sub1->init;
$new_sub1->append_conf('postgresql.conf', "max_replication_slots = 0");
# pg_upgrade will fail because the new cluster has insufficient
# max_replication_slots.
command_checks_all(
[
'pg_upgrade', '--no-sync',
'-d', $old_sub->data_dir,
'-D', $new_sub1->data_dir,
'-b', $oldbindir,
'-B', $newbindir,
'-s', $new_sub1->host,
'-p', $old_sub->port,
'-P', $new_sub1->port,
$mode, '--check',
],
1,
[
qr/max_replication_slots \(0\) must be greater than or equal to the number of subscriptions \(1\) on the old cluster/
],
[qr//],
'run of pg_upgrade where the new cluster has insufficient max_replication_slots'
);
# Reset max_replication_slots
$new_sub1->append_conf('postgresql.conf', "max_replication_slots = 10");
# Drop the subscription
$old_sub->start;
$old_sub->safe_psql('postgres', "DROP SUBSCRIPTION regress_sub2");
# ------------------------------------------------------
# Check that pg_upgrade refuses to run if:
# a) there's a subscription with tables in a state other than 'r' (ready) or
# 'i' (init) and/or
# b) the subscription has no replication origin.
# ------------------------------------------------------
$publisher->safe_psql(
'postgres', qq[
CREATE TABLE tab_primary_key(id serial PRIMARY KEY);
INSERT INTO tab_primary_key values(1);
CREATE PUBLICATION regress_pub3 FOR TABLE tab_primary_key;
]);
# Insert the same value that is already present in publisher to the primary key
# column of subscriber so that the table sync will fail.
$old_sub->safe_psql(
'postgres', qq[
CREATE TABLE tab_primary_key(id serial PRIMARY KEY);
INSERT INTO tab_primary_key values(1);
CREATE SUBSCRIPTION regress_sub3 CONNECTION '$connstr' PUBLICATION regress_pub3;
]);
# Table will be in 'd' (data is being copied) state as table sync will fail
# because of primary key constraint error.
my $started_query =
"SELECT count(1) = 1 FROM pg_subscription_rel WHERE srsubstate = 'd'";
$old_sub->poll_query_until('postgres', $started_query)
or die
"Timed out while waiting for the table state to become 'd' (datasync)";
# Create another subscription and drop the subscription's replication origin
$old_sub->safe_psql('postgres',
"CREATE SUBSCRIPTION regress_sub4 CONNECTION '$connstr' PUBLICATION regress_pub3 WITH (enabled = false)"
);
$sub_oid = $old_sub->safe_psql('postgres',
"SELECT oid FROM pg_subscription WHERE subname = 'regress_sub4'");
my $reporigin = 'pg_' . qq($sub_oid);
$old_sub->safe_psql('postgres',
"SELECT pg_replication_origin_drop('$reporigin')");
$old_sub->stop;
command_fails(
[
'pg_upgrade', '--no-sync',
'-d', $old_sub->data_dir,
'-D', $new_sub1->data_dir,
'-b', $oldbindir,
'-B', $newbindir,
'-s', $new_sub1->host,
'-p', $old_sub->port,
'-P', $new_sub1->port,
$mode, '--check',
],
'run of pg_upgrade --check for old instance with relation in \'d\' datasync(invalid) state and missing replication origin'
);
# Verify the reason why the subscriber cannot be upgraded
my $sub_relstate_filename;
# Find a txt file that contains a list of tables that cannot be upgraded. We
# cannot predict the file's path because the output directory contains a
# milliseconds timestamp. File::Find::find must be used.
find(
sub {
if ($File::Find::name =~ m/subs_invalid\.txt/)
{
$sub_relstate_filename = $File::Find::name;
}
},
$new_sub1->data_dir . "/pg_upgrade_output.d");
# Check the file content which should have tab_primary_key table in invalid
# state.
like(
slurp_file($sub_relstate_filename),
qr/The table sync state \"d\" is not allowed for database:\"postgres\" subscription:\"regress_sub3\" schema:\"public\" relation:\"tab_primary_key\"/m,
'the previous test failed due to subscription table in invalid state');
# Check the file content which should have regress_sub4 subscription.
like(
slurp_file($sub_relstate_filename),
qr/The replication origin is missing for database:\"postgres\" subscription:\"regress_sub4\"/m,
'the previous test failed due to missing replication origin');
done_testing(); done_testing();