Add injection-point test for new multixact CV usage

Before commit a0e0fb1ba5, multixact.c contained a case in the
multixact-read path where it would loop sleeping 1ms each time until
another multixact-create path completed, which was uncovered by any
tests.  That commit changed the code to rely on a condition variable
instead.  Add a test now, which relies on injection points and "loading"
thereof (because of it being in a critical section), per commit
4b211003ec.

Author: Andrey Borodin <x4mmm@yandex-team.ru>
Reviewed-by: Michaël Paquier <michael@paquier.xyz>
Discussion: https://postgr.es/m/0925F9A9-4D53-4B27-A87E-3D83A757B0E0@yandex-team.ru

src/backend/access/transam/multixact.c

@@ -88,6 +88,7 @@
 #include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/fmgrprotos.h"
+#include "utils/injection_point.h"
 #include "utils/guc_hooks.h"
 #include "utils/memutils.h"
 
@@ -868,6 +869,8 @@ MultiXactIdCreateFromMembers(int nmembers, MultiXactMember *members)
 	 */
 	multi = GetNewMultiXactId(nmembers, &offset);
 
+	INJECTION_POINT("multixact-create-from-members");
+
 	/* Make an XLOG entry describing the new MXID. */
 	xlrec.mid = multi;
 	xlrec.moff = offset;
@@ -1480,6 +1483,8 @@ retry:
 			LWLockRelease(lock);
 			CHECK_FOR_INTERRUPTS();
 
+			INJECTION_POINT("multixact-get-members-cv-sleep");
+
 			ConditionVariableSleep(&MultiXactState->nextoff_cv,
 								   WAIT_EVENT_MULTIXACT_CREATION);
 			slept = true;

src/test/modules/test_slru/Makefile

@@ -3,9 +3,14 @@
 MODULE_big = test_slru
 OBJS = \
 	$(WIN32RES) \
-	test_slru.o
+	test_slru.o \
+	test_multixact.o
 PGFILEDESC = "test_slru - test module for SLRUs"
 
+EXTRA_INSTALL=src/test/modules/injection_points
+export enable_injection_points enable_injection_points
+TAP_TESTS = 1
+
 EXTENSION = test_slru
 DATA = test_slru--1.0.sql
 

src/test/modules/test_slru/meson.build

@@ -2,6 +2,7 @@
 
 test_slru_sources = files(
   'test_slru.c',
+  'test_multixact.c',
 )
 
 if host_system == 'windows'
@@ -32,4 +33,12 @@ tests += {
     'regress_args': ['--temp-config', files('test_slru.conf')],
     'runningcheck': false,
   },
+  'tap': {
+    'env': {
+       'enable_injection_points': get_option('injection_points') ? 'yes' : 'no',
+    },
+    'tests': [
+      't/001_multixact.pl'
+    ],
+  },
 }

src/test/modules/test_slru/t/001_multixact.pl

@@ -0,0 +1,124 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+# This test verifies edge case of reading a multixact:
+# when we have multixact that is followed by exactly one another multixact,
+# and another multixact have no offset yet, we must wait until this offset
+# becomes observable. Previously we used to wait for 1ms in a loop in this
+# case, but now we use CV for this. This test is exercising such a sleep.
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+
+use Test::More;
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+my ($node, $result);
+
+$node = PostgreSQL::Test::Cluster->new('mike');
+$node->init;
+$node->append_conf('postgresql.conf',
+	"shared_preload_libraries = 'test_slru'");
+$node->start;
+$node->safe_psql('postgres', q(CREATE EXTENSION injection_points));
+$node->safe_psql('postgres', q(CREATE EXTENSION test_slru));
+
+# Test for Multixact generation edge case
+$node->safe_psql('postgres',
+	q{select injection_points_attach('test-multixact-read','wait')});
+$node->safe_psql('postgres',
+	q{select injection_points_attach('multixact-get-members-cv-sleep','wait')}
+);
+
+# This session must observe sleep on the condition variable while generating a
+# multixact.  To achieve this it first will create a multixact, then pause
+# before reading it.
+my $observer = $node->background_psql('postgres');
+
+# This query will create a multixact, and hang just before reading it.
+$observer->query_until(
+	qr/start/,
+	q{
+	\echo start
+	SELECT test_read_multixact(test_create_multixact());
+});
+$node->wait_for_event('client backend', 'test-multixact-read');
+
+# This session will create the next Multixact. This is necessary to avoid
+# multixact.c's non-sleeping edge case 1.
+my $creator = $node->background_psql('postgres');
+$node->safe_psql('postgres',
+	q{SELECT injection_points_attach('multixact-create-from-members','wait');}
+);
+
+# We expect this query to hang in the critical section after generating new
+# multixact, but before filling it's offset into SLRU.
+# Running an injection point inside a critical section requires it to be
+# loaded beforehand.
+$creator->query_until(
+	qr/start/, q{
+	\echo start
+	SELECT injection_points_load('multixact-create-from-members');
+	SELECT test_create_multixact();
+});
+
+$node->wait_for_event('client backend', 'multixact-create-from-members');
+
+# Ensure we have the backends waiting that we expect
+is( $node->safe_psql(
+		'postgres',
+		q{SELECT string_agg(wait_event, ', ' ORDER BY wait_event)
+		FROM pg_stat_activity WHERE wait_event_type = 'InjectionPoint'}
+	),
+	'multixact-create-from-members, test-multixact-read',
+	"matching injection point waits");
+
+# Now wake observer to get it to read the initial multixact.  A subsequent
+# multixact already exists, but that one doesn't have an offset assigned, so
+# this will hit multixact.c's edge case 2.
+$node->safe_psql('postgres',
+	q{SELECT injection_points_wakeup('test-multixact-read')});
+$node->wait_for_event('client backend', 'multixact-get-members-cv-sleep');
+
+# Ensure we have the backends waiting that we expect
+is( $node->safe_psql(
+		'postgres',
+		q{SELECT string_agg(wait_event, ', ' ORDER BY wait_event)
+		FROM pg_stat_activity WHERE wait_event_type = 'InjectionPoint'}
+	),
+	'multixact-create-from-members, multixact-get-members-cv-sleep',
+	"matching injection point waits");
+
+# Now we have two backends waiting in multixact-create-from-members and
+# multixact-get-members-cv-sleep.  Also we have 3 injections points set to wait.
+# If we wakeup multixact-get-members-cv-sleep it will happen again, so we must
+# detach it first. So let's detach all injection points, then wake up all
+# backends.
+
+$node->safe_psql('postgres',
+	q{SELECT injection_points_detach('test-multixact-read')});
+$node->safe_psql('postgres',
+	q{SELECT injection_points_detach('multixact-create-from-members')});
+$node->safe_psql('postgres',
+	q{SELECT injection_points_detach('multixact-get-members-cv-sleep')});
+
+$node->safe_psql('postgres',
+	q{SELECT injection_points_wakeup('multixact-create-from-members')});
+$node->safe_psql('postgres',
+	q{SELECT injection_points_wakeup('multixact-get-members-cv-sleep')});
+
+# Background psql will now be able to read the result and disconnect.
+$observer->quit;
+$creator->quit;
+
+$node->stop;
+
+# If we reached this point - everything is OK.
+ok(1);
+done_testing();

src/test/modules/test_slru/test_multixact.c

@@ -0,0 +1,57 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_multixact.c
+ *		Support code for multixact testing
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_slru/test_multixact.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/multixact.h"
+#include "access/xact.h"
+#include "utils/builtins.h"
+#include "utils/injection_point.h"
+
+PG_FUNCTION_INFO_V1(test_create_multixact);
+PG_FUNCTION_INFO_V1(test_read_multixact);
+
+/*
+ * Produces multixact with 2 current xids
+ */
+Datum
+test_create_multixact(PG_FUNCTION_ARGS)
+{
+	MultiXactId id;
+
+	MultiXactIdSetOldestMember();
+	id = MultiXactIdCreate(GetCurrentTransactionId(), MultiXactStatusUpdate,
+						   GetCurrentTransactionId(), MultiXactStatusForShare);
+	PG_RETURN_TRANSACTIONID(id);
+}
+
+/*
+ * Reads given multixact after running an injection point. Discards local cache
+ * to make a real read.  Tailored for multixact testing.
+ */
+Datum
+test_read_multixact(PG_FUNCTION_ARGS)
+{
+	MultiXactId id = PG_GETARG_TRANSACTIONID(0);
+	MultiXactMember *members;
+
+	INJECTION_POINT("test-multixact-read");
+	/* discard caches */
+	AtEOXact_MultiXact();
+
+	if (GetMultiXactIdMembers(id, &members, false, false) == -1)
+		elog(ERROR, "MultiXactId not found");
+
+	PG_RETURN_VOID();
+}

src/test/modules/test_slru/test_slru--1.0.sql

@@ -19,3 +19,9 @@ CREATE OR REPLACE FUNCTION test_slru_page_truncate(bigint) RETURNS VOID
   AS 'MODULE_PATHNAME', 'test_slru_page_truncate' LANGUAGE C;
 CREATE OR REPLACE FUNCTION test_slru_delete_all() RETURNS VOID
   AS 'MODULE_PATHNAME', 'test_slru_delete_all' LANGUAGE C;
+
+
+CREATE OR REPLACE FUNCTION test_create_multixact() RETURNS xid
+  AS 'MODULE_PATHNAME', 'test_create_multixact' LANGUAGE C;
+CREATE OR REPLACE FUNCTION test_read_multixact(xid) RETURNS VOID
+  AS 'MODULE_PATHNAME', 'test_read_multixact'LANGUAGE C;