Move routines to manipulate WAL into PostgreSQL::Test::Cluster

These facilities were originally in the recovery TAP test
039_end_of_wal.pl.  A follow-up bug fix with a TAP test doing similar
WAL manipulations requires them, and all these had better not be
duplicated due to their complexity.  The routine names are tweaked to
use "wal" more consistently, similarly to the existing "advance_wal".

In v14 and v13, the new routines are moved to PostgresNode.pm.
039_end_of_wal.pl is updated to use the refactored routines, without
changing its coverage.

Reviewed-by: Alexander Kukushkin
Discussion: https://postgr.es/m/CAFh8B=mozC+e1wGJq0H=0O65goZju+6ab5AU7DEWCSUA2OtwDg@mail.gmail.com
Backpatch-through: 13

1 files changed, 66 insertions, 191 deletions

diff --git a/src/test/recovery/t/039_end_of_wal.pl b/src/test/recovery/t/039_end_of_wal.pl
index ab751eb271d..47f9bb15e03 100644
--- a/src/test/recovery/t/039_end_of_wal.pl
+++ b/src/test/recovery/t/039_end_of_wal.pl
@@ -20,9 +20,6 @@ use integer;    # causes / operator to use integer math
 # we need to know the endianness to do that.
 my $BIG_ENDIAN = pack("L", 0x12345678) eq pack("N", 0x12345678);
 
-# Header size of record header.
-my $RECORD_HEADER_SIZE = 24;
-
 # Fields retrieved from code headers.
 my @scan_result = scan_server_header('access/xlog_internal.h',
 	'#define\s+XLOG_PAGE_MAGIC\s+(\w+)');
@@ -36,64 +33,6 @@ my $WAL_SEGMENT_SIZE;
 my $WAL_BLOCK_SIZE;
 my $TLI;
 
-# Build path of a WAL segment.
-sub wal_segment_path
-{
-	my $node = shift;
-	my $tli = shift;
-	my $segment = shift;
-	my $wal_path =
-	  sprintf("%s/pg_wal/%08X%08X%08X", $node->data_dir, $tli, 0, $segment);
-	return $wal_path;
-}
-
-# Calculate from a LSN (in bytes) its segment number and its offset.
-sub lsn_to_segment_and_offset
-{
-	my $lsn = shift;
-	return ($lsn / $WAL_SEGMENT_SIZE, $lsn % $WAL_SEGMENT_SIZE);
-}
-
-# Write some arbitrary data in WAL for the given segment at LSN.
-# This should be called while the cluster is not running.
-sub write_wal
-{
-	my $node = shift;
-	my $tli = shift;
-	my $lsn = shift;
-	my $data = shift;
-
-	my ($segment, $offset) = lsn_to_segment_and_offset($lsn);
-	my $path = wal_segment_path($node, $tli, $segment);
-
-	open my $fh, "+<:raw", $path or die;
-	seek($fh, $offset, SEEK_SET) or die;
-	print $fh $data;
-	close $fh;
-}
-
-# Emit a WAL record of arbitrary size.  Returns the end LSN of the
-# record inserted, in bytes.
-sub emit_message
-{
-	my $node = shift;
-	my $size = shift;
-	return int(
-		$node->safe_psql(
-			'postgres',
-			"SELECT pg_logical_emit_message(true, '', repeat('a', $size)) - '0/0'"
-		));
-}
-
-# Get the current insert LSN of a node, in bytes.
-sub get_insert_lsn
-{
-	my $node = shift;
-	return int(
-		$node->safe_psql(
-			'postgres', "SELECT pg_current_wal_insert_lsn() - '0/0'"));
-}
-
 # Get GUC value, converted to an int.
 sub get_int_setting
 {
@@ -167,69 +106,6 @@ sub build_page_header
 		$BIG_ENDIAN ? $xlp_pageaddr : 0, $xlp_rem_len);
 }
 
-# Make sure we are far away enough from the end of a page that we could insert
-# a couple of small records.  This inserts a few records of a fixed size, until
-# the threshold gets close enough to the end of the WAL page inserting records
-# to.
-sub advance_out_of_record_splitting_zone
-{
-	my $node = shift;
-
-	my $page_threshold = $WAL_BLOCK_SIZE / 4;
-	my $end_lsn = get_insert_lsn($node);
-	my $page_offset = $end_lsn % $WAL_BLOCK_SIZE;
-	while ($page_offset >= $WAL_BLOCK_SIZE - $page_threshold)
-	{
-		emit_message($node, $page_threshold);
-		$end_lsn = get_insert_lsn($node);
-		$page_offset = $end_lsn % $WAL_BLOCK_SIZE;
-	}
-	return $end_lsn;
-}
-
-# Advance so close to the end of a page that an XLogRecordHeader would not
-# fit on it.
-sub advance_to_record_splitting_zone
-{
-	my $node = shift;
-
-	my $end_lsn = get_insert_lsn($node);
-	my $page_offset = $end_lsn % $WAL_BLOCK_SIZE;
-
-	# Get fairly close to the end of a page in big steps
-	while ($page_offset <= $WAL_BLOCK_SIZE - 512)
-	{
-		emit_message($node, $WAL_BLOCK_SIZE - $page_offset - 256);
-		$end_lsn = get_insert_lsn($node);
-		$page_offset = $end_lsn % $WAL_BLOCK_SIZE;
-	}
-
-	# Calibrate our message size so that we can get closer 8 bytes at
-	# a time.
-	my $message_size = $WAL_BLOCK_SIZE - 80;
-	while ($page_offset <= $WAL_BLOCK_SIZE - $RECORD_HEADER_SIZE)
-	{
-		emit_message($node, $message_size);
-		$end_lsn = get_insert_lsn($node);
-
-		my $old_offset = $page_offset;
-		$page_offset = $end_lsn % $WAL_BLOCK_SIZE;
-
-		# Adjust the message size until it causes 8 bytes changes in
-		# offset, enough to be able to split a record header.
-		my $delta = $page_offset - $old_offset;
-		if ($delta > 8)
-		{
-			$message_size -= 8;
-		}
-		elsif ($delta <= 0)
-		{
-			$message_size += 8;
-		}
-	}
-	return $end_lsn;
-}
-
 # Setup a new node.  The configuration chosen here minimizes the number
 # of arbitrary records that could get generated in a cluster.  Enlarging
 # checkpoint_timeout avoids noise with checkpoint activity.  wal_level
@@ -265,8 +141,8 @@ note "Single-page end-of-WAL detection";
 ###########################################################################
 
 # xl_tot_len is 0 (a common case, we hit trailing zeroes).
-emit_message($node, 0);
-$end_lsn = advance_out_of_record_splitting_zone($node);
+$node->emit_wal(0);
+$end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
 $node->stop('immediate');
 my $log_size = -s $node->logfile;
 $node->start;
@@ -276,10 +152,10 @@ ok( $node->log_contains(
 	"xl_tot_len zero");
 
 # xl_tot_len is < 24 (presumably recycled garbage).
-emit_message($node, 0);
-$end_lsn = advance_out_of_record_splitting_zone($node);
+$node->emit_wal(0);
+$end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn, build_record_header(23));
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE, build_record_header(23));
 $log_size = -s $node->logfile;
 $node->start;
 ok( $node->log_contains(
@@ -289,10 +165,10 @@ ok( $node->log_contains(
 
 # xl_tot_len in final position, not big enough to span into a new page but
 # also not eligible for regular record header validation
-emit_message($node, 0);
-$end_lsn = advance_to_record_splitting_zone($node);
+$node->emit_wal(0);
+$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn, build_record_header(1));
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE, build_record_header(1));
 $log_size = -s $node->logfile;
 $node->start;
 ok( $node->log_contains(
@@ -301,10 +177,10 @@ ok( $node->log_contains(
 	"xl_tot_len short at end-of-page");
 
 # Need more pages, but xl_prev check fails first.
-emit_message($node, 0);
-$end_lsn = advance_out_of_record_splitting_zone($node);
+$node->emit_wal(0);
+$end_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
 $log_size = -s $node->logfile;
 $node->start;
@@ -313,12 +189,12 @@ ok( $node->log_contains(
 	"xl_prev bad");
 
 # xl_crc check fails.
-emit_message($node, 0);
-advance_out_of_record_splitting_zone($node);
-$end_lsn = emit_message($node, 10);
+$node->emit_wal(0);
+$node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
+$end_lsn = $node->emit_wal(10);
 $node->stop('immediate');
 # Corrupt a byte in that record, breaking its CRC.
-write_wal($node, $TLI, $end_lsn - 8, '!');
+$node->write_wal($TLI, $end_lsn - 8, $WAL_SEGMENT_SIZE, '!');
 $log_size = -s $node->logfile;
 $node->start;
 ok( $node->log_contains(
@@ -335,11 +211,11 @@ note "Multi-page end-of-WAL detection, header is not split";
 # written to WAL.
 
 # Good xl_prev, we hit zero page next (zero magic).
-emit_message($node, 0);
-$prev_lsn = advance_out_of_record_splitting_zone($node);
-$end_lsn = emit_message($node, 0);
+$node->emit_wal(0);
+$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
+$end_lsn = $node->emit_wal(0);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
 $log_size = -s $node->logfile;
 $node->start;
@@ -347,16 +223,14 @@ ok($node->log_contains("invalid magic number 0000 .* LSN .*", $log_size),
 	"xlp_magic zero");
 
 # Good xl_prev, we hit garbage page next (bad magic).
-emit_message($node, 0);
-$prev_lsn = advance_out_of_record_splitting_zone($node);
-$end_lsn = emit_message($node, 0);
+$node->emit_wal(0);
+$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
+$end_lsn = $node->emit_wal(0);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
-write_wal(
-	$node, $TLI,
-	start_of_next_page($end_lsn),
-	build_page_header(0xcafe, 0, 1, 0));
+$node->write_wal($TLI, start_of_next_page($end_lsn),
+	$WAL_SEGMENT_SIZE, build_page_header(0xcafe, 0, 1, 0));
 $log_size = -s $node->logfile;
 $node->start;
 ok($node->log_contains("invalid magic number CAFE .* LSN .*", $log_size),
@@ -364,16 +238,14 @@ ok($node->log_contains("invalid magic number CAFE .* LSN .*", $log_size),
 
 # Good xl_prev, we hit typical recycled page (good xlp_magic, bad
 # xlp_pageaddr).
-emit_message($node, 0);
-$prev_lsn = advance_out_of_record_splitting_zone($node);
-$end_lsn = emit_message($node, 0);
+$node->emit_wal(0);
+$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
+$end_lsn = $node->emit_wal(0);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn));
-write_wal(
-	$node, $TLI,
-	start_of_next_page($end_lsn),
-	build_page_header($XLP_PAGE_MAGIC, 0, 1, 0xbaaaaaad));
+$node->write_wal($TLI, start_of_next_page($end_lsn),
+	$WAL_SEGMENT_SIZE, build_page_header($XLP_PAGE_MAGIC, 0, 1, 0xbaaaaaad));
 $log_size = -s $node->logfile;
 $node->start;
 ok( $node->log_contains(
@@ -381,15 +253,16 @@ ok( $node->log_contains(
 	"xlp_pageaddr bad");
 
 # Good xl_prev, xlp_magic, xlp_pageaddr, but bogus xlp_info.
-emit_message($node, 0);
-$prev_lsn = advance_out_of_record_splitting_zone($node);
-$end_lsn = emit_message($node, 0);
+$node->emit_wal(0);
+$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
+$end_lsn = $node->emit_wal(0);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
-write_wal(
-	$node, $TLI,
+$node->write_wal(
+	$TLI,
 	start_of_next_page($end_lsn),
+	$WAL_SEGMENT_SIZE,
 	build_page_header(
 		$XLP_PAGE_MAGIC, 0x1234, 1, start_of_next_page($end_lsn)));
 $log_size = -s $node->logfile;
@@ -399,15 +272,14 @@ ok($node->log_contains("invalid info bits 1234 in .*, LSN .*,", $log_size),
 
 # Good xl_prev, xlp_magic, xlp_pageaddr, but xlp_info doesn't mention
 # continuation record.
-emit_message($node, 0);
-$prev_lsn = advance_out_of_record_splitting_zone($node);
-$end_lsn = emit_message($node, 0);
+$node->emit_wal(0);
+$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
+$end_lsn = $node->emit_wal(0);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
-write_wal(
-	$node, $TLI,
-	start_of_next_page($end_lsn),
+$node->write_wal($TLI, start_of_next_page($end_lsn),
+	$WAL_SEGMENT_SIZE,
 	build_page_header($XLP_PAGE_MAGIC, 0, 1, start_of_next_page($end_lsn)));
 $log_size = -s $node->logfile;
 $node->start;
@@ -416,15 +288,16 @@ ok($node->log_contains("there is no contrecord flag at .*", $log_size),
 
 # Good xl_prev, xlp_magic, xlp_pageaddr, xlp_info but xlp_rem_len doesn't add
 # up.
-emit_message($node, 0);
-$prev_lsn = advance_out_of_record_splitting_zone($node);
-$end_lsn = emit_message($node, 0);
+$node->emit_wal(0);
+$prev_lsn = $node->advance_wal_out_of_record_splitting_zone($WAL_BLOCK_SIZE);
+$end_lsn = $node->emit_wal(0);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn));
-write_wal(
-	$node, $TLI,
+$node->write_wal(
+	$TLI,
 	start_of_next_page($end_lsn),
+	$WAL_SEGMENT_SIZE,
 	build_page_header(
 		$XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD,
 		1, start_of_next_page($end_lsn),
@@ -441,10 +314,10 @@ note "Multi-page, but header is split, so page checks are done first";
 ###########################################################################
 
 # xl_prev is bad and xl_tot_len is too big, but we'll check xlp_magic first.
-emit_message($node, 0);
-$end_lsn = advance_to_record_splitting_zone($node);
+$node->emit_wal(0);
+$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
 $log_size = -s $node->logfile;
 $node->start;
@@ -452,14 +325,15 @@ ok($node->log_contains("invalid magic number 0000 .* LSN .*", $log_size),
 	"xlp_magic zero (split record header)");
 
 # And we'll also check xlp_pageaddr before any header checks.
-emit_message($node, 0);
-$end_lsn = advance_to_record_splitting_zone($node);
+$node->emit_wal(0);
+$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
-write_wal(
-	$node, $TLI,
+$node->write_wal(
+	$TLI,
 	start_of_next_page($end_lsn),
+	$WAL_SEGMENT_SIZE,
 	build_page_header(
 		$XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD, 1, 0xbaaaaaad));
 $log_size = -s $node->logfile;
@@ -470,14 +344,15 @@ ok( $node->log_contains(
 
 # We'll also discover that xlp_rem_len doesn't add up before any
 # header checks,
-emit_message($node, 0);
-$end_lsn = advance_to_record_splitting_zone($node);
+$node->emit_wal(0);
+$end_lsn = $node->advance_wal_to_record_splitting_zone($WAL_BLOCK_SIZE);
 $node->stop('immediate');
-write_wal($node, $TLI, $end_lsn,
+$node->write_wal($TLI, $end_lsn, $WAL_SEGMENT_SIZE,
 	build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef));
-write_wal(
-	$node, $TLI,
+$node->write_wal(
+	$TLI,
 	start_of_next_page($end_lsn),
+	$WAL_SEGMENT_SIZE,
 	build_page_header(
 		$XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD,
 		1, start_of_next_page($end_lsn),