Add end-to-end testing of pg_basebackup's tar-format output.

The existing test script does run pg_basebackup with the -Ft option,
but it makes no real attempt to verify the sanity of the results.
We wouldn't know if the output is incompatible with standard "tar"
programs, nor if the server fails to start from the restored output.
Notably, this means that xlog.c's read_tablespace_map() is not being
meaningfully tested, since that code is used only in the tar-format
case.  (We do have reasonable coverage of restoring from plain-format
output, though it's over in src/test/recovery not here.)

Hence, attempt to untar the output and start a server from it,
rather just hoping it's OK.

This test assumes that the local "tar" has the "-C directory"
switch.  Although that's not promised by POSIX, my research
suggests that all non-extinct tar implementations have it.
Should the buildfarm's opinion differ, we can complicate the
test a bit to avoid requiring that.

Possibly this should be back-patched, but I'm unsure about
whether it could work on Windows before d66b23b.

Author: tglsfdc

src/bin/pg_basebackup/Makefile

@@ -18,6 +18,9 @@ subdir = src/bin/pg_basebackup
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+# make this available to TAP test scripts
+export TAR
+
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 

src/bin/pg_basebackup/t/010_pg_basebackup.pl

@@ -6,7 +6,7 @@
 use File::Path qw(rmtree);
 use PostgresNode;
 use TestLib;
-use Test::More tests => 109;
+use Test::More tests => 110;
 
 program_help_ok('pg_basebackup');
 program_version_ok('pg_basebackup');
@@ -229,6 +229,7 @@
 
 $node->start;
 
+# Test backup of a tablespace using tar format.
 # Create a temporary directory in the system location and symlink it
 # to our physical temp location.  That way we can use shorter names
 # for the tablespace directories, which hopefully won't run afoul of
@@ -242,14 +243,52 @@
 $node->safe_psql('postgres',
 	"CREATE TABLESPACE tblspc1 LOCATION '$realTsDir';");
 $node->safe_psql('postgres',
-	"CREATE TABLE test1 (a int) TABLESPACE tblspc1;");
-$node->command_ok(
-	[ 'pg_basebackup', '-D', "$real_tempdir/tarbackup2", '-Ft' ],
-	'tar format with tablespaces');
-ok(-f "$tempdir/tarbackup2/base.tar", 'backup tar was created');
-my @tblspc_tars = glob "$tempdir/tarbackup2/[0-9]*.tar";
+	    "CREATE TABLE test1 (a int) TABLESPACE tblspc1;"
+	  . "INSERT INTO test1 VALUES (1234);");
+$node->backup('tarbackup2', backup_options => ['-Ft']);
+# empty test1, just so that it's different from the to-be-restored data
+$node->safe_psql('postgres', "TRUNCATE TABLE test1;");
+
+# basic checks on the output
+my $backupdir = $node->backup_dir . '/tarbackup2';
+ok(-f "$backupdir/base.tar",   'backup tar was created');
+ok(-f "$backupdir/pg_wal.tar", 'WAL tar was created');
+my @tblspc_tars = glob "$backupdir/[0-9]*.tar";
 is(scalar(@tblspc_tars), 1, 'one tablespace tar was created');
-rmtree("$tempdir/tarbackup2");
+
+# Try to verify the tar-format backup by restoring it.
+# For this, we use the tar program identified by configure.
+SKIP:
+{
+	my $tar = $ENV{TAR};
+	skip "no tar program available", 1
+	  if (!defined $tar || $tar eq '');
+
+	my $node2 = get_new_node('replica');
+
+	# Recover main data directory
+	$node2->init_from_backup($node, 'tarbackup2', tar_program => $tar);
+
+	# Recover tablespace into a new directory (not where it was!)
+	mkdir "$tempdir/tblspc1replica";
+	my $realRepTsDir = TestLib::perl2host("$shorter_tempdir/tblspc1replica");
+	TestLib::system_or_bail($tar, 'xf', $tblspc_tars[0], '-C', $realRepTsDir);
+
+	# Update tablespace map to point to new directory.
+	# XXX Ideally pg_basebackup would handle this.
+	$tblspc_tars[0] =~ m|/([0-9]*)\.tar$|;
+	my $tblspcoid       = $1;
+	my $escapedRepTsDir = $realRepTsDir;
+	$escapedRepTsDir =~ s/\\/\\\\/g;
+	open my $mapfile, '>', $node2->data_dir . '/tablespace_map';
+	print $mapfile "$tblspcoid $escapedRepTsDir\n";
+	close $mapfile;
+
+	$node2->start;
+	my $result = $node2->safe_psql('postgres', 'SELECT * FROM test1');
+	is($result, '1234', "tablespace data restored from tar-format backup");
+	$node2->stop;
+}
 
 # Create an unlogged table to test that forks other than init are not copied.
 $node->safe_psql('postgres',

src/test/perl/PostgresNode.pm

@@ -538,8 +538,11 @@ sub append_conf
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
-B<< $node->backup_dir >>, including the WAL. WAL files
-fetched at the end of the backup, not streamed.
+B<< $node->backup_dir >>, including the WAL.
+
+By default, WAL files are fetched at the end of the backup, not streamed.
+You can adjust that and other things by passing an array of additional
+B<pg_basebackup> command line options in the keyword parameter backup_options.
 
 You'll have to configure a suitable B<max_wal_senders> on the
 target server since it isn't done by default.
@@ -548,15 +551,16 @@ target server since it isn't done by default.
 
 sub backup
 {
-	my ($self, $backup_name) = @_;
+	my ($self, $backup_name, %params) = @_;
 	my $backup_path = $self->backup_dir . '/' . $backup_name;
 	my $name        = $self->name;
 
 	print "# Taking pg_basebackup $backup_name from node \"$name\"\n";
 	TestLib::system_or_bail(
 		'pg_basebackup', '-D', $backup_path, '-h',
 		$self->host,     '-p', $self->port,  '--checkpoint',
-		'fast',          '--no-sync');
+		'fast',          '--no-sync',
+		@{ $params{backup_options} });
 	print "# Backup finished\n";
 	return;
 }
@@ -650,6 +654,11 @@ of a backup previously created on that node with $node->backup.
 
 Does not start the node after initializing it.
 
+By default, the backup is assumed to be plain format.  To restore from
+a tar-format backup, pass the name of the tar program to use in the
+keyword parameter tar_program.  Note that tablespace tar files aren't
+handled here.
+
 Streaming replication can be enabled on this node by passing the keyword
 parameter has_streaming => 1. This is disabled by default.
 
@@ -687,8 +696,21 @@ sub init_from_backup
 	mkdir $self->archive_dir;
 
 	my $data_path = $self->data_dir;
-	rmdir($data_path);
-	RecursiveCopy::copypath($backup_path, $data_path);
+	if (defined $params{tar_program})
+	{
+		mkdir($data_path);
+		TestLib::system_or_bail($params{tar_program}, 'xf',
+			$backup_path . '/base.tar',
+			'-C', $data_path);
+		TestLib::system_or_bail($params{tar_program}, 'xf',
+			$backup_path . '/pg_wal.tar',
+			'-C', $data_path . '/pg_wal');
+	}
+	else
+	{
+		rmdir($data_path);
+		RecursiveCopy::copypath($backup_path, $data_path);
+	}
 	chmod(0700, $data_path);
 
 	# Base configuration for this node