From 0ad3c60caf5f77edfefaf8850fbba5ea4fe28640 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 25 Jan 2023 14:36:51 +0900 Subject: Rename contrib module basic_archive to basic_wal_module This rename is in preparation for the introduction of recovery modules, where basic_wal_module will be used as a base template for the set of callbacks introduced. The former name did not really reflect all that. Author: Nathan Bossart Discussion: https://postgr.es/m/20221227192449.GA3672473@nathanxps13 --- contrib/Makefile | 2 +- contrib/basic_archive/.gitignore | 4 - contrib/basic_archive/Makefile | 21 -- contrib/basic_archive/basic_archive.c | 370 --------------------- contrib/basic_archive/basic_archive.conf | 4 - contrib/basic_archive/expected/basic_archive.out | 29 -- contrib/basic_archive/meson.build | 34 -- contrib/basic_archive/sql/basic_archive.sql | 22 -- contrib/basic_wal_module/.gitignore | 4 + contrib/basic_wal_module/Makefile | 21 ++ contrib/basic_wal_module/basic_wal_module.c | 370 +++++++++++++++++++++ contrib/basic_wal_module/basic_wal_module.conf | 4 + .../basic_wal_module/expected/basic_wal_module.out | 29 ++ contrib/basic_wal_module/meson.build | 34 ++ contrib/basic_wal_module/sql/basic_wal_module.sql | 22 ++ contrib/meson.build | 2 +- doc/src/sgml/appendix-obsolete-basic-archive.sgml | 25 ++ doc/src/sgml/appendix-obsolete.sgml | 1 + doc/src/sgml/archive-modules.sgml | 2 +- doc/src/sgml/basic-archive.sgml | 81 ----- doc/src/sgml/basic-wal-module.sgml | 81 +++++ doc/src/sgml/contrib.sgml | 2 +- doc/src/sgml/filelist.sgml | 3 +- 23 files changed, 597 insertions(+), 570 deletions(-) delete mode 100644 contrib/basic_archive/.gitignore delete mode 100644 contrib/basic_archive/Makefile delete mode 100644 contrib/basic_archive/basic_archive.c delete mode 100644 contrib/basic_archive/basic_archive.conf delete mode 100644 contrib/basic_archive/expected/basic_archive.out delete mode 100644 contrib/basic_archive/meson.build delete mode 100644 contrib/basic_archive/sql/basic_archive.sql create mode 100644 contrib/basic_wal_module/.gitignore create mode 100644 contrib/basic_wal_module/Makefile create mode 100644 contrib/basic_wal_module/basic_wal_module.c create mode 100644 contrib/basic_wal_module/basic_wal_module.conf create mode 100644 contrib/basic_wal_module/expected/basic_wal_module.out create mode 100644 contrib/basic_wal_module/meson.build create mode 100644 contrib/basic_wal_module/sql/basic_wal_module.sql create mode 100644 doc/src/sgml/appendix-obsolete-basic-archive.sgml delete mode 100644 doc/src/sgml/basic-archive.sgml create mode 100644 doc/src/sgml/basic-wal-module.sgml diff --git a/contrib/Makefile b/contrib/Makefile index bbf220407b0..98acaf8690b 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -9,7 +9,7 @@ SUBDIRS = \ amcheck \ auth_delay \ auto_explain \ - basic_archive \ + basic_wal_module \ basebackup_to_shell \ bloom \ btree_gin \ diff --git a/contrib/basic_archive/.gitignore b/contrib/basic_archive/.gitignore deleted file mode 100644 index 5dcb3ff9723..00000000000 --- a/contrib/basic_archive/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Generated subdirectories -/log/ -/results/ -/tmp_check/ diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile deleted file mode 100644 index 55d299d650c..00000000000 --- a/contrib/basic_archive/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -# contrib/basic_archive/Makefile - -MODULES = basic_archive -PGFILEDESC = "basic_archive - basic archive module" - -REGRESS = basic_archive -REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf -# Disabled because these tests require "shared_preload_libraries=basic_archive", -# which typical installcheck users do not have (e.g. buildfarm clients). -NO_INSTALLCHECK = 1 - -ifdef USE_PGXS -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) -else -subdir = contrib/basic_archive -top_builddir = ../.. -include $(top_builddir)/src/Makefile.global -include $(top_srcdir)/contrib/contrib-global.mk -endif diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c deleted file mode 100644 index 3d29711a316..00000000000 --- a/contrib/basic_archive/basic_archive.c +++ /dev/null @@ -1,370 +0,0 @@ -/*------------------------------------------------------------------------- - * - * basic_archive.c - * - * This file demonstrates a basic archive library implementation that is - * roughly equivalent to the following shell command: - * - * test ! -f /path/to/dest && cp /path/to/src /path/to/dest - * - * One notable difference between this module and the shell command above - * is that this module first copies the file to a temporary destination, - * syncs it to disk, and then durably moves it to the final destination. - * - * Another notable difference is that if /path/to/dest already exists - * but has contents identical to /path/to/src, archiving will succeed, - * whereas the command shown above would fail. This prevents problems if - * a file is successfully archived and then the system crashes before - * a durable record of the success has been made. - * - * Copyright (c) 2022-2023, PostgreSQL Global Development Group - * - * IDENTIFICATION - * contrib/basic_archive/basic_archive.c - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" - -#include -#include -#include - -#include "common/int.h" -#include "miscadmin.h" -#include "postmaster/pgarch.h" -#include "storage/copydir.h" -#include "storage/fd.h" -#include "utils/guc.h" -#include "utils/memutils.h" - -PG_MODULE_MAGIC; - -static char *archive_directory = NULL; -static MemoryContext basic_archive_context; - -static bool basic_archive_configured(void); -static bool basic_archive_file(const char *file, const char *path); -static void basic_archive_file_internal(const char *file, const char *path); -static bool check_archive_directory(char **newval, void **extra, GucSource source); -static bool compare_files(const char *file1, const char *file2); - -/* - * _PG_init - * - * Defines the module's GUC. - */ -void -_PG_init(void) -{ - DefineCustomStringVariable("basic_archive.archive_directory", - gettext_noop("Archive file destination directory."), - NULL, - &archive_directory, - "", - PGC_SIGHUP, - 0, - check_archive_directory, NULL, NULL); - - MarkGUCPrefixReserved("basic_archive"); - - basic_archive_context = AllocSetContextCreate(TopMemoryContext, - "basic_archive", - ALLOCSET_DEFAULT_SIZES); -} - -/* - * _PG_archive_module_init - * - * Returns the module's archiving callbacks. - */ -void -_PG_archive_module_init(ArchiveModuleCallbacks *cb) -{ - AssertVariableIsOfType(&_PG_archive_module_init, ArchiveModuleInit); - - cb->check_configured_cb = basic_archive_configured; - cb->archive_file_cb = basic_archive_file; -} - -/* - * check_archive_directory - * - * Checks that the provided archive directory exists. - */ -static bool -check_archive_directory(char **newval, void **extra, GucSource source) -{ - struct stat st; - - /* - * The default value is an empty string, so we have to accept that value. - * Our check_configured callback also checks for this and prevents - * archiving from proceeding if it is still empty. - */ - if (*newval == NULL || *newval[0] == '\0') - return true; - - /* - * Make sure the file paths won't be too long. The docs indicate that the - * file names to be archived can be up to 64 characters long. - */ - if (strlen(*newval) + 64 + 2 >= MAXPGPATH) - { - GUC_check_errdetail("Archive directory too long."); - return false; - } - - /* - * Do a basic sanity check that the specified archive directory exists. It - * could be removed at some point in the future, so we still need to be - * prepared for it not to exist in the actual archiving logic. - */ - if (stat(*newval, &st) != 0 || !S_ISDIR(st.st_mode)) - { - GUC_check_errdetail("Specified archive directory does not exist."); - return false; - } - - return true; -} - -/* - * basic_archive_configured - * - * Checks that archive_directory is not blank. - */ -static bool -basic_archive_configured(void) -{ - return archive_directory != NULL && archive_directory[0] != '\0'; -} - -/* - * basic_archive_file - * - * Archives one file. - */ -static bool -basic_archive_file(const char *file, const char *path) -{ - sigjmp_buf local_sigjmp_buf; - MemoryContext oldcontext; - - /* - * We run basic_archive_file_internal() in our own memory context so that - * we can easily reset it during error recovery (thus avoiding memory - * leaks). - */ - oldcontext = MemoryContextSwitchTo(basic_archive_context); - - /* - * Since the archiver operates at the bottom of the exception stack, - * ERRORs turn into FATALs and cause the archiver process to restart. - * However, using ereport(ERROR, ...) when there are problems is easy to - * code and maintain. Therefore, we create our own exception handler to - * catch ERRORs and return false instead of restarting the archiver - * whenever there is a failure. - */ - if (sigsetjmp(local_sigjmp_buf, 1) != 0) - { - /* Since not using PG_TRY, must reset error stack by hand */ - error_context_stack = NULL; - - /* Prevent interrupts while cleaning up */ - HOLD_INTERRUPTS(); - - /* Report the error and clear ErrorContext for next time */ - EmitErrorReport(); - FlushErrorState(); - - /* Close any files left open by copy_file() or compare_files() */ - AtEOSubXact_Files(false, InvalidSubTransactionId, InvalidSubTransactionId); - - /* Reset our memory context and switch back to the original one */ - MemoryContextSwitchTo(oldcontext); - MemoryContextReset(basic_archive_context); - - /* Remove our exception handler */ - PG_exception_stack = NULL; - - /* Now we can allow interrupts again */ - RESUME_INTERRUPTS(); - - /* Report failure so that the archiver retries this file */ - return false; - } - - /* Enable our exception handler */ - PG_exception_stack = &local_sigjmp_buf; - - /* Archive the file! */ - basic_archive_file_internal(file, path); - - /* Remove our exception handler */ - PG_exception_stack = NULL; - - /* Reset our memory context and switch back to the original one */ - MemoryContextSwitchTo(oldcontext); - MemoryContextReset(basic_archive_context); - - return true; -} - -static void -basic_archive_file_internal(const char *file, const char *path) -{ - char destination[MAXPGPATH]; - char temp[MAXPGPATH + 256]; - struct stat st; - struct timeval tv; - uint64 epoch; /* milliseconds */ - - ereport(DEBUG3, - (errmsg("archiving \"%s\" via basic_archive", file))); - - snprintf(destination, MAXPGPATH, "%s/%s", archive_directory, file); - - /* - * First, check if the file has already been archived. If it already - * exists and has the same contents as the file we're trying to archive, - * we can return success (after ensuring the file is persisted to disk). - * This scenario is possible if the server crashed after archiving the - * file but before renaming its .ready file to .done. - * - * If the archive file already exists but has different contents, - * something might be wrong, so we just fail. - */ - if (stat(destination, &st) == 0) - { - if (compare_files(path, destination)) - { - ereport(DEBUG3, - (errmsg("archive file \"%s\" already exists with identical contents", - destination))); - - fsync_fname(destination, false); - fsync_fname(archive_directory, true); - - return; - } - - ereport(ERROR, - (errmsg("archive file \"%s\" already exists", destination))); - } - else if (errno != ENOENT) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not stat file \"%s\": %m", destination))); - - /* - * Pick a sufficiently unique name for the temporary file so that a - * collision is unlikely. This helps avoid problems in case a temporary - * file was left around after a crash or another server happens to be - * archiving to the same directory. - */ - gettimeofday(&tv, NULL); - if (pg_mul_u64_overflow((uint64) 1000, (uint64) tv.tv_sec, &epoch) || - pg_add_u64_overflow(epoch, (uint64) (tv.tv_usec / 1000), &epoch)) - elog(ERROR, "could not generate temporary file name for archiving"); - - snprintf(temp, sizeof(temp), "%s/%s.%s.%d." UINT64_FORMAT, - archive_directory, "archtemp", file, MyProcPid, epoch); - - /* - * Copy the file to its temporary destination. Note that this will fail - * if temp already exists. - */ - copy_file(path, temp); - - /* - * Sync the temporary file to disk and move it to its final destination. - * Note that this will overwrite any existing file, but this is only - * possible if someone else created the file since the stat() above. - */ - (void) durable_rename(temp, destination, ERROR); - - ereport(DEBUG1, - (errmsg("archived \"%s\" via basic_archive", file))); -} - -/* - * compare_files - * - * Returns whether the contents of the files are the same. - */ -static bool -compare_files(const char *file1, const char *file2) -{ -#define CMP_BUF_SIZE (4096) - char buf1[CMP_BUF_SIZE]; - char buf2[CMP_BUF_SIZE]; - int fd1; - int fd2; - bool ret = true; - - fd1 = OpenTransientFile(file1, O_RDONLY | PG_BINARY); - if (fd1 < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open file \"%s\": %m", file1))); - - fd2 = OpenTransientFile(file2, O_RDONLY | PG_BINARY); - if (fd2 < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open file \"%s\": %m", file2))); - - for (;;) - { - int nbytes = 0; - int buf1_len = 0; - int buf2_len = 0; - - while (buf1_len < CMP_BUF_SIZE) - { - nbytes = read(fd1, buf1 + buf1_len, CMP_BUF_SIZE - buf1_len); - if (nbytes < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not read file \"%s\": %m", file1))); - else if (nbytes == 0) - break; - - buf1_len += nbytes; - } - - while (buf2_len < CMP_BUF_SIZE) - { - nbytes = read(fd2, buf2 + buf2_len, CMP_BUF_SIZE - buf2_len); - if (nbytes < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not read file \"%s\": %m", file2))); - else if (nbytes == 0) - break; - - buf2_len += nbytes; - } - - if (buf1_len != buf2_len || memcmp(buf1, buf2, buf1_len) != 0) - { - ret = false; - break; - } - else if (buf1_len == 0) - break; - } - - if (CloseTransientFile(fd1) != 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not close file \"%s\": %m", file1))); - - if (CloseTransientFile(fd2) != 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not close file \"%s\": %m", file2))); - - return ret; -} diff --git a/contrib/basic_archive/basic_archive.conf b/contrib/basic_archive/basic_archive.conf deleted file mode 100644 index 7c82a4b82b2..00000000000 --- a/contrib/basic_archive/basic_archive.conf +++ /dev/null @@ -1,4 +0,0 @@ -archive_mode = on -archive_library = 'basic_archive' -basic_archive.archive_directory = '.' -wal_level = replica diff --git a/contrib/basic_archive/expected/basic_archive.out b/contrib/basic_archive/expected/basic_archive.out deleted file mode 100644 index 0015053e0f2..00000000000 --- a/contrib/basic_archive/expected/basic_archive.out +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TABLE test (a INT); -SELECT 1 FROM pg_switch_wal(); - ?column? ----------- - 1 -(1 row) - -DO $$ -DECLARE - archived bool; - loops int := 0; -BEGIN - LOOP - archived := count(*) > 0 FROM pg_ls_dir('.', false, false) a - WHERE a ~ '^[0-9A-F]{24}$'; - IF archived OR loops > 120 * 10 THEN EXIT; END IF; - PERFORM pg_sleep(0.1); - loops := loops + 1; - END LOOP; -END -$$; -SELECT count(*) > 0 FROM pg_ls_dir('.', false, false) a - WHERE a ~ '^[0-9A-F]{24}$'; - ?column? ----------- - t -(1 row) - -DROP TABLE test; diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build deleted file mode 100644 index bc1380e6f66..00000000000 --- a/contrib/basic_archive/meson.build +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2022-2023, PostgreSQL Global Development Group - -basic_archive_sources = files( - 'basic_archive.c', -) - -if host_system == 'windows' - basic_archive_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ - '--NAME', 'basic_archive', - '--FILEDESC', 'basic_archive - basic archive module',]) -endif - -basic_archive = shared_module('basic_archive', - basic_archive_sources, - kwargs: contrib_mod_args, -) -contrib_targets += basic_archive - -tests += { - 'name': 'basic_archive', - 'sd': meson.current_source_dir(), - 'bd': meson.current_build_dir(), - 'regress': { - 'sql': [ - 'basic_archive', - ], - 'regress_args': [ - '--temp-config', files('basic_archive.conf'), - ], - # Disabled because these tests require "shared_preload_libraries=basic_archive", - # which typical runningcheck users do not have (e.g. buildfarm clients). - 'runningcheck': false, - }, -} diff --git a/contrib/basic_archive/sql/basic_archive.sql b/contrib/basic_archive/sql/basic_archive.sql deleted file mode 100644 index 14e236d57ab..00000000000 --- a/contrib/basic_archive/sql/basic_archive.sql +++ /dev/null @@ -1,22 +0,0 @@ -CREATE TABLE test (a INT); -SELECT 1 FROM pg_switch_wal(); - -DO $$ -DECLARE - archived bool; - loops int := 0; -BEGIN - LOOP - archived := count(*) > 0 FROM pg_ls_dir('.', false, false) a - WHERE a ~ '^[0-9A-F]{24}$'; - IF archived OR loops > 120 * 10 THEN EXIT; END IF; - PERFORM pg_sleep(0.1); - loops := loops + 1; - END LOOP; -END -$$; - -SELECT count(*) > 0 FROM pg_ls_dir('.', false, false) a - WHERE a ~ '^[0-9A-F]{24}$'; - -DROP TABLE test; diff --git a/contrib/basic_wal_module/.gitignore b/contrib/basic_wal_module/.gitignore new file mode 100644 index 00000000000..5dcb3ff9723 --- /dev/null +++ b/contrib/basic_wal_module/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/basic_wal_module/Makefile b/contrib/basic_wal_module/Makefile new file mode 100644 index 00000000000..1f88aaf4693 --- /dev/null +++ b/contrib/basic_wal_module/Makefile @@ -0,0 +1,21 @@ +# contrib/basic_wal_module/Makefile + +MODULES = basic_wal_module +PGFILEDESC = "basic_wal_module - basic write-ahead log module" + +REGRESS = basic_wal_module +REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_wal_module/basic_wal_module.conf +# Disabled because these tests require "shared_preload_libraries=basic_wal_module", +# which typical installcheck users do not have (e.g. buildfarm clients). +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/basic_wal_module +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/basic_wal_module/basic_wal_module.c b/contrib/basic_wal_module/basic_wal_module.c new file mode 100644 index 00000000000..78c36656a8b --- /dev/null +++ b/contrib/basic_wal_module/basic_wal_module.c @@ -0,0 +1,370 @@ +/*------------------------------------------------------------------------- + * + * basic_wal_module.c + * + * This file demonstrates a basic archive library implementation that is + * roughly equivalent to the following shell command: + * + * test ! -f /path/to/dest && cp /path/to/src /path/to/dest + * + * One notable difference between this module and the shell command above + * is that this module first copies the file to a temporary destination, + * syncs it to disk, and then durably moves it to the final destination. + * + * Another notable difference is that if /path/to/dest already exists + * but has contents identical to /path/to/src, archiving will succeed, + * whereas the command shown above would fail. This prevents problems if + * a file is successfully archived and then the system crashes before + * a durable record of the success has been made. + * + * Copyright (c) 2022-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/basic_wal_module/basic_wal_module.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "common/int.h" +#include "miscadmin.h" +#include "postmaster/pgarch.h" +#include "storage/copydir.h" +#include "storage/fd.h" +#include "utils/guc.h" +#include "utils/memutils.h" + +PG_MODULE_MAGIC; + +static char *archive_directory = NULL; +static MemoryContext basic_wal_module_context; + +static bool basic_archive_configured(void); +static bool basic_archive_file(const char *file, const char *path); +static void basic_archive_file_internal(const char *file, const char *path); +static bool check_archive_directory(char **newval, void **extra, GucSource source); +static bool compare_files(const char *file1, const char *file2); + +/* + * _PG_init + * + * Defines the module's GUC. + */ +void +_PG_init(void) +{ + DefineCustomStringVariable("basic_wal_module.archive_directory", + gettext_noop("Archive file destination directory."), + NULL, + &archive_directory, + "", + PGC_SIGHUP, + 0, + check_archive_directory, NULL, NULL); + + MarkGUCPrefixReserved("basic_wal_module"); + + basic_wal_module_context = AllocSetContextCreate(TopMemoryContext, + "basic_wal_module", + ALLOCSET_DEFAULT_SIZES); +} + +/* + * _PG_archive_module_init + * + * Returns the module's archiving callbacks. + */ +void +_PG_archive_module_init(ArchiveModuleCallbacks *cb) +{ + AssertVariableIsOfType(&_PG_archive_module_init, ArchiveModuleInit); + + cb->check_configured_cb = basic_archive_configured; + cb->archive_file_cb = basic_archive_file; +} + +/* + * check_archive_directory + * + * Checks that the provided archive directory exists. + */ +static bool +check_archive_directory(char **newval, void **extra, GucSource source) +{ + struct stat st; + + /* + * The default value is an empty string, so we have to accept that value. + * Our check_configured callback also checks for this and prevents + * archiving from proceeding if it is still empty. + */ + if (*newval == NULL || *newval[0] == '\0') + return true; + + /* + * Make sure the file paths won't be too long. The docs indicate that the + * file names to be archived can be up to 64 characters long. + */ + if (strlen(*newval) + 64 + 2 >= MAXPGPATH) + { + GUC_check_errdetail("Archive directory too long."); + return false; + } + + /* + * Do a basic sanity check that the specified archive directory exists. It + * could be removed at some point in the future, so we still need to be + * prepared for it not to exist in the actual archiving logic. + */ + if (stat(*newval, &st) != 0 || !S_ISDIR(st.st_mode)) + { + GUC_check_errdetail("Specified archive directory does not exist."); + return false; + } + + return true; +} + +/* + * basic_archive_configured + * + * Checks that archive_directory is not blank. + */ +static bool +basic_archive_configured(void) +{ + return archive_directory != NULL && archive_directory[0] != '\0'; +} + +/* + * basic_archive_file + * + * Archives one file. + */ +static bool +basic_archive_file(const char *file, const char *path) +{ + sigjmp_buf local_sigjmp_buf; + MemoryContext oldcontext; + + /* + * We run basic_archive_file_internal() in our own memory context so that + * we can easily reset it during error recovery (thus avoiding memory + * leaks). + */ + oldcontext = MemoryContextSwitchTo(basic_wal_module_context); + + /* + * Since the archiver operates at the bottom of the exception stack, + * ERRORs turn into FATALs and cause the archiver process to restart. + * However, using ereport(ERROR, ...) when there are problems is easy to + * code and maintain. Therefore, we create our own exception handler to + * catch ERRORs and return false instead of restarting the archiver + * whenever there is a failure. + */ + if (sigsetjmp(local_sigjmp_buf, 1) != 0) + { + /* Since not using PG_TRY, must reset error stack by hand */ + error_context_stack = NULL; + + /* Prevent interrupts while cleaning up */ + HOLD_INTERRUPTS(); + + /* Report the error and clear ErrorContext for next time */ + EmitErrorReport(); + FlushErrorState(); + + /* Close any files left open by copy_file() or compare_files() */ + AtEOSubXact_Files(false, InvalidSubTransactionId, InvalidSubTransactionId); + + /* Reset our memory context and switch back to the original one */ + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(basic_wal_module_context); + + /* Remove our exception handler */ + PG_exception_stack = NULL; + + /* Now we can allow interrupts again */ + RESUME_INTERRUPTS(); + + /* Report failure so that the archiver retries this file */ + return false; + } + + /* Enable our exception handler */ + PG_exception_stack = &local_sigjmp_buf; + + /* Archive the file! */ + basic_archive_file_internal(file, path); + + /* Remove our exception handler */ + PG_exception_stack = NULL; + + /* Reset our memory context and switch back to the original one */ + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(basic_wal_module_context); + + return true; +} + +static void +basic_archive_file_internal(const char *file, const char *path) +{ + char destination[MAXPGPATH]; + char temp[MAXPGPATH + 256]; + struct stat st; + struct timeval tv; + uint64 epoch; /* milliseconds */ + + ereport(DEBUG3, + (errmsg("archiving \"%s\" via basic_wal_module", file))); + + snprintf(destination, MAXPGPATH, "%s/%s", archive_directory, file); + + /* + * First, check if the file has already been archived. If it already + * exists and has the same contents as the file we're trying to archive, + * we can return success (after ensuring the file is persisted to disk). + * This scenario is possible if the server crashed after archiving the + * file but before renaming its .ready file to .done. + * + * If the archive file already exists but has different contents, + * something might be wrong, so we just fail. + */ + if (stat(destination, &st) == 0) + { + if (compare_files(path, destination)) + { + ereport(DEBUG3, + (errmsg("archive file \"%s\" already exists with identical contents", + destination))); + + fsync_fname(destination, false); + fsync_fname(archive_directory, true); + + return; + } + + ereport(ERROR, + (errmsg("archive file \"%s\" already exists", destination))); + } + else if (errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", destination))); + + /* + * Pick a sufficiently unique name for the temporary file so that a + * collision is unlikely. This helps avoid problems in case a temporary + * file was left around after a crash or another server happens to be + * archiving to the same directory. + */ + gettimeofday(&tv, NULL); + if (pg_mul_u64_overflow((uint64) 1000, (uint64) tv.tv_sec, &epoch) || + pg_add_u64_overflow(epoch, (uint64) (tv.tv_usec / 1000), &epoch)) + elog(ERROR, "could not generate temporary file name for archiving"); + + snprintf(temp, sizeof(temp), "%s/%s.%s.%d." UINT64_FORMAT, + archive_directory, "archtemp", file, MyProcPid, epoch); + + /* + * Copy the file to its temporary destination. Note that this will fail + * if temp already exists. + */ + copy_file(path, temp); + + /* + * Sync the temporary file to disk and move it to its final destination. + * Note that this will overwrite any existing file, but this is only + * possible if someone else created the file since the stat() above. + */ + (void) durable_rename(temp, destination, ERROR); + + ereport(DEBUG1, + (errmsg("archived \"%s\" via basic_wal_module", file))); +} + +/* + * compare_files + * + * Returns whether the contents of the files are the same. + */ +static bool +compare_files(const char *file1, const char *file2) +{ +#define CMP_BUF_SIZE (4096) + char buf1[CMP_BUF_SIZE]; + char buf2[CMP_BUF_SIZE]; + int fd1; + int fd2; + bool ret = true; + + fd1 = OpenTransientFile(file1, O_RDONLY | PG_BINARY); + if (fd1 < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", file1))); + + fd2 = OpenTransientFile(file2, O_RDONLY | PG_BINARY); + if (fd2 < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", file2))); + + for (;;) + { + int nbytes = 0; + int buf1_len = 0; + int buf2_len = 0; + + while (buf1_len < CMP_BUF_SIZE) + { + nbytes = read(fd1, buf1 + buf1_len, CMP_BUF_SIZE - buf1_len); + if (nbytes < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", file1))); + else if (nbytes == 0) + break; + + buf1_len += nbytes; + } + + while (buf2_len < CMP_BUF_SIZE) + { + nbytes = read(fd2, buf2 + buf2_len, CMP_BUF_SIZE - buf2_len); + if (nbytes < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", file2))); + else if (nbytes == 0) + break; + + buf2_len += nbytes; + } + + if (buf1_len != buf2_len || memcmp(buf1, buf2, buf1_len) != 0) + { + ret = false; + break; + } + else if (buf1_len == 0) + break; + } + + if (CloseTransientFile(fd1) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", file1))); + + if (CloseTransientFile(fd2) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", file2))); + + return ret; +} diff --git a/contrib/basic_wal_module/basic_wal_module.conf b/contrib/basic_wal_module/basic_wal_module.conf new file mode 100644 index 00000000000..9a4ffacf683 --- /dev/null +++ b/contrib/basic_wal_module/basic_wal_module.conf @@ -0,0 +1,4 @@ +archive_mode = on +archive_library = 'basic_wal_module' +basic_wal_module.archive_directory = '.' +wal_level = replica diff --git a/contrib/basic_wal_module/expected/basic_wal_module.out b/contrib/basic_wal_module/expected/basic_wal_module.out new file mode 100644 index 00000000000..0015053e0f2 --- /dev/null +++ b/contrib/basic_wal_module/expected/basic_wal_module.out @@ -0,0 +1,29 @@ +CREATE TABLE test (a INT); +SELECT 1 FROM pg_switch_wal(); + ?column? +---------- + 1 +(1 row) + +DO $$ +DECLARE + archived bool; + loops int := 0; +BEGIN + LOOP + archived := count(*) > 0 FROM pg_ls_dir('.', false, false) a + WHERE a ~ '^[0-9A-F]{24}$'; + IF archived OR loops > 120 * 10 THEN EXIT; END IF; + PERFORM pg_sleep(0.1); + loops := loops + 1; + END LOOP; +END +$$; +SELECT count(*) > 0 FROM pg_ls_dir('.', false, false) a + WHERE a ~ '^[0-9A-F]{24}$'; + ?column? +---------- + t +(1 row) + +DROP TABLE test; diff --git a/contrib/basic_wal_module/meson.build b/contrib/basic_wal_module/meson.build new file mode 100644 index 00000000000..59939d71c4f --- /dev/null +++ b/contrib/basic_wal_module/meson.build @@ -0,0 +1,34 @@ +# Copyright (c) 2022-2023, PostgreSQL Global Development Group + +basic_wal_module_sources = files( + 'basic_wal_module.c', +) + +if host_system == 'windows' + basic_wal_module_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'basic_wal_module', + '--FILEDESC', 'basic_wal_module - basic write-ahead log module',]) +endif + +basic_wal_module = shared_module('basic_wal_module', + basic_wal_module_sources, + kwargs: contrib_mod_args, +) +contrib_targets += basic_wal_module + +tests += { + 'name': 'basic_wal_module', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'basic_wal_module', + ], + 'regress_args': [ + '--temp-config', files('basic_wal_module.conf'), + ], + # Disabled because these tests require "shared_preload_libraries=basic_wal_module", + # which typical runningcheck users do not have (e.g. buildfarm clients). + 'runningcheck': false, + }, +} diff --git a/contrib/basic_wal_module/sql/basic_wal_module.sql b/contrib/basic_wal_module/sql/basic_wal_module.sql new file mode 100644 index 00000000000..14e236d57ab --- /dev/null +++ b/contrib/basic_wal_module/sql/basic_wal_module.sql @@ -0,0 +1,22 @@ +CREATE TABLE test (a INT); +SELECT 1 FROM pg_switch_wal(); + +DO $$ +DECLARE + archived bool; + loops int := 0; +BEGIN + LOOP + archived := count(*) > 0 FROM pg_ls_dir('.', false, false) a + WHERE a ~ '^[0-9A-F]{24}$'; + IF archived OR loops > 120 * 10 THEN EXIT; END IF; + PERFORM pg_sleep(0.1); + loops := loops + 1; + END LOOP; +END +$$; + +SELECT count(*) > 0 FROM pg_ls_dir('.', false, false) a + WHERE a ~ '^[0-9A-F]{24}$'; + +DROP TABLE test; diff --git a/contrib/meson.build b/contrib/meson.build index bd4a57c43c0..2db77a18d74 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -11,7 +11,7 @@ subdir('adminpack') subdir('amcheck') subdir('auth_delay') subdir('auto_explain') -subdir('basic_archive') +subdir('basic_wal_module') subdir('bloom') subdir('basebackup_to_shell') subdir('bool_plperl') diff --git a/doc/src/sgml/appendix-obsolete-basic-archive.sgml b/doc/src/sgml/appendix-obsolete-basic-archive.sgml new file mode 100644 index 00000000000..5070b3b6abf --- /dev/null +++ b/doc/src/sgml/appendix-obsolete-basic-archive.sgml @@ -0,0 +1,25 @@ + + + + + <command>basic_archive</command> renamed to <command>basic_wal_module</command> + + + basic_archive + basic_wal_module + + + + PostgreSQL 15 provided an archive module named + basic_archive + basic_archive. + This module was renamed to basic_wal_module. See + for documentation of + basic_wal_module, and see + the release notes for PostgreSQL 16 + for details on this change. + + + diff --git a/doc/src/sgml/appendix-obsolete.sgml b/doc/src/sgml/appendix-obsolete.sgml index b1a00c8ce67..87c1b1020df 100644 --- a/doc/src/sgml/appendix-obsolete.sgml +++ b/doc/src/sgml/appendix-obsolete.sgml @@ -38,5 +38,6 @@ &obsolete-pgxlogdump; &obsolete-pgresetxlog; &obsolete-pgreceivexlog; + &obsolete-basic-archive; diff --git a/doc/src/sgml/archive-modules.sgml b/doc/src/sgml/archive-modules.sgml index ef02051f7f0..1a32006e2c7 100644 --- a/doc/src/sgml/archive-modules.sgml +++ b/doc/src/sgml/archive-modules.sgml @@ -32,7 +32,7 @@ - The contrib/basic_archive module contains a working + The contrib/basic_wal_module module contains a working example, which demonstrates some useful techniques. diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml deleted file mode 100644 index b4d43ced203..00000000000 --- a/doc/src/sgml/basic-archive.sgml +++ /dev/null @@ -1,81 +0,0 @@ - - - - basic_archive — an example WAL archive module - - - basic_archive - - - - basic_archive is an example of an archive module. This - module copies completed WAL segment files to the specified directory. This - may not be especially useful, but it can serve as a starting point for - developing your own archive module. For more information about archive - modules, see . - - - - In order to function, this module must be loaded via - , and - must be enabled. - - - - Configuration Parameters - - - - - basic_archive.archive_directory (string) - - basic_archive.archive_directory configuration parameter - - - - - The directory where the server should copy WAL segment files. This - directory must already exist. The default is an empty string, which - effectively halts WAL archiving, but if - is enabled, the server will accumulate WAL segment files in the - expectation that a value will soon be provided. - - - - - - - These parameters must be set in postgresql.conf. - Typical usage might be: - - - -# postgresql.conf -archive_mode = 'on' -archive_library = 'basic_archive' -basic_archive.archive_directory = '/path/to/archive/directory' - - - - - Notes - - - Server crashes may leave temporary files with the prefix - archtemp in the archive directory. It is recommended to - delete such files before restarting the server after a crash. It is safe to - remove such files while the server is running as long as they are unrelated - to any archiving still in progress, but users should use extra caution when - doing so. - - - - - Author - - - Nathan Bossart - - - - diff --git a/doc/src/sgml/basic-wal-module.sgml b/doc/src/sgml/basic-wal-module.sgml new file mode 100644 index 00000000000..c418b01eb85 --- /dev/null +++ b/doc/src/sgml/basic-wal-module.sgml @@ -0,0 +1,81 @@ + + + + basic_wal_module — an example write-ahead log module + + + basic_wal_module + + + + basic_wal_module is an example of an archive module. + This module copies completed WAL segment files to the specified directory. + This may not be especially useful, but it can serve as a starting point for + developing your own archive module. For more information about archive + modules, see . + + + + In order to function, this module must be loaded via + , and + must be enabled. + + + + Configuration Parameters + + + + + basic_wal_module.archive_directory (string) + + basic_wal_module.archive_directory configuration parameter + + + + + The directory where the server should copy WAL segment files. This + directory must already exist. The default is an empty string, which + effectively halts WAL archiving, but if + is enabled, the server will accumulate WAL segment files in the + expectation that a value will soon be provided. + + + + + + + These parameters must be set in postgresql.conf. + Typical usage might be: + + + +# postgresql.conf +archive_mode = 'on' +archive_library = 'basic_wal_module' +basic_wal_module.archive_directory = '/path/to/archive/directory' + + + + + Notes + + + Server crashes may leave temporary files with the prefix + archtemp in the archive directory. It is recommended to + delete such files before restarting the server after a crash. It is safe to + remove such files while the server is running as long as they are unrelated + to any archiving still in progress, but users should use extra caution when + doing so. + + + + + Author + + + Nathan Bossart + + + + diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 12c79b798ba..3225284eead 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -105,7 +105,7 @@ CREATE EXTENSION extension_name; &auth-delay; &auto-explain; &basebackup-to-shell; - &basic-archive; + &basic-wal-module; &bloom; &btree-gin; &btree-gist; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 0d6be9a2faf..2d36c34ce81 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -116,7 +116,7 @@ - + @@ -200,3 +200,4 @@ + -- cgit v1.2.3