Make WAL segment size configurable at initdb time.

For performance reasons a larger segment size than the default 16MB
can be useful. A larger segment size has two main benefits: Firstly,
in setups using archiving, it makes it easier to write scripts that
can keep up with higher amounts of WAL, secondly, the WAL has to be
written and synced to disk less frequently.

But at the same time large segment size are disadvantageous for
smaller databases. So far the segment size had to be configured at
compile time, often making it unrealistic to choose one fitting to a
particularly load. Therefore change it to a initdb time setting.

This includes a breaking changes to the xlogreader.h API, which now
requires the current segment size to be configured.  For that and
similar reasons a number of binaries had to be taught how to recognize
the current segment size.

Author: Beena Emerson, editorialized by Andres Freund
Reviewed-By: Andres Freund, David Steele, Kuntal Ghosh, Michael
    Paquier, Peter Eisentraut, Robert Hass, Tushar Ahuja
Discussion: https://postgr.es/m/CAOG9ApEAcQ--1ieKbhFzXSQPw_YLmepaa4hNdnY5+ZULpt81Mw@mail.gmail.com

6 files changed, 60 insertions, 38 deletions

diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 7213af0e813..0f2b8bd53fb 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -94,6 +94,7 @@ extern PGDLLIMPORT XLogRecPtr XactLastCommitEnd;
 extern bool reachedConsistency;
 
 /* these variables are GUC parameters related to XLOG */
+extern int	wal_segment_size;
 extern int	min_wal_size_mb;
 extern int	max_wal_size_mb;
 extern int	wal_keep_segments;
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 7453dcbd0eb..22a8e63658e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -85,15 +85,27 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
 #define XLogPageHeaderSize(hdr)		\
 	(((hdr)->xlp_info & XLP_LONG_HEADER) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD)
 
-/*
- * The XLOG is split into WAL segments (physical files) of the size indicated
- * by XLOG_SEG_SIZE.
- */
-#define XLogSegSize		((uint32) XLOG_SEG_SIZE)
-#define XLogSegmentsPerXLogId	(UINT64CONST(0x100000000) / XLOG_SEG_SIZE)
+/* wal_segment_size can range from 1MB to 1GB */
+#define WalSegMinSize 1024 * 1024
+#define WalSegMaxSize 1024 * 1024 * 1024
+/* default number of min and max wal segments */
+#define DEFAULT_MIN_WAL_SEGS 5
+#define DEFAULT_MAX_WAL_SEGS 64
+
+/* check that the given size is a valid wal_segment_size */
+#define IsPowerOf2(x) (x > 0 && ((x) & ((x)-1)) == 0)
+#define IsValidWalSegSize(size) \
+	 (IsPowerOf2(size) && \
+	 ((size) >= WalSegMinSize && (size) <= WalSegMaxSize))
+
+#define XLogSegmentsPerXLogId(wal_segsz_bytes)	\
+	(UINT64CONST(0x100000000) / (wal_segsz_bytes))
+
+#define XLogSegNoOffsetToRecPtr(segno, offset, dest, wal_segsz_bytes) \
+		(dest) = (segno) * (wal_segsz_bytes) + (offset)
 
-#define XLogSegNoOffsetToRecPtr(segno, offset, dest) \
-		(dest) = (segno) * XLOG_SEG_SIZE + (offset)
+#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)	\
+	((xlogptr) & ((wal_segsz_bytes) - 1))
 
 /*
  * Compute a segment number from an XLogRecPtr.
@@ -103,11 +115,11 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
  * for deciding which segment to write given a pointer to a record end,
  * for example.
  */
-#define XLByteToSeg(xlrp, logSegNo) \
-	logSegNo = (xlrp) / XLogSegSize
+#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) \
+	logSegNo = (xlrp) / (wal_segsz_bytes)
 
-#define XLByteToPrevSeg(xlrp, logSegNo) \
-	logSegNo = ((xlrp) - 1) / XLogSegSize
+#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes) \
+	logSegNo = ((xlrp) - 1) / (wal_segsz_bytes)
 
 /*
  * Is an XLogRecPtr within a particular XLOG segment?
@@ -115,11 +127,11 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
  * For XLByteInSeg, do the computation at face value.  For XLByteInPrevSeg,
  * a boundary byte is taken to be in the previous segment.
  */
-#define XLByteInSeg(xlrp, logSegNo) \
-	(((xlrp) / XLogSegSize) == (logSegNo))
+#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) \
+	(((xlrp) / (wal_segsz_bytes)) == (logSegNo))
 
-#define XLByteInPrevSeg(xlrp, logSegNo) \
-	((((xlrp) - 1) / XLogSegSize) == (logSegNo))
+#define XLByteInPrevSeg(xlrp, logSegNo, wal_segsz_bytes) \
+	((((xlrp) - 1) / (wal_segsz_bytes)) == (logSegNo))
 
 /* Check if an XLogRecPtr value is in a plausible range */
 #define XRecOffIsValid(xlrp) \
@@ -140,10 +152,10 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
 /* Length of XLog file name */
 #define XLOG_FNAME_LEN	   24
 
-#define XLogFileName(fname, tli, logSegNo)	\
+#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)	\
 	snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli,		\
-			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \
-			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId))
+			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)))
 
 #define XLogFileNameById(fname, tli, log, seg)	\
 	snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli, log, seg)
@@ -162,18 +174,18 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
 	 strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN &&		\
 	 strcmp((fname) + XLOG_FNAME_LEN, ".partial") == 0)
 
-#define XLogFromFileName(fname, tli, logSegNo)	\
+#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)	\
 	do {												\
 		uint32 log;										\
 		uint32 seg;										\
 		sscanf(fname, "%08X%08X%08X", tli, &log, &seg); \
-		*logSegNo = (uint64) log * XLogSegmentsPerXLogId + seg; \
+		*logSegNo = (uint64) log * XLogSegmentsPerXLogId(wal_segsz_bytes) + seg; \
 	} while (0)
 
-#define XLogFilePath(path, tli, logSegNo)	\
-	snprintf(path, MAXPGPATH, XLOGDIR "/%08X%08X%08X", tli,				\
-			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId),				\
-			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId))
+#define XLogFilePath(path, tli, logSegNo, wal_segsz_bytes)	\
+	snprintf(path, MAXPGPATH, XLOGDIR "/%08X%08X%08X", tli,	\
+			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)))
 
 #define TLHistoryFileName(fname, tli)	\
 	snprintf(fname, MAXFNAMELEN, "%08X.history", tli)
@@ -189,20 +201,22 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
 #define StatusFilePath(path, xlog, suffix)	\
 	snprintf(path, MAXPGPATH, XLOGDIR "/archive_status/%s%s", xlog, suffix)
 
-#define BackupHistoryFileName(fname, tli, logSegNo, offset) \
+#define BackupHistoryFileName(fname, tli, logSegNo, startpoint, wal_segsz_bytes) \
 	snprintf(fname, MAXFNAMELEN, "%08X%08X%08X.%08X.backup", tli, \
-			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId),		  \
-			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId), offset)
+			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+			 (uint32) (XLogSegmentOffset(startpoint, wal_segsz_bytes)))
 
 #define IsBackupHistoryFileName(fname) \
 	(strlen(fname) > XLOG_FNAME_LEN && \
 	 strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
 	 strcmp((fname) + strlen(fname) - strlen(".backup"), ".backup") == 0)
 
-#define BackupHistoryFilePath(path, tli, logSegNo, offset)	\
+#define BackupHistoryFilePath(path, tli, logSegNo, startpoint, wal_segsz_bytes)	\
 	snprintf(path, MAXPGPATH, XLOGDIR "/%08X%08X%08X.%08X.backup", tli, \
-			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \
-			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId), offset)
+			 (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+			 (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+			 (uint32) (XLogSegmentOffset((startpoint), wal_segsz_bytes)))
 
 /*
  * Information logged when we detect a change in one of the parameters
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 76715983349..3a9ebd43543 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -74,6 +74,11 @@ struct XLogReaderState
 	 */
 
 	/*
+	 * Segment size of the to-be-parsed data (mandatory).
+	 */
+	int			wal_segment_size;
+
+	/*
 	 * Data input callback (mandatory).
 	 *
 	 * This callback shall read at least reqLen valid bytes of the xlog page
@@ -189,7 +194,8 @@ struct XLogReaderState
 };
 
 /* Get a new XLogReader */
-extern XLogReaderState *XLogReaderAllocate(XLogPageReadCB pagereadfunc,
+extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
+				   XLogPageReadCB pagereadfunc,
 				   void *private_data);
 
 /* Free an XLogReader */
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 1ec03caf5fb..3fed3b64318 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -21,7 +21,7 @@
 
 
 /* Version identifier for this pg_control format */
-#define PG_CONTROL_VERSION	1002
+#define PG_CONTROL_VERSION	1003
 
 /* Nonce key length, see below */
 #define MOCK_AUTH_NONCE_LEN		32
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 579d195663c..85deb29d831 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -895,11 +895,6 @@
    */
 #undef XLOG_BLCKSZ
 
-/* XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2
-   and larger than XLOG_BLCKSZ (preferably, a great deal larger than
-   XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
-#undef XLOG_SEG_SIZE
-
 
 
 /* Number of bits in a file offset, on hosts where this is settable. */
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f3b35297d18..9615a389af2 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -14,6 +14,12 @@
  */
 
 /*
+ * This is default value for wal_segment_size to be used at intidb when run
+ * without --walsegsize option. Must be a valid segment size.
+ */
+#define DEFAULT_XLOG_SEG_SIZE	(16*1024*1024)
+
+/*
  * Maximum length for identifiers (e.g. table names, column names,
  * function names).  Names actually are limited to one less byte than this,
  * because the length must include a trailing zero byte.