From b8e288fd70f538c51a924259396a220f2311ccd1 Mon Sep 17 00:00:00 2001 From: benoit Date: Tue, 8 Oct 2024 12:39:41 +0200 Subject: [PATCH 1/4] Add a guc for parallel worker logging The new guc log_parallel_workers controls whether a log message is produced to display information on the number of workers spawned when a parallel query or utility is executed. The default value is `none` which disables logging. `all` displays information for all parallel queries, whereas `shortage` displays information only when the number of workers launched is lower than the number of planned workers. This new parameter can help database administrators and developers diagnose performance issues related to parallelism and optimize the configuration of the system accordingly. --- doc/src/sgml/config.sgml | 18 ++++++++++++++++++ src/backend/access/transam/parallel.c | 19 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 19 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/access/parallel.h | 10 ++++++++++ src/include/utils/guc.h | 1 + 6 files changed, 68 insertions(+) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a8542fe41cec..52533ac52e63 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7912,6 +7912,24 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' + + log_parallel_workers (enum) + + log_parallel_workers configuration parameter + + + + + Controls whether a log message about the number of workers is emitted during the + execution of a parallel query or utility statement. The default value is + none which disables logging. all emits + information for all parallel queries or utilities, whereas shortage + emits information only when the number of workers launched is lower than the number + of planned workers. + + + + log_parameter_max_length (integer) diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c index 94db1ec30126..77a8deff30d9 100644 --- a/src/backend/access/transam/parallel.c +++ b/src/backend/access/transam/parallel.c @@ -1663,3 +1663,22 @@ LookupParallelWorkerFunction(const char *libraryname, const char *funcname) return (parallel_worker_main_type) load_external_function(libraryname, funcname, true, NULL); } + +/* + * If required, emit information about parallel workers usage in + * the logs. + */ +void +LogParallelWorkersIfNeeded(int log_parallel_workers, + int parallel_workers_to_launch, + int parallel_workers_launched) +{ + if ((log_parallel_workers == LOG_PARALLEL_WORKERS_ALL && + parallel_workers_to_launch > 0) || + (log_parallel_workers == LOG_PARALLEL_WORKERS_SHORTAGE && + parallel_workers_to_launch != parallel_workers_launched)) + ereport(LOG, + (errmsg("launched %i parallel workers (planned: %i)", + parallel_workers_launched, + parallel_workers_to_launch))); +} diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index d54df555fba9..9c5f0134e8cc 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -28,6 +28,7 @@ #include "access/commit_ts.h" #include "access/gin.h" +#include "access/parallel.h" #include "access/slru.h" #include "access/toast_compression.h" #include "access/twophase.h" @@ -428,6 +429,13 @@ static const struct config_enum_entry debug_logical_replication_streaming_option {NULL, 0, false} }; +static const struct config_enum_entry log_parallel_workers_options[] = { + {"none", LOG_PARALLEL_WORKERS_NONE, false}, + {"all", LOG_PARALLEL_WORKERS_ALL, false}, + {"shortage", LOG_PARALLEL_WORKERS_SHORTAGE, false}, + {NULL, 0, false} +}; + StaticAssertDecl(lengthof(ssl_protocol_versions_info) == (PG_TLS1_3_VERSION + 2), "array length mismatch"); @@ -531,6 +539,7 @@ int log_min_duration_statement = -1; int log_parameter_max_length = -1; int log_parameter_max_length_on_error = 0; int log_temp_files = -1; +int log_parallel_workers = LOG_PARALLEL_WORKERS_NONE; double log_statement_sample_rate = 1.0; double log_xact_sample_rate = 0; char *backtrace_functions; @@ -5340,6 +5349,16 @@ struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"log_parallel_workers", PGC_SUSET, LOGGING_WHAT, + gettext_noop("Log information about parallel worker usage"), + NULL + }, + &log_parallel_workers, + LOG_PARALLEL_WORKERS_NONE, log_parallel_workers_options, + NULL, NULL, NULL + }, + { {"ssl_min_protocol_version", PGC_SIGHUP, CONN_AUTH_SSL, gettext_noop("Sets the minimum SSL/TLS protocol version to use."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 25fe90a430f4..60f87617ca05 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -638,6 +638,7 @@ #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; # -1 disables, 0 logs all temp files +#log_parallel_workers = none # none, all, shortage #log_timezone = 'GMT' # - Process Title - diff --git a/src/include/access/parallel.h b/src/include/access/parallel.h index f37be6d56909..f865cd755e39 100644 --- a/src/include/access/parallel.h +++ b/src/include/access/parallel.h @@ -53,6 +53,12 @@ typedef struct ParallelWorkerContext shm_toc *toc; } ParallelWorkerContext; +typedef enum { + LOG_PARALLEL_WORKERS_NONE=0, + LOG_PARALLEL_WORKERS_ALL, + LOG_PARALLEL_WORKERS_SHORTAGE, +} log_parallel_workers_option_list; + extern PGDLLIMPORT volatile sig_atomic_t ParallelMessagePending; extern PGDLLIMPORT int ParallelWorkerNumber; extern PGDLLIMPORT bool InitializingParallelWorker; @@ -78,4 +84,8 @@ extern void ParallelWorkerReportLastRecEnd(XLogRecPtr last_xlog_end); extern void ParallelWorkerMain(Datum main_arg); +extern void LogParallelWorkersIfNeeded(int log_parallel_workers, + int parallel_workers_to_launch, + int parallel_workers_launched); + #endif /* PARALLEL_H */ diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index f619100467df..7986dc3e5417 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -279,6 +279,7 @@ extern PGDLLIMPORT int log_temp_files; extern PGDLLIMPORT double log_statement_sample_rate; extern PGDLLIMPORT double log_xact_sample_rate; extern PGDLLIMPORT char *backtrace_functions; +extern PGDLLIMPORT int log_parallel_workers; extern PGDLLIMPORT int temp_file_limit; From 5ec83f9716aead62a4dbcb8f22e2f02f0f015d6b Mon Sep 17 00:00:00 2001 From: benoit Date: Mon, 20 Jan 2025 15:48:32 +0100 Subject: [PATCH 2/4] Setup counters for parallel vacuums This can be used by other patches such as the one for pg_stat_database. --- src/backend/commands/vacuumparallel.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 2b9d548cdeb1..454d4a94eaf7 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -208,6 +208,9 @@ struct ParallelVacuumState int nindexes_parallel_cleanup; int nindexes_parallel_condcleanup; + int nworkers_to_launch; + int nworkers_launched; + /* Buffer access strategy used by leader process */ BufferAccessStrategy bstrategy; @@ -362,6 +365,9 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, if ((vacoptions & VACUUM_OPTION_PARALLEL_COND_CLEANUP) != 0) pvs->nindexes_parallel_condcleanup++; } + pvs->nworkers_to_launch = 0; + pvs->nworkers_launched = 0; + shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_INDEX_STATS, indstats); pvs->indstats = indstats; @@ -738,6 +744,9 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan for (int i = 0; i < pvs->pcxt->nworkers_launched; i++) InstrAccumParallelQuery(&pvs->buffer_usage[i], &pvs->wal_usage[i]); + + pvs->nworkers_to_launch += pvs->pcxt->nworkers_to_launch; + pvs->nworkers_launched += pvs->pcxt->nworkers_launched; } /* From 4be318d23ae97d1c503924aee0d04f083a7a3901 Mon Sep 17 00:00:00 2001 From: benoit Date: Wed, 29 Jan 2025 17:10:57 +0100 Subject: [PATCH 3/4] Implements logging for parallel worker usage in utilities This patch implements logging of parallel worker usage for: * the index cleanup and bulkdelete phases of vacuum; * btree and brin index builds. --- src/backend/access/brin/brin.c | 4 ++++ src/backend/access/nbtree/nbtsort.c | 4 ++++ src/backend/commands/vacuumparallel.c | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index 01e1db7f856b..b69caeed0ad2 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -2552,6 +2552,10 @@ _brin_end_parallel(BrinLeader *brinleader, BrinBuildState *state) /* Shutdown worker processes */ WaitForParallelWorkersToFinish(brinleader->pcxt); + LogParallelWorkersIfNeeded(log_parallel_workers, + brinleader->pcxt->nworkers_to_launch, + brinleader->pcxt->nworkers_launched); + /* * Next, accumulate WAL usage. (This must wait for the workers to finish, * or we might get incomplete data.) diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c index 3794cc924ad4..e48deed90352 100644 --- a/src/backend/access/nbtree/nbtsort.c +++ b/src/backend/access/nbtree/nbtsort.c @@ -1613,6 +1613,10 @@ _bt_end_parallel(BTLeader *btleader) /* Shutdown worker processes */ WaitForParallelWorkersToFinish(btleader->pcxt); + LogParallelWorkersIfNeeded(log_parallel_workers, + btleader->pcxt->nworkers_to_launch, + btleader->pcxt->nworkers_launched); + /* * Next, accumulate WAL usage. (This must wait for the workers to finish, * or we might get incomplete data.) diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 454d4a94eaf7..01dee17741a1 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -443,6 +443,10 @@ parallel_vacuum_end(ParallelVacuumState *pvs, IndexBulkDeleteResult **istats) { Assert(!IsParallelWorker()); + LogParallelWorkersIfNeeded(log_parallel_workers, + pvs->nworkers_to_launch, + pvs->nworkers_launched); + /* Copy the updated statistics */ for (int i = 0; i < pvs->nindexes; i++) { From 2eb6426f856907748d0231d10a020b8b288a5be8 Mon Sep 17 00:00:00 2001 From: benoit Date: Wed, 29 Jan 2025 17:15:25 +0100 Subject: [PATCH 4/4] Implements logging for parallel worker usage in queries --- src/backend/executor/execMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 2da848970be0..798ae3136915 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -560,6 +560,10 @@ standard_ExecutorEnd(QueryDesc *queryDesc) pgstat_update_parallel_workers_stats((PgStat_Counter) estate->es_parallel_workers_to_launch, (PgStat_Counter) estate->es_parallel_workers_launched); + LogParallelWorkersIfNeeded(log_parallel_workers, + estate->es_parallel_workers_to_launch, + estate->es_parallel_workers_launched); + /* * Check that ExecutorFinish was called, unless in EXPLAIN-only mode or if * execution was aborted.