From 40965dfef0f26a92249cda7a956bd03c9358a026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Herrera?= Date: Sat, 26 Jul 2025 19:57:26 +0200 Subject: [PATCH v21 2/6] Add REPACK command REPACK absorbs the functionality of VACUUM FULL and CLUSTER in a single command. Because this functionality is completely different from regular VACUUM, having it separate from VACUUM makes it easier for users to understand; as for CLUSTER, the term is heavily overloaded in the TI world and even in Postgres itself, so it's good that we can avoid it. This also adds pg_repackdb, a new utility that can invoke the new commands. This is heavily based on vacuumdb. We may still change the implementation, depending on how does Windows like this one. Author: Antonin Houska Reviewed-by: To fill in Discussion: https://postgr.es/m/82651.1720540558@antos Discussion: https://postgr.es/m/202507262156.sb455angijk6@alvherre.pgsql --- doc/src/sgml/monitoring.sgml | 223 ++++++- doc/src/sgml/ref/allfiles.sgml | 2 + doc/src/sgml/ref/cluster.sgml | 97 +-- doc/src/sgml/ref/clusterdb.sgml | 5 + doc/src/sgml/ref/pg_repackdb.sgml | 479 ++++++++++++++ doc/src/sgml/ref/repack.sgml | 284 +++++++++ doc/src/sgml/ref/vacuum.sgml | 33 +- doc/src/sgml/reference.sgml | 2 + src/backend/access/heap/heapam_handler.c | 32 +- src/backend/catalog/index.c | 2 +- src/backend/catalog/system_views.sql | 26 + src/backend/commands/cluster.c | 758 +++++++++++++++-------- src/backend/commands/vacuum.c | 3 +- src/backend/parser/gram.y | 88 ++- src/backend/tcop/utility.c | 20 +- src/backend/utils/adt/pgstatfuncs.c | 2 + src/bin/psql/tab-complete.in.c | 33 +- src/bin/scripts/Makefile | 4 +- src/bin/scripts/meson.build | 2 + src/bin/scripts/pg_repackdb.c | 226 +++++++ src/bin/scripts/t/103_repackdb.pl | 24 + src/bin/scripts/vacuuming.c | 60 +- src/bin/scripts/vacuuming.h | 11 +- src/include/commands/cluster.h | 8 +- src/include/commands/progress.h | 61 +- src/include/nodes/parsenodes.h | 20 +- src/include/parser/kwlist.h | 1 + src/include/tcop/cmdtaglist.h | 1 + src/include/utils/backend_progress.h | 1 + src/test/regress/expected/cluster.out | 125 +++- src/test/regress/expected/rules.out | 23 + src/test/regress/sql/cluster.sql | 59 ++ src/tools/pgindent/typedefs.list | 3 + 33 files changed, 2271 insertions(+), 447 deletions(-) create mode 100644 doc/src/sgml/ref/pg_repackdb.sgml create mode 100644 doc/src/sgml/ref/repack.sgml create mode 100644 src/bin/scripts/pg_repackdb.c create mode 100644 src/bin/scripts/t/103_repackdb.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 3f4a27a736e..12e103d319d 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -405,6 +405,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_progress_repackpg_stat_progress_repack + One row for each backend running + REPACK, showing current progress. See + . + + + pg_stat_progress_basebackuppg_stat_progress_basebackup One row for each WAL sender process streaming a base backup, @@ -5506,7 +5514,8 @@ FROM pg_stat_get_backend_idset() AS backendid; certain commands during command execution. Currently, the only commands which support progress reporting are ANALYZE, CLUSTER, - CREATE INDEX, VACUUM, + CREATE INDEX, REPACK, + VACUUM, COPY, and (i.e., replication command that issues to take @@ -5965,6 +5974,218 @@ FROM pg_stat_get_backend_idset() AS backendid; + + REPACK Progress Reporting + + + pg_stat_progress_repack + + + + Whenever REPACK is running, + the pg_stat_progress_repack view will contain a + row for each backend that is currently running the command. The tables + below describe the information that will be reported and provide + information about how to interpret it. + + + + <structname>pg_stat_progress_repack</structname> View + + + + + Column Type + + + Description + + + + + + + + pid integer + + + Process ID of backend. + + + + + + datid oid + + + OID of the database to which this backend is connected. + + + + + + datname name + + + Name of the database to which this backend is connected. + + + + + + relid oid + + + OID of the table being repacked. + + + + + + phase text + + + Current processing phase. See . + + + + + + repack_index_relid oid + + + If the table is being scanned using an index, this is the OID of the + index being used; otherwise, it is zero. + + + + + + heap_tuples_scanned bigint + + + Number of heap tuples scanned. + This counter only advances when the phase is + seq scanning heap, + index scanning heap + or writing new heap. + + + + + + heap_tuples_written bigint + + + Number of heap tuples written. + This counter only advances when the phase is + seq scanning heap, + index scanning heap + or writing new heap. + + + + + + heap_blks_total bigint + + + Total number of heap blocks in the table. This number is reported + as of the beginning of seq scanning heap. + + + + + + heap_blks_scanned bigint + + + Number of heap blocks scanned. This counter only advances when the + phase is seq scanning heap. + + + + + + index_rebuild_count bigint + + + Number of indexes rebuilt. This counter only advances when the phase + is rebuilding index. + + + + +
+ + + REPACK Phases + + + + + + Phase + Description + + + + + + initializing + + The command is preparing to begin scanning the heap. This phase is + expected to be very brief. + + + + seq scanning heap + + The command is currently scanning the table using a sequential scan. + + + + index scanning heap + + REPACK is currently scanning the table using an index scan. + + + + sorting tuples + + REPACK is currently sorting tuples. + + + + writing new heap + + REPACK is currently writing the new heap. + + + + swapping relation files + + The command is currently swapping newly-built files into place. + + + + rebuilding index + + The command is currently rebuilding an index. + + + + performing final cleanup + + The command is performing final cleanup. When this phase is + completed, REPACK will end. + + + + +
+ + COPY Progress Reporting diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index f5be638867a..eabf92e3536 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -167,6 +167,7 @@ Complete list of usable sgml source files in this directory. + @@ -212,6 +213,7 @@ Complete list of usable sgml source files in this directory. + diff --git a/doc/src/sgml/ref/cluster.sgml b/doc/src/sgml/ref/cluster.sgml index 8811f169ea0..cfcfb65e349 100644 --- a/doc/src/sgml/ref/cluster.sgml +++ b/doc/src/sgml/ref/cluster.sgml @@ -33,51 +33,13 @@ CLUSTER [ ( option [, ...] ) ] [ Description - CLUSTER instructs PostgreSQL - to cluster the table specified - by table_name - based on the index specified by - index_name. The index must - already have been defined on - table_name. + The CLUSTER command is equivalent to + with an USING INDEX + clause. See there for more details. - - When a table is clustered, it is physically reordered - based on the index information. Clustering is a one-time operation: - when the table is subsequently updated, the changes are - not clustered. That is, no attempt is made to store new or - updated rows according to their index order. (If one wishes, one can - periodically recluster by issuing the command again. Also, setting - the table's fillfactor storage parameter to less than - 100% can aid in preserving cluster ordering during updates, since updated - rows are kept on the same page if enough space is available there.) - - - - When a table is clustered, PostgreSQL - remembers which index it was clustered by. The form - CLUSTER table_name - reclusters the table using the same index as before. You can also - use the CLUSTER or SET WITHOUT CLUSTER - forms of ALTER TABLE to set the index to be used for - future cluster operations, or to clear any previous setting. - + - - CLUSTER without a - table_name reclusters all the - previously-clustered tables in the current database that the calling user - has privileges for. This form of CLUSTER cannot be - executed inside a transaction block. - - - - When a table is being clustered, an ACCESS - EXCLUSIVE lock is acquired on it. This prevents any other - database operations (both reads and writes) from operating on the - table until the CLUSTER is finished. - @@ -136,63 +98,12 @@ CLUSTER [ ( option [, ...] ) ] [ - - In cases where you are accessing single rows randomly - within a table, the actual order of the data in the - table is unimportant. However, if you tend to access some - data more than others, and there is an index that groups - them together, you will benefit from using CLUSTER. - If you are requesting a range of indexed values from a table, or a - single indexed value that has multiple rows that match, - CLUSTER will help because once the index identifies the - table page for the first row that matches, all other rows - that match are probably already on the same table page, - and so you save disk accesses and speed up the query. - - - - CLUSTER can re-sort the table using either an index scan - on the specified index, or (if the index is a b-tree) a sequential - scan followed by sorting. It will attempt to choose the method that - will be faster, based on planner cost parameters and available statistical - information. - - While CLUSTER is running, the is temporarily changed to pg_catalog, pg_temp. - - When an index scan is used, a temporary copy of the table is created that - contains the table data in the index order. Temporary copies of each - index on the table are created as well. Therefore, you need free space on - disk at least equal to the sum of the table size and the index sizes. - - - - When a sequential scan and sort is used, a temporary sort file is - also created, so that the peak temporary space requirement is as much - as double the table size, plus the index sizes. This method is often - faster than the index scan method, but if the disk space requirement is - intolerable, you can disable this choice by temporarily setting to off. - - - - It is advisable to set to - a reasonably large value (but not more than the amount of RAM you can - dedicate to the CLUSTER operation) before clustering. - - - - Because the planner records statistics about the ordering of - tables, it is advisable to run ANALYZE - on the newly clustered table. - Otherwise, the planner might make poor choices of query plans. - - Because CLUSTER remembers which indexes are clustered, one can cluster the tables one wants clustered manually the first time, diff --git a/doc/src/sgml/ref/clusterdb.sgml b/doc/src/sgml/ref/clusterdb.sgml index 0d2051bf6f1..546c1289c31 100644 --- a/doc/src/sgml/ref/clusterdb.sgml +++ b/doc/src/sgml/ref/clusterdb.sgml @@ -64,6 +64,11 @@ PostgreSQL documentation this utility and via other methods for accessing the server. + + clusterdb has been superceded by + pg_repackdb. + + diff --git a/doc/src/sgml/ref/pg_repackdb.sgml b/doc/src/sgml/ref/pg_repackdb.sgml new file mode 100644 index 00000000000..32570d071cb --- /dev/null +++ b/doc/src/sgml/ref/pg_repackdb.sgml @@ -0,0 +1,479 @@ + + + + + pg_repackdb + + + + pg_repackdb + 1 + Application + + + + pg_repackdb + repack and analyze a PostgreSQL + database + + + + + pg_repackdb + connection-option + option + + + + + + + + table + ( column [,...] ) + + + + + + dbname + + + + + + + + pg_repackdb + connection-option + option + + + + + + + + schema + + + + + + dbname + + + + + + + + pg_repackdb + connection-option + option + + + + + + + + schema + + + + + + dbname + + + + + + + + + Description + + + pg_repackdb is a utility for repacking a + PostgreSQL database. + pg_repackdb will also generate internal + statistics used by the PostgreSQL query + optimizer. + + + + pg_repackdb is a wrapper around the SQL + command REPACK There + is no effective difference between repacking and analyzing databases via + this utility and via other methods for accessing the server. + + + + + + + Options + + + pg_repackdb accepts the following command-line arguments: + + + + + + + Repack all databases. + + + + + + + + + + Specifies the name of the database to be repacked or analyzed, + when / is not used. If this + is not specified, the database name is read from the environment + variable PGDATABASE. If that is not set, the user name + specified for the connection is used. + The dbname can be + a connection string. If so, + connection string parameters will override any conflicting command + line options. + + + + + + + + + + Echo the commands that pg_repackdb + generates and sends to the server. + + + + + + + + + + Execute the repack or analyze commands in parallel by running + njobs + commands simultaneously. This option may reduce the processing time + but it also increases the load on the database server. + + + pg_repackdb will open + njobs connections to the + database, so make sure your + setting is high enough to accommodate all connections. + + + Note that using this mode might cause deadlock failures if certain + system catalogs are processed in parallel. + + + + + + + + + + Repack or analyze all tables in + schema only. Multiple + schemas can be repacked by writing multiple + switches. + + + + + + + + + + Do not repack or analyze any tables in + schema. Multiple schemas + can be excluded by writing multiple switches. + + + + + + + + + + Do not display progress messages. + + + + + + + + + + Repack or analyze table + only. Column names can be specified only in conjunction with + the option. Multiple tables can be + repacked by writing multiple + switches. + + + + If you specify columns, you probably have to escape the parentheses + from the shell. (See examples below.) + + + + + + + + + + + Print detailed information during processing. + + + + + + + + + + Print the pg_repackdb version and exit. + + + + + + + + + + Also calculate statistics for use by the optimizer. + + + + + + + + + + Show help about pg_repackdb command line + arguments, and exit. + + + + + + + + + pg_repackdb also accepts + the following command-line arguments for connection parameters: + + + + + + + Specifies the host name of the machine on which the server + is running. If the value begins with a slash, it is used + as the directory for the Unix domain socket. + + + + + + + + + + Specifies the TCP port or local Unix domain socket file + extension on which the server + is listening for connections. + + + + + + + + + + User name to connect as. + + + + + + + + + + Never issue a password prompt. If the server requires + password authentication and a password is not available by + other means such as a .pgpass file, the + connection attempt will fail. This option can be useful in + batch jobs and scripts where no user is present to enter a + password. + + + + + + + + + + Force pg_repackdb to prompt for a + password before connecting to a database. + + + + This option is never essential, since + pg_repackdb will automatically prompt + for a password if the server demands password authentication. + However, pg_repackdb will waste a + connection attempt finding out that the server wants a password. + In some cases it is worth typing to avoid the extra + connection attempt. + + + + + + + + + When the / is used, connect + to this database to gather the list of databases to repack. + If not specified, the postgres database will be used, + or if that does not exist, template1 will be used. + This can be a connection + string. If so, connection string parameters will override any + conflicting command line options. Also, connection string parameters + other than the database name itself will be re-used when connecting + to other databases. + + + + + + + + + + Environment + + + + PGDATABASE + PGHOST + PGPORT + PGUSER + + + + Default connection parameters + + + + + + PG_COLOR + + + Specifies whether to use color in diagnostic messages. Possible values + are always, auto and + never. + + + + + + + This utility, like most other PostgreSQL utilities, + also uses the environment variables supported by libpq + (see ). + + + + + + + Diagnostics + + + In case of difficulty, see + and for + discussions of potential problems and error messages. + The database server must be running at the + targeted host. Also, any default connection settings and environment + variables used by the libpq front-end + library will apply. + + + + + + Examples + + + To repack the database test: + +$ pg_repackdb test + + + + + To repack and analyze for the optimizer a database named + bigdb: + +$ pg_repackdb --analyze bigdb + + + + + To repack a single table + foo in a database named + xyzzy, and analyze a single column + bar of the table for the optimizer: + +$ pg_repackdb --analyze --verbose --table='foo(bar)' xyzzy + + + + To repack all tables in the foo and bar schemas + in a database named xyzzy: + +$ pg_repackdb --schema='foo' --schema='bar' xyzzy + + + + + + + See Also + + + + + + + diff --git a/doc/src/sgml/ref/repack.sgml b/doc/src/sgml/ref/repack.sgml new file mode 100644 index 00000000000..fd9d89f8aaa --- /dev/null +++ b/doc/src/sgml/ref/repack.sgml @@ -0,0 +1,284 @@ + + + + + REPACK + + + + REPACK + 7 + SQL - Language Statements + + + + REPACK + rewrite a table to reclaim disk space + + + + +REPACK [ ( option [, ...] ) ] [ table_name [ USING INDEX [ index_name ] ] ] + +where option can be one of: + + VERBOSE [ boolean ] + ANALYSE | ANALYZE + + + + + Description + + + REPACK reclaims storage occupied by dead + tuples. Unlike VACUUM, it does so by rewriting the + entire contents of the table specified + by table_name into a new disk + file with no extra space (except for the space guaranteed by + the fillfactor storage parameter), allowing unused space + to be returned to the operating system. + + + + Without + a table_name, REPACK + processes every table and materialized view in the current database that + the current user has the MAINTAIN privilege on. This + form of REPACK cannot be executed inside a transaction + block. + + + + If a USING INDEX clause is specified, the rows are + physically reordered based on information from an index. Please see the + notes on clustering below. + + + + When a table is being repacked, an ACCESS EXCLUSIVE lock + is acquired on it. This prevents any other database operations (both reads + and writes) from operating on the table until the REPACK + is finished. + + + + Notes on Clustering + + + If the USING INDEX clause is specified, the rows in + the table are physically reordered following an index: if an index name + is specified in the command, then that index is used; if no index name + is specified, then the index that has been configured as the index to + cluster on. If no index has been configured in this way, an error is + thrown. The index given in the USING INDEX clause + is configured as the index to cluster on, as well as an index given + to the CLUSTER command. An index can be set + manually using ALTER TABLE ... CLUSTER ON, and reset + with ALTER TABLE ... SET WITHOUT CLUSTER. + + + + If no table name is specified in REPACK USING INDEX, + all tables which have a clustering index defined and which the calling + user has privileges for are processed. + + + + Clustering is a one-time operation: when the table is + subsequently updated, the changes are not clustered. That is, no attempt + is made to store new or updated rows according to their index order. (If + one wishes, one can periodically recluster by issuing the command again. + Also, setting the table's fillfactor storage parameter + to less than 100% can aid in preserving cluster ordering during updates, + since updated rows are kept on the same page if enough space is available + there.) + + + + In cases where you are accessing single rows randomly within a table, the + actual order of the data in the table is unimportant. However, if you tend + to access some data more than others, and there is an index that groups + them together, you will benefit from using clustering. If + you are requesting a range of indexed values from a table, or a single + indexed value that has multiple rows that match, + REPACK will help because once the index identifies the + table page for the first row that matches, all other rows that match are + probably already on the same table page, and so you save disk accesses and + speed up the query. + + + + REPACK can re-sort the table using either an index scan + on the specified index (if the index is a b-tree), or a sequential scan + followed by sorting. It will attempt to choose the method that will be + faster, based on planner cost parameters and available statistical + information. + + + + Because the planner records statistics about the ordering of tables, it is + advisable to + run ANALYZE on the + newly repacked table. Otherwise, the planner might make poor choices of + query plans. + + + + + Notes on Resources + + + When an index scan or a sequential scan without sort is used, a temporary + copy of the table is created that contains the table data in the index + order. Temporary copies of each index on the table are created as well. + Therefore, you need free space on disk at least equal to the sum of the + table size and the index sizes. + + + + When a sequential scan and sort is used, a temporary sort file is also + created, so that the peak temporary space requirement is as much as double + the table size, plus the index sizes. This method is often faster than + the index scan method, but if the disk space requirement is intolerable, + you can disable this choice by temporarily setting + to off. + + + + It is advisable to set to a + reasonably large value (but not more than the amount of RAM you can + dedicate to the REPACK operation) before repacking. + + + + + + + Parameters + + + + table_name + + + The name (possibly schema-qualified) of a table. + + + + + + index_name + + + The name of an index. + + + + + + VERBOSE + + + Prints a progress report as each table is repacked + at INFO level. + + + + + + ANALYZE + ANALYSE + + + Applies on the table after repacking. This is + currently only supported when a single (non-partitioned) table is specified. + + + + + + boolean + + + Specifies whether the selected option should be turned on or off. + You can write TRUE, ON, or + 1 to enable the option, and FALSE, + OFF, or 0 to disable it. The + boolean value can also + be omitted, in which case TRUE is assumed. + + + + + + + + Notes + + + To repack a table, one must have the MAINTAIN privilege + on the table. + + + + While REPACK is running, the is temporarily changed to pg_catalog, + pg_temp. + + + + Each backend running REPACK will report its progress + in the pg_stat_progress_repack view. See + for details. + + + + Repacking a partitioned table repacks each of its partitions. If an index + is specified, each partition is repacked using the partition of that + index. REPACK on a partitioned table cannot be executed + inside a transaction block. + + + + + + Examples + + + Repack the table employees: + +REPACK employees; + + + + + Repack the table employees on the basis of its + index employees_ind (Since index is used here, this is + effectively clustering): + +REPACK employees USING INDEX employees_ind; + + + + + Repack all tables in the database on which you have + the MAINTAIN privilege: + +REPACK; + + + + + Compatibility + + + There is no REPACK statement in the SQL standard. + + + + + diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index bd5dcaf86a5..062b658cfcd 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -25,7 +25,6 @@ VACUUM [ ( option [, ...] ) ] [ where option can be one of: - FULL [ boolean ] FREEZE [ boolean ] VERBOSE [ boolean ] ANALYZE [ boolean ] @@ -39,6 +38,7 @@ VACUUM [ ( option [, ...] ) ] [ boolean ] ONLY_DATABASE_STATS [ boolean ] BUFFER_USAGE_LIMIT size + FULL [ boolean ] and table_and_columns is: @@ -95,20 +95,6 @@ VACUUM [ ( option [, ...] ) ] [ Parameters - - FULL - - - Selects full vacuum, which can reclaim more - space, but takes much longer and exclusively locks the table. - This method also requires extra disk space, since it writes a - new copy of the table and doesn't release the old copy until - the operation is complete. Usually this should only be used when a - significant amount of space needs to be reclaimed from within the table. - - - - FREEZE @@ -362,6 +348,23 @@ VACUUM [ ( option [, ...] ) ] [ + + FULL + + + This option, which is deprecated, makes VACUUM + behave like REPACK without a + USING INDEX clause. + This method of compacting the table takes much longer than + VACUUM and exclusively locks the table. + This method also requires extra disk space, since it writes a + new copy of the table and doesn't release the old copy until + the operation is complete. Usually this should only be used when a + significant amount of space needs to be reclaimed from within the table. + + + + boolean diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index ff85ace83fc..2ee08e21f41 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -195,6 +195,7 @@ &refreshMaterializedView; &reindex; &releaseSavepoint; + &repack; &reset; &revoke; &rollback; @@ -257,6 +258,7 @@ &pgIsready; &pgReceivewal; &pgRecvlogical; + &pgRepackdb; &pgRestore; &pgVerifyBackup; &psqlRef; diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index bcbac844bb6..79f9de5d760 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -741,13 +741,13 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, if (OldIndex != NULL && !use_sort) { const int ci_index[] = { - PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_INDEX_RELID + PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_INDEX_RELID }; int64 ci_val[2]; /* Set phase and OIDOldIndex to columns */ - ci_val[0] = PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP; + ci_val[0] = PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP; ci_val[1] = RelationGetRelid(OldIndex); pgstat_progress_update_multi_param(2, ci_index, ci_val); @@ -759,15 +759,15 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, else { /* In scan-and-sort mode and also VACUUM FULL, set phase */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP); tableScan = table_beginscan(OldHeap, SnapshotAny, 0, (ScanKey) NULL); heapScan = (HeapScanDesc) tableScan; indexScan = NULL; /* Set total heap blocks */ - pgstat_progress_update_param(PROGRESS_CLUSTER_TOTAL_HEAP_BLKS, + pgstat_progress_update_param(PROGRESS_REPACK_TOTAL_HEAP_BLKS, heapScan->rs_nblocks); } @@ -809,7 +809,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, * is manually updated to the correct value when the table * scan finishes. */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_BLKS_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_BLKS_SCANNED, heapScan->rs_nblocks); break; } @@ -825,7 +825,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, */ if (prev_cblock != heapScan->rs_cblock) { - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_BLKS_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_BLKS_SCANNED, (heapScan->rs_cblock + heapScan->rs_nblocks - heapScan->rs_startblock @@ -912,14 +912,14 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, * In scan-and-sort mode, report increase in number of tuples * scanned */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_SCANNED, *num_tuples); } else { const int ct_index[] = { - PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, - PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN + PROGRESS_REPACK_HEAP_TUPLES_SCANNED, + PROGRESS_REPACK_HEAP_TUPLES_WRITTEN }; int64 ct_val[2]; @@ -952,14 +952,14 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, double n_tuples = 0; /* Report that we are now sorting tuples */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SORT_TUPLES); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SORT_TUPLES); tuplesort_performsort(tuplesort); /* Report that we are now writing new heap */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_WRITE_NEW_HEAP); for (;;) { @@ -977,7 +977,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, values, isnull, rwstate); /* Report n_tuples */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_WRITTEN, n_tuples); } diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index c4029a4f3d3..3063abff9a5 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -4079,7 +4079,7 @@ reindex_relation(const ReindexStmt *stmt, Oid relid, int flags, Assert(!ReindexIsProcessingIndex(indexOid)); /* Set index rebuild count */ - pgstat_progress_update_param(PROGRESS_CLUSTER_INDEX_REBUILD_COUNT, + pgstat_progress_update_param(PROGRESS_REPACK_INDEX_REBUILD_COUNT, i); i++; } diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1b3c5a55882..b2b7b10c2be 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1279,6 +1279,32 @@ CREATE VIEW pg_stat_progress_cluster AS FROM pg_stat_get_progress_info('CLUSTER') AS S LEFT JOIN pg_database D ON S.datid = D.oid; +CREATE VIEW pg_stat_progress_repack AS + SELECT + S.pid AS pid, + S.datid AS datid, + D.datname AS datname, + S.relid AS relid, + -- param1 is currently unused + CASE S.param2 WHEN 0 THEN 'initializing' + WHEN 1 THEN 'seq scanning heap' + WHEN 2 THEN 'index scanning heap' + WHEN 3 THEN 'sorting tuples' + WHEN 4 THEN 'writing new heap' + WHEN 5 THEN 'swapping relation files' + WHEN 6 THEN 'rebuilding index' + WHEN 7 THEN 'performing final cleanup' + END AS phase, + CAST(S.param3 AS oid) AS repack_index_relid, + S.param4 AS heap_tuples_scanned, + S.param5 AS heap_tuples_written, + S.param6 AS heap_blks_total, + S.param7 AS heap_blks_scanned, + S.param8 AS index_rebuild_count + FROM pg_stat_get_progress_info('REPACK') AS S + LEFT JOIN pg_database D ON S.datid = D.oid; + + CREATE VIEW pg_stat_progress_create_index AS SELECT S.pid AS pid, S.datid AS datid, D.datname AS datname, diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index b55221d44cd..8b64f9e6795 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -67,18 +67,41 @@ typedef struct Oid indexOid; } RelToCluster; - -static void cluster_multiple_rels(List *rtcs, ClusterParams *params); -static void rebuild_relation(Relation OldHeap, Relation index, bool verbose); +static bool cluster_rel_recheck(RepackCommand cmd, Relation OldHeap, + Oid indexOid, Oid userid, int options); +static void rebuild_relation(RepackCommand cmd, bool usingindex, + Relation OldHeap, Relation index, bool verbose); static void copy_table_data(Relation NewHeap, Relation OldHeap, Relation OldIndex, bool verbose, bool *pSwapToastByContent, TransactionId *pFreezeXid, MultiXactId *pCutoffMulti); -static List *get_tables_to_cluster(MemoryContext cluster_context); -static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context, - Oid indexOid); -static bool cluster_is_permitted_for_relation(Oid relid, Oid userid); +static List *get_tables_to_repack(RepackCommand cmd, bool usingindex, + MemoryContext permcxt); +static List *get_tables_to_repack_partitioned(RepackCommand cmd, + MemoryContext cluster_context, + Oid relid, bool rel_is_index); +static bool cluster_is_permitted_for_relation(RepackCommand cmd, + Oid relid, Oid userid); +static Relation process_single_relation(RepackStmt *stmt, + ClusterParams *params); +static Oid determine_clustered_index(Relation rel, bool usingindex, + const char *indexname); +static const char * +RepackCommandAsString(RepackCommand cmd) +{ + switch (cmd) + { + case REPACK_COMMAND_REPACK: + return "REPACK"; + case REPACK_COMMAND_VACUUMFULL: + return "VACUUM"; + case REPACK_COMMAND_CLUSTER: + return "CLUSTER"; + } + return "???"; +} + /*--------------------------------------------------------------------------- * This cluster code allows for clustering multiple tables at once. Because * of this, we cannot just run everything on a single transaction, or we @@ -104,191 +127,155 @@ static bool cluster_is_permitted_for_relation(Oid relid, Oid userid); *--------------------------------------------------------------------------- */ void -cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) +ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel) { - ListCell *lc; ClusterParams params = {0}; - bool verbose = false; Relation rel = NULL; - Oid indexOid = InvalidOid; - MemoryContext cluster_context; + MemoryContext repack_context; List *rtcs; /* Parse option list */ - foreach(lc, stmt->params) + foreach_node(DefElem, opt, stmt->params) { - DefElem *opt = (DefElem *) lfirst(lc); - if (strcmp(opt->defname, "verbose") == 0) - verbose = defGetBoolean(opt); + params.options |= defGetBoolean(opt) ? CLUOPT_VERBOSE : 0; + else if (strcmp(opt->defname, "analyze") == 0 || + strcmp(opt->defname, "analyse") == 0) + params.options |= defGetBoolean(opt) ? CLUOPT_ANALYZE : 0; else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized CLUSTER option \"%s\"", + errmsg("unrecognized %s option \"%s\"", + RepackCommandAsString(stmt->command), opt->defname), parser_errposition(pstate, opt->location))); } - params.options = (verbose ? CLUOPT_VERBOSE : 0); - + /* + * If a single relation is specified, process it and we're done ... unless + * the relation is a partitioned table, in which case we fall through. + */ if (stmt->relation != NULL) { - /* This is the single-relation case. */ - Oid tableOid; - - /* - * Find, lock, and check permissions on the table. We obtain - * AccessExclusiveLock right away to avoid lock-upgrade hazard in the - * single-transaction case. - */ - tableOid = RangeVarGetRelidExtended(stmt->relation, - AccessExclusiveLock, - 0, - RangeVarCallbackMaintainsTable, - NULL); - rel = table_open(tableOid, NoLock); - - /* - * Reject clustering a remote temp table ... their local buffer - * manager is not going to cope. - */ - if (RELATION_IS_OTHER_TEMP(rel)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster temporary tables of other sessions"))); - - if (stmt->indexname == NULL) - { - ListCell *index; - - /* We need to find the index that has indisclustered set. */ - foreach(index, RelationGetIndexList(rel)) - { - indexOid = lfirst_oid(index); - if (get_index_isclustered(indexOid)) - break; - indexOid = InvalidOid; - } - - if (!OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("there is no previously clustered index for table \"%s\"", - stmt->relation->relname))); - } - else - { - /* - * The index is expected to be in the same namespace as the - * relation. - */ - indexOid = get_relname_relid(stmt->indexname, - rel->rd_rel->relnamespace); - if (!OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("index \"%s\" for table \"%s\" does not exist", - stmt->indexname, stmt->relation->relname))); - } - - /* For non-partitioned tables, do what we came here to do. */ - if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) - { - cluster_rel(rel, indexOid, ¶ms); - /* cluster_rel closes the relation, but keeps lock */ - + rel = process_single_relation(stmt, ¶ms); + if (rel == NULL) return; - } } + /* Don't allow this for now. Maybe we can add support for this later */ + if (params.options & CLUOPT_ANALYZE) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot ANALYZE multiple tables")); + /* * By here, we know we are in a multi-table situation. In order to avoid * holding locks for too long, we want to process each table in its own * transaction. This forces us to disallow running inside a user * transaction block. */ - PreventInTransactionBlock(isTopLevel, "CLUSTER"); + PreventInTransactionBlock(isTopLevel, RepackCommandAsString(stmt->command)); /* Also, we need a memory context to hold our list of relations */ - cluster_context = AllocSetContextCreate(PortalContext, - "Cluster", - ALLOCSET_DEFAULT_SIZES); + repack_context = AllocSetContextCreate(PortalContext, + "Repack", + ALLOCSET_DEFAULT_SIZES); - /* - * Either we're processing a partitioned table, or we were not given any - * table name at all. In either case, obtain a list of relations to - * process. - * - * In the former case, an index name must have been given, so we don't - * need to recheck its "indisclustered" bit, but we have to check that it - * is an index that we can cluster on. In the latter case, we set the - * option bit to have indisclustered verified. - * - * Rechecking the relation itself is necessary here in all cases. - */ params.options |= CLUOPT_RECHECK; - if (rel != NULL) + + /* + * If we don't have a relation yet, determine a relation list. If we do, + * then it must be a partitioned table, and we want to process its + * partitions. + */ + if (rel == NULL) { + Assert(stmt->indexname == NULL); + rtcs = get_tables_to_repack(stmt->command, stmt->usingindex, + repack_context); + } + else + { + Oid relid; + bool rel_is_index; + Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); - check_index_is_clusterable(rel, indexOid, AccessShareLock); - rtcs = get_tables_to_cluster_partitioned(cluster_context, indexOid); - /* close relation, releasing lock on parent table */ + /* + * If an index name was specified, resolve it now and pass it down. + */ + if (stmt->usingindex) + { + /* + * XXX how should this behave? Passing no index to a partitioned + * table could be useful to have certain partitions clustered by + * some index, and other partitions by a different index. + */ + if (!stmt->indexname) + ereport(ERROR, + errmsg("there is no previously clustered index for table \"%s\"", + RelationGetRelationName(rel))); + + relid = determine_clustered_index(rel, true, stmt->indexname); + if (!OidIsValid(relid)) + elog(ERROR, "unable to determine index to cluster on"); + /* XXX is this the right place for this check? */ + check_index_is_clusterable(rel, relid, AccessExclusiveLock); + rel_is_index = true; + } + else + { + relid = RelationGetRelid(rel); + rel_is_index = false; + } + + rtcs = get_tables_to_repack_partitioned(stmt->command, repack_context, + relid, rel_is_index); + + /* close parent relation, releasing lock on it */ table_close(rel, AccessExclusiveLock); + rel = NULL; } - else - { - rtcs = get_tables_to_cluster(cluster_context); - params.options |= CLUOPT_RECHECK_ISCLUSTERED; - } - - /* Do the job. */ - cluster_multiple_rels(rtcs, ¶ms); - - /* Start a new transaction for the cleanup work. */ - StartTransactionCommand(); - - /* Clean up working storage */ - MemoryContextDelete(cluster_context); -} - -/* - * Given a list of relations to cluster, process each of them in a separate - * transaction. - * - * We expect to be in a transaction at start, but there isn't one when we - * return. - */ -static void -cluster_multiple_rels(List *rtcs, ClusterParams *params) -{ - ListCell *lc; /* Commit to get out of starting transaction */ PopActiveSnapshot(); CommitTransactionCommand(); /* Cluster the tables, each in a separate transaction */ - foreach(lc, rtcs) + Assert(rel == NULL); + foreach_ptr(RelToCluster, rtc, rtcs) { - RelToCluster *rtc = (RelToCluster *) lfirst(lc); - Relation rel; - /* Start a new transaction for each relation. */ StartTransactionCommand(); + /* + * Open the target table, coping with the case where it has been + * dropped. + */ + rel = try_table_open(rtc->tableOid, AccessExclusiveLock); + if (rel == NULL) + { + CommitTransactionCommand(); + continue; + } + /* functions in indexes may want a snapshot set */ PushActiveSnapshot(GetTransactionSnapshot()); - rel = table_open(rtc->tableOid, AccessExclusiveLock); - /* Process this table */ - cluster_rel(rel, rtc->indexOid, params); + cluster_rel(stmt->command, stmt->usingindex, + rel, rtc->indexOid, ¶ms); /* cluster_rel closes the relation, but keeps lock */ PopActiveSnapshot(); CommitTransactionCommand(); } + + /* Start a new transaction for the cleanup work. */ + StartTransactionCommand(); + + /* Clean up working storage */ + MemoryContextDelete(repack_context); } /* @@ -304,11 +291,14 @@ cluster_multiple_rels(List *rtcs, ClusterParams *params) * them incrementally while we load the table. * * If indexOid is InvalidOid, the table will be rewritten in physical order - * instead of index order. This is the new implementation of VACUUM FULL, - * and error messages should refer to the operation as VACUUM not CLUSTER. + * instead of index order. + * + * 'cmd' indicates which command is being executed, to be used for error + * messages. */ void -cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) +cluster_rel(RepackCommand cmd, bool usingindex, + Relation OldHeap, Oid indexOid, ClusterParams *params) { Oid tableOid = RelationGetRelid(OldHeap); Oid save_userid; @@ -323,13 +313,25 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) /* Check for user-requested abort. */ CHECK_FOR_INTERRUPTS(); - pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid); - if (OidIsValid(indexOid)) - pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, + if (cmd == REPACK_COMMAND_REPACK) + pgstat_progress_start_command(PROGRESS_COMMAND_REPACK, tableOid); + else + pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid); + + if (cmd == REPACK_COMMAND_REPACK) + pgstat_progress_update_param(PROGRESS_REPACK_COMMAND, + PROGRESS_REPACK_COMMAND_REPACK); + else if (cmd == REPACK_COMMAND_CLUSTER) + { + pgstat_progress_update_param(PROGRESS_REPACK_COMMAND, PROGRESS_CLUSTER_COMMAND_CLUSTER); + } else - pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, + { + Assert(cmd == REPACK_COMMAND_VACUUMFULL); + pgstat_progress_update_param(PROGRESS_REPACK_COMMAND, PROGRESS_CLUSTER_COMMAND_VACUUM_FULL); + } /* * Switch to the table owner's userid, so that any index functions are run @@ -351,63 +353,21 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) * to cluster a not-previously-clustered index. */ if (recheck) - { - /* Check that the user still has privileges for the relation */ - if (!cluster_is_permitted_for_relation(tableOid, save_userid)) - { - relation_close(OldHeap, AccessExclusiveLock); + if (!cluster_rel_recheck(cmd, OldHeap, indexOid, save_userid, + params->options)) goto out; - } - - /* - * Silently skip a temp table for a remote session. Only doing this - * check in the "recheck" case is appropriate (which currently means - * somebody is executing a database-wide CLUSTER or on a partitioned - * table), because there is another check in cluster() which will stop - * any attempt to cluster remote temp tables by name. There is - * another check in cluster_rel which is redundant, but we leave it - * for extra safety. - */ - if (RELATION_IS_OTHER_TEMP(OldHeap)) - { - relation_close(OldHeap, AccessExclusiveLock); - goto out; - } - - if (OidIsValid(indexOid)) - { - /* - * Check that the index still exists - */ - if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid))) - { - relation_close(OldHeap, AccessExclusiveLock); - goto out; - } - - /* - * Check that the index is still the one with indisclustered set, - * if needed. - */ - if ((params->options & CLUOPT_RECHECK_ISCLUSTERED) != 0 && - !get_index_isclustered(indexOid)) - { - relation_close(OldHeap, AccessExclusiveLock); - goto out; - } - } - } /* - * We allow VACUUM FULL, but not CLUSTER, on shared catalogs. CLUSTER - * would work in most respects, but the index would only get marked as - * indisclustered in the current database, leading to unexpected behavior - * if CLUSTER were later invoked in another database. + * We allow repacking shared catalogs only when not using an index. It + * would work to use an index in most respects, but the index would only + * get marked as indisclustered in the current database, leading to + * unexpected behavior if CLUSTER were later invoked in another database. */ - if (OidIsValid(indexOid) && OldHeap->rd_rel->relisshared) + if (usingindex && OldHeap->rd_rel->relisshared) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster a shared catalog"))); + errmsg("cannot run \"%s\" on a shared catalog", + RepackCommandAsString(cmd)))); /* * Don't process temp tables of other backends ... their local buffer @@ -415,21 +375,30 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) */ if (RELATION_IS_OTHER_TEMP(OldHeap)) { - if (OidIsValid(indexOid)) + if (cmd == REPACK_COMMAND_CLUSTER) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot cluster temporary tables of other sessions"))); + else if (cmd == REPACK_COMMAND_REPACK) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot repack temporary tables of other sessions"))); + } else + { + Assert(cmd == REPACK_COMMAND_VACUUMFULL); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot vacuum temporary tables of other sessions"))); + } } /* * Also check for active uses of the relation in the current transaction, * including open scans and pending AFTER trigger events. */ - CheckTableNotInUse(OldHeap, OidIsValid(indexOid) ? "CLUSTER" : "VACUUM"); + CheckTableNotInUse(OldHeap, RepackCommandAsString(cmd)); /* Check heap and index are valid to cluster on */ if (OidIsValid(indexOid)) @@ -469,7 +438,7 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) TransferPredicateLocksToHeapRelation(OldHeap); /* rebuild_relation does all the dirty work */ - rebuild_relation(OldHeap, index, verbose); + rebuild_relation(cmd, usingindex, OldHeap, index, verbose); /* rebuild_relation closes OldHeap, and index if valid */ out: @@ -482,6 +451,63 @@ out: pgstat_progress_end_command(); } +/* + * Check if the table (and its index) still meets the requirements of + * cluster_rel(). + */ +static bool +cluster_rel_recheck(RepackCommand cmd, Relation OldHeap, Oid indexOid, + Oid userid, int options) +{ + Oid tableOid = RelationGetRelid(OldHeap); + + /* Check that the user still has privileges for the relation */ + if (!cluster_is_permitted_for_relation(cmd, tableOid, userid)) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + + /* + * Silently skip a temp table for a remote session. Only doing this check + * in the "recheck" case is appropriate (which currently means somebody is + * executing a database-wide CLUSTER or on a partitioned table), because + * there is another check in cluster() which will stop any attempt to + * cluster remote temp tables by name. There is another check in + * cluster_rel which is redundant, but we leave it for extra safety. + */ + if (RELATION_IS_OTHER_TEMP(OldHeap)) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + + if (OidIsValid(indexOid)) + { + /* + * Check that the index still exists + */ + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid))) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + + /* + * Check that the index is still the one with indisclustered set, if + * needed. + */ + if ((options & CLUOPT_RECHECK_ISCLUSTERED) != 0 && + !get_index_isclustered(indexOid)) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + } + + return true; +} + /* * Verify that the specified heap and index are valid to cluster on * @@ -626,7 +652,8 @@ mark_index_clustered(Relation rel, Oid indexOid, bool is_internal) * On exit, they are closed, but locks on them are not released. */ static void -rebuild_relation(Relation OldHeap, Relation index, bool verbose) +rebuild_relation(RepackCommand cmd, bool usingindex, + Relation OldHeap, Relation index, bool verbose) { Oid tableOid = RelationGetRelid(OldHeap); Oid accessMethod = OldHeap->rd_rel->relam; @@ -642,8 +669,8 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose) Assert(CheckRelationLockedByMe(OldHeap, AccessExclusiveLock, false) && (index == NULL || CheckRelationLockedByMe(index, AccessExclusiveLock, false))); - if (index) - /* Mark the correct index as clustered */ + /* for CLUSTER or REPACK USING INDEX, mark the index as the one to use */ + if (usingindex) mark_index_clustered(OldHeap, RelationGetRelid(index), true); /* Remember info about rel before closing OldHeap */ @@ -1458,8 +1485,8 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, int i; /* Report that we are now swapping relation files */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SWAP_REL_FILES); /* Zero out possible results from swapped_relation_files */ memset(mapped_tables, 0, sizeof(mapped_tables)); @@ -1509,14 +1536,14 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; /* Report that we are now reindexing relations */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_REBUILD_INDEX); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_REBUILD_INDEX); reindex_relation(NULL, OIDOldHeap, reindex_flags, &reindex_params); /* Report that we are now doing clean up */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_FINAL_CLEANUP); /* * If the relation being rebuilt is pg_class, swap_relation_files() @@ -1632,69 +1659,137 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, } } - /* - * Get a list of tables that the current user has privileges on and - * have indisclustered set. Return the list in a List * of RelToCluster - * (stored in the specified memory context), each one giving the tableOid - * and the indexOid on which the table is already clustered. + * Determine which relations to process, when REPACK/CLUSTER is called + * without specifying a table name. The exact process depends on whether + * USING INDEX was given or not, and in any case we only return tables and + * materialized views that the current user has privileges to repack/cluster. + * + * If USING INDEX was given, we scan pg_index to find those that have + * indisclustered set; if it was not given, scan pg_class and return all + * tables. + * + * Return it as a list of RelToCluster in the given memory context. */ static List * -get_tables_to_cluster(MemoryContext cluster_context) +get_tables_to_repack(RepackCommand command, bool usingindex, + MemoryContext permcxt) { - Relation indRelation; + Relation catalog; TableScanDesc scan; - ScanKeyData entry; - HeapTuple indexTuple; - Form_pg_index index; + HeapTuple tuple; MemoryContext old_context; List *rtcs = NIL; - /* - * Get all indexes that have indisclustered set and that the current user - * has the appropriate privileges for. - */ - indRelation = table_open(IndexRelationId, AccessShareLock); - ScanKeyInit(&entry, - Anum_pg_index_indisclustered, - BTEqualStrategyNumber, F_BOOLEQ, - BoolGetDatum(true)); - scan = table_beginscan_catalog(indRelation, 1, &entry); - while ((indexTuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + if (usingindex) { - RelToCluster *rtc; + ScanKeyData entry; - index = (Form_pg_index) GETSTRUCT(indexTuple); + catalog = table_open(IndexRelationId, AccessShareLock); + ScanKeyInit(&entry, + Anum_pg_index_indisclustered, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(true)); + scan = table_beginscan_catalog(catalog, 1, &entry); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + RelToCluster *rtc; + Form_pg_index index; - if (!cluster_is_permitted_for_relation(index->indrelid, GetUserId())) - continue; + index = (Form_pg_index) GETSTRUCT(tuple); - /* Use a permanent memory context for the result list */ - old_context = MemoryContextSwitchTo(cluster_context); + /* + * XXX I think the only reason there's no test failure here is + * that we seldom have clustered indexes that would be affected by + * concurrency. Maybe we should also do the + * ConditionalLockRelationOid+SearchSysCacheExists dance that we + * do below. + */ + if (!cluster_is_permitted_for_relation(command, index->indrelid, + GetUserId())) + continue; - rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); - rtc->tableOid = index->indrelid; - rtc->indexOid = index->indexrelid; - rtcs = lappend(rtcs, rtc); + /* Use a permanent memory context for the result list */ + old_context = MemoryContextSwitchTo(permcxt); - MemoryContextSwitchTo(old_context); + rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); + rtc->tableOid = index->indrelid; + rtc->indexOid = index->indexrelid; + rtcs = lappend(rtcs, rtc); + + MemoryContextSwitchTo(old_context); + } } + else + { + catalog = table_open(RelationRelationId, AccessShareLock); + scan = table_beginscan_catalog(catalog, 0, NULL); + + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + RelToCluster *rtc; + Form_pg_class class; + + class = (Form_pg_class) GETSTRUCT(tuple); + + /* + * Try to obtain a light lock on the table, to ensure it doesn't + * go away while we collect the list. If we cannot, just + * disregard the table. XXX we could release at the bottom of the + * loop, but for now just hold it until this transaction is + * finished. + */ + if (!ConditionalLockRelationOid(class->oid, AccessShareLock)) + continue; + + /* Verify that the table still exists. */ + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(class->oid))) + { + /* Release useless lock */ + UnlockRelationOid(class->oid, AccessShareLock); + continue; + } + + /* Can only process plain tables and matviews */ + if (class->relkind != RELKIND_RELATION && + class->relkind != RELKIND_MATVIEW) + continue; + + if (!cluster_is_permitted_for_relation(command, class->oid, + GetUserId())) + continue; + + /* Use a permanent memory context for the result list */ + old_context = MemoryContextSwitchTo(permcxt); + + rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); + rtc->tableOid = class->oid; + rtc->indexOid = InvalidOid; + rtcs = lappend(rtcs, rtc); + + MemoryContextSwitchTo(old_context); + } + } + table_endscan(scan); - - relation_close(indRelation, AccessShareLock); + relation_close(catalog, AccessShareLock); return rtcs; } /* - * Given an index on a partitioned table, return a list of RelToCluster for + * Given a partitioned table or its index, return a list of RelToCluster for * all the children leaves tables/indexes. * * Like expand_vacuum_rel, but here caller must hold AccessExclusiveLock * on the table containing the index. + * + * 'rel_is_index' tells whether 'relid' is that of an index (true) or of the + * owning relation. */ static List * -get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) +get_tables_to_repack_partitioned(RepackCommand cmd, MemoryContext cluster_context, + Oid relid, bool rel_is_index) { List *inhoids; ListCell *lc; @@ -1702,17 +1797,33 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) MemoryContext old_context; /* Do not lock the children until they're processed */ - inhoids = find_all_inheritors(indexOid, NoLock, NULL); + inhoids = find_all_inheritors(relid, NoLock, NULL); foreach(lc, inhoids) { - Oid indexrelid = lfirst_oid(lc); - Oid relid = IndexGetRelation(indexrelid, false); + Oid inhoid = lfirst_oid(lc); + Oid inhrelid, + inhindid; RelToCluster *rtc; - /* consider only leaf indexes */ - if (get_rel_relkind(indexrelid) != RELKIND_INDEX) - continue; + if (rel_is_index) + { + /* consider only leaf indexes */ + if (get_rel_relkind(inhoid) != RELKIND_INDEX) + continue; + + inhrelid = IndexGetRelation(inhoid, false); + inhindid = inhoid; + } + else + { + /* consider only leaf relations */ + if (get_rel_relkind(inhoid) != RELKIND_RELATION) + continue; + + inhrelid = inhoid; + inhindid = InvalidOid; + } /* * It's possible that the user does not have privileges to CLUSTER the @@ -1720,15 +1831,15 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) * table. We skip any partitions which the user is not permitted to * CLUSTER. */ - if (!cluster_is_permitted_for_relation(relid, GetUserId())) + if (!cluster_is_permitted_for_relation(cmd, inhrelid, GetUserId())) continue; /* Use a permanent memory context for the result list */ old_context = MemoryContextSwitchTo(cluster_context); rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); - rtc->tableOid = relid; - rtc->indexOid = indexrelid; + rtc->tableOid = inhrelid; + rtc->indexOid = inhindid; rtcs = lappend(rtcs, rtc); MemoryContextSwitchTo(old_context); @@ -1742,13 +1853,148 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) * function emits a WARNING. */ static bool -cluster_is_permitted_for_relation(Oid relid, Oid userid) +cluster_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid) { if (pg_class_aclcheck(relid, userid, ACL_MAINTAIN) == ACLCHECK_OK) return true; + Assert(cmd == REPACK_COMMAND_CLUSTER || cmd == REPACK_COMMAND_REPACK); ereport(WARNING, - (errmsg("permission denied to cluster \"%s\", skipping it", - get_rel_name(relid)))); + errmsg("permission denied to execute %s on \"%s\", skipping it", + cmd == REPACK_COMMAND_CLUSTER ? "CLUSTER" : "REPACK", + get_rel_name(relid))); + return false; } + + +/* + * Given a RepackStmt with an indicated relation name, resolve the relation + * name, obtain lock on it, then determine what to do based on the relation + * type: if it's not a partitioned table, repack it as indicated (using an + * existing clustered index, or following the indicated index), and return + * NULL. + * + * On the other hand, if the table is partitioned, do nothing further and + * instead return the opened relcache entry, so that caller can process the + * partitions using the multiple-table handling code. The index name is not + * resolve in this case. + */ +static Relation +process_single_relation(RepackStmt *stmt, ClusterParams *params) +{ + Relation rel; + Oid tableOid; + + Assert(stmt->relation != NULL); + Assert(stmt->command == REPACK_COMMAND_CLUSTER || + stmt->command == REPACK_COMMAND_REPACK); + + /* + * Find, lock, and check permissions on the table. We obtain + * AccessExclusiveLock right away to avoid lock-upgrade hazard in the + * single-transaction case. + */ + tableOid = RangeVarGetRelidExtended(stmt->relation, + AccessExclusiveLock, + 0, + RangeVarCallbackMaintainsTable, + NULL); + rel = table_open(tableOid, NoLock); + + /* + * Reject clustering a remote temp table ... their local buffer manager is + * not going to cope. + */ + if (RELATION_IS_OTHER_TEMP(rel)) + { + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot execute %s on temporary tables of other sessions", + RepackCommandAsString(stmt->command))); + } + + /* + * For partitioned tables, let caller handle this. Otherwise, process it + * here and we're done. + */ + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + return rel; + else + { + Oid indexOid; + + indexOid = determine_clustered_index(rel, stmt->usingindex, + stmt->indexname); + if (OidIsValid(indexOid)) + check_index_is_clusterable(rel, indexOid, AccessExclusiveLock); + cluster_rel(stmt->command, stmt->usingindex, rel, indexOid, params); + + /* Do an analyze, if requested */ + if (params->options & CLUOPT_ANALYZE) + { + VacuumParams vac_params = {0}; + + vac_params.options |= VACOPT_ANALYZE; + if (params->options & CLUOPT_VERBOSE) + vac_params.options |= VACOPT_VERBOSE; + analyze_rel(RelationGetRelid(rel), NULL, vac_params, NIL, true, + NULL); + } + + return NULL; + } +} + +/* + * Given a relation and the usingindex/indexname options in a + * REPACK USING INDEX or CLUSTER command, return the OID of the index to use + * for clustering the table. + * + * Caller must hold lock on the relation so that the set of indexes doesn't + * change, and must call check_index_is_clusterable. + */ +static Oid +determine_clustered_index(Relation rel, bool usingindex, const char *indexname) +{ + Oid indexOid; + + if (indexname == NULL && usingindex) + { + ListCell *lc; + + /* Find an index with indisclustered set, or report error */ + foreach(lc, RelationGetIndexList(rel)) + { + indexOid = lfirst_oid(lc); + + if (get_index_isclustered(indexOid)) + break; + indexOid = InvalidOid; + } + + if (!OidIsValid(indexOid)) + ereport(ERROR, + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("there is no previously clustered index for table \"%s\"", + RelationGetRelationName(rel))); + } + else if (indexname != NULL) + { + /* + * An index was specified; figure out its OID. It must be in the same + * namespace as the relation. + */ + indexOid = get_relname_relid(indexname, + rel->rd_rel->relnamespace); + if (!OidIsValid(indexOid)) + ereport(ERROR, + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("index \"%s\" for table \"%s\" does not exist", + indexname, RelationGetRelationName(rel))); + } + else + indexOid = InvalidOid; + + return indexOid; +} diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 733ef40ae7c..8863ad0e8bd 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -2287,7 +2287,8 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, cluster_params.options |= CLUOPT_VERBOSE; /* VACUUM FULL is now a variant of CLUSTER; see cluster.c */ - cluster_rel(rel, InvalidOid, &cluster_params); + cluster_rel(REPACK_COMMAND_VACUUMFULL, false, rel, InvalidOid, + &cluster_params); /* cluster_rel closes the relation, but keeps lock */ rel = NULL; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index db43034b9db..f9152728021 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -280,7 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); AlterCompositeTypeStmt AlterUserMappingStmt AlterRoleStmt AlterRoleSetStmt AlterPolicyStmt AlterStatsStmt AlterDefaultPrivilegesStmt DefACLAction - AnalyzeStmt CallStmt ClosePortalStmt ClusterStmt CommentStmt + AnalyzeStmt CallStmt ClosePortalStmt CommentStmt ConstraintsSetStmt CopyStmt CreateAsStmt CreateCastStmt CreateDomainStmt CreateExtensionStmt CreateGroupStmt CreateOpClassStmt CreateOpFamilyStmt AlterOpFamilyStmt CreatePLangStmt @@ -297,7 +297,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); GrantStmt GrantRoleStmt ImportForeignSchemaStmt IndexStmt InsertStmt ListenStmt LoadStmt LockStmt MergeStmt NotifyStmt ExplainableStmt PreparableStmt CreateFunctionStmt AlterFunctionStmt ReindexStmt RemoveAggrStmt - RemoveFuncStmt RemoveOperStmt RenameStmt ReturnStmt RevokeStmt RevokeRoleStmt + RemoveFuncStmt RemoveOperStmt RenameStmt RepackStmt ReturnStmt RevokeStmt RevokeRoleStmt RuleActionStmt RuleActionStmtOrEmpty RuleStmt SecLabelStmt SelectStmt TransactionStmt TransactionStmtLegacy TruncateStmt UnlistenStmt UpdateStmt VacuumStmt @@ -316,7 +316,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type opt_single_name %type opt_qualified_name -%type opt_concurrently +%type opt_concurrently opt_usingindex %type opt_drop_behavior %type opt_utility_option_list %type utility_option_list @@ -763,7 +763,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); QUOTE QUOTES RANGE READ REAL REASSIGN RECURSIVE REF_P REFERENCES REFERENCING - REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA + REFRESH REINDEX RELATIVE_P RELEASE RENAME REPACK REPEATABLE REPLACE REPLICA RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROUTINE ROUTINES ROW ROWS RULE @@ -1025,7 +1025,6 @@ stmt: | CallStmt | CheckPointStmt | ClosePortalStmt - | ClusterStmt | CommentStmt | ConstraintsSetStmt | CopyStmt @@ -1099,6 +1098,7 @@ stmt: | RemoveFuncStmt | RemoveOperStmt | RenameStmt + | RepackStmt | RevokeStmt | RevokeRoleStmt | RuleStmt @@ -1135,6 +1135,11 @@ opt_concurrently: | /*EMPTY*/ { $$ = false; } ; +opt_usingindex: + USING INDEX { $$ = true; } + | /* EMPTY */ { $$ = false; } + ; + opt_drop_behavior: CASCADE { $$ = DROP_CASCADE; } | RESTRICT { $$ = DROP_RESTRICT; } @@ -11912,38 +11917,91 @@ CreateConversionStmt: /***************************************************************************** * * QUERY: + * REPACK [ (options) ] [ [ USING INDEX ] ] + * + * obsolete variants: * CLUSTER (options) [ [ USING ] ] * CLUSTER [VERBOSE] [ [ USING ] ] * CLUSTER [VERBOSE] ON (for pre-8.3) * *****************************************************************************/ -ClusterStmt: - CLUSTER '(' utility_option_list ')' qualified_name cluster_index_specification +RepackStmt: + REPACK opt_utility_option_list qualified_name USING INDEX name { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); + n->command = REPACK_COMMAND_REPACK; + n->relation = $3; + n->indexname = $6; + n->usingindex = true; + n->params = $2; + $$ = (Node *) n; + } + | REPACK opt_utility_option_list qualified_name opt_usingindex + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_REPACK; + n->relation = $3; + n->indexname = NULL; + n->usingindex = $4; + n->params = $2; + $$ = (Node *) n; + } + | REPACK '(' utility_option_list ')' + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_REPACK; + n->relation = NULL; + n->indexname = NULL; + n->usingindex = false; + n->params = $3; + $$ = (Node *) n; + } + | REPACK opt_usingindex + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_REPACK; + n->relation = NULL; + n->indexname = NULL; + n->usingindex = $2; + n->params = NIL; + $$ = (Node *) n; + } + | CLUSTER '(' utility_option_list ')' qualified_name cluster_index_specification + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_CLUSTER; n->relation = $5; n->indexname = $6; + n->usingindex = true; n->params = $3; $$ = (Node *) n; } | CLUSTER opt_utility_option_list { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); + n->command = REPACK_COMMAND_CLUSTER; n->relation = NULL; n->indexname = NULL; + n->usingindex = true; n->params = $2; $$ = (Node *) n; } /* unparenthesized VERBOSE kept for pre-14 compatibility */ | CLUSTER opt_verbose qualified_name cluster_index_specification { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); + n->command = REPACK_COMMAND_CLUSTER; n->relation = $3; n->indexname = $4; + n->usingindex = true; if ($2) n->params = list_make1(makeDefElem("verbose", NULL, @2)); $$ = (Node *) n; @@ -11951,20 +12009,24 @@ ClusterStmt: /* unparenthesized VERBOSE kept for pre-17 compatibility */ | CLUSTER VERBOSE { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); + n->command = REPACK_COMMAND_CLUSTER; n->relation = NULL; n->indexname = NULL; + n->usingindex = true; n->params = list_make1(makeDefElem("verbose", NULL, @2)); $$ = (Node *) n; } /* kept for pre-8.3 compatibility */ | CLUSTER opt_verbose name ON qualified_name { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); + n->command = REPACK_COMMAND_CLUSTER; n->relation = $5; n->indexname = $3; + n->usingindex = true; if ($2) n->params = list_make1(makeDefElem("verbose", NULL, @2)); $$ = (Node *) n; @@ -17960,6 +18022,7 @@ unreserved_keyword: | RELATIVE_P | RELEASE | RENAME + | REPACK | REPEATABLE | REPLACE | REPLICA @@ -18592,6 +18655,7 @@ bare_label_keyword: | RELATIVE_P | RELEASE | RENAME + | REPACK | REPEATABLE | REPLACE | REPLICA diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 5f442bc3bd4..cf6db581007 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -277,9 +277,9 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) return COMMAND_OK_IN_RECOVERY | COMMAND_OK_IN_READ_ONLY_TXN; } - case T_ClusterStmt: case T_ReindexStmt: case T_VacuumStmt: + case T_RepackStmt: { /* * These commands write WAL, so they're not strictly @@ -854,14 +854,14 @@ standard_ProcessUtility(PlannedStmt *pstmt, ExecuteCallStmt(castNode(CallStmt, parsetree), params, isAtomicContext, dest); break; - case T_ClusterStmt: - cluster(pstate, (ClusterStmt *) parsetree, isTopLevel); - break; - case T_VacuumStmt: ExecVacuum(pstate, (VacuumStmt *) parsetree, isTopLevel); break; + case T_RepackStmt: + ExecRepack(pstate, (RepackStmt *) parsetree, isTopLevel); + break; + case T_ExplainStmt: ExplainQuery(pstate, (ExplainStmt *) parsetree, params, dest); break; @@ -2851,10 +2851,6 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_CALL; break; - case T_ClusterStmt: - tag = CMDTAG_CLUSTER; - break; - case T_VacuumStmt: if (((VacuumStmt *) parsetree)->is_vacuumcmd) tag = CMDTAG_VACUUM; @@ -2862,6 +2858,10 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_ANALYZE; break; + case T_RepackStmt: + tag = CMDTAG_REPACK; + break; + case T_ExplainStmt: tag = CMDTAG_EXPLAIN; break; @@ -3499,7 +3499,7 @@ GetCommandLogLevel(Node *parsetree) lev = LOGSTMT_ALL; break; - case T_ClusterStmt: + case T_RepackStmt: lev = LOGSTMT_DDL; break; diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index c756c2bebaa..a1e10e8c2f6 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -268,6 +268,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) cmdtype = PROGRESS_COMMAND_ANALYZE; else if (pg_strcasecmp(cmd, "CLUSTER") == 0) cmdtype = PROGRESS_COMMAND_CLUSTER; + else if (pg_strcasecmp(cmd, "REPACK") == 0) + cmdtype = PROGRESS_COMMAND_REPACK; else if (pg_strcasecmp(cmd, "CREATE INDEX") == 0) cmdtype = PROGRESS_COMMAND_CREATE_INDEX; else if (pg_strcasecmp(cmd, "BASEBACKUP") == 0) diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 8b10f2313f3..59ff6e0923b 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -1247,7 +1247,7 @@ static const char *const sql_commands[] = { "DELETE FROM", "DISCARD", "DO", "DROP", "END", "EXECUTE", "EXPLAIN", "FETCH", "GRANT", "IMPORT FOREIGN SCHEMA", "INSERT INTO", "LISTEN", "LOAD", "LOCK", "MERGE INTO", "MOVE", "NOTIFY", "PREPARE", - "REASSIGN", "REFRESH MATERIALIZED VIEW", "REINDEX", "RELEASE", + "REASSIGN", "REFRESH MATERIALIZED VIEW", "REINDEX", "RELEASE", "REPACK", "RESET", "REVOKE", "ROLLBACK", "SAVEPOINT", "SECURITY LABEL", "SELECT", "SET", "SHOW", "START", "TABLE", "TRUNCATE", "UNLISTEN", "UPDATE", "VACUUM", "VALUES", "WITH", @@ -4997,6 +4997,37 @@ match_previous_words(int pattern_id, COMPLETE_WITH_QUERY(Query_for_list_of_tablespaces); } +/* REPACK */ + else if (Matches("REPACK")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_clusterables); + else if (Matches("REPACK", "(*)")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_clusterables); + /* If we have REPACK , then add "USING INDEX" */ + else if (Matches("REPACK", MatchAnyExcept("("))) + COMPLETE_WITH("USING INDEX"); + /* If we have REPACK (*) , then add "USING INDEX" */ + else if (Matches("REPACK", "(*)", MatchAny)) + COMPLETE_WITH("USING INDEX"); + /* If we have REPACK USING, then add the index as well */ + else if (Matches("REPACK", MatchAny, "USING", "INDEX")) + { + set_completion_reference(prev3_wd); + COMPLETE_WITH_SCHEMA_QUERY(Query_for_index_of_table); + } + else if (HeadMatches("REPACK", "(*") && + !HeadMatches("REPACK", "(*)")) + { + /* + * This fires if we're in an unfinished parenthesized option list. + * get_previous_words treats a completed parenthesized option list as + * one word, so the above test is correct. + */ + if (ends_with(prev_wd, '(') || ends_with(prev_wd, ',')) + COMPLETE_WITH("VERBOSE"); + else if (TailMatches("VERBOSE")) + COMPLETE_WITH("ON", "OFF"); + } + /* SECURITY LABEL */ else if (Matches("SECURITY")) COMPLETE_WITH("LABEL"); diff --git a/src/bin/scripts/Makefile b/src/bin/scripts/Makefile index 019ca06455d..f0c1bd4175c 100644 --- a/src/bin/scripts/Makefile +++ b/src/bin/scripts/Makefile @@ -16,7 +16,7 @@ subdir = src/bin/scripts top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS = createdb createuser dropdb dropuser clusterdb vacuumdb reindexdb pg_isready +PROGRAMS = createdb createuser dropdb dropuser clusterdb vacuumdb reindexdb pg_isready pg_repackdb override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) @@ -31,6 +31,7 @@ clusterdb: clusterdb.o common.o $(WIN32RES) | submake-libpq submake-libpgport su vacuumdb: vacuumdb.o vacuuming.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils reindexdb: reindexdb.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils pg_isready: pg_isready.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils +pg_repackdb: pg_repackdb.o vacuuming.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) createdb$(X) '$(DESTDIR)$(bindir)'/createdb$(X) @@ -41,6 +42,7 @@ install: all installdirs $(INSTALL_PROGRAM) vacuumdb$(X) '$(DESTDIR)$(bindir)'/vacuumdb$(X) $(INSTALL_PROGRAM) reindexdb$(X) '$(DESTDIR)$(bindir)'/reindexdb$(X) $(INSTALL_PROGRAM) pg_isready$(X) '$(DESTDIR)$(bindir)'/pg_isready$(X) + $(INSTALL_PROGRAM) pg_repackdb$(X) '$(DESTDIR)$(bindir)'/pg_repackdb$(X) installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' diff --git a/src/bin/scripts/meson.build b/src/bin/scripts/meson.build index a4fed59d1c9..18410fb80dd 100644 --- a/src/bin/scripts/meson.build +++ b/src/bin/scripts/meson.build @@ -42,6 +42,7 @@ vacuuming_common = static_library('libvacuuming_common', binaries = [ 'vacuumdb', + 'pg_repackdb' ] foreach binary : binaries binary_sources = files('@0@.c'.format(binary)) @@ -80,6 +81,7 @@ tests += { 't/100_vacuumdb.pl', 't/101_vacuumdb_all.pl', 't/102_vacuumdb_stages.pl', + 't/103_repackdb.pl', 't/200_connstr.pl', ], }, diff --git a/src/bin/scripts/pg_repackdb.c b/src/bin/scripts/pg_repackdb.c new file mode 100644 index 00000000000..23326372a77 --- /dev/null +++ b/src/bin/scripts/pg_repackdb.c @@ -0,0 +1,226 @@ +/*------------------------------------------------------------------------- + * + * pg_repackdb + * An utility to run REPACK + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * FIXME: this is missing a way to specify the index to use to repack one + * table, or whether to pass a WITH INDEX clause when multiple tables are + * used. Something like --index[=indexname]. Adding that bleeds into + * vacuuming.c as well. + * + * src/bin/scripts/pg_repackdb.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include + +#include "common.h" +#include "common/logging.h" +#include "fe_utils/option_utils.h" +#include "vacuuming.h" + +static void help(const char *progname); +void check_objfilter(void); + +int +main(int argc, char *argv[]) +{ + static struct option long_options[] = { + {"host", required_argument, NULL, 'h'}, + {"port", required_argument, NULL, 'p'}, + {"username", required_argument, NULL, 'U'}, + {"no-password", no_argument, NULL, 'w'}, + {"password", no_argument, NULL, 'W'}, + {"echo", no_argument, NULL, 'e'}, + {"quiet", no_argument, NULL, 'q'}, + {"dbname", required_argument, NULL, 'd'}, + {"all", no_argument, NULL, 'a'}, + {"table", required_argument, NULL, 't'}, + {"verbose", no_argument, NULL, 'v'}, + {"jobs", required_argument, NULL, 'j'}, + {"schema", required_argument, NULL, 'n'}, + {"exclude-schema", required_argument, NULL, 'N'}, + {"maintenance-db", required_argument, NULL, 2}, + {NULL, 0, NULL, 0} + }; + + const char *progname; + int optindex; + int c; + const char *dbname = NULL; + const char *maintenance_db = NULL; + ConnParams cparams; + bool echo = false; + bool quiet = false; + vacuumingOptions vacopts; + SimpleStringList objects = {NULL, NULL}; + int concurrentCons = 1; + int tbl_count = 0; + + /* initialize options */ + memset(&vacopts, 0, sizeof(vacopts)); + vacopts.mode = MODE_REPACK; + + /* the same for connection parameters */ + memset(&cparams, 0, sizeof(cparams)); + cparams.prompt_password = TRI_DEFAULT; + + pg_logging_init(argv[0]); + progname = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts")); + + handle_help_version_opts(argc, argv, progname, help); + + while ((c = getopt_long(argc, argv, "ad:eh:j:n:N:p:qt:U:vwW", + long_options, &optindex)) != -1) + { + switch (c) + { + case 'a': + objfilter |= OBJFILTER_ALL_DBS; + break; + case 'd': + objfilter |= OBJFILTER_DATABASE; + dbname = pg_strdup(optarg); + break; + case 'e': + echo = true; + break; + case 'h': + cparams.pghost = pg_strdup(optarg); + break; + case 'j': + if (!option_parse_int(optarg, "-j/--jobs", 1, INT_MAX, + &concurrentCons)) + exit(1); + break; + case 'n': + objfilter |= OBJFILTER_SCHEMA; + simple_string_list_append(&objects, optarg); + break; + case 'N': + objfilter |= OBJFILTER_SCHEMA_EXCLUDE; + simple_string_list_append(&objects, optarg); + break; + case 'p': + cparams.pgport = pg_strdup(optarg); + break; + case 'q': + quiet = true; + break; + case 't': + objfilter |= OBJFILTER_TABLE; + simple_string_list_append(&objects, optarg); + tbl_count++; + break; + case 'U': + cparams.pguser = pg_strdup(optarg); + break; + case 'v': + vacopts.verbose = true; + break; + case 'w': + cparams.prompt_password = TRI_NO; + break; + case 'W': + cparams.prompt_password = TRI_YES; + break; + case 2: + maintenance_db = pg_strdup(optarg); + break; + default: + /* getopt_long already emitted a complaint */ + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + } + + /* + * Non-option argument specifies database name as long as it wasn't + * already specified with -d / --dbname + */ + if (optind < argc && dbname == NULL) + { + objfilter |= OBJFILTER_DATABASE; + dbname = argv[optind]; + optind++; + } + + if (optind < argc) + { + pg_log_error("too many command-line arguments (first is \"%s\")", + argv[optind]); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + + /* + * Validate the combination of filters specified in the command-line + * options. + */ + check_objfilter(); + + vacuuming_main(&cparams, dbname, maintenance_db, &vacopts, &objects, + false, tbl_count, concurrentCons, + progname, echo, quiet); + exit(0); +} + +/* + * Verify that the filters used at command line are compatible. + */ +void +check_objfilter(void) +{ + if ((objfilter & OBJFILTER_ALL_DBS) && + (objfilter & OBJFILTER_DATABASE)) + pg_fatal("cannot repack all databases and a specific one at the same time"); + + if ((objfilter & OBJFILTER_TABLE) && + (objfilter & OBJFILTER_SCHEMA)) + pg_fatal("cannot repack all tables in schema(s) and specific table(s) at the same time"); + + if ((objfilter & OBJFILTER_TABLE) && + (objfilter & OBJFILTER_SCHEMA_EXCLUDE)) + pg_fatal("cannot repack specific table(s) and exclude schema(s) at the same time"); + + if ((objfilter & OBJFILTER_SCHEMA) && + (objfilter & OBJFILTER_SCHEMA_EXCLUDE)) + pg_fatal("cannot repack all tables in schema(s) and exclude schema(s) at the same time"); +} + +static void +help(const char *progname) +{ + printf(_("%s repacks a PostgreSQL database.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]... [DBNAME]\n"), progname); + printf(_("\nOptions:\n")); + printf(_(" -a, --all repack all databases\n")); + printf(_(" -d, --dbname=DBNAME database to repack\n")); + printf(_(" -e, --echo show the commands being sent to the server\n")); + printf(_(" -j, --jobs=NUM use this many concurrent connections to repack\n")); + printf(_(" -n, --schema=SCHEMA repack tables in the specified schema(s) only\n")); + printf(_(" -N, --exclude-schema=SCHEMA do not repack tables in the specified schema(s)\n")); + printf(_(" -q, --quiet don't write any messages\n")); + printf(_(" -t, --table='TABLE' repack specific table(s) only\n")); + printf(_(" -v, --verbose write a lot of output\n")); + printf(_(" -V, --version output version information, then exit\n")); + printf(_(" -?, --help show this help, then exit\n")); + printf(_("\nConnection options:\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); + printf(_(" -p, --port=PORT database server port\n")); + printf(_(" -U, --username=USERNAME user name to connect as\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n")); + printf(_(" --maintenance-db=DBNAME alternate maintenance database\n")); + printf(_("\nRead the description of the SQL command REPACK for details.\n")); + printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT); + printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); +} diff --git a/src/bin/scripts/t/103_repackdb.pl b/src/bin/scripts/t/103_repackdb.pl new file mode 100644 index 00000000000..51de4d7ab34 --- /dev/null +++ b/src/bin/scripts/t/103_repackdb.pl @@ -0,0 +1,24 @@ +# Copyright (c) 2021-2025, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +program_help_ok('pg_repackdb'); +program_version_ok('pg_repackdb'); +program_options_handling_ok('pg_repackdb'); + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->start; + +$node->issues_sql_like( + [ 'pg_repackdb', 'postgres' ], + qr/statement: REPACK.*;/, + 'SQL REPACK run'); + + +done_testing(); diff --git a/src/bin/scripts/vacuuming.c b/src/bin/scripts/vacuuming.c index 9be37fcc45a..e07071c38ee 100644 --- a/src/bin/scripts/vacuuming.c +++ b/src/bin/scripts/vacuuming.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * vacuuming.c - * Common routines for vacuumdb + * Common routines for vacuumdb and pg_repackdb * * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -166,6 +166,14 @@ vacuum_one_database(ConnParams *cparams, conn = connectDatabase(cparams, progname, echo, false, true); + if (vacopts->mode == MODE_REPACK && PQserverVersion(conn) < 190000) + { + /* XXX arguably, here we should use VACUUM FULL instead of failing */ + PQfinish(conn); + pg_fatal("cannot use the \"%s\" command on server versions older than PostgreSQL %s", + "REPACK", "19"); + } + if (vacopts->disable_page_skipping && PQserverVersion(conn) < 90600) { PQfinish(conn); @@ -258,9 +266,15 @@ vacuum_one_database(ConnParams *cparams, if (stage != ANALYZE_NO_STAGE) printf(_("%s: processing database \"%s\": %s\n"), progname, PQdb(conn), _(stage_messages[stage])); - else + else if (vacopts->mode == MODE_VACUUM) printf(_("%s: vacuuming database \"%s\"\n"), progname, PQdb(conn)); + else + { + Assert(vacopts->mode == MODE_REPACK); + printf(_("%s: repacking database \"%s\"\n"), + progname, PQdb(conn)); + } fflush(stdout); } @@ -350,7 +364,7 @@ vacuum_one_database(ConnParams *cparams, * through ParallelSlotsGetIdle. */ ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL); - run_vacuum_command(free_slot->connection, sql.data, + run_vacuum_command(free_slot->connection, vacopts, sql.data, echo, tabname); cell = cell->next; @@ -363,7 +377,7 @@ vacuum_one_database(ConnParams *cparams, } /* If we used SKIP_DATABASE_STATS, mop up with ONLY_DATABASE_STATS */ - if (vacopts->skip_database_stats && + if (vacopts->mode == MODE_VACUUM && vacopts->skip_database_stats && stage == ANALYZE_NO_STAGE) { const char *cmd = "VACUUM (ONLY_DATABASE_STATS);"; @@ -376,7 +390,7 @@ vacuum_one_database(ConnParams *cparams, } ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL); - run_vacuum_command(free_slot->connection, cmd, echo, NULL); + run_vacuum_command(free_slot->connection, vacopts, cmd, echo, NULL); if (!ParallelSlotsWaitCompletion(sa)) failed = true; @@ -708,6 +722,12 @@ vacuum_all_databases(ConnParams *cparams, int i; conn = connectMaintenanceDatabase(cparams, progname, echo); + if (vacopts->mode == MODE_REPACK && PQserverVersion(conn) < 190000) + { + PQfinish(conn); + pg_fatal("cannot use the \"%s\" command on server versions older than PostgreSQL %s", + "REPACK", "19"); + } result = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn AND datconnlimit <> -2 ORDER BY 1;", echo); @@ -761,7 +781,7 @@ vacuum_all_databases(ConnParams *cparams, } /* - * Construct a vacuum/analyze command to run based on the given + * Construct a vacuum/analyze/repack command to run based on the given * options, in the given string buffer, which may contain previous garbage. * * The table name used must be already properly quoted. The command generated @@ -777,7 +797,13 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, resetPQExpBuffer(sql); - if (vacopts->analyze_only) + if (vacopts->mode == MODE_REPACK) + { + appendPQExpBufferStr(sql, "REPACK"); + if (vacopts->verbose) + appendPQExpBufferStr(sql, " (VERBOSE)"); + } + else if (vacopts->analyze_only) { appendPQExpBufferStr(sql, "ANALYZE"); @@ -938,8 +964,8 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, * Any errors during command execution are reported to stderr. */ void -run_vacuum_command(PGconn *conn, const char *sql, bool echo, - const char *table) +run_vacuum_command(PGconn *conn, vacuumingOptions *vacopts, + const char *sql, bool echo, const char *table) { bool status; @@ -952,13 +978,21 @@ run_vacuum_command(PGconn *conn, const char *sql, bool echo, { if (table) { - pg_log_error("vacuuming of table \"%s\" in database \"%s\" failed: %s", - table, PQdb(conn), PQerrorMessage(conn)); + if (vacopts->mode == MODE_VACUUM) + pg_log_error("vacuuming of table \"%s\" in database \"%s\" failed: %s", + table, PQdb(conn), PQerrorMessage(conn)); + else + pg_log_error("repacking of table \"%s\" in database \"%s\" failed: %s", + table, PQdb(conn), PQerrorMessage(conn)); } else { - pg_log_error("vacuuming of database \"%s\" failed: %s", - PQdb(conn), PQerrorMessage(conn)); + if (vacopts->mode == MODE_VACUUM) + pg_log_error("vacuuming of database \"%s\" failed: %s", + PQdb(conn), PQerrorMessage(conn)); + else + pg_log_error("repacking of database \"%s\" failed: %s", + PQdb(conn), PQerrorMessage(conn)); } } } diff --git a/src/bin/scripts/vacuuming.h b/src/bin/scripts/vacuuming.h index d3f000840fa..154bc9925c0 100644 --- a/src/bin/scripts/vacuuming.h +++ b/src/bin/scripts/vacuuming.h @@ -17,6 +17,12 @@ #include "fe_utils/connect_utils.h" #include "fe_utils/simple_list.h" +typedef enum +{ + MODE_VACUUM, + MODE_REPACK +} RunMode; + /* For analyze-in-stages mode */ #define ANALYZE_NO_STAGE -1 #define ANALYZE_NUM_STAGES 3 @@ -24,6 +30,7 @@ /* vacuum options controlled by user flags */ typedef struct vacuumingOptions { + RunMode mode; bool analyze_only; bool verbose; bool and_analyze; @@ -87,8 +94,8 @@ extern void vacuum_all_databases(ConnParams *cparams, extern void prepare_vacuum_command(PQExpBuffer sql, int serverVersion, vacuumingOptions *vacopts, const char *table); -extern void run_vacuum_command(PGconn *conn, const char *sql, bool echo, - const char *table); +extern void run_vacuum_command(PGconn *conn, vacuumingOptions *vacopts, + const char *sql, bool echo, const char *table); extern char *escape_quotes(const char *src); diff --git a/src/include/commands/cluster.h b/src/include/commands/cluster.h index 60088a64cbb..890998d84bb 100644 --- a/src/include/commands/cluster.h +++ b/src/include/commands/cluster.h @@ -24,6 +24,7 @@ #define CLUOPT_RECHECK 0x02 /* recheck relation state */ #define CLUOPT_RECHECK_ISCLUSTERED 0x04 /* recheck relation state for * indisclustered */ +#define CLUOPT_ANALYZE 0x08 /* do an ANALYZE */ /* options for CLUSTER */ typedef struct ClusterParams @@ -31,8 +32,11 @@ typedef struct ClusterParams bits32 options; /* bitmask of CLUOPT_* */ } ClusterParams; -extern void cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel); -extern void cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params); + +extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); + +extern void cluster_rel(RepackCommand command, bool usingindex, + Relation OldHeap, Oid indexOid, ClusterParams *params); extern void check_index_is_clusterable(Relation OldHeap, Oid indexOid, LOCKMODE lockmode); extern void mark_index_clustered(Relation rel, Oid indexOid, bool is_internal); diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index 1cde4bd9bcf..5b6639c114c 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -56,24 +56,51 @@ #define PROGRESS_ANALYZE_PHASE_COMPUTE_EXT_STATS 4 #define PROGRESS_ANALYZE_PHASE_FINALIZE_ANALYZE 5 -/* Progress parameters for cluster */ -#define PROGRESS_CLUSTER_COMMAND 0 -#define PROGRESS_CLUSTER_PHASE 1 -#define PROGRESS_CLUSTER_INDEX_RELID 2 -#define PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED 3 -#define PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN 4 -#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5 -#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6 -#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7 +/* + * Progress parameters for REPACK. + * + * Note: Since REPACK shares some code with CLUSTER, these values are also + * used by CLUSTER. (CLUSTER is now deprecated, so it makes little sense to + * introduce a separate set of constants.) + */ +#define PROGRESS_REPACK_COMMAND 0 +#define PROGRESS_REPACK_PHASE 1 +#define PROGRESS_REPACK_INDEX_RELID 2 +#define PROGRESS_REPACK_HEAP_TUPLES_SCANNED 3 +#define PROGRESS_REPACK_HEAP_TUPLES_WRITTEN 4 +#define PROGRESS_REPACK_TOTAL_HEAP_BLKS 5 +#define PROGRESS_REPACK_HEAP_BLKS_SCANNED 6 +#define PROGRESS_REPACK_INDEX_REBUILD_COUNT 7 -/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */ -#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1 -#define PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP 2 -#define PROGRESS_CLUSTER_PHASE_SORT_TUPLES 3 -#define PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP 4 -#define PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES 5 -#define PROGRESS_CLUSTER_PHASE_REBUILD_INDEX 6 -#define PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP 7 +/* + * Phases of repack (as advertised via PROGRESS_REPACK_PHASE). + */ +#define PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP 1 +#define PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP 2 +#define PROGRESS_REPACK_PHASE_SORT_TUPLES 3 +#define PROGRESS_REPACK_PHASE_WRITE_NEW_HEAP 4 +#define PROGRESS_REPACK_PHASE_SWAP_REL_FILES 5 +#define PROGRESS_REPACK_PHASE_REBUILD_INDEX 6 +#define PROGRESS_REPACK_PHASE_FINAL_CLEANUP 7 + +/* + * Commands of PROGRESS_REPACK + * + * Currently we only have one command, so the PROGRESS_REPACK_COMMAND + * parameter is not necessary. However it makes cluster.c simpler if we have + * the same set of parameters for CLUSTER and REPACK - see the note on REPACK + * parameters above. + */ +#define PROGRESS_REPACK_COMMAND_REPACK 1 + +/* + * Progress parameters for cluster. + * + * Although we need to report REPACK and CLUSTER in separate views, the + * parameters and phases of CLUSTER are a subset of those of REPACK. Therefore + * we just use the appropriate values defined for REPACK above instead of + * defining a separate set of constants here. + */ /* Commands of PROGRESS_CLUSTER */ #define PROGRESS_CLUSTER_COMMAND_CLUSTER 1 diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 86a236bd58b..fcc25a0c592 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3949,16 +3949,26 @@ typedef struct AlterSystemStmt } AlterSystemStmt; /* ---------------------- - * Cluster Statement (support pbrown's cluster index implementation) + * Repack Statement * ---------------------- */ -typedef struct ClusterStmt +typedef enum RepackCommand +{ + REPACK_COMMAND_CLUSTER, + REPACK_COMMAND_REPACK, + REPACK_COMMAND_VACUUMFULL, +} RepackCommand; + +typedef struct RepackStmt { NodeTag type; - RangeVar *relation; /* relation being indexed, or NULL if all */ - char *indexname; /* original index defined */ + RepackCommand command; /* type of command being run */ + RangeVar *relation; /* relation being repacked */ + char *indexname; /* order tuples by this index */ + bool usingindex; /* whether USING INDEX is specified */ List *params; /* list of DefElem nodes */ -} ClusterStmt; +} RepackStmt; + /* ---------------------- * Vacuum and Analyze Statements diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index a4af3f717a1..22559369e2c 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -374,6 +374,7 @@ PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("release", RELEASE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("rename", RENAME, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("repack", REPACK, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index d250a714d59..cceb312f2b3 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -196,6 +196,7 @@ PG_CMDTAG(CMDTAG_REASSIGN_OWNED, "REASSIGN OWNED", false, false, false) PG_CMDTAG(CMDTAG_REFRESH_MATERIALIZED_VIEW, "REFRESH MATERIALIZED VIEW", true, false, false) PG_CMDTAG(CMDTAG_REINDEX, "REINDEX", true, false, false) PG_CMDTAG(CMDTAG_RELEASE, "RELEASE", false, false, false) +PG_CMDTAG(CMDTAG_REPACK, "REPACK", false, false, false) PG_CMDTAG(CMDTAG_RESET, "RESET", false, false, false) PG_CMDTAG(CMDTAG_REVOKE, "REVOKE", true, false, false) PG_CMDTAG(CMDTAG_REVOKE_ROLE, "REVOKE ROLE", false, false, false) diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h index dda813ab407..e69e366dcdc 100644 --- a/src/include/utils/backend_progress.h +++ b/src/include/utils/backend_progress.h @@ -28,6 +28,7 @@ typedef enum ProgressCommandType PROGRESS_COMMAND_CREATE_INDEX, PROGRESS_COMMAND_BASEBACKUP, PROGRESS_COMMAND_COPY, + PROGRESS_COMMAND_REPACK, } ProgressCommandType; #define PGSTAT_NUM_PROGRESS_PARAM 20 diff --git a/src/test/regress/expected/cluster.out b/src/test/regress/expected/cluster.out index 4d40a6809ab..5256628b51d 100644 --- a/src/test/regress/expected/cluster.out +++ b/src/test/regress/expected/cluster.out @@ -254,6 +254,63 @@ ORDER BY 1; clstr_tst_pkey (3 rows) +-- REPACK handles individual tables identically to CLUSTER, but it's worth +-- checking if it handles table hierarchies identically as well. +REPACK clstr_tst USING INDEX clstr_tst_c; +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 2'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + a | b | c | substring | length +----+-----+------------------+--------------------------------+-------- + 10 | 14 | catorce | | + 18 | 5 | cinco | | + 9 | 4 | cuatro | | + 26 | 19 | diecinueve | | + 12 | 18 | dieciocho | | + 30 | 16 | dieciseis | | + 24 | 17 | diecisiete | | + 2 | 10 | diez | | + 23 | 12 | doce | | + 11 | 2 | dos | | + 25 | 9 | nueve | | + 31 | 8 | ocho | | + 1 | 11 | once | | + 28 | 15 | quince | | + 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 29 | 7 | siete | | + 15 | 13 | trece | | + 22 | 30 | treinta | | + 17 | 32 | treinta y dos | | + 3 | 31 | treinta y uno | | + 5 | 3 | tres | | + 20 | 1 | uno | | + 6 | 20 | veinte | | + 14 | 25 | veinticinco | | + 21 | 24 | veinticuatro | | + 4 | 22 | veintidos | | + 19 | 29 | veintinueve | | + 16 | 28 | veintiocho | | + 27 | 26 | veintiseis | | + 13 | 27 | veintisiete | | + 7 | 23 | veintitres | | + 8 | 21 | veintiuno | | + 0 | 100 | in child table | | + 0 | 100 | in child table 2 | | +(34 rows) + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); +ERROR: insert or update on table "clstr_tst" violates foreign key constraint "clstr_tst_con" +DETAIL: Key (b)=(1111) is not present in table "clstr_tst_s". +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; + conname +---------------------- + clstr_tst_a_not_null + clstr_tst_con + clstr_tst_pkey +(3 rows) + SELECT relname, relkind, EXISTS(SELECT 1 FROM pg_class WHERE oid = c.reltoastrelid) AS hastoast FROM pg_class c WHERE relname LIKE 'clstr_tst%' ORDER BY relname; @@ -381,6 +438,35 @@ SELECT * FROM clstr_1; 2 (2 rows) +-- REPACK w/o argument performs no ordering, so we can only check which tables +-- have the relfilenode changed. +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_old AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); +SET SESSION AUTHORIZATION regress_clstr_user; +SET client_min_messages = ERROR; -- order of "skipping" warnings may vary +REPACK; +RESET client_min_messages; +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_new AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); +-- Do the actual comparison. Unlike CLUSTER, clstr_3 should have been +-- processed because there is nothing like clustering index here. +SELECT o.relname FROM relnodes_old o +JOIN relnodes_new n ON o.relname = n.relname +WHERE o.relfilenode <> n.relfilenode +ORDER BY o.relname; + relname +--------- + clstr_1 + clstr_3 +(2 rows) + +SET SESSION AUTHORIZATION regress_clstr_user; -- Test MVCC-safety of cluster. There isn't much we can do to verify the -- results with a single backend... CREATE TABLE clustertest (key int PRIMARY KEY); @@ -495,6 +581,43 @@ ALTER TABLE clstrpart SET WITHOUT CLUSTER; ERROR: cannot mark index clustered in partitioned table ALTER TABLE clstrpart CLUSTER ON clstrpart_idx; ERROR: cannot mark index clustered in partitioned table +-- Check that REPACK sets new relfilenodes: it should process exactly the same +-- tables as CLUSTER did. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart USING INDEX clstrpart_idx; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + relname | level | relkind | ?column? +-------------+-------+---------+---------- + clstrpart | 0 | p | t + clstrpart1 | 1 | p | t + clstrpart11 | 2 | r | f + clstrpart12 | 2 | p | t + clstrpart2 | 1 | r | f + clstrpart3 | 1 | p | t + clstrpart33 | 2 | r | f +(7 rows) + +-- And finally the same for REPACK w/o index. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + relname | level | relkind | ?column? +-------------+-------+---------+---------- + clstrpart | 0 | p | t + clstrpart1 | 1 | p | t + clstrpart11 | 2 | r | f + clstrpart12 | 2 | p | t + clstrpart2 | 1 | r | f + clstrpart3 | 1 | p | t + clstrpart33 | 2 | r | f +(7 rows) + DROP TABLE clstrpart; -- Ownership of partitions is checked CREATE TABLE ptnowner(i int unique) PARTITION BY LIST (i); @@ -513,7 +636,7 @@ CREATE TEMP TABLE ptnowner_oldnodes AS JOIN pg_class AS c ON c.oid=tree.relid; SET SESSION AUTHORIZATION regress_ptnowner; CLUSTER ptnowner USING ptnowner_i_idx; -WARNING: permission denied to cluster "ptnowner2", skipping it +WARNING: permission denied to execute CLUSTER on "ptnowner2", skipping it RESET SESSION AUTHORIZATION; SELECT a.relname, a.relfilenode=b.relfilenode FROM pg_class a JOIN ptnowner_oldnodes b USING (oid) ORDER BY a.relname COLLATE "C"; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 35e8aad7701..3a1d1d28282 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2071,6 +2071,29 @@ pg_stat_progress_create_index| SELECT s.pid, s.param15 AS partitions_done FROM (pg_stat_get_progress_info('CREATE INDEX'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); +pg_stat_progress_repack| SELECT s.pid, + s.datid, + d.datname, + s.relid, + CASE s.param2 + WHEN 0 THEN 'initializing'::text + WHEN 1 THEN 'seq scanning heap'::text + WHEN 2 THEN 'index scanning heap'::text + WHEN 3 THEN 'sorting tuples'::text + WHEN 4 THEN 'writing new heap'::text + WHEN 5 THEN 'swapping relation files'::text + WHEN 6 THEN 'rebuilding index'::text + WHEN 7 THEN 'performing final cleanup'::text + ELSE NULL::text + END AS phase, + (s.param3)::oid AS repack_index_relid, + s.param4 AS heap_tuples_scanned, + s.param5 AS heap_tuples_written, + s.param6 AS heap_blks_total, + s.param7 AS heap_blks_scanned, + s.param8 AS index_rebuild_count + FROM (pg_stat_get_progress_info('REPACK'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) + LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_vacuum| SELECT s.pid, s.datid, d.datname, diff --git a/src/test/regress/sql/cluster.sql b/src/test/regress/sql/cluster.sql index b7115f86104..cfcc3dc9761 100644 --- a/src/test/regress/sql/cluster.sql +++ b/src/test/regress/sql/cluster.sql @@ -76,6 +76,19 @@ INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass ORDER BY 1; +-- REPACK handles individual tables identically to CLUSTER, but it's worth +-- checking if it handles table hierarchies identically as well. +REPACK clstr_tst USING INDEX clstr_tst_c; + +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 2'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); + +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; SELECT relname, relkind, EXISTS(SELECT 1 FROM pg_class WHERE oid = c.reltoastrelid) AS hastoast @@ -159,6 +172,34 @@ INSERT INTO clstr_1 VALUES (1); CLUSTER clstr_1; SELECT * FROM clstr_1; +-- REPACK w/o argument performs no ordering, so we can only check which tables +-- have the relfilenode changed. +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_old AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); + +SET SESSION AUTHORIZATION regress_clstr_user; +SET client_min_messages = ERROR; -- order of "skipping" warnings may vary +REPACK; +RESET client_min_messages; + +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_new AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); + +-- Do the actual comparison. Unlike CLUSTER, clstr_3 should have been +-- processed because there is nothing like clustering index here. +SELECT o.relname FROM relnodes_old o +JOIN relnodes_new n ON o.relname = n.relname +WHERE o.relfilenode <> n.relfilenode +ORDER BY o.relname; + +SET SESSION AUTHORIZATION regress_clstr_user; + -- Test MVCC-safety of cluster. There isn't much we can do to verify the -- results with a single backend... @@ -229,6 +270,24 @@ SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM o CLUSTER clstrpart; ALTER TABLE clstrpart SET WITHOUT CLUSTER; ALTER TABLE clstrpart CLUSTER ON clstrpart_idx; + +-- Check that REPACK sets new relfilenodes: it should process exactly the same +-- tables as CLUSTER did. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart USING INDEX clstrpart_idx; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + +-- And finally the same for REPACK w/o index. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + DROP TABLE clstrpart; -- Ownership of partitions is checked diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a13e8162890..98242e25432 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2537,6 +2537,8 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepOriginId +RepackCommand +RepackStmt ReparameterizeForeignPathByChild_function ReplaceVarsFromTargetList_context ReplaceVarsNoMatchOption @@ -2603,6 +2605,7 @@ RtlNtStatusToDosError_t RuleInfo RuleLock RuleStmt +RunMode RunningTransactions RunningTransactionsData SASLStatus -- 2.43.0