[CF 5378] v30 - Conflict detection for update_deleted in logical replication

This branch was automatically generated by a robot using patches from an
email thread registered at:

https://commitfest.postgresql.org/patch/5378

The branch will be overwritten each time a new patch version is posted to
the thread, and also periodically to check for bitrot caused by changes
on the master branch.

Patch(es): https://www.postgresql.org/message-id/TYAPR01MB572428A7A4A875C069567B8E94852@TYAPR01MB5724.jpnprd01.prod.outlook.com
Author(s): Zhijie Hou

Author: Commitfest Bot

doc/src/sgml/catalogs.sgml

@@ -8088,6 +8088,18 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       </para></entry>
      </row>
 
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>subretainconflictinfo</structfield> <type>bool</type>
+      </para>
+      <para>
+       If true, the detection of <xref linkend="conflict-update-deleted"/> is
+       enabled and the information (e.g., dead tuples, commit timestamps, and
+       origins) on the subscriber that is still useful for conflict detection
+       is retained.
+      </para></entry>
+     </row>
+
      <row>
       <entry role="catalog_table_entry"><para role="column_definition">
        <structfield>subconninfo</structfield> <type>text</type>

doc/src/sgml/config.sgml

@@ -4945,6 +4945,8 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
           new setting.
           This setting has no effect if <varname>primary_conninfo</varname> is not
           set or the server is not in standby mode.
+          The name cannot be <literal>pg_conflict_detection</literal>, as it is
+          reserved for logical replication conflict detection.
          </para>
         </listitem>
        </varlistentry>
@@ -5377,6 +5379,50 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-max-conflict-retention-duration" xreflabel="max_conflict_retention_duration">
+      <term><varname>max_conflict_retention_duration</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>max_conflict_retention_duration</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Maximum duration (in milliseconds) for which conflict
+        information can be retained for conflict detection by the apply worker.
+        The default value is <literal>0</literal>, indicating that conflict
+        information is retained until it is no longer needed for detection
+        purposes.
+       </para>
+       <para>
+        The replication slot
+        <quote><literal>pg_conflict_detection</literal></quote> that used to
+        retain conflict information will be invalidated if all apply workers
+        associated with the subscriptions, where
+        <literal>retain_conflict_info</literal> is enabled, confirm that the
+        retention duration exceeded the
+        <literal>max_conflict_retention_duration</literal>. If the replication
+        slot is invalidated, you can disable
+        <literal>retain_conflict_info</literal> and re-enable it after
+        confirming this replication slot has been dropped. Alternatively, the
+        invalidated slot will be automatically dropped and re-created once the
+        apply worker confirms that the retention duration is within the
+        specified limit.
+       </para>
+       <para>
+        This option is effective only if a subscription with
+        <literal>retain_conflict_info</literal> enabled is present, and the
+        associated apply worker is active.
+       </para>
+       <warning>
+        <para>
+         Note that setting a non-zero value for this option could lead to
+         conflict information being removed prematurely, potentially missing
+         some conflict detections.
+        </para>
+       </warning>
+      </listitem>
+     </varlistentry>
+
      </variablelist>
     </sect2>
 

doc/src/sgml/func.sgml

@@ -29710,7 +29710,9 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
        </para>
        <para>
         Creates a new physical replication slot named
-        <parameter>slot_name</parameter>. The optional second parameter,
+        <parameter>slot_name</parameter>. The name cannot be
+        <literal>pg_conflict_detection</literal>, as it is reserved for
+        logical replication conflict detection. The optional second parameter,
         when <literal>true</literal>, specifies that the <acronym>LSN</acronym> for this
         replication slot be reserved immediately; otherwise
         the <acronym>LSN</acronym> is reserved on first connection from a streaming
@@ -29754,7 +29756,9 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
        <para>
         Creates a new logical (decoding) replication slot named
         <parameter>slot_name</parameter> using the output plugin
-        <parameter>plugin</parameter>. The optional third
+        <parameter>plugin</parameter>. The name cannot be
+        <literal>pg_conflict_detection</literal>, as it is reserved for
+        logical replication conflict detection. The optional third
         parameter, <parameter>temporary</parameter>, when set to true, specifies that
         the slot should not be permanently stored to disk and is only meant
         for use by the current session. Temporary slots are also
@@ -29784,6 +29788,8 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
        <para>
         Copies an existing physical replication slot named <parameter>src_slot_name</parameter>
         to a physical replication slot named <parameter>dst_slot_name</parameter>.
+        The new slot name cannot be <literal>pg_conflict_detection</literal>,
+        as it is reserved for logical replication conflict detection.
         The copied physical slot starts to reserve WAL from the same <acronym>LSN</acronym> as the
         source slot.
         <parameter>temporary</parameter> is optional. If <parameter>temporary</parameter>
@@ -29806,7 +29812,9 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
         Copies an existing logical replication slot
         named <parameter>src_slot_name</parameter> to a logical replication
         slot named <parameter>dst_slot_name</parameter>, optionally changing
-        the output plugin and persistence.  The copied logical slot starts
+        the output plugin and persistence.  The name cannot be
+        <literal>pg_conflict_detection</literal>, as it is reserved for
+        logical replication conflict detection.  The copied logical slot starts
         from the same <acronym>LSN</acronym> as the source logical slot.  Both
         <parameter>temporary</parameter> and <parameter>plugin</parameter> are
         optional; if they are omitted, the values of the source slot are used.

doc/src/sgml/logical-replication.sgml

@@ -1818,6 +1818,21 @@ test_sub=# SELECT * from tab_gen_to_gen;
       </para>
      </listitem>
     </varlistentry>
+    <varlistentry id="conflict-update-deleted" xreflabel="update_deleted">
+     <term><literal>update_deleted</literal></term>
+     <listitem>
+      <para>
+       The tuple to be updated was deleted by another origin. The update will
+       simply be skipped in this scenario.
+       Note that this conflict can only be detected when
+       <xref linkend="guc-track-commit-timestamp"/>
+       and <link linkend="sql-createsubscription-params-with-retain-conflict-info"><literal>retain_conflict_info</literal></link>
+       are enabled. Note that if a tuple cannot be found due to the table being
+       truncated only a <literal>update_missing</literal> conflict will arise.
+       arise
+      </para>
+     </listitem>
+    </varlistentry>
     <varlistentry id="conflict-update-origin-differs" xreflabel="update_origin_differs">
      <term><literal>update_origin_differs</literal></term>
      <listitem>
@@ -2408,7 +2423,8 @@ CONTEXT:  processing remote data for replication origin "pg_16395" during "INSER
    <para>
     <link linkend="guc-max-replication-slots"><varname>max_replication_slots</varname></link>
     must be set to at least the number of subscriptions expected to connect,
-    plus some reserve for table synchronization.
+    plus some reserve for table synchronization and one if
+    <link linkend="sql-createsubscription-params-with-retain-conflict-info"><literal>retain_conflict_info</literal></link> is enabled.
    </para>
 
    <para>

doc/src/sgml/monitoring.sgml

@@ -2109,6 +2109,19 @@ description | Waiting for a newly initialized WAL file to reach durable storage
        sender; NULL for parallel apply workers
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>retain_conflict_info</structfield> <type>boolean</type>
+      </para>
+      <para>
+       True if <link linkend="sql-createsubscription-params-with-retain-conflict-info"><literal>retain_conflict_info</literal></link>
+       is enabled and the duration for which conflict information is
+       retained for conflict detection by this apply worker does not exceed
+       <link linkend="guc-max-conflict-retention-duration"><literal>max_conflict_retention_duration</literal></link>; NULL for
+       parallel apply workers and table synchronization workers.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -2194,6 +2207,17 @@ description | Waiting for a newly initialized WAL file to reach durable storage
       </para></entry>
      </row>
 
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>confl_update_deleted</structfield> <type>bigint</type>
+      </para>
+      <para>
+       Number of times the tuple to be updated was deleted by another origin
+       during the application of changes. See <xref linkend="conflict-update-deleted"/>
+       for details about this conflict.
+      </para></entry>
+     </row>
+
      <row>
       <entry role="catalog_table_entry"><para role="column_definition">
        <structfield>confl_update_origin_differs</structfield> <type>bigint</type>

doc/src/sgml/protocol.sgml

@@ -2220,6 +2220,8 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
          <para>
           The name of the slot to create. Must be a valid replication slot
           name (see <xref linkend="streaming-replication-slots-manipulation"/>).
+          The name cannot be <literal>pg_conflict_detection</literal>, as it
+          is reserved for logical replication conflict detection.
          </para>
         </listitem>
        </varlistentry>
@@ -2638,6 +2640,69 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
          </variablelist>
         </listitem>
        </varlistentry>
+
+       <varlistentry id="protocol-replication-primary-status-update">
+        <term>Primary status update (B)</term>
+        <listitem>
+         <variablelist>
+          <varlistentry>
+           <term>Byte1('s')</term>
+           <listitem>
+            <para>
+             Identifies the message as a primary status update.
+            </para>
+           </listitem>
+          </varlistentry>
+
+          <varlistentry>
+           <term>Int64</term>
+           <listitem>
+            <para>
+             The latest WAL write position on the server.
+            </para>
+           </listitem>
+          </varlistentry>
+
+          <varlistentry>
+           <term>Int32</term>
+           <listitem>
+            <para>
+             The oldest transaction ID that is currently in the commit
+             phase on the server.
+            </para>
+           </listitem>
+          </varlistentry>
+
+          <varlistentry>
+           <term>Int32</term>
+           <listitem>
+            <para>
+             The next transaction ID to be assigned on the server.
+            </para>
+           </listitem>
+          </varlistentry>
+
+          <varlistentry>
+           <term>Int32</term>
+           <listitem>
+            <para>
+             The epoch of the next transaction ID to be assigned.
+            </para>
+           </listitem>
+          </varlistentry>
+
+          <varlistentry>
+           <term>Int64</term>
+           <listitem>
+            <para>
+             The server's system clock at the time of transmission, as
+             microseconds since midnight on 2000-01-01.
+            </para>
+           </listitem>
+          </varlistentry>
+         </variablelist>
+        </listitem>
+       </varlistentry>
       </variablelist>
 
       <para>
@@ -2782,6 +2847,33 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
          </variablelist>
         </listitem>
        </varlistentry>
+
+       <varlistentry id="protocol-replication-standby-wal-status-request">
+        <term>Request primary status update (F)</term>
+        <listitem>
+         <variablelist>
+          <varlistentry>
+           <term>Byte1('p')</term>
+           <listitem>
+            <para>
+             Identifies the message as a request for a primary status update.
+            </para>
+           </listitem>
+          </varlistentry>
+
+          <varlistentry>
+           <term>Int64</term>
+           <listitem>
+            <para>
+             The client's system clock at the time of transmission, as
+             microseconds since midnight on 2000-01-01.
+            </para>
+           </listitem>
+          </varlistentry>
+         </variablelist>
+        </listitem>
+       </varlistentry>
+
       </variablelist>
      </listitem>
     </varlistentry>

doc/src/sgml/ref/alter_subscription.sgml

@@ -235,8 +235,9 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO <
       <link linkend="sql-createsubscription-params-with-password-required"><literal>password_required</literal></link>,
       <link linkend="sql-createsubscription-params-with-run-as-owner"><literal>run_as_owner</literal></link>,
       <link linkend="sql-createsubscription-params-with-origin"><literal>origin</literal></link>,
-      <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link>, and
-      <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link>.
+      <link linkend="sql-createsubscription-params-with-failover"><literal>failover</literal></link>,
+      <link linkend="sql-createsubscription-params-with-two-phase"><literal>two_phase</literal></link>, and
+      <link linkend="sql-createsubscription-params-with-retain-conflict-info"><literal>retain_conflict_info</literal></link>.
       Only a superuser can set <literal>password_required = false</literal>.
      </para>
 
@@ -285,6 +286,13 @@ ALTER SUBSCRIPTION <replaceable class="parameter">name</replaceable> RENAME TO <
       option is changed from <literal>true</literal> to <literal>false</literal>,
       the publisher will replicate the transactions again when they are committed.
      </para>
+
+     <para>
+      If the <link linkend="sql-createsubscription-params-with-retain-conflict-info"><literal>retain_conflict_info</literal></link>
+      option is altered to <literal>false</literal> and no other subscription
+      has this option enabled, the additional replication slot that was created
+      to retain conflict information will be dropped.
+     </para>
     </listitem>
    </varlistentry>
 

doc/src/sgml/ref/create_subscription.sgml

@@ -169,7 +169,9 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
         <listitem>
          <para>
           Name of the publisher's replication slot to use.  The default is
-          to use the name of the subscription for the slot name.
+          to use the name of the subscription for the slot name. The name cannot
+          be <literal>pg_conflict_detection</literal>, as it is reserved for
+          logical replication conflict detection.
          </para>
 
          <para>
@@ -435,6 +437,35 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
          </para>
         </listitem>
        </varlistentry>
+
+      <varlistentry id="sql-createsubscription-params-with-retain-conflict-info">
+        <term><literal>retain_conflict_info</literal> (<type>boolean</type>)</term>
+        <listitem>
+         <para>
+          Specifies whether the information (e.g., dead tuples, commit
+          timestamps, and origins) on the subscriber that is still useful for
+          conflict detection is retained. The default is
+          <literal>false</literal>. If set to true, the detection of
+          <xref linkend="conflict-update-deleted"/> is enabled, and an
+          additional replication slot named
+          <quote><literal>pg_conflict_detection</literal></quote> will be
+          created on the subscriber to prevent the conflict information from
+          being removed.
+         </para>
+
+         <para>
+          Note that the information useful for conflict detection is retained
+          only after the creation of the additional slot. You can verify the
+          existence of this slot by querying <link linkend="view-pg-replication-slots">pg_replication_slots</link>.<structfield>conflicting</structfield>
+          And even if multiple subscriptions on one node enable this option,
+          only one replication slot will be created.
+         </para>
+
+         <para>
+          This option cannot be enabled if the publisher is also a physical standby.
+         </para>
+        </listitem>
+       </varlistentry>
       </variablelist></para>
 
     </listitem>

doc/src/sgml/system-views.sgml

@@ -2936,6 +2936,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
           <xref linkend="guc-idle-replication-slot-timeout"/> duration.
          </para>
         </listitem>
+        <listitem>
+         <para>
+          <literal>conflict_retention_exceeds_max_duration</literal> means that
+          the duration for retaining conflict information, which is used
+          in logical replication conflict detection, has exceeded the maximum
+          allowable limit. It is set only for the slot
+          <literal>pg_conflict_detection</literal>, which is created when
+          <link linkend="sql-createsubscription-params-with-retain-conflict-info"><literal>retain_conflict_info</literal></link>
+          is enabled.
+         </para>
+        </listitem>
        </itemizedlist>
       </para></entry>
      </row>