Add detailed design for EXPLAIN COPY patch series (0001-0003)

doc/src/sgml/ref/copy.sgml

@@ -584,6 +584,13 @@ COPY <replaceable class="parameter">count</replaceable>
  <refsect1 id="sql-copy-notes">
   <title>Notes</title>
 
+   <para>
+    <xref linkend="sql-explain"/> can be used to display a description of
+    a <command>COPY</command> command, including the execution plan of the
+    source query of <literal>COPY
+    (<replaceable class="parameter">query</replaceable>) TO</literal>.
+   </para>
+
    <para>
     <command>COPY TO</command> can be used with plain
     tables, populated materialized views, and partitioned tables.

doc/src/sgml/ref/explain.sgml

@@ -98,7 +98,7 @@ EXPLAIN [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] <rep
     <command>EXPLAIN ANALYZE</command> on an
     <command>INSERT</command>, <command>UPDATE</command>,
     <command>DELETE</command>, <command>MERGE</command>,
-    <command>CREATE TABLE AS</command>,
+    <command>CREATE TABLE AS</command>, <command>COPY FROM</command>,
     or <command>EXECUTE</command> statement
     without letting the command affect your data, use this approach:
 <programlisting>
@@ -350,10 +350,43 @@ ROLLBACK;
       Any <command>SELECT</command>, <command>INSERT</command>, <command>UPDATE</command>,
       <command>DELETE</command>, <command>MERGE</command>,
       <command>VALUES</command>, <command>EXECUTE</command>,
-      <command>DECLARE</command>, <command>CREATE TABLE AS</command>, or
-      <command>CREATE MATERIALIZED VIEW AS</command> statement, whose execution
+      <command>DECLARE</command>, <command>CREATE TABLE AS</command>,
+      <command>CREATE MATERIALIZED VIEW AS</command>, or
+      <command>COPY</command> statement, whose execution
       plan you wish to see.
      </para>
+     <para>
+      For a <command>COPY</command> statement, <command>EXPLAIN</command>
+      shows a description of the <command>COPY</command> operation; when
+      copying the result of a query with
+      <command>COPY (<replaceable>query</replaceable>) TO</command>, the
+      execution plan of the query is also shown.  Without
+      <literal>ANALYZE</literal>, the <command>COPY</command> source or
+      destination file is not accessed, so a nonexistent file is not
+      detected by plain <command>EXPLAIN</command>.
+     </para>
+     <para>
+      <command>EXPLAIN ANALYZE</command> of a
+      <command>COPY (<replaceable>query</replaceable>) TO</command> statement
+      executes the source query, but does not produce the
+      <command>COPY</command> output: no file is written, and no data is
+      sent to the client.
+     </para>
+     <para>
+      <command>EXPLAIN ANALYZE</command> of a <command>COPY FROM</command>
+      statement executes the <command>COPY</command> and actually loads the
+      data.  In addition to the row counts, it reports a breakdown of the
+      time spent reading, parsing and converting the input data
+      (<literal>Input Time</literal>), inserting tuples into the table
+      (<literal>Insert Time</literal>), and updating indexes
+      (<literal>Index Update Time</literal>), as well as trigger execution
+      times.  The time breakdown is omitted when <literal>TIMING
+      OFF</literal> is specified.  Time spent elsewhere, for example
+      evaluating the <literal>WHERE</literal> clause or routing tuples to
+      partitions, is included only in the total execution time.
+      <command>COPY FROM STDIN</command> cannot be executed under
+      <command>EXPLAIN ANALYZE</command>.
+     </para>
     </listitem>
    </varlistentry>
   </variablelist>

src/backend/commands/Makefile

@@ -33,6 +33,7 @@ OBJS = \
 	dropcmds.o \
 	event_trigger.o \
 	explain.o \
+	explain_copy.o \
 	explain_dr.o \
 	explain_format.o \
 	explain_state.o \

src/backend/commands/copy.c

@@ -41,28 +41,37 @@
 #include "utils/rls.h"
 
 /*
- *	 DoCopy executes the SQL COPY statement
- *
- * Either unload or reload contents of table <relation>, depending on <from>.
- * (<from> = true means we are inserting into the table.)  In the "TO" case
- * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
- * or DELETE query.
+ * ProcessCopyTarget
+ *		Prepare the target of a COPY statement for execution
  *
- * If <pipe> is false, transfer is between the table and the file named
- * <filename>.  Otherwise, transfer is between the table and our regular
- * input/output stream. The latter could be either stdin/stdout or a
- * socket, depending on whether we're running under Postmaster control.
+ * This performs the permission checks and preparatory transformations
+ * needed before executing a COPY statement: checking the right to use
+ * COPY TO/FROM a file or program, opening and locking the target relation
+ * (if any), transforming and validating the WHERE clause, checking
+ * privileges on the target columns, and converting the statement into a
+ * query-based COPY when row-level security is enabled on the target
+ * relation.
  *
  * Do not allow a Postgres user without the 'pg_read_server_files' or
  * 'pg_write_server_files' role to read from or write to a file.
  *
  * Do not allow the copy if user doesn't have proper permission to access
  * the table or the specifically requested columns.
+ *
+ * The results are returned in *rel_p (the opened target relation, or NULL
+ * when the COPY uses a query), *relid_p (the OID of the target relation,
+ * or InvalidOid), *query_p (the source query for a query-based COPY TO,
+ * or NULL) and *whereClause_p (the transformed WHERE clause for COPY
+ * FROM, or NULL).
+ *
+ * When *rel_p is set, the relation is left open and locked; the caller is
+ * responsible for closing it (keeping the lock until end of transaction).
  */
 void
-DoCopy(ParseState *pstate, const CopyStmt *stmt,
-	   int stmt_location, int stmt_len,
-	   uint64 *processed)
+ProcessCopyTarget(ParseState *pstate, const CopyStmt *stmt,
+				  int stmt_location, int stmt_len,
+				  Relation *rel_p, Oid *relid_p,
+				  RawStmt **query_p, Node **whereClause_p)
 {
 	bool		is_from = stmt->is_from;
 	bool		pipe = (stmt->filename == NULL);
@@ -354,6 +363,39 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt,
 		rel = NULL;
 	}
 
+	*rel_p = rel;
+	*relid_p = relid;
+	*query_p = query;
+	*whereClause_p = whereClause;
+}
+
+/*
+ *	 DoCopy executes the SQL COPY statement
+ *
+ * Either unload or reload contents of table <relation>, depending on <from>.
+ * (<from> = true means we are inserting into the table.)  In the "TO" case
+ * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE
+ * or DELETE query.
+ *
+ * If <pipe> is false, transfer is between the table and the file named
+ * <filename>.  Otherwise, transfer is between the table and our regular
+ * input/output stream. The latter could be either stdin/stdout or a
+ * socket, depending on whether we're running under Postmaster control.
+ */
+void
+DoCopy(ParseState *pstate, const CopyStmt *stmt,
+	   int stmt_location, int stmt_len,
+	   uint64 *processed)
+{
+	bool		is_from = stmt->is_from;
+	Relation	rel;
+	Oid			relid;
+	RawStmt    *query;
+	Node	   *whereClause;
+
+	ProcessCopyTarget(pstate, stmt, stmt_location, stmt_len,
+					  &rel, &relid, &query, &whereClause);
+
 	if (is_from)
 	{
 		CopyFromState cstate;

src/backend/commands/copyfrom.c

@@ -35,6 +35,7 @@
 #include "commands/trigger.h"
 #include "executor/execPartition.h"
 #include "executor/executor.h"
+#include "executor/instrument.h"
 #include "executor/nodeModifyTable.h"
 #include "executor/tuptable.h"
 #include "foreign/fdwapi.h"
@@ -481,12 +482,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
 			TupleTableSlot **rslots;
 
 			/* insert into foreign table: let the FDW do it */
+			CopyFromInstrStartPhase(cstate);
 			rslots =
 				resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert(estate,
 																	 resultRelInfo,
 																	 &slots[sent],
 																	 NULL,
 																	 &inserted);
+			CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT);
 
 			sent += size;
 
@@ -553,12 +556,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
 		 * context before calling it.
 		 */
 		oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
+		CopyFromInstrStartPhase(cstate);
 		table_multi_insert(resultRelInfo->ri_RelationDesc,
 						   slots,
 						   nused,
 						   mycid,
 						   ti_options,
 						   buffer->bistate);
+		CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT);
 		MemoryContextSwitchTo(oldcontext);
 
 		for (i = 0; i < nused; i++)
@@ -572,10 +577,12 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo,
 				List	   *recheckIndexes;
 
 				cstate->cur_lineno = buffer->linenos[i];
+				CopyFromInstrStartPhase(cstate);
 				recheckIndexes =
 					ExecInsertIndexTuples(resultRelInfo,
 										  estate, 0, buffer->slots[i],
 										  NIL, NULL);
+				CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INDEX);
 				ExecARInsertTriggers(estate, resultRelInfo,
 									 slots[i], recheckIndexes,
 									 cstate->transition_capture);
@@ -774,6 +781,58 @@ CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri,
 	miinfo->bufferedBytes += tuplen;
 }
 
+/*
+ * Attach EXPLAIN ANALYZE instrumentation to a COPY FROM operation.  Must be
+ * called between BeginCopyFrom() and CopyFrom().
+ */
+void
+CopyFromSetInstrumentation(CopyFromState cstate, CopyFromInstrumentation *instr)
+{
+	cstate->instr = instr;
+}
+
+/*
+ * Collect the per-trigger statistics of the given result relations into the
+ * EXPLAIN ANALYZE instrumentation, in the current memory context.  This has
+ * to be done before the executor state of the COPY FROM operation is
+ * destroyed.
+ */
+static void
+CopyFromCollectTriggerStats(CopyFromInstrumentation *instr, List *resultrels)
+{
+	ListCell   *lc;
+
+	foreach(lc, resultrels)
+	{
+		ResultRelInfo *rInfo = (ResultRelInfo *) lfirst(lc);
+
+		if (!rInfo->ri_TrigDesc || !rInfo->ri_TrigInstrument)
+			continue;
+
+		for (int nt = 0; nt < rInfo->ri_TrigDesc->numtriggers; nt++)
+		{
+			Trigger    *trig = rInfo->ri_TrigDesc->triggers + nt;
+			TriggerInstrumentation *tginstr = rInfo->ri_TrigInstrument + nt;
+			CopyFromTriggerStats *stats;
+
+			/* ignore triggers that were never invoked */
+			if (tginstr->firings == 0)
+				continue;
+
+			stats = palloc0_object(CopyFromTriggerStats);
+			stats->trigger_name = pstrdup(trig->tgname);
+			if (OidIsValid(trig->tgconstraint))
+				stats->constraint_name = get_constraint_name(trig->tgconstraint);
+			stats->relation_name =
+				pstrdup(RelationGetRelationName(rInfo->ri_RelationDesc));
+			stats->firings = tginstr->firings;
+			stats->total = tginstr->instr.total;
+
+			instr->triggers = lappend(instr->triggers, stats);
+		}
+	}
+}
+
 /*
  * Copy FROM file to relation.
  */
@@ -910,6 +969,16 @@ CopyFrom(CopyFromState cstate)
 		ti_options |= TABLE_INSERT_FROZEN;
 	}
 
+	/*
+	 * If EXPLAIN ANALYZE instrumentation is attached, request trigger
+	 * instrumentation; InitResultRelInfo then sets up ri_TrigInstrument for
+	 * the target relation as well as for any partitions we route tuples to,
+	 * and trigger.c collects the statistics.
+	 */
+	if (cstate->instr)
+		estate->es_instrument = cstate->instr->collect_timing ?
+			INSTRUMENT_TIMER : INSTRUMENT_ROWS;
+
 	/*
 	 * We need a ResultRelInfo so we can use the regular executor's
 	 * index-entry-making machinery.  (There used to be a huge amount of code
@@ -1148,8 +1217,13 @@ CopyFrom(CopyFromState cstate)
 		ExecClearTuple(myslot);
 
 		/* Directly store the values/nulls array in the slot */
+		CopyFromInstrStartPhase(cstate);
 		if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull))
+		{
+			CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INPUT);
 			break;
+		}
+		CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INPUT);
 
 		if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE &&
 			cstate->escontext->error_occurred)
@@ -1408,10 +1482,12 @@ CopyFrom(CopyFromState cstate)
 					/* OK, store the tuple */
 					if (resultRelInfo->ri_FdwRoutine != NULL)
 					{
+						CopyFromInstrStartPhase(cstate);
 						myslot = resultRelInfo->ri_FdwRoutine->ExecForeignInsert(estate,
 																				 resultRelInfo,
 																				 myslot,
 																				 NULL);
+						CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT);
 
 						if (myslot == NULL) /* "do nothing" */
 							continue;	/* next tuple please */
@@ -1426,14 +1502,20 @@ CopyFrom(CopyFromState cstate)
 					else
 					{
 						/* OK, store the tuple and create index entries for it */
+						CopyFromInstrStartPhase(cstate);
 						table_tuple_insert(resultRelInfo->ri_RelationDesc,
 										   myslot, mycid, ti_options, bistate);
+						CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT);
 
 						if (resultRelInfo->ri_NumIndices > 0)
+						{
+							CopyFromInstrStartPhase(cstate);
 							recheckIndexes = ExecInsertIndexTuples(resultRelInfo,
 																   estate, 0,
 																   myslot, NIL,
 																   NULL);
+							CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INDEX);
+						}
 					}
 
 					/* AFTER ROW INSERT Triggers */
@@ -1493,6 +1575,25 @@ CopyFrom(CopyFromState cstate)
 	/* Handle queued AFTER triggers */
 	AfterTriggerEndQuery(estate);
 
+	/*
+	 * Hand the collected statistics over to the attached EXPLAIN ANALYZE
+	 * instrumentation, if any, before the executor state goes away.
+	 */
+	if (cstate->instr)
+	{
+		CopyFromInstrumentation *ci = cstate->instr;
+
+		ci->excluded = excluded;
+		ci->skipped = cstate->num_errors;
+		ci->show_relname =
+			(list_length(estate->es_opened_result_relations) > 1 ||
+			 estate->es_tuple_routing_result_relations != NIL ||
+			 estate->es_trig_target_relations != NIL);
+		CopyFromCollectTriggerStats(ci, estate->es_opened_result_relations);
+		CopyFromCollectTriggerStats(ci, estate->es_tuple_routing_result_relations);
+		CopyFromCollectTriggerStats(ci, estate->es_trig_target_relations);
+	}
+
 	ExecResetTupleTable(estate->es_tupleTable, false);
 
 	/* Allow the FDW to shut down */