From 486075a960dc035460cef325be2c07280f4954f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 8 Jul 2026 08:26:08 +0000 Subject: [PATCH 1/9] Add detailed design for EXPLAIN COPY patch series (0001-0003) Design document covering: - 0001: grammar (ExplainableStmt + CopyStmt), ExplainOneUtility dispatch, new explain_copy.c, DoCopy/BeginCopyTo refactoring, non-ANALYZE output - 0002: EXPLAIN ANALYZE COPY (query) TO executing the query while producing no COPY output - 0003: EXPLAIN ANALYZE COPY FROM with per-phase timing breakdown (input/insert/index/trigger), instrumentation plumbing, overhead mitigation, output schema, and test plan Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- EXPLAIN_COPY_DESIGN.md | 604 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 604 insertions(+) create mode 100644 EXPLAIN_COPY_DESIGN.md diff --git a/EXPLAIN_COPY_DESIGN.md b/EXPLAIN_COPY_DESIGN.md new file mode 100644 index 0000000000000..c82ef7255ddcf --- /dev/null +++ b/EXPLAIN_COPY_DESIGN.md @@ -0,0 +1,604 @@ +# EXPLAIN COPY 詳細設計(パッチ 0001〜0003) + +対象: PostgreSQL 20devel (master) +目的: EXPLAIN の対象コマンドに COPY を追加し、COPY FROM の時間内訳を表示する。 + +パッチシリーズの意味論(最初に固定し、以後変更しない): + +| パッチ | できるようになること | 実行の有無 | +|---|---|---| +| 0001 | `EXPLAIN COPY ...`(全バリアント、ANALYZE なし) | 一切実行しない | +| 0002 | `EXPLAIN ANALYZE COPY (query) TO ...` | クエリを実行、COPY 出力(ファイル/STDOUT)は生成しない | +| 0003 | `EXPLAIN ANALYZE COPY ... FROM ...` + 時間内訳 | 実際にデータをロードする | + +--- + +## 0001: 文法・ディスパッチ・非 ANALYZE の EXPLAIN + +### 0001-1. 文法 (src/backend/parser/gram.y) + +`ExplainableStmt`(gram.y:12846)に `| CopyStmt` を追加する。 + +``` +ExplainableStmt: + SelectStmt + | ... + | ExecuteStmt + | CopyStmt /* by default all are $$=$1 */ +``` + +bison 3.8.2 で shift/reduce 競合が発生しないことは検証済み(gram.y は +`%expect 0` のため、競合があればビルドが失敗する)。 + +`PreparableStmt` には追加しない(PREPARE 対象外のため EXPLAIN EXECUTE 経由で +CopyStmt が来ることはない)。 + +### 0001-2. パース解析 + +変更不要。`transformExplainStmt`(analyze.c:3461)は内包文を +`transformOptionalSelectInto` → `transformStmt` に渡し、CopyStmt は +transformStmt の default 分岐(analyze.c:435-444)で CMD_UTILITY の Query に +包まれる。実行時は ExplainQuery → ExplainOneQuery → ExplainOneUtility に +到達する(NotifyStmt と同じ経路)。 + +### 0001-3. ディスパッチ (src/backend/commands/explain.c) + +`ExplainOneUtility()`(explain.c:396)の NotifyStmt 分岐の前に追加: + +```c +else if (IsA(utilityStmt, CopyStmt)) + ExplainCopyStmt(castNode(CopyStmt, utilityStmt), es, pstate, params); +``` + +CopyStmt は変更しない(const 扱い)ため、EXPLAIN EXECUTE 経路のような +copyObject は不要。 + +### 0001-4. copy.c のリファクタリング + +DoCopy(copy.c:63-386)の前半(74〜355 行: 権限チェック〜RLS 変換)を +共通関数に抽出する。**挙動変更なし**。 + +```c +/* copy.c / copy.h */ +void ProcessCopyTarget(ParseState *pstate, const CopyStmt *stmt, + int stmt_location, int stmt_len, + Relation *rel_p, /* out: 対象リレーション(query TO は NULL) */ + Oid *relid_p, /* out: RLS 再確認用 relid */ + RawStmt **query_p, /* out: query TO / RLS 変換後クエリ */ + Node **whereClause_p); /* out: FROM の WHERE(変換済み) */ +``` + +抽出内容: +- file/PROGRAM のロール権限チェック(pg_read_server_files 等) +- リレーションの open + lock(FROM: RowExclusiveLock, TO: AccessShareLock) +- WHERE 句の transform と検証 +- 列単位 ACL チェック(ExecCheckPermissions) +- RLS 有効時の relation TO → query TO 変換(copy.c:242-342) + +DoCopy は `ProcessCopyTarget` + 実行部(357-382 行)+ table_close となる。 + +EXPLAIN(非 ANALYZE)でもこの関数を通すことで、存在しないテーブル・権限 +不足は通常の COPY と同様に検出される(EXPLAIN INSERT が ExecutorStart で +ACL チェックされるのと整合)。ファイルは open しないため、ファイル不存在は +検出されない(EXPLAIN の一般的な性質として許容し、ドキュメントに記載)。 + +### 0001-5. copyto.c のリファクタリング + +BeginCopyTo のクエリ解析・検証部(copyto.c:909-986)を抽出する。 +**挙動変更なし**。 + +```c +/* copyto.c / copy.h */ +Query *CopyToTransformQuery(ParseState *pstate, RawStmt *raw_query); +``` + +内容: pg_analyze_and_rewrite_fixedparams、DO INSTEAD ルール拒否、 +SELECT INTO 拒否、ユーティリティ文拒否、RETURNING 必須チェック。 +プランニングと relationOids 再確認(copyto.c:989-1015)は呼び出し元に残す。 + +### 0001-6. explain_copy.c(新規ファイル) + +explain.c は近年 explain_dr.c / explain_format.c / explain_state.c に分割 +されており、この流れに沿って `src/backend/commands/explain_copy.c` を新設。 + +```c +/* explain.h に宣言 */ +void ExplainCopyStmt(CopyStmt *stmt, ExplainState *es, + ParseState *pstate, ParamListInfo params); +``` + +0001 時点の処理フロー: + +``` +ProcessCopyTarget(pstate, stmt, ..., &rel, &relid, &query, &whereClause); + +if (stmt->is_from) +{ + if (es->analyze) + ereport(ERROR, "EXPLAIN ANALYZE is not yet supported for COPY FROM"); + /* 0003 で解除 */ + ExplainCopyFromInfo(stmt, rel, es); /* 静的情報のみ */ +} +else if (query != NULL) /* COPY (query) TO / RLS 変換された relation TO */ +{ + if (es->analyze) + ereport(ERROR, "..."); /* 0002 で解除 */ + ExplainCopyToQuery(stmt, query, relid, es, pstate, params); +} +else /* COPY relation TO */ +{ + if (es->analyze) + ereport(ERROR, "..."); /* relation TO の ANALYZE は将来課題(Phase 4) */ + ExplainCopyToInfo(stmt, rel, es); +} + +if (rel) + table_close(rel, NoLock); +``` + +STDIN/STDOUT は 0001 では**許可**する(実行しないためプロトコル問題は +発生しない)。ANALYZE 時の制限は 0002/0003 で導入する。 + +#### ExplainCopyToQuery の実装(0001 の中核) + +standard_ExplainOneQuery(explain.c:324-381)を模倣し、COPY 用の追加情報を +ExplainOnePlan に渡す: + +```c +static void +ExplainCopyToQuery(CopyStmt *stmt, RawStmt *raw_query, Oid queryRelId, + ExplainState *es, ParseState *pstate, ParamListInfo params) +{ + Query *query; + PlannedStmt *plan; + instr_time planstart, planduration; + BufferUsage bufusage_start, bufusage; /* es->buffers 時 */ + MemoryContextCounters mem_counters; /* es->memory 時 */ + JumbleState *jstate = NULL; + + query = CopyToTransformQuery(pstate, raw_query); + + /* CTAS 分岐(explain.c:429-434)と同様に jumble + フック */ + if (IsQueryIdEnabled()) + jstate = JumbleQuery(query); + if (post_parse_analyze_hook) + (*post_parse_analyze_hook) (pstate, query, jstate); + + /* standard_ExplainOneQuery と同じ計測付きでプランニング */ + ... INSTR_TIME_SET_CURRENT(planstart) ... + plan = pg_plan_query(query, pstate->p_sourcetext, + CURSOR_OPT_PARALLEL_OK, params, es); + ... planduration 算出 ... + + /* BeginCopyTo(copyto.c:1003-1015)と同じ RLS 再確認 */ + if (OidIsValid(queryRelId) && + !list_member_oid(plan->relationOids, queryRelId)) + ereport(ERROR, "relation referenced by COPY statement has changed"); + + ExplainOnePlan(plan, NULL, es, pstate->p_sourcetext, params, + pstate->p_queryEnv, &planduration, + (es->buffers ? &bufusage : NULL), + (es->memory ? &mem_counters : NULL), + stmt); /* ← 新パラメータ */ +} +``` + +#### ExplainOnePlan のシグネチャ拡張 + +COPY の静的情報を「Query」グループの**内側**に出力するため、 +`ExplainOnePlan()`(explain.c:500)に `const CopyStmt *copystmt` パラメータを +追加する(NULL 可)。既存呼び出し元(explain.c、prepare.c)は NULL を渡す。 + +理由: JSON 等の構造化フォーマットで、ExplainOnePlan が開く無ラベルの +"Query" グループ(explain.c:603)の外側にラベル付きグループを重ねると +不正な JSON になる。CTAS が `into` パラメータで同じ問題を解決している +前例に従う。ABI 変化はメジャーリリースの通例として許容(planduration / +bufusage / mem_counters 追加時と同じ)。 + +ExplainOnePlan 内では ExplainPrintPlan 直後に、copystmt が非 NULL なら +ExplainPrintCopyInfo(explain_copy.c で定義、後述の共通出力関数)を呼ぶ。 + +#### 静的情報の出力関数(FROM / TO 共通) + +```c +static void ExplainPrintCopyInfo(const CopyStmt *stmt, Relation rel, + ExplainState *es); +``` + +出力プロパティ(値が既定値のものは省略): + +| プロパティ | 出力条件 | 値 | +|---|---|---| +| Relation Name / Schema | rel != NULL(Schema は VERBOSE 時) | 対象テーブル | +| Format | 常時 | text / csv / binary(拡張フォーマット名も可) | +| Source(FROM)/ Target(TO) | 常時 | file / program / stdin / stdout | +| File | file/program 時 | ファイル名 or コマンド(文中に既出のため常時表示) | +| Freeze | FREEZE 時 | true | +| On Error | on_error != stop 時 | ignore / set_null | + +オプションの単純な復唱(DELIMITER 等)は行わない(文面に書いてある情報で +あり、レビューで削られるのが常のため最小限とする)。 + +#### TEXT フォーマット出力例(0001) + +``` +=# EXPLAIN COPY tab FROM '/tmp/tab.csv' WITH (FORMAT csv); + QUERY PLAN +------------------------------------------ + Copy From on tab + Format: csv + Source: file + File: /tmp/tab.csv + +=# EXPLAIN COPY (SELECT * FROM t WHERE a > 0) TO stdout; + QUERY PLAN +------------------------------------------ + Seq Scan on t (cost=0.00..41.88 rows=850 width=8) + Filter: (a > 0) + Copy To + Format: text + Target: stdout +``` + +JSON 出力例: + +```json +[{ "Copy From": { "Relation Name": "tab", "Format": "csv", + "Source": "file", "File": "/tmp/tab.csv" } }] + +[{ "Plan": { ... }, + "Copy": { "Format": "text", "Target": "stdout" } }] +``` + +FROM / relation TO は ExplainOpenGroup("Query", NULL, true, es) → +ExplainOpenGroup("Copy From", "Copy From", true, es) → プロパティ、の +入れ子で出力する(TEXT の見出し行 "Copy From on tab" は +format == EXPLAIN_FORMAT_TEXT のとき手動出力)。 + +### 0001-7. その他の変更 + +- **psql タブ補完** (src/bin/psql/tab-complete.in.c): EXPLAIN の後続候補に + COPY を追加。 +- **ドキュメント**: explain.sgml の対象文リスト(SELECT, INSERT, ...)に + COPY を追加。「ANALYZE は 0001 時点では COPY に未対応」の footnote は + シリーズ全体コミット時には不要になるため、0002/0003 と同時期の + コミットを想定して段階ごとに記述を更新。copy.sgml から explain.sgml への + 相互参照を追加。 +- **リグレッションテスト**: 新規 `src/test/regress/sql/explain_copy.sql` + (+ expected、parallel_schedule への追加)。0001 分: + - EXPLAIN COPY tbl FROM '/nonexistent'(実行しないため成功する) + - EXPLAIN COPY tbl TO stdout / FROM stdin + - EXPLAIN (FORMAT JSON) を explain.sql の explain_filter / + explain_filter_to_json と同様のフィルタ関数で安定化 + - 権限エラー(非特権ロールで FROM 'file')、存在しないテーブル + - EXPLAIN ANALYZE COPY ... が ERROR になること(0002/0003 で期待値更新) + +### 0001-8. 変更ファイル一覧 + +``` +src/backend/parser/gram.y | ExplainableStmt に CopyStmt +src/backend/commands/explain.c | ディスパッチ + ExplainOnePlan 拡張 +src/backend/commands/explain_copy.c | 新規 +src/backend/commands/copy.c | ProcessCopyTarget 抽出 +src/backend/commands/copyto.c | CopyToTransformQuery 抽出 +src/backend/commands/Makefile, meson.build | explain_copy.o 追加 +src/include/commands/explain.h | ExplainCopyStmt / ExplainOnePlan +src/include/commands/copy.h | ProcessCopyTarget / CopyToTransformQuery +src/bin/psql/tab-complete.in.c +doc/src/sgml/ref/explain.sgml, copy.sgml +src/test/regress/{sql,expected}/explain_copy.*, parallel_schedule +``` + +--- + +## 0002: EXPLAIN ANALYZE COPY (query) TO + +### 0002-1. 意味論(シリーズとして固定) + +**ANALYZE 時、内包クエリは実行するが、COPY の整形出力は一切生成しない。** +ファイルは書かれず、STDOUT にもデータは送られない。 + +根拠: EXPLAIN ANALYZE SELECT が結果行をクライアントに送らないのと同じ +扱い。COPY TO の「書き込み」は外部への出力であり、CTAS / INSERT のような +データベース内の副作用(WAL・トリガ等、計測対象そのもの)とは性質が +異なる。この整理により: + +- `TO STDOUT` でも CopyOutResponse が送られないためプロトコル問題が + 発生せず、**ANALYZE 時も STDIN/STDOUT 制限が不要**(TO 側)。 +- 将来整形時間を計測する場合(Phase 4)も、EXPLAIN (SERIALIZE) と同様に + 「整形するが捨てる」DestReceiver を追加するだけで意味論が変わらない。 + +### 0002-2. 実装 + +0001 の ExplainCopyToQuery から `if (es->analyze) ereport(ERROR ...)` を +削除するだけで、実行は ExplainOnePlan が担う: + +- es->analyze 時、ExplainOnePlan は into == NULL かつ SERIALIZE なしなら + None_Receiver で ExecutorRun する(explain.c:550-594)。行は破棄される。 +- ノード別実測時間・BUFFERS・WAL・Planning/Execution Time は既存機構が + そのまま出力する。追加の計測コードはゼロ。 + +スナップショット処理は ExplainOnePlan の PushCopiedSnapshot + +UpdateActiveSnapshotCommandId(explain.c:539-540)が BeginCopyTo +(copyto.c:1021-1022)と同等のため追加対応不要。 + +`COPY relation TO`(query なし)の ANALYZE はプランを持たないため 0002 の +対象外とし、引き続き ERROR(メッセージ: "EXPLAIN ANALYZE is not supported +for COPY relation TO", HINT: "Use the COPY (SELECT ...) TO variant.")。 +RLS 有効時は relation TO でも query に変換されるため ANALYZE 可能になる +点をテストで確認する。 + +### 0002-3. 出力例 + +``` +=# EXPLAIN (ANALYZE, COSTS OFF) COPY (SELECT * FROM t) TO '/tmp/t.out'; + QUERY PLAN +------------------------------------------ + Seq Scan on t (actual time=0.010..45.2 rows=100000 loops=1) + Copy To + Format: text + Target: file + File: /tmp/t.out + Planning Time: 0.100 ms + Execution Time: 60.123 ms +``` + +(注: /tmp/t.out は作成されない。ドキュメントに明記する。) + +### 0002-4. テスト・ドキュメント + +- explain_filter 経由で (ANALYZE, TIMING OFF, SUMMARY OFF, COSTS OFF, + BUFFERS OFF) の TEXT / JSON 出力を検証。 +- `TO STDOUT` + ANALYZE でデータが送られないこと(psql 出力がプランのみ)。 +- ファイルが作成されないことの確認(pg_stat_file がエラーになる等)。 +- RLS 付き relation TO の ANALYZE が動くこと。 +- explain.sgml: 「EXPLAIN ANALYZE COPY ... TO はクエリを実行するが出力先 + には何も書かれない」を明記。 + +### 0002-5. 変更ファイル一覧 + +``` +src/backend/commands/explain_copy.c | ANALYZE 許可(エラー分岐の整理) +doc/src/sgml/ref/explain.sgml +src/test/regress/{sql,expected}/explain_copy.* +``` + +--- + +## 0003: EXPLAIN ANALYZE COPY FROM + 時間内訳 + +### 0003-1. 意味論 + +- **実際にデータをロードする**(EXPLAIN ANALYZE INSERT と同じ)。 + ドキュメントに BEGIN; EXPLAIN ANALYZE COPY ...; ROLLBACK; の例を記載。 +- `FROM STDIN` は ANALYZE 時 ERROR: + "EXPLAIN ANALYZE cannot be used with COPY FROM STDIN" + HINT "Use COPY FROM a file or PROGRAM." + (EXPLAIN 応答中に CopyInResponse を送るとプロトコル上クライアントの + 想定を壊すため。非 ANALYZE は 0001 どおり許可。) +- 読み取り専用トランザクションでは既存 COPY FROM と同じく + PreventCommandIfReadOnly(copy.c:364-365 と同一条件)。 +- FREEZE / ON_ERROR / WHERE / トリガ / パーティションはすべて通常どおり + 動作する。 + +### 0003-2. 計測構造体と受け渡し + +```c +/* src/include/commands/copy.h */ +typedef struct CopyFromInstrumentation +{ + bool collect_timing; /* es->timing: 位相別時間を計測するか */ + + /* collect_timing 時のみ更新 */ + instr_time input_time; /* NextCopyFrom 合計(読込+パース+型変換) */ + instr_time insert_time; /* table_(multi_)insert / FDW insert */ + instr_time index_time; /* ExecInsertIndexTuples */ + + /* 常時更新(既存カウンタの転記) */ + uint64 excluded; /* WHERE で除外された行数 */ + uint64 skipped; /* ON_ERROR でスキップされた行数 */ + + /* トリガ実測(ri_TrigInstrument から集約、下記参照) */ + List *triggers; /* ExplainCopyTrigger のリスト */ +} CopyFromInstrumentation; + +/* copyfrom.c がエクスポート */ +void CopyFromSetInstrumentation(CopyFromState cstate, + CopyFromInstrumentation *instr); +``` + +- `CopyFromStateData`(copyfrom_internal.h)に + `CopyFromInstrumentation *instr`(既定 NULL)を追加。 +- **BeginCopyFrom のシグネチャは変更しない**。BeginCopyFrom は file_fdw + 等の拡張から呼ばれる公開 API のため、セッター方式で後付けする。 + +### 0003-3. 計測ポイント(copyfrom.c) + +タイマーはすべて `if (cstate->instr && cstate->instr->collect_timing)` で +ガードし、EXPLAIN 経由でない通常 COPY(instr == NULL)には**分岐 1 回以外の +コストを一切追加しない**。 + +1. **input_time**: メインループの NextCopyFrom 呼び出し + (copyfrom.c:1151)を INSTR_TIME_SET_CURRENT 対で挟む。 + - 計測点が CopyFromRoutine->CopyFromOneRow のコールバック境界に一致 + するため、text/csv/binary だけでなくカスタムフォーマットも計測される。 + - ON_ERROR でスキップされる行のパースコストも含まれる(入力時間の + 一部として妥当)。 +2. **insert_time / index_time**(2 経路): + - 単一挿入経路: table_tuple_insert(copyfrom.c:1429)と + ExecForeignInsert(copyfrom.c:1411)を insert_time で、 + ExecInsertIndexTuples(copyfrom.c:1433)を index_time で挟む。 + - マルチ挿入経路: CopyMultiInsertBufferFlush 内の + table_multi_insert(copyfrom.c:556)を insert_time で、 + ExecInsertIndexTuples(copyfrom.c:576)を index_time で挟む + (miinfo->cstate から instr に到達可能)。 + - フラッシュ内の AFTER ROW トリガ処理はタイマーの外に置き、トリガ + 計測(下記)に委ねる。 +3. **トリガ**: EXPLAIN ANALYZE の既存トリガ計測機構を再利用する。 + - CopyFrom の初期化部で、instr 設定時に対象 ResultRelInfo の + ri_TrigInstrument に InstrAlloc(numTriggers, INSTRUMENT_TIMER, false) + を設定。パーティションルーティング時は resultRelInfo 切替ブロック + (copyfrom.c:1220-1262)で未設定なら設定する。 + - trigger.c は ri_TrigInstrument が非 NULL なら自動的に計測する + (trigger.c:2468 ほか)。 + - CopyFrom の終了処理(FreeExecutorState 前)で、estate の + result-relation 群から {トリガ名, リレーション名, calls, total time} + を instr->triggers に集約する(EState は CopyFrom のローカルで + ある解放されるため、ここで転記が必要)。 +4. **カウンタ**: excluded(copyfrom.c:1202 のローカル変数)と + cstate->num_errors を終了時に instr へ転記。処理行数は CopyFrom の + 戻り値を使用。 + +### 0003-4. explain_copy.c の ANALYZE FROM 経路 + +```c +/* 0001 の ereport(ERROR) を置き換え */ +if (es->analyze) +{ + CopyFromInstrumentation ci = {0}; + BufferUsage bufusage_start; /* es->buffers 時 */ + WalUsage walusage_start; /* es->wal 時 */ + instr_time starttime; + uint64 processed; + CopyFromState cstate; + + if (stmt->filename == NULL) + ereport(ERROR, ... STDIN 拒否 ...); + if (XactReadOnly && !rel->rd_islocaltemp) + PreventCommandIfReadOnly("COPY FROM"); + + ci.collect_timing = es->timing; + ... bufusage_start / walusage_start 記録、starttime 記録 ... + + cstate = BeginCopyFrom(pstate, rel, whereClause, stmt->filename, + stmt->is_program, NULL, stmt->attlist, + stmt->options); + CopyFromSetInstrumentation(cstate, &ci); + processed = CopyFrom(cstate); + EndCopyFrom(cstate); + + ... 差分算出、出力(下記) ... +} +``` + +BUFFERS / WAL は pgBufferUsage / pgWalUsage の差分を取り、explain.c の +show_buffer_usage() / show_wal_usage()(現在 static、explain.c:150-151)を +extern 化して再利用する。トリガ出力は report_triggers(explain.c:74)と +同一書式を explain_copy.c 側で生成する(データ源が ci.triggers のため)。 + +全体時間は starttime からの経過を es->summary 時に "Execution Time" として +出力(ExplainOnePlan と同書式)。プランニングが存在しないため +"Planning Time" は出力しない。 + +### 0003-5. 出力設計 + +TEXT(TIMING ON): + +``` +=# BEGIN; +=# EXPLAIN (ANALYZE, BUFFERS) COPY lineitem FROM '/tmp/lineitem.csv' (FORMAT csv); + QUERY PLAN +------------------------------------------------------------------ + Copy From on lineitem (actual rows=6001215) + Format: csv + Source: file + File: /tmp/lineitem.csv + Input Time: 3210.456 ms + Insert Time: 2103.222 ms + Index Update Time: 812.345 ms + Rows Skipped: 5 + Buffers: shared hit=12345 read=678 dirtied=91011 written=1213 + WAL: records=6001220 fpi=123 bytes=987654321 + Trigger trg_audit: time=102.334 calls=6001215 + Execution Time: 6543.210 ms +=# ROLLBACK; +``` + +表示規則: +- TIMING OFF 時は Input/Insert/Index Update Time の 3 行を出力しない + (actual rows と各カウンタのみ)。 +- Rows Excluded by Filter は WHERE 句がある場合のみ、Rows Skipped は + ON_ERROR が stop 以外の場合のみ出力。 +- Input Time は「読み込み+パース+型変換」の合計であることを + ドキュメントに明記(細分化は Phase 4)。名称は bikeshed 対象として + 提案メールで代替案(Read Time / Parse Time 分離等)とともに提示する。 + +JSON: + +```json +[{ "Copy From": { + "Relation Name": "lineitem", + "Format": "csv", "Source": "file", "File": "/tmp/lineitem.csv", + "Actual Rows": 6001215, + "Input Time": 3210.456, + "Insert Time": 2103.222, + "Index Update Time": 812.345, + "Rows Skipped": 5, + "Shared Hit Blocks": 12345, ..., + "WAL Records": 6001220, ... }, + "Triggers": [ { "Trigger Name": "trg_audit", "Relation": "lineitem", + "Time": 102.334, "Calls": 6001215 } ], + "Execution Time": 6543.210 }] +``` + +### 0003-6. オーバーヘッドの見積りと緩和 + +- TIMING ON 時の追加コストは行あたり clock_gettime × 2(input)+ + 単一挿入経路なら × 4。Linux vDSO で 1 回 20〜30ns として 10^7 行で + 0.5〜1 秒程度、細い行の高速ロードでは 5〜10% 級になり得る。 +- マルチ挿入経路のフラッシュは約 1000 行単位のため insert/index タイマーの + コストは無視できる。支配項は input タイマー。 +- 緩和策: `EXPLAIN (ANALYZE, TIMING OFF)` では位相別タイマーを完全に + 無効化し、カウンタのみ収集する(追加 clock 呼び出しゼロ)。 +- 提案メールには pgbench 相当のベンチ結果を添付する: + unpatched / patched+非EXPLAIN / ANALYZE+TIMING OFF / ANALYZE+TIMING ON を + narrow(2 列)・wide(30 列)× 1000 万行で比較。 + +### 0003-7. テスト + +``` +\set filename :abs_builddir '/results/explain_copy.data' +COPY src_tbl TO :'filename'; +BEGIN; +EXPLAIN (ANALYZE, TIMING OFF, SUMMARY OFF, COSTS OFF) + COPY dst_tbl FROM :'filename'; +ROLLBACK; +``` + +- 出力の時間値・ブロック数は不定のため、基本は TIMING OFF + SUMMARY OFF + + BUFFERS OFF で安定化し、TIMING ON の形は explain_filter(数値→N 置換)で + 1 ケースのみ検証。 +- ケース: 単純ロード / WHERE 付き / ON_ERROR ignore(Rows Skipped)/ + BEFORE・AFTER トリガ付き(Trigger 行)/ パーティション先ロード / + FDW 先(postgres_fdw regress は不可のため file_fdw 圏外、コアでは + 対象外とし contrib テスト追加は保留)/ FREEZE。 +- エラー系: ANALYZE + STDIN、read-only トランザクション、REJECT_LIMIT 超過 + (途中エラーでも壊れないこと)。 +- トランザクション内 ROLLBACK でデータが残らないこと。 + +### 0003-8. 変更ファイル一覧 + +``` +src/backend/commands/copyfrom.c | タイマー挿入、セッター、トリガ集約 +src/include/commands/copyfrom_internal.h | cstate->instr フィールド +src/include/commands/copy.h | CopyFromInstrumentation / セッター +src/backend/commands/explain_copy.c | ANALYZE FROM 経路と出力 +src/backend/commands/explain.c | show_buffer_usage 等の extern 化 +src/include/commands/explain.h +doc/src/sgml/ref/explain.sgml, copy.sgml +src/test/regress/{sql,expected}/explain_copy.* +``` + +--- + +## シリーズ全体の残論点(提案メールに明記するもの) + +1. **位相名の命名**(Input Time / Insert Time / Index Update Time)。 +2. **EXPLAIN ANALYZE COPY TO が出力を生成しない**という意味論の是非 + (CTAS はテーブルを作るという非対称の説明)。 +3. ExplainOnePlan のシグネチャ変更(拡張への影響)の許容。 +4. 非 ANALYZE EXPLAIN でファイルを open しない(不存在を検出しない)ことの + 是非。 +5. 将来課題: input の細分化(read/parse/convert)、COPY relation TO の + ANALYZE、整形時間計測(SERIALIZE 型 DestReceiver)、パーティション別 + 内訳、pg_stat_progress_copy との関係整理。 From 54ac0ab46515f4ea5d7ee899d0e32044a96fc1e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 8 Jul 2026 23:50:08 +0000 Subject: [PATCH 2/9] Split refactoring into its own patch; restructure series as 0001-0004 Per review feedback, separate the behavior-preserving refactoring of copy.c/copyto.c (ProcessCopyTarget, CopyToTransformQuery) into patch 0001 so reviewers can verify 'no functional change' in isolation. Renumber the feature patches accordingly (grammar/dispatch -> 0002, ANALYZE COPY TO -> 0003, ANALYZE COPY FROM -> 0004) and document the split policy: pure code motion gets its own patch, while API additions with no in-tree caller (ExplainOnePlan parameter, instrumentation setter) stay with the feature that uses them. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- EXPLAIN_COPY_DESIGN.md | 235 ++++++++++++++++++++++++----------------- 1 file changed, 137 insertions(+), 98 deletions(-) diff --git a/EXPLAIN_COPY_DESIGN.md b/EXPLAIN_COPY_DESIGN.md index c82ef7255ddcf..452996017f328 100644 --- a/EXPLAIN_COPY_DESIGN.md +++ b/EXPLAIN_COPY_DESIGN.md @@ -1,21 +1,100 @@ -# EXPLAIN COPY 詳細設計(パッチ 0001〜0003) +# EXPLAIN COPY 詳細設計(パッチ 0001〜0004) 対象: PostgreSQL 20devel (master) 目的: EXPLAIN の対象コマンドに COPY を追加し、COPY FROM の時間内訳を表示する。 -パッチシリーズの意味論(最初に固定し、以後変更しない): +## パッチ構成 -| パッチ | できるようになること | 実行の有無 | +リファクタリング(挙動変更なし)と機能追加を分離した 4 パッチ構成とする。 + +| パッチ | 内容 | 挙動変更 | +|---|---|---| +| 0001 | copy.c / copyto.c のリファクタリング(共通関数の抽出) | なし | +| 0002 | `EXPLAIN COPY ...`(全バリアント、ANALYZE なし) | 新機能 | +| 0003 | `EXPLAIN ANALYZE COPY (query) TO ...` | 新機能 | +| 0004 | `EXPLAIN ANALYZE COPY ... FROM ...` + 時間内訳 | 新機能 | + +分割の方針: +- **純粋なコード移動は独立パッチにする**(0001)。レビュアーが + 「挙動変更なし」を diff だけで確認でき、機能部分の議論と切り離して + 先行コミットできる。 +- **利用者のいない API 変更は機能パッチ側に残す**。ExplainOnePlan の + パラメータ追加(0002)や CopyFromInstrumentation のセッター(0004)を + 先行パッチに切り出すと、in-tree の呼び出し元が存在しない dead code に + なるため、それを使う機能と同じパッチに含める。 + +実行意味論(シリーズとして最初に固定し、以後変更しない): + +| コマンド | ANALYZE なし | ANALYZE あり | |---|---|---| -| 0001 | `EXPLAIN COPY ...`(全バリアント、ANALYZE なし) | 一切実行しない | -| 0002 | `EXPLAIN ANALYZE COPY (query) TO ...` | クエリを実行、COPY 出力(ファイル/STDOUT)は生成しない | -| 0003 | `EXPLAIN ANALYZE COPY ... FROM ...` + 時間内訳 | 実際にデータをロードする | +| COPY ... FROM | 実行しない | 実際にロードする(0004) | +| COPY (query) TO | 実行しない(プランのみ表示) | クエリを実行、COPY 出力は生成しない(0003) | +| COPY relation TO | 実行しない | ERROR(将来課題) | + +--- + +## 0001: copy.c / copyto.c のリファクタリング + +挙動変更を一切含まない、共通関数の抽出のみのパッチ。コミットメッセージに +"No functional changes." と、0002 以降で EXPLAIN から再利用する予定である +ことを明記する。新規テストは追加しない(既存リグレッションテストが +そのまま通ることが検証条件)。 + +### 0001-1. copy.c: ProcessCopyTarget の抽出 + +DoCopy(copy.c:63-386)の前半(74〜355 行: 権限チェック〜RLS 変換)を +共通関数に抽出する。 + +```c +/* copy.c / copy.h */ +void ProcessCopyTarget(ParseState *pstate, const CopyStmt *stmt, + int stmt_location, int stmt_len, + Relation *rel_p, /* out: 対象リレーション(query TO は NULL) */ + Oid *relid_p, /* out: RLS 再確認用 relid */ + RawStmt **query_p, /* out: query TO / RLS 変換後クエリ */ + Node **whereClause_p); /* out: FROM の WHERE(変換済み) */ +``` + +抽出内容: +- file/PROGRAM のロール権限チェック(pg_read_server_files 等) +- リレーションの open + lock(FROM: RowExclusiveLock, TO: AccessShareLock) +- WHERE 句の transform と検証 +- 列単位 ACL チェック(ExecCheckPermissions) +- RLS 有効時の relation TO → query TO 変換(copy.c:242-342) + +DoCopy は `ProcessCopyTarget` + 実行部(357-382 行)+ table_close となる。 + +### 0001-2. copyto.c: CopyToTransformQuery の抽出 + +BeginCopyTo のクエリ解析・検証部(copyto.c:909-986)を抽出する。 + +```c +/* copyto.c / copy.h */ +Query *CopyToTransformQuery(ParseState *pstate, RawStmt *raw_query); +``` + +内容: pg_analyze_and_rewrite_fixedparams、DO INSTEAD ルール拒否、 +SELECT INTO 拒否、ユーティリティ文拒否、RETURNING 必須チェック。 +プランニングと relationOids 再確認(copyto.c:989-1015)は BeginCopyTo に +残す(0003 の EXPLAIN 経路は自前でプランニングするため)。 + +### 0001-3. 変更ファイル一覧 + +``` +src/backend/commands/copy.c | ProcessCopyTarget 抽出 +src/backend/commands/copyto.c | CopyToTransformQuery 抽出 +src/include/commands/copy.h | 両関数の宣言 +``` + +備考: 2 関数を 1 パッチにまとめるのは、いずれも「COPY 文のセットアップを +EXPLAIN から再利用可能にする」という同一目的の抽出であるため。レビューで +分割を求められた場合はファイル単位(copy.c / copyto.c)で容易に分割できる。 --- -## 0001: 文法・ディスパッチ・非 ANALYZE の EXPLAIN +## 0002: 文法・ディスパッチ・非 ANALYZE の EXPLAIN -### 0001-1. 文法 (src/backend/parser/gram.y) +### 0002-1. 文法 (src/backend/parser/gram.y) `ExplainableStmt`(gram.y:12846)に `| CopyStmt` を追加する。 @@ -33,7 +112,7 @@ bison 3.8.2 で shift/reduce 競合が発生しないことは検証済み(gram. `PreparableStmt` には追加しない(PREPARE 対象外のため EXPLAIN EXECUTE 経由で CopyStmt が来ることはない)。 -### 0001-2. パース解析 +### 0002-2. パース解析 変更不要。`transformExplainStmt`(analyze.c:3461)は内包文を `transformOptionalSelectInto` → `transformStmt` に渡し、CopyStmt は @@ -41,7 +120,7 @@ transformStmt の default 分岐(analyze.c:435-444)で CMD_UTILITY の Query に 包まれる。実行時は ExplainQuery → ExplainOneQuery → ExplainOneUtility に 到達する(NotifyStmt と同じ経路)。 -### 0001-3. ディスパッチ (src/backend/commands/explain.c) +### 0002-3. ディスパッチ (src/backend/commands/explain.c) `ExplainOneUtility()`(explain.c:396)の NotifyStmt 分岐の前に追加: @@ -53,50 +132,7 @@ else if (IsA(utilityStmt, CopyStmt)) CopyStmt は変更しない(const 扱い)ため、EXPLAIN EXECUTE 経路のような copyObject は不要。 -### 0001-4. copy.c のリファクタリング - -DoCopy(copy.c:63-386)の前半(74〜355 行: 権限チェック〜RLS 変換)を -共通関数に抽出する。**挙動変更なし**。 - -```c -/* copy.c / copy.h */ -void ProcessCopyTarget(ParseState *pstate, const CopyStmt *stmt, - int stmt_location, int stmt_len, - Relation *rel_p, /* out: 対象リレーション(query TO は NULL) */ - Oid *relid_p, /* out: RLS 再確認用 relid */ - RawStmt **query_p, /* out: query TO / RLS 変換後クエリ */ - Node **whereClause_p); /* out: FROM の WHERE(変換済み) */ -``` - -抽出内容: -- file/PROGRAM のロール権限チェック(pg_read_server_files 等) -- リレーションの open + lock(FROM: RowExclusiveLock, TO: AccessShareLock) -- WHERE 句の transform と検証 -- 列単位 ACL チェック(ExecCheckPermissions) -- RLS 有効時の relation TO → query TO 変換(copy.c:242-342) - -DoCopy は `ProcessCopyTarget` + 実行部(357-382 行)+ table_close となる。 - -EXPLAIN(非 ANALYZE)でもこの関数を通すことで、存在しないテーブル・権限 -不足は通常の COPY と同様に検出される(EXPLAIN INSERT が ExecutorStart で -ACL チェックされるのと整合)。ファイルは open しないため、ファイル不存在は -検出されない(EXPLAIN の一般的な性質として許容し、ドキュメントに記載)。 - -### 0001-5. copyto.c のリファクタリング - -BeginCopyTo のクエリ解析・検証部(copyto.c:909-986)を抽出する。 -**挙動変更なし**。 - -```c -/* copyto.c / copy.h */ -Query *CopyToTransformQuery(ParseState *pstate, RawStmt *raw_query); -``` - -内容: pg_analyze_and_rewrite_fixedparams、DO INSTEAD ルール拒否、 -SELECT INTO 拒否、ユーティリティ文拒否、RETURNING 必須チェック。 -プランニングと relationOids 再確認(copyto.c:989-1015)は呼び出し元に残す。 - -### 0001-6. explain_copy.c(新規ファイル) +### 0002-4. explain_copy.c(新規ファイル) explain.c は近年 explain_dr.c / explain_format.c / explain_state.c に分割 されており、この流れに沿って `src/backend/commands/explain_copy.c` を新設。 @@ -107,7 +143,7 @@ void ExplainCopyStmt(CopyStmt *stmt, ExplainState *es, ParseState *pstate, ParamListInfo params); ``` -0001 時点の処理フロー: +0002 時点の処理フロー: ``` ProcessCopyTarget(pstate, stmt, ..., &rel, &relid, &query, &whereClause); @@ -116,19 +152,19 @@ if (stmt->is_from) { if (es->analyze) ereport(ERROR, "EXPLAIN ANALYZE is not yet supported for COPY FROM"); - /* 0003 で解除 */ + /* 0004 で解除 */ ExplainCopyFromInfo(stmt, rel, es); /* 静的情報のみ */ } else if (query != NULL) /* COPY (query) TO / RLS 変換された relation TO */ { if (es->analyze) - ereport(ERROR, "..."); /* 0002 で解除 */ + ereport(ERROR, "..."); /* 0003 で解除 */ ExplainCopyToQuery(stmt, query, relid, es, pstate, params); } else /* COPY relation TO */ { if (es->analyze) - ereport(ERROR, "..."); /* relation TO の ANALYZE は将来課題(Phase 4) */ + ereport(ERROR, "..."); /* relation TO の ANALYZE は将来課題 */ ExplainCopyToInfo(stmt, rel, es); } @@ -136,10 +172,16 @@ if (rel) table_close(rel, NoLock); ``` -STDIN/STDOUT は 0001 では**許可**する(実行しないためプロトコル問題は -発生しない)。ANALYZE 時の制限は 0002/0003 で導入する。 +STDIN/STDOUT は 0002 では**許可**する(実行しないためプロトコル問題は +発生しない)。ANALYZE 時の制限は 0003/0004 で導入する。 + +非 ANALYZE の EXPLAIN でも ProcessCopyTarget を通すことで、存在しない +テーブル・権限不足は通常の COPY と同様に検出される(EXPLAIN INSERT が +ExecutorStart で ACL チェックされるのと整合)。ファイルは open しないため、 +ファイル不存在は検出されない(EXPLAIN の一般的な性質として許容し、 +ドキュメントに記載)。 -#### ExplainCopyToQuery の実装(0001 の中核) +#### ExplainCopyToQuery の実装(0002 の中核) standard_ExplainOneQuery(explain.c:324-381)を模倣し、COPY 用の追加情報を ExplainOnePlan に渡す: @@ -195,6 +237,9 @@ COPY の静的情報を「Query」グループの**内側**に出力するため 前例に従う。ABI 変化はメジャーリリースの通例として許容(planduration / bufusage / mem_counters 追加時と同じ)。 +このパラメータ追加は 0002 で初めて使うため、0001 には含めず本パッチに +含める(dead code を作らない)。 + ExplainOnePlan 内では ExplainPrintPlan 直後に、copystmt が非 NULL なら ExplainPrintCopyInfo(explain_copy.c で定義、後述の共通出力関数)を呼ぶ。 @@ -219,7 +264,7 @@ static void ExplainPrintCopyInfo(const CopyStmt *stmt, Relation rel, オプションの単純な復唱(DELIMITER 等)は行わない(文面に書いてある情報で あり、レビューで削られるのが常のため最小限とする)。 -#### TEXT フォーマット出力例(0001) +#### TEXT フォーマット出力例(0002) ``` =# EXPLAIN COPY tab FROM '/tmp/tab.csv' WITH (FORMAT csv); @@ -255,35 +300,29 @@ ExplainOpenGroup("Copy From", "Copy From", true, es) → プロパティ、の 入れ子で出力する(TEXT の見出し行 "Copy From on tab" は format == EXPLAIN_FORMAT_TEXT のとき手動出力)。 -### 0001-7. その他の変更 +### 0002-5. その他の変更 - **psql タブ補完** (src/bin/psql/tab-complete.in.c): EXPLAIN の後続候補に COPY を追加。 - **ドキュメント**: explain.sgml の対象文リスト(SELECT, INSERT, ...)に - COPY を追加。「ANALYZE は 0001 時点では COPY に未対応」の footnote は - シリーズ全体コミット時には不要になるため、0002/0003 と同時期の - コミットを想定して段階ごとに記述を更新。copy.sgml から explain.sgml への - 相互参照を追加。 + COPY を追加。copy.sgml から explain.sgml への相互参照を追加。 - **リグレッションテスト**: 新規 `src/test/regress/sql/explain_copy.sql` - (+ expected、parallel_schedule への追加)。0001 分: + (+ expected、parallel_schedule への追加)。0002 分: - EXPLAIN COPY tbl FROM '/nonexistent'(実行しないため成功する) - EXPLAIN COPY tbl TO stdout / FROM stdin - EXPLAIN (FORMAT JSON) を explain.sql の explain_filter / explain_filter_to_json と同様のフィルタ関数で安定化 - 権限エラー(非特権ロールで FROM 'file')、存在しないテーブル - - EXPLAIN ANALYZE COPY ... が ERROR になること(0002/0003 で期待値更新) + - EXPLAIN ANALYZE COPY ... が ERROR になること(0003/0004 で期待値更新) -### 0001-8. 変更ファイル一覧 +### 0002-6. 変更ファイル一覧 ``` src/backend/parser/gram.y | ExplainableStmt に CopyStmt src/backend/commands/explain.c | ディスパッチ + ExplainOnePlan 拡張 src/backend/commands/explain_copy.c | 新規 -src/backend/commands/copy.c | ProcessCopyTarget 抽出 -src/backend/commands/copyto.c | CopyToTransformQuery 抽出 src/backend/commands/Makefile, meson.build | explain_copy.o 追加 src/include/commands/explain.h | ExplainCopyStmt / ExplainOnePlan -src/include/commands/copy.h | ProcessCopyTarget / CopyToTransformQuery src/bin/psql/tab-complete.in.c doc/src/sgml/ref/explain.sgml, copy.sgml src/test/regress/{sql,expected}/explain_copy.*, parallel_schedule @@ -291,9 +330,9 @@ src/test/regress/{sql,expected}/explain_copy.*, parallel_schedule --- -## 0002: EXPLAIN ANALYZE COPY (query) TO +## 0003: EXPLAIN ANALYZE COPY (query) TO -### 0002-1. 意味論(シリーズとして固定) +### 0003-1. 意味論(シリーズとして固定) **ANALYZE 時、内包クエリは実行するが、COPY の整形出力は一切生成しない。** ファイルは書かれず、STDOUT にもデータは送られない。 @@ -305,12 +344,12 @@ src/test/regress/{sql,expected}/explain_copy.*, parallel_schedule - `TO STDOUT` でも CopyOutResponse が送られないためプロトコル問題が 発生せず、**ANALYZE 時も STDIN/STDOUT 制限が不要**(TO 側)。 -- 将来整形時間を計測する場合(Phase 4)も、EXPLAIN (SERIALIZE) と同様に +- 将来整形時間を計測する場合も、EXPLAIN (SERIALIZE) と同様に 「整形するが捨てる」DestReceiver を追加するだけで意味論が変わらない。 -### 0002-2. 実装 +### 0003-2. 実装 -0001 の ExplainCopyToQuery から `if (es->analyze) ereport(ERROR ...)` を +0002 の ExplainCopyToQuery から `if (es->analyze) ereport(ERROR ...)` を 削除するだけで、実行は ExplainOnePlan が担う: - es->analyze 時、ExplainOnePlan は into == NULL かつ SERIALIZE なしなら @@ -322,13 +361,13 @@ src/test/regress/{sql,expected}/explain_copy.*, parallel_schedule UpdateActiveSnapshotCommandId(explain.c:539-540)が BeginCopyTo (copyto.c:1021-1022)と同等のため追加対応不要。 -`COPY relation TO`(query なし)の ANALYZE はプランを持たないため 0002 の +`COPY relation TO`(query なし)の ANALYZE はプランを持たないため 0003 の 対象外とし、引き続き ERROR(メッセージ: "EXPLAIN ANALYZE is not supported for COPY relation TO", HINT: "Use the COPY (SELECT ...) TO variant.")。 RLS 有効時は relation TO でも query に変換されるため ANALYZE 可能になる 点をテストで確認する。 -### 0002-3. 出力例 +### 0003-3. 出力例 ``` =# EXPLAIN (ANALYZE, COSTS OFF) COPY (SELECT * FROM t) TO '/tmp/t.out'; @@ -345,7 +384,7 @@ RLS 有効時は relation TO でも query に変換されるため ANALYZE 可 (注: /tmp/t.out は作成されない。ドキュメントに明記する。) -### 0002-4. テスト・ドキュメント +### 0003-4. テスト・ドキュメント - explain_filter 経由で (ANALYZE, TIMING OFF, SUMMARY OFF, COSTS OFF, BUFFERS OFF) の TEXT / JSON 出力を検証。 @@ -355,7 +394,7 @@ RLS 有効時は relation TO でも query に変換されるため ANALYZE 可 - explain.sgml: 「EXPLAIN ANALYZE COPY ... TO はクエリを実行するが出力先 には何も書かれない」を明記。 -### 0002-5. 変更ファイル一覧 +### 0003-5. 変更ファイル一覧 ``` src/backend/commands/explain_copy.c | ANALYZE 許可(エラー分岐の整理) @@ -365,9 +404,9 @@ src/test/regress/{sql,expected}/explain_copy.* --- -## 0003: EXPLAIN ANALYZE COPY FROM + 時間内訳 +## 0004: EXPLAIN ANALYZE COPY FROM + 時間内訳 -### 0003-1. 意味論 +### 0004-1. 意味論 - **実際にデータをロードする**(EXPLAIN ANALYZE INSERT と同じ)。 ドキュメントに BEGIN; EXPLAIN ANALYZE COPY ...; ROLLBACK; の例を記載。 @@ -375,13 +414,13 @@ src/test/regress/{sql,expected}/explain_copy.* "EXPLAIN ANALYZE cannot be used with COPY FROM STDIN" HINT "Use COPY FROM a file or PROGRAM." (EXPLAIN 応答中に CopyInResponse を送るとプロトコル上クライアントの - 想定を壊すため。非 ANALYZE は 0001 どおり許可。) + 想定を壊すため。非 ANALYZE は 0002 どおり許可。) - 読み取り専用トランザクションでは既存 COPY FROM と同じく PreventCommandIfReadOnly(copy.c:364-365 と同一条件)。 - FREEZE / ON_ERROR / WHERE / トリガ / パーティションはすべて通常どおり 動作する。 -### 0003-2. 計測構造体と受け渡し +### 0004-2. 計測構造体と受け渡し ```c /* src/include/commands/copy.h */ @@ -411,8 +450,10 @@ void CopyFromSetInstrumentation(CopyFromState cstate, `CopyFromInstrumentation *instr`(既定 NULL)を追加。 - **BeginCopyFrom のシグネチャは変更しない**。BeginCopyFrom は file_fdw 等の拡張から呼ばれる公開 API のため、セッター方式で後付けする。 +- セッターとフィールドは本パッチで初めて使われるため、リファクタリング + パッチ(0001)には含めない(dead code を作らない)。 -### 0003-3. 計測ポイント(copyfrom.c) +### 0004-3. 計測ポイント(copyfrom.c) タイマーはすべて `if (cstate->instr && cstate->instr->collect_timing)` で ガードし、EXPLAIN 経由でない通常 COPY(instr == NULL)には**分岐 1 回以外の @@ -444,15 +485,15 @@ void CopyFromSetInstrumentation(CopyFromState cstate, - CopyFrom の終了処理(FreeExecutorState 前)で、estate の result-relation 群から {トリガ名, リレーション名, calls, total time} を instr->triggers に集約する(EState は CopyFrom のローカルで - ある解放されるため、ここで転記が必要)。 + あり解放されるため、ここで転記が必要)。 4. **カウンタ**: excluded(copyfrom.c:1202 のローカル変数)と cstate->num_errors を終了時に instr へ転記。処理行数は CopyFrom の 戻り値を使用。 -### 0003-4. explain_copy.c の ANALYZE FROM 経路 +### 0004-4. explain_copy.c の ANALYZE FROM 経路 ```c -/* 0001 の ereport(ERROR) を置き換え */ +/* 0002 の ereport(ERROR) を置き換え */ if (es->analyze) { CopyFromInstrumentation ci = {0}; @@ -490,7 +531,7 @@ extern 化して再利用する。トリガ出力は report_triggers(explain.c:7 出力(ExplainOnePlan と同書式)。プランニングが存在しないため "Planning Time" は出力しない。 -### 0003-5. 出力設計 +### 0004-5. 出力設計 TEXT(TIMING ON): @@ -520,7 +561,7 @@ TEXT(TIMING ON): - Rows Excluded by Filter は WHERE 句がある場合のみ、Rows Skipped は ON_ERROR が stop 以外の場合のみ出力。 - Input Time は「読み込み+パース+型変換」の合計であることを - ドキュメントに明記(細分化は Phase 4)。名称は bikeshed 対象として + ドキュメントに明記(細分化は将来課題)。名称は bikeshed 対象として 提案メールで代替案(Read Time / Parse Time 分離等)とともに提示する。 JSON: @@ -541,7 +582,7 @@ JSON: "Execution Time": 6543.210 }] ``` -### 0003-6. オーバーヘッドの見積りと緩和 +### 0004-6. オーバーヘッドの見積りと緩和 - TIMING ON 時の追加コストは行あたり clock_gettime × 2(input)+ 単一挿入経路なら × 4。Linux vDSO で 1 回 20〜30ns として 10^7 行で @@ -554,7 +595,7 @@ JSON: unpatched / patched+非EXPLAIN / ANALYZE+TIMING OFF / ANALYZE+TIMING ON を narrow(2 列)・wide(30 列)× 1000 万行で比較。 -### 0003-7. テスト +### 0004-7. テスト ``` \set filename :abs_builddir '/results/explain_copy.data' @@ -569,14 +610,12 @@ ROLLBACK; BUFFERS OFF で安定化し、TIMING ON の形は explain_filter(数値→N 置換)で 1 ケースのみ検証。 - ケース: 単純ロード / WHERE 付き / ON_ERROR ignore(Rows Skipped)/ - BEFORE・AFTER トリガ付き(Trigger 行)/ パーティション先ロード / - FDW 先(postgres_fdw regress は不可のため file_fdw 圏外、コアでは - 対象外とし contrib テスト追加は保留)/ FREEZE。 + BEFORE・AFTER トリガ付き(Trigger 行)/ パーティション先ロード / FREEZE。 - エラー系: ANALYZE + STDIN、read-only トランザクション、REJECT_LIMIT 超過 (途中エラーでも壊れないこと)。 - トランザクション内 ROLLBACK でデータが残らないこと。 -### 0003-8. 変更ファイル一覧 +### 0004-8. 変更ファイル一覧 ``` src/backend/commands/copyfrom.c | タイマー挿入、セッター、トリガ集約 From 970ad481d4b47f2848079f917d8f40921c886e55 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 00:10:10 +0000 Subject: [PATCH 3/9] Design the start/stop instrumentation API for COPY FROM timing Replace the loose 'wrap call sites with INSTR_TIME pairs' description with a concrete API: a CopyFromPhase enum, a phase_time[] array in CopyFromInstrumentation, and static inline CopyFromInstrStartPhase()/CopyFromInstrStopPhase() helpers in copyfrom_internal.h. Document the non-nesting invariant (single phase_start, Assert-checked), error-path behavior, the exact insertion points in copyfrom.c, and the rejected alternative of reusing instrument.c's InstrStartNode/InstrStopNode. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- EXPLAIN_COPY_DESIGN.md | 175 +++++++++++++++++++++++++++++++++-------- 1 file changed, 143 insertions(+), 32 deletions(-) diff --git a/EXPLAIN_COPY_DESIGN.md b/EXPLAIN_COPY_DESIGN.md index 452996017f328..b2e336759852a 100644 --- a/EXPLAIN_COPY_DESIGN.md +++ b/EXPLAIN_COPY_DESIGN.md @@ -422,16 +422,30 @@ src/test/regress/{sql,expected}/explain_copy.* ### 0004-2. 計測構造体と受け渡し +計測フェーズを enum で定義し、位相別時間は enum で添字付けする配列で +保持する(EXPLAIN 出力側・将来のフェーズ追加の双方で扱いやすい)。 + ```c /* src/include/commands/copy.h */ + +/* COPY FROM の計測フェーズ */ +typedef enum CopyFromPhase +{ + COPY_FROM_PHASE_INPUT, /* NextCopyFrom: 読込+パース+型変換 + * (DEFAULT 式の評価を含む) */ + COPY_FROM_PHASE_INSERT, /* table_(multi_)insert / FDW insert */ + COPY_FROM_PHASE_INDEX, /* ExecInsertIndexTuples */ +} CopyFromPhase; + +#define COPY_FROM_NUM_PHASES (COPY_FROM_PHASE_INDEX + 1) + typedef struct CopyFromInstrumentation { bool collect_timing; /* es->timing: 位相別時間を計測するか */ /* collect_timing 時のみ更新 */ - instr_time input_time; /* NextCopyFrom 合計(読込+パース+型変換) */ - instr_time insert_time; /* table_(multi_)insert / FDW insert */ - instr_time index_time; /* ExecInsertIndexTuples */ + instr_time phase_start; /* 実行中フェーズの開始時刻 */ + instr_time phase_time[COPY_FROM_NUM_PHASES]; /* フェーズ別累積時間 */ /* 常時更新(既存カウンタの転記) */ uint64 excluded; /* WHERE で除外された行数 */ @@ -453,29 +467,126 @@ void CopyFromSetInstrumentation(CopyFromState cstate, - セッターとフィールドは本パッチで初めて使われるため、リファクタリング パッチ(0001)には含めない(dead code を作らない)。 -### 0004-3. 計測ポイント(copyfrom.c) - -タイマーはすべて `if (cstate->instr && cstate->instr->collect_timing)` で -ガードし、EXPLAIN 経由でない通常 COPY(instr == NULL)には**分岐 1 回以外の -コストを一切追加しない**。 - -1. **input_time**: メインループの NextCopyFrom 呼び出し - (copyfrom.c:1151)を INSTR_TIME_SET_CURRENT 対で挟む。 - - 計測点が CopyFromRoutine->CopyFromOneRow のコールバック境界に一致 - するため、text/csv/binary だけでなくカスタムフォーマットも計測される。 - - ON_ERROR でスキップされる行のパースコストも含まれる(入力時間の - 一部として妥当)。 -2. **insert_time / index_time**(2 経路): - - 単一挿入経路: table_tuple_insert(copyfrom.c:1429)と - ExecForeignInsert(copyfrom.c:1411)を insert_time で、 - ExecInsertIndexTuples(copyfrom.c:1433)を index_time で挟む。 - - マルチ挿入経路: CopyMultiInsertBufferFlush 内の - table_multi_insert(copyfrom.c:556)を insert_time で、 - ExecInsertIndexTuples(copyfrom.c:576)を index_time で挟む - (miinfo->cstate から instr に到達可能)。 - - フラッシュ内の AFTER ROW トリガ処理はタイマーの外に置き、トリガ - 計測(下記)に委ねる。 -3. **トリガ**: EXPLAIN ANALYZE の既存トリガ計測機構を再利用する。 +### 0004-3. 計測 API(スタート/ストップ) + +呼び出し箇所(メインループとマルチ挿入フラッシュ)はいずれも copyfrom.c +内だが、将来 copyfromparse.c での read 時間細分化にも使えるよう、 +copyfrom_internal.h に static inline で定義する。 + +```c +/* src/include/commands/copyfrom_internal.h */ + +/* + * フェーズ計測の開始。計測が無効(instr == NULL または timing off)なら + * 何もしない。フェーズはネスト不可(単一の phase_start を共有するため)。 + */ +static inline void +CopyFromInstrStartPhase(CopyFromState cstate) +{ + CopyFromInstrumentation *ci = cstate->instr; + + if (ci == NULL || !ci->collect_timing) + return; + Assert(INSTR_TIME_IS_ZERO(ci->phase_start)); /* ネスト検出 */ + INSTR_TIME_SET_CURRENT(ci->phase_start); +} + +/* + * フェーズ計測の終了。経過時間を phase_time[phase] に加算する。 + */ +static inline void +CopyFromInstrStopPhase(CopyFromState cstate, CopyFromPhase phase) +{ + CopyFromInstrumentation *ci = cstate->instr; + instr_time now; + + if (ci == NULL || !ci->collect_timing) + return; + INSTR_TIME_SET_CURRENT(now); + INSTR_TIME_ACCUM_DIFF(ci->phase_time[phase], now, ci->phase_start); + INSTR_TIME_SET_ZERO(ci->phase_start); /* ネスト検出用 */ +} +``` + +設計上のポイント: + +- **ガードは関数内**に置き、呼び出し箇所は常に 1 行にする(ホットループの + 可読性維持)。通常 COPY(instr == NULL)の追加コストは分岐 1 回のみ。 +- **start はフェーズ引数を取らない**。フェーズの区別が必要なのは累積先を + 決める stop 側だけであり、start を軽くする。 +- **フェーズはネストしない**という不変条件を置く。3 フェーズの計測区間は + すべて逐次(NextCopyFrom → [flush: multi_insert → index] → 次行)で + 重ならないため、単一の phase_start で足りる。誤用は + Assert(INSTR_TIME_IS_ZERO) がアサートビルドで検出する + (INSTR_TIME_SET_ZERO は数命令なので非アサートビルドでも許容)。 +- **エラー時の後始末は不要**。計測途中で ereport(ERROR) が起きた場合 + (ON_ERROR stop のパースエラー等)は文全体が中断され EXPLAIN 出力自体が + 行われないため、走りかけのフェーズ時間は捨てられるだけでよい。 + ON_ERROR ignore のソフトエラーは NextCopyFrom が正常返却するので、 + 直後の stop が通常どおり実行される。 + +### 0004-4. 計測ポイント(copyfrom.c への挿入箇所) + +| フェーズ | start | stop | 箇所 | +|---|---|---|---| +| INPUT | NextCopyFrom 呼び出し直前 | 直後 | copyfrom.c:1151(メインループ) | +| INSERT | table_tuple_insert 直前 | 直後 | copyfrom.c:1429(単一挿入) | +| INSERT | ExecForeignInsert 直前 | 直後 | copyfrom.c:1411(FDW 単一挿入) | +| INSERT | table_multi_insert 直前 | 直後 | copyfrom.c:556(フラッシュ) | +| INDEX | ExecInsertIndexTuples 直前 | 直後 | copyfrom.c:1433(単一挿入) | +| INDEX | ExecInsertIndexTuples 直前 | 直後 | copyfrom.c:576(フラッシュ内の行ループ) | + +挿入例(メインループ、copyfrom.c:1151): + +```c + CopyFromInstrStartPhase(cstate); + if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull)) + { + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INPUT); + break; + } + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INPUT); +``` + +挿入例(フラッシュ内、copyfrom.c:556 / 576。cstate は +miinfo->cstate から取得): + +```c + CopyFromInstrStartPhase(cstate); + table_multi_insert(resultRelInfo->ri_RelationDesc, + slots, nused, mycid, ti_options, buffer->bistate); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT); + + for (i = 0; i < nused; i++) + { + ... + CopyFromInstrStartPhase(cstate); + recheckIndexes = ExecInsertIndexTuples(...); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INDEX); + ExecARInsertTriggers(...); /* タイマー外(トリガ計測に委ねる) */ + ... + } +``` + +計測意味論の注記: + +- INPUT は CopyFromRoutine->CopyFromOneRow のコールバック境界に一致する + ため、text/csv/binary だけでなくカスタムフォーマットも計測される。 +- INPUT には ON_ERROR でスキップされる行のパースコスト、および DEFAULT 式 + の評価コストが含まれる。 +- INDEX のフラッシュ内計測は行単位の start/stop になるが、発生するのは + インデックスが存在する場合のみで、コストは INPUT タイマー(全行で発生) + と同オーダー以下。 +- 検討した代替案: 既存の instrument.c(InstrStartNode / InstrStopNode + + NodeInstrumentation)をフェーズごとに割り当てる案。フェーズ別 + BufferUsage が将来ほぼ無償で得られる利点はあるが、firsttuple / + tuplecount 等プランノード前提の管理が不要に付いてくること、構造体が + EXPLAIN 出力側に対して過剰であることから、軽量な専用 API を採用した。 + 提案メールには代替案として記載する。 + +### 0004-5. トリガ・カウンタの計測 + +1. **トリガ**: EXPLAIN ANALYZE の既存トリガ計測機構を再利用する。 - CopyFrom の初期化部で、instr 設定時に対象 ResultRelInfo の ri_TrigInstrument に InstrAlloc(numTriggers, INSTRUMENT_TIMER, false) を設定。パーティションルーティング時は resultRelInfo 切替ブロック @@ -486,11 +597,11 @@ void CopyFromSetInstrumentation(CopyFromState cstate, result-relation 群から {トリガ名, リレーション名, calls, total time} を instr->triggers に集約する(EState は CopyFrom のローカルで あり解放されるため、ここで転記が必要)。 -4. **カウンタ**: excluded(copyfrom.c:1202 のローカル変数)と +2. **カウンタ**: excluded(copyfrom.c:1202 のローカル変数)と cstate->num_errors を終了時に instr へ転記。処理行数は CopyFrom の 戻り値を使用。 -### 0004-4. explain_copy.c の ANALYZE FROM 経路 +### 0004-6. explain_copy.c の ANALYZE FROM 経路 ```c /* 0002 の ereport(ERROR) を置き換え */ @@ -531,7 +642,7 @@ extern 化して再利用する。トリガ出力は report_triggers(explain.c:7 出力(ExplainOnePlan と同書式)。プランニングが存在しないため "Planning Time" は出力しない。 -### 0004-5. 出力設計 +### 0004-7. 出力設計 TEXT(TIMING ON): @@ -582,7 +693,7 @@ JSON: "Execution Time": 6543.210 }] ``` -### 0004-6. オーバーヘッドの見積りと緩和 +### 0004-8. オーバーヘッドの見積りと緩和 - TIMING ON 時の追加コストは行あたり clock_gettime × 2(input)+ 単一挿入経路なら × 4。Linux vDSO で 1 回 20〜30ns として 10^7 行で @@ -595,7 +706,7 @@ JSON: unpatched / patched+非EXPLAIN / ANALYZE+TIMING OFF / ANALYZE+TIMING ON を narrow(2 列)・wide(30 列)× 1000 万行で比較。 -### 0004-7. テスト +### 0004-9. テスト ``` \set filename :abs_builddir '/results/explain_copy.data' @@ -615,7 +726,7 @@ ROLLBACK; (途中エラーでも壊れないこと)。 - トランザクション内 ROLLBACK でデータが残らないこと。 -### 0004-8. 変更ファイル一覧 +### 0004-10. 変更ファイル一覧 ``` src/backend/commands/copyfrom.c | タイマー挿入、セッター、トリガ集約 From 0c69e1a153167a75c4def911c717ab2643089b73 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 03:20:00 +0000 Subject: [PATCH 4/9] Document rationale for extending EXPLAIN rather than adding COPY (VERBOSE) Add a design-rationale section comparing the EXPLAIN extension against the alternative of a COPY option emitting a NOTICE summary: role consistency with DML statements, structured output formats, orthogonal composition with existing EXPLAIN options, result-set vs NOTICE output channel, plan display for COPY (query) TO, and the no-execution EXPLAIN mode. Also records the counterarguments (STDIN coverage, VACUUM VERBOSE precedent, 'EXPLAIN is for plans') with rebuttals for the proposal mail. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- EXPLAIN_COPY_DESIGN.md | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/EXPLAIN_COPY_DESIGN.md b/EXPLAIN_COPY_DESIGN.md index b2e336759852a..129d608dfe031 100644 --- a/EXPLAIN_COPY_DESIGN.md +++ b/EXPLAIN_COPY_DESIGN.md @@ -23,6 +23,74 @@ 先行パッチに切り出すと、in-tree の呼び出し元が存在しない dead code に なるため、それを使う機能と同じパッチに含める。 +## 設計根拠: なぜ COPY (VERBOSE) オプションではなく EXPLAIN 拡張か + +対抗案は「COPY にオプション(例: `COPY ... WITH (VERBOSE)`)を追加し、 +完了時に NOTICE で内訳を出す」である。UI 層以外(計測コード = 0004 の +本体)は両案で共通であり、これは計測結果をどの機構に載せるかの選択で +ある。以下の理由で EXPLAIN を採る。 + +### 積極的根拠 + +1. **実行統計の取得は EXPLAIN ANALYZE の責務、という既存の役割分担**。 + INSERT / UPDATE / DELETE / MERGE / CTAS の実行統計はすべて + EXPLAIN ANALYZE で取る。COPY FROM は意味的にはバルク INSERT であり + (WHERE 句・トリガ・パーティションルーティングを持つ DML 的性格)、 + これだけ別の入口になるのはユーザーのメンタルモデル + (「遅い文を調べる → EXPLAIN ANALYZE」)に反する。 + +2. **構造化出力(FORMAT JSON/XML/YAML)が無償で手に入る**。 + NOTICE は自由書式テキストで、監視ツールや解析ツールが機械可読に + 取れず、文言変更が事実上の互換性破壊になる。EXPLAIN の JSON なら + フィールド追加が後方互換で行える。 + +3. **既存 EXPLAIN オプション体系との直交な合成**。TIMING ON/OFF・ + BUFFERS・WAL・SUMMARY・FORMAT がそのまま効き、計測オーバーヘッドの + 制御語彙(TIMING OFF)も再発明不要。VERBOSE 案ではこれらを COPY + オプションとして重複定義していくことになる。 + +4. **出力チャネルの問題**。COPY の結果はコマンドタグのみで行を返す場所が + なく、VERBOSE 案は必然的に NOTICE/ログになる。NOTICE はドライバに + よって扱いが異なり、捨てられることもある。EXPLAIN は結果セットとして + 返すため全クライアントで一様に扱える。リグレッションテストも + explain_filter 等の既存基盤に乗る。 + +5. **COPY (query) TO のプラン表示は EXPLAIN でしか成立しない**。内包 + クエリの実行計画を NOTICE に吐くのは auto_explain のログ形式問題の + 再来である。TO/FROM を一貫した UI で扱うなら EXPLAIN 側しかない。 + +6. **「実行しないで確認」(非 ANALYZE)が自然に手に入る**。COPY + オプション案には「実行しない COPY」という概念がなく、dry-run 相当を + 別途発明する必要がある。 + +7. **副作用と計測の意味論が確立済み**。「EXPLAIN ANALYZE は実際に実行 + する」は INSERT/CTAS で周知・文書化済みで、COPY FROM にもそのまま + 適用できる。 + +### 対抗案の利点と反論 + +- **STDIN が計測できない(EXPLAIN 案の弱点)**: VERBOSE 案なら + COPY FROM STDIN でも動くのは事実。ただし (a) チューニング目的の計測は + サーバサイドファイル / PROGRAM での再現実験として行うのが通常で、 + 実運用ストリームの常時計測は progress ビューやログの領分、 + (b) CopyFromInstrumentation はコマンド非依存に設計してあるため、 + STDIN 込みの軽量サマリが将来必要になれば、同じ計測基盤の上に + log_verbosity 拡張等を後付けできる。**EXPLAIN 案は VERBOSE 案を排除 + しないが、逆(NOTICE テキストから構造化出力・プラン表示へ)は + 成り立たない**。 +- **VACUUM (VERBOSE) 等の前例がある**: VACUUM / CLUSTER は プランを + 持たず、複数リレーションを対象にし得る純ユーティリティで、EXPLAIN の + 対象になりようがない。COPY は半分(query TO)が文字どおりプランを持ち、 + FROM も DML 的性格を持つ点で異なる。また VACUUM (VERBOSE) の出力が + 機械可読でないことは長年の不満で、pg_stat_progress_vacuum や + log_autovacuum 系が別途必要になった経緯は、むしろ「VERBOSE テキストは + 行き止まり」であることの証左。 +- **「EXPLAIN はプランの説明であり、プランのない COPY FROM は対象外」**: + EXPLAIN は既に NOTIFY / EXECUTE / DECLARE CURSOR / CTAS という + ユーティリティ文を扱っており(ExplainOneUtility)、実態は「文の実行 + 内容と実行統計の説明」である。プランノードを持たない文への内訳表示は + その自然な延長にある。 + 実行意味論(シリーズとして最初に固定し、以後変更しない): | コマンド | ANALYZE なし | ANALYZE あり | From cef0691361517db6e2dfdc325e9638adfe30c3ef Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 04:18:20 +0000 Subject: [PATCH 5/9] Expand anticipated objections in the design rationale Add three groups of additional counterarguments with rebuttals: alternatives (pg_stat_progress_copy, file_fdw workaround, profilers, cumulative stats), semantics/UX (\copy-over-STDIN practicality, phase sum vs Execution Time gap, observer effect, ANALYZE side effects, partial support matrix), and implementation/maintenance (auto_explain gap, pg_stat_statements consistency, hot-loop instrumentation rot, future parallel COPY). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- EXPLAIN_COPY_DESIGN.md | 72 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/EXPLAIN_COPY_DESIGN.md b/EXPLAIN_COPY_DESIGN.md index 129d608dfe031..ba216f2777e9d 100644 --- a/EXPLAIN_COPY_DESIGN.md +++ b/EXPLAIN_COPY_DESIGN.md @@ -91,6 +91,78 @@ 内容と実行統計の説明」である。プランノードを持たない文への内訳表示は その自然な延長にある。 +### その他想定される反論 + +#### (A) 代替手段系 + +- **「pg_stat_progress_copy を拡張すべき」**: progress ビューは実行中の + 外部観測用で、int64 スロット固定の軽量カウンタが設計思想。位相別時間の + ような「終了時に確定する統計」を载せる場所ではなく、事後に結果が + 消える点でチューニング用途に合わない。両者は補完関係(実行中 = + progress、事後の内訳 = EXPLAIN)と整理する。 +- **「file_fdw + EXPLAIN ANALYZE INSERT ... SELECT で既に可能」**: + 計測されるのは file_fdw と ModifyTable の経路であり、COPY 本体 + (マルチ挿入バッファ、FREEZE、ON_ERROR、binary フォーマット)とは + 別コードパス。COPY の性能問題を調べたいのに COPY を通らない計測では + 答えにならない。file_fdw は contrib であり binary 非対応。 +- **「perf / eBPF でプロファイルすれば取れる」**: 開発者向けの手段で + あり、本番 DBA が権限なしで・文単位で・全プラットフォームで取れる + 手段にならない。EXPLAIN ANALYZE が存在するのと同じ理由がそのまま + 当てはまる。 +- **「pg_stat_io 等の累積統計に載せるべき」**: 累積統計はインスタンス + 全体の傾向把握用で、特定の 1 文の内訳という問いに答えられない。 + +#### (B) 意味論・UX 系 + +- **「実際のロードは psql の \copy やドライバ経由(= FROM STDIN)が + 大半で、ANALYZE 不可では実用にならない」**(STDIN 反論の実務版): + 最も痛い反論として想定する。回答: (a) チューニングはサーバサイド + ファイル / PROGRAM への再現で行うのが現実的で、同一データなら計測 + 結果は転用できる(STDIN 固有コストはフロントエンド読込のみで、それは + Input Time に現れる性質のもの)。(b) プロトコル上は EXPLAIN 応答中の + CopyInResponse も可能で、psql は処理できる見込みがあるため、将来 + 緩和の余地を残した「初期実装の制限」と位置づける。 +- **「フェーズ合計 ≠ Execution Time で、差分は何かと必ず聞かれる」**: + WHERE 評価・パーティションルーティング・BEFORE トリガ・制約検査・ + タプル具現化などは 3 フェーズのどれにも入らない。回答: 内訳は網羅 + ではなく支配項の特定が目的であり、差分は「その他(ルーティング・ + 制約等)」としてドキュメントに明記する。enum + 配列の設計により + フェーズの追加は後方互換で行える。合計行を出さないことで「完全な + 分解」に見せない。 +- **「計測が計測対象を歪める(観測者効果)」**: TIMING ON の行単位 + タイマーで Input Time 自体が膨らむ。回答: EXPLAIN ANALYZE のプラン + 計測と同じ既知の性質で、既にドキュメント化された注意事項の適用範囲を + 広げるだけ。相対比較(どのフェーズが支配的か)には有効で、絶対値が + 必要なら TIMING OFF + 全体時間で確認できる。 +- **「EXPLAIN ANALYZE でデータが入るのは危険 / 驚き」**: INSERT / CTAS + で確立済みの意味論であり、ドキュメントに BEGIN/ROLLBACK 例を載せる。 + 「挿入せずパースだけ計測する dry-run」は実挿入と数値が乖離するため + 採らない(要望が強ければ将来の別オプション)。 +- **「サポートマトリクスが歯抜け(STDIN は非 ANALYZE のみ、relation TO + は ANALYZE 不可)で分かりにくい」**: 各制限に HINT 付きの明確な + エラーを出し、ドキュメントに対応表を載せる。歯抜けを理由に全部入りを + 待つより、段階的に埋める方が本体の流儀(MERGE の段階的拡張等)。 + +#### (C) 実装・保守系 + +- **「auto_explain で拾えないため、本番で遅かった夜間ロードの事後調査 + には使えない」**: auto_explain は Executor フック依存でユーティリティ + 文を扱えない。回答: 本パッチの計測基盤(CopyFromInstrumentation)は + EXPLAIN 非依存なので、将来 auto_explain のユーティリティ対応や + log_verbosity 拡張を同じ基盤で実装できる。本提案はその第一歩。 +- **「pg_stat_statements との整合」**: EXPLAIN ANALYZE COPY は内側の + CopyStmt が ProcessUtility を通らないため pgss に COPY として計上 + されない。これは EXPLAIN ANALYZE INSERT 等の既存挙動と同等であり、 + 新たな非一貫性ではないことを説明する。 +- **「ホットループに計測点を撒くと将来の COPY 改修で腐る」**: 挿入経路の + 追加時に計測漏れが起き得る。回答: 計測点は CopyFromRoutine 境界と + tableam / index / trigger 呼び出しという安定した抽象境界に限定して + おり、無秩序に散らしていない。リグレッションテストで各フェーズが + 非ゼロになるケースを持つことで漏れを検出する。 +- **「将来のパラレル COPY と衝突しないか」**: フェーズ別 instr_time + 配列はワーカー毎に持って合算可能な構造(Instrumentation の集約と + 同型)であり、設計上の障害にならない。 + 実行意味論(シリーズとして最初に固定し、以後変更しない): | コマンド | ANALYZE なし | ANALYZE あり | From 2b8c717837c4312cdfe20c56726929a84861b93d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 01:10:48 +0000 Subject: [PATCH 6/9] Refactor COPY statement setup into reusable functions Extract the setup portion of DoCopy() -- permission checks, opening and locking the target relation, WHERE clause transformation, column privilege checks, and the RLS conversion to a query-based COPY -- into ProcessCopyTarget(). Likewise extract the parse analysis, rewrite and validation of the source query of COPY (query) TO from BeginCopyTo() into CopyToTransformQuery(). No functional changes. These functions will be reused by an upcoming patch adding EXPLAIN support for COPY. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- src/backend/commands/copy.c | 68 ++++++++++++--- src/backend/commands/copyto.c | 155 ++++++++++++++++++---------------- src/include/commands/copy.h | 5 ++ 3 files changed, 144 insertions(+), 84 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 003b70852bb90..2833e44354b4b 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -41,28 +41,37 @@ #include "utils/rls.h" /* - * DoCopy executes the SQL COPY statement - * - * Either unload or reload contents of table , depending on . - * ( = true means we are inserting into the table.) In the "TO" case - * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE - * or DELETE query. + * ProcessCopyTarget + * Prepare the target of a COPY statement for execution * - * If is false, transfer is between the table and the file named - * . Otherwise, transfer is between the table and our regular - * input/output stream. The latter could be either stdin/stdout or a - * socket, depending on whether we're running under Postmaster control. + * This performs the permission checks and preparatory transformations + * needed before executing a COPY statement: checking the right to use + * COPY TO/FROM a file or program, opening and locking the target relation + * (if any), transforming and validating the WHERE clause, checking + * privileges on the target columns, and converting the statement into a + * query-based COPY when row-level security is enabled on the target + * relation. * * Do not allow a Postgres user without the 'pg_read_server_files' or * 'pg_write_server_files' role to read from or write to a file. * * Do not allow the copy if user doesn't have proper permission to access * the table or the specifically requested columns. + * + * The results are returned in *rel_p (the opened target relation, or NULL + * when the COPY uses a query), *relid_p (the OID of the target relation, + * or InvalidOid), *query_p (the source query for a query-based COPY TO, + * or NULL) and *whereClause_p (the transformed WHERE clause for COPY + * FROM, or NULL). + * + * When *rel_p is set, the relation is left open and locked; the caller is + * responsible for closing it (keeping the lock until end of transaction). */ void -DoCopy(ParseState *pstate, const CopyStmt *stmt, - int stmt_location, int stmt_len, - uint64 *processed) +ProcessCopyTarget(ParseState *pstate, const CopyStmt *stmt, + int stmt_location, int stmt_len, + Relation *rel_p, Oid *relid_p, + RawStmt **query_p, Node **whereClause_p) { bool is_from = stmt->is_from; bool pipe = (stmt->filename == NULL); @@ -354,6 +363,39 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, rel = NULL; } + *rel_p = rel; + *relid_p = relid; + *query_p = query; + *whereClause_p = whereClause; +} + +/* + * DoCopy executes the SQL COPY statement + * + * Either unload or reload contents of table , depending on . + * ( = true means we are inserting into the table.) In the "TO" case + * we also support copying the output of an arbitrary SELECT, INSERT, UPDATE + * or DELETE query. + * + * If is false, transfer is between the table and the file named + * . Otherwise, transfer is between the table and our regular + * input/output stream. The latter could be either stdin/stdout or a + * socket, depending on whether we're running under Postmaster control. + */ +void +DoCopy(ParseState *pstate, const CopyStmt *stmt, + int stmt_location, int stmt_len, + uint64 *processed) +{ + bool is_from = stmt->is_from; + Relation rel; + Oid relid; + RawStmt *query; + Node *whereClause; + + ProcessCopyTarget(pstate, stmt, stmt_location, stmt_len, + &rel, &relid, &query, &whereClause); + if (is_from) { CopyFromState cstate; diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index d3adc752ae30c..4f8ce5b78ed18 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -769,6 +769,89 @@ EndCopy(CopyToState cstate) pfree(cstate); } +/* + * CopyToTransformQuery + * Run parse analysis and rewrite on the source query of a COPY TO + * command, and check that it is something COPY can handle. + * + * Note this also acquires sufficient locks on the source table(s). + */ +Query * +CopyToTransformQuery(ParseState *pstate, RawStmt *raw_query) +{ + List *rewritten; + Query *query; + + rewritten = pg_analyze_and_rewrite_fixedparams(raw_query, + pstate->p_sourcetext, NULL, 0, + NULL); + + /* check that we got back something we can work with */ + if (rewritten == NIL) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("DO INSTEAD NOTHING rules are not supported for COPY"))); + } + else if (list_length(rewritten) > 1) + { + ListCell *lc; + + /* examine queries to determine which error message to issue */ + foreach(lc, rewritten) + { + Query *q = lfirst_node(Query, lc); + + if (q->querySource == QSRC_QUAL_INSTEAD_RULE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("conditional DO INSTEAD rules are not supported for COPY"))); + if (q->querySource == QSRC_NON_INSTEAD_RULE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("DO ALSO rules are not supported for COPY"))); + } + + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("multi-statement DO INSTEAD rules are not supported for COPY"))); + } + + query = linitial_node(Query, rewritten); + + /* The grammar allows SELECT INTO, but we don't support that */ + if (query->utilityStmt != NULL && + IsA(query->utilityStmt, CreateTableAsStmt)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY (SELECT INTO) is not supported"))); + + /* The only other utility command we could see is NOTIFY */ + if (query->utilityStmt != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY query must not be a utility command"))); + + /* + * Similarly the grammar doesn't enforce the presence of a RETURNING + * clause, but this is required here. + */ + if (query->commandType != CMD_SELECT && + query->returningList == NIL) + { + Assert(query->commandType == CMD_INSERT || + query->commandType == CMD_UPDATE || + query->commandType == CMD_DELETE || + query->commandType == CMD_MERGE); + + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY query must have a RETURNING clause"))); + } + + return query; +} + /* * Setup CopyToState to read tuples from a table or a query for COPY TO. * @@ -906,7 +989,6 @@ BeginCopyTo(ParseState *pstate, } else { - List *rewritten; Query *query; PlannedStmt *plan; DestReceiver *dest; @@ -914,76 +996,7 @@ BeginCopyTo(ParseState *pstate, cstate->rel = NULL; cstate->partitions = NIL; - /* - * Run parse analysis and rewrite. Note this also acquires sufficient - * locks on the source table(s). - */ - rewritten = pg_analyze_and_rewrite_fixedparams(raw_query, - pstate->p_sourcetext, NULL, 0, - NULL); - - /* check that we got back something we can work with */ - if (rewritten == NIL) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("DO INSTEAD NOTHING rules are not supported for COPY"))); - } - else if (list_length(rewritten) > 1) - { - ListCell *lc; - - /* examine queries to determine which error message to issue */ - foreach(lc, rewritten) - { - Query *q = lfirst_node(Query, lc); - - if (q->querySource == QSRC_QUAL_INSTEAD_RULE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("conditional DO INSTEAD rules are not supported for COPY"))); - if (q->querySource == QSRC_NON_INSTEAD_RULE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("DO ALSO rules are not supported for COPY"))); - } - - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("multi-statement DO INSTEAD rules are not supported for COPY"))); - } - - query = linitial_node(Query, rewritten); - - /* The grammar allows SELECT INTO, but we don't support that */ - if (query->utilityStmt != NULL && - IsA(query->utilityStmt, CreateTableAsStmt)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY (SELECT INTO) is not supported"))); - - /* The only other utility command we could see is NOTIFY */ - if (query->utilityStmt != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY query must not be a utility command"))); - - /* - * Similarly the grammar doesn't enforce the presence of a RETURNING - * clause, but this is required here. - */ - if (query->commandType != CMD_SELECT && - query->returningList == NIL) - { - Assert(query->commandType == CMD_INSERT || - query->commandType == CMD_UPDATE || - query->commandType == CMD_DELETE || - query->commandType == CMD_MERGE); - - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY query must have a RETURNING clause"))); - } + query = CopyToTransformQuery(pstate, raw_query); /* plan the query */ plan = pg_plan_query(query, pstate->p_sourcetext, diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index abecfe510981e..e34350a86f2af 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -109,6 +109,10 @@ typedef void (*copy_data_dest_cb) (void *data, int len); extern void DoCopy(ParseState *pstate, const CopyStmt *stmt, int stmt_location, int stmt_len, uint64 *processed); +extern void ProcessCopyTarget(ParseState *pstate, const CopyStmt *stmt, + int stmt_location, int stmt_len, + Relation *rel_p, Oid *relid_p, + RawStmt **query_p, Node **whereClause_p); extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, List *options); extern CopyFromState BeginCopyFrom(ParseState *pstate, Relation rel, Node *whereClause, @@ -129,6 +133,7 @@ extern DestReceiver *CreateCopyDestReceiver(void); /* * internal prototypes */ +extern Query *CopyToTransformQuery(ParseState *pstate, RawStmt *raw_query); extern CopyToState BeginCopyTo(ParseState *pstate, Relation rel, RawStmt *raw_query, Oid queryRelId, const char *filename, bool is_program, copy_data_dest_cb data_dest_cb, List *attnamelist, List *options); From 1dc189570db1d14edfa6d676c7afb43c23d76ba2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 01:21:02 +0000 Subject: [PATCH 7/9] Allow EXPLAIN on COPY statements Add CopyStmt to ExplainableStmt in the grammar and dispatch it from ExplainOneUtility() to the new ExplainCopyStmt() in explain_copy.c. Plain EXPLAIN never executes the COPY: it performs the same permission checks and preparatory transformations as execution would (reusing ProcessCopyTarget()), then prints a description of the COPY operation: target relation, format, source/target and non-default options. For COPY (query) TO, the execution plan of the source query is shown as well; to print the COPY details inside the query's output group, ExplainOnePlan() gains a CopyStmt parameter, following the precedent of its IntoClause parameter for CREATE TABLE AS. EXPLAIN ANALYZE of COPY is rejected for now; support for it is added separately for COPY (query) TO and COPY FROM by upcoming patches. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- doc/src/sgml/ref/copy.sgml | 7 + doc/src/sgml/ref/explain.sgml | 15 +- src/backend/commands/Makefile | 1 + src/backend/commands/explain.c | 11 +- src/backend/commands/explain_copy.c | 318 +++++++++++++++++++++ src/backend/commands/meson.build | 1 + src/backend/commands/prepare.c | 2 +- src/backend/parser/gram.y | 3 +- src/bin/psql/tab-complete.in.c | 6 +- src/include/commands/explain.h | 8 +- src/test/regress/expected/explain_copy.out | 138 +++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/explain_copy.sql | 49 ++++ 13 files changed, 550 insertions(+), 11 deletions(-) create mode 100644 src/backend/commands/explain_copy.c create mode 100644 src/test/regress/expected/explain_copy.out create mode 100644 src/test/regress/sql/explain_copy.sql diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 4706c9a44100c..3ff1e94c242b0 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -584,6 +584,13 @@ COPY count Notes + + can be used to display a description of + a COPY command, including the execution plan of the + source query of COPY + (query) TO. + + COPY TO can be used with plain tables, populated materialized views, and partitioned tables. diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index f38d31e106a39..4122b3c6be851 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -350,10 +350,21 @@ ROLLBACK; Any SELECT, INSERT, UPDATE, DELETE, MERGE, VALUES, EXECUTE, - DECLARE, CREATE TABLE AS, or - CREATE MATERIALIZED VIEW AS statement, whose execution + DECLARE, CREATE TABLE AS, + CREATE MATERIALIZED VIEW AS, or + COPY statement, whose execution plan you wish to see. + + For a COPY statement, EXPLAIN + shows a description of the COPY operation; when + copying the result of a query with + COPY (query) TO, the + execution plan of the query is also shown. Without + ANALYZE, the COPY source or + destination file is not accessed, so a nonexistent file is not + detected by plain EXPLAIN. + diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index 5b9d084977e48..a0e37c72fe023 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -33,6 +33,7 @@ OBJS = \ dropcmds.o \ event_trigger.o \ explain.o \ + explain_copy.o \ explain_dr.o \ explain_format.o \ explain_state.o \ diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index a40d03d35f3ba..cb824f41a5dda 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -377,7 +377,7 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, /* run it (if needed) and produce output */ ExplainOnePlan(plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL), - es->memory ? &mem_counters : NULL); + es->memory ? &mem_counters : NULL, NULL); } /* @@ -467,6 +467,8 @@ ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, else if (IsA(utilityStmt, ExecuteStmt)) ExplainExecuteQuery((ExecuteStmt *) utilityStmt, into, es, pstate, params); + else if (IsA(utilityStmt, CopyStmt)) + ExplainCopyStmt(castNode(CopyStmt, utilityStmt), es, pstate, params); else if (IsA(utilityStmt, NotifyStmt)) { if (es->format == EXPLAIN_FORMAT_TEXT) @@ -501,7 +503,8 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, - const MemoryContextCounters *mem_counters) + const MemoryContextCounters *mem_counters, + const CopyStmt *copystmt) { DestReceiver *dest; QueryDesc *queryDesc; @@ -605,6 +608,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, /* Create textual dump of plan tree */ ExplainPrintPlan(es, queryDesc); + /* Show details of the COPY statement, if we are explaining one */ + if (copystmt) + ExplainPrintCopyInfo(copystmt, es); + /* Show buffer and/or memory usage in planning */ if (peek_buffer_usage(es, bufusage) || mem_counters) { diff --git a/src/backend/commands/explain_copy.c b/src/backend/commands/explain_copy.c new file mode 100644 index 0000000000000..27a2fc80609b9 --- /dev/null +++ b/src/backend/commands/explain_copy.c @@ -0,0 +1,318 @@ +/*------------------------------------------------------------------------- + * + * explain_copy.c + * Explain a COPY statement + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994-5, Regents of the University of California + * + * IDENTIFICATION + * src/backend/commands/explain_copy.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/table.h" +#include "commands/copy.h" +#include "commands/explain.h" +#include "commands/explain_format.h" +#include "commands/explain_state.h" +#include "executor/instrument.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/ruleutils.h" + +static void ExplainCopyToQuery(const CopyStmt *stmt, RawStmt *raw_query, + Oid queryRelId, ExplainState *es, + ParseState *pstate, ParamListInfo params); +static void ExplainCopyGeneric(const CopyStmt *stmt, Relation rel, + const CopyFormatOptions *opts, + ExplainState *es); +static void show_copy_properties(const CopyStmt *stmt, + const CopyFormatOptions *opts, + ExplainState *es); + +/* + * ExplainCopyStmt - + * print out the execution "plan" for one COPY statement + * + * This is called back from ExplainOneUtility. + */ +void +ExplainCopyStmt(CopyStmt *stmt, ExplainState *es, + ParseState *pstate, ParamListInfo params) +{ + Relation rel; + Oid relid; + RawStmt *query; + Node *whereClause; + CopyFormatOptions opts = {0}; /* ProcessCopyOptions expects zeroes */ + + /* + * Perform the same permission checks and preparatory transformations as + * the execution of the COPY statement would, so that EXPLAIN reports the + * same errors. This opens and locks the target relation, and converts + * the statement to a query-based COPY if row-level security applies. + */ + ProcessCopyTarget(pstate, stmt, -1, 0, + &rel, &relid, &query, &whereClause); + + /* Likewise, validate the options; we also need them for the output */ + ProcessCopyOptions(pstate, &opts, stmt->is_from, stmt->options); + + if (stmt->is_from) + { + if (es->analyze) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("EXPLAIN ANALYZE is not supported for COPY FROM"))); + + ExplainCopyGeneric(stmt, rel, &opts, es); + } + else if (query != NULL) + { + /* COPY (query) TO, or COPY relation TO converted because of RLS */ + if (es->analyze) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("EXPLAIN ANALYZE is not supported for COPY TO"))); + + ExplainCopyToQuery(stmt, query, relid, es, pstate, params); + } + else + { + /* COPY relation TO, no plan to show */ + if (es->analyze) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("EXPLAIN ANALYZE is not supported for COPY relation TO"), + errhint("Try the COPY (SELECT ...) TO variant."))); + + ExplainCopyGeneric(stmt, rel, &opts, es); + } + + if (rel != NULL) + table_close(rel, NoLock); +} + +/* + * ExplainCopyToQuery - + * print the plan of the source query of a query-based COPY TO + * + * This parallels standard_ExplainOneQuery, but starts from the raw source + * query of the COPY statement and validates it as BeginCopyTo would. + */ +static void +ExplainCopyToQuery(const CopyStmt *stmt, RawStmt *raw_query, Oid queryRelId, + ExplainState *es, ParseState *pstate, ParamListInfo params) +{ + Query *query; + PlannedStmt *plan; + instr_time planstart, + planduration; + BufferUsage bufusage_start, + bufusage; + MemoryContextCounters mem_counters; + MemoryContext planner_ctx = NULL; + MemoryContext saved_ctx = NULL; + + /* + * Run parse analysis, rewrite and COPY-specific validation. Note that + * parse analysis also computes the query identifier and invokes the + * post_parse_analyze_hook, as the execution of the COPY statement would. + */ + query = CopyToTransformQuery(pstate, raw_query); + + if (es->memory) + { + /* See standard_ExplainOneQuery for the reasoning here */ + planner_ctx = AllocSetContextCreate(CurrentMemoryContext, + "explain analyze planner context", + ALLOCSET_DEFAULT_SIZES); + saved_ctx = MemoryContextSwitchTo(planner_ctx); + } + + if (es->buffers) + bufusage_start = pgBufferUsage; + INSTR_TIME_SET_CURRENT(planstart); + + /* plan the query */ + plan = pg_plan_query(query, pstate->p_sourcetext, CURSOR_OPT_PARALLEL_OK, + params, es); + + INSTR_TIME_SET_CURRENT(planduration); + INSTR_TIME_SUBTRACT(planduration, planstart); + + if (es->memory) + { + MemoryContextSwitchTo(saved_ctx); + MemoryContextMemConsumed(planner_ctx, &mem_counters); + } + + /* calc differences of buffer counters. */ + if (es->buffers) + { + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); + } + + /* + * When a relation-based COPY TO was converted to a query because of + * row-level security, check that the planner resolved the same relation + * we originally locked, as BeginCopyTo does. + */ + if (OidIsValid(queryRelId) && + !list_member_oid(plan->relationOids, queryRelId)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("relation referenced by COPY statement has changed"))); + + /* produce output (and run the query, if ANALYZE was given) */ + ExplainOnePlan(plan, NULL, es, pstate->p_sourcetext, params, + pstate->p_queryEnv, &planduration, + (es->buffers ? &bufusage : NULL), + (es->memory ? &mem_counters : NULL), + stmt); +} + +/* + * ExplainCopyGeneric - + * print the description of a COPY statement that has no plan to show + * (COPY ... FROM ..., or table-based COPY ... TO ...) + */ +static void +ExplainCopyGeneric(const CopyStmt *stmt, Relation rel, + const CopyFormatOptions *opts, ExplainState *es) +{ + const char *label = stmt->is_from ? "Copy From" : "Copy To"; + + ExplainOpenGroup("Query", NULL, true, es); + ExplainOpenGroup(label, label, true, es); + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + ExplainIndentText(es); + if (es->verbose) + appendStringInfo(es->str, "%s on %s.%s\n", label, + quote_identifier(get_namespace_name(RelationGetNamespace(rel))), + quote_identifier(RelationGetRelationName(rel))); + else + appendStringInfo(es->str, "%s on %s\n", label, + quote_identifier(RelationGetRelationName(rel))); + es->indent++; + } + else + { + ExplainPropertyText("Relation Name", + RelationGetRelationName(rel), es); + if (es->verbose) + ExplainPropertyText("Schema", + get_namespace_name(RelationGetNamespace(rel)), + es); + } + + show_copy_properties(stmt, opts, es); + + if (es->format == EXPLAIN_FORMAT_TEXT) + es->indent--; + + ExplainCloseGroup(label, label, true, es); + ExplainCloseGroup("Query", NULL, true, es); +} + +/* + * ExplainPrintCopyInfo - + * print the details of the COPY statement being explained within the + * output of its source query's plan + * + * This is called back from ExplainOnePlan when explaining a query-based + * COPY TO, inside the "Query" output group. + */ +void +ExplainPrintCopyInfo(const CopyStmt *copystmt, ExplainState *es) +{ + CopyFormatOptions opts = {0}; /* ProcessCopyOptions expects zeroes */ + ParseState *tmp_pstate = make_parsestate(NULL); + + /* Re-extract the options; this cannot fail, they were checked before */ + ProcessCopyOptions(tmp_pstate, &opts, copystmt->is_from, + copystmt->options); + free_parsestate(tmp_pstate); + + ExplainOpenGroup("Copy", "Copy", true, es); + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + ExplainIndentText(es); + appendStringInfo(es->str, "%s\n", + copystmt->is_from ? "Copy From" : "Copy To"); + es->indent++; + } + + show_copy_properties(copystmt, &opts, es); + + if (es->format == EXPLAIN_FORMAT_TEXT) + es->indent--; + + ExplainCloseGroup("Copy", "Copy", true, es); +} + +/* + * show_copy_properties - + * print the properties of a COPY statement that are common to all + * EXPLAIN COPY output variants + */ +static void +show_copy_properties(const CopyStmt *stmt, const CopyFormatOptions *opts, + ExplainState *es) +{ + const char *format_name; + + switch (opts->format) + { + case COPY_FORMAT_TEXT: + format_name = "text"; + break; + case COPY_FORMAT_BINARY: + format_name = "binary"; + break; + case COPY_FORMAT_CSV: + format_name = "csv"; + break; + case COPY_FORMAT_JSON: + format_name = "json"; + break; + default: + format_name = "???"; + break; + } + ExplainPropertyText("Format", format_name, es); + + if (stmt->filename == NULL) + ExplainPropertyText(stmt->is_from ? "Source" : "Target", + stmt->is_from ? "stdin" : "stdout", es); + else if (stmt->is_program) + { + ExplainPropertyText(stmt->is_from ? "Source" : "Target", + "program", es); + ExplainPropertyText("Program", stmt->filename, es); + } + else + { + ExplainPropertyText(stmt->is_from ? "Source" : "Target", + "file", es); + ExplainPropertyText("File", stmt->filename, es); + } + + if (opts->freeze) + ExplainPropertyBool("Freeze", true, es); + + if (opts->on_error != COPY_ON_ERROR_STOP) + ExplainPropertyText("On Error", + opts->on_error == COPY_ON_ERROR_IGNORE ? + "ignore" : "set_null", es); +} diff --git a/src/backend/commands/meson.build b/src/backend/commands/meson.build index 9f258d566ebf6..93ff22df5664a 100644 --- a/src/backend/commands/meson.build +++ b/src/backend/commands/meson.build @@ -21,6 +21,7 @@ backend_sources += files( 'dropcmds.c', 'event_trigger.c', 'explain.c', + 'explain_copy.c', 'explain_dr.c', 'explain_format.c', 'explain_state.c', diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c index 876aad2100aeb..604ce0de1376b 100644 --- a/src/backend/commands/prepare.c +++ b/src/backend/commands/prepare.c @@ -661,7 +661,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, if (pstmt->commandType != CMD_UTILITY) ExplainOnePlan(pstmt, into, es, query_string, paramLI, pstate->p_queryEnv, &planduration, (es->buffers ? &bufusage : NULL), - es->memory ? &mem_counters : NULL); + es->memory ? &mem_counters : NULL, NULL); else ExplainOneUtility(pstmt->utilityStmt, into, es, pstate, paramLI); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index ff4e1388c5500..48e8c204ab4c5 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -12853,7 +12853,8 @@ ExplainableStmt: | CreateAsStmt | CreateMatViewStmt | RefreshMatViewStmt - | ExecuteStmt /* by default all are $$=$1 */ + | ExecuteStmt + | CopyStmt /* by default all are $$=$1 */ ; /***************************************************************************** diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 49ea584cd4f9d..d282776376f44 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -4516,7 +4516,7 @@ match_previous_words(int pattern_id, */ else if (Matches("EXPLAIN")) COMPLETE_WITH("SELECT", "INSERT INTO", "DELETE FROM", "UPDATE", "DECLARE", - "MERGE INTO", "EXECUTE", "ANALYZE", "VERBOSE"); + "MERGE INTO", "EXECUTE", "COPY", "ANALYZE", "VERBOSE"); else if (HeadMatches("EXPLAIN", "(*") && !HeadMatches("EXPLAIN", "(*)")) { @@ -4538,12 +4538,12 @@ match_previous_words(int pattern_id, } else if (Matches("EXPLAIN", "ANALYZE")) COMPLETE_WITH("SELECT", "INSERT INTO", "DELETE FROM", "UPDATE", "DECLARE", - "MERGE INTO", "EXECUTE", "VERBOSE"); + "MERGE INTO", "EXECUTE", "COPY", "VERBOSE"); else if (Matches("EXPLAIN", "(*)") || Matches("EXPLAIN", "VERBOSE") || Matches("EXPLAIN", "ANALYZE", "VERBOSE")) COMPLETE_WITH("SELECT", "INSERT INTO", "DELETE FROM", "UPDATE", "DECLARE", - "MERGE INTO", "EXECUTE"); + "MERGE INTO", "EXECUTE", "COPY"); /* FETCH && MOVE */ diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 472e141bba3ff..526e0cea6eefd 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -69,7 +69,13 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, - const MemoryContextCounters *mem_counters); + const MemoryContextCounters *mem_counters, + const CopyStmt *copystmt); + +/* in commands/explain_copy.c */ +extern void ExplainCopyStmt(CopyStmt *stmt, ExplainState *es, + ParseState *pstate, ParamListInfo params); +extern void ExplainPrintCopyInfo(const CopyStmt *copystmt, ExplainState *es); extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc); extern void ExplainPrintTriggers(ExplainState *es, diff --git a/src/test/regress/expected/explain_copy.out b/src/test/regress/expected/explain_copy.out new file mode 100644 index 0000000000000..a5efd1aebb3ec --- /dev/null +++ b/src/test/regress/expected/explain_copy.out @@ -0,0 +1,138 @@ +-- +-- Tests for EXPLAIN of COPY statements +-- +create table explain_copy_tbl (a int, b text); +insert into explain_copy_tbl values (1, 'one'), (2, 'two'); +-- Plain EXPLAIN does not execute the COPY, and does not access the +-- source/destination file. +explain copy explain_copy_tbl from '/no/such/file'; + QUERY PLAN +------------------------------- + Copy From on explain_copy_tbl + Format: text + Source: file + File: /no/such/file +(4 rows) + +explain (verbose) copy explain_copy_tbl to '/no/such/file' with (format binary); + QUERY PLAN +------------------------------------ + Copy To on public.explain_copy_tbl + Format: binary + Target: file + File: /no/such/file +(4 rows) + +explain copy explain_copy_tbl from stdin with (format csv, on_error ignore); + QUERY PLAN +------------------------------- + Copy From on explain_copy_tbl + Format: csv + Source: stdin + On Error: ignore +(4 rows) + +explain copy explain_copy_tbl (a) from program '/no/such/program'; + QUERY PLAN +------------------------------- + Copy From on explain_copy_tbl + Format: text + Source: program + Program: /no/such/program +(4 rows) + +explain copy explain_copy_tbl to stdout; + QUERY PLAN +----------------------------- + Copy To on explain_copy_tbl + Format: text + Target: stdout +(3 rows) + +-- The plan of the source query of COPY (query) TO is shown. +explain (costs off) copy (select * from explain_copy_tbl where a > 0) to stdout; + QUERY PLAN +------------------------------ + Seq Scan on explain_copy_tbl + Filter: (a > 0) + Copy To + Format: text + Target: stdout +(5 rows) + +explain (costs off, format json) copy (select a from explain_copy_tbl) to stdout (format csv); + QUERY PLAN +-------------------------------------------- + [ + + { + + "Plan": { + + "Node Type": "Seq Scan", + + "Parallel Aware": false, + + "Async Capable": false, + + "Relation Name": "explain_copy_tbl",+ + "Alias": "explain_copy_tbl", + + "Disabled": false + + }, + + "Copy": { + + "Format": "csv", + + "Target": "stdout" + + } + + } + + ] +(1 row) + +-- invalid cases +explain copy explain_copy_no_such_table from stdin; +ERROR: relation "explain_copy_no_such_table" does not exist +explain copy explain_copy_tbl from stdin with (format nosuch); +ERROR: COPY format "nosuch" not recognized +LINE 1: explain copy explain_copy_tbl from stdin with (format nosuch... + ^ +explain (analyze) copy explain_copy_tbl from stdin; +ERROR: EXPLAIN ANALYZE is not supported for COPY FROM +explain (analyze) copy explain_copy_tbl to stdout; +ERROR: EXPLAIN ANALYZE is not supported for COPY relation TO +HINT: Try the COPY (SELECT ...) TO variant. +explain (analyze) copy (select 1) to stdout; +ERROR: EXPLAIN ANALYZE is not supported for COPY TO +-- Plain EXPLAIN performs the same permission checks as COPY would. +create role regress_explain_copy; +grant select, insert on explain_copy_tbl to regress_explain_copy; +set role regress_explain_copy; +explain copy explain_copy_tbl from '/no/such/file'; -- fail, no file access +ERROR: permission denied to COPY from a file +DETAIL: Only roles with privileges of the "pg_read_server_files" role may COPY from a file. +HINT: Anyone can COPY to stdout or from stdin. psql's \copy command also works for anyone. +explain copy explain_copy_tbl from stdin; + QUERY PLAN +------------------------------- + Copy From on explain_copy_tbl + Format: text + Source: stdin +(3 rows) + +reset role; +-- Row-level security converts COPY relation TO into a query-based COPY. +create table explain_copy_rls (a int); +insert into explain_copy_rls values (1), (-1); +alter table explain_copy_rls enable row level security; +create policy explain_copy_policy on explain_copy_rls + for select using (a > 0); +grant select on explain_copy_rls to regress_explain_copy; +set role regress_explain_copy; +explain (costs off) copy explain_copy_rls to stdout; + QUERY PLAN +------------------------------ + Seq Scan on explain_copy_rls + Filter: (a > 0) + Copy To + Format: text + Target: stdout +(5 rows) + +reset role; +revoke all on explain_copy_tbl from regress_explain_copy; +revoke all on explain_copy_rls from regress_explain_copy; +drop role regress_explain_copy; +drop table explain_copy_rls; +drop table explain_copy_tbl; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 8fa0a6c47fb30..e5932826de641 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -123,7 +123,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr # The stats test resets stats, so nothing else needing stats access can be in # this group. # ---------- -test: partition_merge partition_split partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain memoize stats predicate numa eager_aggregate graph_table_rls planner_est +test: partition_merge partition_split partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain explain_copy memoize stats predicate numa eager_aggregate graph_table_rls planner_est # ---------- # Another group of parallel tests (compression) diff --git a/src/test/regress/sql/explain_copy.sql b/src/test/regress/sql/explain_copy.sql new file mode 100644 index 0000000000000..b630053df6815 --- /dev/null +++ b/src/test/regress/sql/explain_copy.sql @@ -0,0 +1,49 @@ +-- +-- Tests for EXPLAIN of COPY statements +-- +create table explain_copy_tbl (a int, b text); +insert into explain_copy_tbl values (1, 'one'), (2, 'two'); + +-- Plain EXPLAIN does not execute the COPY, and does not access the +-- source/destination file. +explain copy explain_copy_tbl from '/no/such/file'; +explain (verbose) copy explain_copy_tbl to '/no/such/file' with (format binary); +explain copy explain_copy_tbl from stdin with (format csv, on_error ignore); +explain copy explain_copy_tbl (a) from program '/no/such/program'; +explain copy explain_copy_tbl to stdout; + +-- The plan of the source query of COPY (query) TO is shown. +explain (costs off) copy (select * from explain_copy_tbl where a > 0) to stdout; +explain (costs off, format json) copy (select a from explain_copy_tbl) to stdout (format csv); + +-- invalid cases +explain copy explain_copy_no_such_table from stdin; +explain copy explain_copy_tbl from stdin with (format nosuch); +explain (analyze) copy explain_copy_tbl from stdin; +explain (analyze) copy explain_copy_tbl to stdout; +explain (analyze) copy (select 1) to stdout; + +-- Plain EXPLAIN performs the same permission checks as COPY would. +create role regress_explain_copy; +grant select, insert on explain_copy_tbl to regress_explain_copy; +set role regress_explain_copy; +explain copy explain_copy_tbl from '/no/such/file'; -- fail, no file access +explain copy explain_copy_tbl from stdin; +reset role; + +-- Row-level security converts COPY relation TO into a query-based COPY. +create table explain_copy_rls (a int); +insert into explain_copy_rls values (1), (-1); +alter table explain_copy_rls enable row level security; +create policy explain_copy_policy on explain_copy_rls + for select using (a > 0); +grant select on explain_copy_rls to regress_explain_copy; +set role regress_explain_copy; +explain (costs off) copy explain_copy_rls to stdout; +reset role; + +revoke all on explain_copy_tbl from regress_explain_copy; +revoke all on explain_copy_rls from regress_explain_copy; +drop role regress_explain_copy; +drop table explain_copy_rls; +drop table explain_copy_tbl; From ee7d9eb59059f92da05e8e55fdab87423e3df727 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 01:22:24 +0000 Subject: [PATCH 8/9] Allow EXPLAIN ANALYZE on COPY (query) TO With ANALYZE, the source query of a query-based COPY TO is now planned and executed through the regular EXPLAIN machinery, reporting node timings, buffer usage and WAL usage as for any other query. The COPY output itself is not produced: no file is written, and no data is sent to the client, paralleling how EXPLAIN ANALYZE discards the rows a SELECT would return. This also means no COPY protocol messages are emitted, so COPY ... TO STDOUT needs no special treatment. EXPLAIN ANALYZE of COPY FROM and of table-based COPY TO remains unsupported for now. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- doc/src/sgml/ref/explain.sgml | 7 ++++++ src/backend/commands/explain_copy.c | 14 ++++++----- src/test/regress/expected/explain_copy.out | 27 ++++++++++++++++++++-- src/test/regress/sql/explain_copy.sql | 9 +++++++- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index 4122b3c6be851..b02d41adc8d6a 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -365,6 +365,13 @@ ROLLBACK; destination file is not accessed, so a nonexistent file is not detected by plain EXPLAIN. + + EXPLAIN ANALYZE of a + COPY (query) TO statement + executes the source query, but does not produce the + COPY output: no file is written, and no data is + sent to the client. + diff --git a/src/backend/commands/explain_copy.c b/src/backend/commands/explain_copy.c index 27a2fc80609b9..dbdabf7a2ada0 100644 --- a/src/backend/commands/explain_copy.c +++ b/src/backend/commands/explain_copy.c @@ -75,12 +75,14 @@ ExplainCopyStmt(CopyStmt *stmt, ExplainState *es, } else if (query != NULL) { - /* COPY (query) TO, or COPY relation TO converted because of RLS */ - if (es->analyze) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("EXPLAIN ANALYZE is not supported for COPY TO"))); - + /* + * COPY (query) TO, or COPY relation TO converted because of RLS. + * + * With ANALYZE, the source query is executed, but the COPY output is + * not produced: no file is written, and no data is sent to the + * client. This parallels EXPLAIN ANALYZE discarding the rows a + * SELECT would return. + */ ExplainCopyToQuery(stmt, query, relid, es, pstate, params); } else diff --git a/src/test/regress/expected/explain_copy.out b/src/test/regress/expected/explain_copy.out index a5efd1aebb3ec..d9d4b961b9e72 100644 --- a/src/test/regress/expected/explain_copy.out +++ b/src/test/regress/expected/explain_copy.out @@ -81,6 +81,31 @@ explain (costs off, format json) copy (select a from explain_copy_tbl) to stdout ] (1 row) +-- EXPLAIN ANALYZE of a query-based COPY TO executes the source query, +-- but produces no COPY output: no file is written, and no data is sent +-- to the client. +explain (analyze, costs off, timing off, summary off, buffers off) + copy (select * from explain_copy_tbl where a > 0) to stdout; + QUERY PLAN +--------------------------------------------------------- + Seq Scan on explain_copy_tbl (actual rows=2.00 loops=1) + Filter: (a > 0) + Copy To + Format: text + Target: stdout +(5 rows) + +explain (analyze, costs off, timing off, summary off, buffers off) + copy (select a from explain_copy_tbl) to '/no/such/dir/file'; + QUERY PLAN +--------------------------------------------------------- + Seq Scan on explain_copy_tbl (actual rows=2.00 loops=1) + Copy To + Format: text + Target: file + File: /no/such/dir/file +(5 rows) + -- invalid cases explain copy explain_copy_no_such_table from stdin; ERROR: relation "explain_copy_no_such_table" does not exist @@ -93,8 +118,6 @@ ERROR: EXPLAIN ANALYZE is not supported for COPY FROM explain (analyze) copy explain_copy_tbl to stdout; ERROR: EXPLAIN ANALYZE is not supported for COPY relation TO HINT: Try the COPY (SELECT ...) TO variant. -explain (analyze) copy (select 1) to stdout; -ERROR: EXPLAIN ANALYZE is not supported for COPY TO -- Plain EXPLAIN performs the same permission checks as COPY would. create role regress_explain_copy; grant select, insert on explain_copy_tbl to regress_explain_copy; diff --git a/src/test/regress/sql/explain_copy.sql b/src/test/regress/sql/explain_copy.sql index b630053df6815..4c0ee22a6c25c 100644 --- a/src/test/regress/sql/explain_copy.sql +++ b/src/test/regress/sql/explain_copy.sql @@ -16,12 +16,19 @@ explain copy explain_copy_tbl to stdout; explain (costs off) copy (select * from explain_copy_tbl where a > 0) to stdout; explain (costs off, format json) copy (select a from explain_copy_tbl) to stdout (format csv); +-- EXPLAIN ANALYZE of a query-based COPY TO executes the source query, +-- but produces no COPY output: no file is written, and no data is sent +-- to the client. +explain (analyze, costs off, timing off, summary off, buffers off) + copy (select * from explain_copy_tbl where a > 0) to stdout; +explain (analyze, costs off, timing off, summary off, buffers off) + copy (select a from explain_copy_tbl) to '/no/such/dir/file'; + -- invalid cases explain copy explain_copy_no_such_table from stdin; explain copy explain_copy_tbl from stdin with (format nosuch); explain (analyze) copy explain_copy_tbl from stdin; explain (analyze) copy explain_copy_tbl to stdout; -explain (analyze) copy (select 1) to stdout; -- Plain EXPLAIN performs the same permission checks as COPY would. create role regress_explain_copy; From e4c605b35094f9ce7bcb2d5dda86467c7d80595d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 01:33:30 +0000 Subject: [PATCH 9/9] Allow EXPLAIN ANALYZE on COPY FROM, reporting a timing breakdown With ANALYZE, COPY FROM is now executed (actually loading the data, like EXPLAIN ANALYZE INSERT) and EXPLAIN reports the row counts and a breakdown of where the time was spent: - Input Time: reading, parsing and converting the input data, measured around the CopyFromOneRow format callback so that custom COPY formats are covered as well - Insert Time: table_(multi_)insert or FDW inserts - Index Update Time: ExecInsertIndexTuples - per-trigger times, reusing the executor's trigger instrumentation by requesting es_instrument on the COPY's EState Buffer and WAL usage are reported from pgBufferUsage/pgWalUsage diffs. Rows excluded by the WHERE clause and rows skipped by ON_ERROR are also shown. The instrumentation is attached to the CopyFromState with the new CopyFromSetInstrumentation() function, keeping the BeginCopyFrom() API unchanged for external callers such as file_fdw. The per-phase timers are inline no-ops when no instrumentation is attached, so regular COPY pays only a branch per call site; with TIMING OFF only row counters are collected. EXPLAIN ANALYZE COPY FROM STDIN is rejected: the CopyInResponse protocol message would be sent while the client expects the result of the EXPLAIN statement. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EZujnAnhcNbVyWUn4dwJVu --- doc/src/sgml/ref/explain.sgml | 17 +- src/backend/commands/copyfrom.c | 101 ++++++++++ src/backend/commands/explain.c | 6 +- src/backend/commands/explain_copy.c | 224 ++++++++++++++++++++- src/include/commands/copy.h | 47 +++++ src/include/commands/copyfrom_internal.h | 35 ++++ src/include/commands/explain.h | 3 + src/test/regress/expected/explain_copy.out | 146 +++++++++++++- src/test/regress/sql/explain_copy.sql | 96 +++++++++ 9 files changed, 664 insertions(+), 11 deletions(-) diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index b02d41adc8d6a..04c05df1c290d 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -98,7 +98,7 @@ EXPLAIN [ ( option [, ...] ) ] EXPLAIN ANALYZE on an INSERT, UPDATE, DELETE, MERGE, - CREATE TABLE AS, + CREATE TABLE AS, COPY FROM, or EXECUTE statement without letting the command affect your data, use this approach: @@ -372,6 +372,21 @@ ROLLBACK; COPY output: no file is written, and no data is sent to the client. + + EXPLAIN ANALYZE of a COPY FROM + statement executes the COPY and actually loads the + data. In addition to the row counts, it reports a breakdown of the + time spent reading, parsing and converting the input data + (Input Time), inserting tuples into the table + (Insert Time), and updating indexes + (Index Update Time), as well as trigger execution + times. The time breakdown is omitted when TIMING + OFF is specified. Time spent elsewhere, for example + evaluating the WHERE clause or routing tuples to + partitions, is included only in the total execution time. + COPY FROM STDIN cannot be executed under + EXPLAIN ANALYZE. + diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 80a527ed4c642..76648560488d9 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -35,6 +35,7 @@ #include "commands/trigger.h" #include "executor/execPartition.h" #include "executor/executor.h" +#include "executor/instrument.h" #include "executor/nodeModifyTable.h" #include "executor/tuptable.h" #include "foreign/fdwapi.h" @@ -481,12 +482,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, TupleTableSlot **rslots; /* insert into foreign table: let the FDW do it */ + CopyFromInstrStartPhase(cstate); rslots = resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert(estate, resultRelInfo, &slots[sent], NULL, &inserted); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT); sent += size; @@ -553,12 +556,14 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, * context before calling it. */ oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); + CopyFromInstrStartPhase(cstate); table_multi_insert(resultRelInfo->ri_RelationDesc, slots, nused, mycid, ti_options, buffer->bistate); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT); MemoryContextSwitchTo(oldcontext); for (i = 0; i < nused; i++) @@ -572,10 +577,12 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, List *recheckIndexes; cstate->cur_lineno = buffer->linenos[i]; + CopyFromInstrStartPhase(cstate); recheckIndexes = ExecInsertIndexTuples(resultRelInfo, estate, 0, buffer->slots[i], NIL, NULL); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INDEX); ExecARInsertTriggers(estate, resultRelInfo, slots[i], recheckIndexes, cstate->transition_capture); @@ -774,6 +781,58 @@ CopyMultiInsertInfoStore(CopyMultiInsertInfo *miinfo, ResultRelInfo *rri, miinfo->bufferedBytes += tuplen; } +/* + * Attach EXPLAIN ANALYZE instrumentation to a COPY FROM operation. Must be + * called between BeginCopyFrom() and CopyFrom(). + */ +void +CopyFromSetInstrumentation(CopyFromState cstate, CopyFromInstrumentation *instr) +{ + cstate->instr = instr; +} + +/* + * Collect the per-trigger statistics of the given result relations into the + * EXPLAIN ANALYZE instrumentation, in the current memory context. This has + * to be done before the executor state of the COPY FROM operation is + * destroyed. + */ +static void +CopyFromCollectTriggerStats(CopyFromInstrumentation *instr, List *resultrels) +{ + ListCell *lc; + + foreach(lc, resultrels) + { + ResultRelInfo *rInfo = (ResultRelInfo *) lfirst(lc); + + if (!rInfo->ri_TrigDesc || !rInfo->ri_TrigInstrument) + continue; + + for (int nt = 0; nt < rInfo->ri_TrigDesc->numtriggers; nt++) + { + Trigger *trig = rInfo->ri_TrigDesc->triggers + nt; + TriggerInstrumentation *tginstr = rInfo->ri_TrigInstrument + nt; + CopyFromTriggerStats *stats; + + /* ignore triggers that were never invoked */ + if (tginstr->firings == 0) + continue; + + stats = palloc0_object(CopyFromTriggerStats); + stats->trigger_name = pstrdup(trig->tgname); + if (OidIsValid(trig->tgconstraint)) + stats->constraint_name = get_constraint_name(trig->tgconstraint); + stats->relation_name = + pstrdup(RelationGetRelationName(rInfo->ri_RelationDesc)); + stats->firings = tginstr->firings; + stats->total = tginstr->instr.total; + + instr->triggers = lappend(instr->triggers, stats); + } + } +} + /* * Copy FROM file to relation. */ @@ -910,6 +969,16 @@ CopyFrom(CopyFromState cstate) ti_options |= TABLE_INSERT_FROZEN; } + /* + * If EXPLAIN ANALYZE instrumentation is attached, request trigger + * instrumentation; InitResultRelInfo then sets up ri_TrigInstrument for + * the target relation as well as for any partitions we route tuples to, + * and trigger.c collects the statistics. + */ + if (cstate->instr) + estate->es_instrument = cstate->instr->collect_timing ? + INSTRUMENT_TIMER : INSTRUMENT_ROWS; + /* * We need a ResultRelInfo so we can use the regular executor's * index-entry-making machinery. (There used to be a huge amount of code @@ -1148,8 +1217,13 @@ CopyFrom(CopyFromState cstate) ExecClearTuple(myslot); /* Directly store the values/nulls array in the slot */ + CopyFromInstrStartPhase(cstate); if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull)) + { + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INPUT); break; + } + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INPUT); if (cstate->opts.on_error == COPY_ON_ERROR_IGNORE && cstate->escontext->error_occurred) @@ -1408,10 +1482,12 @@ CopyFrom(CopyFromState cstate) /* OK, store the tuple */ if (resultRelInfo->ri_FdwRoutine != NULL) { + CopyFromInstrStartPhase(cstate); myslot = resultRelInfo->ri_FdwRoutine->ExecForeignInsert(estate, resultRelInfo, myslot, NULL); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT); if (myslot == NULL) /* "do nothing" */ continue; /* next tuple please */ @@ -1426,14 +1502,20 @@ CopyFrom(CopyFromState cstate) else { /* OK, store the tuple and create index entries for it */ + CopyFromInstrStartPhase(cstate); table_tuple_insert(resultRelInfo->ri_RelationDesc, myslot, mycid, ti_options, bistate); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INSERT); if (resultRelInfo->ri_NumIndices > 0) + { + CopyFromInstrStartPhase(cstate); recheckIndexes = ExecInsertIndexTuples(resultRelInfo, estate, 0, myslot, NIL, NULL); + CopyFromInstrStopPhase(cstate, COPY_FROM_PHASE_INDEX); + } } /* AFTER ROW INSERT Triggers */ @@ -1493,6 +1575,25 @@ CopyFrom(CopyFromState cstate) /* Handle queued AFTER triggers */ AfterTriggerEndQuery(estate); + /* + * Hand the collected statistics over to the attached EXPLAIN ANALYZE + * instrumentation, if any, before the executor state goes away. + */ + if (cstate->instr) + { + CopyFromInstrumentation *ci = cstate->instr; + + ci->excluded = excluded; + ci->skipped = cstate->num_errors; + ci->show_relname = + (list_length(estate->es_opened_result_relations) > 1 || + estate->es_tuple_routing_result_relations != NIL || + estate->es_trig_target_relations != NIL); + CopyFromCollectTriggerStats(ci, estate->es_opened_result_relations); + CopyFromCollectTriggerStats(ci, estate->es_tuple_routing_result_relations); + CopyFromCollectTriggerStats(ci, estate->es_trig_target_relations); + } + ExecResetTupleTable(estate->es_tupleTable, false); /* Allow the FDW to shut down */ diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index cb824f41a5dda..e35504d948bb8 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -147,8 +147,6 @@ static void show_instrumentation_count(const char *qlabel, int which, static void show_foreignscan_info(ForeignScanState *fsstate, ExplainState *es); static const char *explain_get_index_name(Oid indexId); static bool peek_buffer_usage(ExplainState *es, const BufferUsage *usage); -static void show_buffer_usage(ExplainState *es, const BufferUsage *usage); -static void show_wal_usage(ExplainState *es, const WalUsage *usage); static void show_memory_counters(ExplainState *es, const MemoryContextCounters *mem_counters); static void show_result_replacement_info(Result *result, ExplainState *es); @@ -4295,7 +4293,7 @@ peek_buffer_usage(ExplainState *es, const BufferUsage *usage) /* * Show buffer usage details. This better be sync with peek_buffer_usage. */ -static void +void show_buffer_usage(ExplainState *es, const BufferUsage *usage) { if (es->format == EXPLAIN_FORMAT_TEXT) @@ -4464,7 +4462,7 @@ show_buffer_usage(ExplainState *es, const BufferUsage *usage) /* * Show WAL usage details. */ -static void +void show_wal_usage(ExplainState *es, const WalUsage *usage) { if (es->format == EXPLAIN_FORMAT_TEXT) diff --git a/src/backend/commands/explain_copy.c b/src/backend/commands/explain_copy.c index dbdabf7a2ada0..2f2c3a87d4cc7 100644 --- a/src/backend/commands/explain_copy.c +++ b/src/backend/commands/explain_copy.c @@ -14,11 +14,13 @@ #include "postgres.h" #include "access/table.h" +#include "access/xact.h" #include "commands/copy.h" #include "commands/explain.h" #include "commands/explain_format.h" #include "commands/explain_state.h" #include "executor/instrument.h" +#include "miscadmin.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" @@ -29,12 +31,18 @@ static void ExplainCopyToQuery(const CopyStmt *stmt, RawStmt *raw_query, Oid queryRelId, ExplainState *es, ParseState *pstate, ParamListInfo params); +static void ExplainCopyFromExec(const CopyStmt *stmt, Relation rel, + Node *whereClause, + const CopyFormatOptions *opts, + ExplainState *es, ParseState *pstate); static void ExplainCopyGeneric(const CopyStmt *stmt, Relation rel, const CopyFormatOptions *opts, ExplainState *es); static void show_copy_properties(const CopyStmt *stmt, const CopyFormatOptions *opts, ExplainState *es); +static void show_copy_trigger_stats(const CopyFromInstrumentation *ci, + ExplainState *es); /* * ExplainCopyStmt - @@ -67,11 +75,9 @@ ExplainCopyStmt(CopyStmt *stmt, ExplainState *es, if (stmt->is_from) { if (es->analyze) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("EXPLAIN ANALYZE is not supported for COPY FROM"))); - - ExplainCopyGeneric(stmt, rel, &opts, es); + ExplainCopyFromExec(stmt, rel, whereClause, &opts, es, pstate); + else + ExplainCopyGeneric(stmt, rel, &opts, es); } else if (query != NULL) { @@ -181,6 +187,149 @@ ExplainCopyToQuery(const CopyStmt *stmt, RawStmt *raw_query, Oid queryRelId, stmt); } +/* + * ExplainCopyFromExec - + * execute a COPY FROM statement under EXPLAIN ANALYZE and print the + * collected statistics, including the per-phase timing breakdown + */ +static void +ExplainCopyFromExec(const CopyStmt *stmt, Relation rel, Node *whereClause, + const CopyFormatOptions *opts, ExplainState *es, + ParseState *pstate) +{ + CopyFromInstrumentation ci = {0}; + CopyFromState cstate; + BufferUsage bufusage_start, + bufusage; + WalUsage walusage_start, + walusage; + instr_time starttime, + totaltime; + uint64 processed; + + /* + * COPY FROM STDIN cannot be executed under EXPLAIN: the CopyInResponse + * protocol message would be sent while the client expects the result of + * the EXPLAIN statement. + */ + if (stmt->filename == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("EXPLAIN ANALYZE cannot be used with COPY FROM STDIN"), + errhint("Use COPY FROM a file or PROGRAM."))); + + /* check read-only transaction, as the execution of COPY FROM would */ + if (XactReadOnly && !rel->rd_islocaltemp) + PreventCommandIfReadOnly("COPY FROM"); + + ci.collect_timing = es->timing; + + if (es->buffers) + bufusage_start = pgBufferUsage; + if (es->wal) + walusage_start = pgWalUsage; + INSTR_TIME_SET_CURRENT(starttime); + + /* run the COPY as DoCopy would, with the instrumentation attached */ + cstate = BeginCopyFrom(pstate, rel, whereClause, stmt->filename, + stmt->is_program, NULL, stmt->attlist, + stmt->options); + CopyFromSetInstrumentation(cstate, &ci); + processed = CopyFrom(cstate); + EndCopyFrom(cstate); + + /* as in ExplainOnePlan, in case this is used in a multi-command string */ + CommandCounterIncrement(); + + INSTR_TIME_SET_CURRENT(totaltime); + INSTR_TIME_SUBTRACT(totaltime, starttime); + + /* calc differences of buffer and WAL counters */ + if (es->buffers) + { + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); + } + if (es->wal) + { + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &walusage_start); + } + + /* produce the output */ + ExplainOpenGroup("Query", NULL, true, es); + ExplainOpenGroup("Copy From", "Copy From", true, es); + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + ExplainIndentText(es); + if (es->verbose) + appendStringInfo(es->str, "Copy From on %s.%s", + quote_identifier(get_namespace_name(RelationGetNamespace(rel))), + quote_identifier(RelationGetRelationName(rel))); + else + appendStringInfo(es->str, "Copy From on %s", + quote_identifier(RelationGetRelationName(rel))); + appendStringInfo(es->str, " (actual rows=%" PRIu64 ")\n", processed); + es->indent++; + } + else + { + ExplainPropertyText("Relation Name", + RelationGetRelationName(rel), es); + if (es->verbose) + ExplainPropertyText("Schema", + get_namespace_name(RelationGetNamespace(rel)), + es); + ExplainPropertyUInteger("Actual Rows", NULL, processed, es); + } + + show_copy_properties(stmt, opts, es); + + /* the per-phase timing breakdown */ + if (es->timing) + { + ExplainPropertyFloat("Input Time", "ms", + INSTR_TIME_GET_MILLISEC(ci.phase_time[COPY_FROM_PHASE_INPUT]), + 3, es); + ExplainPropertyFloat("Insert Time", "ms", + INSTR_TIME_GET_MILLISEC(ci.phase_time[COPY_FROM_PHASE_INSERT]), + 3, es); + ExplainPropertyFloat("Index Update Time", "ms", + INSTR_TIME_GET_MILLISEC(ci.phase_time[COPY_FROM_PHASE_INDEX]), + 3, es); + } + + if (whereClause != NULL) + ExplainPropertyUInteger("Rows Excluded by Filter", NULL, + ci.excluded, es); + if (opts->on_error != COPY_ON_ERROR_STOP) + ExplainPropertyUInteger("Rows Skipped", NULL, ci.skipped, es); + + if (es->buffers) + show_buffer_usage(es, &bufusage); + if (es->wal) + show_wal_usage(es, &walusage); + + if (es->format == EXPLAIN_FORMAT_TEXT) + es->indent--; + + ExplainCloseGroup("Copy From", "Copy From", true, es); + + /* Print info about runtime of triggers */ + show_copy_trigger_stats(&ci, es); + + /* + * As in ExplainOnePlan, total execution time is only reported when + * summary reporting is enabled. + */ + if (es->summary) + ExplainPropertyFloat("Execution Time", "ms", + INSTR_TIME_GET_MILLISEC(totaltime), 3, es); + + ExplainCloseGroup("Query", NULL, true, es); +} + /* * ExplainCopyGeneric - * print the description of a COPY statement that has no plan to show @@ -318,3 +467,68 @@ show_copy_properties(const CopyStmt *stmt, const CopyFormatOptions *opts, opts->on_error == COPY_ON_ERROR_IGNORE ? "ignore" : "set_null", es); } + +/* + * show_copy_trigger_stats - + * print the per-trigger statistics collected by CopyFrom + * + * This mirrors the output of report_triggers() in explain.c, which cannot + * be used directly because the executor state of the COPY has already been + * destroyed by the time we get here. + */ +static void +show_copy_trigger_stats(const CopyFromInstrumentation *ci, ExplainState *es) +{ + ListCell *lc; + + ExplainOpenGroup("Triggers", "Triggers", false, es); + + foreach(lc, ci->triggers) + { + CopyFromTriggerStats *stats = (CopyFromTriggerStats *) lfirst(lc); + + ExplainOpenGroup("Trigger", NULL, true, es); + + /* + * In text format, we avoid printing both the trigger name and the + * constraint name unless VERBOSE is specified. In non-text formats + * we just print everything. + */ + if (es->format == EXPLAIN_FORMAT_TEXT) + { + if (es->verbose || stats->constraint_name == NULL) + appendStringInfo(es->str, "Trigger %s", stats->trigger_name); + else + appendStringInfoString(es->str, "Trigger"); + if (stats->constraint_name) + appendStringInfo(es->str, " for constraint %s", + stats->constraint_name); + if (ci->show_relname) + appendStringInfo(es->str, " on %s", stats->relation_name); + if (es->timing) + appendStringInfo(es->str, ": time=%.3f calls=%" PRId64 "\n", + INSTR_TIME_GET_MILLISEC(stats->total), + stats->firings); + else + appendStringInfo(es->str, ": calls=%" PRId64 "\n", + stats->firings); + } + else + { + ExplainPropertyText("Trigger Name", stats->trigger_name, es); + if (stats->constraint_name) + ExplainPropertyText("Constraint Name", stats->constraint_name, + es); + ExplainPropertyText("Relation", stats->relation_name, es); + if (es->timing) + ExplainPropertyFloat("Time", "ms", + INSTR_TIME_GET_MILLISEC(stats->total), 3, + es); + ExplainPropertyInteger("Calls", NULL, stats->firings, es); + } + + ExplainCloseGroup("Trigger", NULL, true, es); + } + + ExplainCloseGroup("Triggers", "Triggers", false, es); +} diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index e34350a86f2af..f7b5c4f1ba2f2 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -17,6 +17,7 @@ #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" +#include "portability/instr_time.h" #include "tcop/dest.h" /* @@ -99,6 +100,50 @@ typedef struct CopyFormatOptions List *convert_select; /* list of column names (can be NIL) */ } CopyFormatOptions; +/* + * Execution phases of COPY FROM distinguished by EXPLAIN ANALYZE. + */ +typedef enum CopyFromPhase +{ + COPY_FROM_PHASE_INPUT, /* reading, parsing and converting the input + * data, including the evaluation of default + * expressions */ + COPY_FROM_PHASE_INSERT, /* inserting tuples into the target table */ + COPY_FROM_PHASE_INDEX, /* updating indexes of the target table */ +} CopyFromPhase; + +#define COPY_FROM_NUM_PHASES (COPY_FROM_PHASE_INDEX + 1) + +/* Per-trigger statistics collected for EXPLAIN ANALYZE COPY FROM */ +typedef struct CopyFromTriggerStats +{ + char *trigger_name; + char *constraint_name; /* NULL if not a constraint trigger */ + char *relation_name; + int64 firings; /* number of times the trigger was fired */ + instr_time total; /* total time spent in the trigger */ +} CopyFromTriggerStats; + +/* + * Instrumentation collected while executing COPY FROM under EXPLAIN + * ANALYZE. Attach to a CopyFromState with CopyFromSetInstrumentation() + * between BeginCopyFrom() and CopyFrom(). + */ +typedef struct CopyFromInstrumentation +{ + bool collect_timing; /* collect per-phase timing? */ + + /* Updated only when collect_timing; see copyfrom_internal.h */ + instr_time phase_start; /* start of the currently running phase */ + instr_time phase_time[COPY_FROM_NUM_PHASES]; /* accumulated time */ + + /* Filled in at the end of CopyFrom() */ + uint64 excluded; /* rows excluded by the WHERE clause */ + uint64 skipped; /* rows skipped because of ON_ERROR */ + List *triggers; /* list of CopyFromTriggerStats */ + bool show_relname; /* qualify trigger names with the relation? */ +} CopyFromInstrumentation; + /* These are private in commands/copy[from|to].c */ typedef struct CopyFromStateData *CopyFromState; typedef struct CopyToStateData *CopyToState; @@ -127,6 +172,8 @@ extern void CopyFromErrorCallback(void *arg); extern char *CopyLimitPrintoutLength(const char *str); extern uint64 CopyFrom(CopyFromState cstate); +extern void CopyFromSetInstrumentation(CopyFromState cstate, + CopyFromInstrumentation *instr); extern DestReceiver *CreateCopyDestReceiver(void); diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 9d3e244ee550c..0ee0956bd046c 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -189,8 +189,43 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* EXPLAIN ANALYZE instrumentation, or NULL */ + CopyFromInstrumentation *instr; } CopyFromStateData; +/* + * Start and stop helpers for the per-phase timing of EXPLAIN ANALYZE COPY + * FROM. These are no-ops when no instrumentation is attached or timing is + * not requested, so calling them costs regular COPY only a branch. + * + * Phases never nest; the single shared start-time field both exploits and, + * with assertions enabled, verifies this. + */ +static inline void +CopyFromInstrStartPhase(CopyFromState cstate) +{ + CopyFromInstrumentation *ci = cstate->instr; + + if (ci == NULL || !ci->collect_timing) + return; + Assert(INSTR_TIME_IS_ZERO(ci->phase_start)); + INSTR_TIME_SET_CURRENT(ci->phase_start); +} + +static inline void +CopyFromInstrStopPhase(CopyFromState cstate, CopyFromPhase phase) +{ + CopyFromInstrumentation *ci = cstate->instr; + instr_time now; + + if (ci == NULL || !ci->collect_timing) + return; + INSTR_TIME_SET_CURRENT(now); + INSTR_TIME_ACCUM_DIFF(ci->phase_time[phase], now, ci->phase_start); + INSTR_TIME_SET_ZERO(ci->phase_start); +} + extern void ReceiveCopyBegin(CopyFromState cstate); extern void ReceiveCopyBinaryHeader(CopyFromState cstate); diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 526e0cea6eefd..0e0462959a41c 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -88,4 +88,7 @@ extern void ExplainQueryText(ExplainState *es, QueryDesc *queryDesc); extern void ExplainQueryParameters(ExplainState *es, ParamListInfo params, int maxlen); +extern void show_buffer_usage(ExplainState *es, const BufferUsage *usage); +extern void show_wal_usage(ExplainState *es, const WalUsage *usage); + #endif /* EXPLAIN_H */ diff --git a/src/test/regress/expected/explain_copy.out b/src/test/regress/expected/explain_copy.out index d9d4b961b9e72..6b29649baa2d2 100644 --- a/src/test/regress/expected/explain_copy.out +++ b/src/test/regress/expected/explain_copy.out @@ -114,7 +114,8 @@ ERROR: COPY format "nosuch" not recognized LINE 1: explain copy explain_copy_tbl from stdin with (format nosuch... ^ explain (analyze) copy explain_copy_tbl from stdin; -ERROR: EXPLAIN ANALYZE is not supported for COPY FROM +ERROR: EXPLAIN ANALYZE cannot be used with COPY FROM STDIN +HINT: Use COPY FROM a file or PROGRAM. explain (analyze) copy explain_copy_tbl to stdout; ERROR: EXPLAIN ANALYZE is not supported for COPY relation TO HINT: Try the COPY (SELECT ...) TO variant. @@ -158,4 +159,147 @@ revoke all on explain_copy_tbl from regress_explain_copy; revoke all on explain_copy_rls from regress_explain_copy; drop role regress_explain_copy; drop table explain_copy_rls; +-- +-- EXPLAIN ANALYZE COPY FROM executes the COPY and reports a breakdown of +-- the execution. (The timing values are not stable, so the breakdown +-- itself is exercised with TIMING OFF, which omits it.) +-- +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/explain_copy.data' +copy explain_copy_tbl to :'filename'; +create table explain_copy_target (a int, b text); +create index explain_copy_target_idx on explain_copy_target (a); +-- The File property of the output contains the absolute build directory +-- path, so filter it out. +create function explain_copy_filter(cmd text) returns setof text +language plpgsql as +$$ +declare + ln text; +begin + for ln in execute cmd loop + ln := regexp_replace(ln, '(File|Program): .*', '\1: ...'); + return next ln; + end loop; +end; +$$; +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'filename')); + explain_copy_filter +-------------------------------------------------- + Copy From on explain_copy_target (actual rows=2) + Format: text + Source: file + File: ... +(4 rows) + +-- the data is actually loaded ... +select count(*) from explain_copy_target; + count +------- + 2 +(1 row) + +rollback; +-- ... and was rolled back with the transaction +select count(*) from explain_copy_target; + count +------- + 0 +(1 row) + +-- rows excluded by the WHERE clause are reported +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'filename') + || ' where a > 1'); + explain_copy_filter +-------------------------------------------------- + Copy From on explain_copy_target (actual rows=1) + Format: text + Source: file + File: ... + Rows Excluded by Filter: 1 +(5 rows) + +rollback; +-- trigger statistics are reported +create function explain_copy_trigf() returns trigger language plpgsql + as $$ begin return new; end $$; +create trigger explain_copy_trig before insert on explain_copy_target + for each row execute function explain_copy_trigf(); +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'filename')); + explain_copy_filter +-------------------------------------------------- + Copy From on explain_copy_target (actual rows=2) + Format: text + Source: file + File: ... + Trigger explain_copy_trig: calls=2 +(5 rows) + +rollback; +drop trigger explain_copy_trig on explain_copy_target; +drop function explain_copy_trigf(); +-- rows skipped because of ON_ERROR are reported +create table explain_copy_texts (a text, b text); +insert into explain_copy_texts values ('1', 'one'), ('bogus', 'two'), ('3', 'three'); +\set badfile :abs_builddir '/results/explain_copy_bad.data' +copy explain_copy_texts to :'badfile'; +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'badfile') + || ' with (on_error ignore)'); +NOTICE: 1 row was skipped due to data type incompatibility + explain_copy_filter +-------------------------------------------------- + Copy From on explain_copy_target (actual rows=2) + Format: text + Source: file + File: ... + On Error: ignore + Rows Skipped: 1 +(6 rows) + +rollback; +drop table explain_copy_texts; +-- a partitioned table can be the target +create table explain_copy_part (a int, b text) partition by range (a); +create table explain_copy_part1 partition of explain_copy_part + for values from (0) to (2); +create table explain_copy_part2 partition of explain_copy_part + for values from (2) to (100); +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_part from ' || quote_literal(:'filename')); + explain_copy_filter +------------------------------------------------ + Copy From on explain_copy_part (actual rows=2) + Format: text + Source: file + File: ... +(4 rows) + +rollback; +drop table explain_copy_part; +-- ANALYZE cannot be used with COPY FROM STDIN +explain (analyze) copy explain_copy_target from stdin; +ERROR: EXPLAIN ANALYZE cannot be used with COPY FROM STDIN +HINT: Use COPY FROM a file or PROGRAM. +-- nor in a read-only transaction +begin transaction read only; +explain (analyze, timing off, summary off, buffers off) + copy explain_copy_target from :'filename'; +ERROR: cannot execute COPY FROM in a read-only transaction +rollback; +drop function explain_copy_filter(text); +drop table explain_copy_target; drop table explain_copy_tbl; diff --git a/src/test/regress/sql/explain_copy.sql b/src/test/regress/sql/explain_copy.sql index 4c0ee22a6c25c..b027f7b1523ca 100644 --- a/src/test/regress/sql/explain_copy.sql +++ b/src/test/regress/sql/explain_copy.sql @@ -53,4 +53,100 @@ revoke all on explain_copy_tbl from regress_explain_copy; revoke all on explain_copy_rls from regress_explain_copy; drop role regress_explain_copy; drop table explain_copy_rls; + +-- +-- EXPLAIN ANALYZE COPY FROM executes the COPY and reports a breakdown of +-- the execution. (The timing values are not stable, so the breakdown +-- itself is exercised with TIMING OFF, which omits it.) +-- +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/explain_copy.data' +copy explain_copy_tbl to :'filename'; + +create table explain_copy_target (a int, b text); +create index explain_copy_target_idx on explain_copy_target (a); + +-- The File property of the output contains the absolute build directory +-- path, so filter it out. +create function explain_copy_filter(cmd text) returns setof text +language plpgsql as +$$ +declare + ln text; +begin + for ln in execute cmd loop + ln := regexp_replace(ln, '(File|Program): .*', '\1: ...'); + return next ln; + end loop; +end; +$$; + +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'filename')); +-- the data is actually loaded ... +select count(*) from explain_copy_target; +rollback; +-- ... and was rolled back with the transaction +select count(*) from explain_copy_target; + +-- rows excluded by the WHERE clause are reported +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'filename') + || ' where a > 1'); +rollback; + +-- trigger statistics are reported +create function explain_copy_trigf() returns trigger language plpgsql + as $$ begin return new; end $$; +create trigger explain_copy_trig before insert on explain_copy_target + for each row execute function explain_copy_trigf(); +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'filename')); +rollback; +drop trigger explain_copy_trig on explain_copy_target; +drop function explain_copy_trigf(); + +-- rows skipped because of ON_ERROR are reported +create table explain_copy_texts (a text, b text); +insert into explain_copy_texts values ('1', 'one'), ('bogus', 'two'), ('3', 'three'); +\set badfile :abs_builddir '/results/explain_copy_bad.data' +copy explain_copy_texts to :'badfile'; +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_target from ' || quote_literal(:'badfile') + || ' with (on_error ignore)'); +rollback; +drop table explain_copy_texts; + +-- a partitioned table can be the target +create table explain_copy_part (a int, b text) partition by range (a); +create table explain_copy_part1 partition of explain_copy_part + for values from (0) to (2); +create table explain_copy_part2 partition of explain_copy_part + for values from (2) to (100); +begin; +select explain_copy_filter( + 'explain (analyze, timing off, summary off, buffers off) ' + 'copy explain_copy_part from ' || quote_literal(:'filename')); +rollback; +drop table explain_copy_part; + +-- ANALYZE cannot be used with COPY FROM STDIN +explain (analyze) copy explain_copy_target from stdin; + +-- nor in a read-only transaction +begin transaction read only; +explain (analyze, timing off, summary off, buffers off) + copy explain_copy_target from :'filename'; +rollback; + +drop function explain_copy_filter(text); +drop table explain_copy_target; drop table explain_copy_tbl;