Add metrics for measure delete progress

README.md

@@ -66,6 +66,13 @@ The output consists of 2 categories of metrics:
 |             | `write_req_errors_total` | counter | The total number of errors occurred while processing write requests |
 | Internal    | `request_latency_bucket` | histogram | The number of requests and requests time for each source request |
 |             | `request_size_bucket` | histogram | The number of requests and requests size for each source request |
+| Delete / garbage collection | `delete_process_items` | gauge | Size of the current batch of items found to process by delete handlers |
+|             | `delete_process_remaining` | gauge | Items left to process by delete handlers |
+|             | `delete_process_processed_total` | counter | Cumulative number of items delete handlers attempted to delete or move |
+|             | `delete_process_deleted_total` | counter | Cumulative number of items successfully deleted or moved |
+|             | `delete_process_kept_total` | counter | Cumulative number of items intentionally left in place (skipped, e.g. by the protection window/trash filter, or failed after retries) |
+|             | `delete_request_latency_seconds_bucket` | histogram | Latency of delete-handler list/delete operations |
+|             | `delete_request_size_bucket` | histogram | Number of items returned by a delete-handler list operation |
 
 Internal metrics show latency and size for each source request. Source request is the internal type of query from GP to yproxy. Yproxy performs mostly upload/download requests to S3. But yezzey perform requests multiple species. They are named as source request. We measure source requests:
 ```
@@ -93,6 +100,21 @@ COLLECT OBSOLETE
 DELETE OBSOLETE
 ```
 
+Delete/garbage-collection metrics are labeled per `bucket` (since yproxy can
+manage multiple storage buckets) and per `operation`, one of:
+```
+DELETE_GARBAGE  # DeleteGarbageInBucket
+DELETE_PREFIX   # DeletePrefixInBucket
+```
+`delete_request_latency_seconds` is additionally labeled by `stage`
+(`list` or `delete`), distinguishing the time spent listing objects from the
+time spent on each individual delete/move call.
+
+`HandleUntrashifyFile` restores files from trash rather than deleting them,
+so it does not report these metrics - it still logs an Info-level line when
+it starts and finishes, plus a periodic Info-level progress line every
+50000 processed items.
+
 <details>
   <summary>Example of gathered metrics for a simple yezzey query:</summary>
 

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/jarcoal/httpmock v1.3.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/rs/zerolog v1.31.0
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.11.1
@@ -37,9 +38,9 @@ require (
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect

pkg/metrics/delete_metrics.go

@@ -0,0 +1,103 @@
+package metrics
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// ProgressLogInterval is how often (in cumulative processed items) callers
+// should emit an Info-level progress log line, to keep log volume bounded
+// during long-running delete/untrashify operations.
+const ProgressLogInterval = 50000
+
+var (
+	itemCountBuckets = []float64{1, 10, 50, 100, 500, 1000, 5000, 10000, 50000}
+
+	DeleteProcessTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "delete_process_items",
+		Help: "Size of the current batch of items found to process by delete/untrashify handlers",
+	}, []string{"bucket", "operation"})
+
+	DeleteProcessRemaining = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "delete_process_remaining",
+		Help: "Items left to process by delete/untrashify handlers",
+	}, []string{"bucket", "operation"})
+
+	DeleteProcessProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "delete_process_processed_total",
+		Help: "Cumulative number of items delete/untrashify handlers attempted to delete or move",
+	}, []string{"bucket", "operation"})
+
+	DeleteProcessDeleted = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "delete_process_deleted_total",
+		Help: "Cumulative number of items successfully deleted or moved",
+	}, []string{"bucket", "operation"})
+
+	DeleteProcessKept = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "delete_process_kept_total",
+		Help: "Cumulative number of items intentionally left in place (skipped or failed after retries)",
+	}, []string{"bucket", "operation"})
+
+	DeleteRequestLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "delete_request_latency_seconds",
+		Help:    "Latency of delete-handler list/delete operations",
+		Buckets: latencyBuckets,
+	}, []string{"bucket", "operation", "stage"})
+
+	DeleteRequestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "delete_request_size",
+		Help:    "Number of items returned by a delete-handler list operation",
+		Buckets: itemCountBuckets,
+	}, []string{"bucket", "operation"})
+)
+
+// DeleteOpTracker reports per-bucket, per-operation progress metrics for the
+// long-running delete/untrashify handlers in pkg/proc/delete_handler.go.
+type DeleteOpTracker struct {
+	bucket    string
+	operation string
+	processed atomic.Int64
+}
+
+func NewDeleteOpTracker(bucket, operation string) *DeleteOpTracker {
+	return &DeleteOpTracker{bucket: bucket, operation: operation}
+}
+
+func (t *DeleteOpTracker) labels() prometheus.Labels {
+	return prometheus.Labels{"bucket": t.bucket, "operation": t.operation}
+}
+
+func (t *DeleteOpTracker) SetTotal(n int) {
+	DeleteProcessTotal.With(t.labels()).Set(float64(n))
+}
+
+func (t *DeleteOpTracker) SetRemaining(n int) {
+	DeleteProcessRemaining.With(t.labels()).Set(float64(n))
+}
+
+func (t *DeleteOpTracker) ObserveList(d time.Duration, itemCount int) {
+	DeleteRequestLatency.With(prometheus.Labels{"bucket": t.bucket, "operation": t.operation, "stage": "list"}).Observe(d.Seconds())
+	DeleteRequestSize.With(t.labels()).Observe(float64(itemCount))
+}
+
+func (t *DeleteOpTracker) ObserveDelete(d time.Duration) {
+	DeleteRequestLatency.With(prometheus.Labels{"bucket": t.bucket, "operation": t.operation, "stage": "delete"}).Observe(d.Seconds())
+}
+
+// AddProcessed records n more processed items and returns the cumulative
+// processed count for this tracker, for use with ProgressLogInterval.
+func (t *DeleteOpTracker) AddProcessed(n int) int64 {
+	DeleteProcessProcessed.With(t.labels()).Add(float64(n))
+	return t.processed.Add(int64(n))
+}
+
+func (t *DeleteOpTracker) AddDeleted(n int) {
+	DeleteProcessDeleted.With(t.labels()).Add(float64(n))
+}
+
+func (t *DeleteOpTracker) AddKept(n int) {
+	DeleteProcessKept.With(t.labels()).Add(float64(n))
+}