Skip to content

Commit

Permalink
mvcc: Export a "Last DB compaction" timestamp metric (etcd-io#12176)
Browse files Browse the repository at this point in the history
This is to aid with debugging the effectiveness of systems that
manually take care of cluster compaction, and have greater visibity
into recent compactions.

It can be handy to alert on the exactly how long it was since a
compaction (and also to put on dashboards) had happened.

---

Tested using a test cluster, the final result looks like this:

```
	root@etcd-1:~# ETCDCTL_API=3 /tmp/test-etcd/etcdctl --endpoints=192.168.232.10:2379 compact 1012
	compacted revision 1012

	root@etcd-1:~# curl -s 192.168.232.10:2379/metrics | grep last
	# HELP etcd_debugging_mvcc_db_compaction_last The unix time since the last db compaction.  Resets to 0 on start.
	# TYPE etcd_debugging_mvcc_db_compaction_last gauge
	etcd_debugging_mvcc_db_compaction_last 1.595873939e+09
```
  • Loading branch information
benjojo authored and YoSmudge committed Dec 19, 2024
1 parent f893034 commit 79f8cf9
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 0 deletions.
1 change: 1 addition & 0 deletions mvcc/kvstore_compaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc
defer func() { dbCompactionTotalMs.Observe(float64(time.Since(totalStart) / time.Millisecond)) }()
keyCompactions := 0
defer func() { dbCompactionKeysCounter.Add(float64(keyCompactions)) }()
defer func() { dbCompactionLast.Set(float64(time.Now().Unix())) }()

end := make([]byte, 8)
binary.BigEndian.PutUint64(end, uint64(compactMainRev+1))
Expand Down
9 changes: 9 additions & 0 deletions mvcc/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,14 @@ var (
Buckets: prometheus.ExponentialBuckets(100, 2, 14),
})

dbCompactionLast = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "etcd_debugging",
Subsystem: "mvcc",
Name: "db_compaction_last",
Help: "The unix time of the last db compaction. Resets to 0 on start.",
})

dbCompactionKeysCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "etcd_debugging",
Expand Down Expand Up @@ -324,6 +332,7 @@ func init() {
prometheus.MustRegister(indexCompactionPauseMs)
prometheus.MustRegister(dbCompactionPauseMs)
prometheus.MustRegister(dbCompactionTotalMs)
prometheus.MustRegister(dbCompactionLast)
prometheus.MustRegister(dbCompactionKeysCounter)
prometheus.MustRegister(dbTotalSize)
prometheus.MustRegister(dbTotalSizeDebug)
Expand Down

0 comments on commit 79f8cf9

Please # to comment.