diff --git a/cmd/decommetric_string.go b/cmd/decommetric_string.go new file mode 100644 index 000000000..fb485ab01 --- /dev/null +++ b/cmd/decommetric_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=decomMetric -trimprefix=decomMetric erasure-server-pool-decom.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[decomMetricDecommissionBucket-0] + _ = x[decomMetricDecommissionObject-1] + _ = x[decomMetricDecommissionRemoveObject-2] +} + +const _decomMetric_name = "DecommissionBucketDecommissionObjectDecommissionRemoveObject" + +var _decomMetric_index = [...]uint8{0, 18, 36, 60} + +func (i decomMetric) String() string { + if i >= decomMetric(len(_decomMetric_index)-1) { + return "decomMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _decomMetric_name[_decomMetric_index[i]:_decomMetric_index[i+1]] +} diff --git a/cmd/erasure-server-pool-decom.go b/cmd/erasure-server-pool-decom.go index d05bcfd5a..3e2dc479b 100644 --- a/cmd/erasure-server-pool-decom.go +++ b/cmd/erasure-server-pool-decom.go @@ -800,11 +800,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool logger.LogIf(ctx, err) continue } + stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID) if err = z.decommissionObject(ctx, bi.Name, gr); err != nil { + stopFn(err) failure = true logger.LogIf(ctx, err) continue } + stopFn(nil) failure = false break } @@ -819,6 +822,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool // if all versions were decommissioned, then we can delete the object versions. if decommissionedCount == len(fivs.Versions) { + stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name) _, err := set.DeleteObject(ctx, bi.Name, encodeDirObject(entry.name), @@ -826,6 +830,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool DeletePrefix: true, // use prefix delete to delete all versions at once. }, ) + stopFn(err) auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err) if err != nil { logger.LogIf(ctx, err) @@ -881,6 +886,47 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool return nil } +//msgp:ignore decomMetrics +type decomMetrics struct{} + +var globalDecommissionMetrics decomMetrics + +//msgp:ignore decomMetric +//go:generate stringer -type=decomMetric -trimprefix=decomMetric $GOFILE +type decomMetric uint8 + +const ( + decomMetricDecommissionBucket decomMetric = iota + decomMetricDecommissionObject + decomMetricDecommissionRemoveObject +) + +func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + return madmin.TraceInfo{ + TraceType: madmin.TraceDecommission, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: fmt.Sprintf("decommission.%s (pool-id=%d)", d.String(), poolIdx), + Duration: duration, + Path: path, + Error: errStr, + } +} + +func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err error) { + startTime := time.Now() + return func(err error) { + duration := time.Since(startTime) + if globalTrace.NumSubscribers(madmin.TraceDecommission) > 0 { + globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err)) + } + } +} + func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error { pool := z.serverPools[idx] for _, bucket := range z.poolMeta.PendingBuckets(idx) { @@ -898,9 +944,13 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i if serverDebugLog { console.Debugln("decommission: currently on bucket", bucket.Name) } + stopFn := globalDecommissionMetrics.log(decomMetricDecommissionBucket, idx, bucket.Name) if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil { + stopFn(err) return err } + stopFn(nil) + z.poolMetaMutex.Lock() z.poolMeta.BucketDone(idx, bucket) z.poolMeta.save(ctx, z.serverPools) diff --git a/go.mod b/go.mod index bd9d60f94..fb78324e5 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,6 @@ require ( github.com/rs/cors v1.7.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/secure-io/sio-go v0.3.1 - github.com/shirou/gopsutil v3.21.11+incompatible github.com/shirou/gopsutil/v3 v3.22.6 github.com/streadway/amqp v1.0.0 github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e diff --git a/go.sum b/go.sum index 8714791e2..3e704116b 100644 --- a/go.sum +++ b/go.sum @@ -804,8 +804,6 @@ github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88= github.com/shirou/gopsutil/v3 v3.22.6 h1:FnHOFOh+cYAM0C30P+zysPISzlknLC5Z1G4EAElznfQ= github.com/shirou/gopsutil/v3 v3.22.6/go.mod h1:EdIubSnZhbAvBS1yJ7Xi+AShB/hxwLHOMz4MCYz7yMs=