From 0767980550eec7c220ab3cf408dd46504cb1541e Mon Sep 17 00:00:00 2001 From: Hridoy Roy Date: Fri, 13 Nov 2020 10:26:58 -0800 Subject: [PATCH] Port: Telemetry For Lease Expiration Times (#10375) * port lease metrics * go mod vendor * caught a bug --- command/server/config_test.go | 4 + command/server/config_test_helpers.go | 154 ++++++++++++--- command/server/test-fixtures/config5.hcl | 51 +++++ helper/metricsutil/bucket.go | 19 ++ helper/metricsutil/wrapped_metrics.go | 14 +- internalshared/configutil/config.go | 3 + internalshared/configutil/telemetry.go | 44 ++++- vault/core_metrics.go | 17 ++ vault/expiration.go | 85 ++++++++ vault/expiration_test.go | 182 ++++++++++++++++++ .../github.com/hashicorp/vault/api/client.go | 110 ++++++++--- 11 files changed, 624 insertions(+), 59 deletions(-) create mode 100644 command/server/test-fixtures/config5.hcl diff --git a/command/server/config_test.go b/command/server/config_test.go index 46075e5346..779518766e 100644 --- a/command/server/config_test.go +++ b/command/server/config_test.go @@ -30,6 +30,10 @@ func TestLoadConfigFileIntegerAndBooleanValuesJson(t *testing.T) { testLoadConfigFileIntegerAndBooleanValuesJson(t) } +func TestLoadConfigFileWithLeaseMetricTelemetry(t *testing.T) { + testLoadConfigFileLeaseMetrics(t) +} + func TestLoadConfigDir(t *testing.T) { testLoadConfigDir(t) } diff --git a/command/server/config_test_helpers.go b/command/server/config_test_helpers.go index 26049912cd..c03b53f9b8 100644 --- a/command/server/config_test_helpers.go +++ b/command/server/config_test_helpers.go @@ -62,14 +62,17 @@ func testLoadConfigFile_topLevel(t *testing.T, entropy *configutil.Entropy) { }, Telemetry: &configutil.Telemetry{ - StatsdAddr: "bar", - StatsiteAddr: "foo", - DisableHostname: false, - DogStatsDAddr: "127.0.0.1:7254", - DogStatsDTags: []string{"tag_1:val_1", "tag_2:val_2"}, - PrometheusRetentionTime: 30 * time.Second, - UsageGaugePeriod: 5 * time.Minute, - MaximumGaugeCardinality: 125, + StatsdAddr: "bar", + StatsiteAddr: "foo", + DisableHostname: false, + DogStatsDAddr: "127.0.0.1:7254", + DogStatsDTags: []string{"tag_1:val_1", "tag_2:val_2"}, + PrometheusRetentionTime: 30 * time.Second, + UsageGaugePeriod: 5 * time.Minute, + MaximumGaugeCardinality: 125, + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 168, + LeaseMetricsNameSpaceLabels: false, }, DisableMlock: true, @@ -192,6 +195,9 @@ func testLoadConfigFile_json2(t *testing.T, entropy *configutil.Entropy) { CirconusBrokerID: "0", CirconusBrokerSelectTag: "dc:sfo", PrometheusRetentionTime: 30 * time.Second, + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 168, + LeaseMetricsNameSpaceLabels: false, }, }, @@ -371,15 +377,18 @@ func testLoadConfigFile(t *testing.T) { }, Telemetry: &configutil.Telemetry{ - StatsdAddr: "bar", - StatsiteAddr: "foo", - DisableHostname: false, - UsageGaugePeriod: 5 * time.Minute, - MaximumGaugeCardinality: 100, - DogStatsDAddr: "127.0.0.1:7254", - DogStatsDTags: []string{"tag_1:val_1", "tag_2:val_2"}, - PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime, - MetricsPrefix: "myprefix", + StatsdAddr: "bar", + StatsiteAddr: "foo", + DisableHostname: false, + UsageGaugePeriod: 5 * time.Minute, + MaximumGaugeCardinality: 100, + DogStatsDAddr: "127.0.0.1:7254", + DogStatsDTags: []string{"tag_1:val_1", "tag_2:val_2"}, + PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime, + MetricsPrefix: "myprefix", + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 168, + LeaseMetricsNameSpaceLabels: false, }, DisableMlock: true, @@ -477,6 +486,9 @@ func testLoadConfigFile_json(t *testing.T) { CirconusBrokerID: "", CirconusBrokerSelectTag: "", PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime, + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 168, + LeaseMetricsNameSpaceLabels: false, }, PidFile: "./pidfile", @@ -540,12 +552,15 @@ func testLoadConfigDir(t *testing.T) { }, Telemetry: &configutil.Telemetry{ - StatsiteAddr: "qux", - StatsdAddr: "baz", - DisableHostname: true, - UsageGaugePeriod: 5 * time.Minute, - MaximumGaugeCardinality: 100, - PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime, + StatsiteAddr: "qux", + StatsdAddr: "baz", + DisableHostname: true, + UsageGaugePeriod: 5 * time.Minute, + MaximumGaugeCardinality: 100, + PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime, + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 168, + LeaseMetricsNameSpaceLabels: false, }, ClusterName: "testcluster", }, @@ -668,6 +683,9 @@ func testConfig_Sanitized(t *testing.T) { "stackdriver_debug_logs": false, "statsd_address": "bar", "statsite_address": "", + "lease_metrics_epsilon": time.Hour, + "num_lease_metrics_buckets": 168, + "add_lease_metrics_namespace_labels": false, }, } @@ -787,3 +805,93 @@ func testParseSeals(t *testing.T) { } require.Equal(t, config, expected) } + +func testLoadConfigFileLeaseMetrics(t *testing.T) { + config, err := LoadConfigFile("./test-fixtures/config5.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + Listeners: []*configutil.Listener{ + { + Type: "tcp", + Address: "127.0.0.1:443", + }, + }, + + Telemetry: &configutil.Telemetry{ + StatsdAddr: "bar", + StatsiteAddr: "foo", + DisableHostname: false, + UsageGaugePeriod: 5 * time.Minute, + MaximumGaugeCardinality: 100, + DogStatsDAddr: "127.0.0.1:7254", + DogStatsDTags: []string{"tag_1:val_1", "tag_2:val_2"}, + PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime, + MetricsPrefix: "myprefix", + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 2, + LeaseMetricsNameSpaceLabels: true, + }, + + DisableMlock: true, + + Entropy: nil, + + PidFile: "./pidfile", + + ClusterName: "testcluster", + }, + + Storage: &Storage{ + Type: "consul", + RedirectAddr: "foo", + Config: map[string]string{ + "foo": "bar", + }, + }, + + HAStorage: &Storage{ + Type: "consul", + RedirectAddr: "snafu", + Config: map[string]string{ + "bar": "baz", + }, + DisableClustering: true, + }, + + ServiceRegistration: &ServiceRegistration{ + Type: "consul", + Config: map[string]string{ + "foo": "bar", + }, + }, + + DisableCache: true, + DisableCacheRaw: true, + DisablePrintableCheckRaw: true, + DisablePrintableCheck: true, + EnableUI: true, + EnableUIRaw: true, + + EnableRawEndpoint: true, + EnableRawEndpointRaw: true, + + DisableSealWrap: true, + DisableSealWrapRaw: true, + + MaxLeaseTTL: 10 * time.Hour, + MaxLeaseTTLRaw: "10h", + DefaultLeaseTTL: 10 * time.Hour, + DefaultLeaseTTLRaw: "10h", + } + + addExpectedEntConfig(expected, []string{}) + + config.Listeners[0].RawConfig = nil + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} diff --git a/command/server/test-fixtures/config5.hcl b/command/server/test-fixtures/config5.hcl new file mode 100644 index 0000000000..acf58a8978 --- /dev/null +++ b/command/server/test-fixtures/config5.hcl @@ -0,0 +1,51 @@ +disable_cache = true + disable_mlock = true + + ui = true + + listener "tcp" { + address = "127.0.0.1:443" + allow_stuff = true + } + + backend "consul" { + foo = "bar" + advertise_addr = "foo" + } + + ha_backend "consul" { + bar = "baz" + advertise_addr = "snafu" + disable_clustering = "true" + } + + service_registration "consul" { + foo = "bar" + } + + telemetry { + statsd_address = "bar" + usage_gauge_period = "5m" + maximum_gauge_cardinality = 100 + + statsite_address = "foo" + dogstatsd_addr = "127.0.0.1:7254" + dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"] + metrics_prefix = "myprefix" + + lease_metrics_epsilon = "1h" + num_lease_metrics_buckets = 2 + add_lease_metrics_namespace_labels = true + } + + sentinel { + additional_enabled_modules = [] + } + + max_lease_ttl = "10h" + default_lease_ttl = "10h" + cluster_name = "testcluster" + pid_file = "./pidfile" + raw_storage_endpoint = true + disable_sealwrap = true + disable_printable_check = true \ No newline at end of file diff --git a/helper/metricsutil/bucket.go b/helper/metricsutil/bucket.go index 759a5207e6..088516bd6e 100644 --- a/helper/metricsutil/bucket.go +++ b/helper/metricsutil/bucket.go @@ -37,3 +37,22 @@ func TTLBucket(ttl time.Duration) string { } } + +func ExpiryBucket(expiryTime time.Time, leaseEpsilon time.Duration, rollingWindow time.Time, labelNS string, useNS bool) *LeaseExpiryLabel { + if !useNS { + labelNS = "" + } + leaseExpiryLabel := LeaseExpiryLabel{LabelNS: labelNS} + + // calculate rolling window + if expiryTime.Before(rollingWindow) { + leaseExpiryLabel.LabelName = expiryTime.Round(leaseEpsilon).String() + return &leaseExpiryLabel + } + return nil +} + +type LeaseExpiryLabel = struct { + LabelName string + LabelNS string +} diff --git a/helper/metricsutil/wrapped_metrics.go b/helper/metricsutil/wrapped_metrics.go index 8240f2d1cb..a1695cdf5d 100644 --- a/helper/metricsutil/wrapped_metrics.go +++ b/helper/metricsutil/wrapped_metrics.go @@ -28,6 +28,15 @@ type ClusterMetricSink struct { // Sink is the go-metrics instance to send to. Sink metrics.MetricSink + + // Constants that are helpful for metrics within the metrics sink + TelemetryConsts TelemetryConstConfig +} + +type TelemetryConstConfig struct { + LeaseMetricsEpsilon time.Duration + NumLeaseMetricsTimeBuckets int + LeaseMetricsNameSpaceLabels bool } type Metrics interface { @@ -83,8 +92,9 @@ func BlackholeSink() *ClusterMetricSink { func NewClusterMetricSink(clusterName string, sink metrics.MetricSink) *ClusterMetricSink { cms := &ClusterMetricSink{ - ClusterName: atomic.Value{}, - Sink: sink, + ClusterName: atomic.Value{}, + Sink: sink, + TelemetryConsts: TelemetryConstConfig{}, } cms.ClusterName.Store(clusterName) return cms diff --git a/internalshared/configutil/config.go b/internalshared/configutil/config.go index 5e642a7314..6ef434cccb 100644 --- a/internalshared/configutil/config.go +++ b/internalshared/configutil/config.go @@ -214,6 +214,9 @@ func (c *SharedConfig) Sanitized() map[string]interface{} { "stackdriver_location": c.Telemetry.StackdriverLocation, "stackdriver_namespace": c.Telemetry.StackdriverNamespace, "stackdriver_debug_logs": c.Telemetry.StackdriverDebugLogs, + "lease_metrics_epsilon": c.Telemetry.LeaseMetricsEpsilon, + "num_lease_metrics_buckets": c.Telemetry.NumLeaseMetricsTimeBuckets, + "add_lease_metrics_namespace_labels": c.Telemetry.LeaseMetricsNameSpaceLabels, } result["telemetry"] = sanitizedTelemetry } diff --git a/internalshared/configutil/telemetry.go b/internalshared/configutil/telemetry.go index a7d49a70cc..4f31605b73 100644 --- a/internalshared/configutil/telemetry.go +++ b/internalshared/configutil/telemetry.go @@ -24,9 +24,11 @@ import ( ) const ( - PrometheusDefaultRetentionTime = 24 * time.Hour - UsageGaugeDefaultPeriod = 10 * time.Minute - MaximumGaugeCardinalityDefault = 500 + PrometheusDefaultRetentionTime = 24 * time.Hour + UsageGaugeDefaultPeriod = 10 * time.Minute + MaximumGaugeCardinalityDefault = 500 + LeaseMetricsEpsilonDefault = time.Hour + NumLeaseMetricsTimeBucketsDefault = 168 ) // Telemetry is the telemetry configuration for the server @@ -137,6 +139,16 @@ type Telemetry struct { StackdriverNamespace string `hcl:"stackdriver_namespace"` // StackdriverDebugLogs will write additional stackdriver related debug logs to stderr. StackdriverDebugLogs bool `hcl:"stackdriver_debug_logs"` + + // How often metrics for lease expiry will be aggregated + LeaseMetricsEpsilon time.Duration + LeaseMetricsEpsilonRaw interface{} `hcl:"lease_metrics_epsilon"` + + // Number of buckets by time that will be used in lease aggregation + NumLeaseMetricsTimeBuckets int `hcl:"num_lease_metrics_buckets"` + + // Whether or not telemetry should add labels for namespaces + LeaseMetricsNameSpaceLabels bool `hcl:"add_lease_metrics_namespace_labels"` } func (t *Telemetry) GoString() string { @@ -151,11 +163,6 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error { // Get our one item item := list.Items[0] - var t Telemetry - if err := hcl.DecodeObject(&t, item.Val); err != nil { - return multierror.Prefix(err, "telemetry:") - } - if result.Telemetry == nil { result.Telemetry = &Telemetry{} } @@ -192,6 +199,24 @@ func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error { result.Telemetry.MaximumGaugeCardinality = MaximumGaugeCardinalityDefault } + if result.Telemetry.LeaseMetricsEpsilonRaw != nil { + if result.Telemetry.LeaseMetricsEpsilonRaw == "none" { + result.Telemetry.LeaseMetricsEpsilonRaw = 0 + } else { + var err error + if result.Telemetry.LeaseMetricsEpsilon, err = parseutil.ParseDurationSecond(result.Telemetry.LeaseMetricsEpsilonRaw); err != nil { + return err + } + result.Telemetry.LeaseMetricsEpsilonRaw = nil + } + } else { + result.Telemetry.LeaseMetricsEpsilon = LeaseMetricsEpsilonDefault + } + + if result.Telemetry.NumLeaseMetricsTimeBuckets == 0 { + result.Telemetry.NumLeaseMetricsTimeBuckets = NumLeaseMetricsTimeBucketsDefault + } + return nil } @@ -355,6 +380,9 @@ func SetupTelemetry(opts *SetupTelemetryOpts) (*metrics.InmemSink, *metricsutil. wrapper := metricsutil.NewClusterMetricSink(opts.ClusterName, globalMetrics) wrapper.MaxGaugeCardinality = opts.Config.MaximumGaugeCardinality wrapper.GaugeInterval = opts.Config.UsageGaugePeriod + wrapper.TelemetryConsts.LeaseMetricsEpsilon = opts.Config.LeaseMetricsEpsilon + wrapper.TelemetryConsts.LeaseMetricsNameSpaceLabels = opts.Config.LeaseMetricsNameSpaceLabels + wrapper.TelemetryConsts.NumLeaseMetricsTimeBuckets = opts.Config.NumLeaseMetricsTimeBuckets return inm, wrapper, prometheusEnabled, nil } diff --git a/vault/core_metrics.go b/vault/core_metrics.go index 10e42ddffc..3d24c2edda 100644 --- a/vault/core_metrics.go +++ b/vault/core_metrics.go @@ -116,6 +116,17 @@ func (c *Core) tokenGaugePolicyCollector(ctx context.Context) ([]metricsutil.Gau return ts.gaugeCollectorByPolicy(ctx) } +func (c *Core) leaseExpiryGaugeCollector(ctx context.Context) ([]metricsutil.GaugeLabelValues, error) { + c.stateLock.RLock() + e := c.expiration + metricsConsts := c.MetricSink().TelemetryConsts + c.stateLock.RUnlock() + if e == nil { + return []metricsutil.GaugeLabelValues{}, errors.New("nil expiration manager") + } + return e.leaseAggregationMetrics(ctx, metricsConsts) +} + func (c *Core) tokenGaugeMethodCollector(ctx context.Context) ([]metricsutil.GaugeLabelValues, error) { c.stateLock.RLock() ts := c.tokenStore @@ -164,6 +175,12 @@ func (c *Core) emitMetrics(stopCh chan struct{}) { c.tokenGaugePolicyCollector, "", }, + { + []string{"expire", "leases", "by_expiration"}, + []metrics.Label{{"gauge", "leases_by_expiration"}}, + c.leaseExpiryGaugeCollector, + "", + }, { []string{"token", "count", "by_auth"}, []metrics.Label{{"gauge", "token_by_auth"}}, diff --git a/vault/expiration.go b/vault/expiration.go index ea09bffbbe..49d5a937c8 100644 --- a/vault/expiration.go +++ b/vault/expiration.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/errwrap" log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/base62" @@ -1997,6 +1998,66 @@ func (m *ExpirationManager) emitMetrics() { } } +func (m *ExpirationManager) leaseAggregationMetrics(ctx context.Context, consts metricsutil.TelemetryConstConfig) ([]metricsutil.GaugeLabelValues, error) { + expiryTimes := make(map[metricsutil.LeaseExpiryLabel]int) + leaseEpsilon := consts.LeaseMetricsEpsilon + nsLabel := consts.LeaseMetricsNameSpaceLabels + + rollingWindow := time.Now().Add(time.Duration(consts.NumLeaseMetricsTimeBuckets) * leaseEpsilon) + + err := m.walkLeases(func(entryID string, expireTime time.Time) bool { + select { + // Abort and return empty collection if it's taking too much time, nonblocking check. + case <-ctx.Done(): + return false + default: + if entryID == "" { + return true + } + _, nsID := namespace.SplitIDFromString(entryID) + if nsID == "" { + nsID = "root" // this is what metricsutil.NamespaceLabel does + } + label := metricsutil.ExpiryBucket(expireTime, leaseEpsilon, rollingWindow, nsID, nsLabel) + if label != nil { + expiryTimes[*label] += 1 + } + return true + } + }) + + if err != nil { + return []metricsutil.GaugeLabelValues{}, suppressRestoreModeError(err) + } + + // If collection was cancelled, return an empty array. + select { + case <-ctx.Done(): + return []metricsutil.GaugeLabelValues{}, nil + default: + break + } + + flattenedResults := make([]metricsutil.GaugeLabelValues, 0, len(expiryTimes)) + + for bucket, count := range expiryTimes { + if nsLabel { + flattenedResults = append(flattenedResults, + metricsutil.GaugeLabelValues{ + Labels: []metrics.Label{{"expiring", bucket.LabelName}, {"namespace", bucket.LabelNS}}, + Value: float32(count), + }) + } else { + flattenedResults = append(flattenedResults, + metricsutil.GaugeLabelValues{ + Labels: []metrics.Label{{"expiring", bucket.LabelName}}, + Value: float32(count), + }) + } + } + return flattenedResults, nil +} + // Callback function type to walk tokens referenced in the expiration // manager. Don't want to use leaseEntry here because it's an unexported // type (though most likely we would only call this from within the "vault" core package.) @@ -2031,6 +2092,30 @@ func (m *ExpirationManager) WalkTokens(walkFn ExpirationWalkFunction) error { return nil } +// leaseWalkFunction can only be used by the core package. +type leaseWalkFunction = func(leaseID string, expireTime time.Time) bool + +func (m *ExpirationManager) walkLeases(walkFn leaseWalkFunction) error { + if m.inRestoreMode() { + return ErrInRestoreMode + } + + callback := func(key, value interface{}) bool { + p := value.(pendingInfo) + if p.cachedLeaseInfo == nil { + return true + } + lease := p.cachedLeaseInfo + expireTime := lease.ExpireTime + return walkFn(key.(string), expireTime) + } + + m.pending.Range(callback) + m.nonexpiring.Range(callback) + + return nil +} + // leaseEntry is used to structure the values the expiration // manager stores. This is used to handle renew and revocation. type leaseEntry struct { diff --git a/vault/expiration_test.go b/vault/expiration_test.go index 5a65df4df0..9033ecd338 100644 --- a/vault/expiration_test.go +++ b/vault/expiration_test.go @@ -13,8 +13,10 @@ import ( "testing" "time" + metrics "github.com/armon/go-metrics" log "github.com/hashicorp/go-hclog" uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/logging" @@ -38,6 +40,186 @@ func mockBackendExpiration(t testing.TB, backend physical.Backend) (*Core, *Expi return c, c.expiration } +func TestExpiration_Metrics(t *testing.T) { + var err error + + testCore := TestCore(t) + testCore.baseLogger = logger + testCore.logger = logger.Named("core") + testCoreUnsealed(t, testCore) + + exp := testCore.expiration + + if err := exp.Restore(nil); err != nil { + t.Fatal(err) + } + + // Set up a count function to calculate number of leases + count := 0 + countFunc := func(leaseID string) { + count++ + } + + // Scan the storage with the count func set + if err = logical.ScanView(namespace.RootContext(nil), exp.idView, countFunc); err != nil { + t.Fatal(err) + } + + // Check that there are no leases to begin with + if count != 0 { + t.Fatalf("bad: lease count; expected:0 actual:%d", count) + } + + for i := 0; i < 50; i++ { + le := &leaseEntry{ + LeaseID: "lease" + fmt.Sprintf("%d", i), + Path: "foo/bar/" + fmt.Sprintf("%d", i), + namespace: namespace.RootNamespace, + IssueTime: time.Now(), + ExpireTime: time.Now().Add(time.Hour), + } + + otherNS := &namespace.Namespace{ + ID: "nsid", + Path: "foo/bar", + } + + otherNSle := &leaseEntry{ + LeaseID: "lease" + fmt.Sprintf("%d", i) + "/blah.nsid", + Path: "foo/bar/" + fmt.Sprintf("%d", i) + "/blah.nsid", + namespace: otherNS, + IssueTime: time.Now(), + ExpireTime: time.Now().Add(time.Hour), + } + + exp.pendingLock.Lock() + if err := exp.persistEntry(namespace.RootContext(nil), le); err != nil { + exp.pendingLock.Unlock() + t.Fatalf("error persisting entry: %v", err) + } + exp.updatePendingInternal(le) + + if err := exp.persistEntry(namespace.RootContext(nil), otherNSle); err != nil { + exp.pendingLock.Unlock() + t.Fatalf("error persisting entry: %v", err) + } + exp.updatePendingInternal(otherNSle) + exp.pendingLock.Unlock() + } + + for i := 50; i < 250; i++ { + le := &leaseEntry{ + LeaseID: "lease" + fmt.Sprintf("%d", i+1), + Path: "foo/bar/" + fmt.Sprintf("%d", i+1), + namespace: namespace.RootNamespace, + IssueTime: time.Now(), + ExpireTime: time.Now().Add(2 * time.Hour), + } + + exp.pendingLock.Lock() + if err := exp.persistEntry(namespace.RootContext(nil), le); err != nil { + exp.pendingLock.Unlock() + t.Fatalf("error persisting entry: %v", err) + } + exp.updatePendingInternal(le) + exp.pendingLock.Unlock() + } + + count = 0 + if err = logical.ScanView(context.Background(), exp.idView, countFunc); err != nil { + t.Fatal(err) + } + + var conf metricsutil.TelemetryConstConfig = metricsutil.TelemetryConstConfig{ + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 2, + LeaseMetricsNameSpaceLabels: true, + } + + flattenedResults, err := exp.leaseAggregationMetrics(context.Background(), conf) + if err != nil { + t.Fatal(err) + } + if flattenedResults == nil { + t.Fatal("lease aggregation returns nil metrics") + } + + labelOneHour := metrics.Label{"expiring", time.Now().Add(time.Hour).Round(time.Hour).String()} + labelTwoHours := metrics.Label{"expiring", time.Now().Add(2 * time.Hour).Round(time.Hour).String()} + nsLabel := metrics.Label{"namespace", "root"} + nsLabelNonRoot := metrics.Label{"namespace", "nsid"} + + foundLabelOne := false + foundLabelTwo := false + foundLabelThree := false + + for _, labelVal := range flattenedResults { + retNsLabel := labelVal.Labels[1] + retTimeLabel := labelVal.Labels[0] + if nsLabel == retNsLabel { + if labelVal.Value == 50 { + if retTimeLabel == labelOneHour { + foundLabelOne = true + } + } + if labelVal.Value == 200 { + if retTimeLabel == labelTwoHours { + foundLabelTwo = true + } + } + } else if retNsLabel == nsLabelNonRoot { + if labelVal.Value == 50 { + if retTimeLabel == labelOneHour { + foundLabelThree = true + } + } + } + } + + if !foundLabelOne || !foundLabelTwo || !foundLabelThree { + t.Errorf("One of the labels is missing") + } + + // test the same leases while ignoring namespaces so the 2 different namespaces get aggregated + conf = metricsutil.TelemetryConstConfig{ + LeaseMetricsEpsilon: time.Hour, + NumLeaseMetricsTimeBuckets: 2, + LeaseMetricsNameSpaceLabels: false, + } + + flattenedResults, err = exp.leaseAggregationMetrics(context.Background(), conf) + if err != nil { + t.Fatal(err) + } + if flattenedResults == nil { + t.Fatal("lease aggregation returns nil metrics") + } + + foundLabelOne = false + foundLabelTwo = false + + for _, labelVal := range flattenedResults { + if len(labelVal.Labels) != 1 { + t.Errorf("Namespace label is returned when explicitly not requested.") + } + retTimeLabel := labelVal.Labels[0] + if labelVal.Value == 100 { + if retTimeLabel == labelOneHour { + foundLabelOne = true + } + } + if labelVal.Value == 200 { + if retTimeLabel == labelTwoHours { + foundLabelTwo = true + } + } + } + if !foundLabelOne || !foundLabelTwo { + t.Errorf("One of the labels is missing") + } + +} + func TestExpiration_Tidy(t *testing.T) { var err error diff --git a/vendor/github.com/hashicorp/vault/api/client.go b/vendor/github.com/hashicorp/vault/api/client.go index f7c5c61fd5..56de24921c 100644 --- a/vendor/github.com/hashicorp/vault/api/client.go +++ b/vendor/github.com/hashicorp/vault/api/client.go @@ -475,6 +475,9 @@ func (c *Client) SetAddress(addr string) error { return errwrap.Wrapf("failed to set address: {{err}}", err) } + c.config.modifyLock.Lock() + c.config.Address = addr + c.config.modifyLock.Unlock() c.addr = parsedAddr return nil } @@ -492,57 +495,111 @@ func (c *Client) Address() string { // rateLimit and burst are specified according to https://godoc.org/golang.org/x/time/rate#NewLimiter func (c *Client) SetLimiter(rateLimit float64, burst int) { c.modifyLock.RLock() + defer c.modifyLock.RUnlock() c.config.modifyLock.Lock() defer c.config.modifyLock.Unlock() - c.modifyLock.RUnlock() c.config.Limiter = rate.NewLimiter(rate.Limit(rateLimit), burst) } +func (c *Client) Limiter() *rate.Limiter { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.RLock() + defer c.config.modifyLock.RUnlock() + + return c.config.Limiter +} + // SetMaxRetries sets the number of retries that will be used in the case of certain errors func (c *Client) SetMaxRetries(retries int) { c.modifyLock.RLock() + defer c.modifyLock.RUnlock() c.config.modifyLock.Lock() defer c.config.modifyLock.Unlock() - c.modifyLock.RUnlock() c.config.MaxRetries = retries } +func (c *Client) MaxRetries() int { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.RLock() + defer c.config.modifyLock.RUnlock() + + return c.config.MaxRetries +} + +func (c *Client) SetSRVLookup(srv bool) { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.Lock() + defer c.config.modifyLock.Unlock() + + c.config.SRVLookup = srv +} + +func (c *Client) SRVLookup() bool { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.RLock() + defer c.config.modifyLock.RUnlock() + + return c.config.SRVLookup +} + // SetCheckRetry sets the CheckRetry function to be used for future requests. func (c *Client) SetCheckRetry(checkRetry retryablehttp.CheckRetry) { c.modifyLock.RLock() + defer c.modifyLock.RUnlock() c.config.modifyLock.Lock() defer c.config.modifyLock.Unlock() - c.modifyLock.RUnlock() c.config.CheckRetry = checkRetry } +func (c *Client) CheckRetry() retryablehttp.CheckRetry { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.RLock() + defer c.config.modifyLock.RUnlock() + + return c.config.CheckRetry +} + // SetClientTimeout sets the client request timeout func (c *Client) SetClientTimeout(timeout time.Duration) { c.modifyLock.RLock() + defer c.modifyLock.RUnlock() c.config.modifyLock.Lock() defer c.config.modifyLock.Unlock() - c.modifyLock.RUnlock() c.config.Timeout = timeout } -func (c *Client) OutputCurlString() bool { +func (c *Client) ClientTimeout() time.Duration { c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.RLock() + defer c.config.modifyLock.RUnlock() + + return c.config.Timeout +} + +func (c *Client) OutputCurlString() bool { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() c.config.modifyLock.RLock() defer c.config.modifyLock.RUnlock() - c.modifyLock.RUnlock() return c.config.OutputCurlString } func (c *Client) SetOutputCurlString(curl bool) { c.modifyLock.RLock() + defer c.modifyLock.RUnlock() c.config.modifyLock.Lock() defer c.config.modifyLock.Unlock() - c.modifyLock.RUnlock() c.config.OutputCurlString = curl } @@ -552,7 +609,6 @@ func (c *Client) SetOutputCurlString(curl bool) { func (c *Client) CurrentWrappingLookupFunc() WrappingLookupFunc { c.modifyLock.RLock() defer c.modifyLock.RUnlock() - return c.wrappingLookupFunc } @@ -561,7 +617,6 @@ func (c *Client) CurrentWrappingLookupFunc() WrappingLookupFunc { func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) { c.modifyLock.Lock() defer c.modifyLock.Unlock() - c.wrappingLookupFunc = lookupFunc } @@ -570,7 +625,6 @@ func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) { func (c *Client) SetMFACreds(creds []string) { c.modifyLock.Lock() defer c.modifyLock.Unlock() - c.mfaCreds = creds } @@ -595,7 +649,6 @@ func (c *Client) setNamespace(namespace string) { func (c *Client) Token() string { c.modifyLock.RLock() defer c.modifyLock.RUnlock() - return c.token } @@ -604,7 +657,6 @@ func (c *Client) Token() string { func (c *Client) SetToken(v string) { c.modifyLock.Lock() defer c.modifyLock.Unlock() - c.token = v } @@ -612,7 +664,6 @@ func (c *Client) SetToken(v string) { func (c *Client) ClearToken() { c.modifyLock.Lock() defer c.modifyLock.Unlock() - c.token = "" } @@ -655,9 +706,9 @@ func (c *Client) SetHeaders(headers http.Header) { // SetBackoff sets the backoff function to be used for future requests. func (c *Client) SetBackoff(backoff retryablehttp.Backoff) { c.modifyLock.RLock() + defer c.modifyLock.RUnlock() c.config.modifyLock.Lock() defer c.config.modifyLock.Unlock() - c.modifyLock.RUnlock() c.config.Backoff = backoff } @@ -672,22 +723,30 @@ func (c *Client) SetBackoff(backoff retryablehttp.Backoff) { // behavior, must currently then be set as desired on the new client. func (c *Client) Clone() (*Client, error) { c.modifyLock.RLock() - c.config.modifyLock.RLock() + defer c.modifyLock.RUnlock() + config := c.config - c.modifyLock.RUnlock() + config.modifyLock.RLock() + defer config.modifyLock.RUnlock() newConfig := &Config{ - Address: config.Address, - HttpClient: config.HttpClient, - MaxRetries: config.MaxRetries, - Timeout: config.Timeout, - Backoff: config.Backoff, - CheckRetry: config.CheckRetry, - Limiter: config.Limiter, + Address: config.Address, + HttpClient: config.HttpClient, + MaxRetries: config.MaxRetries, + Timeout: config.Timeout, + Backoff: config.Backoff, + CheckRetry: config.CheckRetry, + Limiter: config.Limiter, + OutputCurlString: config.OutputCurlString, + AgentAddress: config.AgentAddress, + SRVLookup: config.SRVLookup, + } + client, err := NewClient(newConfig) + if err != nil { + return nil, err } - config.modifyLock.RUnlock() - return NewClient(newConfig) + return client, nil } // SetPolicyOverride sets whether requests should be sent with the policy @@ -696,7 +755,6 @@ func (c *Client) Clone() (*Client, error) { func (c *Client) SetPolicyOverride(override bool) { c.modifyLock.Lock() defer c.modifyLock.Unlock() - c.policyOverride = override }