From b4b347f0ae51c2039fb33d833e43986e79aed3cf Mon Sep 17 00:00:00 2001 From: Vault Automation Date: Thu, 4 Sep 2025 16:02:24 -0600 Subject: [PATCH] Backport [VAULT-38467] Cumulative namespace client count calculation into ce/main (#9038) * [VAULT-38467] Cumulative namespace client count calculation (#8650) * porting over changes from spike * refactor * move cumulativeCount to new ent file * move struct * add tests * add deleted namespace test * expand comment and remove debug log * make sure inputs are aligned to month start/end * address comments and fix test * fix test * fix test --------- Co-authored-by: Jenny Deng --- vault/activity_log.go | 74 +-- vault/activity_log_util_common.go | 67 ++ vault/activity_log_util_common_test.go | 835 +++++++++++++++++++++++++ 3 files changed, 908 insertions(+), 68 deletions(-) diff --git a/vault/activity_log.go b/vault/activity_log.go index 6f50e2ee9a..9614e04ec4 100644 --- a/vault/activity_log.go +++ b/vault/activity_log.go @@ -1962,76 +1962,14 @@ func (a *ActivityLog) DefaultStartTime(endTime time.Time) time.Time { } func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.Time, limitNamespaces int) (map[string]interface{}, error) { - var computePartial bool - - // Change the start time to the beginning of the month, and the end time to be the end - // of the month. + // Normalize the start time to the beginning of the month, and the end time to be the end of the month. startTime = timeutil.StartOfMonth(startTime) endTime = timeutil.EndOfMonth(endTime) - // At the max, we only want to return data up until the end of the current month. - // Adjust the end time be the current month if a future date has been provided. - endOfCurrentMonth := timeutil.EndOfMonth(a.clock.Now().UTC()) - adjustedEndTime := endTime - if endTime.After(endOfCurrentMonth) { - adjustedEndTime = endOfCurrentMonth - } - - // If the endTime of the query is the current month, request data from the queryStore - // with the endTime equal to the end of the last month, and add in the current month - // data. - precomputedQueryEndTime := adjustedEndTime - if timeutil.IsCurrentMonth(adjustedEndTime, a.clock.Now().UTC()) { - precomputedQueryEndTime = timeutil.EndOfMonth(timeutil.MonthsPreviousTo(1, timeutil.StartOfMonth(adjustedEndTime))) - computePartial = true - } - - pq := &activity.PrecomputedQuery{} - if startTime.After(precomputedQueryEndTime) && timeutil.IsCurrentMonth(startTime, a.clock.Now().UTC()) { - // We're only calculating the partial month client count. Skip the precomputation - // get call. - pq = &activity.PrecomputedQuery{ - StartTime: startTime, - EndTime: endTime, - Namespaces: make([]*activity.NamespaceRecord, 0), - Months: make([]*activity.MonthRecord, 0), - } - } else { - storedQuery, err := a.queryStore.Get(ctx, startTime, precomputedQueryEndTime) - if err != nil { - return nil, err - } - if storedQuery == nil { - // If the storedQuery is nil, that means there's no historical data to process. But, it's possible there's - // still current month data to process, so rather than returning a 204, let's proceed along like we're - // just querying the current month. - storedQuery = &activity.PrecomputedQuery{ - StartTime: startTime, - EndTime: endTime, - Namespaces: make([]*activity.NamespaceRecord, 0), - Months: make([]*activity.MonthRecord, 0), - } - } - pq = storedQuery - } - - var partialByMonth map[int64]*processMonth - if computePartial { - // Traverse through current month's activitylog data and group clients - // into months and namespaces - a.fragmentLock.RLock() - partialByMonth, _ = a.populateNamespaceAndMonthlyBreakdowns() - a.fragmentLock.RUnlock() - - // Estimate the current month totals. These record contains is complete with all the - // current month data, grouped by namespace and mounts - currentMonth, err := a.computeCurrentMonthForBillingPeriod(partialByMonth, startTime, adjustedEndTime) - if err != nil { - return nil, err - } - - // Combine the existing months precomputed query with the current month data - pq.CombineWithCurrentMonth(currentMonth) + // Compute the total clients in the billing period, and get the breakdown by namespace. + pq, err := a.computeClientsInBillingPeriod(ctx, startTime, endTime) + if err != nil { + return nil, err } // Convert the namespace data into a protobuf format that can be returned in the response @@ -2071,7 +2009,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T a.sortActivityLogMonthsResponse(months) // Modify the final month output to make response more consumable based on API request - months = a.modifyResponseMonths(months, startTime, adjustedEndTime) + months = a.modifyResponseMonths(months, startTime, endTime) responseData["months"] = months return responseData, nil diff --git a/vault/activity_log_util_common.go b/vault/activity_log_util_common.go index 973392577d..201b8512eb 100644 --- a/vault/activity_log_util_common.go +++ b/vault/activity_log_util_common.go @@ -19,6 +19,73 @@ import ( "google.golang.org/protobuf/proto" ) +// computeClientsInBillingPeriod computes the clients in a billing period given a start and end time. +// It retrieves the precomputed query data for the specified period, and if the current month is included, +// it computes the current month's data by aggregating the activity log fragments. It returns the counts +// separated by namespace as well as the total counts for the billing period. +func (a *ActivityLog) computeClientsInBillingPeriod(ctx context.Context, startTime time.Time, endTime time.Time) (*activity.PrecomputedQuery, error) { + var computePartial bool + + // If the endTime of the query is the current month, request data from the queryStore + // with the endTime equal to the end of the last month, and add in the current month + // data. + precomputedQueryEndTime := endTime + if timeutil.IsCurrentMonth(endTime, a.clock.Now().UTC()) { + precomputedQueryEndTime = timeutil.EndOfMonth(timeutil.MonthsPreviousTo(1, timeutil.StartOfMonth(endTime))) + computePartial = true + } + + pq := &activity.PrecomputedQuery{} + if startTime.After(precomputedQueryEndTime) && timeutil.IsCurrentMonth(startTime, a.clock.Now().UTC()) { + // We're only calculating the partial month client count. Skip the precomputation + // get call. + pq = &activity.PrecomputedQuery{ + StartTime: startTime, + EndTime: endTime, + Namespaces: make([]*activity.NamespaceRecord, 0), + Months: make([]*activity.MonthRecord, 0), + } + } else { + storedQuery, err := a.queryStore.Get(ctx, startTime, precomputedQueryEndTime) + if err != nil { + return nil, err + } + if storedQuery == nil { + // If the storedQuery is nil, that means there's no historical data to process. But, it's possible there's + // still current month data to process, so rather than returning a 204, let's proceed along like we're + // just querying the current month. + storedQuery = &activity.PrecomputedQuery{ + StartTime: startTime, + EndTime: endTime, + Namespaces: make([]*activity.NamespaceRecord, 0), + Months: make([]*activity.MonthRecord, 0), + } + } + pq = storedQuery + } + + var partialByMonth map[int64]*processMonth + if computePartial { + // Traverse through current month's activitylog data and group clients + // into months and namespaces + a.fragmentLock.RLock() + partialByMonth, _ = a.populateNamespaceAndMonthlyBreakdowns() + a.fragmentLock.RUnlock() + + // Estimate the current month totals. These record contains is complete with all the + // current month data, grouped by namespace and mounts + currentMonth, err := a.computeCurrentMonthForBillingPeriod(partialByMonth, startTime, endTime) + if err != nil { + return nil, err + } + + // Combine the existing months precomputed query with the current month data + pq.CombineWithCurrentMonth(currentMonth) + } + + return pq, nil +} + // computeCurrentMonthForBillingPeriod computes the current month's data with respect // to a billing period. func (a *ActivityLog) computeCurrentMonthForBillingPeriod(byMonth map[int64]*processMonth, startTime time.Time, endTime time.Time) (*activity.MonthRecord, error) { diff --git a/vault/activity_log_util_common_test.go b/vault/activity_log_util_common_test.go index c2ea1cf99f..e3965c99ee 100644 --- a/vault/activity_log_util_common_test.go +++ b/vault/activity_log_util_common_test.go @@ -5,13 +5,16 @@ package vault import ( "context" + "encoding/json" "errors" "fmt" "io" "sort" + "sync" "testing" "time" + "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/timeutil" "github.com/hashicorp/vault/vault/activity" "github.com/stretchr/testify/require" @@ -40,6 +43,19 @@ func equalActivityMonthRecords(t *testing.T, expected, got *activity.MonthRecord require.Equal(t, expected, got) } +// equalActivityNamespaceRecords is a helper to sort the namespaces in activity.NamespaceRecord's, +// then compare their equality +func equalActivityNamespaceRecords(t *testing.T, expected, got []*activity.NamespaceRecord) { + t.Helper() + sort.SliceStable(expected, func(i, j int) bool { + return expected[i].NamespaceID < expected[j].NamespaceID + }) + sort.SliceStable(got, func(i, j int) bool { + return got[i].NamespaceID < got[j].NamespaceID + }) + require.Equal(t, expected, got) +} + // Test_ActivityLog_ComputeCurrentMonthForBillingPeriodInternal test calls // computeCurrentMonthForBillingPeriodInternal with the current month map having // some overlap with the previous months. The test then verifies that the @@ -1036,6 +1052,825 @@ func Test_ActivityLog_ComputeCurrentMonth_NamespaceMounts(t *testing.T) { } } +// Test_ActivityLog_ComputeClientsInBillingPeriod tests the computeClientsInBillingPeriod method +// by creating a set of clients across multiple namespaces and mounts, some new and some repeated over 3 months. +// It verifies that the response matches the expected results for billing periods that only include past months, +// as well as for billing periods that include the current month. Also tests a period with no clients. +func Test_ActivityLog_ComputeClientsInBillingPeriod(t *testing.T) { + // Create 20 clients of the 4 client types across 3 namespaces and 3 mounts. + clients := []*activity.EntityRecord{ + {ClientID: "client_1", ClientType: entityActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_2", ClientType: nonEntityTokenActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_3", ClientType: secretSyncActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_4", ClientType: ACMEActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_5", ClientType: entityActivityType, NamespaceID: "ns2", MountAccessor: "mount2"}, + {ClientID: "client_6", ClientType: nonEntityTokenActivityType, NamespaceID: "ns2", MountAccessor: "mount2"}, + {ClientID: "client_7", ClientType: secretSyncActivityType, NamespaceID: "ns3", MountAccessor: "mount3"}, + {ClientID: "client_8", ClientType: ACMEActivityType, NamespaceID: "ns3", MountAccessor: "mount3"}, + + {ClientID: "client_9", ClientType: entityActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_10", ClientType: nonEntityTokenActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_11", ClientType: secretSyncActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_12", ClientType: ACMEActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_13", ClientType: entityActivityType, NamespaceID: "ns2", MountAccessor: "mount2"}, + {ClientID: "client_14", ClientType: nonEntityTokenActivityType, NamespaceID: "ns2", MountAccessor: "mount2"}, + {ClientID: "client_15", ClientType: secretSyncActivityType, NamespaceID: "ns3", MountAccessor: "mount3"}, + {ClientID: "client_16", ClientType: ACMEActivityType, NamespaceID: "ns3", MountAccessor: "mount3"}, + + {ClientID: "client_17", ClientType: entityActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_18", ClientType: nonEntityTokenActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_19", ClientType: secretSyncActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + {ClientID: "client_20", ClientType: ACMEActivityType, NamespaceID: "ns1", MountAccessor: "mount1"}, + } + + // The clients are distributed as follows: + // First month (2 months ago): new clients 1-8 + // Second month (1 month ago): new clients 9-16, repeated clients 1-8 + // Current month: new clients 17-20, repeated clients 1-16 + segments := []struct { + StartTime time.Time + Clients []*activity.EntityRecord + }{ + { + StartTime: timeutil.StartOfMonth(timeutil.MonthsPreviousTo(2, time.Now().UTC())), + Clients: clients[:8], + }, + { + StartTime: timeutil.StartOfMonth(timeutil.MonthsPreviousTo(1, time.Now().UTC())), + Clients: clients[:16], + }, + { + StartTime: timeutil.StartOfMonth(time.Now().UTC()), + Clients: clients, + }, + } + + testCases := []struct { + name string + startTime time.Time + endTime time.Time + expectedNamespaces []*activity.NamespaceRecord + expectedMonths []*activity.MonthRecord + }{ + { + name: "past billing period two months", + startTime: segments[0].StartTime, + endTime: timeutil.EndOfMonth(segments[1].StartTime), + expectedNamespaces: []*activity.NamespaceRecord{ + { + NamespaceID: "ns1", + Entities: 2, + NonEntityTokens: 2, + SecretSyncs: 2, + ACMEClients: 2, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{EntityClients: 2, NonEntityClients: 2, SecretSyncs: 2, ACMEClients: 2}, + }, + }, + }, + { + NamespaceID: "ns2", + Entities: 2, + NonEntityTokens: 2, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{EntityClients: 2, NonEntityClients: 2}, + }, + }, + }, + { + NamespaceID: "ns3", + SecretSyncs: 2, + ACMEClients: 2, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{SecretSyncs: 2, ACMEClients: 2}, + }, + }, + }, + }, + expectedMonths: []*activity.MonthRecord{ + { + Timestamp: segments[0].StartTime.Unix(), + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + }, + NewClients: &activity.NewClientRecord{ + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + }, + }, + }, + { + Timestamp: segments[1].StartTime.Unix(), + Counts: &activity.CountsRecord{ + EntityClients: 4, + NonEntityClients: 4, + SecretSyncs: 4, + ACMEClients: 4, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 2, + ACMEClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 2, + ACMEClients: 2, + }, + }, + }, + }, + }, + NewClients: &activity.NewClientRecord{ + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "current billing period three months", + startTime: segments[0].StartTime, + endTime: timeutil.EndOfMonth(segments[2].StartTime), + expectedNamespaces: []*activity.NamespaceRecord{ + { + NamespaceID: "ns1", + Entities: 3, + NonEntityTokens: 3, + SecretSyncs: 3, + ACMEClients: 3, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{EntityClients: 3, NonEntityClients: 3, SecretSyncs: 3, ACMEClients: 3}, + }, + }, + }, + { + NamespaceID: "ns2", + Entities: 2, + NonEntityTokens: 2, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{EntityClients: 2, NonEntityClients: 2}, + }, + }, + }, + { + NamespaceID: "ns3", + SecretSyncs: 2, + ACMEClients: 2, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{SecretSyncs: 2, ACMEClients: 2}, + }, + }, + }, + }, + expectedMonths: []*activity.MonthRecord{ + { + Timestamp: segments[0].StartTime.Unix(), + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + }, + NewClients: &activity.NewClientRecord{ + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + }, + }, + }, + { + Timestamp: segments[1].StartTime.Unix(), + Counts: &activity.CountsRecord{ + EntityClients: 4, + NonEntityClients: 4, + SecretSyncs: 4, + ACMEClients: 4, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 2, + ACMEClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 2, + ACMEClients: 2, + }, + }, + }, + }, + }, + NewClients: &activity.NewClientRecord{ + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + SecretSyncs: 2, + ACMEClients: 2, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + }, + }, + }, + { + Timestamp: segments[2].StartTime.Unix(), + Counts: &activity.CountsRecord{ + EntityClients: 5, + NonEntityClients: 5, + SecretSyncs: 5, + ACMEClients: 5, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 3, + NonEntityClients: 3, + SecretSyncs: 3, + ACMEClients: 3, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 3, + NonEntityClients: 3, + SecretSyncs: 3, + ACMEClients: 3, + }, + }, + }, + }, + { + NamespaceID: "ns2", + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"), + Counts: &activity.CountsRecord{ + EntityClients: 2, + NonEntityClients: 2, + }, + }, + }, + }, + { + NamespaceID: "ns3", + Counts: &activity.CountsRecord{ + SecretSyncs: 2, + ACMEClients: 2, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"), + Counts: &activity.CountsRecord{ + SecretSyncs: 2, + ACMEClients: 2, + }, + }, + }, + }, + }, + NewClients: &activity.NewClientRecord{ + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Namespaces: []*activity.MonthlyNamespaceRecord{ + { + NamespaceID: "ns1", + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + Mounts: []*activity.MountRecord{ + { + MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"), + Counts: &activity.CountsRecord{ + EntityClients: 1, + NonEntityClients: 1, + SecretSyncs: 1, + ACMEClients: 1, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "past billing period no activity", + startTime: timeutil.MonthsPreviousTo(5, time.Now().UTC()), + endTime: timeutil.EndOfMonth(timeutil.StartOfPreviousMonth(segments[0].StartTime)), + expectedNamespaces: []*activity.NamespaceRecord{}, + expectedMonths: []*activity.MonthRecord{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + core, _, _ := TestCoreUnsealed(t) + a := core.activityLog + a.SetEnable(true) + ctx := namespace.RootContext(nil) + + // Write segments to storage + for i := 0; i < len(segments); i++ { + writeEntitySegment(t, core, segments[i].StartTime, 0, &activity.EntityActivityLog{Clients: segments[i].Clients}) + } + + // Write intent logs for previous months and create precomputed queries + writeIntentLog(t, core, segments[0].StartTime) + a.SetStartTimestamp(segments[1].StartTime.Unix()) + err := a.precomputedQueryWorker(ctx, nil) + require.NoError(t, err) + expectMissingSegment(t, core, "sys/counters/activity/endofmonth") + + writeIntentLog(t, core, segments[1].StartTime) + a.SetStartTimestamp(segments[2].StartTime.Unix()) + err = a.precomputedQueryWorker(ctx, nil) + require.NoError(t, err) + expectMissingSegment(t, core, "sys/counters/activity/endofmonth") + + // add repeated clients seen till last month to in-memory map + repeatedClients := make(map[string]struct{}) + for _, c := range clients[:16] { + repeatedClients[c.ClientID] = struct{}{} + } + a.SetClientIDsUsageInfo(repeatedClients) + + // Refresh partialMonthClientTracker with current segments + var wg sync.WaitGroup + err = a.refreshFromStoredLog(namespace.RootContext(nil), &wg, time.Now().UTC()) + require.NoError(t, err, "error loading clients") + wg.Wait() + + // Validate computeClientsInBillingPeriod for the specified billing period + pq, err := a.computeClientsInBillingPeriod(ctx, tc.startTime, tc.endTime) + require.NoError(t, err) + require.NotNil(t, pq) + require.Len(t, pq.Namespaces, len(tc.expectedNamespaces)) + equalActivityNamespaceRecords(t, tc.expectedNamespaces, pq.Namespaces) + require.Len(t, pq.Months, len(tc.expectedMonths)) + // Ensure months are in chronological order + sort.SliceStable(pq.Months, func(i, j int) bool { + return pq.Months[i].Timestamp < pq.Months[j].Timestamp + }) + for i := range pq.Months { + require.Equal(t, tc.expectedMonths[i].Timestamp, pq.Months[i].Timestamp) + equalActivityMonthRecords(t, tc.expectedMonths[i], pq.Months[i]) + } + }) + } +} + +// writeIntentLog writes an intent log for the end of a given month +func writeIntentLog(t *testing.T, core *Core, ts time.Time) { + t.Helper() + intent := &ActivityIntentLog{ + PreviousMonth: ts.Unix(), + NextMonth: timeutil.StartOfNextMonth(ts).Unix(), + } + data, err := json.Marshal(intent) + require.NoError(t, err) + WriteToStorage(t, core, "sys/counters/activity/endofmonth", data) +} + // writeEntitySegment writes a single segment file with the given time and index for an entity func writeEntitySegment(t *testing.T, core *Core, ts time.Time, index int, item *activity.EntityActivityLog) { t.Helper()