mirror of
https://github.com/hashicorp/vault.git
synced 2025-09-19 21:01:09 +02:00
Backport [VAULT-38467] Cumulative namespace client count calculation into ce/main (#9038)
* [VAULT-38467] Cumulative namespace client count calculation (#8650) * porting over changes from spike * refactor * move cumulativeCount to new ent file * move struct * add tests * add deleted namespace test * expand comment and remove debug log * make sure inputs are aligned to month start/end * address comments and fix test * fix test * fix test --------- Co-authored-by: Jenny Deng <jenny.deng@hashicorp.com>
This commit is contained in:
Vault Automation
GitHub
parent
4bb8810b2c
commit
b4b347f0ae
@ -1962,77 +1962,15 @@ func (a *ActivityLog) DefaultStartTime(endTime time.Time) time.Time {
|
||||
}
|
||||
|
||||
func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.Time, limitNamespaces int) (map[string]interface{}, error) {
|
||||
var computePartial bool
|
||||
|
||||
// Change the start time to the beginning of the month, and the end time to be the end
|
||||
// of the month.
|
||||
// Normalize the start time to the beginning of the month, and the end time to be the end of the month.
|
||||
startTime = timeutil.StartOfMonth(startTime)
|
||||
endTime = timeutil.EndOfMonth(endTime)
|
||||
|
||||
// At the max, we only want to return data up until the end of the current month.
|
||||
// Adjust the end time be the current month if a future date has been provided.
|
||||
endOfCurrentMonth := timeutil.EndOfMonth(a.clock.Now().UTC())
|
||||
adjustedEndTime := endTime
|
||||
if endTime.After(endOfCurrentMonth) {
|
||||
adjustedEndTime = endOfCurrentMonth
|
||||
}
|
||||
|
||||
// If the endTime of the query is the current month, request data from the queryStore
|
||||
// with the endTime equal to the end of the last month, and add in the current month
|
||||
// data.
|
||||
precomputedQueryEndTime := adjustedEndTime
|
||||
if timeutil.IsCurrentMonth(adjustedEndTime, a.clock.Now().UTC()) {
|
||||
precomputedQueryEndTime = timeutil.EndOfMonth(timeutil.MonthsPreviousTo(1, timeutil.StartOfMonth(adjustedEndTime)))
|
||||
computePartial = true
|
||||
}
|
||||
|
||||
pq := &activity.PrecomputedQuery{}
|
||||
if startTime.After(precomputedQueryEndTime) && timeutil.IsCurrentMonth(startTime, a.clock.Now().UTC()) {
|
||||
// We're only calculating the partial month client count. Skip the precomputation
|
||||
// get call.
|
||||
pq = &activity.PrecomputedQuery{
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
Namespaces: make([]*activity.NamespaceRecord, 0),
|
||||
Months: make([]*activity.MonthRecord, 0),
|
||||
}
|
||||
} else {
|
||||
storedQuery, err := a.queryStore.Get(ctx, startTime, precomputedQueryEndTime)
|
||||
// Compute the total clients in the billing period, and get the breakdown by namespace.
|
||||
pq, err := a.computeClientsInBillingPeriod(ctx, startTime, endTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if storedQuery == nil {
|
||||
// If the storedQuery is nil, that means there's no historical data to process. But, it's possible there's
|
||||
// still current month data to process, so rather than returning a 204, let's proceed along like we're
|
||||
// just querying the current month.
|
||||
storedQuery = &activity.PrecomputedQuery{
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
Namespaces: make([]*activity.NamespaceRecord, 0),
|
||||
Months: make([]*activity.MonthRecord, 0),
|
||||
}
|
||||
}
|
||||
pq = storedQuery
|
||||
}
|
||||
|
||||
var partialByMonth map[int64]*processMonth
|
||||
if computePartial {
|
||||
// Traverse through current month's activitylog data and group clients
|
||||
// into months and namespaces
|
||||
a.fragmentLock.RLock()
|
||||
partialByMonth, _ = a.populateNamespaceAndMonthlyBreakdowns()
|
||||
a.fragmentLock.RUnlock()
|
||||
|
||||
// Estimate the current month totals. These record contains is complete with all the
|
||||
// current month data, grouped by namespace and mounts
|
||||
currentMonth, err := a.computeCurrentMonthForBillingPeriod(partialByMonth, startTime, adjustedEndTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Combine the existing months precomputed query with the current month data
|
||||
pq.CombineWithCurrentMonth(currentMonth)
|
||||
}
|
||||
|
||||
// Convert the namespace data into a protobuf format that can be returned in the response
|
||||
totalCounts, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, pq.Namespaces)
|
||||
@ -2071,7 +2009,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
|
||||
a.sortActivityLogMonthsResponse(months)
|
||||
|
||||
// Modify the final month output to make response more consumable based on API request
|
||||
months = a.modifyResponseMonths(months, startTime, adjustedEndTime)
|
||||
months = a.modifyResponseMonths(months, startTime, endTime)
|
||||
responseData["months"] = months
|
||||
|
||||
return responseData, nil
|
||||
|
||||
@ -19,6 +19,73 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// computeClientsInBillingPeriod computes the clients in a billing period given a start and end time.
|
||||
// It retrieves the precomputed query data for the specified period, and if the current month is included,
|
||||
// it computes the current month's data by aggregating the activity log fragments. It returns the counts
|
||||
// separated by namespace as well as the total counts for the billing period.
|
||||
func (a *ActivityLog) computeClientsInBillingPeriod(ctx context.Context, startTime time.Time, endTime time.Time) (*activity.PrecomputedQuery, error) {
|
||||
var computePartial bool
|
||||
|
||||
// If the endTime of the query is the current month, request data from the queryStore
|
||||
// with the endTime equal to the end of the last month, and add in the current month
|
||||
// data.
|
||||
precomputedQueryEndTime := endTime
|
||||
if timeutil.IsCurrentMonth(endTime, a.clock.Now().UTC()) {
|
||||
precomputedQueryEndTime = timeutil.EndOfMonth(timeutil.MonthsPreviousTo(1, timeutil.StartOfMonth(endTime)))
|
||||
computePartial = true
|
||||
}
|
||||
|
||||
pq := &activity.PrecomputedQuery{}
|
||||
if startTime.After(precomputedQueryEndTime) && timeutil.IsCurrentMonth(startTime, a.clock.Now().UTC()) {
|
||||
// We're only calculating the partial month client count. Skip the precomputation
|
||||
// get call.
|
||||
pq = &activity.PrecomputedQuery{
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
Namespaces: make([]*activity.NamespaceRecord, 0),
|
||||
Months: make([]*activity.MonthRecord, 0),
|
||||
}
|
||||
} else {
|
||||
storedQuery, err := a.queryStore.Get(ctx, startTime, precomputedQueryEndTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if storedQuery == nil {
|
||||
// If the storedQuery is nil, that means there's no historical data to process. But, it's possible there's
|
||||
// still current month data to process, so rather than returning a 204, let's proceed along like we're
|
||||
// just querying the current month.
|
||||
storedQuery = &activity.PrecomputedQuery{
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
Namespaces: make([]*activity.NamespaceRecord, 0),
|
||||
Months: make([]*activity.MonthRecord, 0),
|
||||
}
|
||||
}
|
||||
pq = storedQuery
|
||||
}
|
||||
|
||||
var partialByMonth map[int64]*processMonth
|
||||
if computePartial {
|
||||
// Traverse through current month's activitylog data and group clients
|
||||
// into months and namespaces
|
||||
a.fragmentLock.RLock()
|
||||
partialByMonth, _ = a.populateNamespaceAndMonthlyBreakdowns()
|
||||
a.fragmentLock.RUnlock()
|
||||
|
||||
// Estimate the current month totals. These record contains is complete with all the
|
||||
// current month data, grouped by namespace and mounts
|
||||
currentMonth, err := a.computeCurrentMonthForBillingPeriod(partialByMonth, startTime, endTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Combine the existing months precomputed query with the current month data
|
||||
pq.CombineWithCurrentMonth(currentMonth)
|
||||
}
|
||||
|
||||
return pq, nil
|
||||
}
|
||||
|
||||
// computeCurrentMonthForBillingPeriod computes the current month's data with respect
|
||||
// to a billing period.
|
||||
func (a *ActivityLog) computeCurrentMonthForBillingPeriod(byMonth map[int64]*processMonth, startTime time.Time, endTime time.Time) (*activity.MonthRecord, error) {
|
||||
|
||||
@ -5,13 +5,16 @@ package vault
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/vault/helper/namespace"
|
||||
"github.com/hashicorp/vault/helper/timeutil"
|
||||
"github.com/hashicorp/vault/vault/activity"
|
||||
"github.com/stretchr/testify/require"
|
||||
@ -40,6 +43,19 @@ func equalActivityMonthRecords(t *testing.T, expected, got *activity.MonthRecord
|
||||
require.Equal(t, expected, got)
|
||||
}
|
||||
|
||||
// equalActivityNamespaceRecords is a helper to sort the namespaces in activity.NamespaceRecord's,
|
||||
// then compare their equality
|
||||
func equalActivityNamespaceRecords(t *testing.T, expected, got []*activity.NamespaceRecord) {
|
||||
t.Helper()
|
||||
sort.SliceStable(expected, func(i, j int) bool {
|
||||
return expected[i].NamespaceID < expected[j].NamespaceID
|
||||
})
|
||||
sort.SliceStable(got, func(i, j int) bool {
|
||||
return got[i].NamespaceID < got[j].NamespaceID
|
||||
})
|
||||
require.Equal(t, expected, got)
|
||||
}
|
||||
|
||||
// Test_ActivityLog_ComputeCurrentMonthForBillingPeriodInternal test calls
|
||||
// computeCurrentMonthForBillingPeriodInternal with the current month map having
|
||||
// some overlap with the previous months. The test then verifies that the
|
||||
@ -1036,6 +1052,825 @@ func Test_ActivityLog_ComputeCurrentMonth_NamespaceMounts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test_ActivityLog_ComputeClientsInBillingPeriod tests the computeClientsInBillingPeriod method
|
||||
// by creating a set of clients across multiple namespaces and mounts, some new and some repeated over 3 months.
|
||||
// It verifies that the response matches the expected results for billing periods that only include past months,
|
||||
// as well as for billing periods that include the current month. Also tests a period with no clients.
|
||||
func Test_ActivityLog_ComputeClientsInBillingPeriod(t *testing.T) {
|
||||
// Create 20 clients of the 4 client types across 3 namespaces and 3 mounts.
|
||||
clients := []*activity.EntityRecord{
|
||||
{ClientID: "client_1", ClientType: entityActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_2", ClientType: nonEntityTokenActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_3", ClientType: secretSyncActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_4", ClientType: ACMEActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_5", ClientType: entityActivityType, NamespaceID: "ns2", MountAccessor: "mount2"},
|
||||
{ClientID: "client_6", ClientType: nonEntityTokenActivityType, NamespaceID: "ns2", MountAccessor: "mount2"},
|
||||
{ClientID: "client_7", ClientType: secretSyncActivityType, NamespaceID: "ns3", MountAccessor: "mount3"},
|
||||
{ClientID: "client_8", ClientType: ACMEActivityType, NamespaceID: "ns3", MountAccessor: "mount3"},
|
||||
|
||||
{ClientID: "client_9", ClientType: entityActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_10", ClientType: nonEntityTokenActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_11", ClientType: secretSyncActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_12", ClientType: ACMEActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_13", ClientType: entityActivityType, NamespaceID: "ns2", MountAccessor: "mount2"},
|
||||
{ClientID: "client_14", ClientType: nonEntityTokenActivityType, NamespaceID: "ns2", MountAccessor: "mount2"},
|
||||
{ClientID: "client_15", ClientType: secretSyncActivityType, NamespaceID: "ns3", MountAccessor: "mount3"},
|
||||
{ClientID: "client_16", ClientType: ACMEActivityType, NamespaceID: "ns3", MountAccessor: "mount3"},
|
||||
|
||||
{ClientID: "client_17", ClientType: entityActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_18", ClientType: nonEntityTokenActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_19", ClientType: secretSyncActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
{ClientID: "client_20", ClientType: ACMEActivityType, NamespaceID: "ns1", MountAccessor: "mount1"},
|
||||
}
|
||||
|
||||
// The clients are distributed as follows:
|
||||
// First month (2 months ago): new clients 1-8
|
||||
// Second month (1 month ago): new clients 9-16, repeated clients 1-8
|
||||
// Current month: new clients 17-20, repeated clients 1-16
|
||||
segments := []struct {
|
||||
StartTime time.Time
|
||||
Clients []*activity.EntityRecord
|
||||
}{
|
||||
{
|
||||
StartTime: timeutil.StartOfMonth(timeutil.MonthsPreviousTo(2, time.Now().UTC())),
|
||||
Clients: clients[:8],
|
||||
},
|
||||
{
|
||||
StartTime: timeutil.StartOfMonth(timeutil.MonthsPreviousTo(1, time.Now().UTC())),
|
||||
Clients: clients[:16],
|
||||
},
|
||||
{
|
||||
StartTime: timeutil.StartOfMonth(time.Now().UTC()),
|
||||
Clients: clients,
|
||||
},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
startTime time.Time
|
||||
endTime time.Time
|
||||
expectedNamespaces []*activity.NamespaceRecord
|
||||
expectedMonths []*activity.MonthRecord
|
||||
}{
|
||||
{
|
||||
name: "past billing period two months",
|
||||
startTime: segments[0].StartTime,
|
||||
endTime: timeutil.EndOfMonth(segments[1].StartTime),
|
||||
expectedNamespaces: []*activity.NamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Entities: 2,
|
||||
NonEntityTokens: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{EntityClients: 2, NonEntityClients: 2, SecretSyncs: 2, ACMEClients: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Entities: 2,
|
||||
NonEntityTokens: 2,
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{EntityClients: 2, NonEntityClients: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{SecretSyncs: 2, ACMEClients: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedMonths: []*activity.MonthRecord{
|
||||
{
|
||||
Timestamp: segments[0].StartTime.Unix(),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NewClients: &activity.NewClientRecord{
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Timestamp: segments[1].StartTime.Unix(),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 4,
|
||||
NonEntityClients: 4,
|
||||
SecretSyncs: 4,
|
||||
ACMEClients: 4,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NewClients: &activity.NewClientRecord{
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "current billing period three months",
|
||||
startTime: segments[0].StartTime,
|
||||
endTime: timeutil.EndOfMonth(segments[2].StartTime),
|
||||
expectedNamespaces: []*activity.NamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Entities: 3,
|
||||
NonEntityTokens: 3,
|
||||
SecretSyncs: 3,
|
||||
ACMEClients: 3,
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{EntityClients: 3, NonEntityClients: 3, SecretSyncs: 3, ACMEClients: 3},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Entities: 2,
|
||||
NonEntityTokens: 2,
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{EntityClients: 2, NonEntityClients: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{SecretSyncs: 2, ACMEClients: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedMonths: []*activity.MonthRecord{
|
||||
{
|
||||
Timestamp: segments[0].StartTime.Unix(),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NewClients: &activity.NewClientRecord{
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Timestamp: segments[1].StartTime.Unix(),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 4,
|
||||
NonEntityClients: 4,
|
||||
SecretSyncs: 4,
|
||||
ACMEClients: 4,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NewClients: &activity.NewClientRecord{
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Timestamp: segments[2].StartTime.Unix(),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 5,
|
||||
NonEntityClients: 5,
|
||||
SecretSyncs: 5,
|
||||
ACMEClients: 5,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 3,
|
||||
NonEntityClients: 3,
|
||||
SecretSyncs: 3,
|
||||
ACMEClients: 3,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 3,
|
||||
NonEntityClients: 3,
|
||||
SecretSyncs: 3,
|
||||
ACMEClients: 3,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns2",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount2"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 2,
|
||||
NonEntityClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
NamespaceID: "ns3",
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount3"),
|
||||
Counts: &activity.CountsRecord{
|
||||
SecretSyncs: 2,
|
||||
ACMEClients: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NewClients: &activity.NewClientRecord{
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Namespaces: []*activity.MonthlyNamespaceRecord{
|
||||
{
|
||||
NamespaceID: "ns1",
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
Mounts: []*activity.MountRecord{
|
||||
{
|
||||
MountPath: fmt.Sprintf(DeletedMountFmt, "mount1"),
|
||||
Counts: &activity.CountsRecord{
|
||||
EntityClients: 1,
|
||||
NonEntityClients: 1,
|
||||
SecretSyncs: 1,
|
||||
ACMEClients: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "past billing period no activity",
|
||||
startTime: timeutil.MonthsPreviousTo(5, time.Now().UTC()),
|
||||
endTime: timeutil.EndOfMonth(timeutil.StartOfPreviousMonth(segments[0].StartTime)),
|
||||
expectedNamespaces: []*activity.NamespaceRecord{},
|
||||
expectedMonths: []*activity.MonthRecord{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
core, _, _ := TestCoreUnsealed(t)
|
||||
a := core.activityLog
|
||||
a.SetEnable(true)
|
||||
ctx := namespace.RootContext(nil)
|
||||
|
||||
// Write segments to storage
|
||||
for i := 0; i < len(segments); i++ {
|
||||
writeEntitySegment(t, core, segments[i].StartTime, 0, &activity.EntityActivityLog{Clients: segments[i].Clients})
|
||||
}
|
||||
|
||||
// Write intent logs for previous months and create precomputed queries
|
||||
writeIntentLog(t, core, segments[0].StartTime)
|
||||
a.SetStartTimestamp(segments[1].StartTime.Unix())
|
||||
err := a.precomputedQueryWorker(ctx, nil)
|
||||
require.NoError(t, err)
|
||||
expectMissingSegment(t, core, "sys/counters/activity/endofmonth")
|
||||
|
||||
writeIntentLog(t, core, segments[1].StartTime)
|
||||
a.SetStartTimestamp(segments[2].StartTime.Unix())
|
||||
err = a.precomputedQueryWorker(ctx, nil)
|
||||
require.NoError(t, err)
|
||||
expectMissingSegment(t, core, "sys/counters/activity/endofmonth")
|
||||
|
||||
// add repeated clients seen till last month to in-memory map
|
||||
repeatedClients := make(map[string]struct{})
|
||||
for _, c := range clients[:16] {
|
||||
repeatedClients[c.ClientID] = struct{}{}
|
||||
}
|
||||
a.SetClientIDsUsageInfo(repeatedClients)
|
||||
|
||||
// Refresh partialMonthClientTracker with current segments
|
||||
var wg sync.WaitGroup
|
||||
err = a.refreshFromStoredLog(namespace.RootContext(nil), &wg, time.Now().UTC())
|
||||
require.NoError(t, err, "error loading clients")
|
||||
wg.Wait()
|
||||
|
||||
// Validate computeClientsInBillingPeriod for the specified billing period
|
||||
pq, err := a.computeClientsInBillingPeriod(ctx, tc.startTime, tc.endTime)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, pq)
|
||||
require.Len(t, pq.Namespaces, len(tc.expectedNamespaces))
|
||||
equalActivityNamespaceRecords(t, tc.expectedNamespaces, pq.Namespaces)
|
||||
require.Len(t, pq.Months, len(tc.expectedMonths))
|
||||
// Ensure months are in chronological order
|
||||
sort.SliceStable(pq.Months, func(i, j int) bool {
|
||||
return pq.Months[i].Timestamp < pq.Months[j].Timestamp
|
||||
})
|
||||
for i := range pq.Months {
|
||||
require.Equal(t, tc.expectedMonths[i].Timestamp, pq.Months[i].Timestamp)
|
||||
equalActivityMonthRecords(t, tc.expectedMonths[i], pq.Months[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeIntentLog writes an intent log for the end of a given month
|
||||
func writeIntentLog(t *testing.T, core *Core, ts time.Time) {
|
||||
t.Helper()
|
||||
intent := &ActivityIntentLog{
|
||||
PreviousMonth: ts.Unix(),
|
||||
NextMonth: timeutil.StartOfNextMonth(ts).Unix(),
|
||||
}
|
||||
data, err := json.Marshal(intent)
|
||||
require.NoError(t, err)
|
||||
WriteToStorage(t, core, "sys/counters/activity/endofmonth", data)
|
||||
}
|
||||
|
||||
// writeEntitySegment writes a single segment file with the given time and index for an entity
|
||||
func writeEntitySegment(t *testing.T, core *Core, ts time.Time, index int, item *activity.EntityActivityLog) {
|
||||
t.Helper()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user