diff --git a/changelog/31422.txt b/changelog/31422.txt new file mode 100644 index 0000000000..e34df8321a --- /dev/null +++ b/changelog/31422.txt @@ -0,0 +1,3 @@ +```release-note:improvement +activity: The [activity export API](https://developer.hashicorp.com/vault/api-docs/system/internal-counters#activity-export) response now includes a new timestamp that denotes the first time the client was used within the specified query period. +``` \ No newline at end of file diff --git a/vault/activity_log.go b/vault/activity_log.go index d6c10af271..6f50e2ee9a 100644 --- a/vault/activity_log.go +++ b/vault/activity_log.go @@ -288,9 +288,12 @@ type ActivityLogExportRecord struct { // MountPath is the path of the auth mount associated with the token used MountPath string `json:"mount_path" mapstructure:"mount_path"` - // TokenCreationTime denotes the time at which the activity occurred formatted using RFC3339 + // TokenCreationTime denotes the token creation timestamp formatted using RFC3339 TokenCreationTime string `json:"token_creation_time" mapstructure:"token_creation_time"` + // ClientFirstUsedTime denotes the timestamp at which the activity first occurred in the query period formatted using RFC3339 + ClientFirstUsedTime string `json:"client_first_used_time,omitempty" mapstructure:"client_first_used_time"` + // Policies are the list of policy names attached to the token used Policies []string `json:"policies" mapstructure:"policies"` @@ -3209,6 +3212,13 @@ func (a *ActivityLog) writeExport(ctx context.Context, rw http.ResponseWriter, f EntityGroupIDs: []string{}, } + // if a client does not have usage time (clients used before upgrade to 1.21 and not seen yet after the upgrade), + // do not include first used time in response. + if e.UsageTime != 0 { + clientFirstUsedTimeStamp := time.Unix(e.UsageTime, 0) + record.ClientFirstUsedTime = clientFirstUsedTimeStamp.UTC().Format(time.RFC3339) + } + if e.MountAccessor != "" { cacheKey := e.NamespaceID + mountPathIdentity @@ -3484,6 +3494,7 @@ func baseActivityExportCSVHeader() []string { "mount_path", "mount_type", "token_creation_time", + "client_first_used_time", } } diff --git a/vault/external_tests/activity_testonly/activity_testonly_test.go b/vault/external_tests/activity_testonly/activity_testonly_test.go index 44a66d8a03..c4011919c7 100644 --- a/vault/external_tests/activity_testonly/activity_testonly_test.go +++ b/vault/external_tests/activity_testonly/activity_testonly_test.go @@ -410,3 +410,70 @@ path "sys/internal/counters/activity/export" { require.NoError(t, err) require.Len(t, clients, 10) } + +// Test_ActivityLog_Export_FirstUsedTime ensures that the export API returns the timestamp the +// clients were first seen in the query period for ClientFirstUsedTime field. +func Test_ActivityLog_Export_FirstUsedTime(t *testing.T) { + timeutil.SkipAtEndOfMonth(t) + t.Parallel() + + now := time.Now().UTC() + + var err error + + cluster := minimal.NewTestSoloCluster(t, nil) + client := cluster.Cores[0].Client + _, err = client.Logical().Write("sys/internal/counters/config", map[string]interface{}{ + "enabled": "enable", + }) + require.NoError(t, err) + + // create new clients and repeated clients for every month. + // this is to verify that only the first used timestamp appears in the response + _, err = clientcountutil.NewActivityLogData(client). + NewPreviousMonthData(2). + NewClientsSeen(4). + NewPreviousMonthData(1). + NewClientsSeen(6). + RepeatedClientsSeen(4). + NewCurrentMonthData(). + NewClientsSeen(10). + RepeatedClientsSeen(10). + Write(context.Background(), generation.WriteOptions_WRITE_ENTITIES) + + require.NoError(t, err) + + startTime := timeutil.StartOfMonth(timeutil.MonthsPreviousTo(3, now)) + + // Call export api + clients, err := getJSONExport(t, client, startTime, now) + require.NoError(t, err) + + // total number of clients present in the query period + require.Len(t, clients, 20) + + clientCountByFirstUsedMonth := make(map[time.Time]int) + currMonthStart := timeutil.StartOfMonth(now.UTC()) + oneMonthAgoStart := timeutil.StartOfPreviousMonth(currMonthStart) + twoMonthsAgoStart := timeutil.StartOfPreviousMonth(oneMonthAgoStart) + + // count number of clients by month the clients the first seen + // this actually corresponds to new clients added every month + // since clientcountutil assigns a random time for usage time we can check by month here as we do not know the exact expected usage time + for _, clientDetails := range clients { + require.NotEmpty(t, clientDetails.ClientFirstUsedTime) + parsedTime, err := time.Parse(time.RFC3339, clientDetails.ClientFirstUsedTime) // RFC3339 handles 'Z' for UTC + require.NoError(t, err) + startOfMonth := timeutil.StartOfMonth(parsedTime) + clientCountByFirstUsedMonth[startOfMonth]++ + } + + // 4 clients were first seen two months ago + require.Equal(t, clientCountByFirstUsedMonth[twoMonthsAgoStart], 4) + + // 6 new clients were first seen one month ago + require.Equal(t, clientCountByFirstUsedMonth[oneMonthAgoStart], 6) + + // 10 clients were first seen in the current month + require.Equal(t, clientCountByFirstUsedMonth[currMonthStart], 10) +} diff --git a/vault/logical_system_activity_write_testonly.go b/vault/logical_system_activity_write_testonly.go index 0636361ab4..3796585df0 100644 --- a/vault/logical_system_activity_write_testonly.go +++ b/vault/logical_system_activity_write_testonly.go @@ -357,8 +357,9 @@ func (m *multipleMonthsActivityClients) addRepeatedClients(monthsAgo int32, c *g for _, client := range repeatedFrom.clients { if c.ClientType == client.ClientType && mountAccessor == client.MountAccessor && c.Namespace == client.NamespaceID { - client.UsageTime = usageTime.Unix() - addingTo.addEntityRecord(client, segmentIndex) + repeatedClient := *client + repeatedClient.UsageTime = usageTime.Unix() + addingTo.addEntityRecord(&repeatedClient, segmentIndex) numClients-- if numClients == 0 { break diff --git a/vault/logical_system_activity_write_testonly_test.go b/vault/logical_system_activity_write_testonly_test.go index 6ea429339b..0179d3d4fb 100644 --- a/vault/logical_system_activity_write_testonly_test.go +++ b/vault/logical_system_activity_write_testonly_test.go @@ -352,26 +352,42 @@ func Test_multipleMonthsActivityClients_addRepeatedClients(t *testing.T) { month2Clients := m.months[2].clients month1Clients := m.months[1].clients + // checks if all the clients in "containsClients" array exists in "allClients" array + hasClients := func(allClients []*activity.EntityRecord, containsClients []*activity.EntityRecord) bool { + allClientsList := make(map[string]struct{}) + + for _, client := range allClients { + allClientsList[client.ClientID] = struct{}{} + } + + for _, client := range containsClients { + if _, exists := allClientsList[client.ClientID]; !exists { + return false + } + } + return true + } + thisMonth := m.months[0] // this will match the first client in month 1 require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, Repeated: true}, defaultMount, nil, time.Now().UTC())) - require.Contains(t, month1Clients, thisMonth.clients[0]) + require.True(t, hasClients(month1Clients, []*activity.EntityRecord{thisMonth.clients[0]})) // this will match the 3rd client in month 1 require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, Repeated: true, ClientType: "non-entity"}, defaultMount, nil, time.Now().UTC())) - require.Equal(t, month1Clients[2], thisMonth.clients[1]) + require.True(t, hasClients([]*activity.EntityRecord{month1Clients[2]}, []*activity.EntityRecord{thisMonth.clients[1]})) // this will match the first two clients in month 1 require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 2, Repeated: true}, defaultMount, nil, time.Now().UTC())) - require.Equal(t, month1Clients[0:2], thisMonth.clients[2:4]) + require.True(t, hasClients(month1Clients[0:2], thisMonth.clients[2:4])) // this will match the first client in month 2 require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2}, "identity", nil, time.Now().UTC())) - require.Equal(t, month2Clients[0], thisMonth.clients[4]) + require.True(t, hasClients([]*activity.EntityRecord{month2Clients[0]}, []*activity.EntityRecord{thisMonth.clients[4]})) // this will match the 3rd client in month 2 require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2, Namespace: "other_ns"}, defaultMount, nil, time.Now().UTC())) - require.Equal(t, month2Clients[2], thisMonth.clients[5]) + require.True(t, hasClients([]*activity.EntityRecord{month2Clients[2]}, []*activity.EntityRecord{thisMonth.clients[5]})) require.Error(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2, Namespace: "other_ns"}, "other_mount", nil, time.Now().UTC())) }