Export api changes to add first used timestamp column in query period - CE changes (#31422)

* moving ce changes from ent pr

* add changelog

* Update changelog/31422.txt

Co-authored-by: Amir Aslamov <amir.aslamov@hashicorp.com>

---------

Co-authored-by: Amir Aslamov <amir.aslamov@hashicorp.com>

changelog/31422.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+activity: The [activity export API](https://developer.hashicorp.com/vault/api-docs/system/internal-counters#activity-export) response now includes a new timestamp that denotes the first time the client was used within the specified query period.
+```

vault/activity_log.go

@@ -288,9 +288,12 @@ type ActivityLogExportRecord struct {
 	// MountPath is the path of the auth mount associated with the token used
 	MountPath string `json:"mount_path" mapstructure:"mount_path"`
 
-	// TokenCreationTime denotes the time at which the activity occurred formatted using RFC3339
+	// TokenCreationTime denotes the token creation timestamp formatted using RFC3339
 	TokenCreationTime string `json:"token_creation_time" mapstructure:"token_creation_time"`
 
+	// ClientFirstUsedTime denotes the timestamp at which the activity first occurred in the query period formatted using RFC3339
+	ClientFirstUsedTime string `json:"client_first_used_time,omitempty" mapstructure:"client_first_used_time"`
+
 	// Policies are the list of policy names attached to the token used
 	Policies []string `json:"policies" mapstructure:"policies"`
 
@@ -3209,6 +3212,13 @@ func (a *ActivityLog) writeExport(ctx context.Context, rw http.ResponseWriter, f
 				EntityGroupIDs:            []string{},
 			}
 
+			// if a client does not have usage time (clients used before upgrade to 1.21 and not seen yet after the upgrade),
+			// do not include first used time in response.
+			if e.UsageTime != 0 {
+				clientFirstUsedTimeStamp := time.Unix(e.UsageTime, 0)
+				record.ClientFirstUsedTime = clientFirstUsedTimeStamp.UTC().Format(time.RFC3339)
+			}
+
 			if e.MountAccessor != "" {
 				cacheKey := e.NamespaceID + mountPathIdentity
 
@@ -3484,6 +3494,7 @@ func baseActivityExportCSVHeader() []string {
 		"mount_path",
 		"mount_type",
 		"token_creation_time",
+		"client_first_used_time",
 	}
 }
 

vault/external_tests/activity_testonly/activity_testonly_test.go

@@ -410,3 +410,70 @@ path "sys/internal/counters/activity/export" {
 	require.NoError(t, err)
 	require.Len(t, clients, 10)
 }
+
+// Test_ActivityLog_Export_FirstUsedTime ensures that the export API returns the timestamp the
+// clients were first seen in the query period for ClientFirstUsedTime field.
+func Test_ActivityLog_Export_FirstUsedTime(t *testing.T) {
+	timeutil.SkipAtEndOfMonth(t)
+	t.Parallel()
+
+	now := time.Now().UTC()
+
+	var err error
+
+	cluster := minimal.NewTestSoloCluster(t, nil)
+	client := cluster.Cores[0].Client
+	_, err = client.Logical().Write("sys/internal/counters/config", map[string]interface{}{
+		"enabled": "enable",
+	})
+	require.NoError(t, err)
+
+	// create new clients and repeated clients for every month.
+	// this is to verify that only the first used timestamp appears in the response
+	_, err = clientcountutil.NewActivityLogData(client).
+		NewPreviousMonthData(2).
+		NewClientsSeen(4).
+		NewPreviousMonthData(1).
+		NewClientsSeen(6).
+		RepeatedClientsSeen(4).
+		NewCurrentMonthData().
+		NewClientsSeen(10).
+		RepeatedClientsSeen(10).
+		Write(context.Background(), generation.WriteOptions_WRITE_ENTITIES)
+
+	require.NoError(t, err)
+
+	startTime := timeutil.StartOfMonth(timeutil.MonthsPreviousTo(3, now))
+
+	// Call export api
+	clients, err := getJSONExport(t, client, startTime, now)
+	require.NoError(t, err)
+
+	// total number of clients present in the query period
+	require.Len(t, clients, 20)
+
+	clientCountByFirstUsedMonth := make(map[time.Time]int)
+	currMonthStart := timeutil.StartOfMonth(now.UTC())
+	oneMonthAgoStart := timeutil.StartOfPreviousMonth(currMonthStart)
+	twoMonthsAgoStart := timeutil.StartOfPreviousMonth(oneMonthAgoStart)
+
+	// count number of clients by month the clients the first seen
+	// this actually corresponds to new clients added every month
+	// since clientcountutil assigns a random time for usage time we can check by month here as we do not know the exact expected usage time
+	for _, clientDetails := range clients {
+		require.NotEmpty(t, clientDetails.ClientFirstUsedTime)
+		parsedTime, err := time.Parse(time.RFC3339, clientDetails.ClientFirstUsedTime) // RFC3339 handles 'Z' for UTC
+		require.NoError(t, err)
+		startOfMonth := timeutil.StartOfMonth(parsedTime)
+		clientCountByFirstUsedMonth[startOfMonth]++
+	}
+
+	// 4 clients were first seen two months ago
+	require.Equal(t, clientCountByFirstUsedMonth[twoMonthsAgoStart], 4)
+
+	// 6 new clients were first seen one month ago
+	require.Equal(t, clientCountByFirstUsedMonth[oneMonthAgoStart], 6)
+
+	// 10 clients were first seen in the current month
+	require.Equal(t, clientCountByFirstUsedMonth[currMonthStart], 10)
+}

vault/logical_system_activity_write_testonly.go

@@ -357,8 +357,9 @@ func (m *multipleMonthsActivityClients) addRepeatedClients(monthsAgo int32, c *g
 
 	for _, client := range repeatedFrom.clients {
 		if c.ClientType == client.ClientType && mountAccessor == client.MountAccessor && c.Namespace == client.NamespaceID {
-			client.UsageTime = usageTime.Unix()
-			addingTo.addEntityRecord(client, segmentIndex)
+			repeatedClient := *client
+			repeatedClient.UsageTime = usageTime.Unix()
+			addingTo.addEntityRecord(&repeatedClient, segmentIndex)
 			numClients--
 			if numClients == 0 {
 				break

vault/logical_system_activity_write_testonly_test.go

@@ -352,26 +352,42 @@ func Test_multipleMonthsActivityClients_addRepeatedClients(t *testing.T) {
 	month2Clients := m.months[2].clients
 	month1Clients := m.months[1].clients
 
+	// checks if all the clients in "containsClients" array exists in "allClients" array
+	hasClients := func(allClients []*activity.EntityRecord, containsClients []*activity.EntityRecord) bool {
+		allClientsList := make(map[string]struct{})
+
+		for _, client := range allClients {
+			allClientsList[client.ClientID] = struct{}{}
+		}
+
+		for _, client := range containsClients {
+			if _, exists := allClientsList[client.ClientID]; !exists {
+				return false
+			}
+		}
+		return true
+	}
+
 	thisMonth := m.months[0]
 	// this will match the first client in month 1
 	require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, Repeated: true}, defaultMount, nil, time.Now().UTC()))
-	require.Contains(t, month1Clients, thisMonth.clients[0])
+	require.True(t, hasClients(month1Clients, []*activity.EntityRecord{thisMonth.clients[0]}))
 
 	// this will match the 3rd client in month 1
 	require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, Repeated: true, ClientType: "non-entity"}, defaultMount, nil, time.Now().UTC()))
-	require.Equal(t, month1Clients[2], thisMonth.clients[1])
+	require.True(t, hasClients([]*activity.EntityRecord{month1Clients[2]}, []*activity.EntityRecord{thisMonth.clients[1]}))
 
 	// this will match the first two clients in month 1
 	require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 2, Repeated: true}, defaultMount, nil, time.Now().UTC()))
-	require.Equal(t, month1Clients[0:2], thisMonth.clients[2:4])
+	require.True(t, hasClients(month1Clients[0:2], thisMonth.clients[2:4]))
 
 	// this will match the first client in month 2
 	require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2}, "identity", nil, time.Now().UTC()))
-	require.Equal(t, month2Clients[0], thisMonth.clients[4])
+	require.True(t, hasClients([]*activity.EntityRecord{month2Clients[0]}, []*activity.EntityRecord{thisMonth.clients[4]}))
 
 	// this will match the 3rd client in month 2
 	require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2, Namespace: "other_ns"}, defaultMount, nil, time.Now().UTC()))
-	require.Equal(t, month2Clients[2], thisMonth.clients[5])
+	require.True(t, hasClients([]*activity.EntityRecord{month2Clients[2]}, []*activity.EntityRecord{thisMonth.clients[5]}))
 
 	require.Error(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2, Namespace: "other_ns"}, "other_mount", nil, time.Now().UTC()))
 }