Audit the client token accessors (#2037)

audit/format.go

@@ -64,9 +64,18 @@ func (f *AuditFormatter) FormatRequest(
 		if err := Hash(config.Salt, auth); err != nil {
 			return err
 		}
+
+		// Cache and restore accessor in the request
+		var clientTokenAccessor string
+		if !config.HMACAccessor && req != nil && req.ClientTokenAccessor != "" {
+			clientTokenAccessor = req.ClientTokenAccessor
+		}
 		if err := Hash(config.Salt, req); err != nil {
 			return err
 		}
+		if clientTokenAccessor != "" {
+			req.ClientTokenAccessor = clientTokenAccessor
+		}
 	}
 
 	// If auth is nil, make an empty one
@@ -89,13 +98,14 @@ func (f *AuditFormatter) FormatRequest(
 		},
 
 		Request: AuditRequest{
-			ID:          req.ID,
+			ID:                  req.ID,
-			ClientToken: req.ClientToken,
+			ClientToken:         req.ClientToken,
-			Operation:   req.Operation,
+			ClientTokenAccessor: req.ClientTokenAccessor,
-			Path:        req.Path,
+			Operation:           req.Operation,
-			Data:        req.Data,
+			Path:                req.Path,
-			RemoteAddr:  getRemoteAddr(req),
+			Data:                req.Data,
-			WrapTTL:     int(req.WrapTTL / time.Second),
+			RemoteAddr:          getRemoteAddr(req),
+			WrapTTL:             int(req.WrapTTL / time.Second),
 		},
 	}
 
@@ -167,9 +177,17 @@ func (f *AuditFormatter) FormatResponse(
 			auth.Accessor = accessor
 		}
 
+		// Cache and restore accessor in the request
+		var clientTokenAccessor string
+		if !config.HMACAccessor && req != nil && req.ClientTokenAccessor != "" {
+			clientTokenAccessor = req.ClientTokenAccessor
+		}
 		if err := Hash(config.Salt, req); err != nil {
 			return err
 		}
+		if clientTokenAccessor != "" {
+			req.ClientTokenAccessor = clientTokenAccessor
+		}
 
 		// Cache and restore accessor in the response
 		accessor = ""
@@ -241,13 +259,14 @@ func (f *AuditFormatter) FormatResponse(
 		},
 
 		Request: AuditRequest{
-			ID:          req.ID,
+			ID:                  req.ID,
-			ClientToken: req.ClientToken,
+			ClientToken:         req.ClientToken,
-			Operation:   req.Operation,
+			ClientTokenAccessor: req.ClientTokenAccessor,
-			Path:        req.Path,
+			Operation:           req.Operation,
-			Data:        req.Data,
+			Path:                req.Path,
-			RemoteAddr:  getRemoteAddr(req),
+			Data:                req.Data,
-			WrapTTL:     int(req.WrapTTL / time.Second),
+			RemoteAddr:          getRemoteAddr(req),
+			WrapTTL:             int(req.WrapTTL / time.Second),
 		},
 
 		Response: AuditResponse{
@@ -286,13 +305,14 @@ type AuditResponseEntry struct {
 }
 
 type AuditRequest struct {
-	ID          string                 `json:"id"`
+	ID                  string                 `json:"id"`
-	Operation   logical.Operation      `json:"operation"`
+	Operation           logical.Operation      `json:"operation"`
-	ClientToken string                 `json:"client_token"`
+	ClientToken         string                 `json:"client_token"`
-	Path        string                 `json:"path"`
+	ClientTokenAccessor string                 `json:"client_token_accessor"`
-	Data        map[string]interface{} `json:"data"`
+	Path                string                 `json:"path"`
-	RemoteAddr  string                 `json:"remote_address"`
+	Data                map[string]interface{} `json:"data"`
-	WrapTTL     int                    `json:"wrap_ttl"`
+	RemoteAddr          string                 `json:"remote_address"`
+	WrapTTL             int                    `json:"wrap_ttl"`
 }
 
 type AuditResponse struct {

audit/format_jsonx_test.go

@@ -32,7 +32,7 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 			},
 			errors.New("this is an error"),
 			"",
-			`<json:object name="auth"><json:string name="accessor"></json:string><json:string name="client_token"></json:string><json:string name="display_name"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:null name="data" /><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
+			`<json:object name="auth"><json:string name="accessor"></json:string><json:string name="client_token"></json:string><json:string name="display_name"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
 		},
 	}
 

audit/hashstructure.go

@@ -49,6 +49,10 @@ func Hash(salter *salt.Salt, raw interface{}) error {
 			s.ClientToken = fn(s.ClientToken)
 		}
 
+		if s.ClientTokenAccessor != "" {
+			s.ClientTokenAccessor = fn(s.ClientTokenAccessor)
+		}
+
 		data, err := HashStructure(s.Data, fn)
 		if err != nil {
 			return err

http/handler.go

@@ -245,10 +245,19 @@ func respondStandby(core *vault.Core, w http.ResponseWriter, reqURL *url.URL) {
 }
 
 // requestAuth adds the token to the logical.Request if it exists.
-func requestAuth(r *http.Request, req *logical.Request) *logical.Request {
+func requestAuth(core *vault.Core, r *http.Request, req *logical.Request) *logical.Request {
 	// Attach the header value if we have it
 	if v := r.Header.Get(AuthHeaderName); v != "" {
 		req.ClientToken = v
+
+		// Also attach the accessor if we have it. This doesn't fail if it
+		// doesn't exist because the request may be to an unauthenticated
+		// endpoint/login endpoint where a bad current token doesn't matter, or
+		// a token from a Vault version pre-accessors.
+		te, err := core.LookupToken(v)
+		if err == nil && te != nil {
+			req.ClientTokenAccessor = te.Accessor
+		}
 	}
 
 	return req

http/help.go

@@ -27,11 +27,13 @@ func handleHelp(core *vault.Core, w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	resp, err := core.HandleRequest(requestAuth(req, &logical.Request{
+	lreq := requestAuth(core, req, &logical.Request{
 		Operation:  logical.HelpOperation,
 		Path:       path,
 		Connection: getConnection(req),
-	}))
+	})
+
+	resp, err := core.HandleRequest(lreq)
 	if err != nil {
 		respondError(w, http.StatusInternalServerError, err)
 		return

http/logical.go

@@ -16,7 +16,7 @@ import (
 
 type PrepareRequestFunc func(*vault.Core, *logical.Request) error
 
-func buildLogicalRequest(w http.ResponseWriter, r *http.Request) (*logical.Request, int, error) {
+func buildLogicalRequest(core *vault.Core, w http.ResponseWriter, r *http.Request) (*logical.Request, int, error) {
 	// Determine the path...
 	if !strings.HasPrefix(r.URL.Path, "/v1/") {
 		return nil, http.StatusNotFound, nil
@@ -72,13 +72,14 @@ func buildLogicalRequest(w http.ResponseWriter, r *http.Request) (*logical.Reque
 		return nil, http.StatusBadRequest, errwrap.Wrapf("failed to generate identifier for the request: {{err}}", err)
 	}
 
-	req := requestAuth(r, &logical.Request{
+	req := requestAuth(core, r, &logical.Request{
 		ID:         request_id,
 		Operation:  op,
 		Path:       path,
 		Data:       data,
 		Connection: getConnection(r),
 	})
+
 	req, err = requestWrapTTL(r, req)
 	if err != nil {
 		return nil, http.StatusBadRequest, errwrap.Wrapf("error parsing X-Vault-Wrap-TTL header: {{err}}", err)
@@ -89,7 +90,7 @@ func buildLogicalRequest(w http.ResponseWriter, r *http.Request) (*logical.Reque
 
 func handleLogical(core *vault.Core, dataOnly bool, prepareRequestCallback PrepareRequestFunc) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		req, statusCode, err := buildLogicalRequest(w, r)
+		req, statusCode, err := buildLogicalRequest(core, w, r)
 		if err != nil || statusCode != 0 {
 			respondError(w, statusCode, err)
 			return

http/sys_seal.go

@@ -15,7 +15,7 @@ import (
 
 func handleSysSeal(core *vault.Core) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		req, statusCode, err := buildLogicalRequest(w, r)
+		req, statusCode, err := buildLogicalRequest(core, w, r)
 		if err != nil || statusCode != 0 {
 			respondError(w, statusCode, err)
 			return
@@ -40,7 +40,7 @@ func handleSysSeal(core *vault.Core) http.Handler {
 
 func handleSysStepDown(core *vault.Core) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		req, statusCode, err := buildLogicalRequest(w, r)
+		req, statusCode, err := buildLogicalRequest(core, w, r)
 		if err != nil || statusCode != 0 {
 			respondError(w, statusCode, err)
 			return

logical/request.go

@@ -47,6 +47,10 @@ type Request struct {
 	// hashed.
 	ClientToken string `json:"client_token" structs:"client_token" mapstructure:"client_token"`
 
+	// ClientTokenAccessor is provided to the core so that the it can get
+	// logged as part of request audit logging.
+	ClientTokenAccessor string `json:"client_token_accessor" structs:"client_token_accessor" mapstructure:"client_token_accessor"`
+
 	// DisplayName is provided to the logical backend to help associate
 	// dynamic secrets with the source entity. This is not a sensitive
 	// name, but is useful for operators.

vault/core.go

@@ -13,12 +13,12 @@ import (
 	"sync"
 	"time"
 
+	"github.com/armon/go-metrics"
 	log "github.com/mgutz/logxi/v1"
 
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 
-	"github.com/armon/go-metrics"
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-uuid"
@@ -492,6 +492,23 @@ func (c *Core) Shutdown() error {
 	return c.sealInternal()
 }
 
+// LookupToken returns the properties of the token from the token store. This
+// is particularly useful to fetch the accessor of the client token and get it
+// populated in the logical request along with the client token. The accessor
+// of the client token can get audit logged.
+func (c *Core) LookupToken(token string) (*TokenEntry, error) {
+	if token == "" {
+		return nil, fmt.Errorf("missing client token")
+	}
+
+	// Many tests don't have a token store running
+	if c.tokenStore == nil {
+		return nil, nil
+	}
+
+	return c.tokenStore.Lookup(token)
+}
+
 func (c *Core) fetchACLandTokenEntry(req *logical.Request) (*ACL, *TokenEntry, error) {
 	defer metrics.MeasureSince([]string{"core", "fetch_acl_and_token"}, time.Now())