Merge pull request #786 from hashicorp/issue-784

Reintroduce the ability to look up obfuscated values in the audit log

CHANGELOG.md

@@ -39,6 +39,9 @@ IMPROVEMENTS:
  * audit: HMAC-SHA256'd client tokens are now stored with each request entry.
    Previously they were only displayed at creation time; this allows much
    better traceability of client actions. [GH-713]
+ * audit: There is now a `sys/audit-hash` endpoint that can be used to generate
+   an HMAC-SHA256'd value from provided data using the given audit backend's
+   salt [GH-784]
  * core: The physical storage read cache can now be disabled via
    "disable_cache" [GH-674]
  * core: The unsealing process can now be reset midway through (this feature

api/sys_audit.go

@@ -4,6 +4,31 @@ import (
 	"fmt"
 )
 
+func (c *Sys) AuditHash(path string, input string) (string, error) {
+	body := map[string]interface{}{
+		"input": input,
+	}
+
+	r := c.c.NewRequest("PUT", fmt.Sprintf("/v1/sys/audit-hash/%s", path))
+	if err := r.SetJSONBody(body); err != nil {
+		return "", err
+	}
+
+	resp, err := c.c.RawRequest(r)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	type d struct {
+		Hash string
+	}
+
+	var result d
+	err = resp.DecodeJSON(&result)
+	return result.Hash, err
+}
+
 func (c *Sys) ListAudit() (map[string]*Audit, error) {
 	r := c.c.NewRequest("GET", "/v1/sys/audit")
 	resp, err := c.c.RawRequest(r)

audit/audit.go

@@ -21,6 +21,11 @@ type Backend interface {
 	// MUST not be modified in anyway. They should be deep copied if this is
 	// a possibility.
 	LogResponse(*logical.Auth, *logical.Request, *logical.Response, error) error
+
+	// GetHash is used to return the given data with the backend's hash,
+	// so that a caller can determine if a value in the audit log matches
+	// an expected plaintext value
+	GetHash(string) string
 }
 
 type BackendConfig struct {

audit/hashstructure.go

@@ -10,6 +10,11 @@ import (
 	"github.com/mitchellh/reflectwalk"
 )
 
+// HashString hashes the given opaque string and returns it
+func HashString(salter *salt.Salt, data string) string {
+	return salter.GetIdentifiedHMAC(data)
+}
+
 // Hash will hash the given type. This has built-in support for auth,
 // requests, and responses. If it is a type that isn't recognized, then
 // it will be passed through.

audit/hashstructure_test.go

@@ -81,6 +81,25 @@ func TestCopy_response(t *testing.T) {
 	}
 }
 
+func TestHashString(t *testing.T) {
+	inmemStorage := &logical.InmemStorage{}
+	inmemStorage.Put(&logical.StorageEntry{
+		Key:   "salt",
+		Value: []byte("foo"),
+	})
+	localSalt, err := salt.NewSalt(inmemStorage, &salt.Config{
+		HMAC:     sha256.New,
+		HMACType: "hmac-sha256",
+	})
+	if err != nil {
+		t.Fatalf("Error instantiating salt: %s", err)
+	}
+	out := HashString(localSalt, "foo")
+	if out != "hmac-sha256:08ba357e274f528065766c770a639abf6809b39ccfd37c2a3157c7f51954da0a" {
+		t.Fatalf("err: HashString output did not match expected")
+	}
+}
+
 func TestHash(t *testing.T) {
 	now := time.Now().UTC()
 

builtin/audit/file/backend.go

@@ -63,6 +63,10 @@ type Backend struct {
 	f    *os.File
 }
 
+func (b *Backend) GetHash(data string) string {
+	return audit.HashString(b.salt, data)
+}
+
 func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) error {
 	if err := b.open(); err != nil {
 		return err

builtin/audit/syslog/backend.go

@@ -60,6 +60,10 @@ type Backend struct {
 	salt   *salt.Salt
 }
 
+func (b *Backend) GetHash(data string) string {
+	return audit.HashString(b.salt, data)
+}
+
 func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) error {
 	if !b.logRaw {
 		// Before we copy the structure we must nil out some data

helper/salt/salt.go

@@ -110,18 +110,18 @@ func (s *Salt) SaltID(id string) string {
 	return SaltID(s.salt, id, s.config.HashFunc)
 }
 
-// GetHMAC is used to apply a salt and hash function to an ID to make sure
+// GetHMAC is used to apply a salt and hash function to data to make sure it is
-// it is not reversible, with an additional HMAC
+// not reversible, with an additional HMAC
-func (s *Salt) GetHMAC(id string) string {
+func (s *Salt) GetHMAC(data string) string {
 	hm := hmac.New(s.config.HMAC, []byte(s.salt))
-	hm.Write([]byte(id))
+	hm.Write([]byte(data))
 	return hex.EncodeToString(hm.Sum(nil))
 }
 
-// GetIdentifiedHMAC is used to apply a salt and hash function to an ID to make sure
+// GetIdentifiedHMAC is used to apply a salt and hash function to data to make
-// it is not reversible, with an additional HMAC, and ID prepended
+// sure it is not reversible, with an additional HMAC, and ID prepended
-func (s *Salt) GetIdentifiedHMAC(id string) string {
+func (s *Salt) GetIdentifiedHMAC(data string) string {
-	return s.hmacType + ":" + s.GetHMAC(id)
+	return s.hmacType + ":" + s.GetHMAC(data)
 }
 
 // DidGenerate returns if the underlying salt value was generated

http/handler.go

@@ -34,8 +34,9 @@ func Handler(core *vault.Core) http.Handler {
 	mux.Handle("/v1/sys/revoke-prefix/", proxySysRequest(core))
 	mux.Handle("/v1/sys/auth", proxySysRequest(core))
 	mux.Handle("/v1/sys/auth/", proxySysRequest(core))
-	mux.Handle("/v1/sys/audit", handleSysListAudit(core))
+	mux.Handle("/v1/sys/audit-hash/", proxySysRequest(core))
-	mux.Handle("/v1/sys/audit/", handleSysAudit(core))
+	mux.Handle("/v1/sys/audit", proxySysRequest(core))
+	mux.Handle("/v1/sys/audit/", proxySysRequest(core))
 	mux.Handle("/v1/sys/leader", handleSysLeader(core))
 	mux.Handle("/v1/sys/health", handleSysHealth(core))
 	mux.Handle("/v1/sys/rotate", proxySysRequest(core))

http/sys_audit.go

@@ -1,114 +0,0 @@
-package http
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/hashicorp/vault/logical"
-	"github.com/hashicorp/vault/vault"
-)
-
-func handleSysListAudit(core *vault.Core) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != "GET" {
-			respondError(w, http.StatusMethodNotAllowed, nil)
-			return
-		}
-
-		resp, ok := request(core, w, r, requestAuth(r, &logical.Request{
-			Operation:  logical.ReadOperation,
-			Path:       "sys/audit",
-			Connection: getConnection(r),
-		}))
-		if !ok {
-			return
-		}
-
-		respondOk(w, resp.Data)
-	})
-}
-
-func handleSysAudit(core *vault.Core) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.Method {
-		case "POST":
-			fallthrough
-		case "PUT":
-			handleSysEnableAudit(core, w, r)
-		case "DELETE":
-			handleSysDisableAudit(core, w, r)
-		default:
-			respondError(w, http.StatusMethodNotAllowed, nil)
-			return
-		}
-	})
-}
-
-func handleSysDisableAudit(core *vault.Core, w http.ResponseWriter, r *http.Request) {
-	// Determine the path...
-	prefix := "/v1/sys/audit/"
-	if !strings.HasPrefix(r.URL.Path, prefix) {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-	path := r.URL.Path[len(prefix):]
-	if path == "" {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-
-	_, ok := request(core, w, r, requestAuth(r, &logical.Request{
-		Operation:  logical.DeleteOperation,
-		Path:       "sys/audit/" + path,
-		Connection: getConnection(r),
-	}))
-	if !ok {
-		return
-	}
-
-	respondOk(w, nil)
-}
-
-func handleSysEnableAudit(core *vault.Core, w http.ResponseWriter, r *http.Request) {
-	// Determine the path...
-	prefix := "/v1/sys/audit/"
-	if !strings.HasPrefix(r.URL.Path, prefix) {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-
-	path := r.URL.Path[len(prefix):]
-	if path == "" {
-		respondError(w, http.StatusNotFound, nil)
-		return
-	}
-
-	// Parse the request if we can
-	var req enableAuditRequest
-	if err := parseRequest(r, &req); err != nil {
-		respondError(w, http.StatusBadRequest, err)
-		return
-	}
-
-	_, ok := request(core, w, r, requestAuth(r, &logical.Request{
-		Operation:  logical.WriteOperation,
-		Path:       "sys/audit/" + path,
-		Connection: getConnection(r),
-		Data: map[string]interface{}{
-			"type":        req.Type,
-			"description": req.Description,
-			"options":     req.Options,
-		},
-	}))
-	if !ok {
-		return
-	}
-
-	respondOk(w, nil)
-}
-
-type enableAuditRequest struct {
-	Type        string            `json:"type"`
-	Description string            `json:"description"`
-	Options     map[string]string `json:"options"`
-}

http/sys_audit_test.go

@@ -59,3 +59,29 @@ func TestSysDisableAudit(t *testing.T) {
 		t.Fatalf("bad: %#v", actual)
 	}
 }
+
+func TestSysAuditHash(t *testing.T) {
+	core, _, token := vault.TestCoreUnsealed(t)
+	ln, addr := TestServer(t, core)
+	defer ln.Close()
+	TestServerAuth(t, addr, token)
+
+	resp := testHttpPost(t, token, addr+"/v1/sys/audit/noop", map[string]interface{}{
+		"type": "noop",
+	})
+	testResponseStatus(t, resp, 204)
+
+	resp = testHttpPost(t, token, addr+"/v1/sys/audit-hash/noop", map[string]interface{}{
+		"input": "bar",
+	})
+
+	var actual map[string]interface{}
+	expected := map[string]interface{}{
+		"hash": "hmac-sha256:f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317",
+	}
+	testResponseStatus(t, resp, 200)
+	testResponseBody(t, resp, &actual)
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatalf("bad: expected:\n%#v\n, got:\n%#v\n", expected, actual)
+	}
+}

vault/audit.go

@@ -287,6 +287,18 @@ func (a *AuditBroker) IsRegistered(name string) bool {
 	return ok
 }
 
+// GetHash returns a hash using the salt of the given backend
+func (a *AuditBroker) GetHash(name string, input string) (string, error) {
+	a.l.RLock()
+	defer a.l.RUnlock()
+	be, ok := a.backends[name]
+	if !ok {
+		return "", fmt.Errorf("unknown audit backend %s", name)
+	}
+
+	return be.backend.GetHash(input), nil
+}
+
 // LogRequest is used to ensure all the audit backends have an opportunity to
 // log the given request and that *at least one* succeeds.
 func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (reterr error) {

vault/audit_test.go

@@ -43,6 +43,10 @@ func (n *NoopAudit) LogResponse(a *logical.Auth, r *logical.Request, re *logical
 	return n.RespErr
 }
 
+func (n *NoopAudit) GetHash(data string) string {
+	return n.Config.Salt.GetIdentifiedHMAC(data)
+}
+
 func TestCore_EnableAudit(t *testing.T) {
 	c, key, _ := TestCoreUnsealed(t)
 	c.auditBackends["noop"] = func(config *audit.BackendConfig) (audit.Backend, error) {

vault/logical_system.go

@@ -263,6 +263,28 @@ func NewSystemBackend(core *Core, config *logical.BackendConfig) logical.Backend
 				HelpDescription: strings.TrimSpace(sysHelp["policy"][1]),
 			},
 
+			&framework.Path{
+				Pattern: "audit-hash/(?P<path>.+)",
+
+				Fields: map[string]*framework.FieldSchema{
+					"path": &framework.FieldSchema{
+						Type:        framework.TypeString,
+						Description: strings.TrimSpace(sysHelp["audit_path"][0]),
+					},
+
+					"input": &framework.FieldSchema{
+						Type: framework.TypeString,
+					},
+				},
+
+				Callbacks: map[logical.Operation]framework.OperationFunc{
+					logical.WriteOperation: b.handleAuditHash,
+				},
+
+				HelpSynopsis:    strings.TrimSpace(sysHelp["audit-hash"][0]),
+				HelpDescription: strings.TrimSpace(sysHelp["audit-hash"][1]),
+			},
+
 			&framework.Path{
 				Pattern: "audit$",
 
@@ -822,6 +844,32 @@ func (b *SystemBackend) handleAuditTable(
 	return resp, nil
 }
 
+// handleAuditHash is used to fetch the hash of the given input data with the
+// specified audit backend's salt
+func (b *SystemBackend) handleAuditHash(
+	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	path := data.Get("path").(string)
+	input := data.Get("input").(string)
+	if input == "" {
+		return logical.ErrorResponse("the \"input\" parameter is empty"), nil
+	}
+
+	if !strings.HasSuffix(path, "/") {
+		path += "/"
+	}
+
+	hash, err := b.Core.auditBroker.GetHash(path, input)
+	if err != nil {
+		return logical.ErrorResponse(err.Error()), nil
+	}
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"hash": hash,
+		},
+	}, nil
+}
+
 // handleEnableAudit is used to enable a new audit backend
 func (b *SystemBackend) handleEnableAudit(
 	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
@@ -1167,6 +1215,11 @@ or delete a policy.
 		"",
 	},
 
+	"audit-hash": {
+		"The hash of the given string via the given audit backend",
+		"",
+	},
+
 	"audit-table": {
 		"List the currently enabled audit backends.",
 		`

vault/logical_system_test.go

@@ -1,12 +1,14 @@
 package vault
 
 import (
+	"crypto/sha256"
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/fatih/structs"
 	"github.com/hashicorp/vault/audit"
+	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
 )
 
@@ -543,6 +545,57 @@ func TestSystemBackend_enableAudit(t *testing.T) {
 	}
 }
 
+func TestSystemBackend_auditHash(t *testing.T) {
+	c, b, _ := testCoreSystemBackend(t)
+	c.auditBackends["noop"] = func(config *audit.BackendConfig) (audit.Backend, error) {
+		view := &logical.InmemStorage{}
+		view.Put(&logical.StorageEntry{
+			Key:   "salt",
+			Value: []byte("foo"),
+		})
+		var err error
+		config.Salt, err = salt.NewSalt(view, &salt.Config{
+			HMAC:     sha256.New,
+			HMACType: "hmac-sha256",
+		})
+		if err != nil {
+			t.Fatal("error getting new salt: %v", err)
+		}
+		return &NoopAudit{
+			Config: config,
+		}, nil
+	}
+
+	req := logical.TestRequest(t, logical.WriteOperation, "audit/foo")
+	req.Data["type"] = "noop"
+
+	resp, err := b.HandleRequest(req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if resp != nil {
+		t.Fatalf("bad: %v", resp)
+	}
+
+	req = logical.TestRequest(t, logical.WriteOperation, "audit-hash/foo")
+	req.Data["input"] = "bar"
+
+	resp, err = b.HandleRequest(req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if resp == nil || resp.Data == nil {
+		t.Fatalf("response or its data was nil")
+	}
+	hash, ok := resp.Data["hash"]
+	if !ok {
+		t.Fatalf("did not get hash back in response, response was %#v", resp.Data)
+	}
+	if hash.(string) != "hmac-sha256:f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317" {
+		t.Fatalf("bad hash back: %s", hash.(string))
+	}
+}
+
 func TestSystemBackend_enableAudit_invalid(t *testing.T) {
 	b := testSystemBackend(t)
 	req := logical.TestRequest(t, logical.WriteOperation, "audit/foo")

vault/testing.go

@@ -2,6 +2,7 @@ package vault
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"fmt"
 	"net"
 	"os/exec"
@@ -11,6 +12,7 @@ import (
 	"golang.org/x/crypto/ssh"
 
 	"github.com/hashicorp/vault/audit"
+	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
 	"github.com/hashicorp/vault/physical"
@@ -58,6 +60,19 @@ oOyBJU/HMVvBfv4g+OVFLVgSwwm6owwsouZ0+D/LasbuHqYyqYqdyPJQYzWA2Y+F
 func TestCore(t *testing.T) *Core {
 	noopAudits := map[string]audit.Factory{
 		"noop": func(config *audit.BackendConfig) (audit.Backend, error) {
+			view := &logical.InmemStorage{}
+			view.Put(&logical.StorageEntry{
+				Key:   "salt",
+				Value: []byte("foo"),
+			})
+			var err error
+			config.Salt, err = salt.NewSalt(view, &salt.Config{
+				HMAC:     sha256.New,
+				HMACType: "hmac-sha256",
+			})
+			if err != nil {
+				t.Fatal("error getting new salt: %v", err)
+			}
 			return &noopAudit{
 				Config: config,
 			}, nil
@@ -247,6 +262,10 @@ type noopAudit struct {
 	Config *audit.BackendConfig
 }
 
+func (n *noopAudit) GetHash(data string) string {
+	return n.Config.Salt.GetIdentifiedHMAC(data)
+}
+
 func (n *noopAudit) LogRequest(a *logical.Auth, r *logical.Request, e error) error {
 	return nil
 }

website/source/docs/audit/index.html.md

@@ -25,11 +25,11 @@ interaction with Vault. The data in the request and the data in the
 response (including secrets and authentication tokens) will be hashed
 with a salt using HMAC-SHA256.
 
-<!---
+The purpose of the hash is so that secrets aren't in plaintext within your
-The purpose of the hash is so that secrets aren't in plaintext within
+audit logs. However, you're still able to check the value of secrets by
-your audit logs. However, you're still able to check the value of
+generating HMACs yourself; this can be done with the audit backend's hash
-secrets by SHA-ing it yourself.
+function and salt by using the `/sys/audit-hash` API endpoint (see the
---->
+documentation for more details).
 
 ## Enabling/Disabling Audit Backends
 

website/source/docs/http/sys-audit-hash.html.md

@@ -0,0 +1,53 @@
+---
+layout: "http"
+page_title: "HTTP API: /sys/audit-hash"
+sidebar_current: "docs-http-audits-hash"
+description: |-
+  The `/sys/audit-hash` endpoint is used to hash data using an audit backend's hash function and salt.
+---
+
+# /sys/audit-hash
+
+## POST
+
+<dl>
+  <dt>Description</dt>
+  <dd>
+    Hash the given input data with the specified audit backend's hash function
+    and salt. This endpoint can be used to discover whether a given plaintext
+    string (the `input` parameter) appears in the audit log in obfuscated form.
+    Note that the audit log records requests and responses; since the Vault API
+    is JSON-based, any binary data returned from an API call (such as a
+    DER-format certificate) is base64-encoded by the Vault server in the
+    response, and as a result such information should also be base64-encoded to
+    supply into the `input` parameter.
+  </dd>
+
+  <dt>Method</dt>
+  <dd>POST</dd>
+
+  <dt>URL</dt>
+  <dd>`/sys/audit-hash/<name>`</dd>
+
+  <dt>Parameters</dt>
+  <dd>
+    <ul>
+      <li>
+        <span class="param">input</span>
+        <span class="param-flags">required</span>
+        The input string to hash.
+      </li>
+    </ul>
+  </dd>
+
+  <dt>Returns</dt>
+  <dd>
+
+    ```javascript
+    {
+      "hash": "hmac-sha256:08ba357e274f528065766c770a639abf6809b39ccfd37c2a3157c7f51954da0a"
+    }
+    ```
+
+  </dd>
+</dl>

website/source/layouts/http.erb

@@ -74,6 +74,9 @@
 						<li<%= sidebar_current("docs-http-audits-audits") %>>
 							<a href="/docs/http/sys-audit.html">/sys/audit</a>
 						</li>
+						<li<%= sidebar_current("docs-http-audits-hash") %>>
+							<a href="/docs/http/sys-audit-hash.html">/sys/audit-hash</a>
+						</li>
 					</ul>
 				</li>