From 49d525ebf3c15c4b5477ac9e62d621cca9533156 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Wed, 18 Nov 2015 20:26:03 -0500 Subject: [PATCH] Reintroduce the ability to look up obfuscated values in the audit log with a new endpoint '/sys/audit-hash', which returns the given input string hashed with the given audit backend's hash function and salt (currently, always HMAC-SHA256 and a backend-specific salt). In the process of adding the HTTP handler, this also removes the custom HTTP handlers for the other audit endpoints, which were simply forwarding to the logical system backend. This means that the various audit functions will now redirect correctly from a standby to master. (Tests all pass.) Fixes #784 --- CHANGELOG.md | 3 + api/sys_audit.go | 25 ++++ audit/audit.go | 5 + audit/hashstructure.go | 5 + audit/hashstructure_test.go | 19 +++ builtin/audit/file/backend.go | 4 + builtin/audit/syslog/backend.go | 4 + helper/salt/salt.go | 16 +-- http/handler.go | 5 +- http/sys_audit.go | 114 ------------------ http/sys_audit_test.go | 26 ++++ vault/audit.go | 12 ++ vault/audit_test.go | 4 + vault/logical_system.go | 53 ++++++++ vault/logical_system_test.go | 53 ++++++++ vault/testing.go | 19 +++ website/source/docs/audit/index.html.md | 10 +- .../source/docs/http/sys-audit-hash.html.md | 53 ++++++++ website/source/layouts/http.erb | 3 + 19 files changed, 304 insertions(+), 129 deletions(-) delete mode 100644 http/sys_audit.go create mode 100644 website/source/docs/http/sys-audit-hash.html.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a11de89c0..d588212309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ IMPROVEMENTS: * audit: HMAC-SHA256'd client tokens are now stored with each request entry. Previously they were only displayed at creation time; this allows much better traceability of client actions. [GH-713] + * audit: There is now a `sys/audit-hash` endpoint that can be used to generate + an HMAC-SHA256'd value from provided data using the given audit backend's + salt [GH-784] * core: The physical storage read cache can now be disabled via "disable_cache" [GH-674] * core: The unsealing process can now be reset midway through (this feature diff --git a/api/sys_audit.go b/api/sys_audit.go index 885baf4850..bf688541e3 100644 --- a/api/sys_audit.go +++ b/api/sys_audit.go @@ -4,6 +4,31 @@ import ( "fmt" ) +func (c *Sys) AuditHash(path string, input string) (string, error) { + body := map[string]interface{}{ + "input": input, + } + + r := c.c.NewRequest("PUT", fmt.Sprintf("/v1/sys/audit-hash/%s", path)) + if err := r.SetJSONBody(body); err != nil { + return "", err + } + + resp, err := c.c.RawRequest(r) + if err != nil { + return "", err + } + defer resp.Body.Close() + + type d struct { + Hash string + } + + var result d + err = resp.DecodeJSON(&result) + return result.Hash, err +} + func (c *Sys) ListAudit() (map[string]*Audit, error) { r := c.c.NewRequest("GET", "/v1/sys/audit") resp, err := c.c.RawRequest(r) diff --git a/audit/audit.go b/audit/audit.go index 9c704042b9..ddcf6426bd 100644 --- a/audit/audit.go +++ b/audit/audit.go @@ -21,6 +21,11 @@ type Backend interface { // MUST not be modified in anyway. They should be deep copied if this is // a possibility. LogResponse(*logical.Auth, *logical.Request, *logical.Response, error) error + + // GetHash is used to return the given data with the backend's hash, + // so that a caller can determine if a value in the audit log matches + // an expected plaintext value + GetHash(string) string } type BackendConfig struct { diff --git a/audit/hashstructure.go b/audit/hashstructure.go index 98633e3727..3394ead456 100644 --- a/audit/hashstructure.go +++ b/audit/hashstructure.go @@ -10,6 +10,11 @@ import ( "github.com/mitchellh/reflectwalk" ) +// HashString hashes the given opaque string and returns it +func HashString(salter *salt.Salt, data string) string { + return salter.GetIdentifiedHMAC(data) +} + // Hash will hash the given type. This has built-in support for auth, // requests, and responses. If it is a type that isn't recognized, then // it will be passed through. diff --git a/audit/hashstructure_test.go b/audit/hashstructure_test.go index ce7e55d76f..1813008440 100644 --- a/audit/hashstructure_test.go +++ b/audit/hashstructure_test.go @@ -81,6 +81,25 @@ func TestCopy_response(t *testing.T) { } } +func TestHashString(t *testing.T) { + inmemStorage := &logical.InmemStorage{} + inmemStorage.Put(&logical.StorageEntry{ + Key: "salt", + Value: []byte("foo"), + }) + localSalt, err := salt.NewSalt(inmemStorage, &salt.Config{ + HMAC: sha256.New, + HMACType: "hmac-sha256", + }) + if err != nil { + t.Fatalf("Error instantiating salt: %s", err) + } + out := HashString(localSalt, "foo") + if out != "hmac-sha256:08ba357e274f528065766c770a639abf6809b39ccfd37c2a3157c7f51954da0a" { + t.Fatalf("err: HashString output did not match expected") + } +} + func TestHash(t *testing.T) { now := time.Now().UTC() diff --git a/builtin/audit/file/backend.go b/builtin/audit/file/backend.go index 4d636c2a12..f52086745d 100644 --- a/builtin/audit/file/backend.go +++ b/builtin/audit/file/backend.go @@ -63,6 +63,10 @@ type Backend struct { f *os.File } +func (b *Backend) GetHash(data string) string { + return audit.HashString(b.salt, data) +} + func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) error { if err := b.open(); err != nil { return err diff --git a/builtin/audit/syslog/backend.go b/builtin/audit/syslog/backend.go index a93df385e7..a44ff0c8f3 100644 --- a/builtin/audit/syslog/backend.go +++ b/builtin/audit/syslog/backend.go @@ -60,6 +60,10 @@ type Backend struct { salt *salt.Salt } +func (b *Backend) GetHash(data string) string { + return audit.HashString(b.salt, data) +} + func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) error { if !b.logRaw { // Before we copy the structure we must nil out some data diff --git a/helper/salt/salt.go b/helper/salt/salt.go index cfc9336941..0abd9054b6 100644 --- a/helper/salt/salt.go +++ b/helper/salt/salt.go @@ -110,18 +110,18 @@ func (s *Salt) SaltID(id string) string { return SaltID(s.salt, id, s.config.HashFunc) } -// GetHMAC is used to apply a salt and hash function to an ID to make sure -// it is not reversible, with an additional HMAC -func (s *Salt) GetHMAC(id string) string { +// GetHMAC is used to apply a salt and hash function to data to make sure it is +// not reversible, with an additional HMAC +func (s *Salt) GetHMAC(data string) string { hm := hmac.New(s.config.HMAC, []byte(s.salt)) - hm.Write([]byte(id)) + hm.Write([]byte(data)) return hex.EncodeToString(hm.Sum(nil)) } -// GetIdentifiedHMAC is used to apply a salt and hash function to an ID to make sure -// it is not reversible, with an additional HMAC, and ID prepended -func (s *Salt) GetIdentifiedHMAC(id string) string { - return s.hmacType + ":" + s.GetHMAC(id) +// GetIdentifiedHMAC is used to apply a salt and hash function to data to make +// sure it is not reversible, with an additional HMAC, and ID prepended +func (s *Salt) GetIdentifiedHMAC(data string) string { + return s.hmacType + ":" + s.GetHMAC(data) } // DidGenerate returns if the underlying salt value was generated diff --git a/http/handler.go b/http/handler.go index 81bf6b3dfa..9747380aea 100644 --- a/http/handler.go +++ b/http/handler.go @@ -34,8 +34,9 @@ func Handler(core *vault.Core) http.Handler { mux.Handle("/v1/sys/revoke-prefix/", proxySysRequest(core)) mux.Handle("/v1/sys/auth", proxySysRequest(core)) mux.Handle("/v1/sys/auth/", proxySysRequest(core)) - mux.Handle("/v1/sys/audit", handleSysListAudit(core)) - mux.Handle("/v1/sys/audit/", handleSysAudit(core)) + mux.Handle("/v1/sys/audit-hash/", proxySysRequest(core)) + mux.Handle("/v1/sys/audit", proxySysRequest(core)) + mux.Handle("/v1/sys/audit/", proxySysRequest(core)) mux.Handle("/v1/sys/leader", handleSysLeader(core)) mux.Handle("/v1/sys/health", handleSysHealth(core)) mux.Handle("/v1/sys/rotate", proxySysRequest(core)) diff --git a/http/sys_audit.go b/http/sys_audit.go deleted file mode 100644 index 65cb7baa20..0000000000 --- a/http/sys_audit.go +++ /dev/null @@ -1,114 +0,0 @@ -package http - -import ( - "net/http" - "strings" - - "github.com/hashicorp/vault/logical" - "github.com/hashicorp/vault/vault" -) - -func handleSysListAudit(core *vault.Core) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { - respondError(w, http.StatusMethodNotAllowed, nil) - return - } - - resp, ok := request(core, w, r, requestAuth(r, &logical.Request{ - Operation: logical.ReadOperation, - Path: "sys/audit", - Connection: getConnection(r), - })) - if !ok { - return - } - - respondOk(w, resp.Data) - }) -} - -func handleSysAudit(core *vault.Core) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "POST": - fallthrough - case "PUT": - handleSysEnableAudit(core, w, r) - case "DELETE": - handleSysDisableAudit(core, w, r) - default: - respondError(w, http.StatusMethodNotAllowed, nil) - return - } - }) -} - -func handleSysDisableAudit(core *vault.Core, w http.ResponseWriter, r *http.Request) { - // Determine the path... - prefix := "/v1/sys/audit/" - if !strings.HasPrefix(r.URL.Path, prefix) { - respondError(w, http.StatusNotFound, nil) - return - } - path := r.URL.Path[len(prefix):] - if path == "" { - respondError(w, http.StatusNotFound, nil) - return - } - - _, ok := request(core, w, r, requestAuth(r, &logical.Request{ - Operation: logical.DeleteOperation, - Path: "sys/audit/" + path, - Connection: getConnection(r), - })) - if !ok { - return - } - - respondOk(w, nil) -} - -func handleSysEnableAudit(core *vault.Core, w http.ResponseWriter, r *http.Request) { - // Determine the path... - prefix := "/v1/sys/audit/" - if !strings.HasPrefix(r.URL.Path, prefix) { - respondError(w, http.StatusNotFound, nil) - return - } - - path := r.URL.Path[len(prefix):] - if path == "" { - respondError(w, http.StatusNotFound, nil) - return - } - - // Parse the request if we can - var req enableAuditRequest - if err := parseRequest(r, &req); err != nil { - respondError(w, http.StatusBadRequest, err) - return - } - - _, ok := request(core, w, r, requestAuth(r, &logical.Request{ - Operation: logical.WriteOperation, - Path: "sys/audit/" + path, - Connection: getConnection(r), - Data: map[string]interface{}{ - "type": req.Type, - "description": req.Description, - "options": req.Options, - }, - })) - if !ok { - return - } - - respondOk(w, nil) -} - -type enableAuditRequest struct { - Type string `json:"type"` - Description string `json:"description"` - Options map[string]string `json:"options"` -} diff --git a/http/sys_audit_test.go b/http/sys_audit_test.go index f67af78f7d..010f4bb06e 100644 --- a/http/sys_audit_test.go +++ b/http/sys_audit_test.go @@ -59,3 +59,29 @@ func TestSysDisableAudit(t *testing.T) { t.Fatalf("bad: %#v", actual) } } + +func TestSysAuditHash(t *testing.T) { + core, _, token := vault.TestCoreUnsealed(t) + ln, addr := TestServer(t, core) + defer ln.Close() + TestServerAuth(t, addr, token) + + resp := testHttpPost(t, token, addr+"/v1/sys/audit/noop", map[string]interface{}{ + "type": "noop", + }) + testResponseStatus(t, resp, 204) + + resp = testHttpPost(t, token, addr+"/v1/sys/audit-hash/noop", map[string]interface{}{ + "input": "bar", + }) + + var actual map[string]interface{} + expected := map[string]interface{}{ + "hash": "hmac-sha256:f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317", + } + testResponseStatus(t, resp, 200) + testResponseBody(t, resp, &actual) + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("bad: expected:\n%#v\n, got:\n%#v\n", expected, actual) + } +} diff --git a/vault/audit.go b/vault/audit.go index 144a3efd17..96e2253484 100644 --- a/vault/audit.go +++ b/vault/audit.go @@ -287,6 +287,18 @@ func (a *AuditBroker) IsRegistered(name string) bool { return ok } +// GetHash returns a hash using the salt of the given backend +func (a *AuditBroker) GetHash(name string, input string) (string, error) { + a.l.RLock() + defer a.l.RUnlock() + be, ok := a.backends[name] + if !ok { + return "", fmt.Errorf("unknown audit backend %s", name) + } + + return be.backend.GetHash(input), nil +} + // LogRequest is used to ensure all the audit backends have an opportunity to // log the given request and that *at least one* succeeds. func (a *AuditBroker) LogRequest(auth *logical.Auth, req *logical.Request, outerErr error) (reterr error) { diff --git a/vault/audit_test.go b/vault/audit_test.go index 502805e301..385c716d74 100644 --- a/vault/audit_test.go +++ b/vault/audit_test.go @@ -43,6 +43,10 @@ func (n *NoopAudit) LogResponse(a *logical.Auth, r *logical.Request, re *logical return n.RespErr } +func (n *NoopAudit) GetHash(data string) string { + return n.Config.Salt.GetIdentifiedHMAC(data) +} + func TestCore_EnableAudit(t *testing.T) { c, key, _ := TestCoreUnsealed(t) c.auditBackends["noop"] = func(config *audit.BackendConfig) (audit.Backend, error) { diff --git a/vault/logical_system.go b/vault/logical_system.go index a55f4271ed..972d1c99b4 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -263,6 +263,28 @@ func NewSystemBackend(core *Core, config *logical.BackendConfig) logical.Backend HelpDescription: strings.TrimSpace(sysHelp["policy"][1]), }, + &framework.Path{ + Pattern: "audit-hash/(?P.+)", + + Fields: map[string]*framework.FieldSchema{ + "path": &framework.FieldSchema{ + Type: framework.TypeString, + Description: strings.TrimSpace(sysHelp["audit_path"][0]), + }, + + "input": &framework.FieldSchema{ + Type: framework.TypeString, + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.WriteOperation: b.handleAuditHash, + }, + + HelpSynopsis: strings.TrimSpace(sysHelp["audit-hash"][0]), + HelpDescription: strings.TrimSpace(sysHelp["audit-hash"][1]), + }, + &framework.Path{ Pattern: "audit$", @@ -822,6 +844,32 @@ func (b *SystemBackend) handleAuditTable( return resp, nil } +// handleAuditHash is used to fetch the hash of the given input data with the +// specified audit backend's salt +func (b *SystemBackend) handleAuditHash( + req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + path := data.Get("path").(string) + input := data.Get("input").(string) + if input == "" { + return logical.ErrorResponse("the \"input\" parameter is empty"), nil + } + + if !strings.HasSuffix(path, "/") { + path += "/" + } + + hash, err := b.Core.auditBroker.GetHash(path, input) + if err != nil { + return logical.ErrorResponse(err.Error()), nil + } + + return &logical.Response{ + Data: map[string]interface{}{ + "hash": hash, + }, + }, nil +} + // handleEnableAudit is used to enable a new audit backend func (b *SystemBackend) handleEnableAudit( req *logical.Request, data *framework.FieldData) (*logical.Response, error) { @@ -1167,6 +1215,11 @@ or delete a policy. "", }, + "audit-hash": { + "The hash of the given string via the given audit backend", + "", + }, + "audit-table": { "List the currently enabled audit backends.", ` diff --git a/vault/logical_system_test.go b/vault/logical_system_test.go index 8b783047c0..fdca81efa4 100644 --- a/vault/logical_system_test.go +++ b/vault/logical_system_test.go @@ -1,12 +1,14 @@ package vault import ( + "crypto/sha256" "reflect" "testing" "time" "github.com/fatih/structs" "github.com/hashicorp/vault/audit" + "github.com/hashicorp/vault/helper/salt" "github.com/hashicorp/vault/logical" ) @@ -543,6 +545,57 @@ func TestSystemBackend_enableAudit(t *testing.T) { } } +func TestSystemBackend_auditHash(t *testing.T) { + c, b, _ := testCoreSystemBackend(t) + c.auditBackends["noop"] = func(config *audit.BackendConfig) (audit.Backend, error) { + view := &logical.InmemStorage{} + view.Put(&logical.StorageEntry{ + Key: "salt", + Value: []byte("foo"), + }) + var err error + config.Salt, err = salt.NewSalt(view, &salt.Config{ + HMAC: sha256.New, + HMACType: "hmac-sha256", + }) + if err != nil { + t.Fatal("error getting new salt: %v", err) + } + return &NoopAudit{ + Config: config, + }, nil + } + + req := logical.TestRequest(t, logical.WriteOperation, "audit/foo") + req.Data["type"] = "noop" + + resp, err := b.HandleRequest(req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp != nil { + t.Fatalf("bad: %v", resp) + } + + req = logical.TestRequest(t, logical.WriteOperation, "audit-hash/foo") + req.Data["input"] = "bar" + + resp, err = b.HandleRequest(req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp == nil || resp.Data == nil { + t.Fatalf("response or its data was nil") + } + hash, ok := resp.Data["hash"] + if !ok { + t.Fatalf("did not get hash back in response, response was %#v", resp.Data) + } + if hash.(string) != "hmac-sha256:f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317" { + t.Fatalf("bad hash back: %s", hash.(string)) + } +} + func TestSystemBackend_enableAudit_invalid(t *testing.T) { b := testSystemBackend(t) req := logical.TestRequest(t, logical.WriteOperation, "audit/foo") diff --git a/vault/testing.go b/vault/testing.go index 764dbbefa1..6df06e11fa 100644 --- a/vault/testing.go +++ b/vault/testing.go @@ -2,6 +2,7 @@ package vault import ( "bytes" + "crypto/sha256" "fmt" "net" "os/exec" @@ -11,6 +12,7 @@ import ( "golang.org/x/crypto/ssh" "github.com/hashicorp/vault/audit" + "github.com/hashicorp/vault/helper/salt" "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/logical/framework" "github.com/hashicorp/vault/physical" @@ -58,6 +60,19 @@ oOyBJU/HMVvBfv4g+OVFLVgSwwm6owwsouZ0+D/LasbuHqYyqYqdyPJQYzWA2Y+F func TestCore(t *testing.T) *Core { noopAudits := map[string]audit.Factory{ "noop": func(config *audit.BackendConfig) (audit.Backend, error) { + view := &logical.InmemStorage{} + view.Put(&logical.StorageEntry{ + Key: "salt", + Value: []byte("foo"), + }) + var err error + config.Salt, err = salt.NewSalt(view, &salt.Config{ + HMAC: sha256.New, + HMACType: "hmac-sha256", + }) + if err != nil { + t.Fatal("error getting new salt: %v", err) + } return &noopAudit{ Config: config, }, nil @@ -247,6 +262,10 @@ type noopAudit struct { Config *audit.BackendConfig } +func (n *noopAudit) GetHash(data string) string { + return n.Config.Salt.GetIdentifiedHMAC(data) +} + func (n *noopAudit) LogRequest(a *logical.Auth, r *logical.Request, e error) error { return nil } diff --git a/website/source/docs/audit/index.html.md b/website/source/docs/audit/index.html.md index 58a08f0d98..131e60e0ba 100644 --- a/website/source/docs/audit/index.html.md +++ b/website/source/docs/audit/index.html.md @@ -25,11 +25,11 @@ interaction with Vault. The data in the request and the data in the response (including secrets and authentication tokens) will be hashed with a salt using HMAC-SHA256. - +The purpose of the hash is so that secrets aren't in plaintext within your +audit logs. However, you're still able to check the value of secrets by +generating HMACs yourself; this can be done with the audit backend's hash +function and salt by using the `/sys/audit-hash` API endpoint (see the +documentation for more details). ## Enabling/Disabling Audit Backends diff --git a/website/source/docs/http/sys-audit-hash.html.md b/website/source/docs/http/sys-audit-hash.html.md new file mode 100644 index 0000000000..07c44590b8 --- /dev/null +++ b/website/source/docs/http/sys-audit-hash.html.md @@ -0,0 +1,53 @@ +--- +layout: "http" +page_title: "HTTP API: /sys/audit-hash" +sidebar_current: "docs-http-audits-hash" +description: |- + The `/sys/audit-hash` endpoint is used to hash data using an audit backend's hash function and salt. +--- + +# /sys/audit-hash + +## POST + +
+
Description
+
+ Hash the given input data with the specified audit backend's hash function + and salt. This endpoint can be used to discover whether a given plaintext + string (the `input` parameter) appears in the audit log in obfuscated form. + Note that the audit log records requests and responses; since the Vault API + is JSON-based, any binary data returned from an API call (such as a + DER-format certificate) is base64-encoded by the Vault server in the + response, and as a result such information should also be base64-encoded to + supply into the `input` parameter. +
+ +
Method
+
POST
+ +
URL
+
`/sys/audit-hash/`
+ +
Parameters
+
+
    +
  • + input + required + The input string to hash. +
    • +
+
+ +
Returns
+
+ + ```javascript + { + "hash": "hmac-sha256:08ba357e274f528065766c770a639abf6809b39ccfd37c2a3157c7f51954da0a" + } + ``` + +
+
diff --git a/website/source/layouts/http.erb b/website/source/layouts/http.erb index 00b0be5c5f..7e3e03c3e6 100644 --- a/website/source/layouts/http.erb +++ b/website/source/layouts/http.erb @@ -74,6 +74,9 @@ > /sys/audit + > + /sys/audit-hash +