From d8eb7d3e1563e3ce46e48c2367258abc88c70b8d Mon Sep 17 00:00:00 2001 From: Andreas Auernhammer Date: Tue, 11 May 2021 03:15:11 +0200 Subject: [PATCH] kms: replace KES client implementation with minio/kes (#12207) This commit replaces the custom KES client implementation with the KES SDK from https://github.com/minio/kes The SDK supports multi-server client load-balancing and requests retry out of the box. Therefore, this change reduces the overall complexity within the MinIO server and there is no need to maintain two separate client implementations. Signed-off-by: Andreas Auernhammer --- cmd/admin-handlers.go | 4 +- cmd/bucket-handlers.go | 3 +- cmd/bucket-metadata-sys.go | 4 +- cmd/bucket-targets.go | 3 +- cmd/common-main.go | 41 +- cmd/crypto/{kms.go => auto-encryption.go} | 26 +- cmd/crypto/config.go | 84 ---- cmd/crypto/help.go | 18 - cmd/crypto/json.go | 203 --------- cmd/crypto/kes.go | 489 ---------------------- cmd/crypto/retry.go | 67 --- cmd/crypto/sse-kms.go | 8 +- cmd/crypto/sse-s3.go | 5 +- cmd/disk-cache-utils.go | 3 +- cmd/encryption-v1.go | 10 +- cmd/object-handlers.go | 3 +- cmd/utils.go | 40 -- go.mod | 10 +- go.sum | 125 ++++-- pkg/certs/ca-certs.go | 66 ++- pkg/kms/kes.go | 136 ++++++ 21 files changed, 350 insertions(+), 998 deletions(-) rename cmd/crypto/{kms.go => auto-encryption.go} (53%) delete mode 100644 cmd/crypto/config.go delete mode 100644 cmd/crypto/help.go delete mode 100644 cmd/crypto/json.go delete mode 100644 cmd/crypto/kes.go delete mode 100644 cmd/crypto/retry.go create mode 100644 pkg/kms/kes.go diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 923bcef04..30f484d8b 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -37,9 +37,9 @@ import ( "time" "github.com/gorilla/mux" + "github.com/minio/kes" "github.com/minio/madmin-go" "github.com/minio/minio/cmd/config" - "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger/message/log" @@ -1003,7 +1003,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError { Description: err.Error(), HTTPStatusCode: http.StatusServiceUnavailable, } - case errors.Is(err, crypto.ErrKESKeyExists): + case errors.Is(err, kes.ErrKeyExists): apiErr = APIError{ Code: "XMinioKMSKeyExists", Description: err.Error(), diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 34097daf3..651eed362 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -50,6 +50,7 @@ import ( "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/hash" iampolicy "github.com/minio/minio/pkg/iam/policy" + "github.com/minio/minio/pkg/kms" "github.com/minio/minio/pkg/sync/errgroup" ) @@ -1015,7 +1016,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h reader io.Reader keyID string key []byte - kmsCtx crypto.Context + kmsCtx kms.Context ) kind, _ := crypto.IsRequested(formValues) switch kind { diff --git a/cmd/bucket-metadata-sys.go b/cmd/bucket-metadata-sys.go index 2cae5ecfa..dc5894bcf 100644 --- a/cmd/bucket-metadata-sys.go +++ b/cmd/bucket-metadata-sys.go @@ -26,7 +26,6 @@ import ( "github.com/minio/madmin-go" "github.com/minio/minio-go/v7/pkg/tags" - "github.com/minio/minio/cmd/crypto" "github.com/minio/minio/cmd/logger" bucketsse "github.com/minio/minio/pkg/bucket/encryption" "github.com/minio/minio/pkg/bucket/lifecycle" @@ -35,6 +34,7 @@ import ( "github.com/minio/minio/pkg/bucket/replication" "github.com/minio/minio/pkg/bucket/versioning" "github.com/minio/minio/pkg/event" + "github.com/minio/minio/pkg/kms" "github.com/minio/minio/pkg/sync/errgroup" ) @@ -170,7 +170,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat } meta.ReplicationConfigXML = configData case bucketTargetsFile: - meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, crypto.Context{ + meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, kms.Context{ bucket: meta.Name, bucketTargetsFile: bucketTargetsFile, }) diff --git a/cmd/bucket-targets.go b/cmd/bucket-targets.go index deb37e57a..15cba3704 100644 --- a/cmd/bucket-targets.go +++ b/cmd/bucket-targets.go @@ -33,6 +33,7 @@ import ( "github.com/minio/minio/cmd/crypto" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/bucket/versioning" + "github.com/minio/minio/pkg/kms" ) const ( @@ -390,7 +391,7 @@ func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.Bu return nil, err } if crypto.S3.IsEncrypted(meta) { - if data, err = decryptBucketMetadata(cdata, bucket, meta, crypto.Context{ + if data, err = decryptBucketMetadata(cdata, bucket, meta, kms.Context{ bucket: bucket, bucketTargetsFile: bucketTargetsFile, }); err != nil { diff --git a/cmd/common-main.go b/cmd/common-main.go index 83baad223..95d93f6a1 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -40,12 +40,12 @@ import ( "github.com/minio/cli" "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio/cmd/config" - "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/certs" "github.com/minio/minio/pkg/console" + "github.com/minio/minio/pkg/ellipses" "github.com/minio/minio/pkg/env" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/kms" @@ -361,18 +361,37 @@ func handleCommonEnvVars() { } } if env.IsSet(config.EnvKESEndpoint) { - kesEndpoints, err := crypto.ParseKESEndpoints(env.Get(config.EnvKESEndpoint, "")) - if err != nil { - logger.Fatal(err, "Unable to parse the KES endpoints inherited from the shell environment") + var endpoints []string + for _, endpoint := range strings.Split(env.Get(config.EnvKESEndpoint, ""), ",") { + if strings.TrimSpace(endpoint) == "" { + continue + } + if !ellipses.HasEllipses(endpoint) { + endpoints = append(endpoints, endpoint) + continue + } + patterns, err := ellipses.FindEllipsesPatterns(endpoint) + if err != nil { + logger.Fatal(err, fmt.Sprintf("Invalid KES endpoint %q", endpoint)) + } + for _, lbls := range patterns.Expand() { + endpoints = append(endpoints, strings.Join(lbls, "")) + } } - KMS, err := crypto.NewKes(crypto.KesConfig{ - Enabled: true, - Endpoint: kesEndpoints, + certificate, err := tls.LoadX509KeyPair(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, "")) + if err != nil { + logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment") + } + rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get())) + if err != nil { + logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))) + } + + KMS, err := kms.NewWithConfig(kms.Config{ + Endpoints: endpoints, DefaultKeyID: env.Get(config.EnvKESKeyName, ""), - CertFile: env.Get(config.EnvKESClientCert, ""), - KeyFile: env.Get(config.EnvKESClientKey, ""), - CAPath: env.Get(config.EnvKESServerCA, globalCertsCADir.Get()), - Transport: newCustomHTTPTransportWithHTTP2(&tls.Config{RootCAs: globalRootCAs}, defaultDialTimeout)(), + Certificate: certificate, + RootCAs: rootCAs, }) if err != nil { logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment") diff --git a/cmd/crypto/kms.go b/cmd/crypto/auto-encryption.go similarity index 53% rename from cmd/crypto/kms.go rename to cmd/crypto/auto-encryption.go index d78fe9a84..103011ca7 100644 --- a/cmd/crypto/kms.go +++ b/cmd/crypto/auto-encryption.go @@ -18,15 +18,23 @@ package crypto import ( - "github.com/minio/minio/pkg/kms" + "github.com/minio/minio/cmd/config" + "github.com/minio/minio/pkg/env" ) -// Context is a list of key-value pairs cryptographically -// associated with a certain object. -type Context = kms.Context +const ( + // EnvKMSAutoEncryption is the environment variable used to en/disable + // SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled, + // requires a valid KMS configuration and turns any non-SSE-C + // request into an SSE-S3 request. + // If present EnvAutoEncryption must be either "on" or "off". + EnvKMSAutoEncryption = "MINIO_KMS_AUTO_ENCRYPTION" +) -// KMS represents an active and authenticted connection -// to a Key-Management-Service. It supports generating -// data key generation and unsealing of KMS-generated -// data keys. -type KMS = kms.KMS +// LookupAutoEncryption returns true if and only if +// the MINIO_KMS_AUTO_ENCRYPTION env. variable is +// set to "on". +func LookupAutoEncryption() bool { + auto, _ := config.ParseBool(env.Get(EnvKMSAutoEncryption, config.EnableOff)) + return auto +} diff --git a/cmd/crypto/config.go b/cmd/crypto/config.go deleted file mode 100644 index a1f30f0f2..000000000 --- a/cmd/crypto/config.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2015-2021 MinIO, Inc. -// -// This file is part of MinIO Object Storage stack -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package crypto - -import ( - "math/rand" - "strings" - - "github.com/minio/minio/cmd/config" - "github.com/minio/minio/pkg/ellipses" - "github.com/minio/minio/pkg/env" - xnet "github.com/minio/minio/pkg/net" -) - -const ( - // EnvKMSAutoEncryption is the environment variable used to en/disable - // SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled, - // requires a valid KMS configuration and turns any non-SSE-C - // request into an SSE-S3 request. - // If present EnvAutoEncryption must be either "on" or "off". - EnvKMSAutoEncryption = "MINIO_KMS_AUTO_ENCRYPTION" -) - -// ParseKESEndpoints parses the given endpoint string and -// returns a list of valid endpoint URLs. The order of the -// returned endpoints is randomized. -func ParseKESEndpoints(endpointStr string) ([]string, error) { - var rawEndpoints []string - for _, endpoint := range strings.Split(endpointStr, ",") { - if strings.TrimSpace(endpoint) == "" { - continue - } - if !ellipses.HasEllipses(endpoint) { - rawEndpoints = append(rawEndpoints, endpoint) - continue - } - pattern, err := ellipses.FindEllipsesPatterns(endpoint) - if err != nil { - return nil, Errorf("Invalid KES endpoint %q: %v", endpointStr, err) - } - for _, p := range pattern { - rawEndpoints = append(rawEndpoints, p.Expand()...) - } - } - if len(rawEndpoints) == 0 { - return nil, Errorf("Invalid KES endpoint %q", endpointStr) - } - - var ( - randNum = rand.Intn(len(rawEndpoints)) - endpoints = make([]string, len(rawEndpoints)) - ) - for i, endpoint := range rawEndpoints { - endpoint, err := xnet.ParseHTTPURL(endpoint) - if err != nil { - return nil, Errorf("Invalid KES endpoint %q: %v", endpointStr, err) - } - endpoints[(randNum+i)%len(rawEndpoints)] = endpoint.String() - } - return endpoints, nil -} - -// LookupAutoEncryption returns true if and only if -// the MINIO_KMS_AUTO_ENCRYPTION env. variable is -// set to "on". -func LookupAutoEncryption() bool { - auto, _ := config.ParseBool(env.Get(EnvKMSAutoEncryption, config.EnableOff)) - return auto -} diff --git a/cmd/crypto/help.go b/cmd/crypto/help.go deleted file mode 100644 index 9fde078bd..000000000 --- a/cmd/crypto/help.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2015-2021 MinIO, Inc. -// -// This file is part of MinIO Object Storage stack -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package crypto diff --git a/cmd/crypto/json.go b/cmd/crypto/json.go deleted file mode 100644 index 0faf56bfe..000000000 --- a/cmd/crypto/json.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright (c) 2015-2021 MinIO, Inc. -// -// This file is part of MinIO Object Storage stack -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package crypto - -import ( - "bytes" - "unicode/utf8" -) - -// Adapted from Go stdlib. - -var hexTable = "0123456789abcdef" - -// EscapeStringJSON will escape a string for JSON and write it to dst. -func EscapeStringJSON(dst *bytes.Buffer, s string) { - start := 0 - for i := 0; i < len(s); { - if b := s[i]; b < utf8.RuneSelf { - if htmlSafeSet[b] { - i++ - continue - } - if start < i { - dst.WriteString(s[start:i]) - } - dst.WriteByte('\\') - switch b { - case '\\', '"': - dst.WriteByte(b) - case '\n': - dst.WriteByte('n') - case '\r': - dst.WriteByte('r') - case '\t': - dst.WriteByte('t') - default: - // This encodes bytes < 0x20 except for \t, \n and \r. - // If escapeHTML is set, it also escapes <, >, and & - // because they can lead to security holes when - // user-controlled strings are rendered into JSON - // and served to some browsers. - dst.WriteString(`u00`) - dst.WriteByte(hexTable[b>>4]) - dst.WriteByte(hexTable[b&0xF]) - } - i++ - start = i - continue - } - c, size := utf8.DecodeRuneInString(s[i:]) - if c == utf8.RuneError && size == 1 { - if start < i { - dst.WriteString(s[start:i]) - } - dst.WriteString(`\ufffd`) - i += size - start = i - continue - } - // U+2028 is LINE SEPARATOR. - // U+2029 is PARAGRAPH SEPARATOR. - // They are both technically valid characters in JSON strings, - // but don't work in JSONP, which has to be evaluated as JavaScript, - // and can lead to security holes there. It is valid JSON to - // escape them, so we do so unconditionally. - // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. - if c == '\u2028' || c == '\u2029' { - if start < i { - dst.WriteString(s[start:i]) - } - dst.WriteString(`\u202`) - dst.WriteByte(hexTable[c&0xF]) - i += size - start = i - continue - } - i += size - } - if start < len(s) { - dst.WriteString(s[start:]) - } -} - -// htmlSafeSet holds the value true if the ASCII character with the given -// array position can be safely represented inside a JSON string, embedded -// inside of HTML