diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index 1f71c1dc9..66b5ae9d6 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -36,6 +36,7 @@ import ( "github.com/minio/minio/pkg/auth" objectlock "github.com/minio/minio/pkg/bucket/object/lock" "github.com/minio/minio/pkg/bucket/policy" + "github.com/minio/minio/pkg/etag" "github.com/minio/minio/pkg/hash" iampolicy "github.com/minio/minio/pkg/iam/policy" ) @@ -430,19 +431,14 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty return errCode } - var ( - err error - contentMD5, contentSHA256 []byte - ) - - // Extract 'Content-Md5' if present. - contentMD5, err = checkValidMD5(r.Header) + clientETag, err := etag.FromContentMD5(r.Header) if err != nil { return ErrInvalidDigest } // Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned) // Do not verify 'X-Amz-Content-Sha256' if skipSHA256. + var contentSHA256 []byte if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) { if sha256Sum, ok := r.URL.Query()[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 { contentSHA256, err = hex.DecodeString(sha256Sum[0]) @@ -459,7 +455,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty // Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present. // The verification happens implicit during reading. - reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1) + reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1) if err != nil { return toAPIErrorCode(ctx, err) } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index c01f1ef4b..4f9bfa597 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -19,7 +19,6 @@ package cmd import ( "bufio" "context" - "encoding/hex" "encoding/xml" "fmt" "io" @@ -1414,8 +1413,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req } } - // Get Content-Md5 sent by client and verify if valid - md5Bytes, err := checkValidMD5(r.Header) + clientETag, err := etag.FromContentMD5(r.Header) if err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r)) return @@ -1469,7 +1467,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req } var ( - md5hex = hex.EncodeToString(md5Bytes) + md5hex = clientETag.String() sha256hex = "" reader io.Reader = r.Body s3Err APIErrorCode @@ -2165,8 +2163,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return } - // get Content-Md5 sent by client and verify if valid - md5Bytes, err := checkValidMD5(r.Header) + clientETag, err := etag.FromContentMD5(r.Header) if err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r)) return @@ -2217,7 +2214,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http } var ( - md5hex = hex.EncodeToString(md5Bytes) + md5hex = clientETag.String() sha256hex = "" reader io.Reader = r.Body s3Error APIErrorCode diff --git a/cmd/utils.go b/cmd/utils.go index 8a0fa6897..5f67f3d81 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "crypto/tls" - "encoding/base64" "encoding/json" "encoding/xml" "errors" @@ -143,18 +142,6 @@ func xmlDecoder(body io.Reader, v interface{}, size int64) error { return d.Decode(v) } -// checkValidMD5 - verify if valid md5, returns md5 in bytes. -func checkValidMD5(h http.Header) ([]byte, error) { - md5B64, ok := h[xhttp.ContentMD5] - if ok { - if md5B64[0] == "" { - return nil, fmt.Errorf("Content-Md5 header set to empty value") - } - return base64.StdEncoding.Strict().DecodeString(md5B64[0]) - } - return []byte{}, nil -} - // hasContentMD5 returns true if Content-MD5 header is set. func hasContentMD5(h http.Header) bool { _, ok := h[xhttp.ContentMD5] diff --git a/pkg/etag/etag.go b/pkg/etag/etag.go index 8c8085afe..83a6e8646 100644 --- a/pkg/etag/etag.go +++ b/pkg/etag/etag.go @@ -105,6 +105,8 @@ package etag import ( "bytes" + "crypto/md5" + "encoding/base64" "encoding/hex" "errors" "fmt" @@ -176,6 +178,50 @@ var _ Tagger = ETag{} // compiler check // the Tagger interface. func (e ETag) ETag() ETag { return e } +// FromContentMD5 decodes and returns the Content-MD5 +// as ETag, if set. If no Content-MD5 header is set +// it returns an empty ETag and no error. +func FromContentMD5(h http.Header) (ETag, error) { + v, ok := h["Content-Md5"] + if !ok { + return nil, nil + } + if v[0] == "" { + return nil, errors.New("etag: content-md5 is set but contains no value") + } + b, err := base64.StdEncoding.Strict().DecodeString(v[0]) + if err != nil { + return nil, err + } + if len(b) != md5.Size { + return nil, errors.New("etag: invalid content-md5") + } + return ETag(b), nil +} + +// Multipart computes an S3 multipart ETag given a list of +// S3 singlepart ETags. It returns nil if the list of +// ETags is empty. +// +// Any encrypted or multipart ETag will be ignored and not +// used to compute the returned ETag. +func Multipart(etags ...ETag) ETag { + if len(etags) == 0 { + return nil + } + + var n int64 + h := md5.New() + for _, etag := range etags { + if !etag.IsMultipart() && !etag.IsEncrypted() { + h.Write(etag) + n++ + } + } + etag := append(h.Sum(nil), '-') + return strconv.AppendInt(etag, n, 10) +} + // Set adds the ETag to the HTTP headers. It overwrites any // existing ETag entry. // diff --git a/pkg/etag/etag_test.go b/pkg/etag/etag_test.go index 281584016..f6e0a6e09 100644 --- a/pkg/etag/etag_test.go +++ b/pkg/etag/etag_test.go @@ -17,6 +17,7 @@ package etag import ( "io" "io/ioutil" + "net/http" "strings" "testing" ) @@ -137,3 +138,85 @@ func TestReader(t *testing.T) { } } } + +var multipartTests = []struct { // Test cases have been generated using AWS S3 + ETags []ETag + Multipart ETag +}{ + { + ETags: []ETag{}, + Multipart: ETag{}, + }, + { + ETags: []ETag{must("b10a8db164e0754105b7a99be72e3fe5")}, + Multipart: must("7b976cc68452e003eec7cb0eb631a19a-1"), + }, + { + ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6")}, + Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"), + }, + { + ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("a096eb5968d607c2975fb2c4af9ab225"), must("b10a8db164e0754105b7a99be72e3fe5")}, + Multipart: must("9a0d1febd9265f59f368ceb652770bc2-3"), + }, + { // Check that multipart ETags are ignored + ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("ceb8853ddc5086cc4ab9e149f8f09c88-1")}, + Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"), + }, + { // Check that encrypted ETags are ignored + ETags: []ETag{ + must("90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941"), + must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"), + }, + Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"), + }, +} + +func TestMultipart(t *testing.T) { + for i, test := range multipartTests { + if multipart := Multipart(test.ETags...); !Equal(multipart, test.Multipart) { + t.Fatalf("Test %d: got %q - want %q", i, multipart, test.Multipart) + } + } +} + +var fromContentMD5Tests = []struct { + Header http.Header + ETag ETag + ShouldFail bool +}{ + {Header: http.Header{}, ETag: nil}, // 0 + {Header: http.Header{"Content-Md5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("d41d8cd98f00b204e9800998ecf8427e")}, // 1 + {Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 2 + {Header: http.Header{"Content-MD5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: nil}, // 3 (Content-MD5 vs Content-Md5) + {Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q==", "1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 4 + + {Header: http.Header{"Content-Md5": []string{""}}, ShouldFail: true}, // 5 (empty value) + {Header: http.Header{"Content-Md5": []string{"", "sQqNsWTgdUEFt6mb5y4/5Q=="}}, ShouldFail: true}, // 6 (empty value) + {Header: http.Header{"Content-Md5": []string{"d41d8cd98f00b204e9800998ecf8427e"}}, ShouldFail: true}, // 7 (content-md5 is invalid b64 / of invalid length) +} + +func TestFromContentMD5(t *testing.T) { + for i, test := range fromContentMD5Tests { + ETag, err := FromContentMD5(test.Header) + if err != nil && !test.ShouldFail { + t.Fatalf("Test %d: failed to convert Content-MD5 to ETag: %v", i, err) + } + if err == nil && test.ShouldFail { + t.Fatalf("Test %d: should have failed but succeeded", i) + } + if err == nil { + if !Equal(ETag, test.ETag) { + t.Fatalf("Test %d: got %q - want %q", i, ETag, test.ETag) + } + } + } +} + +func must(s string) ETag { + t, err := Parse(s) + if err != nil { + panic(err) + } + return t +}