diff --git a/cmd/api-response.go b/cmd/api-response.go index 183c7b1d0..68a8c780b 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -743,7 +743,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag. func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse { - cs := oi.decryptChecksums() + cs := oi.decryptChecksums(0) c := CompleteMultipartUploadResponse{ Location: location, Bucket: bucket, diff --git a/cmd/encryption-v1.go b/cmd/encryption-v1.go index 517f0b9a8..ecc35a9c0 100644 --- a/cmd/encryption-v1.go +++ b/cmd/encryption-v1.go @@ -1124,7 +1124,8 @@ func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn } // decryptChecksums will attempt to decode checksums and return it/them if set. -func (o *ObjectInfo) decryptChecksums() map[string]string { +// if part > 0, and we have the checksum for the part that will be returned. +func (o *ObjectInfo) decryptChecksums(part int) map[string]string { data := o.Checksum if len(data) == 0 { return nil @@ -1137,5 +1138,5 @@ func (o *ObjectInfo) decryptChecksums() map[string]string { } data = decrypted } - return hash.ReadCheckSums(data) + return hash.ReadCheckSums(data, part) } diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 9238c8302..59c833bce 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -1135,9 +1135,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } if checksumType.IsSet() { - checksumType |= hash.ChecksumMultipart + checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart cs := hash.NewChecksumFromData(checksumType, checksumCombined) - fi.Checksum = cs.AppendTo(nil, len(fi.Parts)) + fi.Checksum = cs.AppendTo(nil, checksumCombined) if opts.EncryptFn != nil { fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 3c76574e0..81e96fdb1 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1086,7 +1086,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st } fi.DataDir = mustGetUUID() - fi.Checksum = opts.WantChecksum.AppendTo(nil, 0) + fi.Checksum = opts.WantChecksum.AppendTo(nil, nil) if opts.EncryptFn != nil { fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) } diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index 7c81ae357..62d99ff66 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -340,7 +340,7 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) { lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts()) } } - hash.AddChecksumHeader(w, objInfo.decryptChecksums()) + hash.AddChecksumHeader(w, objInfo.decryptChecksums(0)) } func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete) { diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 6ddca0b07..a8d1d8cf1 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -513,8 +513,9 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) } - if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" { - hash.AddChecksumHeader(w, objInfo.decryptChecksums()) + if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil { + // AWS S3 silently drops checksums on range requests. + hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber)) } if err = setObjectHeaders(w, objInfo, rs, opts); err != nil { @@ -800,8 +801,9 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) } - if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" { - hash.AddChecksumHeader(w, objInfo.decryptChecksums()) + if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil { + // AWS S3 silently drops checksums on range requests. + hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber)) } // Set standard object headers. diff --git a/internal/hash/checksum.go b/internal/hash/checksum.go index 98781e4b2..7e548672e 100644 --- a/internal/hash/checksum.go +++ b/internal/hash/checksum.go @@ -19,6 +19,7 @@ package hash import ( "bytes" + "context" "crypto/sha1" "encoding/base64" "encoding/binary" @@ -30,6 +31,7 @@ import ( "github.com/minio/minio/internal/hash/sha256" xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" ) // MinIOMultipartChecksum is as metadata on multipart uploads to indicate checksum type. @@ -56,6 +58,8 @@ const ( ChecksumInvalid // ChecksumMultipart indicates the checksum is from a multipart upload. ChecksumMultipart + // ChecksumIncludesMultipart indicates the checksum also contains part checksums. + ChecksumIncludesMultipart // ChecksumNone indicates no checksum. ChecksumNone ChecksumType = 0 @@ -183,7 +187,7 @@ func NewChecksumFromData(t ChecksumType, data []byte) *Checksum { } // ReadCheckSums will read checksums from b and return them. -func ReadCheckSums(b []byte) map[string]string { +func ReadCheckSums(b []byte, part int) map[string]string { res := make(map[string]string, 1) for len(b) > 0 { t, n := binary.Uvarint(b) @@ -206,8 +210,29 @@ func ReadCheckSums(b []byte) map[string]string { } cs = fmt.Sprintf("%s-%d", cs, t) b = b[n:] + if part > 0 { + cs = "" + } + if typ.Is(ChecksumIncludesMultipart) { + wantLen := int(t) * length + if len(b) < wantLen { + break + } + // Read part checksum + if part > 0 && uint64(part) <= t { + offset := (part - 1) * length + partCs := b[offset:] + cs = base64.StdEncoding.EncodeToString(partCs[:length]) + } + b = b[wantLen:] + } + } else if part > 1 { + // For non-multipart, checksum is part 1. + cs = "" + } + if cs != "" { + res[typ.String()] = cs } - res[typ.String()] = cs } if len(res) == 0 { res = nil @@ -239,7 +264,7 @@ func NewChecksumString(alg, value string) *Checksum { // AppendTo will append the checksum to b. // 'parts' is used when checksum has ChecksumMultipart set. // ReadCheckSums reads the values back. -func (c *Checksum) AppendTo(b []byte, parts int) []byte { +func (c *Checksum) AppendTo(b []byte, parts []byte) []byte { if c == nil { return nil } @@ -252,11 +277,23 @@ func (c *Checksum) AppendTo(b []byte, parts int) []byte { b = append(b, tmp[:n]...) b = append(b, crc...) if c.Type.Is(ChecksumMultipart) { - if parts < 0 { - parts = 0 + var checksums int + // Ensure we don't divide by 0: + if c.Type.RawByteLen() == 0 || len(parts)%c.Type.RawByteLen() != 0 { + logger.LogIf(context.Background(), fmt.Errorf("internal error: Unexpected checksum length: %d, each checksum %d", len(parts), c.Type.RawByteLen())) + checksums = 0 + parts = nil + } else { + checksums = len(parts) / c.Type.RawByteLen() } - n := binary.PutUvarint(tmp[:], uint64(parts)) + if !c.Type.Is(ChecksumIncludesMultipart) { + parts = nil + } + n := binary.PutUvarint(tmp[:], uint64(checksums)) b = append(b, tmp[:n]...) + if len(parts) > 0 { + b = append(b, parts...) + } } return b }