From 82b493adfad540c04ae4c7b0fbc0dd51e17c705a Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Fri, 30 Mar 2018 12:42:48 -0400 Subject: [PATCH] Switch reading from S3 to io.Copy from io.ReadFull (#4225) * Switch reading from S3 to io.Copy from io.ReadFull If the Content-Length header wasn't being sent back, the current behavior could panic. It's unclear when it will not be sent; it appears to be CORS dependent. But this works around it by not trying to preallocate a buffer of a specific size and instead just read until EOF. In addition I noticed that Close wasn't being called. https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#GetObjectOutput specifies that Body is an io.ReadCloser so I added a call to Close. Fixes #4222 * Add some extra efficiency --- physical/s3/s3.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/physical/s3/s3.go b/physical/s3/s3.go index 9f8084ac65..2a22b3b3f8 100644 --- a/physical/s3/s3.go +++ b/physical/s3/s3.go @@ -188,16 +188,20 @@ func (s *S3Backend) Get(ctx context.Context, key string) (*physical.Entry, error if resp == nil { return nil, fmt.Errorf("got nil response from S3 but no error") } + defer resp.Body.Close() - data := make([]byte, *resp.ContentLength) - _, err = io.ReadFull(resp.Body, data) + data := bytes.NewBuffer(nil) + if resp.ContentLength != nil { + data = bytes.NewBuffer(make([]byte, 0, *resp.ContentLength)) + } + _, err = io.Copy(data, resp.Body) if err != nil { return nil, err } ent := &physical.Entry{ Key: key, - Value: data, + Value: data.Bytes(), } return ent, nil