diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go index 82f1e2586..aa89287a8 100644 --- a/cmd/bucket-handlers_test.go +++ b/cmd/bucket-handlers_test.go @@ -300,3 +300,74 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType string, t Tes t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code) } } + +// Wrapper for calling TestListBucketsHandler tests for both XL multiple disks and single node setup. +func TestListBucketsHandler(t *testing.T) { + ExecObjectLayerTest(t, testListBuckets) +} + +// testListBucketsHandler - Tests validate listing of buckets. +func testListBucketsHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + // get random bucket name. + bucketName := getRandomBucketName() + + // Register the API end points with XL/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"ListBuckets"}) + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + credentials := serverConfig.GetCredential() + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err = obj.MakeBucket(bucketName) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + testCases := []struct { + bucketName string + accessKey string + secretKey string + expectedRespStatus int + }{ + // Validate a good case request succeeds. + { + bucketName: bucketName, + accessKey: credentials.AccessKeyID, + secretKey: credentials.SecretAccessKey, + expectedRespStatus: http.StatusOK, + }, + // Validate a bad case request fails with http.StatusForbidden. + { + bucketName: bucketName, + accessKey: "", + secretKey: "", + expectedRespStatus: http.StatusForbidden, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + req, lerr := newTestSignedRequest("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey) + if lerr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: %v", i+1, instanceType, lerr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } +} diff --git a/cmd/erasure-readfile_test.go b/cmd/erasure-readfile_test.go index 0d972062f..05a7c2296 100644 --- a/cmd/erasure-readfile_test.go +++ b/cmd/erasure-readfile_test.go @@ -222,7 +222,7 @@ func TestErasureReadUtils(t *testing.T) { if err != nil { t.Fatal(err) } - objLayer, err := getXLObjectLayer(disks) + objLayer, err := getXLObjectLayer(disks, nil) if err != nil { t.Fatal(err) } diff --git a/cmd/event-notifier_test.go b/cmd/event-notifier_test.go index 6b077dba1..d6a1814a0 100644 --- a/cmd/event-notifier_test.go +++ b/cmd/event-notifier_test.go @@ -99,7 +99,7 @@ func TestInitEventNotifier(t *testing.T) { if err != nil { t.Fatal("Unable to create directories for XL backend. ", err) } - xl, err := getXLObjectLayer(disks) + xl, err := getXLObjectLayer(disks, nil) if err != nil { t.Fatal("Unable to initialize XL backend.", err) } diff --git a/cmd/format-config-v1_test.go b/cmd/format-config-v1_test.go index 515ea31fd..22bd58c99 100644 --- a/cmd/format-config-v1_test.go +++ b/cmd/format-config-v1_test.go @@ -268,7 +268,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, err := getXLObjectLayer(fsDirs) + obj, err := getXLObjectLayer(fsDirs, nil) if err != nil { t.Error(err) } @@ -300,7 +300,7 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, err := getXLObjectLayer(fsDirs) + obj, err := getXLObjectLayer(fsDirs, nil) if err != nil { t.Error(err) } @@ -584,7 +584,7 @@ func TestInitFormatXLErrors(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, err := getXLObjectLayer(fsDirs) + obj, err := getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -680,7 +680,7 @@ func TestLoadFormatXLErrs(t *testing.T) { } // Create an instance of xl backend. - obj, err := getXLObjectLayer(fsDirs) + obj, err := getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -705,7 +705,7 @@ func TestLoadFormatXLErrs(t *testing.T) { t.Fatal(err) } - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -730,7 +730,7 @@ func TestLoadFormatXLErrs(t *testing.T) { t.Fatal(err) } - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -753,7 +753,7 @@ func TestLoadFormatXLErrs(t *testing.T) { t.Fatal(err) } - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -779,7 +779,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Everything is fine, should return nil - obj, err := getXLObjectLayer(fsDirs) + obj, err := getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -796,7 +796,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Disks 0..15 are nil - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -815,7 +815,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // One disk returns Faulty Disk - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -836,7 +836,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // One disk is not found, heal corrupted disks should return nil - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -853,7 +853,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -874,7 +874,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Corrupted format json in one disk - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -899,7 +899,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Everything is fine, should return nil - obj, err := getXLObjectLayer(fsDirs) + obj, err := getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -915,7 +915,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Disks 0..15 are nil - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -934,7 +934,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // One disk returns Faulty Disk - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -955,7 +955,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // One disk is not found, heal corrupted disks should return nil - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -972,7 +972,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } @@ -993,7 +993,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, err = getXLObjectLayer(fsDirs) + obj, err = getXLObjectLayer(fsDirs, nil) if err != nil { t.Fatal(err) } diff --git a/cmd/post-policy_test.go b/cmd/post-policy_test.go index 434f62863..91bfe4e92 100644 --- a/cmd/post-policy_test.go +++ b/cmd/post-policy_test.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/base64" "fmt" + "io/ioutil" "mime/multipart" "net/http" "net/http/httptest" @@ -80,6 +81,8 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle // remove the root folder after the test ends. defer removeAll(rootPath) + credentials := serverConfig.GetCredential() + // bucketnames[0]. // objectNames[0]. // uploadIds [0]. @@ -96,31 +99,50 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle objectName string data []byte expectedRespStatus int - shouldPass bool + accessKey string + secretKey string + malformedBody bool }{ // Success case. { objectName: "test", data: []byte("Hello, World"), expectedRespStatus: http.StatusNoContent, - shouldPass: true, + accessKey: credentials.AccessKeyID, + secretKey: credentials.SecretAccessKey, + malformedBody: false, }, - // Bad case. + // Bad case invalid request. { objectName: "test", data: []byte("Hello, World"), expectedRespStatus: http.StatusBadRequest, - shouldPass: false, + accessKey: "", + secretKey: "", + malformedBody: false, + }, + // Bad case malformed input. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKeyID, + secretKey: credentials.SecretAccessKey, + malformedBody: true, }, } for i, testCase := range testCases { // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. rec := httptest.NewRecorder() - req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.shouldPass) + req, perr := newPostRequest("", bucketName, testCase.objectName, testCase.data, testCase.accessKey, testCase.secretKey) if perr != nil { t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: %v", i+1, instanceType, perr) } + if testCase.malformedBody { + // Change the request body. + req.Body = ioutil.NopCloser(bytes.NewReader([]byte("Hello,"))) + } // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler. // Call the ServeHTTP to execute the handler. apiRouter.ServeHTTP(rec, req) @@ -128,7 +150,6 @@ func testPostPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandle t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) } } - } // postPresignSignatureV4 - presigned signature for PostPolicy requests. @@ -140,33 +161,29 @@ func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l return signature } -func newPostRequest(endPoint, bucketName, objectName string, objData []byte, shouldPass bool) (*http.Request, error) { +func newPostRequest(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) { // Keep time. t := time.Now().UTC() // Expire the request five minutes from now. expirationTime := t.Add(time.Minute * 5) // Get the user credential. - credentials := serverConfig.GetCredential() - credStr := getCredential(credentials.AccessKeyID, serverConfig.GetRegion(), t) + credStr := getCredential(accessKey, serverConfig.GetRegion(), t) // Create a new post policy. policy := newPostPolicyBytes(credStr, bucketName, objectName, expirationTime) // Only need the encoding. encodedPolicy := base64.StdEncoding.EncodeToString(policy) - formData := make(map[string]string) - if shouldPass { - // Presign with V4 signature based on the policy. - signature := postPresignSignatureV4(encodedPolicy, t, credentials.SecretAccessKey, serverConfig.GetRegion()) + // Presign with V4 signature based on the policy. + signature := postPresignSignatureV4(encodedPolicy, t, secretKey, serverConfig.GetRegion()) - formData = map[string]string{ - "bucket": bucketName, - "key": objectName, - "x-amz-credential": credStr, - "policy": encodedPolicy, - "x-amz-signature": signature, - "x-amz-date": t.Format(iso8601DateFormat), - "x-amz-algorithm": "AWS4-HMAC-SHA256", - } + formData := map[string]string{ + "bucket": bucketName, + "key": objectName, + "x-amz-credential": credStr, + "policy": encodedPolicy, + "x-amz-signature": signature, + "x-amz-date": t.Format(iso8601DateFormat), + "x-amz-algorithm": "AWS4-HMAC-SHA256", } // Create the multipart form. diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 98ae06cbb..c74e79bb1 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -211,6 +211,7 @@ func waitForFormattingDisks(disks, ignoredDisks []string) ([]StorageAPI, error) for index, disk := range disks { // Check if disk is ignored. if disksSet.Contains(disk) { + // Set this situation as disk not found. storageDisks[index] = nil continue } @@ -222,6 +223,6 @@ func waitForFormattingDisks(disks, ignoredDisks []string) ([]StorageAPI, error) } storageDisks[index] = storage } - + // Start wait loop retrying formatting disks. return retryFormattingDisks(disks, storageDisks) } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index b39a887dc..b6dc5950b 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -68,7 +68,7 @@ func prepareXL() (ObjectLayer, []string, error) { if err != nil { return nil, nil, err } - obj, err := getXLObjectLayer(fsDirs) + obj, err := getXLObjectLayer(fsDirs, nil) if err != nil { removeRoots(fsDirs) return nil, nil, err @@ -498,6 +498,8 @@ func signRequest(req *http.Request, accessKey, secretKey string) error { } sort.Strings(headers) + region := serverConfig.GetRegion() + // Get canonical headers. var buf bytes.Buffer for _, k := range headers { @@ -549,7 +551,7 @@ func signRequest(req *http.Request, accessKey, secretKey string) error { // Get scope. scope := strings.Join([]string{ currTime.Format(yyyymmdd), - "us-east-1", + region, "s3", "aws4_request", }, "/") @@ -559,8 +561,8 @@ func signRequest(req *http.Request, accessKey, secretKey string) error { stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest))) date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd))) - region := sumHMAC(date, []byte("us-east-1")) - service := sumHMAC(region, []byte("s3")) + regionHMAC := sumHMAC(date, []byte(region)) + service := sumHMAC(regionHMAC, []byte("s3")) signingKey := sumHMAC(service, []byte("aws4_request")) signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) @@ -731,7 +733,7 @@ func makeTestBackend(disks []string, instanceType string) (ObjectLayer, error) { return objLayer, err case "XL": - objectLayer, err := getXLObjectLayer(disks) + objectLayer, err := getXLObjectLayer(disks, nil) if err != nil { return nil, err } @@ -1107,12 +1109,12 @@ func getRandomDisks(N int) ([]string, error) { } // getXLObjectLayer - Instantiates XL object layer and returns it. -func getXLObjectLayer(erasureDisks []string) (ObjectLayer, error) { - err := formatDisks(erasureDisks, nil) +func getXLObjectLayer(erasureDisks []string, ignoredDisks []string) (ObjectLayer, error) { + err := formatDisks(erasureDisks, ignoredDisks) if err != nil { return nil, err } - objLayer, err := newXLObjects(erasureDisks, nil) + objLayer, err := newXLObjects(erasureDisks, ignoredDisks) if err != nil { return nil, err } @@ -1260,7 +1262,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) if err != nil { t.Fatalf("Initialization of disks for XL setup: %s", err) } - objLayer, err := getXLObjectLayer(erasureDisks) + objLayer, err := getXLObjectLayer(erasureDisks, nil) if err != nil { t.Fatalf("Initialization of object layer failed for XL setup: %s", err) } @@ -1288,6 +1290,9 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand // Iterate the list of API functions requested for and register them in mux HTTP handler. for _, apiFunction := range apiFunctions { switch apiFunction { + // Register ListBuckets handler. + case "ListBuckets": + apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler) // Register GetObject handler. case "GetObject`": bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler) diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index 5a98d503f..1c9d9622a 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -200,6 +200,10 @@ func (d byDiskTotal) Less(i, j int) bool { func (xl xlObjects) StorageInfo() StorageInfo { var disksInfo []disk.Info for _, storageDisk := range xl.storageDisks { + if storageDisk == nil { + // Storage disk is empty, perhaps ignored disk or not available. + continue + } info, err := storageDisk.DiskInfo() if err != nil { errorIf(err, "Unable to fetch disk info for %#v", storageDisk) diff --git a/cmd/xl-v1_test.go b/cmd/xl-v1_test.go index 5542c109a..a17dec5ea 100644 --- a/cmd/xl-v1_test.go +++ b/cmd/xl-v1_test.go @@ -115,6 +115,25 @@ func TestStorageInfo(t *testing.T) { if disks16Info.Total <= 0 { t.Fatalf("Diskinfo total values should be greater 0") } + + objLayer, err = newXLObjects(fsDirs, fsDirs[:4]) + if err != nil { + t.Fatalf("Unable to initialize 'XL' object layer with ignored disks %s.", fsDirs[:4]) + } + + // Get storage info first attempt. + disks16Info = objLayer.StorageInfo() + + // This test assumes homogenity between all disks, + // i.e if we loose one disk the effective storage + // usage values is assumed to decrease. If we have + // heterogenous environment this is not true all the time. + if disks16Info.Free <= 0 { + t.Fatalf("Diskinfo total free values should be greater 0") + } + if disks16Info.Total <= 0 { + t.Fatalf("Diskinfo total values should be greater 0") + } } // TestNewXL - tests initialization of all input disks