From 07a556a10b934326f3e4d7115232b2a124342e69 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 1 Nov 2019 16:58:11 -0700 Subject: [PATCH] Avoid ListBuckets() call instead rely on simple HTTP GET (#8475) This is to avoid making calls to backend and requiring gateways to allow permissions for ListBuckets() operation just for Liveness checks, we can avoid this and make our liveness checks to be more performant. --- cmd/gateway-common.go | 24 ++++++++++++++++++++++++ cmd/gateway/azure/gateway-azure.go | 10 ++++++++-- cmd/gateway/b2/gateway-b2.go | 17 +++++++++++------ cmd/gateway/gcs/gateway-gcs.go | 14 ++++++++++---- cmd/gateway/hdfs/gateway-hdfs.go | 3 ++- cmd/gateway/nas/gateway-nas.go | 3 ++- cmd/gateway/oss/gateway-oss.go | 8 ++++++-- cmd/gateway/s3/gateway-s3.go | 9 ++++++++- cmd/healthcheck-handler.go | 9 +++------ cmd/object-api-datatypes.go | 7 ++++++- 10 files changed, 80 insertions(+), 24 deletions(-) diff --git a/cmd/gateway-common.go b/cmd/gateway-common.go index f69c2656e..af91941ca 100644 --- a/cmd/gateway-common.go +++ b/cmd/gateway-common.go @@ -17,8 +17,10 @@ package cmd import ( + "context" "net/http" "strings" + "time" "github.com/minio/minio/cmd/config" xhttp "github.com/minio/minio/cmd/http" @@ -286,6 +288,28 @@ func ToMinioClientCompleteParts(parts []CompletePart) []minio.CompletePart { return mparts } +// IsBackendOnline - verifies if the backend is reachable +// by performing a GET request on the URL. returns 'true' +// if backend is reachable. +func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool { + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + // never follow redirects + clnt.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return false + } + if _, err = clnt.Do(req); err != nil { + return !xnet.IsNetworkOrHostDown(err) + } + return true +} + // ErrorRespToObjectError converts MinIO errors to minio object layer errors. func ErrorRespToObjectError(err error, params ...string) error { if err == nil { diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index a2001f6c0..82a33dc55 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -185,7 +185,9 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro c.HTTPClient = &http.Client{Transport: minio.NewCustomHTTPTransport()} return &azureObjects{ - client: c.GetBlobService(), + endpoint: fmt.Sprintf("https://%s.blob.core.windows.net", creds.AccessKey), + httpClient: c.HTTPClient, + client: c.GetBlobService(), }, nil } @@ -343,7 +345,9 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper // azureObjects - Implements Object layer for Azure blob storage. type azureObjects struct { minio.GatewayUnsupported - client storage.BlobStorageClient // Azure sdk client + endpoint string + httpClient *http.Client + client storage.BlobStorageClient // Azure sdk client } // Convert azure errors to minio object layer errors. @@ -447,6 +451,8 @@ func (a *azureObjects) Shutdown(ctx context.Context) error { // StorageInfo - Not relevant to Azure backend. func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { + si.Backend.Type = minio.BackendGateway + si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint) return si } diff --git a/cmd/gateway/b2/gateway-b2.go b/cmd/gateway/b2/gateway-b2.go index 833bc17ad..18c6bc4d8 100644 --- a/cmd/gateway/b2/gateway-b2.go +++ b/cmd/gateway/b2/gateway-b2.go @@ -123,7 +123,10 @@ func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) return &b2Objects{ creds: creds, b2Client: client, - ctx: ctx, + httpClient: &http.Client{ + Transport: minio.NewCustomHTTPTransport(), + }, + ctx: ctx, }, nil } @@ -135,10 +138,11 @@ func (g *B2) Production() bool { // b2Object implements gateway for MinIO and BackBlaze B2 compatible object storage servers. type b2Objects struct { minio.GatewayUnsupported - mu sync.Mutex - creds auth.Credentials - b2Client *b2.B2 - ctx context.Context + mu sync.Mutex + creds auth.Credentials + b2Client *b2.B2 + httpClient *http.Client + ctx context.Context } // Convert B2 errors to minio object layer errors. @@ -225,12 +229,13 @@ func b2MsgCodeToObjectError(code int, msgCode string, msg string, params ...stri // Shutdown saves any gateway metadata to disk // if necessary and reload upon next restart. func (l *b2Objects) Shutdown(ctx context.Context) error { - // TODO return nil } // StorageInfo is not relevant to B2 backend. func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { + si.Backend.Type = minio.BackendGateway + si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1") return si } diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 86d3fc8f4..c6f5da989 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -208,6 +208,9 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) gcs := &gcsGateway{ client: client, projectID: g.projectID, + httpClient: &http.Client{ + Transport: minio.NewCustomHTTPTransport(), + }, } // Start background process to cleanup old files in minio.sys.tmp @@ -346,8 +349,9 @@ func isValidGCSProjectIDFormat(projectID string) bool { // gcsGateway - Implements gateway for MinIO and GCS compatible object storage servers. type gcsGateway struct { minio.GatewayUnsupported - client *storage.Client - projectID string + client *storage.Client + httpClient *http.Client + projectID string } // Returns projectID from the GOOGLE_APPLICATION_CREDENTIALS file. @@ -412,8 +416,10 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error { } // StorageInfo - Not relevant to GCS backend. -func (l *gcsGateway) StorageInfo(ctx context.Context) minio.StorageInfo { - return minio.StorageInfo{} +func (l *gcsGateway) StorageInfo(ctx context.Context) (si minio.StorageInfo) { + si.Backend.Type = minio.BackendGateway + si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com") + return si } // MakeBucketWithLocation - Create a new container on GCS backend. diff --git a/cmd/gateway/hdfs/gateway-hdfs.go b/cmd/gateway/hdfs/gateway-hdfs.go index e58ae4183..8a01aac4b 100644 --- a/cmd/gateway/hdfs/gateway-hdfs.go +++ b/cmd/gateway/hdfs/gateway-hdfs.go @@ -227,7 +227,8 @@ func (n *hdfsObjects) StorageInfo(ctx context.Context) minio.StorageInfo { } sinfo := minio.StorageInfo{} sinfo.Used = []uint64{fsInfo.Used} - sinfo.Backend.Type = minio.Unknown + sinfo.Backend.Type = minio.BackendGateway + sinfo.Backend.GatewayOnline = true return sinfo } diff --git a/cmd/gateway/nas/gateway-nas.go b/cmd/gateway/nas/gateway-nas.go index 71e162285..28e32d787 100644 --- a/cmd/gateway/nas/gateway-nas.go +++ b/cmd/gateway/nas/gateway-nas.go @@ -125,7 +125,8 @@ func (n *nasObjects) IsListenBucketSupported() bool { func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo { sinfo := n.ObjectLayer.StorageInfo(ctx) - sinfo.Backend.Type = minio.Unknown + sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS + sinfo.Backend.Type = minio.BackendGateway return sinfo } diff --git a/cmd/gateway/oss/gateway-oss.go b/cmd/gateway/oss/gateway-oss.go index 38286eaa0..350b4ad89 100644 --- a/cmd/gateway/oss/gateway-oss.go +++ b/cmd/gateway/oss/gateway-oss.go @@ -145,7 +145,9 @@ func (g *OSS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) if err != nil { return nil, err } - + client.HTTPClient = &http.Client{ + Transport: minio.NewCustomHTTPTransport(), + } return &ossObjects{ Client: client, }, nil @@ -341,7 +343,9 @@ func (l *ossObjects) Shutdown(ctx context.Context) error { // StorageInfo is not relevant to OSS backend. func (l *ossObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { - return + si.Backend.Type = minio.BackendGateway + si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint) + return si } // ossIsValidBucketName verifies whether a bucket name is valid. diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index bdcf96d10..906439936 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -261,7 +261,11 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) s := s3Objects{ Client: clnt, + HTTPClient: &http.Client{ + Transport: minio.NewCustomHTTPTransport(), + }, } + // Enables single encryption of KMS is configured. if minio.GlobalKMS != nil { encS := s3EncObjects{s} @@ -283,7 +287,8 @@ func (g *S3) Production() bool { // s3Objects implements gateway for MinIO and S3 compatible object storage servers. type s3Objects struct { minio.GatewayUnsupported - Client *miniogo.Core + Client *miniogo.Core + HTTPClient *http.Client } // Shutdown saves any gateway metadata to disk @@ -294,6 +299,8 @@ func (l *s3Objects) Shutdown(ctx context.Context) error { // StorageInfo is not relevant to S3 backend. func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { + si.Backend.Type = minio.BackendGateway + si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String()) return si } diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go index f77b420d7..081f328fb 100644 --- a/cmd/healthcheck-handler.go +++ b/cmd/healthcheck-handler.go @@ -59,18 +59,15 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) { // is able to start on orchestration platforms like Docker Swarm. // Refer https://github.com/minio/minio/issues/8140 for more details. // Make sure to add server not initialized status in header - w.Header().Set(xhttp.MinIOServerStatus, "Server-not-initialized") + w.Header().Set(xhttp.MinIOServerStatus, "server-not-initialized") writeSuccessResponseHeadersOnly(w) return } if !globalIsXL && !globalIsDistXL { s := objLayer.StorageInfo(ctx) - // Gateways don't provide disk info. - if s.Backend.Type == Unknown { - // ListBuckets to confirm gateway backend is up - if _, err := objLayer.ListBuckets(ctx); err != nil { - logger.LogOnceIf(ctx, err, struct{}{}) + if s.Backend.Type == BackendGateway { + if !s.Backend.GatewayOnline { writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) return } diff --git a/cmd/object-api-datatypes.go b/cmd/object-api-datatypes.go index 043530ad4..91830160b 100644 --- a/cmd/object-api-datatypes.go +++ b/cmd/object-api-datatypes.go @@ -34,6 +34,8 @@ const ( BackendFS // Multi disk BackendErasure (single, distributed) backend. BackendErasure + // Gateway backend. + BackendGateway // Add your own backend. ) @@ -49,9 +51,12 @@ type StorageInfo struct { // Backend type. Backend struct { - // Represents various backend types, currently on FS and Erasure. + // Represents various backend types, currently on FS, Erasure and Gateway Type BackendType + // Following fields are only meaningful if BackendType is Gateway. + GatewayOnline bool + // Following fields are only meaningful if BackendType is Erasure. OnlineDisks madmin.BackendDisks // Online disks during server startup. OfflineDisks madmin.BackendDisks // Offline disks during server startup.