mirror of
				https://github.com/minio/minio.git
				synced 2025-10-31 00:01:27 +01:00 
			
		
		
		
	With this change, MinIO's ILM supports transitioning objects to a remote tier. This change includes support for Azure Blob Storage, AWS S3 compatible object storage incl. MinIO and Google Cloud Storage as remote tier storage backends. Some new additions include: - Admin APIs remote tier configuration management - Simple journal to track remote objects to be 'collected' This is used by object API handlers which 'mutate' object versions by overwriting/replacing content (Put/CopyObject) or removing the version itself (e.g DeleteObjectVersion). - Rework of previous ILM transition to fit the new model In the new model, a storage class (a.k.a remote tier) is defined by the 'remote' object storage type (one of s3, azure, GCS), bucket name and a prefix. * Fixed bugs, review comments, and more unit-tests - Leverage inline small object feature - Migrate legacy objects to the latest object format before transitioning - Fix restore to particular version if specified - Extend SharedDataDirCount to handle transitioned and restored objects - Restore-object should accept version-id for version-suspended bucket (#12091) - Check if remote tier creds have sufficient permissions - Bonus minor fixes to existing error messages Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io> Co-authored-by: Krishna Srinivas <krishna@minio.io> Signed-off-by: Harshavardhana <harsha@minio.io>
		
			
				
	
	
		
			185 lines
		
	
	
		
			6.8 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
			
		
		
	
	
			185 lines
		
	
	
		
			6.8 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
| // Copyright (c) 2015-2021 MinIO, Inc.
 | |
| //
 | |
| // This file is part of MinIO Object Storage stack
 | |
| //
 | |
| // This program is free software: you can redistribute it and/or modify
 | |
| // it under the terms of the GNU Affero General Public License as published by
 | |
| // the Free Software Foundation, either version 3 of the License, or
 | |
| // (at your option) any later version.
 | |
| //
 | |
| // This program is distributed in the hope that it will be useful
 | |
| // but WITHOUT ANY WARRANTY; without even the implied warranty of
 | |
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 | |
| // GNU Affero General Public License for more details.
 | |
| //
 | |
| // You should have received a copy of the GNU Affero General Public License
 | |
| // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 | |
| 
 | |
| package http
 | |
| 
 | |
| // Standard S3 HTTP response constants
 | |
| const (
 | |
| 	LastModified       = "Last-Modified"
 | |
| 	Date               = "Date"
 | |
| 	ETag               = "ETag"
 | |
| 	ContentType        = "Content-Type"
 | |
| 	ContentMD5         = "Content-Md5"
 | |
| 	ContentEncoding    = "Content-Encoding"
 | |
| 	Expires            = "Expires"
 | |
| 	ContentLength      = "Content-Length"
 | |
| 	ContentLanguage    = "Content-Language"
 | |
| 	ContentRange       = "Content-Range"
 | |
| 	Connection         = "Connection"
 | |
| 	AcceptRanges       = "Accept-Ranges"
 | |
| 	AmzBucketRegion    = "X-Amz-Bucket-Region"
 | |
| 	ServerInfo         = "Server"
 | |
| 	RetryAfter         = "Retry-After"
 | |
| 	Location           = "Location"
 | |
| 	CacheControl       = "Cache-Control"
 | |
| 	ContentDisposition = "Content-Disposition"
 | |
| 	Authorization      = "Authorization"
 | |
| 	Action             = "Action"
 | |
| 	Range              = "Range"
 | |
| )
 | |
| 
 | |
| // Non standard S3 HTTP response constants
 | |
| const (
 | |
| 	XCache       = "X-Cache"
 | |
| 	XCacheLookup = "X-Cache-Lookup"
 | |
| )
 | |
| 
 | |
| // Standard S3 HTTP request constants
 | |
| const (
 | |
| 	IfModifiedSince   = "If-Modified-Since"
 | |
| 	IfUnmodifiedSince = "If-Unmodified-Since"
 | |
| 	IfMatch           = "If-Match"
 | |
| 	IfNoneMatch       = "If-None-Match"
 | |
| 
 | |
| 	// S3 storage class
 | |
| 	AmzStorageClass = "x-amz-storage-class"
 | |
| 
 | |
| 	// S3 object version ID
 | |
| 	AmzVersionID    = "x-amz-version-id"
 | |
| 	AmzDeleteMarker = "x-amz-delete-marker"
 | |
| 
 | |
| 	// S3 object tagging
 | |
| 	AmzObjectTagging = "X-Amz-Tagging"
 | |
| 	AmzTagCount      = "x-amz-tagging-count"
 | |
| 	AmzTagDirective  = "X-Amz-Tagging-Directive"
 | |
| 
 | |
| 	// S3 transition restore
 | |
| 	AmzRestore            = "x-amz-restore"
 | |
| 	AmzRestoreExpiryDays  = "X-Amz-Restore-Expiry-Days"
 | |
| 	AmzRestoreRequestDate = "X-Amz-Restore-Request-Date"
 | |
| 	AmzRestoreOutputPath  = "x-amz-restore-output-path"
 | |
| 
 | |
| 	// S3 extensions
 | |
| 	AmzCopySourceIfModifiedSince   = "x-amz-copy-source-if-modified-since"
 | |
| 	AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since"
 | |
| 
 | |
| 	AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match"
 | |
| 	AmzCopySourceIfMatch     = "x-amz-copy-source-if-match"
 | |
| 
 | |
| 	AmzCopySource                 = "X-Amz-Copy-Source"
 | |
| 	AmzCopySourceVersionID        = "X-Amz-Copy-Source-Version-Id"
 | |
| 	AmzCopySourceRange            = "X-Amz-Copy-Source-Range"
 | |
| 	AmzMetadataDirective          = "X-Amz-Metadata-Directive"
 | |
| 	AmzObjectLockMode             = "X-Amz-Object-Lock-Mode"
 | |
| 	AmzObjectLockRetainUntilDate  = "X-Amz-Object-Lock-Retain-Until-Date"
 | |
| 	AmzObjectLockLegalHold        = "X-Amz-Object-Lock-Legal-Hold"
 | |
| 	AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention"
 | |
| 	AmzBucketReplicationStatus    = "X-Amz-Replication-Status"
 | |
| 	AmzSnowballExtract            = "X-Amz-Meta-Snowball-Auto-Extract"
 | |
| 
 | |
| 	// Multipart parts count
 | |
| 	AmzMpPartsCount = "x-amz-mp-parts-count"
 | |
| 
 | |
| 	// Object date/time of expiration
 | |
| 	AmzExpiration = "x-amz-expiration"
 | |
| 
 | |
| 	// Dummy putBucketACL
 | |
| 	AmzACL = "x-amz-acl"
 | |
| 
 | |
| 	// Signature V4 related contants.
 | |
| 	AmzContentSha256        = "X-Amz-Content-Sha256"
 | |
| 	AmzDate                 = "X-Amz-Date"
 | |
| 	AmzAlgorithm            = "X-Amz-Algorithm"
 | |
| 	AmzExpires              = "X-Amz-Expires"
 | |
| 	AmzSignedHeaders        = "X-Amz-SignedHeaders"
 | |
| 	AmzSignature            = "X-Amz-Signature"
 | |
| 	AmzCredential           = "X-Amz-Credential"
 | |
| 	AmzSecurityToken        = "X-Amz-Security-Token"
 | |
| 	AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
 | |
| 
 | |
| 	AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length"
 | |
| 	AmzMetaUnencryptedContentMD5    = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5"
 | |
| 
 | |
| 	// AWS server-side encryption headers for SSE-S3, SSE-KMS and SSE-C.
 | |
| 	AmzServerSideEncryption                      = "X-Amz-Server-Side-Encryption"
 | |
| 	AmzServerSideEncryptionKmsID                 = AmzServerSideEncryption + "-Aws-Kms-Key-Id"
 | |
| 	AmzServerSideEncryptionKmsContext            = AmzServerSideEncryption + "-Context"
 | |
| 	AmzServerSideEncryptionCustomerAlgorithm     = AmzServerSideEncryption + "-Customer-Algorithm"
 | |
| 	AmzServerSideEncryptionCustomerKey           = AmzServerSideEncryption + "-Customer-Key"
 | |
| 	AmzServerSideEncryptionCustomerKeyMD5        = AmzServerSideEncryption + "-Customer-Key-Md5"
 | |
| 	AmzServerSideEncryptionCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
 | |
| 	AmzServerSideEncryptionCopyCustomerKey       = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
 | |
| 	AmzServerSideEncryptionCopyCustomerKeyMD5    = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
 | |
| 
 | |
| 	AmzEncryptionAES = "AES256"
 | |
| 	AmzEncryptionKMS = "aws:kms"
 | |
| 
 | |
| 	// Signature v2 related constants
 | |
| 	AmzSignatureV2 = "Signature"
 | |
| 	AmzAccessKeyID = "AWSAccessKeyId"
 | |
| 
 | |
| 	// Response request id.
 | |
| 	AmzRequestID = "x-amz-request-id"
 | |
| 
 | |
| 	// Deployment id.
 | |
| 	MinioDeploymentID = "x-minio-deployment-id"
 | |
| 
 | |
| 	// Server-Status
 | |
| 	MinIOServerStatus = "x-minio-server-status"
 | |
| 
 | |
| 	// Delete special flag to force delete a bucket
 | |
| 	MinIOForceDelete = "x-minio-force-delete"
 | |
| 
 | |
| 	// Header indicates if the mtime should be preserved by client
 | |
| 	MinIOSourceMTime = "x-minio-source-mtime"
 | |
| 
 | |
| 	// Header indicates if the etag should be preserved by client
 | |
| 	MinIOSourceETag = "x-minio-source-etag"
 | |
| 
 | |
| 	// Writes expected write quorum
 | |
| 	MinIOWriteQuorum = "x-minio-write-quorum"
 | |
| 
 | |
| 	// Reports number of drives currently healing
 | |
| 	MinIOHealingDrives = "x-minio-healing-drives"
 | |
| 
 | |
| 	// Header indicates if the delete marker should be preserved by client
 | |
| 	MinIOSourceDeleteMarker = "x-minio-source-deletemarker"
 | |
| 
 | |
| 	// Header indicates if the delete marker version needs to be purged.
 | |
| 	MinIOSourceDeleteMarkerDelete = "x-minio-source-deletemarker-delete"
 | |
| 
 | |
| 	// Header indicates permanent delete replication status.
 | |
| 	MinIODeleteReplicationStatus = "X-Minio-Replication-Delete-Status"
 | |
| 	// Header indicates delete-marker replication status.
 | |
| 	MinIODeleteMarkerReplicationStatus = "X-Minio-Replication-DeleteMarker-Status"
 | |
| 	// Header indicates if its a GET/HEAD proxy request for active-active replication
 | |
| 	MinIOSourceProxyRequest = "X-Minio-Source-Proxy-Request"
 | |
| 	// Header indicates that this request is a replication request to create a REPLICA
 | |
| 	MinIOSourceReplicationRequest = "X-Minio-Source-Replication-Request"
 | |
| 	// predicted date/time of transition
 | |
| 	MinIOTransition = "X-Minio-Transition"
 | |
| )
 | |
| 
 | |
| // Common http query params S3 API
 | |
| const (
 | |
| 	VersionID = "versionId"
 | |
| 
 | |
| 	PartNumber = "partNumber"
 | |
| 
 | |
| 	UploadID = "uploadId"
 | |
| )
 |