mirror of
				https://github.com/minio/minio.git
				synced 2025-10-31 16:21:49 +01:00 
			
		
		
		
	With this change, MinIO's ILM supports transitioning objects to a remote tier. This change includes support for Azure Blob Storage, AWS S3 compatible object storage incl. MinIO and Google Cloud Storage as remote tier storage backends. Some new additions include: - Admin APIs remote tier configuration management - Simple journal to track remote objects to be 'collected' This is used by object API handlers which 'mutate' object versions by overwriting/replacing content (Put/CopyObject) or removing the version itself (e.g DeleteObjectVersion). - Rework of previous ILM transition to fit the new model In the new model, a storage class (a.k.a remote tier) is defined by the 'remote' object storage type (one of s3, azure, GCS), bucket name and a prefix. * Fixed bugs, review comments, and more unit-tests - Leverage inline small object feature - Migrate legacy objects to the latest object format before transitioning - Fix restore to particular version if specified - Extend SharedDataDirCount to handle transitioned and restored objects - Restore-object should accept version-id for version-suspended bucket (#12091) - Check if remote tier creds have sufficient permissions - Bonus minor fixes to existing error messages Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io> Co-authored-by: Krishna Srinivas <krishna@minio.io> Signed-off-by: Harshavardhana <harsha@minio.io>
		
			
				
	
	
		
			133 lines
		
	
	
		
			4.6 KiB
		
	
	
	
		
			Python
		
	
	
	
	
	
			
		
		
	
	
			133 lines
		
	
	
		
			4.6 KiB
		
	
	
	
		
			Python
		
	
	
	
	
	
| # -*- coding: utf-8 -*-
 | |
| import json
 | |
| # standard.
 | |
| import os
 | |
| 
 | |
| import certifi
 | |
| # Dependencies
 | |
| import urllib3
 | |
| from botocore.credentials import CredentialProvider, RefreshableCredentials
 | |
| from botocore.exceptions import CredentialRetrievalError
 | |
| from dateutil.parser import parse
 | |
| 
 | |
| from .sts_element import STSElement
 | |
| 
 | |
| 
 | |
| class ClientGrantsCredentialProvider(CredentialProvider):
 | |
|     """
 | |
|     ClientGrantsCredentialProvider implements CredentialProvider compatible
 | |
|     implementation to be used with boto_session
 | |
|     """
 | |
|     METHOD = 'assume-role-client-grants'
 | |
|     CANONICAL_NAME = 'AssumeRoleClientGrants'
 | |
| 
 | |
|     def __init__(self, cid, csec,
 | |
|                  idp_ep='http://localhost:8080/auth/realms/minio/protocol/openid-connect/token',
 | |
|                  sts_ep='http://localhost:9000'):
 | |
|         self.cid = cid
 | |
|         self.csec = csec
 | |
|         self.idp_ep = idp_ep
 | |
|         self.sts_ep = sts_ep
 | |
| 
 | |
|         # Load CA certificates from SSL_CERT_FILE file if set
 | |
|         ca_certs = os.environ.get('SSL_CERT_FILE')
 | |
|         if not ca_certs:
 | |
|             ca_certs = certifi.where()
 | |
| 
 | |
|         self._http = urllib3.PoolManager(
 | |
|             timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
 | |
|             maxsize=10,
 | |
|             cert_reqs='CERT_NONE',
 | |
|             ca_certs=ca_certs,
 | |
|             retries=urllib3.Retry(
 | |
|                 total=5,
 | |
|                 backoff_factor=0.2,
 | |
|                 status_forcelist=[500, 502, 503, 504]
 | |
|             )
 | |
|         )
 | |
| 
 | |
|     def load(self):
 | |
|         """
 | |
|         Search for credentials with client_grants
 | |
|         """
 | |
|         if self.cid is not None:
 | |
|             fetcher = self._create_credentials_fetcher()
 | |
|             return RefreshableCredentials.create_from_metadata(
 | |
|                 metadata=fetcher(),
 | |
|                 refresh_using=fetcher,
 | |
|                 method=self.METHOD,
 | |
|             )
 | |
|         else:
 | |
|             return None
 | |
| 
 | |
|     def _create_credentials_fetcher(self):
 | |
|         method = self.METHOD
 | |
| 
 | |
|         def fetch_credentials():
 | |
|             # HTTP headers are case insensitive filter out
 | |
|             # all duplicate headers and pick one.
 | |
|             headers = {}
 | |
|             headers['content-type'] = 'application/x-www-form-urlencoded'
 | |
|             headers['authorization'] = urllib3.make_headers(
 | |
|                 basic_auth='%s:%s' % (self.cid, self.csec))['authorization']
 | |
| 
 | |
|             response = self._http.urlopen('POST', self.idp_ep,
 | |
|                                           body="grant_type=client_credentials",
 | |
|                                           headers=headers,
 | |
|                                           preload_content=True,
 | |
|                                           )
 | |
|             if response.status != 200:
 | |
|                 message = "Credential refresh failed, response: %s"
 | |
|                 raise CredentialRetrievalError(
 | |
|                     provider=method,
 | |
|                     error_msg=message % response.status,
 | |
|                 )
 | |
| 
 | |
|             creds = json.loads(response.data)
 | |
| 
 | |
|             query = {}
 | |
|             query['Action'] = 'AssumeRoleWithClientGrants'
 | |
|             query['Token'] = creds['access_token']
 | |
|             query['DurationSeconds'] = creds['expires_in']
 | |
|             query['Version'] = '2011-06-15'
 | |
| 
 | |
|             query_components = []
 | |
|             for key in query:
 | |
|                 if query[key] is not None:
 | |
|                     query_components.append("%s=%s" % (key, query[key]))
 | |
| 
 | |
|             query_string = '&'.join(query_components)
 | |
|             sts_ep_url = self.sts_ep
 | |
|             if query_string:
 | |
|                 sts_ep_url = self.sts_ep + '?' + query_string
 | |
| 
 | |
|             response = self._http.urlopen(
 | |
|                 'POST', sts_ep_url, preload_content=True)
 | |
|             if response.status != 200:
 | |
|                 message = "Credential refresh failed, response: %s"
 | |
|                 raise CredentialRetrievalError(
 | |
|                     provider=method,
 | |
|                     error_msg=message % response.status,
 | |
|                 )
 | |
| 
 | |
|             return parse_grants_response(response.data)
 | |
| 
 | |
|         def parse_grants_response(data):
 | |
|             """
 | |
|             Parser for AssumeRoleWithClientGrants response
 | |
| 
 | |
|             :param data: Response data for AssumeRoleWithClientGrants request
 | |
|             :return: dict
 | |
|             """
 | |
|             root = STSElement.fromstring(
 | |
|                 'AssumeRoleWithClientGrantsResponse', data)
 | |
|             result = root.find('AssumeRoleWithClientGrantsResult')
 | |
|             creds = result.find('Credentials')
 | |
|             return dict(
 | |
|                 access_key=creds.get_child_text('AccessKeyId'),
 | |
|                 secret_key=creds.get_child_text('SecretAccessKey'),
 | |
|                 token=creds.get_child_text('SessionToken'),
 | |
|                 expiry_time=parse(creds.get_child_text('Expiration')).isoformat())
 | |
| 
 | |
|         return fetch_credentials
 |