Source code for NGPIris.hcp.hcp


from NGPIris.parse_credentials import CredentialsHandler
from NGPIris.hcp.helpers import (
    raise_path_error,
    create_access_control_policy,
    check_mounted
)
from NGPIris.hcp.exceptions import *

from boto3 import client
from botocore.client import Config
from botocore.exceptions import EndpointConnectionError, ClientError
from boto3.s3.transfer import TransferConfig
from configparser import ConfigParser

from os import (
    path,
    stat,
    listdir
)
from json import dumps
from parse import (
    parse,
    search,
    Result
)
from requests import get
from urllib3 import disable_warnings
from tqdm import tqdm

_KB = 1024
_MB = _KB * _KB

[docs] class HCPHandler: def __init__(self, credentials_path : str, use_ssl : bool = False, proxy_path : str = "", custom_config_path : str = "") -> None: """ Class for handling HCP requests. :param credentials_path: Path to the JSON credentials file :type credentials_path: str :param use_ssl: Boolean choice between using SSL, defaults to False :type use_ssl: bool, optional :param custom_config_path: Path to a .ini file for customs settings regarding download and upload :type custom_config_path: str, optional """ credentials_handler = CredentialsHandler(credentials_path) self.hcp = credentials_handler.hcp self.endpoint = "https://" + self.hcp["endpoint"] tenant_parse = parse("https://{}.hcp1.vgregion.se", self.endpoint) if type(tenant_parse) is Result: self.tenant = str(tenant_parse[0]) else: # pragma: no cover raise RuntimeError("Unable to parse endpoint. Make sure that you have entered the correct endpoint in your credentials JSON file. Hint: The endpoint should *not* contain \"https://\" or port numbers") self.base_request_url = self.endpoint + ":9090/mapi/tenants/" + self.tenant self.aws_access_key_id = self.hcp["aws_access_key_id"] self.aws_secret_access_key = self.hcp["aws_secret_access_key"] self.token = self.aws_access_key_id + ":" + self.aws_secret_access_key self.bucket_name = None self.use_ssl = use_ssl if not self.use_ssl: disable_warnings() if proxy_path: # pragma: no cover s3_config = Config( s3 = { "addressing_style": "path", "payload_signing_enabled": True }, signature_version = "s3v4", proxies = CredentialsHandler(proxy_path).hcp ) else: s3_config = Config( s3 = { "addressing_style": "path", "payload_signing_enabled": True }, signature_version = "s3v4" ) self.s3_client = client( "s3", aws_access_key_id = self.aws_access_key_id, aws_secret_access_key = self.aws_secret_access_key, endpoint_url = self.endpoint, verify = self.use_ssl, config = s3_config ) if custom_config_path: # pragma: no cover ini_config = ConfigParser() ini_config.read(custom_config_path) self.transfer_config = TransferConfig( multipart_threshold = ini_config.getint("hcp", "multipart_threshold"), max_concurrency = ini_config.getint("hcp", "max_concurrency"), multipart_chunksize = ini_config.getint("hcp", "multipart_chunksize"), use_threads = ini_config.getboolean("hcp", "use_threads") ) else: self.transfer_config = TransferConfig( multipart_threshold = 10 * _MB, max_concurrency = 60, multipart_chunksize = 40 * _MB, use_threads = True )
[docs] def get_response(self, path_extension : str = "") -> dict: """ Make a request to the HCP in order to use the builtin MAPI :param path_extension: Extension for the base request URL, defaults to the empty string :type path_extension: str, optional :return: The response as a dictionary :rtype: dict """ url = self.base_request_url + path_extension headers = { "Authorization": "HCP " + self.token, "Cookie": "hcp-ns-auth=" + self.token, "Accept": "application/json" } response = get( url, headers=headers, verify=self.use_ssl ) response.raise_for_status() return dict(response.json())
[docs] def test_connection(self, bucket_name : str = "") -> dict: """ Test the connection to the mounted bucket or another bucket which is supplied as the argument :py:obj:`bucket_name`. :param bucket_name: The name of the bucket to be mounted. Defaults to the empty string :type bucket_name: str, optional :raises RuntimeError: If no bucket is selected :raises VPNConnectionError: If there is no VPN connection :raises BucketNotFound: If no bucket of that name was found :raises Exception: Other exceptions :return: A dictionary of the response :rtype: dict """ if not bucket_name and self.bucket_name: bucket_name = self.bucket_name elif bucket_name: pass else: raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` paramter for `test_connection`") try: response = dict(self.s3_client.head_bucket(Bucket = bucket_name)) except EndpointConnectionError as e: # pragma: no cover print(e) raise VPNConnectionError("Please check your connection and that you have your VPN enabled") except ClientError as e: print(e) raise BucketNotFound("Bucket \"" + bucket_name + "\" was not found") except Exception as e: # pragma: no cover raise Exception(e) if response["ResponseMetadata"].get("HTTPStatusCode", -1) != 200: # pragma: no cover error_msg = "The response code from the request made at " + self.endpoint + " returned status code " + response["ResponseMetadata"]["HTTPStatusCode"] raise Exception(error_msg) return response
[docs] def mount_bucket(self, bucket_name : str) -> None: """ Mount bucket that is to be used. This method needs to executed in order for most of the other methods to work. It mainly concerns operations with download and upload. :param bucket_name: The name of the bucket to be mounted :type bucket_name: str """ # Check if bucket exist self.test_connection(bucket_name = bucket_name) self.bucket_name = bucket_name
[docs] def list_buckets(self) -> list[str]: """ List all available buckets at endpoint. :return: A list of buckets :rtype: list[str] """ response = self.get_response("/namespaces") list_of_buckets : list[str] = response["name"] return list_of_buckets
@check_mounted def list_objects(self, name_only : bool = False) -> list: """ List all objects in the mounted bucket :param name_only: If True, return only a list of the object names. If False, return the full metadata about each object. Defaults to False. :type name_only: bool, optional :return: A list of of either strings or a list of object metadata (the form of a dictionary) :rtype: list """ response_list_objects = dict(self.s3_client.list_objects_v2( Bucket = self.bucket_name )) if "Contents" not in response_list_objects.keys(): # pragma: no cover return [] list_of_objects : list[dict] = response_list_objects["Contents"] if name_only: return [object["Key"] for object in list_of_objects] else: return list_of_objects @check_mounted def get_object(self, key : str) -> dict: """ Retrieve object metadata :param key: The object name :type key: str :return: A dictionary containing the object metadata :rtype: dict """ response = dict(self.s3_client.get_object( Bucket = self.bucket_name, Key = key )) return response @check_mounted def object_exists(self, key : str) -> bool: """ Check if a given object is in the mounted bucket :param key: The object name :type key: str :return: True if the object exist, otherwise False :rtype: bool """ try: response = self.get_object(key) if response["ResponseMetadata"]["HTTPStatusCode"] == 200: return True else: # pragma: no cover return False except: # pragma: no cover return False @check_mounted def download_file(self, key : str, local_file_path : str) -> None: """ Download one object file from the mounted bucket :param key: Name of the object :type key: str :param local_file_path: Path to a file on your local system where the contents of the object file can be put. :type local_file_path: str """ try: file_size : int = self.s3_client.head_object(Bucket = self.bucket_name, Key = key)["ContentLength"] with tqdm( total = file_size, unit = "B", unit_scale = True, desc = key ) as pbar: self.s3_client.download_file( Bucket = self.bucket_name, Key = key, Filename = local_file_path, Config = self.transfer_config, Callback = lambda bytes_transferred : pbar.update(bytes_transferred) ) except ClientError as e0: print(str(e0)) raise Exception("Could not find object", "\"" + key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"") except Exception as e: # pragma: no cover raise Exception(e) @check_mounted def upload_file(self, local_file_path : str, key : str = "") -> None: """ Upload one file to the mounted bucket :param local_file_path: Path to the file to be uploaded :type local_file_path: str :param key: An optional new name for the file object on the bucket. Defaults to the same name as the file :type key: str, optional """ raise_path_error(local_file_path) if not key: file_name = path.basename(local_file_path) key = file_name file_size : int = stat(local_file_path).st_size with tqdm( total = file_size, unit = "B", unit_scale = True, desc = local_file_path ) as pbar: self.s3_client.upload_file( Filename = local_file_path, Bucket = self.bucket_name, Key = key, Config = self.transfer_config, Callback = lambda bytes_transferred : pbar.update(bytes_transferred) ) @check_mounted def upload_folder(self, local_folder_path : str, key : str = "") -> None: """ Upload the contents of a folder to the mounted bucket :param local_folder_path: Path to the folder to be uploaded :type local_folder_path: str :param key: An optional new name for the folder path on the bucket. Defaults to the same name as the local folder path :type key: str, optional """ raise_path_error(local_folder_path) if not key: key = local_folder_path filenames = listdir(local_folder_path) for filename in filenames: self.upload_file(local_folder_path + filename, key + filename) @check_mounted def delete_objects(self, keys : list[str], verbose : bool = True) -> None: """Delete a list of objects on the mounted bucket :param keys: List of object names to be deleted :type keys: list[str] :param verbose: Print the result of the deletion. Defaults to True :type verbose: bool, optional """ object_list = [] for key in keys: object_list.append({"Key" : key}) deletion_dict = {"Objects": object_list} list_of_objects_before = self.list_objects(True) response : dict = self.s3_client.delete_objects( Bucket = self.bucket_name, Delete = deletion_dict ) if verbose: print(dumps(response, indent=4)) diff : set[str] = set(keys) - set(list_of_objects_before) if diff: does_not_exist = [] for key in diff: does_not_exist.append("- " + key + "\n") print("The following could not be deleted because they didn't exist: \n" + "".join(does_not_exist)) @check_mounted def delete_object(self, key : str, verbose : bool = True) -> None: """ Delete a single object in the mounted bucket :param key: The object to be deleted :type key: str :param verbose: Print the result of the deletion. Defaults to True :type verbose: bool, optional """ self.delete_objects([key], verbose = verbose) @check_mounted def delete_folder(self, key : str, verbose : bool = True) -> None: """ Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raisesd :param key: The folder of objects to be deleted :type key: str :param verbose: Print the result of the deletion. defaults to True :type verbose: bool, optional :raises RuntimeError: If there are subfolders, a RuntimeError is raisesd """ if key[-1] != "/": key += "/" object_path_in_folder = [] for s in self.search_objects_in_bucket(key): parse_object = parse(key + "{}", s) if type(parse_object) is Result: object_path_in_folder.append(s) for object_path in object_path_in_folder: if object_path[-1] == "/": raise RuntimeError("There are subfolders in this folder. Please remove these first, before deleting this one") self.delete_objects(object_path_in_folder + [key], verbose = verbose) @check_mounted def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> list[str]: """ Simple search method using substrings in order to find certain objects. Case insensitive by default. :param search_string: Substring to be used in the search :type search_string: str :param case_sensitive: Case sensitivity. Defaults to False :type case_sensitive: bool, optional :return: List of object names that match the in some way to the object names :rtype: list[str] """ search_result : list[str] = [] for key in self.list_objects(True): parse_object = search( search_string, key, case_sensitive = case_sensitive ) if type(parse_object) is Result: search_result.append(key) return search_result @check_mounted def get_object_acl(self, key : str) -> dict: """ Get the object Access Control List (ACL) :param key: The name of the object :type key: str :return: Return the ACL in the shape of a dictionary :rtype: dict """ response : dict = self.s3_client.get_object_acl( Bucket = self.bucket_name, Key = key ) return response @check_mounted def get_bucket_acl(self) -> dict: """ Get the bucket Access Control List (ACL) :return: Return the ACL in the shape of a dictionary :rtype: dict """ response : dict = self.s3_client.get_bucket_acl( Bucket = self.bucket_name ) return response @check_mounted def modify_single_object_acl(self, key : str, user_ID : str, permission : str) -> None: """ Modify permissions for a user in the Access Control List (ACL) for one object :param key: The name of the object :type key: str :param user_ID: The user name. Can either be the DisplayName or user_ID :type user_ID: str :param permission: What permission to be set. Valid options are: * FULL_CONTROL * WRITE * WRITE_ACP * READ * READ_ACP\n :type permission: str """ self.s3_client.put_object_acl( Bucket = self.bucket_name, Key = key, AccessControlPolicy = create_access_control_policy({user_ID : permission}) ) @check_mounted def modify_single_bucket_acl(self, user_ID : str, permission : str) -> None: """ Modify permissions for a user in the Access Control List (ACL) for the mounted bucket :param user_ID: The user name. Can either be the DisplayName or user_ID :type user_ID: str :param permission: What permission to be set. Valid options are: * FULL_CONTROL * WRITE * WRITE_ACP * READ * READ_ACP\n :type permission: str """ self.s3_client.put_bucket_acl( Bucket = self.bucket_name, AccessControlPolicy = create_access_control_policy({user_ID : permission}) ) @check_mounted def modify_object_acl(self, key_user_ID_permissions : dict[str, dict[str, str]]) -> None: """ Modifies permissions to multiple objects, see below. In order to add permissions for multiple objects, we make use of a dictionary of a dictionary: :py:obj:`key_user_ID_permissions = {key : {user_ID : permission}}`. So for every object (key), we set the permissions for every user ID for that object. :param key_user_ID_permissions: The dictionary containing object name and user_id-permission dictionary :type key_user_ID_permissions: dict[str, dict[str, str]] """ for key, user_ID_permissions in key_user_ID_permissions.items(): self.s3_client.put_object_acl( Bucket = self.bucket_name, Key = key, AccessControlPolicy = create_access_control_policy(user_ID_permissions) ) @check_mounted def modify_bucket_acl(self, user_ID_permissions : dict[str, str]) -> None: """ Modify permissions for multiple users for the mounted bucket :param user_ID_permissions: The dictionary containing the user name and the corresponding permission to be set to that user :type user_ID_permissions: dict[str, str] """ self.s3_client.put_bucket_acl( Bucket = self.bucket_name, AccessControlPolicy = create_access_control_policy(user_ID_permissions) )