Source code for rucio.client.rseclient

# Copyright European Organization for Nuclear Research (CERN)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Thomas Beermann, <thomas.beermann@cern.ch>, 2012
# - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2013
# - Mario Lassnig, <mario.lassnig@cern.ch>, 2013
# - Ralph Vigne, <ralph.vigne@cern.ch>, 2013-2015
# - Martin Barisits, <martin.barisits@cern.ch>, 2013
# - Cedric Serfon, <cedric.serfon@cern.ch>, 2014

from json import dumps, loads
from requests.status_codes import codes
from urllib import quote

from rucio.client.baseclient import BaseClient
from rucio.client.baseclient import choice
from rucio.common.utils import build_url


[docs]class RSEClient(BaseClient): """RSE client class for working with rucio RSEs""" RSE_BASEURL = 'rses' def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, auth_type=None, creds=None, timeout=None, user_agent='rucio-clients'): super(RSEClient, self).__init__(rucio_host, auth_host, account, ca_cert, auth_type, creds, timeout, user_agent)
[docs] def get_rse(self, rse): """ Returns details about the referred RSE. :param rse: Name of the referred RSE :returns: A dict containing all attributes of the referred RSE :raises RSENotFound: if the referred RSE was not found in the database """ path = '/'.join([self.RSE_BASEURL, rse]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: rse = loads(r.text) return rse else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def add_rse(self, rse, **kwargs): """ Sends the request to create a new RSE. :param rse: the name of the rse. :param deterministic: Boolean to know if the pfn is generated deterministically. :param volatile: Boolean for RSE cache. :param city: City for the RSE. :param region_code: The region code for the RSE. :param country_name: The country. :param continent: The continent. :param time_zone: Timezone. :param staging_area: Staging area. :param ISP: Internet service provider. :return: True if location was created successfully else False. :raises Duplicate: if rse already exists. """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=dumps(kwargs)) if r.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def update_rse(self, rse, parameters): """ Update RSE properties like availability or name. :param rse: the name of the new rse. :param parameters: A dictionnary with property (name, read, write, delete as keys). """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT', data=dumps(parameters)) if r.status_code == codes.created: return True exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def delete_rse(self, rse): """ Sends the request to delete a rse. :param rse: the name of the rse. :return: True if location was created successfully else False. """ path = 'rses/' + rse url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def list_rses(self, rse_expression=None): """ Sends the request to list all rucio locations(RSEs). :rse_expression: RSE Expression to use as filter. :return: a list containing the names of all rucio locations. """ if rse_expression: path = ['rses', "?expression=" + quote(rse_expression)] path = '/'.join(path) else: path = 'rses/' url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def add_rse_attribute(self, rse, key, value): """ Sends the request to add a RSE attribute. :param rse: the name of the rse. :param key: the attribute key. :param value: the attribute value. :return: True if RSE attribute was created successfully else False. :raises Duplicate: if RSE attribute already exists. """ path = '/'.join([self.RSE_BASEURL, rse, 'attr', key]) url = build_url(choice(self.list_hosts), path=path) data = dumps({'value': value}) r = self._send_request(url, type='POST', data=data) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def delete_rse_attribute(self, rse, key): """ Sends the request to delete a RSE attribute. :param rse: the RSE name. :param key: the attribute key. :return: True if RSE attribute was deleted successfully else False. """ path = '/'.join([self.RSE_BASEURL, rse, 'attr', key]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def list_rse_attributes(self, rse): """ Sends the request to get RSE attributes. :param rse: The RSE name. :return: True if RSE attribute was created successfully else False. """ path = '/'.join([self.RSE_BASEURL, rse, 'attr/']) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: attributes = loads(r.text) return attributes else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def add_protocol(self, rse, params): """ Sends the request to create a new protocol for the given RSE. :param rse: the name of the rse. :param scheme: identifier of this protocol :param params: Attributes of the protocol. Supported are: hostname: hostname for this protocol (default = localhost) port: port for this protocol (default = 0) prefix: string used as a prfeix for this protocol when generating the PFN (default = None) impl: qualified name of the implementation class for this protocol (mandatory) read: integer representing the priority of this procotol for read operations (default = -1) write: integer representing the priority of this procotol for write operations (default = -1) delete: integer representing the priority of this procotol for delet operations (default = -1) extended_attributes: miscellaneous protocol specific information e.g. spacetoken for SRM (default = None) :return: True if protocol was created successfully else False. :raises Duplicate: if protocol with same hostname, port and protocol identifier already exists for the given RSE. :raises RSENotFound: if the RSE doesn't exist. :raises KeyNotFound: if params is missing manadtory attributes to create the protocol. :raises AccessDenied: if not authorized. """ scheme = params['scheme'] path = '/'.join([self.RSE_BASEURL, rse, 'protocols', scheme]) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='POST', data=dumps(params)) if r.status_code == codes.created: return True else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def get_protocols(self, rse, protocol_domain='ALL', operation=None, default=False, scheme=None): """ Returns protocol information. Parameter comibantions are: (operation OR default) XOR protocol. :param rse: the RSE name. :param protocol_domain: The scope of the protocol. Supported are 'LAN', 'WAN', and 'ALL' (as default). :param operation: The name of the requested operation (read, write, or delete). If None, all operations are queried. :param default: Indicates if only the default operations should be returned. :param scheme: The identifier of the requested protocol. :returns: A list with details about each matching protocol. :raises RSENotFound: if the RSE doesn't exist. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSEOperationNotSupported: if no matching protocol entry for the requested operation could be found. """ path = None params = {} if scheme: path = '/'.join([self.RSE_BASEURL, rse, 'protocols', scheme]) else: path = '/'.join([self.RSE_BASEURL, rse, 'protocols']) if operation: params['operation'] = operation if default: params['default'] = default params['protocol_domain'] = protocol_domain url = build_url(choice(self.list_hosts), path=path, params=params) r = self._send_request(url, type='GET') if r.status_code == codes.ok: protocols = loads(r.text) return protocols else: exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code) raise exc_cls(exc_msg)
[docs] def delete_protocols(self, rse, scheme, hostname=None, port=None): """ Deletes matching protocols from RSE. Protocols using the same identifier can be distinguished by hostname and port. :param rse: the RSE name. :param scheme: identifier of the protocol. :param hostname: hostname of the protocol. :param port: port of the protocol. :returns: True if success. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSENotFound: if the RSE doesn't exist. :raises AccessDenied: if not authorized. """ path = [self.RSE_BASEURL, rse, 'protocols', scheme] if hostname: path.append(hostname) if port: path.append(str(port)) path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='DEL') if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def update_protocols(self, rse, scheme, data, hostname=None, port=None): """ Updates matching protocols from RSE. Protocol using the same identifier can be distinguished by hostname and port. :param rse: the RSE name. :param scheme: identifier of the protocol. :param data: A dict providing the new values of the protocol attibutes. Keys must match column names in database. :param hostname: hostname of the protocol. :param port: port of the protocol. :returns: True if success. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSENotFound: if the RSE doesn't exist. :raises KeyNotFound: if invalid data was provided for update. :raises AccessDenied: if not authorized. """ path = [self.RSE_BASEURL, rse, 'protocols', scheme] if hostname: path.append(hostname) if port: path.append(str(port)) path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT', data=dumps(data)) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def swap_protocols(self, rse, domain, operation, scheme_a, scheme_b): """ Swaps the priorities of the provided operation. :param rse: the RSE name. :param domain: the domain in which priorities should be swapped i.e. wan or lan. :param operation: the operation that should be swapped i.e. read, write, or delete. :param scheme_a: the scheme of one of the two protocols to be swapped, e.g. srm. :param scheme_b: the scheme of the other of the two protocols to be swapped, e.g. http. :returns: True if success. :raises RSEProtocolNotSupported: if no matching protocol entry could be found. :raises RSENotFound: if the RSE doesn't exist. :raises KeyNotFound: if invalid data was provided for update. :raises AccessDenied: if not authorized. """ protocol_a = protocol_b = None protocols = self.get_protocols(rse, domain, operation, False, scheme_a)['protocols'] for p in protocols: if p['scheme'] == scheme_a: protocol_a = p if p['scheme'] == scheme_b: protocol_b = p if (protocol_a or protocol_b) is None: return False priority_a = protocol_a['domains'][domain][operation] priority_b = protocol_b['domains'][domain][operation] self.update_protocols(rse, protocol_a['scheme'], {'domains': {domain: {operation: priority_b}}}, protocol_a['hostname'], protocol_a['port']) self.update_protocols(rse, protocol_b['scheme'], {'domains': {domain: {operation: priority_a}}}, protocol_b['hostname'], protocol_b['port']) return True
[docs] def set_rse_usage(self, rse, source, used, free): """ Set RSE usage information. :param rse: the RSE name. :param source: the information source, e.g. srm. :param used: the used space in bytes. :param free: the free in bytes. :returns: True if successful, otherwise false. """ path = [self.RSE_BASEURL, rse, 'usage'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) data = {'source': source, 'used': used, 'free': free} r = self._send_request(url, type='PUT', data=dumps(data)) if r.status_code == codes.ok: return True else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def get_rse_usage(self, rse, filters=None): """ Get RSE usage information. :param rse: the RSE name. :param filters: dictionary of attributes by which the results should be filtered :returns: True if successful, otherwise false. """ path = [self.RSE_BASEURL, rse, 'usage'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET', params=filters) if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def list_rse_usage_history(self, rse, filters=None): """ List RSE usage history information. :param rse: The RSE name. :param filters: dictionary of attributes by which the results should be filtered. :returns: list of dictionnaries. """ path = [self.RSE_BASEURL, rse, 'usage', 'history'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET', params=filters) if r.status_code == codes.ok: return self._load_json_data(r) else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def set_rse_limits(self, rse, name, value): """ Set RSE limit information. :param rse: The RSE name. :param name: The name of the limit. :param value: The feature value. Set to -1 to remove the limit. :returns: True if successful, otherwise false. """ path = [self.RSE_BASEURL, rse, 'limits'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='PUT', data=dumps({'name': name, 'value': value})) if r.status_code == codes.ok: return True exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)
[docs] def get_rse_limits(self, rse): """ Get RSE limits. :param rse: The RSE name. :returns: True if successful, otherwise false. """ path = [self.RSE_BASEURL, rse, 'limits'] path = '/'.join(path) url = build_url(choice(self.list_hosts), path=path) r = self._send_request(url, type='GET') if r.status_code == codes.ok: return self._load_json_data(r).next() else: exc_cls, exc_msg = self._get_exception(r.headers, r.status_code) raise exc_cls(exc_msg)