diff --git a/gns3dms/cloud/base_cloud_ctrl.py b/gns3dms/cloud/base_cloud_ctrl.py index b9335aa8..236cdccc 100644 --- a/gns3dms/cloud/base_cloud_ctrl.py +++ b/gns3dms/cloud/base_cloud_ctrl.py @@ -29,7 +29,7 @@ import logging from io import StringIO, BytesIO from libcloud.compute.base import NodeAuthSSHKey -from libcloud.storage.types import ContainerAlreadyExistsError, ContainerDoesNotExistError +from libcloud.storage.types import ContainerAlreadyExistsError, ContainerDoesNotExistError, ObjectDoesNotExistError from .exceptions import ItemNotFound, KeyPairExists, MethodNotAllowed from .exceptions import OverLimit, BadRequest, ServiceUnavailable @@ -216,11 +216,11 @@ class BaseCloudCtrl(object): return self.driver.list_key_pairs() - def upload_file(self, file_path, folder): + def upload_file(self, file_path, cloud_object_name): """ Uploads file to cloud storage (if it is not identical to a file already in cloud storage). :param file_path: path to file to upload - :param folder: folder in cloud storage to save file in + :param cloud_object_name: name of file saved in cloud storage :return: True if file was uploaded, False if it was skipped because it already existed and was identical """ try: @@ -231,7 +231,6 @@ class BaseCloudCtrl(object): with open(file_path, 'rb') as file: local_file_hash = hashlib.md5(file.read()).hexdigest() - cloud_object_name = folder + '/' + os.path.basename(file_path) cloud_hash_name = cloud_object_name + '.md5' cloud_objects = [obj.name for obj in gns3_container.list_objects()] @@ -254,23 +253,24 @@ class BaseCloudCtrl(object): def list_projects(self): """ Lists projects in cloud storage - :return: List of (project name, object name in storage) + :return: Dictionary where project names are keys and values are names of objects in storage """ try: gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) - projects = [ - (obj.name.replace('projects/', '').replace('.zip', ''), obj.name) + projects = { + obj.name.replace('projects/', '').replace('.zip', ''): obj.name for obj in gns3_container.list_objects() if obj.name.startswith('projects/') and obj.name[-4:] == '.zip' - ] + } return projects except ContainerDoesNotExistError: return [] def download_file(self, file_name, destination=None): """ - Downloads file from cloud storage + Downloads file from cloud storage. If a file exists at destination, and it is identical to the file in cloud + storage, it is not downloaded. :param file_name: name of file in cloud storage to download :param destination: local path to save file to (if None, returns file contents as a file-like object) :return: A file-like object if file contents are returned, or None if file is saved to filesystem @@ -278,7 +278,22 @@ class BaseCloudCtrl(object): gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) storage_object = gns3_container.get_object(file_name) + if destination is not None: + if os.path.isfile(destination): + # if a file exists at destination and its hash matches that of the + # file in cloud storage, don't download it + with open(destination, 'rb') as f: + local_file_hash = hashlib.md5(f.read()).hexdigest() + + hash_object = gns3_container.get_object(file_name + '.md5') + cloud_object_hash = '' + for chunk in hash_object.as_stream(): + cloud_object_hash += chunk.decode('utf8') + + if local_file_hash == cloud_object_hash: + return + storage_object.download(destination) else: contents = b'' @@ -287,3 +302,40 @@ class BaseCloudCtrl(object): contents += chunk return BytesIO(contents) + + def find_storage_image_names(self, images_to_find): + """ + Maps names of image files to their full name in cloud storage + :param images_to_find: list of image names to find + :return: A dictionary where keys are image names, and values are the corresponding names of + the files in cloud storage + """ + gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) + images_in_storage = [obj.name for obj in gns3_container.list_objects() if obj.name.startswith('images/')] + + images = {} + for image_name in images_to_find: + images_with_same_name =\ + list(filter(lambda storage_image_name: storage_image_name.endswith(image_name), images_in_storage)) + + if len(images_with_same_name) == 1: + images[image_name] = images_with_same_name[0] + else: + raise Exception('Image does not exist in cloud storage or is duplicated') + + return images + + def delete_file(self, file_name): + gns3_container = self.storage_driver.get_container(self.GNS3_CONTAINER_NAME) + + try: + object_to_delete = gns3_container.get_object(file_name) + object_to_delete.delete() + except ObjectDoesNotExistError: + pass + + try: + hash_object = gns3_container.get_object(file_name + '.md5') + hash_object.delete() + except ObjectDoesNotExistError: + pass diff --git a/gns3dms/cloud/rackspace_ctrl.py b/gns3dms/cloud/rackspace_ctrl.py index 455f87ba..aee7f46d 100644 --- a/gns3dms/cloud/rackspace_ctrl.py +++ b/gns3dms/cloud/rackspace_ctrl.py @@ -42,11 +42,9 @@ class RackspaceCtrl(BaseCloudCtrl): """ Controller class for interacting with Rackspace API. """ - def __init__(self, username, api_key, gns3_ias_url): + def __init__(self, username, api_key, *args, **kwargs): super(RackspaceCtrl, self).__init__(username, api_key) - self.gns3_ias_url = gns3_ias_url - # set this up so it can be swapped out with a mock for testing self.post_fn = requests.post self.driver_cls = get_driver(Provider.RACKSPACE) @@ -225,55 +223,6 @@ class RackspaceCtrl(BaseCloudCtrl): self.region = region return True - def _get_shared_images(self, username, region, gns3_version): - """ - Given a GNS3 version, ask gns3-ias to share compatible images - - Response: - [{"created_at": "", "schema": "", "status": "", "member_id": "", "image_id": "", "updated_at": ""},] - or, if access was already asked - [{"image_id": "", "member_id": "", "status": "ALREADYREQUESTED"},] - """ - endpoint = self.gns3_ias_url+"/images/grant_access" - params = { - "user_id": username, - "user_region": region.upper(), - "gns3_version": gns3_version, - } - try: - response = requests.get(endpoint, params=params) - except requests.ConnectionError: - raise ApiError("Unable to connect to IAS") - - status = response.status_code - - if status == 200: - return response.json() - elif status == 404: - raise ItemNotFound() - else: - raise ApiError("IAS status code: %d" % status) - - def list_images(self): - """ - Return a dictionary containing RackSpace server images - retrieved from gns3-ias server - """ - if not (self.tenant_id and self.region): - return {} - - try: - shared_images = self._get_shared_images(self.tenant_id, self.region, __version__) - images = {} - for i in shared_images: - images[i['image_id']] = i['image_name'] - return images - except ItemNotFound: - return {} - except ApiError as e: - log.error('Error while retrieving image list: %s' % e) - return {} - def get_image(self, image_id): return self.driver.get_image(image_id) @@ -290,12 +239,11 @@ def get_provider(cloud_settings): username = cloud_settings['cloud_user_name'] apikey = cloud_settings['cloud_api_key'] region = cloud_settings['cloud_region'] - ias_url = cloud_settings.get('gns3_ias_url', '') except KeyError as e: log.error("Unable to create cloud provider: {}".format(e)) return - provider = RackspaceCtrl(username, apikey, ias_url) + provider = RackspaceCtrl(username, apikey) if not provider.authenticate(): log.error("Authentication failed for cloud provider")