Merge remote-tracking branch 'origin/master'

CHANGELOG

@@ -1,5 +1,19 @@
 # Change Log
 
+## 1.3.1 11/04/2015
+
+* Release
+
+## 1.3.1rc4 09/04/2015
+
+* Initial config file content can be empty (fix export issues)
+* Fix crash if IOU initial config is emtpy
+* Return more informations about bad requests for crash reports
+* Allow less strict dependencies for easier install
+* Missing project name in documentation
+* Some spring cleaning
+
+
 ## 1.3.1rc3 07/04/2015
 
 * Fix missing IOU documentation

MANIFEST.in

@@ -4,7 +4,7 @@ include INSTALL
 include LICENSE
 include MANIFEST.in
 include tox.ini
-recursive-exclude tests *
+recursive-include tests *
 recursive-exclude docs *
 recursive-include gns3server *
 recursive-exclude * __pycache__

docs/general.rst

@@ -39,7 +39,7 @@ The next step is to create a project.
 
 .. code-block:: shell-session
 
-    # curl -X POST "http://localhost:8000/v1/projects" -d "{}"
+    # curl -X POST "http://localhost:8000/v1/projects" -d '{"name": "test"}'
     {
         "project_id": "42f9feee-3217-4104-981e-85d5f0a806ec",
         "temporary": false,

gns3server/handlers/api/virtualbox_handler.py

@@ -79,8 +79,9 @@ class VirtualBoxHandler:
                 yield from vm.set_ram(ram)
 
         for name, value in request.json.items():
-            if hasattr(vm, name) and getattr(vm, name) != value:
+            if name != "vm_id":
-                setattr(vm, name, value)
+                if hasattr(vm, name) and getattr(vm, name) != value:
+                    setattr(vm, name, value)
 
         response.set_status(201)
         response.json(vm)

gns3server/modules/base_manager.py

@@ -371,3 +371,40 @@ class BaseManager:
             nio = NIOGenericEthernet(nio_settings["ethernet_device"])
         assert nio is not None
         return nio
+
+    def get_abs_image_path(self, path):
+        """
+        Get the absolute path of an image
+
+        :param path: file path
+        :return: file path
+        """
+
+        img_directory = self.get_images_directory()
+
+        if not os.path.isabs(path):
+            s = os.path.split(path)
+            return os.path.normpath(os.path.join(img_directory, *s))
+        return path
+
+    def get_relative_image_path(self, path):
+        """
+        Get a path relative to images directory path
+        or an abspath if the path is not located inside
+        image directory
+
+        :param path: file path
+        :return: file path
+        """
+
+        img_directory = self.get_images_directory()
+        path = self.get_abs_image_path(path)
+        if os.path.dirname(path) == img_directory:
+            return os.path.basename(path)
+        return path
+
+    def get_images_directory(self):
+        """
+        Get the image directory on disk
+        """
+        raise NotImplementedError

gns3server/modules/dynamips/__init__.py

@@ -616,3 +616,9 @@ class Dynamips(BaseManager):
             if was_auto_started:
                 yield from vm.stop()
         return validated_idlepc
+
+    def get_images_directory(self):
+        """
+        Return the full path of the images directory on disk
+        """
+        return os.path.join(os.path.expanduser(self.config.get_section_config("Server").get("images_path", "~/GNS3/images")), "IOS")

gns3server/modules/dynamips/nodes/router.py

@@ -151,10 +151,7 @@ class Router(BaseVM):
                        "system_id": self._system_id}
 
         # return the relative path if the IOS image is in the images_path directory
-        server_config = self.manager.config.get_section_config("Server")
+        router_info["image"] = self.manager.get_relative_image_path(self._image)
-        relative_image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "IOS", self._image)
-        if os.path.exists(relative_image):
-            router_info["image"] = os.path.basename(self._image)
 
         # add the slots
         slot_number = 0
@@ -419,9 +416,7 @@ class Router(BaseVM):
         :param image: path to IOS image file
         """
 
-        if not os.path.isabs(image):
+        image = self.manager.get_abs_image_path(image)
-            server_config = self.manager.config.get_section_config("Server")
-            image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "IOS", image)
 
         if not os.path.isfile(image):
             raise DynamipsError("IOS image '{}' is not accessible".format(image))

gns3server/modules/iou/__init__.py

@@ -91,3 +91,9 @@ class IOU(BaseManager):
         """
 
         return os.path.join("iou", "device-{}".format(legacy_vm_id))
+
+    def get_images_directory(self):
+        """
+        Return the full path of the images directory on disk
+        """
+        return os.path.join(os.path.expanduser(self.config.get_section_config("Server").get("images_path", "~/GNS3/images")), "IOU")

gns3server/modules/iou/iou_vm.py

@@ -125,14 +125,7 @@ class IOUVM(BaseVM):
         :param path: path to the IOU image executable
         """
 
-        if not os.path.isabs(path):
+        self._path = self.manager.get_abs_image_path(path)
-            server_config = self.manager.config.get_section_config("Server")
-            relative_path = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), path)
-            if not os.path.exists(relative_path):
-                relative_path = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "IOU", path)
-            path = relative_path
-
-        self._path = path
 
         # In 1.2 users uploaded images to the images roots
         # after the migration their images are inside images/IOU
@@ -219,11 +212,7 @@ class IOUVM(BaseVM):
                        "use_default_iou_values": self._use_default_iou_values}
 
         # return the relative path if the IOU image is in the images_path directory
-        server_config = self.manager.config.get_section_config("Server")
+        iou_vm_info["path"] = self.manager.get_relative_image_path(self.path)
-        relative_image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "IOU", self.path)
-        if os.path.exists(relative_image):
-            iou_vm_info["path"] = os.path.basename(self.path)
-
         return iou_vm_info
 
     @property
@@ -975,6 +964,9 @@ class IOUVM(BaseVM):
         try:
             script_file = os.path.join(self.working_dir, "initial-config.cfg")
 
+            if initial_config is None:
+                initial_config = ''
+
             # We disallow erasing the initial config file
             if len(initial_config) == 0 and os.path.exists(script_file):
                 return
@@ -1047,7 +1039,6 @@ class IOUVM(BaseVM):
             raise IOUError("Packet capture is already activated on {adapter_number}/{port_number}".format(adapter_number=adapter_number,
                                                                                                           port_number=port_number))
 
-
         nio.startPacketCapture(output_file, data_link_type)
         log.info('IOU "{name}" [{id}]: starting packet capture on {adapter_number}/{port_number}'.format(name=self._name,
                                                                                                          id=self._id,

gns3server/modules/qemu/__init__.py

@@ -112,3 +112,9 @@ class Qemu(BaseManager):
         """
 
         return os.path.join("qemu", "vm-{}".format(legacy_vm_id))
+
+    def get_images_directory(self):
+        """
+        Return the full path of the images directory on disk
+        """
+        return os.path.join(os.path.expanduser(self.config.get_section_config("Server").get("images_path", "~/GNS3/images")), "QEMU")

gns3server/modules/qemu/qemu_vm.py

@@ -148,14 +148,10 @@ class QemuVM(BaseVM):
         :param hda_disk_image: QEMU hda disk image path
         """
 
-        if not os.path.isabs(hda_disk_image):
+        self._hda_disk_image = self.manager.get_abs_image_path(hda_disk_image)
-            server_config = self.manager.config.get_section_config("Server")
-            hda_disk_image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "QEMU", hda_disk_image)
-
         log.info('QEMU VM "{name}" [{id}] has set the QEMU hda disk image path to {disk_image}'.format(name=self._name,
                                                                                                        id=self._id,
-                                                                                                       disk_image=hda_disk_image))
+                                                                                                       disk_image=self._hda_disk_image))
-        self._hda_disk_image = hda_disk_image
 
     @property
     def hdb_disk_image(self):
@@ -175,14 +171,10 @@ class QemuVM(BaseVM):
         :param hdb_disk_image: QEMU hdb disk image path
         """
 
-        if not os.path.isabs(hdb_disk_image):
+        self._hdb_disk_image = self.manager.get_abs_image_path(hdb_disk_image)
-            server_config = self.manager.config.get_section_config("Server")
-            hdb_disk_image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "QEMU", hdb_disk_image)
-
         log.info('QEMU VM "{name}" [{id}] has set the QEMU hdb disk image path to {disk_image}'.format(name=self._name,
                                                                                                        id=self._id,
-                                                                                                       disk_image=hdb_disk_image))
+                                                                                                       disk_image=self._hdb_disk_image))
-        self._hdb_disk_image = hdb_disk_image
 
     @property
     def hdc_disk_image(self):
@@ -202,14 +194,10 @@ class QemuVM(BaseVM):
         :param hdc_disk_image: QEMU hdc disk image path
         """
 
-        if not os.path.isabs(hdc_disk_image):
+        self._hdc_disk_image = self.manager.get_abs_image_path(hdc_disk_image)
-            server_config = self.manager.config.get_section_config("Server")
-            hdc_disk_image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "QEMU", hdc_disk_image)
-
         log.info('QEMU VM "{name}" [{id}] has set the QEMU hdc disk image path to {disk_image}'.format(name=self._name,
                                                                                                        id=self._id,
-                                                                                                       disk_image=hdc_disk_image))
+                                                                                                       disk_image=self._hdc_disk_image))
-        self._hdc_disk_image = hdc_disk_image
 
     @property
     def hdd_disk_image(self):
@@ -229,14 +217,11 @@ class QemuVM(BaseVM):
         :param hdd_disk_image: QEMU hdd disk image path
         """
 
-        if not os.path.isabs(hdd_disk_image):
+        self._hdd_disk_image = hdd_disk_image
-            server_config = self.manager.config.get_section_config("Server")
+        self._hdd_disk_image = self.manager.get_abs_image_path(hdd_disk_image)
-            hdd_disk_image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "QEMU", hdd_disk_image)
-
         log.info('QEMU VM "{name}" [{id}] has set the QEMU hdd disk image path to {disk_image}'.format(name=self._name,
                                                                                                        id=self._id,
-                                                                                                       disk_image=hdd_disk_image))
+                                                                                                       disk_image=self._hdd_disk_image))
-        self._hdd_disk_image = hdd_disk_image
 
     @property
     def adapters(self):
@@ -778,9 +763,9 @@ class QemuVM(BaseVM):
 
         adapter.add_nio(0, nio)
         log.info('QEMU VM "{name}" [{id}]: {nio} added to adapter {adapter_number}'.format(name=self._name,
-                                                                                          id=self._id,
+                                                                                           id=self._id,
-                                                                                          nio=nio,
+                                                                                           nio=nio,
-                                                                                          adapter_number=adapter_number))
+                                                                                           adapter_number=adapter_number))
 
     @asyncio.coroutine
     def adapter_remove_nio_binding(self, adapter_number):
@@ -1068,23 +1053,6 @@ class QemuVM(BaseVM):
         command.extend(self._graphic())
         return command
 
-    def _get_relative_disk_image_path(self, disk_image):
-        """
-        Returns a relative image path if the disk image is in the images directory.
-
-        :param disk_image: path to the disk image
-
-        :returns: relative or full path
-        """
-
-        if disk_image:
-            # return the relative path if the disk image is in the images_path directory
-            server_config = self.manager.config.get_section_config("Server")
-            relative_image = os.path.join(os.path.expanduser(server_config.get("images_path", "~/GNS3/images")), "QEMU", disk_image)
-            if os.path.exists(relative_image):
-                return os.path.basename(disk_image)
-        return disk_image
-
     def __json__(self):
         answer = {
             "project_id": self.project.id,
@@ -1095,11 +1063,11 @@ class QemuVM(BaseVM):
             if field not in answer:
                 answer[field] = getattr(self, field)
 
-        answer["hda_disk_image"] = self._get_relative_disk_image_path(self._hda_disk_image)
+        answer["hda_disk_image"] = self.manager.get_relative_image_path(self._hda_disk_image)
-        answer["hdb_disk_image"] = self._get_relative_disk_image_path(self._hdb_disk_image)
+        answer["hdb_disk_image"] = self.manager.get_relative_image_path(self._hdb_disk_image)
-        answer["hdc_disk_image"] = self._get_relative_disk_image_path(self._hdc_disk_image)
+        answer["hdc_disk_image"] = self.manager.get_relative_image_path(self._hdc_disk_image)
-        answer["hdd_disk_image"] = self._get_relative_disk_image_path(self._hdd_disk_image)
+        answer["hdd_disk_image"] = self.manager.get_relative_image_path(self._hdd_disk_image)
-        answer["initrd"] = self._get_relative_disk_image_path(self._initrd)
+        answer["initrd"] = self.manager.get_relative_image_path(self._initrd)
-        answer["kernel_image"] = self._get_relative_disk_image_path(self._kernel_image)
+        answer["kernel_image"] = self.manager.get_relative_image_path(self._kernel_image)
 
         return answer

gns3server/modules/virtualbox/__init__.py

@@ -41,6 +41,7 @@ class VirtualBox(BaseManager):
 
         super().__init__()
         self._vboxmanage_path = None
+        self._execute_lock = asyncio.Lock()
 
     @property
     def vboxmanage_path(self):
@@ -82,34 +83,38 @@ class VirtualBox(BaseManager):
     @asyncio.coroutine
     def execute(self, subcommand, args, timeout=60):
 
-        vboxmanage_path = self.vboxmanage_path
+        # We use a lock prevent parallel execution due to strange errors
-        if not vboxmanage_path:
+        # reported by a user and reproduced by us.
-            vboxmanage_path = self.find_vboxmanage()
+        # https://github.com/GNS3/gns3-gui/issues/261
-        command = [vboxmanage_path, "--nologo", subcommand]
+        with (yield from self._execute_lock):
-        command.extend(args)
+            vboxmanage_path = self.vboxmanage_path
-        log.debug("Executing VBoxManage with command: {}".format(command))
+            if not vboxmanage_path:
-        try:
+                vboxmanage_path = self.find_vboxmanage()
-            vbox_user = self.config.get_section_config("VirtualBox").get("vbox_user")
+            command = [vboxmanage_path, "--nologo", subcommand]
-            if vbox_user:
+            command.extend(args)
-                # TODO: test & review this part
+            log.debug("Executing VBoxManage with command: {}".format(command))
-                sudo_command = "sudo -i -u {}".format(vbox_user) + " ".join(command)
+            try:
-                process = yield from asyncio.create_subprocess_shell(sudo_command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+                vbox_user = self.config.get_section_config("VirtualBox").get("vbox_user")
-            else:
+                if vbox_user:
-                process = yield from asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+                    # TODO: test & review this part
-        except (OSError, subprocess.SubprocessError) as e:
+                    sudo_command = "sudo -i -u {}".format(vbox_user) + " ".join(command)
-            raise VirtualBoxError("Could not execute VBoxManage: {}".format(e))
+                    process = yield from asyncio.create_subprocess_shell(sudo_command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+                else:
+                    process = yield from asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+            except (OSError, subprocess.SubprocessError) as e:
+                raise VirtualBoxError("Could not execute VBoxManage: {}".format(e))
 
-        try:
+            try:
-            stdout_data, stderr_data = yield from asyncio.wait_for(process.communicate(), timeout=timeout)
+                stdout_data, stderr_data = yield from asyncio.wait_for(process.communicate(), timeout=timeout)
-        except asyncio.TimeoutError:
+            except asyncio.TimeoutError:
-            raise VirtualBoxError("VBoxManage has timed out after {} seconds!".format(timeout))
+                raise VirtualBoxError("VBoxManage has timed out after {} seconds!".format(timeout))
 
-        if process.returncode:
+            if process.returncode:
-            # only the first line of the output is useful
+                # only the first line of the output is useful
-            vboxmanage_error = stderr_data.decode("utf-8", errors="ignore")
+                vboxmanage_error = stderr_data.decode("utf-8", errors="ignore")
-            raise VirtualBoxError("VirtualBox has returned an error: {}".format(vboxmanage_error))
+                raise VirtualBoxError("VirtualBox has returned an error: {}".format(vboxmanage_error))
 
-        return stdout_data.decode("utf-8", errors="ignore").splitlines()
+            return stdout_data.decode("utf-8", errors="ignore").splitlines()
 
     @asyncio.coroutine
     def get_list(self):

gns3server/modules/virtualbox/virtualbox_vm.py

@@ -105,9 +105,10 @@ class VirtualBoxVM(BaseVM):
 
         results = yield from self.manager.execute("showvminfo", [self._vmname, "--machinereadable"])
         for info in results:
-            name, value = info.split('=', 1)
+            if '=' in info:
-            if name == "VMState":
+                name, value = info.split('=', 1)
-                return value.strip('"')
+                if name == "VMState":
+                    return value.strip('"')
         raise VirtualBoxError("Could not get VM state for {}".format(self._vmname))
 
     @asyncio.coroutine
@@ -139,6 +140,8 @@ class VirtualBoxVM(BaseVM):
     def create(self):
 
         yield from self._get_system_properties()
+        if "API version" not in self._system_properties:
+            raise VirtualBoxError("Can't access to VirtualBox API Version")
         if parse_version(self._system_properties["API version"]) < parse_version("4_3"):
             raise VirtualBoxError("The VirtualBox API version is lower than 4.3")
         log.info("VirtualBox VM '{name}' [{id}] created".format(name=self.name, id=self.id))

gns3server/schemas/iou.py

@@ -220,7 +220,7 @@ IOU_OBJECT_SCHEMA = {
 
 IOU_NIO_SCHEMA = {
     "$schema": "http://json-schema.org/draft-04/schema#",
-    "description": "Request validation to add a NIO for a VPCS instance",
+    "description": "Request validation to add a NIO for a IOU instance",
     "type": "object",
     "definitions": {
         "UDP": {
@@ -317,8 +317,7 @@ IOU_INITIAL_CONFIG_SCHEMA = {
     "properties": {
         "content": {
             "description": "Content of the initial configuration file",
-            "type": ["string", "null"],
+            "type": ["string", "null"]
-            "minLength": 1,
         },
     },
     "additionalProperties": False,

gns3server/schemas/qemu.py

@@ -213,7 +213,7 @@ QEMU_UPDATE_SCHEMA = {
 
 QEMU_NIO_SCHEMA = {
     "$schema": "http://json-schema.org/draft-04/schema#",
-    "description": "Request validation to add a NIO for a VPCS instance",
+    "description": "Request validation to add a NIO for a QEMU instance",
     "type": "object",
     "definitions": {
         "UDP": {

gns3server/version.py

@@ -23,5 +23,5 @@
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
 
-__version__ = "1.3.1.rc3"
+__version__ = "1.3.2.dev1"
-__version_info__ = (1, 3, 0, 99)
+__version_info__ = (1, 3, 2, -99)

gns3server/web/route.py

@@ -40,6 +40,7 @@ def parse_request(request, input_schema):
         try:
             request.json = json.loads(body.decode('utf-8'))
         except ValueError as e:
+            request.json = {"malformed_json": body.decode('utf-8')}
             raise aiohttp.web.HTTPBadRequest(text="Invalid JSON {}".format(e))
     else:
         request.json = {}
@@ -137,6 +138,10 @@ class Route(object):
                             log.warn("Could not write to the record file {}: {}".format(record_file, e))
                     response = Response(route=route, output_schema=output_schema)
                     yield from func(request, response)
+                except aiohttp.web.HTTPBadRequest as e:
+                    response = Response(route=route)
+                    response.set_status(e.status)
+                    response.json({"message": e.text, "status": e.status, "path": route, "request": request.json})
                 except aiohttp.web.HTTPException as e:
                     response = Response(route=route)
                     response.set_status(e.status)

setup.py

@@ -34,16 +34,16 @@ class PyTest(TestCommand):
         sys.exit(errcode)
 
 
-dependencies = ["aiohttp==0.14.4",
+dependencies = ["aiohttp>=0.14.4",
-                "jsonschema==2.4.0",
+                "jsonschema>=2.4.0",
-                "Jinja2==2.7.3",
+                "Jinja2>=2.7.3",
-                "raven==5.2.0"]
+                "raven>=5.2.0"]
 
 #if not sys.platform.startswith("win"):
 #    dependencies.append("netifaces==0.10.4")
 
 if sys.version_info == (3, 3):
-    dependencies.append("asyncio==3.4.2")
+    dependencies.append("asyncio>=3.4.2")
 
 setup(
     name="gns3-server",

tests/modules/iou/test_iou_manager.py

@@ -17,7 +17,9 @@
 
 
 import pytest
+from unittest.mock import patch
 import uuid
+import os
 
 
 from gns3server.modules.iou import IOU
@@ -25,6 +27,15 @@ from gns3server.modules.iou.iou_error import IOUError
 from gns3server.modules.project_manager import ProjectManager
 
 
+@pytest.fixture(scope="function")
+def iou(port_manager):
+    # Cleanup the IOU object
+    IOU._instance = None
+    iou = IOU.instance()
+    iou.port_manager = port_manager
+    return iou
+
+
 def test_get_application_id(loop, project, port_manager):
     # Cleanup the IOU object
     IOU._instance = None
@@ -71,3 +82,8 @@ def test_get_application_id_no_id_available(loop, project, port_manager):
             vm_id = str(uuid.uuid4())
             loop.run_until_complete(iou.create_vm("PC {}".format(i), project.id, vm_id))
             assert iou.get_application_id(vm_id) == i
+
+
+def test_get_images_directory(iou, tmpdir):
+    with patch("gns3server.config.Config.get_section_config", return_value={"images_path": str(tmpdir)}):
+        assert iou.get_images_directory() == str(tmpdir / "IOU")

tests/modules/test_manager.py

@@ -22,6 +22,7 @@ from unittest.mock import patch
 
 
 from gns3server.modules.vpcs import VPCS
+from gns3server.modules.iou import IOU
 
 
 def test_create_vm_new_topology(loop, project, port_manager):
@@ -82,3 +83,33 @@ def test_create_vm_old_topology(loop, project, tmpdir, port_manager):
         vm_dir = os.path.join(project_dir, "project-files", "vpcs", vm.id)
         with open(os.path.join(vm_dir, "startup.vpc")) as f:
             assert f.read() == "1"
+
+
+def test_get_abs_image_path(iou, tmpdir):
+    os.makedirs(str(tmpdir / "IOU"))
+    path1 = str(tmpdir / "test1.bin")
+    open(path1, 'w+').close()
+
+    path2 = str(tmpdir / "IOU" / "test2.bin")
+    open(path2, 'w+').close()
+
+    with patch("gns3server.config.Config.get_section_config", return_value={"images_path": str(tmpdir)}):
+        assert iou.get_abs_image_path(path1) == path1
+        assert iou.get_abs_image_path(path2) == path2
+        assert iou.get_abs_image_path("test2.bin") == path2
+        assert iou.get_abs_image_path("../test1.bin") == path1
+
+
+def test_get_relative_image_path(iou, tmpdir):
+    os.makedirs(str(tmpdir / "IOU"))
+    path1 = str(tmpdir / "test1.bin")
+    open(path1, 'w+').close()
+
+    path2 = str(tmpdir / "IOU" / "test2.bin")
+    open(path2, 'w+').close()
+
+    with patch("gns3server.config.Config.get_section_config", return_value={"images_path": str(tmpdir)}):
+        assert iou.get_relative_image_path(path1) == path1
+        assert iou.get_relative_image_path(path2) == "test2.bin"
+        assert iou.get_relative_image_path("test2.bin") == "test2.bin"
+        assert iou.get_relative_image_path("../test1.bin") == path1