Merge branch '2.1' into embed_shell

.travis.yml

@@ -6,6 +6,7 @@ python:
 sudo: false
 cache: pip
 install:
+- pip install -U setuptools pip
 - python setup.py install
 - pip install -rdev-requirements.txt
 script:

CHANGELOG

@@ -1,5 +1,18 @@
 # Change Log
 
+## 2.0.2 30/05/2017
+
+* Set correct permission on ubridge when doing a remote installation
+* Remote install script should be totally non interactive
+* Duplicate project on remote server use UUID
+* Fix import of some old topologies from 1.3
+* Fix error in logging of error during starting GNS3 VM
+* Fix an error when logging Docker container fail to start
+* Use docker version in error message of outdated docker installation
+* Support images created by "docker commit". Fixes #1039
+* Do not wait auto start to finish in order to complete project opening
+* Improve logging for remote server connection lost
+
 ## 2.0.1 16/05/2017
 
 * Handle HTTP 504 errors from compute node

dev-requirements.txt

@@ -1,7 +1,7 @@
 -rrequirements.txt
 
-sphinx==1.5.6
+sphinx==1.6.2
-pytest==3.0.7
+pytest==3.1.1
 pep8==1.7.0
 pytest-catchlog==1.2.2
 pytest-timeout==1.2.0

gns3server/compute/docker/__init__.py

@@ -33,7 +33,9 @@ from gns3server.compute.docker.docker_error import DockerError, DockerHttp304Err
 log = logging.getLogger(__name__)
 
 
+# Be carefull to keep it consistent
 DOCKER_MINIMUM_API_VERSION = "1.25"
+DOCKER_MINIMUM_VERSION = "1.13"
 
 
 class Docker(BaseManager):
@@ -60,7 +62,7 @@ class Docker(BaseManager):
                 self._connected = False
                 raise DockerError("Can't connect to docker daemon")
             if parse_version(version["ApiVersion"]) < parse_version(DOCKER_MINIMUM_API_VERSION):
-                raise DockerError("Docker API version is {}. GNS3 requires a minimum API version of {}".format(version["ApiVersion"], DOCKER_MINIMUM_API_VERSION))
+                raise DockerError("Docker version is {}. GNS3 requires a minimum version of {}".format(version["Version"], DOCKER_MINIMUM_VERSION))
 
     def connector(self):
         if self._connector is None or self._connector.closed:
@@ -113,11 +115,13 @@ class Docker(BaseManager):
         :returns: HTTP response
         """
         data = json.dumps(data)
-        url = "http://docker/v" + DOCKER_MINIMUM_API_VERSION + "/" + path
-
         if timeout is None:
             timeout = 60 * 60 * 24 * 31  # One month timeout
 
+        if path == 'version':
+            url = "http://docker/v1.12/" + path         # API of docker v1.0
+        else:
+            url = "http://docker/v" + DOCKER_MINIMUM_API_VERSION + "/" + path
         try:
             if path != "version":  # version is use by check connection
                 yield from self._check_connection()
@@ -162,8 +166,7 @@ class Docker(BaseManager):
         """
 
         url = "http://docker/v" + DOCKER_MINIMUM_API_VERSION + "/" + path
-        connection = yield from aiohttp.ws_connect(url,
+        connection = yield from self._session.ws_connect(url,
-                                                   connector=self.connector(),
                                                          origin="http://docker",
                                                          autoping=True)
         return connection

gns3server/compute/docker/docker_vm.py

@@ -213,7 +213,7 @@ class DockerVM(BaseNode):
 
         self._volumes = ["/etc/network"]
 
-        volumes = image_infos.get("ContainerConfig", {}).get("Volumes")
+        volumes = image_infos.get("Config", {}).get("Volumes")
         if volumes is None:
             return binds
         for volume in volumes.keys():
@@ -361,7 +361,7 @@ class DockerVM(BaseNode):
                     try:
                         yield from self._add_ubridge_connection(nio, adapter_number)
                     except UbridgeNamespaceError:
-                        log.error("Container {} failed to start", self.name)
+                        log.error("Container %s failed to start", self.name)
                         yield from self.stop()
 
                         # The container can crash soon after the start, this means we can not move the interface to the container namespace

gns3server/controller/compute.py

@@ -580,8 +580,10 @@ class Compute:
         Forward a call to the emulator on compute
         """
         try:
-            res = yield from self.http_query(method, "/{}/{}".format(type, path), data=data, timeout=None)
+            action = "/{}/{}".format(type, path)
+            res = yield from self.http_query(method, action, data=data, timeout=None)
         except aiohttp.ServerDisconnectedError:
+            log.error("Connection lost to %s during %s %s", self._id, method, action)
             raise aiohttp.web.HTTPGatewayTimeout()
         return res.json
 

gns3server/controller/gns3vm/__init__.py

@@ -250,7 +250,7 @@ class GNS3VM:
                                                             force=True)
                 except aiohttp.web.HTTPConflict:
                     pass
-                log.error("Can't start the GNS3 VM: {}", str(e))
+                log.error("Can't start the GNS3 VM: %s", str(e))
 
     @asyncio.coroutine
     def exit_vm(self):
@@ -287,7 +287,7 @@ class GNS3VM:
                 yield from engine.start()
             except Exception as e:
                 yield from self._controller.delete_compute("vm")
-                log.error("Can't start the GNS3 VM: {}", str(e))
+                log.error("Can't start the GNS3 VM: {}".format(str(e)))
                 yield from compute.update(name="GNS3 VM ({})".format(engine.vmname))
                 raise e
             yield from compute.update(name="GNS3 VM ({})".format(engine.vmname),

gns3server/controller/import_project.py

@@ -24,7 +24,6 @@ import asyncio
 import zipfile
 import aiohttp
 
-from ..config import Config
 from .topology import load_topology
 
 
@@ -74,7 +73,7 @@ def import_project(controller, project_id, stream, location=None, name=None, kee
                 path = location
             else:
                 projects_path = controller.projects_directory()
-                path = os.path.join(projects_path, project_name)
+                path = os.path.join(projects_path, project_id)
             try:
                 os.makedirs(path, exist_ok=True)
             except UnicodeEncodeError as e:

gns3server/controller/project.py

@@ -699,7 +699,10 @@ class Project:
         self._loading = False
         # Should we start the nodes when project is open
         if self._auto_start:
-            yield from self.start_all()
+            # Start all in the background without waiting for completion
+            # we ignore errors because we want to let the user open
+            # their project and fix it
+            asyncio.async(self.start_all())
 
     @asyncio.coroutine
     def wait_loaded(self):

gns3server/controller/topology.py

@@ -321,6 +321,10 @@ def _convert_1_3_later(topo, topo_path):
 
         node["properties"] = {}
 
+        # Some old dynamips node don't have type
+        if "type" not in old_node:
+            old_node["type"] = old_node["properties"]["platform"].upper()
+
         if old_node["type"] == "VPCSDevice":
             node["node_type"] = "vpcs"
         elif old_node["type"] == "QemuVM":
@@ -348,7 +352,7 @@ def _convert_1_3_later(topo, topo_path):
             node["symbol"] = ":/symbols/ethernet_switch.svg"
             node["console_type"] = None
             node["properties"]["ports_mapping"] = []
-            for port in old_node["ports"]:
+            for port in old_node.get("ports", []):
                 node["properties"]["ports_mapping"].append({
                     "name": "Ethernet{}".format(port["port_number"] - 1),
                     "port_number": port["port_number"] - 1,
@@ -359,12 +363,12 @@ def _convert_1_3_later(topo, topo_path):
             node["node_type"] = "frame_relay_switch"
             node["symbol"] = ":/symbols/frame_relay_switch.svg"
             node["console_type"] = None
-        elif old_node["type"] in ["C1700", "C2600", "C2691", "C3600", "C3620", "C3640", "C3660", "C3725", "C3745", "C7200", "EtherSwitchRouter"]:
+        elif old_node["type"].upper() in ["C1700", "C2600", "C2691", "C3600", "C3620", "C3640", "C3660", "C3725", "C3745", "C7200", "EtherSwitchRouter"]:
             if node["symbol"] is None:
                 node["symbol"] = ":/symbols/router.svg"
             node["node_type"] = "dynamips"
             node["properties"]["dynamips_id"] = old_node.get("dynamips_id")
-            if "platform" not in node["properties"] and old_node["type"].startswith("C"):
+            if "platform" not in node["properties"] and old_node["type"].upper().startswith("C"):
                 node["properties"]["platform"] = old_node["type"].lower()
                 if node["properties"]["platform"].startswith("c36"):
                     node["properties"]["platform"] = "c3600"

gns3server/crash_report.py

@@ -57,7 +57,7 @@ class CrashReport:
     Report crash to a third party service
     """
 
-    DSN = "sync+https://9b1156a90ee943eba20e032cf007297d:024b75c7b11844a58df147a4fb059774@sentry.io/38482"
+    DSN = "sync+https://67b93949a78d4ef5978388cc4b8906f9:271ee1dd01db4a39b919097f452cb6c5@sentry.io/38482"
     if hasattr(sys, "frozen"):
         cacert = get_resource("cacert.pem")
         if cacert is not None and os.path.isfile(cacert):

gns3server/handlers/api/controller/project_handler.py

@@ -22,7 +22,6 @@ import tempfile
 
 from gns3server.web.route import Route
 from gns3server.controller import Controller
-from gns3server.controller.project import Project
 from gns3server.controller.import_project import import_project
 from gns3server.controller.export_project import export_project
 from gns3server.config import Config

gns3server/schemas/dynamips_vm.py

@@ -33,7 +33,7 @@ VM_CREATE_SCHEMA = {
         },
         "dynamips_id": {
             "description": "Dynamips ID",
-            "type": "integer"
+            "type": ["integer", "null"]
         },
         "name": {
             "description": "Dynamips VM instance name",

gns3server/web/response.py

@@ -46,6 +46,7 @@ class Response(aiohttp.web.Response):
 
     def enable_chunked_encoding(self):
         # Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
+        if self.content_length:
             self.content_length = None
         super().enable_chunked_encoding()
 

gns3server/web/web_server.py

@@ -43,6 +43,9 @@ import gns3server.handlers
 import logging
 log = logging.getLogger(__name__)
 
+if not aiohttp.__version__.startswith("2.0"):
+    raise RuntimeError("You need aiohttp 2.0 for running GNS3")
+
 
 class WebServer:
 

scripts/remote-install.sh

@@ -152,7 +152,7 @@ log "Update system packages"
 apt-get update
 
 log "Upgrade packages"
-apt-get upgrade -y
+apt-get upgrade --yes --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"
 
 log " Install GNS3 packages"
 apt-get install -y gns3-server
@@ -163,6 +163,10 @@ then
   useradd -d /opt/gns3/ -m gns3
 fi
 
+
+log "Add GNS3 to the ubridge group"
+usermod -aG ubridge gns3
+
 log "Install docker"
 if [ ! -f "/usr/bin/docker" ]
 then

tests/compute/docker/test_docker_vm.py

@@ -840,7 +840,7 @@ def test_get_image_informations(project, manager, loop):
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Not supported on Windows")
 def test_mount_binds(vm, tmpdir):
     image_infos = {
-        "ContainerConfig": {
+        "Config": {
             "Volumes": {
                 "/test/experimental": {}
             }

tests/conftest.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
+import gc
 import pytest
 import socket
 import asyncio

tests/controller/test_export_project.py

@@ -27,13 +27,12 @@ from unittest.mock import MagicMock
 from tests.utils import AsyncioMagicMock, AsyncioBytesIO
 
 from gns3server.controller.project import Project
-from gns3server.controller.compute import Compute
 from gns3server.controller.export_project import export_project, _filter_files
 
 
 @pytest.fixture
 def project(controller):
-    p = Project(controller=controller, name="Test")
+    p = Project(controller=controller, name="test")
     p.dump = MagicMock()
     return p
 
@@ -179,7 +178,7 @@ def test_export_disallow_running(tmpdir, project, node, async_run):
 
     node._status = "started"
     with pytest.raises(aiohttp.web.HTTPConflict):
-        z = async_run(export_project(project, str(tmpdir)))
+        async_run(export_project(project, str(tmpdir)))
 
 
 def test_export_disallow_some_type(tmpdir, project, async_run):

tests/controller/test_import_project.py

@@ -35,6 +35,8 @@ def test_import_project(async_run, tmpdir, controller):
     topology = {
         "project_id": str(uuid.uuid4()),
         "name": "test",
+        "auto_open": True,
+        "auto_start": True,
         "topology": {
         },
         "version": "2.0.0"
@@ -67,6 +69,8 @@ def test_import_project(async_run, tmpdir, controller):
     # A new project name is generated when you import twice the same name
     with open(zip_path, "rb") as f:
         project = async_run(import_project(controller, str(uuid.uuid4()), f))
+    assert project.auto_open is False
+    assert project.auto_start is False
     assert project.name != "test"
 
 

tests/topologies/1_3_dynamips_missing_type/after/1_3_dynamips.gns3

@@ -0,0 +1,76 @@
+{
+    "auto_start": false,
+    "name": "1_3_dynamips",
+    "project_id": "ba5790e1-2f51-443e-a3cc-1a2eee132888",
+    "revision": 6,
+    "topology": {
+        "computes": [
+            {
+                "compute_id": "local",
+                "host": "127.0.0.1",
+                "name": "Local",
+                "port": 8000,
+                "protocol": "http"
+            }
+        ],
+        "drawings": [],
+        "links": [],
+        "nodes": [
+            {
+                "symbol": ":/symbols/iosv_virl.svg",
+                "compute_id": "local",
+                "console": 2001,
+                "console_type": "telnet",
+                "label": {
+                    "rotation": 0,
+                    "style": "font-family: TypeWriter;font-size: 10;font-weight: bold;fill: #000000;fill-opacity: 1.0;",
+                    "text": "R1",
+                    "x": 22,
+                    "y": -25
+                },
+                "name": "R1",
+                "node_id": "0bce6ad5-c688-4d4d-a425-f21aaf3927e2",
+                "node_type": "dynamips",
+                "port_name_format": "Ethernet{0}",
+                "port_segment_size": 0,
+                "first_port_name": null,
+                "properties": {
+                    "dynamips_id": 1,
+                    "auto_delete_disks": true,
+                    "clock_divisor": 4,
+                    "disk0": 0,
+                    "disk1": 0,
+                    "exec_area": 64,
+                    "idlemax": 500,
+                    "idlesleep": 30,
+                    "image": "c7200-adventerprisek9-mz.124-24.T8.image",
+                    "mac_addr": "ca01.2f39.0000",
+                    "midplane": "vxr",
+                    "mmap": true,
+                    "npe": "npe-400",
+                    "nvram": 512,
+                    "platform": "c7200",
+                    "power_supplies": [
+                        1,
+                        1
+                    ],
+                    "ram": 512,
+                    "sensors": [
+                        22,
+                        22,
+                        22,
+                        22
+                    ],
+                    "slot0": "C7200-IO-FE",
+                    "sparsemem": true,
+                    "system_id": "FTX0945W0MY"
+                },
+                "x": -112,
+                "y": -100,
+                "z": 1
+            }
+        ]
+    },
+    "type": "topology",
+    "version": "ANYSTR"
+}

tests/topologies/1_3_dynamips_missing_type/before/1_3_dynamips.gns3

@@ -0,0 +1,80 @@
+{
+    "auto_start": false,
+    "name": "1_3_dynamips",
+    "project_id": "ba5790e1-2f51-443e-a3cc-1a2eee132888",
+    "revision": 3,
+    "topology": {
+        "nodes": [
+            {
+                "default_symbol": ":/symbols/iosv_virl.normal.svg",
+                "description": "Router c7200",
+                "dynamips_id": 1,
+                "hover_symbol": ":/symbols/iosv_virl.selected.svg",
+                "id": 1,
+                "label": {
+                    "color": "#000000",
+                    "font": "TypeWriter,10,-1,5,75,0,0,0,0,0",
+                    "text": "R1",
+                    "x": 22.6171875,
+                    "y": -25.0
+                },
+                "ports": [
+                    {
+                        "adapter_number": 0,
+                        "id": 1,
+                        "name": "FastEthernet0/0",
+                        "port_number": 0
+                    }
+                ],
+                "properties": {
+                    "auto_delete_disks": true,
+                    "clock_divisor": 4,
+                    "console": 2001,
+                    "disk0": 0,
+                    "disk1": 0,
+                    "exec_area": 64,
+                    "idlemax": 500,
+                    "idlesleep": 30,
+                    "image": "c7200-adventerprisek9-mz.124-24.T8.image",
+                    "mac_addr": "ca01.2f39.0000",
+                    "midplane": "vxr",
+                    "mmap": true,
+                    "name": "R1",
+                    "npe": "npe-400",
+                    "nvram": 512,
+                    "platform": "c7200",
+                    "power_supplies": [
+                        1,
+                        1
+                    ],
+                    "ram": 512,
+                    "sensors": [
+                        22,
+                        22,
+                        22,
+                        22
+                    ],
+                    "slot0": "C7200-IO-FE",
+                    "sparsemem": true,
+                    "startup_config": "configs/i1_startup-config.cfg",
+                    "system_id": "FTX0945W0MY"
+                },
+                "server_id": 1,
+                "vm_id": "0bce6ad5-c688-4d4d-a425-f21aaf3927e2",
+                "x": -112.0,
+                "y": -100.0
+            }
+        ],
+        "servers": [
+            {
+                "cloud": false,
+                "host": "127.0.0.1",
+                "id": 1,
+                "local": true,
+                "port": 8000
+            }
+        ]
+    },
+    "type": "topology",
+    "version": "1.3.13"
+}

tests/utils.py

@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import io
+import types
 import asyncio
 import unittest.mock
 
@@ -69,6 +70,10 @@ class AsyncioMagicMock(unittest.mock.MagicMock):
     """
     Magic mock returning coroutine
     """
+    try:
+        __class__ = types.CoroutineType
+    except AttributeError:  # Not supported with Python 3.4
+        __class__ = types.GeneratorType
 
     def __init__(self, return_value=None, return_values=None, **kwargs):
         """