Merge branch 'unstable' into api_list_images

Conflicts:
	gns3server/handlers/api/iou_handler.py

README.rst

@@ -26,6 +26,12 @@ unstable
 ********
 *Never* use this branch for production. Major new features pull requests goes here. 
 
+Linux
+-----
+
+GNS3 is perhaps packaged for your distribution:
+* Gentoo: https://packages.gentoo.org/package/net-misc/gns3-server
+
 
 Linux (Debian based)
 --------------------
@@ -62,6 +68,26 @@ To run tests use:
 
    py.test -v
 
+
+Run as daemon 
+***************
+
+You will found init sample script for various systems
+inside the init directory.
+
+upstart
+~~~~~~~
+
+For ubuntu < 15.04
+
+You need to copy init/gns3.conf.upstart to /etc/init/gns3.conf
+
+.. code:: bash
+
+    sudo chown root /etc/init/gns3.conf
+    sudo service gns3 start
+
+
 Windows
 -------
 

gns3server/handlers/__init__.py

@@ -27,6 +27,7 @@ from gns3server.handlers.api.virtualbox_handler import VirtualBoxHandler
 from gns3server.handlers.api.vpcs_handler import VPCSHandler
 from gns3server.handlers.api.config_handler import ConfigHandler
 from gns3server.handlers.api.server_handler import ServerHandler
+from gns3server.handlers.api.file_handler import FileHandler
 from gns3server.handlers.upload_handler import UploadHandler
 from gns3server.handlers.index_handler import IndexHandler
 

gns3server/handlers/api/file_handler.py

@@ -0,0 +1,60 @@
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+import aiohttp
+
+from ...web.route import Route
+from ...schemas.file import FILE_STREAM_SCHEMA
+
+
+class FileHandler:
+
+    @classmethod
+    @Route.get(
+        r"/files/stream",
+        description="Stream a file from the server",
+        status_codes={
+            200: "File retrieved",
+            404: "File doesn't exist",
+            409: "Can't access to file"
+        },
+        input=FILE_STREAM_SCHEMA
+    )
+    def read(request, response):
+        response.enable_chunked_encoding()
+
+        try:
+            with open(request.json.get("location"), "rb") as f:
+                loop = asyncio.get_event_loop()
+                response.content_type = "application/octet-stream"
+                response.set_status(200)
+                # Very important: do not send a content lenght otherwise QT close the connection but curl can consume the Feed
+                response.content_length = None
+
+                response.start(request)
+
+                while True:
+                    data = yield from loop.run_in_executor(None, f.read, 16)
+                    if len(data) == 0:
+                        yield from asyncio.sleep(0.1)
+                    else:
+                        response.write(data)
+        except FileNotFoundError:
+            raise aiohttp.web.HTTPNotFound()
+        except OSError as e:
+            raise aiohttp.web.HTTPConflict(text=str(e))
+

gns3server/handlers/api/iou_handler.py

@@ -310,7 +310,7 @@ class IOUHandler:
         vm = iou_manager.get_vm(request.match_info["vm_id"],
                                 project_id=request.match_info["project_id"])
         response.set_status(200)
-        response.json({"content": vm.initial_config})
+        response.json({"content": vm.initial_config_content})
 
     @Route.get(
         r"/iou/vms",
@@ -325,3 +325,4 @@ class IOUHandler:
         vms = yield from iou_manager.list_images()
         response.set_status(200)
         response.json(vms)
+

gns3server/main.py

@@ -21,6 +21,8 @@ import datetime
 import sys
 import locale
 import argparse
+import asyncio
+import concurrent
 
 from gns3server.server import Server
 from gns3server.web.logger import init_logger
@@ -92,6 +94,7 @@ def parse_arguments(argv, config):
         "quiet": config.getboolean("quiet", False),
         "debug": config.getboolean("debug", False),
         "live": config.getboolean("live", False),
+        "logfile": config.getboolean("logfile", ""),
     }
 
     parser = argparse.ArgumentParser(description="GNS3 server version {}".format(__version__))
@@ -109,6 +112,7 @@ def parse_arguments(argv, config):
     parser.add_argument("-d", "--debug", action="store_true", help="show debug logs")
     parser.add_argument("--live", action="store_true", help="enable code live reload")
     parser.add_argument("--shell", action="store_true", help="start a shell inside the server (debugging purpose only you need to install ptpython before)")
+    parser.add_argument("--log", help="send output to logfile instead of console")
 
     return parser.parse_args(argv)
 
@@ -141,7 +145,7 @@ def main():
     if args.debug:
         level = logging.DEBUG
 
-    user_log = init_logger(level, quiet=args.quiet)
+    user_log = init_logger(level, logfile=args.log, quiet=args.quiet)
     user_log.info("GNS3 server version {}".format(__version__))
     current_year = datetime.date.today().year
     user_log.info("Copyright (c) 2007-{} GNS3 Technologies Inc.".format(current_year))
@@ -173,6 +177,9 @@ def main():
 
     Project.clean_project_directory()
 
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=100)  # We allow 100 parallel executors
+    loop = asyncio.get_event_loop().set_default_executor(executor)
+
     CrashReport.instance()
     host = server_config["host"]
     port = int(server_config["port"])

gns3server/modules/iou/iou_vm.py

@@ -317,9 +317,9 @@ class IOUVM(BaseVM):
         """
 
         if self.initial_config_file:
-            content = self.initial_config
+            content = self.initial_config_content
             content = content.replace(self._name, new_name)
-            self.initial_config = content
+            self.initial_config_content = content
 
         super(IOUVM, IOUVM).name.__set__(self, new_name)
 
@@ -372,8 +372,8 @@ class IOUVM(BaseVM):
         Checks for a valid IOU key in the iourc file (paranoid mode).
         """
 
-        license_check = self._manager.config.get_section_config("IOU").getboolean("license_check", False)
-        if license_check:
+        license_check = self._manager.config.get_section_config("IOU").getboolean("license_check", True)
+        if license_check is False:
             return
 
         config = configparser.ConfigParser()
@@ -939,7 +939,7 @@ class IOUVM(BaseVM):
             log.warn("could not determine if layer 1 keepalive messages are supported by {}: {}".format(os.path.basename(self._path), e))
 
     @property
-    def initial_config(self):
+    def initial_config_content(self):
         """
         Returns the content of the current initial-config file.
         """
@@ -952,10 +952,10 @@ class IOUVM(BaseVM):
             with open(config_file) as f:
                 return f.read()
         except OSError as e:
-            raise IOUError("Can't read configuration file '{}'".format(config_file))
+            raise IOUError("Can't read configuration file '{}': {}".format(config_file, e))
 
-    @initial_config.setter
-    def initial_config(self, initial_config):
+    @initial_config_content.setter
+    def initial_config_content(self, initial_config):
         """
         Update the initial config
 

gns3server/modules/qemu/__init__.py

@@ -67,7 +67,7 @@ class Qemu(BaseManager):
         for path in paths:
             try:
                 for f in os.listdir(path):
-                    if (f.startswith("qemu-system") or f == "qemu" or f == "qemu.exe") and \
+                    if (f.startswith("qemu-system") or f.startswith("qemu-kvm") or f == "qemu" or f == "qemu.exe") and \
                             os.access(os.path.join(path, f), os.X_OK) and \
                             os.path.isfile(os.path.join(path, f)):
                         qemu_path = os.path.join(path, f)

gns3server/schemas/dynamips_vm.py

@@ -284,20 +284,10 @@ VM_UPDATE_SCHEMA = {
             "type": "string",
             "minLength": 1,
         },
-        "startup_config": {
-            "description": "path to the IOS startup configuration file",
-            "type": "string",
-            "minLength": 1,
-        },
         "startup_config_content": {
             "description": "Content of IOS startup configuration file",
             "type": "string",
         },
-        "private_config": {
-            "description": "path to the IOS private configuration file",
-            "type": "string",
-            "minLength": 1,
-        },
         "private_config_content": {
             "description": "Content of IOS private configuration file",
             "type": "string",

gns3server/schemas/file.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+FILE_STREAM_SCHEMA = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "Request validation retrieval of a file stream",
+    "type": "object",
+    "properties": {
+        "location": {
+            "description": "File path",
+            "type": ["string"],
+            "minLength": 1
+        }
+    },
+    "additionalProperties": False,
+    "required": ["location"]
+}

gns3server/schemas/iou.py

@@ -127,10 +127,6 @@ IOU_UPDATE_SCHEMA = {
             "description": "Always up ethernet interface",
             "type": ["boolean", "null"]
         },
-        "initial_config": {
-            "description": "Path to the initial configuration of IOU",
-            "type": ["string", "null"]
-        },
         "initial_config_content": {
             "description": "Initial configuration of IOU",
             "type": ["string", "null"]

gns3server/server.py

@@ -225,6 +225,11 @@ class Server:
 
         try:
             self._loop.run_forever()
+        except OSError as e:
+            # This is to ignore OSError: [WinError 0] The operation completed successfully
+            # exception on Windows.
+            if not sys.platform.startswith("win") and not e.winerror == 0:
+                raise
         except TypeError as e:
             # This is to ignore an asyncio.windows_events exception
             # on Windows when the process gets the SIGBREAK signal

gns3server/web/logger.py

@@ -78,8 +78,11 @@ class ColouredStreamHandler(logging.StreamHandler):
             self.handleError(record)
 
 
-def init_logger(level, quiet=False):
-    if sys.platform.startswith("win"):
+def init_logger(level, logfile=None, quiet=False):
+    if logfile and len(logfile) > 0:
+        stream_handler = logging.FileHandler(logfile)
+        stream_handler.formatter = ColouredFormatter("{asctime} {levelname} {filename}:{lineno} {message}", "%Y-%m-%d %H:%M:%S", "{")
+    elif sys.platform.startswith("win"):
         stream_handler = logging.StreamHandler(sys.stdout)
         stream_handler.formatter = ColouredFormatter("{asctime} {levelname} {filename}:{lineno} {message}", "%Y-%m-%d %H:%M:%S", "{")
     else:

init/gns3.conf.upstart

@@ -0,0 +1,19 @@
+description "GNS3 server"
+author      "GNS3 Team"
+
+start on filesystem or runlevel [2345]
+stop on shutdown
+
+script
+    echo $$ > /var/run/gns3.pid
+    exec start-stop-daemon --start -c gns3 --exec /usr/local/bin/gns3server --log /var/log/gns3.log
+end script
+
+pre-start script
+    echo "[`date`] GNS3 Starting" >> /var/log/gns3.log
+end script
+
+pre-stop script
+    rm /var/run/gns3.pid
+    echo "[`date`] GNS3 Stopping" >> /var/log/gns3.log
+end script

tests/handlers/api/base.py

@@ -44,7 +44,7 @@ class Query:
     def delete(self, path, **kwargs):
         return self._fetch("DELETE", path, **kwargs)
 
-    def _get_url(self, path, version):
+    def get_url(self, path, version):
         if version is None:
             return "http://{}:{}{}".format(self._host, self._port, path)
         return "http://{}:{}/v{}{}".format(self._host, self._port, version, path)
@@ -62,7 +62,7 @@ class Query:
 
         @asyncio.coroutine
         def go(future):
-            response = yield from aiohttp.request(method, self._get_url(path, api_version), data=body)
+            response = yield from aiohttp.request(method, self.get_url(path, api_version), data=body)
             future.set_result(response)
         future = asyncio.Future()
         asyncio.async(go(future))

tests/handlers/api/test_file.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+This test suite check /files endpoint
+"""
+
+import json
+import asyncio
+import aiohttp
+
+from gns3server.version import __version__
+
+
+def test_stream(server, tmpdir, loop):
+    with open(str(tmpdir / "test"), 'w+') as f:
+        f.write("hello")
+
+    def go(future):
+        query = json.dumps({"location": str(tmpdir / "test")})
+        headers = {'content-type': 'application/json'}
+        response = yield from aiohttp.request("GET", server.get_url("/files/stream", 1), data=query, headers=headers)
+        response.body = yield from response.content.read(5)
+        with open(str(tmpdir / "test"), 'a') as f:
+            f.write("world")
+        response.body += yield from response.content.read(5)
+        response.close()
+        future.set_result(response)
+
+    future = asyncio.Future()
+    asyncio.async(go(future))
+    response = loop.run_until_complete(future)
+    assert response.status == 200
+    assert response.body == b'helloworld'
+
+
+def test_stream_file_not_found(server, tmpdir, loop):
+    def go(future):
+        query = json.dumps({"location": str(tmpdir / "test")})
+        headers = {'content-type': 'application/json'}
+        response = yield from aiohttp.request("GET", server.get_url("/files/stream", 1), data=query, headers=headers)
+        response.close()
+        future.set_result(response)
+
+    future = asyncio.Future()
+    asyncio.async(go(future))
+    response = loop.run_until_complete(future)
+    assert response.status == 404

tests/modules/iou/test_iou_vm.py

@@ -81,11 +81,11 @@ def test_vm(project, manager):
     assert vm.id == "00010203-0405-0607-0809-0a0b0c0d0e0f"
 
 
-def test_vm_initial_config(project, manager):
+def test_vm_initial_config_content(project, manager):
     vm = IOUVM("test", "00010203-0405-0607-0808-0a0b0c0d0e0f", project, manager)
-    vm.initial_config = "hostname %h"
+    vm.initial_config_content = "hostname %h"
     assert vm.name == "test"
-    assert vm.initial_config == "hostname test"
+    assert vm.initial_config_content == "hostname test"
     assert vm.id == "00010203-0405-0607-0808-0a0b0c0d0e0f"
 
 
@@ -287,10 +287,10 @@ def test_update_initial_config_empty(vm):
         assert f.read() == content
 
 
-def test_update_initial_config_h(vm):
+def test_update_initial_config_content_hostname(vm):
     content = "hostname %h\n"
     vm.name = "pc1"
-    vm.initial_config = content
+    vm.initial_config_content = content
     with open(vm.initial_config_file) as f:
         assert f.read() == "hostname pc1\n"
 

tests/modules/qemu/test_qemu_manager.py

@@ -32,7 +32,7 @@ def test_get_qemu_version(loop):
 
 def test_binary_list(loop):
 
-    files_to_create = ["qemu-system-x86", "qemu-system-x42", "hello"]
+    files_to_create = ["qemu-system-x86", "qemu-system-x42", "qemu-kvm", "hello"]
 
     for file_to_create in files_to_create:
         path = os.path.join(os.environ["PATH"], file_to_create)
@@ -44,6 +44,7 @@ def test_binary_list(loop):
         qemus = loop.run_until_complete(asyncio.async(Qemu.binary_list()))
 
         assert {"path": os.path.join(os.environ["PATH"], "qemu-system-x86"), "version": "2.2.0"} in qemus
+        assert {"path": os.path.join(os.environ["PATH"], "qemu-kvm"), "version": "2.2.0"} in qemus
         assert {"path": os.path.join(os.environ["PATH"], "qemu-system-x42"), "version": "2.2.0"} in qemus
         assert {"path": os.path.join(os.environ["PATH"], "hello"), "version": "2.2.0"} not in qemus