All current iou code is async

gns3server/modules/iou/iou_vm.py

@@ -22,10 +22,10 @@ order to run an IOU instance.
 
 import os
 import sys
-import subprocess
 import signal
 import re
 import asyncio
+import subprocess
 import shutil
 import argparse
 import threading
@@ -40,6 +40,7 @@ from ..nios.nio_udp import NIO_UDP
 from ..nios.nio_tap import NIO_TAP
 from ..base_vm import BaseVM
 from .ioucon import start_ioucon
+import gns3server.utils.asyncio
 
 
 import logging
@@ -90,6 +91,7 @@ class IOUVM(BaseVM):
         self._console_host = console_host
 
         # IOU settings
+        self._iourc = None
         self._ethernet_adapters = []
         self._serial_adapters = []
         self.ethernet_adapters = 2 if ethernet_adapters is None else ethernet_adapters  # one adapter = 4 interfaces
@@ -327,20 +329,21 @@ class IOUVM(BaseVM):
     def application_id(self):
         return self._manager.get_application_id(self.id)
 
-    # TODO: ASYNCIO
+    @asyncio.coroutine
     def _library_check(self):
         """
         Checks for missing shared library dependencies in the IOU image.
         """
 
         try:
-            output = subprocess.check_output(["ldd", self._path])
+            output = yield from gns3server.utils.asyncio.subprocess_check_output("ldd", self._path)
         except (FileNotFoundError, subprocess.SubprocessError) as e:
-            log.warn("could not determine the shared library dependencies for {}: {}".format(self._path, e))
+            log.warn("Could not determine the shared library dependencies for {}: {}".format(self._path, e))
             return
 
+        print(output)
         p = re.compile("([\.\w]+)\s=>\s+not found")
-        missing_libs = p.findall(output.decode("utf-8"))
+        missing_libs = p.findall(output)
         if missing_libs:
             raise IOUError("The following shared library dependencies cannot be found for IOU image {}: {}".format(self._path,
                                                                                                                    ", ".join(missing_libs)))
@@ -354,10 +357,9 @@ class IOUVM(BaseVM):
         self._check_requirements()
         if not self.is_running():
 
-            self._rename_nvram_file()
+            yield from self._library_check()
 
-            # TODO: ASYNC
+            self._rename_nvram_file()
-            # self._library_check()
 
             if self._iourc_path and not os.path.isfile(self._iourc_path):
                 raise IOUError("A valid iourc file is necessary to start IOU")
@@ -371,7 +373,7 @@ class IOUVM(BaseVM):
             env = os.environ.copy()
             if self._iourc_path:
                 env["IOURC"] = self._iourc_path
-            self._command = self._build_command()
+            self._command = yield from self._build_command()
             try:
                 log.info("Starting IOU: {}".format(self._command))
                 self._iou_stdout_file = os.path.join(self.working_dir, "iou.log")
@@ -394,7 +396,7 @@ class IOUVM(BaseVM):
             # start console support
             self._start_ioucon()
             # connections support
-            self._start_iouyap()
+            yield from self._start_iouyap()
 
     def _rename_nvram_file(self):
         """
@@ -405,6 +407,7 @@ class IOUVM(BaseVM):
         for file_path in glob.glob(os.path.join(self.working_dir, "nvram_*")):
             shutil.move(file_path, destination)
 
+    @asyncio.coroutine
     def _start_iouyap(self):
         """
         Starts iouyap (handles connections to and from this IOU device).
@@ -417,7 +420,7 @@ class IOUVM(BaseVM):
             self._iouyap_stdout_file = os.path.join(self.working_dir, "iouyap.log")
             log.info("logging to {}".format(self._iouyap_stdout_file))
             with open(self._iouyap_stdout_file, "w") as fd:
-                self._iouyap_process = subprocess.Popen(command,
+                self._iouyap_process = yield from asyncio.create_subprocess_exec(*command,
                                                                                  stdout=fd,
                                                                                  stderr=subprocess.STDOUT,
                                                                                  cwd=self.working_dir)
@@ -596,6 +599,7 @@ class IOUVM(BaseVM):
         except OSError as e:
             raise IOUError("Could not create {}: {}".format(netmap_path, e))
 
+    @asyncio.coroutine
     def _build_command(self):
         """
         Command to start the IOU process.
@@ -639,7 +643,7 @@ class IOUVM(BaseVM):
         if initial_config_file:
             command.extend(["-c", initial_config_file])
         if self._l1_keepalives:
-            self._enable_l1_keepalives(command)
+            yield from self._enable_l1_keepalives(command)
         command.extend([str(self.application_id)])
         return command
 
@@ -819,6 +823,7 @@ class IOUVM(BaseVM):
         else:
             log.info("IOU {name} [id={id}]: has deactivated layer 1 keepalive messages".format(name=self._name, id=self._id))
 
+    @asyncio.coroutine
     def _enable_l1_keepalives(self, command):
         """
         Enables L1 keepalive messages if supported.
@@ -828,8 +833,8 @@ class IOUVM(BaseVM):
         env = os.environ.copy()
         env["IOURC"] = self._iourc
         try:
-            output = subprocess.check_output([self._path, "-h"], stderr=subprocess.STDOUT, cwd=self._working_dir, env=env)
+            output = yield from gns3server.utils.asyncio.subprocess_check_output(self._path, "-h", cwd=self.working_dir, env=env)
-            if re.search("-l\s+Enable Layer 1 keepalive messages", output.decode("utf-8")):
+            if re.search("-l\s+Enable Layer 1 keepalive messages", output):
                 command.extend(["-l"])
             else:
                 raise IOUError("layer 1 keepalive messages are not supported by {}".format(os.path.basename(self._path)))

gns3server/modules/vpcs/vpcs_vm.py

@@ -32,6 +32,7 @@ from pkg_resources import parse_version
 from .vpcs_error import VPCSError
 from ..adapters.ethernet_adapter import EthernetAdapter
 from ..base_vm import BaseVM
+from ...utils.asyncio import subprocess_check_output
 
 
 import logging
@@ -195,7 +196,7 @@ class VPCSVM(BaseVM):
         Checks if the VPCS executable version is >= 0.5b1.
         """
         try:
-            output = yield from self._get_vpcs_welcome()
+            output = yield from subprocess_check_output(self.vpcs_path, "-v", cwd=self.working_dir)
             match = re.search("Welcome to Virtual PC Simulator, version ([0-9a-z\.]+)", output)
             if match:
                 version = match.group(1)
@@ -206,12 +207,6 @@ class VPCSVM(BaseVM):
         except (OSError, subprocess.SubprocessError) as e:
             raise VPCSError("Error while looking for the VPCS version: {}".format(e))
 
-    @asyncio.coroutine
-    def _get_vpcs_welcome(self):
-        proc = yield from asyncio.create_subprocess_exec(self.vpcs_path, "-v", stdout=asyncio.subprocess.PIPE, cwd=self.working_dir)
-        out = yield from proc.stdout.read()
-        return out.decode("utf-8")
-
     @asyncio.coroutine
     def start(self):
         """

gns3server/utils/asyncio.py

@@ -17,6 +17,7 @@
 
 
 import asyncio
+import shutil
 
 
 @asyncio.coroutine
@@ -34,3 +35,21 @@ def wait_run_in_executor(func, *args):
     future = loop.run_in_executor(None, func, *args)
     yield from asyncio.wait([future])
     return future.result()
+
+
+@asyncio.coroutine
+def subprocess_check_output(*args, working_dir=None, env=None):
+    """
+    Run a command and capture output
+
+    :param *args: List of command arguments
+    :param working_dir: Working directory
+    :param env: Command environment
+    :returns: Command output
+    """
+
+    proc = yield from asyncio.create_subprocess_exec(*args, stdout=asyncio.subprocess.PIPE, cwd=working_dir, env=env)
+    output = yield from proc.stdout.read()
+    if output is None:
+        return ""
+    return output.decode("utf-8")

tests/conftest.py

@@ -37,6 +37,17 @@ from tests.api.base import Query
 os.environ["PATH"] = tempfile.mkdtemp()
 
 
+@pytest.yield_fixture
+def restore_original_path():
+    """
+    Temporary restore a standard path environnement. This allow
+    to run external binaries.
+    """
+    os.environ["PATH"] = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+    yield
+    os.environ["PATH"] = tempfile.mkdtemp()
+
+
 @pytest.fixture(scope="session")
 def loop(request):
     """Return an event loop and destroy it at the end of test"""

tests/modules/iou/test_iou_vm.py

@@ -184,18 +184,18 @@ def test_create_netmap_config(vm):
     assert "513:15/3    1:15/3" in content
 
 
-def test_build_command(vm):
+def test_build_command(vm, loop):
 
-    assert vm._build_command() == [vm.path, "-L", str(vm.application_id)]
+    assert loop.run_until_complete(asyncio.async(vm._build_command())) == [vm.path, "-L", str(vm.application_id)]
 
 
-def test_build_command_initial_config(vm):
+def test_build_command_initial_config(vm, loop):
 
     filepath = os.path.join(vm.working_dir, "initial-config.cfg")
     with open(filepath, "w+") as f:
         f.write("service timestamps debug datetime msec\nservice timestamps log datetime msec\nno service password-encryption")
 
-    assert vm._build_command() == [vm.path, "-L", "-c", vm.initial_config_file, str(vm.application_id)]
+    assert loop.run_until_complete(asyncio.async(vm._build_command())) == [vm.path, "-L", "-c", vm.initial_config_file, str(vm.application_id)]
 
 
 def test_get_initial_config(vm):
@@ -231,3 +231,30 @@ def test_change_name(vm, tmpdir):
     assert vm.name == "hello"
     with open(path) as f:
         assert f.read() == "hostname hello"
+
+
+def test_library_check(loop, vm):
+
+    with asyncio_patch("gns3server.utils.asyncio.subprocess_check_output", return_value="") as mock:
+
+        loop.run_until_complete(asyncio.async(vm._library_check()))
+
+    with asyncio_patch("gns3server.utils.asyncio.subprocess_check_output", return_value="libssl => not found") as mock:
+        with pytest.raises(IOUError):
+            loop.run_until_complete(asyncio.async(vm._library_check()))
+
+
+def test_enable_l1_keepalives(loop, vm):
+
+    with asyncio_patch("gns3server.utils.asyncio.subprocess_check_output", return_value="***************************************************************\n\n-l		Enable Layer 1 keepalive messages\n-u <n>		UDP port base for distributed networks\n") as mock:
+
+        command = ["test"]
+        loop.run_until_complete(asyncio.async(vm._enable_l1_keepalives(command)))
+        assert command == ["test", "-l"]
+
+    with asyncio_patch("gns3server.utils.asyncio.subprocess_check_output", return_value="***************************************************************\n\n-u <n>		UDP port base for distributed networks\n") as mock:
+
+        command = ["test"]
+        with pytest.raises(IOUError):
+            loop.run_until_complete(asyncio.async(vm._enable_l1_keepalives(command)))
+            assert command == ["test"]

tests/modules/vpcs/test_vpcs_vm.py

@@ -47,7 +47,7 @@ def test_vm(project, manager):
 
 
 def test_vm_invalid_vpcs_version(loop, project, manager):
-    with asyncio_patch("gns3server.modules.vpcs.vpcs_vm.VPCSVM._get_vpcs_welcome", return_value="Welcome to Virtual PC Simulator, version 0.1"):
+    with asyncio_patch("gns3server.utils.asyncio.subprocess_check_output", return_value="Welcome to Virtual PC Simulator, version 0.1"):
         with pytest.raises(VPCSError):
             vm = VPCSVM("test", "00010203-0405-0607-0809-0a0b0c0d0e0f", project, manager)
             nio = manager.create_nio(vm.vpcs_path, {"type": "nio_udp", "lport": 4242, "rport": 4243, "rhost": "127.0.0.1"})

tests/utils/test_asyncio.py

@@ -19,7 +19,7 @@
 import asyncio
 import pytest
 
-from gns3server.utils.asyncio import wait_run_in_executor
+from gns3server.utils.asyncio import wait_run_in_executor, subprocess_check_output
 
 
 def test_wait_run_in_executor(loop):
@@ -40,3 +40,13 @@ def test_exception_wait_run_in_executor(loop):
     exec = wait_run_in_executor(raise_exception)
     with pytest.raises(Exception):
         result = loop.run_until_complete(asyncio.async(exec))
+
+
+def test_subprocess_check_output(loop, tmpdir, restore_original_path):
+
+    path = str(tmpdir / "test")
+    with open(path, "w+") as f:
+        f.write("TEST")
+    exec = subprocess_check_output("cat", path)
+    result = loop.run_until_complete(asyncio.async(exec))
+    assert result == "TEST"