From 902de3dd477d72a5e614212eccdb08a9af4ae7ce Mon Sep 17 00:00:00 2001 From: grossmj Date: Sat, 25 Aug 2018 14:10:47 +0700 Subject: [PATCH] Refactor asyncio locking system for Python 3.7 support. Ref https://github.com/GNS3/gns3-gui/issues/2566 Ref https://github.com/GNS3/gns3-gui/issues/2568 --- gns3server/compute/base_node.py | 5 ++- gns3server/compute/docker/__init__.py | 5 ++- gns3server/compute/iou/iou_vm.py | 5 ++- .../compute/virtualbox/virtualbox_vm.py | 5 ++- gns3server/compute/vmware/vmware_vm.py | 5 ++- gns3server/controller/compute.py | 5 ++- gns3server/controller/gns3vm/__init__.py | 11 ++++-- gns3server/controller/project.py | 8 ++-- gns3server/ubridge/ubridge_hypervisor.py | 5 ++- gns3server/utils/asyncio/__init__.py | 38 ++++++++++--------- tests/conftest.py | 2 +- tests/utils/test_asyncio.py | 5 ++- 12 files changed, 57 insertions(+), 42 deletions(-) diff --git a/gns3server/compute/base_node.py b/gns3server/compute/base_node.py index ec1d8021..82e4eb6a 100644 --- a/gns3server/compute/base_node.py +++ b/gns3server/compute/base_node.py @@ -29,7 +29,7 @@ import re from gns3server.utils.interfaces import interfaces from ..compute.port_manager import PortManager -from ..utils.asyncio import wait_run_in_executor, locked_coroutine +from ..utils.asyncio import wait_run_in_executor, locking from ..utils.asyncio.telnet_server import AsyncioTelnetServer from ..ubridge.hypervisor import Hypervisor from ..ubridge.ubridge_error import UbridgeError @@ -516,7 +516,8 @@ class BaseNode: except UbridgeError as e: raise UbridgeError("Error while sending command '{}': {}: {}".format(command, e, self._ubridge_hypervisor.read_stdout())) - @locked_coroutine + @locking + @asyncio.coroutine def _start_ubridge(self): """ Starts uBridge (handles connections to and from this node). diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index e746f30e..f7ee075e 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -25,7 +25,7 @@ import asyncio import logging import aiohttp from gns3server.utils import parse_version -from gns3server.utils.asyncio import locked_coroutine +from gns3server.utils.asyncio import locking from gns3server.compute.base_manager import BaseManager from gns3server.compute.docker.docker_vm import DockerVM from gns3server.compute.docker.docker_error import DockerError, DockerHttp304Error, DockerHttp404Error @@ -182,7 +182,8 @@ class Docker(BaseManager): autoping=True) return connection - @locked_coroutine + @locking + @asyncio.coroutine def pull_image(self, image, progress_callback=None): """ Pull image from docker repository diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index 0f2cb636..852a05f3 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -43,7 +43,7 @@ from .utils.iou_export import nvram_export from gns3server.ubridge.ubridge_error import UbridgeError from gns3server.utils.file_watcher import FileWatcher from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer -from gns3server.utils.asyncio import locked_coroutine +from gns3server.utils.asyncio import locking import gns3server.utils.asyncio import gns3server.utils.images @@ -550,7 +550,8 @@ class IOUVM(BaseNode): # configure networking support yield from self._networking() - @locked_coroutine + @locking + @asyncio.coroutine def _networking(self): """ Configures the IOL bridge in uBridge. diff --git a/gns3server/compute/virtualbox/virtualbox_vm.py b/gns3server/compute/virtualbox/virtualbox_vm.py index 2f878ffe..caefb0c9 100644 --- a/gns3server/compute/virtualbox/virtualbox_vm.py +++ b/gns3server/compute/virtualbox/virtualbox_vm.py @@ -33,7 +33,7 @@ import xml.etree.ElementTree as ET from gns3server.utils import parse_version from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer from gns3server.utils.asyncio.serial import asyncio_open_serial -from gns3server.utils.asyncio import locked_coroutine +from gns3server.utils.asyncio import locking from gns3server.compute.virtualbox.virtualbox_error import VirtualBoxError from gns3server.compute.nios.nio_udp import NIOUDP from gns3server.compute.adapters.ethernet_adapter import EthernetAdapter @@ -296,7 +296,8 @@ class VirtualBoxVM(BaseNode): if (yield from self.check_hw_virtualization()): self._hw_virtualization = True - @locked_coroutine + @locking + @asyncio.coroutine def stop(self): """ Stops this VirtualBox VM. diff --git a/gns3server/compute/vmware/vmware_vm.py b/gns3server/compute/vmware/vmware_vm.py index e048d1c8..15fac4f5 100644 --- a/gns3server/compute/vmware/vmware_vm.py +++ b/gns3server/compute/vmware/vmware_vm.py @@ -26,7 +26,7 @@ import tempfile from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer from gns3server.utils.asyncio.serial import asyncio_open_serial -from gns3server.utils.asyncio import locked_coroutine +from gns3server.utils.asyncio import locking from collections import OrderedDict from .vmware_error import VMwareError from ..nios.nio_udp import NIOUDP @@ -94,7 +94,8 @@ class VMwareVM(BaseNode): return self._vmnets - @locked_coroutine + @locking + @asyncio.coroutine def _control_vm(self, subcommand, *additional_args): args = [self._vmx_path] diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index c3231a32..ff565a13 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -27,7 +27,7 @@ from operator import itemgetter from ..utils import parse_version from ..utils.images import list_images -from ..utils.asyncio import locked_coroutine, asyncio_ensure_future +from ..utils.asyncio import locking, asyncio_ensure_future from ..controller.controller_error import ControllerError from ..version import __version__, __version_info__ @@ -400,7 +400,8 @@ class Compute: except aiohttp.web.HTTPConflict: pass - @locked_coroutine + @locking + @asyncio.coroutine def connect(self): """ Check if remote server is accessible diff --git a/gns3server/controller/gns3vm/__init__.py b/gns3server/controller/gns3vm/__init__.py index 8ffd25b6..cf38a0f3 100644 --- a/gns3server/controller/gns3vm/__init__.py +++ b/gns3server/controller/gns3vm/__init__.py @@ -21,7 +21,7 @@ import asyncio import aiohttp import ipaddress -from ...utils.asyncio import locked_coroutine, asyncio_ensure_future +from ...utils.asyncio import locking, asyncio_ensure_future from .vmware_gns3_vm import VMwareGNS3VM from .virtualbox_gns3_vm import VirtualBoxGNS3VM from .remote_gns3_vm import RemoteGNS3VM @@ -265,7 +265,8 @@ class GNS3VM: except GNS3VMError as e: log.warn(str(e)) - @locked_coroutine + @locking + @asyncio.coroutine def start(self): """ Start the GNS3 VM @@ -339,7 +340,8 @@ class GNS3VM: except aiohttp.web.HTTPConflict as e: log.warning("Could not check the VM is in the same subnet as the local server: {}".format(e.text)) - @locked_coroutine + @locking + @asyncio.coroutine def _suspend(self): """ Suspend the GNS3 VM @@ -351,7 +353,8 @@ class GNS3VM: log.info("Suspend the GNS3 VM") yield from engine.suspend() - @locked_coroutine + @locking + @asyncio.coroutine def _stop(self): """ Stop the GNS3 VM diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 7c750f80..b067806f 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -36,7 +36,7 @@ from .udp_link import UDPLink from ..config import Config from ..utils.path import check_path_allowed, get_default_project_directory from ..utils.asyncio.pool import Pool -from ..utils.asyncio import locked_coroutine +from ..utils.asyncio import locking from ..utils.asyncio import wait_run_in_executor from ..utils.asyncio import asyncio_ensure_future from .export_project import export_project @@ -525,7 +525,8 @@ class Project: self.dump() return node - @locked_coroutine + @locking + @asyncio.coroutine def __delete_node_links(self, node): """ Delete all link connected to this node. @@ -783,7 +784,8 @@ class Project: def _topology_file(self): return os.path.join(self.path, self._filename) - @locked_coroutine + @locking + @asyncio.coroutine def open(self): """ Load topology elements diff --git a/gns3server/ubridge/ubridge_hypervisor.py b/gns3server/ubridge/ubridge_hypervisor.py index 221ef207..092189ee 100644 --- a/gns3server/ubridge/ubridge_hypervisor.py +++ b/gns3server/ubridge/ubridge_hypervisor.py @@ -20,7 +20,7 @@ import time import logging import asyncio -from ..utils.asyncio import locked_coroutine +from ..utils.asyncio import locking from .ubridge_error import UbridgeError log = logging.getLogger(__name__) @@ -176,7 +176,8 @@ class UBridgeHypervisor: self._host = host - @locked_coroutine + @locking + @asyncio.coroutine def send(self, command): """ Sends commands to this hypervisor. diff --git a/gns3server/utils/asyncio/__init__.py b/gns3server/utils/asyncio/__init__.py index 70a2cae1..adc2f2a6 100644 --- a/gns3server/utils/asyncio/__init__.py +++ b/gns3server/utils/asyncio/__init__.py @@ -138,26 +138,28 @@ def wait_for_named_pipe_creation(pipe_path, timeout=60): return raise asyncio.TimeoutError() +#FIXME: Use the following wrapper when we drop Python 3.4 and use the async def syntax +# def locking(f): +# +# @wraps(f) +# async def wrapper(oself, *args, **kwargs): +# lock_name = "__" + f.__name__ + "_lock" +# if not hasattr(oself, lock_name): +# setattr(oself, lock_name, asyncio.Lock()) +# async with getattr(oself, lock_name): +# return await f(oself, *args, **kwargs) +# return wrapper -def locked_coroutine(f): - """ - Method decorator that replace asyncio.coroutine that warranty - that this specific method of this class instance will not we - executed twice at the same time - """ - @asyncio.coroutine - def new_function(*args, **kwargs): +def locking(f): - # In the instance of the class we will store - # a lock has an attribute. - lock_var_name = "__" + f.__name__ + "_lock" - if not hasattr(args[0], lock_var_name): - setattr(args[0], lock_var_name, asyncio.Lock()) - - with (yield from getattr(args[0], lock_var_name)): - return (yield from f(*args, **kwargs)) - - return new_function + @functools.wraps(f) + def wrapper(oself, *args, **kwargs): + lock_name = "__" + f.__name__ + "_lock" + if not hasattr(oself, lock_name): + setattr(oself, lock_name, asyncio.Lock()) + with (yield from getattr(oself, lock_name)): + return (yield from f(oself, *args, **kwargs)) + return wrapper #FIXME: conservative approach to supported versions, please remove it when we drop the support to Python < 3.4.4 try: diff --git a/tests/conftest.py b/tests/conftest.py index da83b431..0fcdd055 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -104,7 +104,7 @@ def http_server(request, loop, port_manager, monkeypatch, controller): monkeypatch.setattr('gns3server.compute.virtualbox.virtualbox_vm.VirtualBoxVM.close', lambda self: True) loop.run_until_complete(instance.unload()) srv.close() - srv.wait_closed() + loop.run_until_complete(srv.wait_closed()) @pytest.fixture diff --git a/tests/utils/test_asyncio.py b/tests/utils/test_asyncio.py index 942bf455..225cb9d6 100644 --- a/tests/utils/test_asyncio.py +++ b/tests/utils/test_asyncio.py @@ -21,7 +21,7 @@ import pytest import sys from unittest.mock import MagicMock -from gns3server.utils.asyncio import wait_run_in_executor, subprocess_check_output, wait_for_process_termination, locked_coroutine +from gns3server.utils.asyncio import wait_run_in_executor, subprocess_check_output, wait_for_process_termination, locking from tests.utils import AsyncioMagicMock @@ -84,7 +84,8 @@ def test_lock_decorator(loop): def __init__(self): self._test_val = 0 - @locked_coroutine + @locking + @asyncio.coroutine def method_to_lock(self): res = self._test_val yield from asyncio.sleep(0.1)