From 4518404706edce51f863bb87313a7a54a1c42520 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Mon, 26 Jan 2015 12:10:30 +0100 Subject: [PATCH] Run rmtree in a different thread --- gns3server/handlers/project_handler.py | 6 ++-- gns3server/modules/project.py | 16 +++++++--- gns3server/utils/asyncio.py | 36 ++++++++++++++++++++++ tests/modules/test_project.py | 17 ++++++----- tests/utils/test_asyncio.py | 42 ++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 gns3server/utils/asyncio.py create mode 100644 tests/utils/test_asyncio.py diff --git a/gns3server/handlers/project_handler.py b/gns3server/handlers/project_handler.py index 1ce30c18..e72396f3 100644 --- a/gns3server/handlers/project_handler.py +++ b/gns3server/handlers/project_handler.py @@ -92,7 +92,7 @@ class ProjectHandler: pm = ProjectManager.instance() project = pm.get_project(request.match_info["uuid"]) - project.commit() + yield from project.commit() response.set_status(204) @classmethod @@ -110,7 +110,7 @@ class ProjectHandler: pm = ProjectManager.instance() project = pm.get_project(request.match_info["uuid"]) - project.close() + yield from project.close() response.set_status(204) @classmethod @@ -128,5 +128,5 @@ class ProjectHandler: pm = ProjectManager.instance() project = pm.get_project(request.match_info["uuid"]) - project.delete() + yield from project.delete() response.set_status(204) diff --git a/gns3server/modules/project.py b/gns3server/modules/project.py index ca29aa93..4d174c45 100644 --- a/gns3server/modules/project.py +++ b/gns3server/modules/project.py @@ -19,9 +19,11 @@ import aiohttp import os import tempfile import shutil +import asyncio from uuid import UUID, uuid4 -from ..config import Config +from ..config import Config +from ..utils.asyncio import wait_run_in_executor import logging log = logging.getLogger(__name__) @@ -176,11 +178,13 @@ class Project: if vm in self._vms: self._vms.remove(vm) + @asyncio.coroutine def close(self): """Close the project, but keep informations on disk""" - self._close_and_clean(self._temporary) + yield from self._close_and_clean(self._temporary) + @asyncio.coroutine def _close_and_clean(self, cleanup): """ Close the project, and cleanup the disk if cleanup is True @@ -191,8 +195,9 @@ class Project: for vm in self._vms: vm.close() if cleanup and os.path.exists(self.path): - shutil.rmtree(self.path) + yield from wait_run_in_executor(shutil.rmtree, self.path) + @asyncio.coroutine def commit(self): """Write project changes on disk""" @@ -200,10 +205,11 @@ class Project: vm = self._vms_to_destroy.pop() directory = self.vm_working_directory(vm) if os.path.exists(directory): - shutil.rmtree(directory) + yield from wait_run_in_executor(shutil.rmtree, directory) self.remove_vm(vm) + @asyncio.coroutine def delete(self): """Remove project from disk""" - self._close_and_clean(True) + yield from self._close_and_clean(True) diff --git a/gns3server/utils/asyncio.py b/gns3server/utils/asyncio.py new file mode 100644 index 00000000..9b94eaf0 --- /dev/null +++ b/gns3server/utils/asyncio.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import asyncio + + +@asyncio.coroutine +def wait_run_in_executor(func, *args): + """ + Run blocking code in a different thread and wait + the result. + + :param func: Run this function in a different thread + :param args: Parameters of the function + :returns: Return the result of the function + """ + + loop = asyncio.get_event_loop() + future = loop.run_in_executor(None, func, *args) + yield from asyncio.wait([future]) + return future.result() diff --git a/tests/modules/test_project.py b/tests/modules/test_project.py index 909a09b6..c267a690 100644 --- a/tests/modules/test_project.py +++ b/tests/modules/test_project.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import os +import asyncio import pytest import aiohttp from unittest.mock import patch @@ -84,7 +85,7 @@ def test_mark_vm_for_destruction(vm): assert len(project.vms) == 0 -def test_commit(manager): +def test_commit(manager, loop): project = Project() vm = VPCSVM("test", "00010203-0405-0607-0809-0a0b0c0d0e0f", project, manager) project.add_vm(vm) @@ -92,17 +93,17 @@ def test_commit(manager): project.mark_vm_for_destruction(vm) assert len(project._vms_to_destroy) == 1 assert os.path.exists(directory) - project.commit() + loop.run_until_complete(asyncio.async(project.commit())) assert len(project._vms_to_destroy) == 0 assert os.path.exists(directory) is False assert len(project.vms) == 0 -def test_project_delete(): +def test_project_delete(loop): project = Project() directory = project.path assert os.path.exists(directory) - project.delete() + loop.run_until_complete(asyncio.async(project.delete())) assert os.path.exists(directory) is False @@ -113,22 +114,22 @@ def test_project_add_vm(manager): assert len(project.vms) == 1 -def test_project_close(manager): +def test_project_close(loop, manager): project = Project() vm = VPCSVM("test", "00010203-0405-0607-0809-0a0b0c0d0e0f", project, manager) project.add_vm(vm) with patch("gns3server.modules.vpcs.vpcs_vm.VPCSVM.close") as mock: - project.close() + loop.run_until_complete(asyncio.async(project.close())) assert mock.called -def test_project_close_temporary_project(manager): +def test_project_close_temporary_project(loop, manager): """A temporary project is deleted when closed""" project = Project(temporary=True) directory = project.path assert os.path.exists(directory) - project.close() + loop.run_until_complete(asyncio.async(project.close())) assert os.path.exists(directory) is False diff --git a/tests/utils/test_asyncio.py b/tests/utils/test_asyncio.py new file mode 100644 index 00000000..a6a9cc3e --- /dev/null +++ b/tests/utils/test_asyncio.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import asyncio +import pytest + +from gns3server.utils.asyncio import wait_run_in_executor + + +def test_wait_run_in_executor(loop): + + def change_var(param): + return param + + exec = wait_run_in_executor(change_var, "test") + result = loop.run_until_complete(asyncio.async(exec)) + assert result == "test" + + +def test_exception_wait_run_in_executor(loop): + + def raise_exception(): + raise Exception("test") + + exec = wait_run_in_executor(raise_exception) + with pytest.raises(Exception): + result = loop.run_until_complete(asyncio.async(exec))