mirror of
https://github.com/GNS3/gns3-server
synced 2024-11-12 19:38:57 +00:00
Auto upload missing images
This commit is contained in:
parent
972cbd0594
commit
08e482004f
@ -38,7 +38,7 @@ from .project_manager import ProjectManager
|
||||
from .nios.nio_udp import NIOUDP
|
||||
from .nios.nio_tap import NIOTAP
|
||||
from .nios.nio_ethernet import NIOEthernet
|
||||
from ..utils.images import md5sum, remove_checksum
|
||||
from ..utils.images import md5sum, remove_checksum, images_directories
|
||||
from .error import NodeError, ImageMissingError
|
||||
|
||||
|
||||
@ -389,24 +389,6 @@ class BaseManager:
|
||||
assert nio is not None
|
||||
return nio
|
||||
|
||||
def images_directories(self):
|
||||
"""
|
||||
Return all directory where we will look for images
|
||||
by priority
|
||||
"""
|
||||
server_config = self.config.get_section_config("Server")
|
||||
|
||||
paths = []
|
||||
img_directory = self.get_images_directory()
|
||||
os.makedirs(img_directory, exist_ok=True)
|
||||
paths.append(img_directory)
|
||||
for directory in server_config.get("additional_images_path", "").split(":"):
|
||||
paths.append(directory)
|
||||
# Compatibility with old topologies we look in parent directory
|
||||
paths.append(os.path.normpath(os.path.join(self.get_images_directory(), '..')))
|
||||
# Return only the existings paths
|
||||
return [force_unix_path(p) for p in paths if os.path.exists(p)]
|
||||
|
||||
def get_abs_image_path(self, path):
|
||||
"""
|
||||
Get the absolute path of an image
|
||||
@ -417,6 +399,7 @@ class BaseManager:
|
||||
|
||||
if not path:
|
||||
return ""
|
||||
orig_path = path
|
||||
|
||||
server_config = self.config.get_section_config("Server")
|
||||
img_directory = self.get_images_directory()
|
||||
@ -427,8 +410,7 @@ class BaseManager:
|
||||
raise NodeError("{} is not allowed on this remote server. Please use only a filename in {}.".format(path, img_directory))
|
||||
|
||||
if not os.path.isabs(path):
|
||||
orig_path = path
|
||||
for directory in self.images_directories():
|
||||
for directory in images_directories(self._NODE_TYPE):
|
||||
path = self._recursive_search_file_in_directory(directory, orig_path)
|
||||
if path:
|
||||
return force_unix_path(path)
|
||||
@ -438,21 +420,21 @@ class BaseManager:
|
||||
path = force_unix_path(os.path.join(self.get_images_directory(), *s))
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
raise ImageMissingError(path)
|
||||
raise ImageMissingError(orig_path)
|
||||
|
||||
# For non local server we disallow using absolute path outside image directory
|
||||
if server_config.get("local", False) is True:
|
||||
path = force_unix_path(path)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
raise ImageMissingError(path)
|
||||
raise ImageMissingError(orig_path)
|
||||
|
||||
path = force_unix_path(path)
|
||||
for directory in self.images_directories():
|
||||
for directory in images_directories(self._NODE_TYPE):
|
||||
if os.path.commonprefix([directory, path]) == directory:
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
raise ImageMissingError(path)
|
||||
raise ImageMissingError(orig_path)
|
||||
raise NodeError("{} is not allowed on this remote server. Please use only a filename in {}.".format(path, self.get_images_directory()))
|
||||
|
||||
def _recursive_search_file_in_directory(self, directory, searched_file):
|
||||
@ -485,7 +467,7 @@ class BaseManager:
|
||||
if not path:
|
||||
return ""
|
||||
path = force_unix_path(self.get_abs_image_path(path))
|
||||
for directory in self.images_directories():
|
||||
for directory in images_directories(self._NODE_TYPE):
|
||||
if os.path.commonprefix([directory, path]) == directory:
|
||||
return os.path.relpath(path, directory)
|
||||
return path
|
||||
|
@ -103,6 +103,7 @@ WIC_MATRIX = {"WIC-1ENET": WIC_1ENET,
|
||||
class Dynamips(BaseManager):
|
||||
|
||||
_NODE_CLASS = DynamipsVMFactory
|
||||
_NODE_TYPE = "dynamips"
|
||||
_DEVICE_CLASS = DynamipsDeviceFactory
|
||||
_ghost_ios_lock = None
|
||||
|
||||
|
@ -39,3 +39,4 @@ class ImageMissingError(Exception):
|
||||
|
||||
def __init__(self, image):
|
||||
super().__init__("The image {} is missing".format(image))
|
||||
self.image = image
|
||||
|
@ -33,6 +33,7 @@ log = logging.getLogger(__name__)
|
||||
class IOU(BaseManager):
|
||||
|
||||
_NODE_CLASS = IOUVM
|
||||
_NODE_TYPE = "iou"
|
||||
|
||||
def __init__(self):
|
||||
|
||||
|
@ -38,6 +38,7 @@ log = logging.getLogger(__name__)
|
||||
class Qemu(BaseManager):
|
||||
|
||||
_NODE_CLASS = QemuVM
|
||||
_NODE_TYPE = "qemu"
|
||||
|
||||
@staticmethod
|
||||
@asyncio.coroutine
|
||||
|
@ -19,6 +19,7 @@ import aiohttp
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
import io
|
||||
|
||||
from ..utils import parse_version
|
||||
from ..controller.controller_error import ControllerError
|
||||
@ -34,6 +35,17 @@ class ComputeError(ControllerError):
|
||||
pass
|
||||
|
||||
|
||||
class ComputeConflict(aiohttp.web.HTTPConflict):
|
||||
"""
|
||||
Raise when the compute send a 409 that we can handle
|
||||
|
||||
:param response: The response of the compute
|
||||
"""
|
||||
def __init__(self, response):
|
||||
super().__init__(text=response["message"])
|
||||
self.response = response
|
||||
|
||||
|
||||
class Timeout(aiohttp.Timeout):
|
||||
"""
|
||||
Could be removed with aiohttp 0.22 that support None timeout
|
||||
@ -293,7 +305,7 @@ class Compute:
|
||||
if hasattr(data, '__json__'):
|
||||
data = json.dumps(data.__json__())
|
||||
# Stream the request
|
||||
elif isinstance(data, aiohttp.streams.StreamReader):
|
||||
elif isinstance(data, aiohttp.streams.StreamReader) or isinstance(data, io.BufferedIOBase):
|
||||
chunked = True
|
||||
headers['content-type'] = 'application/octet-stream'
|
||||
else:
|
||||
@ -306,10 +318,13 @@ class Compute:
|
||||
|
||||
if response.status >= 300:
|
||||
# Try to decode the GNS3 error
|
||||
try:
|
||||
msg = json.loads(body)["message"]
|
||||
except (KeyError, json.decoder.JSONDecodeError):
|
||||
msg = body
|
||||
if body:
|
||||
try:
|
||||
msg = json.loads(body)["message"]
|
||||
except (KeyError, json.decoder.JSONDecodeError):
|
||||
msg = body
|
||||
else:
|
||||
msg = ""
|
||||
|
||||
if response.status == 400:
|
||||
raise aiohttp.web.HTTPBadRequest(text="Bad request {} {}".format(url, body))
|
||||
@ -320,7 +335,11 @@ class Compute:
|
||||
elif response.status == 404:
|
||||
raise aiohttp.web.HTTPNotFound(text=msg)
|
||||
elif response.status == 409:
|
||||
raise aiohttp.web.HTTPConflict(text=msg)
|
||||
try:
|
||||
raise ComputeConflict(json.loads(body))
|
||||
# If the 409 doesn't come from a GNS3 server
|
||||
except json.decoder.JSONDecodeError:
|
||||
raise aiohttp.web.HTTPConflict(text=msg)
|
||||
elif response.status == 500:
|
||||
raise aiohttp.web.HTTPInternalServerError(text="Internal server error {}".format(url))
|
||||
elif response.status == 503:
|
||||
|
@ -15,12 +15,16 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import copy
|
||||
import uuid
|
||||
import os
|
||||
|
||||
|
||||
from .compute import ComputeConflict
|
||||
from ..utils.images import images_directories
|
||||
|
||||
class Node:
|
||||
|
||||
def __init__(self, project, compute, node_id=None, node_type=None, name=None, console=None, console_type=None, properties={}):
|
||||
@ -101,8 +105,19 @@ class Node:
|
||||
"""
|
||||
data = self._node_data()
|
||||
data["node_id"] = self._id
|
||||
response = yield from self._compute.post("/projects/{}/{}/nodes".format(self._project.id, self._node_type), data=data)
|
||||
self.parse_node_response(response.json)
|
||||
trial = 0
|
||||
while trial != 6:
|
||||
try:
|
||||
response = yield from self._compute.post("/projects/{}/{}/nodes".format(self._project.id, self._node_type), data=data)
|
||||
except ComputeConflict as e:
|
||||
if e.response.get("exception") == "ImageMissingError":
|
||||
res = yield from self._upload_missing_image(self._node_type, e.response["image"])
|
||||
if not res:
|
||||
raise e
|
||||
else:
|
||||
self.parse_node_response(response.json)
|
||||
return True
|
||||
trial += 1
|
||||
|
||||
@asyncio.coroutine
|
||||
def update(self, name=None, console=None, console_type=None, properties={}):
|
||||
@ -236,6 +251,22 @@ class Node:
|
||||
else:
|
||||
return (yield from self._compute.delete("/projects/{}/{}/nodes/{}{}".format(self._project.id, self._node_type, self._id, path)))
|
||||
|
||||
@asyncio.coroutine
|
||||
def _upload_missing_image(self, type, img):
|
||||
"""
|
||||
Search an image on local computer and upload it to remote compute
|
||||
if the image exists
|
||||
"""
|
||||
for directory in images_directories(type):
|
||||
image = os.path.join(directory, img)
|
||||
if os.path.exists(image):
|
||||
self.project.controller.notification.emit("log.info", {"message": "Uploading missing image {}".format(img)})
|
||||
with open(image, 'rb') as f:
|
||||
yield from self._compute.post("/{}/images/{}".format(self._node_type, os.path.basename(img)), data=f, timeout=None)
|
||||
self.project.controller.notification.emit("log.info", {"message": "Upload finished for {}".format(img)})
|
||||
return True
|
||||
return False
|
||||
|
||||
@asyncio.coroutine
|
||||
def dynamips_auto_idlepc(self):
|
||||
"""
|
||||
|
@ -18,10 +18,44 @@
|
||||
import os
|
||||
import hashlib
|
||||
|
||||
|
||||
from ..config import Config
|
||||
from . import force_unix_path
|
||||
|
||||
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def images_directories(type):
|
||||
"""
|
||||
Return all directory where we will look for images
|
||||
by priority
|
||||
|
||||
:param type: Type of emulator
|
||||
"""
|
||||
server_config = Config.instance().get_section_config("Server")
|
||||
|
||||
paths = []
|
||||
img_dir = os.path.expanduser(server_config.get("images_path", "~/GNS3/images"))
|
||||
if type == "qemu":
|
||||
type_img_directory = os.path.join(img_dir, "QEMU")
|
||||
elif type == "iou":
|
||||
type_img_directory = os.path.join(img_dir, "IOU")
|
||||
elif type == "dynamips":
|
||||
type_img_directory = os.path.join(img_dir, "IOS")
|
||||
else:
|
||||
raise NotImplementedError("%s is not supported", type)
|
||||
os.makedirs(type_img_directory, exist_ok=True)
|
||||
paths.append(type_img_directory)
|
||||
for directory in server_config.get("additional_images_path", "").split(":"):
|
||||
paths.append(directory)
|
||||
# Compatibility with old topologies we look in parent directory
|
||||
paths.append(img_dir)
|
||||
# Return only the existings paths
|
||||
return [force_unix_path(p) for p in paths if os.path.exists(p)]
|
||||
|
||||
|
||||
def md5sum(path):
|
||||
"""
|
||||
Return the md5sum of an image and cache it on disk
|
||||
|
@ -197,11 +197,16 @@ class Route(object):
|
||||
response = Response(request=request, route=route)
|
||||
response.set_status(409)
|
||||
response.json({"message": str(e), "status": 409})
|
||||
except (NodeError, UbridgeError, ImageMissingError) as e:
|
||||
except (NodeError, UbridgeError) as e:
|
||||
log.error("Node error detected: {type}".format(type=e.__class__.__name__), exc_info=1)
|
||||
response = Response(request=request, route=route)
|
||||
response.set_status(409)
|
||||
response.json({"message": str(e), "status": 409, "exception": e.__class__.__name__})
|
||||
except (ImageMissingError) as e:
|
||||
log.error("Image missing error detected: {}".format(e.image))
|
||||
response = Response(request=request, route=route)
|
||||
response.set_status(409)
|
||||
response.json({"message": str(e), "status": 409, "image": e.image, "exception": e.__class__.__name__})
|
||||
except asyncio.futures.CancelledError as e:
|
||||
log.error("Request canceled")
|
||||
response = Response(request=request, route=route)
|
||||
|
@ -88,28 +88,6 @@ def test_create_node_old_topology(loop, project, tmpdir, vpcs):
|
||||
assert f.read() == "1"
|
||||
|
||||
|
||||
def test_images_directories(qemu, tmpdir):
|
||||
path1 = tmpdir / "images1" / "QEMU" / "test1.bin"
|
||||
path1.write("1", ensure=True)
|
||||
path1 = force_unix_path(str(path1))
|
||||
|
||||
path2 = tmpdir / "images2" / "test2.bin"
|
||||
path2.write("1", ensure=True)
|
||||
path2 = force_unix_path(str(path2))
|
||||
|
||||
with patch("gns3server.config.Config.get_section_config", return_value={
|
||||
"images_path": str(tmpdir / "images1"),
|
||||
"additional_images_path": "/tmp/null24564:{}".format(tmpdir / "images2"),
|
||||
"local": False}):
|
||||
|
||||
# /tmp/null24564 is ignored because doesn't exists
|
||||
res = qemu.images_directories()
|
||||
assert res[0] == str(tmpdir / "images1" / "QEMU")
|
||||
assert res[1] == str(tmpdir / "images2")
|
||||
assert res[2] == str(tmpdir / "images1")
|
||||
assert len(res) == 3
|
||||
|
||||
|
||||
def test_get_abs_image_path(qemu, tmpdir):
|
||||
os.makedirs(str(tmpdir / "QEMU"))
|
||||
path1 = force_unix_path(str(tmpdir / "test1.bin"))
|
||||
|
@ -23,7 +23,7 @@ import asyncio
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from gns3server.controller.project import Project
|
||||
from gns3server.controller.compute import Compute, ComputeError
|
||||
from gns3server.controller.compute import Compute, ComputeError, ComputeConflict
|
||||
from gns3server.version import __version__
|
||||
from tests.utils import asyncio_patch, AsyncioMagicMock
|
||||
|
||||
@ -139,9 +139,19 @@ def test_compute_httpQueryNotConnectedNonGNS3Server2(compute, async_run):
|
||||
def test_compute_httpQueryError(compute, async_run):
|
||||
response = MagicMock()
|
||||
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
|
||||
response.status = 409
|
||||
response.status = 404
|
||||
|
||||
with pytest.raises(aiohttp.web.HTTPConflict):
|
||||
with pytest.raises(aiohttp.web.HTTPNotFound):
|
||||
async_run(compute.post("/projects", {"a": "b"}))
|
||||
|
||||
|
||||
def test_compute_httpQueryConflictError(compute, async_run):
|
||||
response = MagicMock()
|
||||
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
|
||||
response.status = 409
|
||||
response.read = AsyncioMagicMock(return_value=b'{"message": "Test"}')
|
||||
|
||||
with pytest.raises(ComputeConflict):
|
||||
async_run(compute.post("/projects", {"a": "b"}))
|
||||
|
||||
|
||||
|
@ -18,7 +18,8 @@
|
||||
import pytest
|
||||
import uuid
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock
|
||||
import os
|
||||
from unittest.mock import MagicMock, ANY
|
||||
|
||||
|
||||
from tests.utils import AsyncioMagicMock
|
||||
@ -77,7 +78,7 @@ def test_create(node, compute, project, async_run):
|
||||
response.json = {"console": 2048}
|
||||
compute.post = AsyncioMagicMock(return_value=response)
|
||||
|
||||
async_run(node.create())
|
||||
assert async_run(node.create()) is True
|
||||
data = {
|
||||
"console": 2048,
|
||||
"console_type": "vnc",
|
||||
@ -90,6 +91,28 @@ def test_create(node, compute, project, async_run):
|
||||
assert node._properties == {"startup_script": "echo test"}
|
||||
|
||||
|
||||
def test_create_image_missing(node, compute, project, async_run):
|
||||
node._console = 2048
|
||||
|
||||
node.__calls = 0
|
||||
@asyncio.coroutine
|
||||
def resp(*args, **kwargs):
|
||||
node.__calls += 1
|
||||
response = MagicMock()
|
||||
if node.__calls == 1:
|
||||
response.status = 409
|
||||
response.json = {"image": "linux.img", "exception": "ImageMissingError"}
|
||||
else:
|
||||
response.status = 200
|
||||
return response
|
||||
|
||||
compute.post = AsyncioMagicMock(side_effect=resp)
|
||||
node._upload_missing_image = AsyncioMagicMock(return_value=True)
|
||||
|
||||
assert async_run(node.create()) is True
|
||||
node._upload_missing_image.called is True
|
||||
|
||||
|
||||
def test_update(node, compute, project, async_run):
|
||||
response = MagicMock()
|
||||
response.json = {"console": 2048}
|
||||
@ -193,3 +216,16 @@ def test_dynamips_idlepc_proposals(node, async_run, compute):
|
||||
|
||||
async_run(node.dynamips_idlepc_proposals())
|
||||
compute.get.assert_called_with("/projects/{}/dynamips/nodes/{}/idlepc_proposals".format(node.project.id, node.id), timeout=240)
|
||||
|
||||
|
||||
def test_upload_missing_image(compute, controller, async_run, images_dir):
|
||||
project = Project(str(uuid.uuid4()), controller=controller)
|
||||
node = Node(project, compute,
|
||||
name="demo",
|
||||
node_id=str(uuid.uuid4()),
|
||||
node_type="qemu",
|
||||
properties={"hda_disk_image": "linux.img"})
|
||||
open(os.path.join(images_dir, "linux.img"), 'w+').close()
|
||||
assert async_run(node._upload_missing_image("qemu", "linux.img")) is True
|
||||
compute.post.assert_called_with("/qemu/images/linux.img", data=ANY, timeout=None)
|
||||
|
||||
|
@ -70,7 +70,10 @@ class AsyncioMagicMock(unittest.mock.MagicMock):
|
||||
Magic mock returning coroutine
|
||||
"""
|
||||
|
||||
def __init__(self, return_value=None, **kwargs):
|
||||
def __init__(self, return_value=None, return_values=None, **kwargs):
|
||||
"""
|
||||
:return_values: Array of return value at each call will return the next
|
||||
"""
|
||||
if return_value:
|
||||
future = asyncio.Future()
|
||||
future.set_result(return_value)
|
||||
|
@ -16,8 +16,33 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from gns3server.utils.images import md5sum, remove_checksum
|
||||
|
||||
from gns3server.utils import force_unix_path
|
||||
from gns3server.utils.images import md5sum, remove_checksum, images_directories
|
||||
|
||||
|
||||
def test_images_directories(tmpdir):
|
||||
path1 = tmpdir / "images1" / "QEMU" / "test1.bin"
|
||||
path1.write("1", ensure=True)
|
||||
path1 = force_unix_path(str(path1))
|
||||
|
||||
path2 = tmpdir / "images2" / "test2.bin"
|
||||
path2.write("1", ensure=True)
|
||||
path2 = force_unix_path(str(path2))
|
||||
|
||||
with patch("gns3server.config.Config.get_section_config", return_value={
|
||||
"images_path": str(tmpdir / "images1"),
|
||||
"additional_images_path": "/tmp/null24564:{}".format(tmpdir / "images2"),
|
||||
"local": False}):
|
||||
|
||||
# /tmp/null24564 is ignored because doesn't exists
|
||||
res = images_directories("qemu")
|
||||
assert res[0] == str(tmpdir / "images1" / "QEMU")
|
||||
assert res[1] == str(tmpdir / "images2")
|
||||
assert res[2] == str(tmpdir / "images1")
|
||||
assert len(res) == 3
|
||||
|
||||
|
||||
def test_md5sum(tmpdir):
|
||||
|
@ -19,7 +19,9 @@ import os
|
||||
import pytest
|
||||
import aiohttp
|
||||
|
||||
|
||||
from gns3server.utils.path import check_path_allowed, get_default_project_directory
|
||||
from gns3server.utils import force_unix_path
|
||||
|
||||
|
||||
def test_check_path_allowed(config, tmpdir):
|
||||
|
Loading…
Reference in New Issue
Block a user