From 1a53c9aabf5ae60358496325f06550388ac12c65 Mon Sep 17 00:00:00 2001 From: grossmj Date: Wed, 14 Feb 2024 16:13:45 +0800 Subject: [PATCH] Backport from v3: install Docker resources in a writable location at runtime. --- gns3server/compute/docker/__init__.py | 60 ++++++++++++++++++++++++++ gns3server/compute/docker/docker_vm.py | 13 ++++-- gns3server/controller/__init__.py | 9 ++-- setup.py | 22 ---------- tests/compute/docker/test_docker.py | 50 +++++++++++++++++++++ tests/compute/docker/test_docker_vm.py | 36 ++++++++-------- 6 files changed, 142 insertions(+), 48 deletions(-) diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index 7e0cd195..7a31fb3b 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -19,11 +19,15 @@ Docker server module. """ +import os import sys import json import asyncio import logging import aiohttp +import shutil +import platformdirs + from gns3server.utils import parse_version from gns3server.utils.asyncio import locking from gns3server.compute.base_manager import BaseManager @@ -55,6 +59,62 @@ class Docker(BaseManager): self._session = None self._api_version = DOCKER_MINIMUM_API_VERSION + @staticmethod + async def install_busybox(dst_dir): + + dst_busybox = os.path.join(dst_dir, "bin", "busybox") + if os.path.isfile(dst_busybox): + return + for busybox_exec in ("busybox-static", "busybox.static", "busybox"): + busybox_path = shutil.which(busybox_exec) + if busybox_path: + try: + # check that busybox is statically linked + # (dynamically linked busybox will fail to run in a container) + proc = await asyncio.create_subprocess_exec( + "ldd", + busybox_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL + ) + stdout, _ = await proc.communicate() + if proc.returncode == 1: + # ldd returns 1 if the file is not a dynamic executable + log.info(f"Installing busybox from '{busybox_path}' to '{dst_busybox}'") + shutil.copy2(busybox_path, dst_busybox, follow_symlinks=True) + return + else: + log.warning(f"Busybox '{busybox_path}' is dynamically linked\n" + f"{stdout.decode('utf-8', errors='ignore').strip()}") + except OSError as e: + raise DockerError(f"Could not install busybox: {e}") + raise DockerError("No busybox executable could be found") + + @staticmethod + def resources_path(): + """ + Get the Docker resources storage directory + """ + + appname = vendor = "GNS3" + docker_resources_dir = os.path.join(platformdirs.user_data_dir(appname, vendor, roaming=True), "docker", "resources") + os.makedirs(docker_resources_dir, exist_ok=True) + return docker_resources_dir + + async def install_resources(self): + """ + Copy the necessary resources to a writable location and install busybox + """ + + try: + dst_path = self.resources_path() + log.info(f"Installing Docker resources in '{dst_path}'") + from gns3server.controller import Controller + Controller.instance().install_resource_files(dst_path, "compute/docker/resources") + await self.install_busybox(dst_path) + except OSError as e: + raise DockerError(f"Could not install Docker resources to {dst_path}: {e}") + async def _check_connection(self): if not self._connected: diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 500e526d..d1ad1456 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -242,10 +242,13 @@ class DockerVM(BaseNode): :returns: Return the path that we need to map to local folders """ - resources = get_resource("compute/docker/resources") - if not os.path.exists(resources): - raise DockerError("{} is missing can't start Docker containers".format(resources)) - binds = ["{}:/gns3:ro".format(resources)] + try: + resources_path = self.manager.resources_path() + except OSError as e: + raise DockerError(f"Cannot access resources: {e}") + + log.info(f'Mount resources from "{resources_path}"') + binds = ["{}:/gns3:ro".format(resources_path)] # We mount our own etc/network try: @@ -460,6 +463,8 @@ class DockerVM(BaseNode): Starts this Docker container. """ + await self.manager.install_resources() + try: state = await self._get_container_state() except DockerHttp404Error: diff --git a/gns3server/controller/__init__.py b/gns3server/controller/__init__.py index 4a66fe2d..34ddf273 100644 --- a/gns3server/controller/__init__.py +++ b/gns3server/controller/__init__.py @@ -297,9 +297,12 @@ class Controller: else: for entry in importlib_resources.files('gns3server').joinpath(resource_name).iterdir(): full_path = os.path.join(dst_path, entry.name) - if entry.is_file() and not os.path.exists(full_path): - log.debug(f'Installing {resource_name} resource file "{entry.name}" to "{full_path}"') - shutil.copy(str(entry), os.path.join(dst_path, entry.name)) + if not os.path.exists(full_path): + if entry.is_file(): + log.debug(f'Installing {resource_name} resource file "{entry.name}" to "{full_path}"') + shutil.copy(str(entry), os.path.join(dst_path, entry.name)) + elif entry.is_dir(): + os.makedirs(full_path, exist_ok=True) def _install_base_configs(self): """ diff --git a/setup.py b/setup.py index 4fdee869..167d64d0 100644 --- a/setup.py +++ b/setup.py @@ -43,28 +43,6 @@ class PyTest(TestCommand): sys.exit(errcode) -BUSYBOX_PATH = "gns3server/compute/docker/resources/bin/busybox" - - -def copy_busybox(): - if not sys.platform.startswith("linux"): - return - if os.path.isfile(BUSYBOX_PATH): - return - for bb_cmd in ("busybox-static", "busybox.static", "busybox"): - bb_path = shutil.which(bb_cmd) - if bb_path: - if subprocess.call(["ldd", bb_path], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL): - shutil.copy2(bb_path, BUSYBOX_PATH, follow_symlinks=True) - break - else: - raise SystemExit("No static busybox found") - - -copy_busybox() dependencies = open("requirements.txt", "r").read().splitlines() setup( diff --git a/tests/compute/docker/test_docker.py b/tests/compute/docker/test_docker.py index 2e9193ad..76a30ce5 100644 --- a/tests/compute/docker/test_docker.py +++ b/tests/compute/docker/test_docker.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import asyncio import pytest from unittest.mock import MagicMock, patch @@ -200,3 +201,52 @@ async def test_docker_check_connection_docker_preferred_version_against_older(vm vm._connected = False await vm._check_connection() assert vm._api_version == DOCKER_MINIMUM_API_VERSION + + +@pytest.mark.asyncio +async def test_install_busybox(): + + mock_process = MagicMock() + mock_process.returncode = 1 # means that busybox is not dynamically linked + mock_process.communicate = AsyncioMagicMock(return_value=(b"", b"not a dynamic executable")) + + with patch("gns3server.compute.docker.os.path.isfile", return_value=False): + with patch("gns3server.compute.docker.shutil.which", return_value="/usr/bin/busybox"): + with asyncio_patch("gns3server.compute.docker.asyncio.create_subprocess_exec", return_value=mock_process) as create_subprocess_mock: + with patch("gns3server.compute.docker.shutil.copy2") as copy2_mock: + dst_dir = Docker.resources_path() + await Docker.install_busybox(dst_dir) + create_subprocess_mock.assert_called_with( + "ldd", + "/usr/bin/busybox", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + assert copy2_mock.called + + +@pytest.mark.asyncio +async def test_install_busybox_dynamic_linked(): + + mock_process = MagicMock() + mock_process.returncode = 0 # means that busybox is dynamically linked + mock_process.communicate = AsyncioMagicMock(return_value=(b"Dynamically linked library", b"")) + + with patch("os.path.isfile", return_value=False): + with patch("gns3server.compute.docker.shutil.which", return_value="/usr/bin/busybox"): + with asyncio_patch("gns3server.compute.docker.asyncio.create_subprocess_exec", return_value=mock_process): + with pytest.raises(DockerError) as e: + dst_dir = Docker.resources_path() + await Docker.install_busybox(dst_dir) + assert str(e.value) == "No busybox executable could be found" + + +@pytest.mark.asyncio +async def test_install_busybox_no_executables(): + + with patch("gns3server.compute.docker.os.path.isfile", return_value=False): + with patch("gns3server.compute.docker.shutil.which", return_value=None): + with pytest.raises(DockerError) as e: + dst_dir = Docker.resources_path() + await Docker.install_busybox(dst_dir) + assert str(e.value) == "No busybox executable could be found" diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 3a395a56..097130bf 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -25,10 +25,8 @@ from tests.utils import asyncio_patch, AsyncioMagicMock from gns3server.ubridge.ubridge_error import UbridgeNamespaceError from gns3server.compute.docker.docker_vm import DockerVM -from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error, DockerHttp304Error +from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error from gns3server.compute.docker import Docker -from gns3server.utils.get_resource import get_resource - from unittest.mock import patch, MagicMock, call @@ -101,7 +99,7 @@ async def test_create(compute_project, manager): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True @@ -139,7 +137,7 @@ async def test_create_with_tag(compute_project, manager): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True @@ -180,7 +178,7 @@ async def test_create_vnc(compute_project, manager): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "/tmp/.X11-unix/X{0}:/tmp/.X11-unix/X{0}:ro".format(vm._display) ], @@ -310,7 +308,7 @@ async def test_create_start_cmd(compute_project, manager): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True @@ -408,7 +406,7 @@ async def test_create_image_not_available(compute_project, manager): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True @@ -451,7 +449,7 @@ async def test_create_with_user(compute_project, manager): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True @@ -533,7 +531,7 @@ async def test_create_with_extra_volumes_duplicate_1_image(compute_project, mana { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")), ], @@ -572,7 +570,7 @@ async def test_create_with_extra_volumes_duplicate_2_user(compute_project, manag { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")), ], @@ -611,7 +609,7 @@ async def test_create_with_extra_volumes_duplicate_3_subdir(compute_project, man { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")), ], @@ -650,7 +648,7 @@ async def test_create_with_extra_volumes_duplicate_4_backslash(compute_project, { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")), ], @@ -689,7 +687,7 @@ async def test_create_with_extra_volumes_duplicate_5_subdir_issue_1595(compute_p { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")), ], "Privileged": True @@ -727,7 +725,7 @@ async def test_create_with_extra_volumes_duplicate_6_subdir_issue_1595(compute_p { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")), ], "Privileged": True @@ -771,7 +769,7 @@ async def test_create_with_extra_volumes(compute_project, manager): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")), "{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")), @@ -996,7 +994,7 @@ async def test_update(vm): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True @@ -1064,7 +1062,7 @@ async def test_update_running(vm): { "CapAdd": ["ALL"], "Binds": [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True @@ -1325,7 +1323,7 @@ async def test_mount_binds(vm): dst = os.path.join(vm.working_dir, "test/experimental") assert vm._mount_binds(image_infos) == [ - "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3:ro".format(Docker.resources_path()), "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes{}".format(dst, "/test/experimental") ]