Merge pull request #2355 from GNS3/refactor/busybox

Install Docker resources in writable location

gns3server/compute/docker/__init__.py

@@ -19,11 +19,15 @@
 Docker server module.
 """
 
+import os
 import sys
 import json
 import asyncio
 import logging
 import aiohttp
+import shutil
+import platformdirs
+
 from gns3server.utils import parse_version
 from gns3server.utils.asyncio import locking
 from gns3server.compute.base_manager import BaseManager
@@ -55,6 +59,62 @@ class Docker(BaseManager):
         self._session = None
         self._api_version = DOCKER_MINIMUM_API_VERSION
 
+    @staticmethod
+    async def install_busybox(dst_dir):
+
+        dst_busybox = os.path.join(dst_dir, "bin", "busybox")
+        if os.path.isfile(dst_busybox):
+            return
+        for busybox_exec in ("busybox-static", "busybox.static", "busybox"):
+            busybox_path = shutil.which(busybox_exec)
+            if busybox_path:
+                try:
+                    # check that busybox is statically linked
+                    # (dynamically linked busybox will fail to run in a container)
+                    proc = await asyncio.create_subprocess_exec(
+                        "ldd",
+                        busybox_path,
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.DEVNULL
+                    )
+                    stdout, _ = await proc.communicate()
+                    if proc.returncode == 1:
+                        # ldd returns 1 if the file is not a dynamic executable
+                        log.info(f"Installing busybox from '{busybox_path}' to '{dst_busybox}'")
+                        shutil.copy2(busybox_path, dst_busybox, follow_symlinks=True)
+                        return
+                    else:
+                        log.warning(f"Busybox '{busybox_path}' is dynamically linked\n"
+                                    f"{stdout.decode('utf-8', errors='ignore').strip()}")
+                except OSError as e:
+                    raise DockerError(f"Could not install busybox: {e}")
+        raise DockerError("No busybox executable could be found")
+
+    @staticmethod
+    def resources_path():
+        """
+        Get the Docker resources storage directory
+        """
+
+        appname = vendor = "GNS3"
+        docker_resources_dir = os.path.join(platformdirs.user_data_dir(appname, vendor, roaming=True), "docker", "resources")
+        os.makedirs(docker_resources_dir, exist_ok=True)
+        return docker_resources_dir
+
+    async def install_resources(self):
+        """
+        Copy the necessary resources to a writable location and install busybox
+        """
+
+        try:
+            dst_path = self.resources_path()
+            log.info(f"Installing Docker resources in '{dst_path}'")
+            from gns3server.controller import Controller
+            Controller.instance().install_resource_files(dst_path, "compute/docker/resources")
+            await self.install_busybox(dst_path)
+        except OSError as e:
+            raise DockerError(f"Could not install Docker resources to {dst_path}: {e}")
+
     async def _check_connection(self):
 
         if not self._connected:

gns3server/compute/docker/docker_vm.py

@@ -242,10 +242,13 @@ class DockerVM(BaseNode):
         :returns: Return the path that we need to map to local folders
         """
 
-        resources = get_resource("compute/docker/resources")
-        if not os.path.exists(resources):
-            raise DockerError("{} is missing can't start Docker containers".format(resources))
-        binds = ["{}:/gns3:ro".format(resources)]
+        try:
+            resources_path = self.manager.resources_path()
+        except OSError as e:
+            raise DockerError(f"Cannot access resources: {e}")
+
+        log.info(f'Mount resources from "{resources_path}"')
+        binds = ["{}:/gns3:ro".format(resources_path)]
 
         # We mount our own etc/network
         try:
@@ -460,6 +463,8 @@ class DockerVM(BaseNode):
         Starts this Docker container.
         """
 
+        await self.manager.install_resources()
+
         try:
             state = await self._get_container_state()
         except DockerHttp404Error:

gns3server/controller/__init__.py

@@ -297,9 +297,12 @@ class Controller:
         else:
             for entry in importlib_resources.files('gns3server').joinpath(resource_name).iterdir():
                 full_path = os.path.join(dst_path, entry.name)
-                if entry.is_file() and not os.path.exists(full_path):
-                    log.debug(f'Installing {resource_name} resource file "{entry.name}" to "{full_path}"')
-                    shutil.copy(str(entry), os.path.join(dst_path, entry.name))
+                if not os.path.exists(full_path):
+                    if entry.is_file():
+                        log.debug(f'Installing {resource_name} resource file "{entry.name}" to "{full_path}"')
+                        shutil.copy(str(entry), os.path.join(dst_path, entry.name))
+                    elif entry.is_dir():
+                        os.makedirs(full_path, exist_ok=True)
 
     def _install_base_configs(self):
         """

setup.py

@@ -43,28 +43,6 @@ class PyTest(TestCommand):
         sys.exit(errcode)
 
 
-BUSYBOX_PATH = "gns3server/compute/docker/resources/bin/busybox"
-
-
-def copy_busybox():
-    if not sys.platform.startswith("linux"):
-        return
-    if os.path.isfile(BUSYBOX_PATH):
-        return
-    for bb_cmd in ("busybox-static", "busybox.static", "busybox"):
-        bb_path = shutil.which(bb_cmd)
-        if bb_path:
-            if subprocess.call(["ldd", bb_path],
-                               stdin=subprocess.DEVNULL,
-                               stdout=subprocess.DEVNULL,
-                               stderr=subprocess.DEVNULL):
-                shutil.copy2(bb_path, BUSYBOX_PATH, follow_symlinks=True)
-                break
-    else:
-        raise SystemExit("No static busybox found")
-
-
-copy_busybox()
 dependencies = open("requirements.txt", "r").read().splitlines()
 
 setup(

tests/compute/docker/test_docker.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import asyncio
 import pytest
 from unittest.mock import MagicMock, patch
 
@@ -200,3 +201,52 @@ async def test_docker_check_connection_docker_preferred_version_against_older(vm
         vm._connected = False
         await vm._check_connection()
         assert vm._api_version == DOCKER_MINIMUM_API_VERSION
+
+
+@pytest.mark.asyncio
+async def test_install_busybox():
+
+    mock_process = MagicMock()
+    mock_process.returncode = 1 # means that busybox is not dynamically linked
+    mock_process.communicate = AsyncioMagicMock(return_value=(b"", b"not a dynamic executable"))
+
+    with patch("gns3server.compute.docker.os.path.isfile", return_value=False):
+        with patch("gns3server.compute.docker.shutil.which", return_value="/usr/bin/busybox"):
+            with asyncio_patch("gns3server.compute.docker.asyncio.create_subprocess_exec", return_value=mock_process) as create_subprocess_mock:
+                with patch("gns3server.compute.docker.shutil.copy2") as copy2_mock:
+                    dst_dir = Docker.resources_path()
+                    await Docker.install_busybox(dst_dir)
+                    create_subprocess_mock.assert_called_with(
+                        "ldd",
+                        "/usr/bin/busybox",
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.DEVNULL,
+                    )
+                    assert copy2_mock.called
+
+
+@pytest.mark.asyncio
+async def test_install_busybox_dynamic_linked():
+
+    mock_process = MagicMock()
+    mock_process.returncode = 0  # means that busybox is dynamically linked
+    mock_process.communicate = AsyncioMagicMock(return_value=(b"Dynamically linked library", b""))
+
+    with patch("os.path.isfile", return_value=False):
+        with patch("gns3server.compute.docker.shutil.which", return_value="/usr/bin/busybox"):
+            with asyncio_patch("gns3server.compute.docker.asyncio.create_subprocess_exec", return_value=mock_process):
+                with pytest.raises(DockerError) as e:
+                    dst_dir = Docker.resources_path()
+                    await Docker.install_busybox(dst_dir)
+                assert str(e.value) == "No busybox executable could be found"
+
+
+@pytest.mark.asyncio
+async def test_install_busybox_no_executables():
+
+    with patch("gns3server.compute.docker.os.path.isfile", return_value=False):
+        with patch("gns3server.compute.docker.shutil.which", return_value=None):
+            with pytest.raises(DockerError) as e:
+                dst_dir = Docker.resources_path()
+                await Docker.install_busybox(dst_dir)
+            assert str(e.value) == "No busybox executable could be found"

tests/compute/docker/test_docker_vm.py

@@ -25,10 +25,8 @@ from tests.utils import asyncio_patch, AsyncioMagicMock
 
 from gns3server.ubridge.ubridge_error import UbridgeNamespaceError
 from gns3server.compute.docker.docker_vm import DockerVM
-from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error, DockerHttp304Error
+from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error
 from gns3server.compute.docker import Docker
-from gns3server.utils.get_resource import get_resource
-
 
 from unittest.mock import patch, MagicMock, call
 
@@ -101,7 +99,7 @@ async def test_create(compute_project, manager):
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
                         "Privileged": True
@@ -139,7 +137,7 @@ async def test_create_with_tag(compute_project, manager):
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
                         "Privileged": True
@@ -180,7 +178,7 @@ async def test_create_vnc(compute_project, manager):
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "/tmp/.X11-unix/X{0}:/tmp/.X11-unix/X{0}:ro".format(vm._display)
                         ],
@@ -310,7 +308,7 @@ async def test_create_start_cmd(compute_project, manager):
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
                         "Privileged": True
@@ -408,7 +406,7 @@ async def test_create_image_not_available(compute_project, manager):
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
                         "Privileged": True
@@ -451,7 +449,7 @@ async def test_create_with_user(compute_project, manager):
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
                         "Privileged": True
@@ -533,7 +531,7 @@ async def test_create_with_extra_volumes_duplicate_1_image(compute_project, mana
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
                         ],
@@ -572,7 +570,7 @@ async def test_create_with_extra_volumes_duplicate_2_user(compute_project, manag
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
                         ],
@@ -611,7 +609,7 @@ async def test_create_with_extra_volumes_duplicate_3_subdir(compute_project, man
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
                         ],
@@ -650,7 +648,7 @@ async def test_create_with_extra_volumes_duplicate_4_backslash(compute_project,
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
                         ],
@@ -689,7 +687,7 @@ async def test_create_with_extra_volumes_duplicate_5_subdir_issue_1595(compute_p
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
                         ],
                         "Privileged": True
@@ -727,7 +725,7 @@ async def test_create_with_extra_volumes_duplicate_6_subdir_issue_1595(compute_p
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
                         ],
                         "Privileged": True
@@ -771,7 +769,7 @@ async def test_create_with_extra_volumes(compute_project, manager):
                     {
                         "CapAdd": ["ALL"],
                         "Binds": [
-                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3:ro".format(Docker.resources_path()),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
                             "{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")),
@@ -996,7 +994,7 @@ async def test_update(vm):
         {
             "CapAdd": ["ALL"],
             "Binds": [
-                "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                "{}:/gns3:ro".format(Docker.resources_path()),
                 "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
             ],
             "Privileged": True
@@ -1064,7 +1062,7 @@ async def test_update_running(vm):
         {
             "CapAdd": ["ALL"],
             "Binds": [
-                "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                "{}:/gns3:ro".format(Docker.resources_path()),
                 "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
             ],
             "Privileged": True
@@ -1325,7 +1323,7 @@ async def test_mount_binds(vm):
 
     dst = os.path.join(vm.working_dir, "test/experimental")
     assert vm._mount_binds(image_infos) == [
-        "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+        "{}:/gns3:ro".format(Docker.resources_path()),
         "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
         "{}:/gns3volumes{}".format(dst, "/test/experimental")
     ]