Merge pull request #2270 from GNS3/packaging-migration

Packaging migration

.github/workflows/testing.yml

@@ -30,10 +30,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -r dev-requirements.txt
-    - name: Install Windows dependencies
-      if: matrix.os == 'windows-latest'
-      run: python -m pip install -r win-requirements.txt
+        python -m pip install .[dev]
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names

AUTHORS

@@ -1,2 +0,0 @@
-Jeremy Grossmann
-Julien Duponchelle

MANIFEST.in

@@ -1,11 +1,7 @@
 include README.md
-include AUTHORS
 include LICENSE
-include MANIFEST.in
-include requirements.txt
-include conf/*.conf
-recursive-include tests *
-recursive-exclude docs *
+include CHANGELOG
 recursive-include gns3server *
+recursive-exclude docs *
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]

README.md

@@ -85,7 +85,7 @@ cd gns3-server
 git checkout 3.0
 python3 -m venv venv-gns3server
 source venv-gns3server/bin/activate
-python3 setup.py install
+python3 -m pip install .
 python3 -m gns3server
 ```
 

dev-requirements.txt

@@ -1,8 +1,6 @@
--r requirements.txt
-
 pytest==7.4.0
 flake8==5.0.4  # v5.0.4 is the last to support Python 3.7
 pytest-timeout==2.1.0
 pytest-asyncio==0.21.1
 requests==2.31.0
-httpx==0.24.1
+httpx==0.24.1

gns3server/compute/docker/__init__.py

@@ -18,11 +18,15 @@
 Docker server module.
 """
 
+import os
 import sys
 import json
 import asyncio
 import logging
 import aiohttp
+import shutil
+import subprocess
+
 from gns3server.utils import parse_version
 from gns3server.utils.asyncio import locking
 from gns3server.compute.base_manager import BaseManager
@@ -54,6 +58,39 @@ class Docker(BaseManager):
         self._session = None
         self._api_version = DOCKER_MINIMUM_API_VERSION
 
+    @staticmethod
+    async def install_busybox():
+
+        if not sys.platform.startswith("linux"):
+            return
+        dst_busybox = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "bin", "busybox")
+        if os.path.isfile(dst_busybox):
+            return
+        for busybox_exec in ("busybox-static", "busybox.static", "busybox"):
+            busybox_path = shutil.which(busybox_exec)
+            if busybox_path:
+                try:
+                    # check that busybox is statically linked
+                    # (dynamically linked busybox will fail to run in a container)
+                    proc = await asyncio.create_subprocess_exec(
+                        "ldd",
+                        busybox_path,
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.DEVNULL
+                    )
+                    stdout, _ = await proc.communicate()
+                    if proc.returncode == 1:
+                        # ldd returns 1 if the file is not a dynamic executable
+                        log.info(f"Installing busybox from '{busybox_path}' to '{dst_busybox}'")
+                        shutil.copy2(busybox_path, dst_busybox, follow_symlinks=True)
+                        return
+                    else:
+                        log.warning(f"Busybox '{busybox_path}' is dynamically linked\n"
+                                    f"{stdout.decode('utf-8', errors='ignore').strip()}")
+                except OSError as e:
+                    raise DockerError(f"Could not install busybox: {e}")
+        raise DockerError("No busybox executable could be found")
+
     async def _check_connection(self):
 
         if not self._connected:

gns3server/compute/docker/docker_vm.py

@@ -523,7 +523,7 @@ class DockerVM(BaseNode):
 
     async def update(self):
         """
-        Destroy an recreate the container with the new settings
+        Destroy and recreate the container with the new settings
         """
 
         # We need to save the console and state and restore it
@@ -544,6 +544,9 @@ class DockerVM(BaseNode):
         Starts this Docker container.
         """
 
+        # make sure busybox is installed
+        await self.manager.install_busybox()
+
         try:
             state = await self._get_container_state()
         except DockerHttp404Error:

gns3server/controller/appliance_manager.py

@@ -22,11 +22,6 @@ import asyncio
 import aiofiles
 import shutil
 
-try:
-    import importlib_resources
-except ImportError:
-    from importlib import resources as importlib_resources
-
 
 from typing import Tuple, List
 from aiohttp.client_exceptions import ClientError

gns3server/utils/asyncio/__init__.py

@@ -77,7 +77,7 @@ async def subprocess_check_output(*args, cwd=None, env=None, stderr=False):
     if output is None:
         return ""
     # If we received garbage we ignore invalid characters
-    # it should happens only when user try to use another binary
+    # it should happen only when user try to use another binary
     # and the code of VPCS, dynamips... Will detect it's not the correct binary
     return output.decode("utf-8", errors="ignore")
 

pyproject.toml

@@ -7,7 +7,7 @@ name = "gns3-server"
 description = "GNS3 graphical interface for the GNS3 server."
 license = {file = "LICENSE"}
 authors = [
-  { name="Jeremy Grossmann" }
+  { name = "Jeremy Grossmann", email = "developers@gns3.com" }
 ]
 readme = "README.md"
 requires-python = ">=3.7"
@@ -29,7 +29,7 @@ classifiers = [
         "Programming Language :: Python :: Implementation :: CPython"
 ]
 
-dynamic = ["version", "dependencies"]
+dynamic = ["version", "dependencies", "optional-dependencies"]
 
 [tool.setuptools]
 packages = ["gns3server"]
@@ -38,15 +38,8 @@ packages = ["gns3server"]
 version = {attr = "gns3server.version.__version__"}
 dependencies = {file = "requirements.txt"}
 
-[project.optional-dependencies]
-test = [
-        "pytest==7.2.2",
-        "flake8==5.0.4",  # v5.0.4 is the last to support Python 3.7
-        "pytest-timeout==2.1.0",
-        "pytest-asyncio==0.20.3",
-        "requests==2.28.2",
-        "httpx==0.23.3"
-]
+[tool.setuptools.dynamic.optional-dependencies]
+dev = {file = ['dev-requirements.txt']}
 
 [project.urls]
 "Homepage" = "http://gns3.com"

requirements.txt

@@ -19,4 +19,3 @@ email-validator==2.0.0.post2
 watchfiles==0.19.0
 zstandard==0.21.0
 importlib_resources>=1.3
-setuptools>=60.8.1

setup.py

@@ -1,47 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013 GNS3 Technologies Inc.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import os
-import shutil
-import subprocess
-
-from setuptools import setup
-
-BUSYBOX_PATH = "gns3server/compute/docker/resources/bin/busybox"
-
-
-def copy_busybox():
-    if not sys.platform.startswith("linux"):
-        return
-    if os.path.isfile(BUSYBOX_PATH):
-        return
-    for bb_cmd in ("busybox-static", "busybox.static", "busybox"):
-        bb_path = shutil.which(bb_cmd)
-        if bb_path:
-            if subprocess.call(["ldd", bb_path],
-                               stdin=subprocess.DEVNULL,
-                               stdout=subprocess.DEVNULL,
-                               stderr=subprocess.DEVNULL):
-                shutil.copy2(bb_path, BUSYBOX_PATH, follow_symlinks=True)
-                break
-    else:
-        raise SystemExit("No static busybox found")
-
-
-copy_busybox()  # TODO: this should probably be done when the first time the server is started
-setup()  # required with setuptools below version 64.0.0

tests/compute/docker/test_docker.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import asyncio
 import pytest
 import pytest_asyncio
 from unittest.mock import MagicMock, patch
@@ -209,3 +210,49 @@ async def test_docker_check_connection_docker_preferred_version_against_older(vm
         vm._connected = False
         await vm._check_connection()
         assert vm._api_version == DOCKER_MINIMUM_API_VERSION
+
+
+@pytest.mark.asyncio
+async def test_install_busybox():
+
+    mock_process = MagicMock()
+    mock_process.returncode = 1 # means that busybox is not dynamically linked
+    mock_process.communicate = AsyncioMagicMock(return_value=(b"", b"not a dynamic executable"))
+
+    with patch("os.path.isfile", return_value=False):
+        with patch("shutil.which", return_value="/usr/bin/busybox"):
+            with asyncio_patch("asyncio.create_subprocess_exec", return_value=mock_process) as create_subprocess_mock:
+                with patch("shutil.copy2") as copy2_mock:
+                    await Docker.install_busybox()
+                    create_subprocess_mock.assert_called_with(
+                        "ldd",
+                        "/usr/bin/busybox",
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.DEVNULL,
+                    )
+                    assert copy2_mock.called
+
+
+@pytest.mark.asyncio
+async def test_install_busybox_dynamic_linked():
+
+    mock_process = MagicMock()
+    mock_process.returncode = 0  # means that busybox is dynamically linked
+    mock_process.communicate = AsyncioMagicMock(return_value=(b"Dynamically linked library", b""))
+
+    with patch("os.path.isfile", return_value=False):
+        with patch("shutil.which", return_value="/usr/bin/busybox"):
+            with asyncio_patch("asyncio.create_subprocess_exec", return_value=mock_process):
+                with pytest.raises(DockerError) as e:
+                    await Docker.install_busybox()
+                assert str(e.value) == "No busybox executable could be found"
+
+
+@pytest.mark.asyncio
+async def test_install_busybox_no_executables():
+
+    with patch("os.path.isfile", return_value=False):
+        with patch("shutil.which", return_value=None):
+            with pytest.raises(DockerError) as e:
+                await Docker.install_busybox()
+            assert str(e.value) == "No busybox executable could be found"

tests/conftest.py

@@ -8,6 +8,7 @@ import os
 import uuid
 import configparser
 import base64
+import stat
 
 from fastapi import FastAPI
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
@@ -405,13 +406,24 @@ def run_around_tests(monkeypatch, config, port_manager):
 
     monkeypatch.setattr("gns3server.utils.path.get_default_project_directory", lambda *args: os.path.join(tmppath, 'projects'))
 
-    # Force sys.platform to the original value. Because it seem not be restore correctly at each tests
+    # Force sys.platform to the original value. Because it seems not be restored correctly after each test
     sys.platform = sys.original_platform
 
     yield
 
-    # An helper should not raise Exception
+    # A helper should not raise Exception
     try:
         shutil.rmtree(tmppath)
     except BaseException:
         pass
+
+
+@pytest.fixture
+def fake_executable(monkeypatch, tmpdir) -> str:
+
+    monkeypatch.setenv("PATH", str(tmpdir))
+    executable_path = os.path.join(os.environ["PATH"], "fake_executable")
+    with open(executable_path, "w+") as f:
+        f.write("1")
+    os.chmod(executable_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+    return executable_path