From 2535e5508d17bae1357018a254656ed1b2f1a785 Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 19 Apr 2022 18:21:39 +0700 Subject: [PATCH 1/3] Remove Qemu binary requirement --- gns3server/api/routes/compute/compute.py | 14 ------ gns3server/api/routes/controller/computes.py | 4 +- gns3server/compute/qemu/__init__.py | 24 ---------- .../controller/appliance_to_template.py | 6 +-- tests/api/routes/compute/test_qemu_nodes.py | 28 ----------- tests/compute/qemu/test_qemu_manager.py | 46 +++++++++---------- 6 files changed, 27 insertions(+), 95 deletions(-) diff --git a/gns3server/api/routes/compute/compute.py b/gns3server/api/routes/compute/compute.py index 5ff3f424..db257965 100644 --- a/gns3server/api/routes/compute/compute.py +++ b/gns3server/api/routes/compute/compute.py @@ -121,20 +121,6 @@ def compute_statistics() -> dict: } -@router.get("/qemu/binaries") -async def get_qemu_binaries( - archs: Optional[List[str]] = Body(None, embed=True) -) -> List[str]: - - return await Qemu.binary_list(archs) - - -@router.get("/qemu/img-binaries") -async def get_image_binaries() -> List[str]: - - return await Qemu.img_binary_list() - - @router.get("/qemu/capabilities") async def get_qemu_capabilities() -> dict: capabilities = {"kvm": []} diff --git a/gns3server/api/routes/controller/computes.py b/gns3server/api/routes/controller/computes.py index aa1f5e58..67652443 100644 --- a/gns3server/api/routes/controller/computes.py +++ b/gns3server/api/routes/controller/computes.py @@ -160,8 +160,8 @@ async def forward_put(compute_id: Union[str, UUID], emulator: str, endpoint_path return await compute.forward("PUT", emulator, endpoint_path, data=compute_data) -@router.post("/{compute_id}/auto_idlepc") -async def autoidlepc(compute_id: Union[str, UUID], auto_idle_pc: schemas.AutoIdlePC) -> str: +@router.post("/{compute_id}/dynamips/auto_idlepc") +async def dynamips_autoidlepc(compute_id: Union[str, UUID], auto_idle_pc: schemas.AutoIdlePC) -> str: """ Find a suitable Idle-PC value for a given IOS image. This may take a few minutes. """ diff --git a/gns3server/compute/qemu/__init__.py b/gns3server/compute/qemu/__init__.py index 58332b2e..836c0ea3 100644 --- a/gns3server/compute/qemu/__init__.py +++ b/gns3server/compute/qemu/__init__.py @@ -159,30 +159,6 @@ class Qemu(BaseManager): return qemus - @staticmethod - async def img_binary_list(): - """ - Gets QEMU-img binaries list available on the host. - - :returns: Array of dictionary {"path": Qemu-img binary path, "version": version of Qemu-img} - """ - qemu_imgs = [] - for path in Qemu.paths_list(): - try: - for f in os.listdir(path): - if ( - (f == "qemu-img" or f == "qemu-img.exe") - and os.access(os.path.join(path, f), os.X_OK) - and os.path.isfile(os.path.join(path, f)) - ): - qemu_path = os.path.join(path, f) - version = await Qemu._get_qemu_img_version(qemu_path) - qemu_imgs.append({"path": qemu_path, "version": version}) - except OSError: - continue - - return qemu_imgs - @staticmethod async def get_qemu_version(qemu_path): """ diff --git a/gns3server/controller/appliance_to_template.py b/gns3server/controller/appliance_to_template.py index aee787f6..f014c568 100644 --- a/gns3server/controller/appliance_to_template.py +++ b/gns3server/controller/appliance_to_template.py @@ -95,10 +95,8 @@ class ApplianceToTemplate: new_config["options"] = options.strip() new_config.update(version.get("images")) - if "path" in appliance_config["qemu"]: - new_config["qemu_path"] = appliance_config["qemu"]["path"] - else: - new_config["qemu_path"] = "qemu-system-{}".format(appliance_config["qemu"]["arch"]) + if "arch" in appliance_config["qemu"]: + new_config["platform"] = appliance_config["qemu"]["arch"] if "first_port_name" in appliance_config: new_config["first_port_name"] = appliance_config["first_port_name"] diff --git a/tests/api/routes/compute/test_qemu_nodes.py b/tests/api/routes/compute/test_qemu_nodes.py index 6c596df0..c57bb2e0 100644 --- a/tests/api/routes/compute/test_qemu_nodes.py +++ b/tests/api/routes/compute/test_qemu_nodes.py @@ -351,34 +351,6 @@ async def test_qemu_delete_nio(app: FastAPI, compute_client: AsyncClient, qemu_v assert response.status_code == status.HTTP_204_NO_CONTENT -async def test_qemu_list_binaries(app: FastAPI, compute_client: AsyncClient) -> None: - - ret = [{"path": "/tmp/1", "version": "2.2.0"}, - {"path": "/tmp/2", "version": "2.1.0"}] - - with asyncio_patch("gns3server.compute.qemu.Qemu.binary_list", return_value=ret) as mock: - response = await compute_client.get(app.url_path_for("compute:get_qemu_binaries")) - assert mock.called_with(None) - assert response.status_code == status.HTTP_200_OK - assert response.json() == ret - - -# async def test_qemu_list_binaries_filter(app: FastAPI, compute_client: AsyncClient, vm: dict) -> None: -# -# ret = [ -# {"path": "/tmp/x86_64", "version": "2.2.0"}, -# {"path": "/tmp/alpha", "version": "2.1.0"}, -# {"path": "/tmp/i386", "version": "2.1.0"} -# ] -# -# with asyncio_patch("gns3server.compute.qemu.Qemu.binary_list", return_value=ret) as mock: -# response = await compute_client.get(app.url_path_for("compute:get_qemu_binaries"), -# json={"archs": ["i386"]}) -# assert response.status_code == status.HTTP_200_OK -# assert mock.called_with(["i386"]) -# assert response.json() == ret - - async def test_images(app: FastAPI, compute_client: AsyncClient, fake_qemu_vm) -> None: response = await compute_client.get(app.url_path_for("compute:get_qemu_images")) diff --git a/tests/compute/qemu/test_qemu_manager.py b/tests/compute/qemu/test_qemu_manager.py index 3c80d6ec..3786121c 100644 --- a/tests/compute/qemu/test_qemu_manager.py +++ b/tests/compute/qemu/test_qemu_manager.py @@ -82,29 +82,29 @@ async def test_binary_list(monkeypatch, tmpdir): assert {"path": os.path.join(os.environ["PATH"], "hello"), "version": version} not in qemus -@pytest.mark.asyncio -async def test_img_binary_list(monkeypatch, tmpdir): - - monkeypatch.setenv("PATH", str(tmpdir)) - files_to_create = ["qemu-img", "qemu-io", "qemu-system-x86", "qemu-system-x42", "qemu-kvm", "hello"] - - for file_to_create in files_to_create: - path = os.path.join(os.environ["PATH"], file_to_create) - with open(path, "w+") as f: - f.write("1") - os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - - with asyncio_patch("gns3server.compute.qemu.subprocess_check_output", return_value="qemu-img version 2.2.0, Copyright (c) 2004-2008 Fabrice Bellard") as mock: - qemus = await Qemu.img_binary_list() - - version = "2.2.0" - - assert {"path": os.path.join(os.environ["PATH"], "qemu-img"), "version": version} in qemus - assert {"path": os.path.join(os.environ["PATH"], "qemu-io"), "version": version} not in qemus - assert {"path": os.path.join(os.environ["PATH"], "qemu-system-x86"), "version": version} not in qemus - assert {"path": os.path.join(os.environ["PATH"], "qemu-kvm"), "version": version} not in qemus - assert {"path": os.path.join(os.environ["PATH"], "qemu-system-x42"), "version": version} not in qemus - assert {"path": os.path.join(os.environ["PATH"], "hello"), "version": version} not in qemus +# @pytest.mark.asyncio +# async def test_img_binary_list(monkeypatch, tmpdir): +# +# monkeypatch.setenv("PATH", str(tmpdir)) +# files_to_create = ["qemu-img", "qemu-io", "qemu-system-x86", "qemu-system-x42", "qemu-kvm", "hello"] +# +# for file_to_create in files_to_create: +# path = os.path.join(os.environ["PATH"], file_to_create) +# with open(path, "w+") as f: +# f.write("1") +# os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) +# +# with asyncio_patch("gns3server.compute.qemu.subprocess_check_output", return_value="qemu-img version 2.2.0, Copyright (c) 2004-2008 Fabrice Bellard") as mock: +# qemus = await Qemu.img_binary_list() +# +# version = "2.2.0" +# +# assert {"path": os.path.join(os.environ["PATH"], "qemu-img"), "version": version} in qemus +# assert {"path": os.path.join(os.environ["PATH"], "qemu-io"), "version": version} not in qemus +# assert {"path": os.path.join(os.environ["PATH"], "qemu-system-x86"), "version": version} not in qemus +# assert {"path": os.path.join(os.environ["PATH"], "qemu-kvm"), "version": version} not in qemus +# assert {"path": os.path.join(os.environ["PATH"], "qemu-system-x42"), "version": version} not in qemus +# assert {"path": os.path.join(os.environ["PATH"], "hello"), "version": version} not in qemus def test_get_legacy_vm_workdir(): From 30f7c0ce740a49d97a5450d3db6c4e920a644643 Mon Sep 17 00:00:00 2001 From: grossmj Date: Fri, 3 Jun 2022 15:35:33 +0700 Subject: [PATCH 2/3] Fix issues when discovering images --- gns3server/api/routes/controller/images.py | 4 +++- gns3server/db/tasks.py | 2 ++ gns3server/utils/images.py | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/gns3server/api/routes/controller/images.py b/gns3server/api/routes/controller/images.py index 6bd3cc13..c23b04b9 100644 --- a/gns3server/api/routes/controller/images.py +++ b/gns3server/api/routes/controller/images.py @@ -129,7 +129,7 @@ async def get_image( async def delete_image( image_path: str, images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)), -) -> None: +) -> Response: """ Delete an image. """ @@ -159,6 +159,8 @@ async def delete_image( if not success: raise ControllerError(f"Image '{image_path}' could not be deleted") + return Response(status_code=status.HTTP_204_NO_CONTENT) + @router.post("/prune", status_code=status.HTTP_204_NO_CONTENT) async def prune_images( diff --git a/gns3server/db/tasks.py b/gns3server/db/tasks.py index 99853993..f6ec4125 100644 --- a/gns3server/db/tasks.py +++ b/gns3server/db/tasks.py @@ -91,6 +91,8 @@ async def get_computes(app: FastAPI) -> List[dict]: def image_filter(change: Change, path: str) -> bool: if change == Change.added: + if path.endswith(".tmp") or path.endswith(".md5sum") or path.startswith("."): + return False header_magic_len = 7 with open(path, "rb") as f: image_header = f.read(header_magic_len) # read the first 7 bytes of the file diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py index 816431fd..ff0986e5 100644 --- a/gns3server/utils/images.py +++ b/gns3server/utils/images.py @@ -139,7 +139,7 @@ async def discover_images(image_type: str, skip_image_paths: list = None) -> Lis for directory in images_directories(image_type): for root, _, filenames in os.walk(os.path.normpath(directory)): for filename in filenames: - if filename.endswith(".md5sum") or filename.startswith("."): + if filename.endswith(".tmp") or filename.endswith(".md5sum") or filename.startswith("."): continue path = os.path.join(root, filename) if not os.path.isfile(path) or skip_image_paths and path in skip_image_paths or path in files: @@ -343,7 +343,8 @@ async def write_image( os.chmod(image_path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC) finally: try: - os.remove(tmp_path) + if os.path.exists(tmp_path): + os.remove(tmp_path) except OSError: log.warning(f"Could not remove '{tmp_path}'") From 7d49b80e6bb4ef5e9dd5776f585be3c593d5956d Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 7 Jun 2022 00:38:59 +0800 Subject: [PATCH 3/3] Add controller endpoints to get VirtualBox VMs, VMware VMs and Docker images --- gns3server/api/routes/controller/computes.py | 58 ++++++++++++++------ gns3server/controller/compute.py | 17 ------ gns3server/schemas/__init__.py | 2 +- gns3server/schemas/controller/computes.py | 25 +++++++++ tests/api/routes/controller/test_computes.py | 32 ++++++++--- tests/controller/test_compute.py | 23 -------- 6 files changed, 91 insertions(+), 66 deletions(-) diff --git a/gns3server/api/routes/controller/computes.py b/gns3server/api/routes/controller/computes.py index 67652443..963ff099 100644 --- a/gns3server/api/routes/controller/computes.py +++ b/gns3server/api/routes/controller/computes.py @@ -115,18 +115,50 @@ async def delete_compute( return Response(status_code=status.HTTP_204_NO_CONTENT) -@router.get("/{compute_id}/{emulator}/images") -async def get_images(compute_id: Union[str, UUID], emulator: str) -> List[str]: +@router.get("/{compute_id}/docker/images", response_model=List[schemas.ComputeDockerImage]) +async def docker_get_images(compute_id: Union[str, UUID]) -> List[schemas.ComputeDockerImage]: """ - Return the list of images available on a compute for a given emulator type. + Get Docker images from a compute. + """ + + compute = Controller.instance().get_compute(str(compute_id)) + result = await compute.forward("GET", "docker", "images") + return result + + +@router.get("/{compute_id}/virtualbox/vms", response_model=List[schemas.ComputeVirtualBoxVM]) +async def virtualbox_vms(compute_id: Union[str, UUID]) -> List[schemas.ComputeVirtualBoxVM]: + """ + Get VirtualBox VMs from a compute. + """ + + compute = Controller.instance().get_compute(str(compute_id)) + result = await compute.forward("GET", "virtualbox", "vms") + return result + + +@router.get("/{compute_id}/vmware/vms", response_model=List[schemas.ComputeVMwareVM]) +async def vmware_vms(compute_id: Union[str, UUID]) -> List[schemas.ComputeVMwareVM]: + """ + Get VMware VMs from a compute. + """ + + compute = Controller.instance().get_compute(str(compute_id)) + result = await compute.forward("GET", "vmware", "vms") + return result + + +@router.post("/{compute_id}/dynamips/auto_idlepc") +async def dynamips_autoidlepc(compute_id: Union[str, UUID], auto_idle_pc: schemas.AutoIdlePC) -> str: + """ + Find a suitable Idle-PC value for a given IOS image. This may take a few minutes. """ controller = Controller.instance() - compute = controller.get_compute(str(compute_id)) - return await compute.images(emulator) + return await controller.autoidlepc(str(compute_id), auto_idle_pc.platform, auto_idle_pc.image, auto_idle_pc.ram) -@router.get("/{compute_id}/{emulator}/{endpoint_path:path}") +@router.get("/{compute_id}/{emulator}/{endpoint_path:path}", deprecated=True) async def forward_get(compute_id: Union[str, UUID], emulator: str, endpoint_path: str) -> dict: """ Forward a GET request to a compute. @@ -138,7 +170,7 @@ async def forward_get(compute_id: Union[str, UUID], emulator: str, endpoint_path return result -@router.post("/{compute_id}/{emulator}/{endpoint_path:path}") +@router.post("/{compute_id}/{emulator}/{endpoint_path:path}", deprecated=True) async def forward_post(compute_id: Union[str, UUID], emulator: str, endpoint_path: str, compute_data: dict) -> dict: """ Forward a POST request to a compute. @@ -149,7 +181,7 @@ async def forward_post(compute_id: Union[str, UUID], emulator: str, endpoint_pat return await compute.forward("POST", emulator, endpoint_path, data=compute_data) -@router.put("/{compute_id}/{emulator}/{endpoint_path:path}") +@router.put("/{compute_id}/{emulator}/{endpoint_path:path}", deprecated=True) async def forward_put(compute_id: Union[str, UUID], emulator: str, endpoint_path: str, compute_data: dict) -> dict: """ Forward a PUT request to a compute. @@ -158,13 +190,3 @@ async def forward_put(compute_id: Union[str, UUID], emulator: str, endpoint_path compute = Controller.instance().get_compute(str(compute_id)) return await compute.forward("PUT", emulator, endpoint_path, data=compute_data) - - -@router.post("/{compute_id}/dynamips/auto_idlepc") -async def dynamips_autoidlepc(compute_id: Union[str, UUID], auto_idle_pc: schemas.AutoIdlePC) -> str: - """ - Find a suitable Idle-PC value for a given IOS image. This may take a few minutes. - """ - - controller = Controller.instance() - return await controller.autoidlepc(str(compute_id), auto_idle_pc.platform, auto_idle_pc.image, auto_idle_pc.ram) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index d8caa17c..29201759 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -620,23 +620,6 @@ class Compute: raise ControllerError(f"Connection lost to {self._id} during {method} {action}") return res.json - async def images(self, type): - """ - Return the list of images available for this type on the compute node. - """ - - res = await self.http_query("GET", f"/{type}/images", timeout=None) - images = res.json - - try: - if type in ["qemu", "dynamips", "iou"]: - images = sorted(images, key=itemgetter("filename")) - else: - images = sorted(images, key=itemgetter("image")) - except OSError as e: - raise ComputeError(f"Cannot list images: {str(e)}") - return images - async def list_files(self, project): """ List files in the project on computes diff --git a/gns3server/schemas/__init__.py b/gns3server/schemas/__init__.py index 77a5c9c3..d9ff9015 100644 --- a/gns3server/schemas/__init__.py +++ b/gns3server/schemas/__init__.py @@ -21,7 +21,7 @@ from .version import Version # Controller schemas from .controller.links import LinkCreate, LinkUpdate, Link -from .controller.computes import ComputeCreate, ComputeUpdate, AutoIdlePC, Compute +from .controller.computes import ComputeCreate, ComputeUpdate, ComputeVirtualBoxVM, ComputeVMwareVM, ComputeDockerImage, AutoIdlePC, Compute from .controller.templates import TemplateCreate, TemplateUpdate, TemplateUsage, Template from .controller.images import Image, ImageType from .controller.appliances import ApplianceVersion, Appliance diff --git a/gns3server/schemas/controller/computes.py b/gns3server/schemas/controller/computes.py index 1c035dc7..8dc6d722 100644 --- a/gns3server/schemas/controller/computes.py +++ b/gns3server/schemas/controller/computes.py @@ -148,6 +148,31 @@ class Compute(DateTimeModelMixin, ComputeBase): orm_mode = True +class ComputeVirtualBoxVM(BaseModel): + """ + VirtualBox VM from compute. + """ + + vmname: str = Field(..., description="VirtualBox VM name") + ram: int = Field(..., description="VirtualBox VM memory") + + +class ComputeVMwareVM(BaseModel): + """ + VMware VM from compute. + """ + + vmname: str = Field(..., description="VMware VM name") + + +class ComputeDockerImage(BaseModel): + """ + Docker image from compute. + """ + + image: str = Field(..., description="Docker image name") + + class AutoIdlePC(BaseModel): """ Data for auto Idle-PC request. diff --git a/tests/api/routes/controller/test_computes.py b/tests/api/routes/controller/test_computes.py index 70134771..bd775d90 100644 --- a/tests/api/routes/controller/test_computes.py +++ b/tests/api/routes/controller/test_computes.py @@ -109,7 +109,7 @@ class TestComputeRoutes: class TestComputeFeatures: - async def test_compute_list_images(self, app: FastAPI, client: AsyncClient) -> None: + async def test_compute_list_docker_images(self, app: FastAPI, client: AsyncClient) -> None: params = { "protocol": "http", @@ -123,12 +123,12 @@ class TestComputeFeatures: assert response.status_code == status.HTTP_201_CREATED compute_id = response.json()["compute_id"] - with asyncio_patch("gns3server.controller.compute.Compute.images", return_value=[{"filename": "linux.qcow2"}, {"filename": "asav.qcow2"}]) as mock: - response = await client.get(app.url_path_for("delete_compute", compute_id=compute_id) + "/qemu/images") - assert response.json() == [{"filename": "linux.qcow2"}, {"filename": "asav.qcow2"}] - mock.assert_called_with("qemu") + with asyncio_patch("gns3server.controller.compute.Compute.forward", return_value=[{"image": "docker1"}, {"image": "docker2"}]) as mock: + response = await client.get(app.url_path_for("docker_get_images", compute_id=compute_id)) + mock.assert_called_with("GET", "docker", "images") + assert response.json() == [{"image": "docker1"}, {"image": "docker2"}] - async def test_compute_list_vms(self, app: FastAPI, client: AsyncClient) -> None: + async def test_compute_list_virtualbox_vms(self, app: FastAPI, client: AsyncClient) -> None: params = { "protocol": "http", @@ -142,10 +142,28 @@ class TestComputeFeatures: compute_id = response.json()["compute_id"] with asyncio_patch("gns3server.controller.compute.Compute.forward", return_value=[]) as mock: - response = await client.get(app.url_path_for("get_compute", compute_id=compute_id) + "/virtualbox/vms") + response = await client.get(app.url_path_for("virtualbox_vms", compute_id=compute_id)) mock.assert_called_with("GET", "virtualbox", "vms") assert response.json() == [] + async def test_compute_list_vmware_vms(self, app: FastAPI, client: AsyncClient) -> None: + + params = { + "protocol": "http", + "host": "localhost", + "port": 4243, + "user": "julien", + "password": "secure" + } + response = await client.post(app.url_path_for("get_computes"), json=params) + assert response.status_code == status.HTTP_201_CREATED + compute_id = response.json()["compute_id"] + + with asyncio_patch("gns3server.controller.compute.Compute.forward", return_value=[]) as mock: + response = await client.get(app.url_path_for("vmware_vms", compute_id=compute_id)) + mock.assert_called_with("GET", "vmware", "vms") + assert response.json() == [] + async def test_compute_create_img(self, app: FastAPI, client: AsyncClient) -> None: params = { diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index 68a8670b..26939b3c 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -397,29 +397,6 @@ async def test_forward_post(compute): await compute.close() -@pytest.mark.asyncio -async def test_images(compute): - """ - Will return image on compute - """ - - response = MagicMock() - response.status = 200 - response.read = AsyncioMagicMock(return_value=json.dumps([{ - "filename": "linux.qcow2", - "path": "linux.qcow2", - "md5sum": "d41d8cd98f00b204e9800998ecf8427e", - "filesize": 0}]).encode()) - with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock: - images = await compute.images("qemu") - mock.assert_called_with("GET", "https://example.com:84/v3/compute/qemu/images", auth=None, data=None, headers={'content-type': 'application/json'}, chunked=None, timeout=None) - await compute.close() - - assert images == [ - {"filename": "linux.qcow2", "path": "linux.qcow2", "md5sum": "d41d8cd98f00b204e9800998ecf8427e", "filesize": 0} - ] - - @pytest.mark.asyncio async def test_list_files(project, compute):