Aiohttp 2.0

Ref #1000
pull/1036/head
Julien Duponchelle 7 years ago
parent e9fef928b0
commit a4245fcf44
No known key found for this signature in database
GPG Key ID: CE8B29639E07F5E8

@ -56,7 +56,7 @@ class Docker(BaseManager):
self._connected = True
connector = self.connector()
version = yield from self.query("GET", "version")
except (aiohttp.errors.ClientOSError, FileNotFoundError):
except (aiohttp.ClientOSError, FileNotFoundError):
self._connected = False
raise DockerError("Can't connect to docker daemon")
if parse_version(version["ApiVersion"]) < parse_version(DOCKER_MINIMUM_API_VERSION):
@ -68,7 +68,7 @@ class Docker(BaseManager):
raise DockerError("Docker is supported only on Linux")
try:
self._connector = aiohttp.connector.UnixConnector(self._server_url, conn_timeout=2, limit=None)
except (aiohttp.errors.ClientOSError, FileNotFoundError):
except (aiohttp.ClientOSError, FileNotFoundError):
raise DockerError("Can't connect to docker daemon")
return self._connector
@ -77,7 +77,7 @@ class Docker(BaseManager):
yield from super().unload()
if self._connected:
if self._connector and not self._connector.closed:
yield from self._connector.close()
self._connector.close()
@asyncio.coroutine
def query(self, method, path, data={}, params={}):
@ -191,7 +191,7 @@ class Docker(BaseManager):
while True:
try:
chunk = yield from response.content.read(1024)
except aiohttp.errors.ServerDisconnectedError:
except aiohttp.ServerDisconnectedError:
break
if not chunk:
break

@ -500,7 +500,7 @@ class DockerVM(BaseNode):
while True:
msg = yield from ws.receive()
if msg.tp == aiohttp.MsgType.text:
if msg.tp == aiohttp.WSMsgType.text:
out.feed_data(msg.data.encode())
else:
out.feed_eof()

@ -421,15 +421,15 @@ class Compute:
"""
try:
self._ws = yield from self._session().ws_connect(self._getUrl("/notifications/ws"), auth=self._auth)
except (aiohttp.errors.WSServerHandshakeError, aiohttp.errors.ClientResponseError):
except (aiohttp.WSServerHandshakeError, aiohttp.ClientResponseError):
self._ws = None
while self._ws is not None:
try:
response = yield from self._ws.receive()
except aiohttp.errors.WSServerHandshakeError:
except aiohttp.WSServerHandshakeError:
self._ws = None
break
if response.tp == aiohttp.MsgType.closed or response.tp == aiohttp.MsgType.error or response.data is None:
if response.tp == aiohttp.WSMsgType.closed or response.tp == aiohttp.WSMsgType.error or response.data is None:
self._connected = False
break
msg = json.loads(response.data)
@ -474,7 +474,7 @@ class Compute:
url = self._getUrl(path)
headers = {}
headers['content-type'] = 'application/json'
chunked = False
chunked = None
if data == {}:
data = None
elif data is not None:
@ -500,12 +500,12 @@ class Compute:
data = send_data(data)
else:
data = json.dumps(data)
data = json.dumps(data).encode("utf-8")
try:
response = yield from self._session().request(method, url, headers=headers, data=data, auth=self._auth, chunked=chunked, timeout=timeout)
except asyncio.TimeoutError as e:
raise ComputeError("Timeout error when connecting to {}".format(url))
except (aiohttp.errors.ClientOSError, aiohttp.errors.ClientRequestError, aiohttp.errors.ServerDisconnectedError, aiohttp.ClientResponseError, ValueError) as e:
except (aiohttp.ClientError, aiohttp.ServerDisconnectedError, ValueError) as e:
raise ComputeError(str(e))
body = yield from response.read()
if body and not raw:
@ -581,7 +581,7 @@ class Compute:
"""
try:
res = yield from self.http_query(method, "/{}/{}".format(type, path), data=data, timeout=None)
except aiohttp.errors.DisconnectedError:
except aiohttp.ServerDisconnectedError:
raise aiohttp.web.HTTPGatewayTimeout()
return res.json

@ -228,7 +228,7 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
try:
resp = None
resp = yield from session.get('http://127.0.0.1:{}/v2/compute/network/interfaces'.format(api_port))
except (OSError, aiohttp.errors.ClientHttpProcessingError, TimeoutError, asyncio.TimeoutError):
except (OSError, aiohttp.ClientError, TimeoutError, asyncio.TimeoutError):
pass
if resp:

@ -457,7 +457,7 @@ class Node:
try:
yield from self.post("/stop", timeout=240, dont_connect=True)
# We don't care if a node is down at this step
except (ComputeError, aiohttp.errors.ClientHttpProcessingError, aiohttp.web.HTTPError):
except (ComputeError, aiohttp.ClientError, aiohttp.web.HTTPError):
pass
except asyncio.TimeoutError:
raise aiohttp.web.HTTPRequestTimeout(text="Timeout when stopping {}".format(self._name))

@ -45,7 +45,10 @@ if __version_info__[3] != 0:
# Display a traceback in case of segfault crash. Usefull when frozen
# Not enabled by default for security reason
log.info("Enable catching segfault")
faulthandler.enable()
try:
faulthandler.enable()
except Exception:
pass # Could fail when loaded into tests
class CrashReport:

@ -16,8 +16,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import asyncio
import aiohttp.errors
import aiohttp
from aiohttp.web import WebSocketResponse
from gns3server.web.route import Route
from gns3server.compute.notification_manager import NotificationManager
@ -30,7 +29,7 @@ def process_websocket(ws):
"""
try:
yield from ws.receive()
except aiohttp.errors.WSServerHandshakeError:
except aiohttp.WSServerHandshakeError:
pass

@ -157,7 +157,6 @@ class ProjectHandler:
response.set_status(200)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
response.start(request)
queue = project.get_listen_queue()
@ -240,11 +239,10 @@ class ProjectHandler:
response.set_status(200)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
try:
with open(path, "rb") as f:
response.start(request)
yield from response.prepare(request)
while True:
data = f.read(4096)
if not data:
@ -274,7 +272,7 @@ class ProjectHandler:
path = request.match_info["path"]
path = os.path.normpath(path)
# Raise an error if user try to escape
# Raise an error if user try to escape
if path[0] == ".":
raise aiohttp.web.HTTPForbidden
path = os.path.join(project.path, path)
@ -283,11 +281,10 @@ class ProjectHandler:
response.set_status(200)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
try:
with open(path, "rb") as f:
response.start(request)
yield from response.prepare(request)
while True:
data = f.read(4096)
if not data:
@ -358,8 +355,7 @@ class ProjectHandler:
response.headers['CONTENT-DISPOSITION'] = 'attachment; filename="{}.gns3project"'.format(project.name)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
response.start(request)
yield from response.prepare(request)
include_images = bool(int(request.json.get("include_images", "0")))
for data in project.export(include_images=include_images):

@ -179,8 +179,7 @@ class LinkHandler:
response.set_status(200)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
response.start(request)
yield from response.prepare(request)
while True:
chunk = f.read(4096)

@ -351,8 +351,7 @@ class NodeHandler:
response.set_status(200)
response.content_type = "application/octet-stream"
response.enable_chunked_encoding()
response.content_length = None
response.start(request)
yield from response.prepare(request)
response.write(res.body)
yield from response.write_eof()

@ -17,7 +17,6 @@
import os
import aiohttp
import aiohttp.errors
import asyncio
import tempfile
@ -47,7 +46,7 @@ def process_websocket(ws):
"""
try:
yield from ws.receive()
except aiohttp.errors.WSServerHandshakeError:
except aiohttp.WSServerHandshakeError:
pass
@ -212,9 +211,8 @@ class ProjectHandler:
response.set_status(200)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
response.start(request)
yield from response.prepare(request)
with controller.notification.queue(project) as queue:
while True:
try:
@ -295,8 +293,7 @@ class ProjectHandler:
response.headers['CONTENT-DISPOSITION'] = 'attachment; filename="{}.gns3project"'.format(project.name)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
response.start(request)
yield from response.prepare(request)
for data in datas:
response.write(data)
@ -408,11 +405,10 @@ class ProjectHandler:
response.set_status(200)
response.enable_chunked_encoding()
# Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed)
response.content_length = None
try:
with open(path, "rb") as f:
response.start(request)
yield from response.prepare(request)
while True:
data = f.read(4096)
if not data:

@ -120,10 +120,10 @@ class Response(aiohttp.web.Response):
st = os.stat(path)
self.last_modified = st.st_mtime
self.content_length = st.st_size
self.enable_chunked_encoding()
with open(path, 'rb') as fobj:
self.start(self._request)
yield from self.prepare(self._request)
chunk_size = 4096
chunk = fobj.read(chunk_size)
while chunk:

@ -220,16 +220,11 @@ class Route(object):
response = Response(request=request, route=route)
response.set_status(408)
response.json({"message": "Request canceled", "status": 408})
except aiohttp.ClientDisconnectedError:
log.warn("Client disconnected")
except aiohttp.ClientError:
log.warn("Client error")
response = Response(request=request, route=route)
response.set_status(408)
response.json({"message": "Client disconnected", "status": 408})
except ConnectionResetError:
log.error("Client connection reset")
response = Response(request=request, route=route)
response.set_status(408)
response.json({"message": "Connection reset", "status": 408})
response.json({"message": "Client error", "status": 408})
except Exception as e:
log.error("Uncaught exception detected: {type}".format(type=type(e)), exc_info=1)
response = Response(request=request, route=route)

@ -1,7 +1,7 @@
jsonschema>=2.4.0
aiohttp>=1.3.5,<=1.4.0 # pyup: ignore
aiohttp-cors==0.5.1 # pyup: ignore
yarl>=0.9.8,<0.10 # pyup: ignore
aiohttp>=2.0.7,<2.1.0 # pyup: ignore
aiohttp-cors>=0.5.3,<0.6.0 # pyup: ignore
yarl>=0.10.2,<0.11 # pyup: ignore
Jinja2>=2.7.3
raven>=5.23.0
psutil>=3.0.0

@ -227,7 +227,7 @@ async def main(loop):
try:
j = await error.response.json()
die("%s %s invalid status %d:\n%s", error.method, error.path, error.response.status, json.dumps(j, indent=4))
except (ValueError, aiohttp.errors.ServerDisconnectedError):
except (ValueError, aiohttp.ServerDisconnectedError):
die("%s %s invalid status %d", error.method, error.path, error.response.status)

@ -75,7 +75,7 @@ def test_compute_httpQuery(compute, async_run):
response.status = 200
async_run(compute.post("/projects", {"a": "b"}))
mock.assert_called_with("POST", "https://example.com:84/v2/compute/projects", data='{"a": "b"}', headers={'content-type': 'application/json'}, auth=None, chunked=False, timeout=20)
mock.assert_called_with("POST", "https://example.com:84/v2/compute/projects", data=b'{"a": "b"}', headers={'content-type': 'application/json'}, auth=None, chunked=None, timeout=20)
assert compute._auth is None
@ -87,7 +87,7 @@ def test_compute_httpQueryAuth(compute, async_run):
compute.user = "root"
compute.password = "toor"
async_run(compute.post("/projects", {"a": "b"}))
mock.assert_called_with("POST", "https://example.com:84/v2/compute/projects", data='{"a": "b"}', headers={'content-type': 'application/json'}, auth=compute._auth, chunked=False, timeout=20)
mock.assert_called_with("POST", "https://example.com:84/v2/compute/projects", data=b'{"a": "b"}', headers={'content-type': 'application/json'}, auth=compute._auth, chunked=None, timeout=20)
assert compute._auth.login == "root"
assert compute._auth.password == "toor"
@ -100,8 +100,8 @@ def test_compute_httpQueryNotConnected(compute, controller, async_run):
response.status = 200
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
async_run(compute.post("/projects", {"a": "b"}))
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=False, timeout=20)
mock.assert_any_call("POST", "https://example.com:84/v2/compute/projects", data='{"a": "b"}', headers={'content-type': 'application/json'}, auth=None, chunked=False, timeout=20)
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=None, timeout=20)
mock.assert_any_call("POST", "https://example.com:84/v2/compute/projects", data=b'{"a": "b"}', headers={'content-type': 'application/json'}, auth=None, chunked=None, timeout=20)
assert compute._connected
assert compute._capabilities["version"] == __version__
controller.notification.emit.assert_called_with("compute.updated", compute.__json__())
@ -122,8 +122,8 @@ def test_compute_httpQueryNotConnectedGNS3vmNotRunning(compute, controller, asyn
response.status = 200
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
async_run(compute.post("/projects", {"a": "b"}))
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=False, timeout=20)
mock.assert_any_call("POST", "https://example.com:84/v2/compute/projects", data='{"a": "b"}', headers={'content-type': 'application/json'}, auth=None, chunked=False, timeout=20)
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=None, timeout=20)
mock.assert_any_call("POST", "https://example.com:84/v2/compute/projects", data=b'{"a": "b"}', headers={'content-type': 'application/json'}, auth=None, chunked=None, timeout=20)
assert controller.gns3vm.start.called
assert compute._connected
@ -139,7 +139,7 @@ def test_compute_httpQueryNotConnectedInvalidVersion(compute, async_run):
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
with pytest.raises(aiohttp.web.HTTPConflict):
async_run(compute.post("/projects", {"a": "b"}))
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=False, timeout=20)
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=None, timeout=20)
def test_compute_httpQueryNotConnectedNonGNS3Server(compute, async_run):
@ -150,7 +150,7 @@ def test_compute_httpQueryNotConnectedNonGNS3Server(compute, async_run):
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
with pytest.raises(aiohttp.web.HTTPConflict):
async_run(compute.post("/projects", {"a": "b"}))
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=False, timeout=20)
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=None, timeout=20)
def test_compute_httpQueryNotConnectedNonGNS3Server2(compute, async_run):
@ -161,7 +161,7 @@ def test_compute_httpQueryNotConnectedNonGNS3Server2(compute, async_run):
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
with pytest.raises(aiohttp.web.HTTPConflict):
async_run(compute.post("/projects", {"a": "b"}))
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=False, timeout=20)
mock.assert_any_call("GET", "https://example.com:84/v2/compute/capabilities", headers={'content-type': 'application/json'}, data=None, auth=None, chunked=None, timeout=20)
def test_compute_httpQueryError(compute, async_run):
@ -190,7 +190,7 @@ def test_compute_httpQuery_project(compute, async_run):
project = Project(name="Test")
async_run(compute.post("/projects", project))
mock.assert_called_with("POST", "https://example.com:84/v2/compute/projects", data=json.dumps(project.__json__()), headers={'content-type': 'application/json'}, auth=None, chunked=False, timeout=20)
mock.assert_called_with("POST", "https://example.com:84/v2/compute/projects", data=json.dumps(project.__json__()), headers={'content-type': 'application/json'}, auth=None, chunked=None, timeout=20)
def test_connectNotification(compute, async_run):
@ -205,11 +205,11 @@ def test_connectNotification(compute, async_run):
if call == 1:
response = MagicMock()
response.data = '{"action": "test", "event": {"a": 1}}'
response.tp = aiohttp.MsgType.text
response.tp = aiohttp.WSMsgType.text
return response
else:
response = MagicMock()
response.tp = aiohttp.MsgType.closed
response.tp = aiohttp.WSMsgType.closed
return response
compute._controller._notification = MagicMock()
@ -238,11 +238,11 @@ def test_connectNotificationPing(compute, async_run):
if call == 1:
response = MagicMock()
response.data = '{"action": "ping", "event": {"cpu_usage_percent": 35.7, "memory_usage_percent": 80.7}}'
response.tp = aiohttp.MsgType.text
response.tp = aiohttp.WSMsgType.text
return response
else:
response = MagicMock()
response.tp = aiohttp.MsgType.closed
response.tp = aiohttp.WSMsgType.closed
return response
compute._controller._notification = MagicMock()
@ -325,7 +325,7 @@ def test_forward_get(compute, async_run):
response.status = 200
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
async_run(compute.forward("GET", "qemu", "images"))
mock.assert_called_with("GET", "https://example.com:84/v2/compute/qemu/images", auth=None, data=None, headers={'content-type': 'application/json'}, chunked=False, timeout=None)
mock.assert_called_with("GET", "https://example.com:84/v2/compute/qemu/images", auth=None, data=None, headers={'content-type': 'application/json'}, chunked=None, timeout=None)
def test_forward_404(compute, async_run):
@ -341,7 +341,7 @@ def test_forward_post(compute, async_run):
response.status = 200
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
async_run(compute.forward("POST", "qemu", "img", data={"id": 42}))
mock.assert_called_with("POST", "https://example.com:84/v2/compute/qemu/img", auth=None, data='{"id": 42}', headers={'content-type': 'application/json'}, chunked=False, timeout=None)
mock.assert_called_with("POST", "https://example.com:84/v2/compute/qemu/img", auth=None, data=b'{"id": 42}', headers={'content-type': 'application/json'}, chunked=None, timeout=None)
def test_images(compute, async_run, images_dir):
@ -358,7 +358,7 @@ def test_images(compute, async_run, images_dir):
open(os.path.join(images_dir, "QEMU", "asa.qcow2"), "w+").close()
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
images = async_run(compute.images("qemu"))
mock.assert_called_with("GET", "https://example.com:84/v2/compute/qemu/images", auth=None, data=None, headers={'content-type': 'application/json'}, chunked=False, timeout=None)
mock.assert_called_with("GET", "https://example.com:84/v2/compute/qemu/images", auth=None, data=None, headers={'content-type': 'application/json'}, chunked=None, timeout=None)
assert images == [
{"filename": "asa.qcow2", "path": "asa.qcow2", "md5sum": "d41d8cd98f00b204e9800998ecf8427e", "filesize": 0},
@ -373,7 +373,7 @@ def test_list_files(project, async_run, compute):
response.status = 200
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
assert async_run(compute.list_files(project)) == res
mock.assert_any_call("GET", "https://example.com:84/v2/compute/projects/{}/files".format(project.id), auth=None, chunked=False, data=None, headers={'content-type': 'application/json'}, timeout=120)
mock.assert_any_call("GET", "https://example.com:84/v2/compute/projects/{}/files".format(project.id), auth=None, chunked=None, data=None, headers={'content-type': 'application/json'}, timeout=120)
def test_interfaces(project, async_run, compute):
@ -392,7 +392,7 @@ def test_interfaces(project, async_run, compute):
response.status = 200
with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
assert async_run(compute.interfaces()) == res
mock.assert_any_call("GET", "https://example.com:84/v2/compute/network/interfaces", auth=None, chunked=False, data=None, headers={'content-type': 'application/json'}, timeout=20)
mock.assert_any_call("GET", "https://example.com:84/v2/compute/network/interfaces", auth=None, chunked=None, data=None, headers={'content-type': 'application/json'}, timeout=20)
def test_get_ip_on_same_subnet(controller, async_run):

@ -85,26 +85,16 @@ class Query:
- example if True the session is included inside documentation
- raw do not JSON encode the query
"""
return self._loop.run_until_complete(asyncio.async(self._async_fetch(method, path, body=body, **kwargs)))
@asyncio.coroutine
def _async_fetch(self, method, path, body=None, **kwargs):
if body is not None and not kwargs.get("raw", False):
body = json.dumps(body)
@asyncio.coroutine
def go_request(future):
response = yield from aiohttp.request(method, self.get_url(path), data=body)
future.set_result(response)
future = asyncio.Future()
asyncio.async(go_request(future))
self._loop.run_until_complete(future)
response = future.result()
@asyncio.coroutine
def go_read(future, response):
response = yield from response.read()
future.set_result(response)
future = asyncio.Future()
asyncio.async(go_read(future, response))
self._loop.run_until_complete(future)
response.body = future.result()
connector = aiohttp.TCPConnector()
response = yield from aiohttp.request(method, self.get_url(path), data=body, loop=self._loop, connector=connector)
response.body = yield from response.read()
x_route = response.headers.get('X-Route', None)
if x_route is not None:
response.route = x_route.replace("/v{}".format(self._api_version), "")

@ -129,19 +129,18 @@ def test_load_project(http_controller, project, config):
assert response.json["project_id"] == project.id
def test_notification(http_controller, project, controller, loop):
def test_notification(http_controller, project, controller, loop, async_run):
@asyncio.coroutine
def go(future):
response = yield from aiohttp.request("GET", http_controller.get_url("/projects/{project_id}/notifications".format(project_id=project.id)))
def go():
connector = aiohttp.TCPConnector()
response = yield from aiohttp.request("GET", http_controller.get_url("/projects/{project_id}/notifications".format(project_id=project.id)), connector=connector)
response.body = yield from response.content.read(200)
controller.notification.emit("node.created", {"a": "b"})
response.body += yield from response.content.read(50)
response.body += yield from response.content.readany()
response.close()
future.set_result(response)
return response
future = asyncio.Future()
asyncio.async(go(future))
response = loop.run_until_complete(future)
response = async_run(asyncio.async(go()))
assert response.status == 200
assert b'"action": "ping"' in response.body
assert b'"cpu_usage_percent"' in response.body

@ -35,7 +35,6 @@ def test_symbols(http_controller):
def test_get(http_controller):
response = http_controller.get('/symbols/' + urllib.parse.quote(':/symbols/firewall.svg') + '/raw')
assert response.status == 200
assert response.headers['CONTENT-LENGTH'] == '9381'
assert response.headers['CONTENT-TYPE'] == 'image/svg+xml'
assert '</svg>' in response.html

Loading…
Cancel
Save