From dae6f173554807d7d04b425a090572a028d7046a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 17:45:45 -0400 Subject: [PATCH 01/13] Upgrade dependencies. --- requirements.txt | 121 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 34 deletions(-) diff --git a/requirements.txt b/requirements.txt index 62b6344..2c736b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,59 +4,112 @@ # # pip-compile --output-file=requirements.txt requirements.in/base.txt # -aiofiles==22.1.0 +aiofiles==23.2.1 # via -r requirements.in/base.txt -anyio==3.6.2 +annotated-types==0.7.0 + # via pydantic +anyio==4.4.0 # via + # httpx # starlette # watchfiles -asgiref==3.5.2 +asgiref==3.8.1 # via django -async-timeout==4.0.2 - # via redis -cffi==1.15.1 +certifi==2024.6.2 + # via + # httpcore + # httpx +cffi==1.16.0 # via pynacl -click==8.1.3 - # via uvicorn -django==4.1.13 +click==8.1.7 + # via + # typer + # uvicorn +django==4.2.13 # via -r requirements.in/base.txt -fastapi==0.88.0 - # via -r requirements.in/base.txt -h11==0.14.0 - # via uvicorn -httptools==0.5.0 - # via uvicorn -idna==3.4 - # via anyio -msgpack==1.0.4 - # via -r requirements.in/base.txt -pycparser==2.21 - # via cffi -pydantic==1.10.2 +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 # via fastapi +fastapi==0.111.0 + # via -r requirements.in/base.txt +fastapi-cli==0.0.4 + # via fastapi +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via fastapi +idna==3.7 + # via + # anyio + # email-validator + # httpx +jinja2==3.1.4 + # via fastapi +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +msgpack==1.0.8 + # via -r requirements.in/base.txt +orjson==3.10.3 + # via fastapi +pycparser==2.22 + # via cffi +pydantic==2.7.3 + # via fastapi +pydantic-core==2.18.4 + # via pydantic +pygments==2.18.0 + # via rich pynacl==1.5.0 # via -r requirements.in/base.txt -python-dotenv==0.21.0 +python-dotenv==1.0.1 # via uvicorn +python-multipart==0.0.9 + # via fastapi pyyaml==6.0.1 # via uvicorn -redis==4.4.0 +redis==5.1.0b6 # via -r requirements.in/base.txt -sniffio==1.3.0 - # via anyio -sqlparse==0.4.3 +rich==13.7.1 + # via typer +shellingham==1.5.4 + # via typer +sniffio==1.3.1 + # via + # anyio + # httpx +sqlparse==0.5.0 # via django -starlette==0.22.0 +starlette==0.37.2 # via fastapi -typing-extensions==4.4.0 +typer==0.12.3 + # via fastapi-cli +typing-extensions==4.12.2 # via # -r requirements.in/base.txt + # fastapi # pydantic -uvicorn[standard]==0.20.0 - # via -r requirements.in/base.txt -uvloop==0.17.0 + # pydantic-core + # typer +ujson==5.10.0 + # via fastapi +uvicorn[standard]==0.30.1 + # via + # -r requirements.in/base.txt + # fastapi +uvloop==0.19.0 # via uvicorn -watchfiles==0.18.1 +watchfiles==0.22.0 # via uvicorn -websockets==10.4 +websockets==12.0 # via uvicorn From df0d1596e2c5cdbf4dca948f9fac1165a61de747 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 17:49:40 -0400 Subject: [PATCH 02/13] Upgrade dev deps and add ruff. --- pyproject.toml | 22 +++++++++ requirements-dev.txt | 81 ++++++++++++++------------------- requirements.in/development.txt | 4 +- 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b622800..02fad2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,25 @@ line-length = 120 [build-system] requires = ["setuptools>=42"] build-backend = "setuptools.build_meta" + +[tool.ruff] +line-length = 120 +exclude = [ + ".git", + ".git-rewrite", + ".mypy_cache", + ".pytype", + ".ruff_cache", + ".venv", + "build", + "dist", + "node_modules", + "migrations", # Alembic migrations +] + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "T20", "W"] +ignore = ["E203", "E501", "E711", "E712", "N803", "N818", "T201"] + +[tool.ruff.lint.isort] +combine-as-imports = true diff --git a/requirements-dev.txt b/requirements-dev.txt index ae24fb4..c48fb6b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,66 +1,53 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements-dev.txt requirements.in/development.txt # -asgiref==3.5.2 - # via django -black==22.10.0 - # via -r requirements.in/development.txt -build==0.9.0 +asgiref==3.8.1 + # via + # django + # django-stubs +build==1.2.1 # via pip-tools -click==8.1.3 +click==8.1.7 + # via pip-tools +coverage==7.5.3 + # via -r requirements.in/development.txt +django==5.0.6 # via - # black + # django-stubs + # django-stubs-ext +django-stubs==5.0.2 + # via -r requirements.in/development.txt +django-stubs-ext==5.0.2 + # via django-stubs +mypy==1.10.0 + # via -r requirements.in/development.txt +mypy-extensions==1.0.0 + # via mypy +packaging==24.0 + # via build +pip-tools==7.4.1 + # via -r requirements.in/development.txt +pyproject-hooks==1.1.0 + # via + # build # pip-tools -coverage==6.5.0 +pywatchman==2.0.0 # via -r requirements.in/development.txt -django==3.2.16 - # via - # -r requirements.in/development.txt - # django-stubs - # django-stubs-ext -django-stubs==1.13.0 +ruff==0.4.8 # via -r requirements.in/development.txt -django-stubs-ext==0.7.0 - # via django-stubs -mypy==0.991 - # via django-stubs -mypy-extensions==0.4.3 - # via - # black - # mypy -packaging==21.3 - # via build -pathspec==0.10.2 - # via black -pep517==0.13.0 - # via build -pip-tools==6.11.0 - # via -r requirements.in/development.txt -platformdirs==2.6.0 - # via black -pyparsing==3.0.9 - # via packaging -pytz==2022.6 +sqlparse==0.5.0 # via django -pywatchman==1.4.1 - # via -r requirements.in/development.txt -sqlparse==0.4.3 - # via django -tomli==2.0.1 +types-pyyaml==6.0.12.20240311 # via django-stubs -types-pytz==2022.6.0.1 - # via django-stubs -types-pyyaml==6.0.12.2 - # via django-stubs -typing-extensions==4.4.0 +typing-extensions==4.12.2 # via # django-stubs # django-stubs-ext # mypy -wheel==0.38.4 +wheel==0.43.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements.in/development.txt b/requirements.in/development.txt index bf565de..488d088 100644 --- a/requirements.in/development.txt +++ b/requirements.in/development.txt @@ -1,6 +1,6 @@ coverage pip-tools pywatchman -black +ruff +mypy django-stubs -django<4.0 From 79d28586c5fd8f9c684b0194dc30b3deac4a5482 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 17:51:44 -0400 Subject: [PATCH 03/13] Run ruff format. --- etebase_server/django/models.py | 5 ++-- etebase_server/django/token_auth/models.py | 2 +- etebase_server/django/utils.py | 4 +-- etebase_server/fastapi/db_hack.py | 1 + etebase_server/fastapi/dependencies.py | 9 +++--- etebase_server/fastapi/exceptions.py | 4 +-- etebase_server/fastapi/main.py | 6 ++-- etebase_server/fastapi/msgpack.py | 3 +- etebase_server/fastapi/redis.py | 1 + .../fastapi/routers/authentication.py | 17 ++++++----- etebase_server/fastapi/routers/collection.py | 29 ++++++++++--------- etebase_server/fastapi/routers/invitation.py | 23 ++++++++------- etebase_server/fastapi/routers/member.py | 10 +++---- .../fastapi/routers/test_reset_view.py | 9 +++--- etebase_server/fastapi/routers/websocket.py | 9 +++--- etebase_server/fastapi/sendfile/utils.py | 8 ++--- etebase_server/fastapi/stoken_handler.py | 1 - etebase_server/fastapi/utils.py | 13 ++++----- etebase_server/myauth/admin.py | 13 +++++++-- etebase_server/myauth/forms.py | 1 + etebase_server/myauth/ldap.py | 12 ++++---- etebase_server/myauth/tests.py | 2 -- etebase_server/myauth/views.py | 2 -- etebase_server/settings.py | 5 ++-- etebase_server/urls.py | 5 ---- etebase_server/utils.py | 3 +- manage.py | 1 + 27 files changed, 100 insertions(+), 98 deletions(-) diff --git a/etebase_server/django/models.py b/etebase_server/django/models.py index 336e3b8..0faf708 100644 --- a/etebase_server/django/models.py +++ b/etebase_server/django/models.py @@ -15,17 +15,16 @@ import typing as t from pathlib import Path -from django.db import models, transaction from django.conf import settings from django.core.validators import RegexValidator +from django.db import models, transaction from django.db.models import Max, Value as V from django.db.models.functions import Coalesce, Greatest -from django.utils.functional import cached_property from django.utils.crypto import get_random_string +from django.utils.functional import cached_property from . import app_settings - UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID") diff --git a/etebase_server/django/token_auth/models.py b/etebase_server/django/token_auth/models.py index de2ffc1..6cb143f 100644 --- a/etebase_server/django/token_auth/models.py +++ b/etebase_server/django/token_auth/models.py @@ -1,6 +1,7 @@ from django.db import models from django.utils import timezone from django.utils.crypto import get_random_string + from etebase_server.myauth.models import get_typed_user_model User = get_typed_user_model() @@ -15,7 +16,6 @@ def get_default_expiry(): class AuthToken(models.Model): - key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key) user = models.ForeignKey(User, null=False, blank=False, related_name="auth_token_set", on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) diff --git a/etebase_server/django/utils.py b/etebase_server/django/utils.py index d4aca72..1cf0f27 100644 --- a/etebase_server/django/utils.py +++ b/etebase_server/django/utils.py @@ -1,13 +1,13 @@ import typing as t from dataclasses import dataclass -from django.db.models import QuerySet from django.core.exceptions import PermissionDenied +from django.db.models import QuerySet + from etebase_server.myauth.models import UserType, get_typed_user_model from . import app_settings - User = get_typed_user_model() diff --git a/etebase_server/fastapi/db_hack.py b/etebase_server/fastapi/db_hack.py index 24d5824..979f1d4 100644 --- a/etebase_server/fastapi/db_hack.py +++ b/etebase_server/fastapi/db_hack.py @@ -2,6 +2,7 @@ FIXME: this whole function is a hack around the django db limitations due to how db connections are cached and cleaned. Essentially django assumes there's the django request dispatcher to automatically clean up after the ORM. """ + import typing as t from functools import wraps diff --git a/etebase_server/fastapi/dependencies.py b/etebase_server/fastapi/dependencies.py index b4d5cf4..a54ac97 100644 --- a/etebase_server/fastapi/dependencies.py +++ b/etebase_server/fastapi/dependencies.py @@ -1,18 +1,17 @@ import dataclasses +from django.db.models import QuerySet +from django.utils import timezone from fastapi import Depends from fastapi.security import APIKeyHeader -from django.utils import timezone -from django.db.models import QuerySet - from etebase_server.django import models from etebase_server.django.token_auth.models import AuthToken, get_default_expiry from etebase_server.myauth.models import UserType, get_typed_user_model + +from .db_hack import django_db_cleanup_decorator from .exceptions import AuthenticationFailed from .utils import get_object_or_404 -from .db_hack import django_db_cleanup_decorator - User = get_typed_user_model() token_scheme = APIKeyHeader(name="Authorization") diff --git a/etebase_server/fastapi/exceptions.py b/etebase_server/fastapi/exceptions.py index 1a98fcb..248558e 100644 --- a/etebase_server/fastapi/exceptions.py +++ b/etebase_server/fastapi/exceptions.py @@ -1,8 +1,8 @@ -from fastapi import status, HTTPException import typing as t -from pydantic import BaseModel from django.core.exceptions import ValidationError as DjangoValidationError +from fastapi import HTTPException, status +from pydantic import BaseModel class HttpErrorField(BaseModel): diff --git a/etebase_server/fastapi/main.py b/etebase_server/fastapi/main.py index e4abd6c..335ca1b 100644 --- a/etebase_server/fastapi/main.py +++ b/etebase_server/fastapi/main.py @@ -6,14 +6,12 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware from fastapi.staticfiles import StaticFiles -from etebase_server.django import app_settings - from .exceptions import CustomHttpException from .msgpack import MsgpackResponse from .routers.authentication import authentication_router from .routers.collection import collection_router, item_router -from .routers.member import member_router from .routers.invitation import invitation_incoming_router, invitation_outgoing_router +from .routers.member import member_router from .routers.websocket import websocket_router @@ -24,7 +22,7 @@ def create_application(prefix="", middlewares=[]): externalDocs={ "url": "https://docs.etebase.com", "description": "Docs about the API specifications and clients.", - } + }, # FIXME: version="2.5.0", ) VERSION = "v1" diff --git a/etebase_server/fastapi/msgpack.py b/etebase_server/fastapi/msgpack.py index 3871d15..6b20416 100644 --- a/etebase_server/fastapi/msgpack.py +++ b/etebase_server/fastapi/msgpack.py @@ -5,8 +5,8 @@ from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response -from .utils import msgpack_encode, msgpack_decode from .db_hack import django_db_cleanup_decorator +from .utils import msgpack_decode, msgpack_encode class MsgpackRequest(Request): @@ -60,7 +60,6 @@ class MsgpackRoute(APIRoute): def get_route_handler(self) -> t.Callable: async def custom_route_handler(request: Request) -> Response: - content_type = request.headers.get("Content-Type") try: request_cls = self.REQUESTS_CLASSES[content_type] diff --git a/etebase_server/fastapi/redis.py b/etebase_server/fastapi/redis.py index b4f7a04..8e9f14c 100644 --- a/etebase_server/fastapi/redis.py +++ b/etebase_server/fastapi/redis.py @@ -1,4 +1,5 @@ import typing as t + from redis import asyncio as aioredis from etebase_server.django import app_settings diff --git a/etebase_server/fastapi/routers/authentication.py b/etebase_server/fastapi/routers/authentication.py index d771a5c..e5cfef8 100644 --- a/etebase_server/fastapi/routers/authentication.py +++ b/etebase_server/fastapi/routers/authentication.py @@ -1,5 +1,4 @@ import typing as t -from typing_extensions import Literal from datetime import datetime import nacl @@ -8,22 +7,24 @@ import nacl.hash import nacl.secret import nacl.signing from django.conf import settings -from django.contrib.auth import user_logged_out, user_logged_in +from django.contrib.auth import user_logged_in, user_logged_out from django.core import exceptions as django_exceptions from django.db import transaction from django.utils.functional import cached_property -from fastapi import APIRouter, Depends, status, Request +from fastapi import APIRouter, Depends, Request, status +from typing_extensions import Literal from etebase_server.django import app_settings, models -from etebase_server.django.token_auth.models import AuthToken from etebase_server.django.models import UserInfo from etebase_server.django.signals import user_signed_up -from etebase_server.django.utils import create_user, get_user_queryset, CallbackContext +from etebase_server.django.token_auth.models import AuthToken +from etebase_server.django.utils import CallbackContext, create_user, get_user_queryset from etebase_server.myauth.models import UserType, get_typed_user_model -from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError -from ..msgpack import MsgpackRoute -from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode, get_user_username_email_kwargs + from ..dependencies import AuthData, get_auth_data, get_authenticated_user +from ..exceptions import AuthenticationFailed, HttpError, transform_validation_error +from ..msgpack import MsgpackRoute +from ..utils import BaseModel, get_user_username_email_kwargs, msgpack_decode, msgpack_encode, permission_responses User = get_typed_user_model() authentication_router = APIRouter(route_class=MsgpackRoute) diff --git a/etebase_server/fastapi/routers/collection.py b/etebase_server/fastapi/routers/collection.py index 97dbbf4..3356a6a 100644 --- a/etebase_server/fastapi/routers/collection.py +++ b/etebase_server/fastapi/routers/collection.py @@ -3,33 +3,34 @@ import typing as t from asgiref.sync import sync_to_async from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from django.db.models import Q, QuerySet -from fastapi import APIRouter, Depends, status, Request, BackgroundTasks +from fastapi import APIRouter, BackgroundTasks, Depends, Request, status from etebase_server.django import models from etebase_server.myauth.models import UserType -from .authentication import get_authenticated_user -from .websocket import get_ticket, TicketRequest, TicketOut -from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError + +from ..db_hack import django_db_cleanup_decorator +from ..dependencies import get_collection, get_collection_queryset, get_item_queryset +from ..exceptions import HttpError, PermissionDenied, ValidationError, transform_validation_error from ..msgpack import MsgpackRoute -from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken +from ..redis import redisw +from ..sendfile import sendfile +from ..stoken_handler import filter_by_stoken, filter_by_stoken_and_limit, get_queryset_stoken, get_stoken_obj from ..utils import ( - get_object_or_404, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, + BaseModel, Context, Prefetch, PrefetchQuery, + get_object_or_404, is_collection_admin, msgpack_encode, - BaseModel, permission_responses, - PERMISSIONS_READ, - PERMISSIONS_READWRITE, ) -from ..dependencies import get_collection_queryset, get_item_queryset, get_collection -from ..sendfile import sendfile -from ..redis import redisw -from ..db_hack import django_db_cleanup_decorator +from .authentication import get_authenticated_user +from .websocket import TicketOut, TicketRequest, get_ticket collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) diff --git a/etebase_server/fastapi/routers/invitation.py b/etebase_server/fastapi/routers/invitation.py index adb51c6..ca7ec23 100644 --- a/etebase_server/fastapi/routers/invitation.py +++ b/etebase_server/fastapi/routers/invitation.py @@ -1,26 +1,27 @@ import typing as t -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Request +from fastapi import APIRouter, Depends, Request, status from etebase_server.django import models -from etebase_server.django.utils import get_user_queryset, CallbackContext +from etebase_server.django.utils import CallbackContext, get_user_queryset from etebase_server.myauth.models import UserType, get_typed_user_model -from .authentication import get_authenticated_user + +from ..db_hack import django_db_cleanup_decorator from ..exceptions import HttpError, PermissionDenied from ..msgpack import MsgpackRoute from ..utils import ( - get_object_or_404, - get_user_username_email_kwargs, - Context, - is_collection_admin, - BaseModel, - permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE, + BaseModel, + Context, + get_object_or_404, + get_user_username_email_kwargs, + is_collection_admin, + permission_responses, ) -from ..db_hack import django_db_cleanup_decorator +from .authentication import get_authenticated_user User = get_typed_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) diff --git a/etebase_server/fastapi/routers/member.py b/etebase_server/fastapi/routers/member.py index 123357b..7c6bc51 100644 --- a/etebase_server/fastapi/routers/member.py +++ b/etebase_server/fastapi/routers/member.py @@ -6,12 +6,12 @@ from fastapi import APIRouter, Depends, status from etebase_server.django import models from etebase_server.myauth.models import UserType, get_typed_user_model -from .authentication import get_authenticated_user -from ..msgpack import MsgpackRoute -from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE -from ..stoken_handler import filter_by_stoken_and_limit -from ..db_hack import django_db_cleanup_decorator +from ..db_hack import django_db_cleanup_decorator +from ..msgpack import MsgpackRoute +from ..stoken_handler import filter_by_stoken_and_limit +from ..utils import PERMISSIONS_READ, PERMISSIONS_READWRITE, BaseModel, get_object_or_404, permission_responses +from .authentication import get_authenticated_user from .collection import get_collection, verify_collection_admin User = get_typed_user_model() diff --git a/etebase_server/fastapi/routers/test_reset_view.py b/etebase_server/fastapi/routers/test_reset_view.py index 7895697..1ec60e2 100644 --- a/etebase_server/fastapi/routers/test_reset_view.py +++ b/etebase_server/fastapi/routers/test_reset_view.py @@ -3,12 +3,13 @@ from django.db import transaction from django.shortcuts import get_object_or_404 from fastapi import APIRouter, Request, status -from etebase_server.django.utils import get_user_queryset, CallbackContext -from .authentication import SignupIn, signup_save -from ..msgpack import MsgpackRoute -from ..exceptions import HttpError +from etebase_server.django.utils import CallbackContext, get_user_queryset from etebase_server.myauth.models import get_typed_user_model +from ..exceptions import HttpError +from ..msgpack import MsgpackRoute +from .authentication import SignupIn, signup_save + test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) User = get_typed_user_model() diff --git a/etebase_server/fastapi/routers/websocket.py b/etebase_server/fastapi/routers/websocket.py index 2caf7c6..a434dbd 100644 --- a/etebase_server/fastapi/routers/websocket.py +++ b/etebase_server/fastapi/routers/websocket.py @@ -1,13 +1,13 @@ import asyncio import typing as t -from redis import asyncio as aioredis -from redis.exceptions import ConnectionError +import nacl.encoding +import nacl.utils from asgiref.sync import sync_to_async from django.db.models import QuerySet from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status -import nacl.encoding -import nacl.utils +from redis import asyncio as aioredis +from redis.exceptions import ConnectionError from etebase_server.django import models from etebase_server.django.utils import CallbackContext, get_user_queryset @@ -19,7 +19,6 @@ from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode from ..redis import redisw from ..utils import BaseModel, permission_responses - User = get_typed_user_model() websocket_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) CollectionQuerySet = QuerySet[models.Collection] diff --git a/etebase_server/fastapi/sendfile/utils.py b/etebase_server/fastapi/sendfile/utils.py index c35d6df..c68559f 100644 --- a/etebase_server/fastapi/sendfile/utils.py +++ b/etebase_server/fastapi/sendfile/utils.py @@ -1,14 +1,14 @@ +import logging from functools import lru_cache from importlib import import_module from pathlib import Path, PurePath from urllib.parse import quote -import logging - -from fastapi import status -from ..exceptions import HttpError from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from fastapi import status + +from ..exceptions import HttpError logger = logging.getLogger(__name__) diff --git a/etebase_server/fastapi/stoken_handler.py b/etebase_server/fastapi/stoken_handler.py index b4c7eab..be5646f 100644 --- a/etebase_server/fastapi/stoken_handler.py +++ b/etebase_server/fastapi/stoken_handler.py @@ -47,7 +47,6 @@ def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]: def filter_by_stoken_and_limit( stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation ) -> t.Tuple[list, t.Optional[Stoken], bool]: - queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation) result = list(queryset[: limit + 1]) diff --git a/etebase_server/fastapi/utils.py b/etebase_server/fastapi/utils.py index 334633c..fbba474 100644 --- a/etebase_server/fastapi/utils.py +++ b/etebase_server/fastapi/utils.py @@ -1,14 +1,13 @@ +import base64 import dataclasses import typing as t -from typing_extensions import Literal + import msgpack -import base64 - -from fastapi import status, Query, Depends -from pydantic import BaseModel as PyBaseModel - -from django.db.models import Model, QuerySet from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Model, QuerySet +from fastapi import Depends, Query, status +from pydantic import BaseModel as PyBaseModel +from typing_extensions import Literal from etebase_server.django import app_settings from etebase_server.django.models import AccessLevels diff --git a/etebase_server/myauth/admin.py b/etebase_server/myauth/admin.py index 1f4b767..8da68b4 100644 --- a/etebase_server/myauth/admin.py +++ b/etebase_server/myauth/admin.py @@ -1,12 +1,21 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin as DjangoUserAdmin -from .models import User + from .forms import AdminUserCreationForm +from .models import User class UserAdmin(DjangoUserAdmin): add_form = AdminUserCreationForm - add_fieldsets = ((None, {"classes": ("wide",), "fields": ("username",),}),) + add_fieldsets = ( + ( + None, + { + "classes": ("wide",), + "fields": ("username",), + }, + ), + ) admin.site.register(User, UserAdmin) diff --git a/etebase_server/myauth/forms.py b/etebase_server/myauth/forms.py index 7681835..b745832 100644 --- a/etebase_server/myauth/forms.py +++ b/etebase_server/myauth/forms.py @@ -1,5 +1,6 @@ from django import forms from django.contrib.auth.forms import UsernameField + from etebase_server.myauth.models import get_typed_user_model User = get_typed_user_model() diff --git a/etebase_server/myauth/ldap.py b/etebase_server/myauth/ldap.py index 00de7a6..65a511d 100644 --- a/etebase_server/myauth/ldap.py +++ b/etebase_server/myauth/ldap.py @@ -1,15 +1,15 @@ import logging -from django.utils import timezone +import ldap from django.conf import settings from django.core.exceptions import PermissionDenied as DjangoPermissionDenied -from etebase_server.django.utils import CallbackContext -from etebase_server.myauth.models import get_typed_user_model, UserType -from etebase_server.fastapi.dependencies import get_authenticated_user -from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied +from django.utils import timezone from fastapi import Depends -import ldap +from etebase_server.django.utils import CallbackContext +from etebase_server.fastapi.dependencies import get_authenticated_user +from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied +from etebase_server.myauth.models import UserType, get_typed_user_model User = get_typed_user_model() diff --git a/etebase_server/myauth/tests.py b/etebase_server/myauth/tests.py index 7ce503c..a39b155 100644 --- a/etebase_server/myauth/tests.py +++ b/etebase_server/myauth/tests.py @@ -1,3 +1 @@ -from django.test import TestCase - # Create your tests here. diff --git a/etebase_server/myauth/views.py b/etebase_server/myauth/views.py index 91ea44a..60f00ef 100644 --- a/etebase_server/myauth/views.py +++ b/etebase_server/myauth/views.py @@ -1,3 +1 @@ -from django.shortcuts import render - # Create your views here. diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 1f9bf3e..03c0338 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -10,8 +10,9 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.0/ref/settings/ """ -import os import configparser +import os + from .utils import get_secret_from_file # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -44,7 +45,7 @@ DATABASES = { } } -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # Application definition diff --git a/etebase_server/urls.py b/etebase_server/urls.py index 574ddf7..17ebc0e 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,11 +1,6 @@ -import os - -from django.conf import settings from django.contrib import admin from django.urls import path from django.views.generic import TemplateView -from django.views.static import serve -from django.contrib.staticfiles import finders urlpatterns = [ path("admin/", admin.site.urls), diff --git a/etebase_server/utils.py b/etebase_server/utils.py index 9f56457..1ccbe1e 100644 --- a/etebase_server/utils.py +++ b/etebase_server/utils.py @@ -12,10 +12,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from django.core.management import utils import os import stat +from django.core.management import utils + def get_secret_from_file(path): try: diff --git a/manage.py b/manage.py index a3a8741..f8ee1ea 100755 --- a/manage.py +++ b/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Django's command-line utility for administrative tasks.""" + import os import sys From 0cdab193087a5f82459f0c9008b9f1d04b378eba Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 17:56:36 -0400 Subject: [PATCH 04/13] Fix rust complaints. --- etebase_server/django/__init__.py | 2 ++ etebase_server/django/app_settings_inner.py | 16 ++++++++-------- etebase_server/django/models.py | 4 ++-- etebase_server/fastapi/main.py | 6 +++--- etebase_server/fastapi/routers/collection.py | 2 +- etebase_server/myauth/models.py | 12 ++---------- etebase_server/settings.py | 2 +- pyproject.toml | 2 +- 8 files changed, 20 insertions(+), 26 deletions(-) diff --git a/etebase_server/django/__init__.py b/etebase_server/django/__init__.py index 99ee8b6..d321c95 100644 --- a/etebase_server/django/__init__.py +++ b/etebase_server/django/__init__.py @@ -1 +1,3 @@ from .app_settings_inner import app_settings + +__all__ = ["app_settings"] diff --git a/etebase_server/django/app_settings_inner.py b/etebase_server/django/app_settings_inner.py index 41fd910..9adf120 100644 --- a/etebase_server/django/app_settings_inner.py +++ b/etebase_server/django/app_settings_inner.py @@ -34,11 +34,11 @@ class AppSettings: return getattr(settings, self.prefix + name, dflt) @cached_property - def REDIS_URI(self) -> t.Optional[str]: # pylint: disable=invalid-name + def REDIS_URI(self) -> t.Optional[str]: # noqa: N802 return self._setting("REDIS_URI", None) @cached_property - def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name + def API_PERMISSIONS_READ(self): # noqa: N802 perms = self._setting("API_PERMISSIONS_READ", tuple()) ret = [] for perm in perms: @@ -46,7 +46,7 @@ class AppSettings: return ret @cached_property - def API_PERMISSIONS_WRITE(self): # pylint: disable=invalid-name + def API_PERMISSIONS_WRITE(self): # noqa: N802 perms = self._setting("API_PERMISSIONS_WRITE", tuple()) ret = [] for perm in perms: @@ -54,35 +54,35 @@ class AppSettings: return ret @cached_property - def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name + def GET_USER_QUERYSET_FUNC(self): # noqa: N802 get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None) if get_user_queryset is not None: return self.import_from_str(get_user_queryset) return None @cached_property - def CREATE_USER_FUNC(self): # pylint: disable=invalid-name + def CREATE_USER_FUNC(self): # noqa: N802 func = self._setting("CREATE_USER_FUNC", None) if func is not None: return self.import_from_str(func) return None @cached_property - def DASHBOARD_URL_FUNC(self): # pylint: disable=invalid-name + def DASHBOARD_URL_FUNC(self): # noqa: N802 func = self._setting("DASHBOARD_URL_FUNC", None) if func is not None: return self.import_from_str(func) return None @cached_property - def CHUNK_PATH_FUNC(self): # pylint: disable=invalid-name + def CHUNK_PATH_FUNC(self): # noqa: N802 func = self._setting("CHUNK_PATH_FUNC", None) if func is not None: return self.import_from_str(func) return None @cached_property - def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name + def CHALLENGE_VALID_SECONDS(self): # noqa: N802 return self._setting("CHALLENGE_VALID_SECONDS", 60) diff --git a/etebase_server/django/models.py b/etebase_server/django/models.py index 0faf708..4e6e5bd 100644 --- a/etebase_server/django/models.py +++ b/etebase_server/django/models.py @@ -18,7 +18,7 @@ from pathlib import Path from django.conf import settings from django.core.validators import RegexValidator from django.db import models, transaction -from django.db.models import Max, Value as V +from django.db.models import Max, Value as Val from django.db.models.functions import Coalesce, Greatest from django.utils.crypto import get_random_string from django.utils.functional import cached_property @@ -29,7 +29,7 @@ UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a val def stoken_annotation_builder(stoken_id_fields: t.List[str]): - aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields] + aggr_fields = [Coalesce(Max(field), Val(0)) for field in stoken_id_fields] return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] diff --git a/etebase_server/fastapi/main.py b/etebase_server/fastapi/main.py index 335ca1b..9e80dba 100644 --- a/etebase_server/fastapi/main.py +++ b/etebase_server/fastapi/main.py @@ -25,9 +25,9 @@ def create_application(prefix="", middlewares=[]): }, # FIXME: version="2.5.0", ) - VERSION = "v1" - BASE_PATH = f"{prefix}/api/{VERSION}" - COLLECTION_UID_MARKER = "{collection_uid}" + VERSION = "v1" # noqa: N806 + BASE_PATH = f"{prefix}/api/{VERSION}" # noqa: N806 + COLLECTION_UID_MARKER = "{collection_uid}" # noqa: N806 app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) diff --git a/etebase_server/fastapi/routers/collection.py b/etebase_server/fastapi/routers/collection.py index 3356a6a..b293f03 100644 --- a/etebase_server/fastapi/routers/collection.py +++ b/etebase_server/fastapi/routers/collection.py @@ -374,7 +374,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val revision_data = item_model.content uid = item_model.uid - Model = models.CollectionItem + Model = models.CollectionItem # noqa: N806 with transaction.atomic(): instance, created = Model.objects.get_or_create( diff --git a/etebase_server/myauth/models.py b/etebase_server/myauth/models.py index 89b94b4..c862a46 100644 --- a/etebase_server/myauth/models.py +++ b/etebase_server/myauth/models.py @@ -1,21 +1,13 @@ import typing as t from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager -from django.core import validators +from django.contrib.auth.validators import UnicodeUsernameValidator from django.db import models -from django.utils.deconstruct import deconstructible from django.utils.translation import gettext_lazy as _ -@deconstructible -class UnicodeUsernameValidator(validators.RegexValidator): - regex = r"^[\w.-]+\Z" - message = _("Enter a valid username. This value may contain only letters, " "numbers, and ./-/_ characters.") - flags = 0 - - class UserManager(DjangoUserManager): - def get_by_natural_key(self, username: str): + def get_by_natural_key(self, username: t.Optional[str]): return self.get(**{self.model.USERNAME_FIELD + "__iexact": username}) diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 03c0338..fc486fe 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -201,7 +201,7 @@ if "DJANGO_MEDIA_ROOT" in os.environ: # Make an `etebase_server_settings` module available to override settings. try: - from etebase_server_settings import * + from etebase_server_settings import * # noqa: F403 except ImportError: pass diff --git a/pyproject.toml b/pyproject.toml index 02fad2c..22072d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ exclude = [ [tool.ruff.lint] select = ["E", "F", "I", "N", "T20", "W"] -ignore = ["E203", "E501", "E711", "E712", "N803", "N818", "T201"] +ignore = ["E203", "E501", "E711", "E712", "N803", "N815", "N818", "T201"] [tool.ruff.lint.isort] combine-as-imports = true From fb9cc701d02fc898a8ec61f5acfcc1fceda50cd0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 18:10:34 -0400 Subject: [PATCH 05/13] Adjust pydantic code to v2. --- etebase_server/fastapi/exceptions.py | 4 ++-- etebase_server/fastapi/routers/authentication.py | 2 +- etebase_server/fastapi/routers/collection.py | 4 ++-- etebase_server/fastapi/routers/invitation.py | 4 ++-- etebase_server/fastapi/routers/member.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/etebase_server/fastapi/exceptions.py b/etebase_server/fastapi/exceptions.py index 248558e..2fbb9a1 100644 --- a/etebase_server/fastapi/exceptions.py +++ b/etebase_server/fastapi/exceptions.py @@ -11,7 +11,7 @@ class HttpErrorField(BaseModel): detail: str class Config: - orm_mode = True + from_attributes = True class HttpErrorOut(BaseModel): @@ -20,7 +20,7 @@ class HttpErrorOut(BaseModel): errors: t.Optional[t.List[HttpErrorField]] class Config: - orm_mode = True + from_attributes = True class CustomHttpException(HTTPException): diff --git a/etebase_server/fastapi/routers/authentication.py b/etebase_server/fastapi/routers/authentication.py index e5cfef8..5f1d8bb 100644 --- a/etebase_server/fastapi/routers/authentication.py +++ b/etebase_server/fastapi/routers/authentication.py @@ -76,7 +76,7 @@ class LoginOut(BaseModel): class Authentication(BaseModel): class Config: - keep_untouched = (cached_property,) + ignored_types= (cached_property,) response: bytes signature: bytes diff --git a/etebase_server/fastapi/routers/collection.py b/etebase_server/fastapi/routers/collection.py index b293f03..04fb8cd 100644 --- a/etebase_server/fastapi/routers/collection.py +++ b/etebase_server/fastapi/routers/collection.py @@ -52,7 +52,7 @@ class CollectionItemRevisionInOut(BaseModel): chunks: t.List[ChunkType] class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm_context( @@ -78,7 +78,7 @@ class CollectionItemCommon(BaseModel): class CollectionItemOut(CollectionItemCommon): class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm_context( diff --git a/etebase_server/fastapi/routers/invitation.py b/etebase_server/fastapi/routers/invitation.py index ca7ec23..b5d841b 100644 --- a/etebase_server/fastapi/routers/invitation.py +++ b/etebase_server/fastapi/routers/invitation.py @@ -34,7 +34,7 @@ class UserInfoOut(BaseModel): pubkey: bytes class Config: - orm_mode = True + from_attributes= True @classmethod def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut": @@ -67,7 +67,7 @@ class CollectionInvitationOut(CollectionInvitationCommon): fromPubkey: bytes class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut": diff --git a/etebase_server/fastapi/routers/member.py b/etebase_server/fastapi/routers/member.py index 7c6bc51..e913fbf 100644 --- a/etebase_server/fastapi/routers/member.py +++ b/etebase_server/fastapi/routers/member.py @@ -39,7 +39,7 @@ class CollectionMemberOut(BaseModel): accessLevel: models.AccessLevels class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut": From 49eeeefef5af2816907b06a7cb2921a93216f023 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 19:28:37 -0400 Subject: [PATCH 06/13] Make fastapi 0.104 and pydantic 2.0 min requirements --- requirements.in/base.txt | 3 ++- requirements.txt | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 6842b02..37de777 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -1,7 +1,8 @@ django>=4.0,<5.0 msgpack pynacl -fastapi +fastapi>=0.104 +pydantic>=2.0.0 typing_extensions uvicorn[standard] aiofiles diff --git a/requirements.txt b/requirements.txt index 2c736b0..a0c480a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,7 +65,9 @@ orjson==3.10.3 pycparser==2.22 # via cffi pydantic==2.7.3 - # via fastapi + # via + # -r requirements.in/base.txt + # fastapi pydantic-core==2.18.4 # via pydantic pygments==2.18.0 From 0d9c9f153dcb93521dbae4f2c20eadcbabca1639 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 18:17:59 -0400 Subject: [PATCH 07/13] Type fix. --- etebase_server/fastapi/msgpack.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/etebase_server/fastapi/msgpack.py b/etebase_server/fastapi/msgpack.py index 6b20416..94a4fa5 100644 --- a/etebase_server/fastapi/msgpack.py +++ b/etebase_server/fastapi/msgpack.py @@ -61,12 +61,13 @@ class MsgpackRoute(APIRoute): def get_route_handler(self) -> t.Callable: async def custom_route_handler(request: Request) -> Response: content_type = request.headers.get("Content-Type") - try: - request_cls = self.REQUESTS_CLASSES[content_type] - request = request_cls(request.scope, request.receive) - except KeyError: - # nothing registered to handle content_type, process given requests as-is - pass + if content_type is not None: + try: + request_cls = self.REQUESTS_CLASSES[content_type] + request = request_cls(request.scope, request.receive) + except KeyError: + # nothing registered to handle content_type, process given requests as-is + pass accept = request.headers.get("Accept") route_handler = self._get_media_type_route_handler(accept) From b9f2cea951e09846cfbc88a922476bb7055ff540 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 18:33:29 -0400 Subject: [PATCH 08/13] more --- etebase_server/fastapi/msgpack.py | 2 +- etebase_server/fastapi/utils.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/etebase_server/fastapi/msgpack.py b/etebase_server/fastapi/msgpack.py index 94a4fa5..496a75a 100644 --- a/etebase_server/fastapi/msgpack.py +++ b/etebase_server/fastapi/msgpack.py @@ -27,7 +27,7 @@ class MsgpackResponse(Response): return b"" if isinstance(content, BaseModel): - content = content.dict() + content = content.model_dump() return msgpack_encode(content) diff --git a/etebase_server/fastapi/utils.py b/etebase_server/fastapi/utils.py index fbba474..0b28875 100644 --- a/etebase_server/fastapi/utils.py +++ b/etebase_server/fastapi/utils.py @@ -25,10 +25,7 @@ T = t.TypeVar("T", bound=Model, covariant=True) class BaseModel(PyBaseModel): - class Config: - json_encoders = { - bytes: lambda x: x, - } + pass @dataclasses.dataclass From 57e676baa1d5a595ae0cba118c1613c0dfaec31b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 19:52:10 -0400 Subject: [PATCH 09/13] Adjust to fastapi changes. --- etebase_server/fastapi/msgpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_server/fastapi/msgpack.py b/etebase_server/fastapi/msgpack.py index 496a75a..d857b16 100644 --- a/etebase_server/fastapi/msgpack.py +++ b/etebase_server/fastapi/msgpack.py @@ -48,7 +48,7 @@ class MsgpackRoute(APIRoute): status_code=self.status_code, # use custom response class or fallback on default self.response_class response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class), - response_field=self.secure_cloned_response_field, + response_field=self.response_field, response_model_include=self.response_model_include, response_model_exclude=self.response_model_exclude, response_model_by_alias=self.response_model_by_alias, From 0be14a7b0e49ff7afaac52e5defe360ea0456779 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 20:17:02 -0400 Subject: [PATCH 10/13] Fixes for fastapi. --- etebase_server/fastapi/msgpack.py | 5 +- .../fastapi/routers/authentication.py | 12 ++--- etebase_server/fastapi/routers/collection.py | 51 ++++++++++++------- etebase_server/fastapi/routers/invitation.py | 14 ++--- etebase_server/fastapi/routers/member.py | 12 +++-- 5 files changed, 56 insertions(+), 38 deletions(-) diff --git a/etebase_server/fastapi/msgpack.py b/etebase_server/fastapi/msgpack.py index d857b16..9820852 100644 --- a/etebase_server/fastapi/msgpack.py +++ b/etebase_server/fastapi/msgpack.py @@ -12,9 +12,12 @@ from .utils import msgpack_decode, msgpack_encode class MsgpackRequest(Request): media_type = "application/msgpack" + async def raw_body(self) -> bytes: + return await super().body() + async def body(self) -> bytes: if not hasattr(self, "_json"): - body = await super().body() + body = await self.raw_body() self._json = msgpack_decode(body) return self._json diff --git a/etebase_server/fastapi/routers/authentication.py b/etebase_server/fastapi/routers/authentication.py index 5f1d8bb..533b0eb 100644 --- a/etebase_server/fastapi/routers/authentication.py +++ b/etebase_server/fastapi/routers/authentication.py @@ -23,7 +23,7 @@ from etebase_server.myauth.models import UserType, get_typed_user_model from ..dependencies import AuthData, get_auth_data, get_authenticated_user from ..exceptions import AuthenticationFailed, HttpError, transform_validation_error -from ..msgpack import MsgpackRoute +from ..msgpack import MsgpackResponse, MsgpackRoute from ..utils import BaseModel, get_user_username_email_kwargs, msgpack_decode, msgpack_encode, permission_responses User = get_typed_user_model() @@ -76,7 +76,7 @@ class LoginOut(BaseModel): class Authentication(BaseModel): class Config: - ignored_types= (cached_property,) + ignored_types = (cached_property,) response: bytes signature: bytes @@ -188,7 +188,7 @@ def login_challenge(user: UserType = Depends(get_login_user)): "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version) + return MsgpackResponse(LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version)) @authentication_router.post("/login/", response_model=LoginOut) @@ -198,7 +198,7 @@ def login(data: Login, request: Request): validate_login_request(data.response_data, data, user, "login", host) ret = LoginOut.from_orm(user) user_logged_in.send(sender=user.__class__, request=None, user=user) - return ret + return MsgpackResponse(ret) @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) @@ -223,7 +223,7 @@ def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_u ret = { "url": get_dashboard_url(CallbackContext(request.path_params, user=user)), } - return ret + return MsgpackResponse(ret) def signup_save(data: SignupIn, request: Request) -> UserType: @@ -261,4 +261,4 @@ def signup(data: SignupIn, request: Request): user = signup_save(data, request) ret = LoginOut.from_orm(user) user_signed_up.send(sender=user.__class__, request=None, user=user) - return ret + return MsgpackResponse(ret) diff --git a/etebase_server/fastapi/routers/collection.py b/etebase_server/fastapi/routers/collection.py index 04fb8cd..9fe74f1 100644 --- a/etebase_server/fastapi/routers/collection.py +++ b/etebase_server/fastapi/routers/collection.py @@ -13,7 +13,7 @@ from etebase_server.myauth.models import UserType from ..db_hack import django_db_cleanup_decorator from ..dependencies import get_collection, get_collection_queryset, get_item_queryset from ..exceptions import HttpError, PermissionDenied, ValidationError, transform_validation_error -from ..msgpack import MsgpackRoute +from ..msgpack import MsgpackRequest, MsgpackResponse, MsgpackRoute from ..redis import redisw from ..sendfile import sendfile from ..stoken_handler import filter_by_stoken, filter_by_stoken_and_limit, get_queryset_stoken, get_stoken_obj @@ -135,7 +135,7 @@ class CollectionListResponse(BaseModel): stoken: t.Optional[str] done: bool - removedMemberships: t.Optional[t.List[RemovedMembershipOut]] + removedMemberships: t.Optional[t.List[RemovedMembershipOut]] = None class CollectionItemListResponse(BaseModel): @@ -275,7 +275,7 @@ def list_multi( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - return collection_list_common(queryset, user, stoken, limit, prefetch) + return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch)) @collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ) @@ -286,7 +286,7 @@ def collection_list( user: UserType = Depends(get_authenticated_user), queryset: CollectionQuerySet = Depends(get_collection_queryset), ): - return collection_list_common(queryset, user, stoken, limit, prefetch) + return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch)) def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): @@ -365,7 +365,7 @@ def collection_get( user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - return CollectionOut.from_orm_context(obj, Context(user, prefetch)) + return MsgpackResponse(CollectionOut.from_orm_context(obj, Context(user, prefetch))) def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool): @@ -418,7 +418,7 @@ def item_get( prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) - return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) + return MsgpackResponse(CollectionItemOut.from_orm_context(obj, Context(user, prefetch))) def item_list_common( @@ -450,7 +450,7 @@ def item_list( queryset = queryset.filter(parent__isnull=True) response = item_list_common(queryset, user, stoken, limit, prefetch) - return response + return MsgpackResponse(response) @item_router.post("/item/subscription-ticket/", response_model=TicketOut, dependencies=PERMISSIONS_READ) @@ -459,7 +459,7 @@ async def item_list_subscription_ticket( user: UserType = Depends(get_authenticated_user), ): """Get an authentication ticket that can be used with the websocket endpoint""" - return await get_ticket(TicketRequest(collection=collection.uid), user) + return MsgpackResponse(await get_ticket(TicketRequest(collection=collection.uid), user)) def item_bulk_common( @@ -527,10 +527,12 @@ def item_revisions( ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result] iterator = ret_data[-1].uid if len(result) > 0 else None - return CollectionItemRevisionListResponse( - data=ret_data, - iterator=iterator, - done=done, + return MsgpackResponse( + CollectionItemRevisionListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) ) @@ -560,10 +562,12 @@ def fetch_updates( new_stoken = new_stoken or stoken_rev_uid context = Context(user, prefetch) - return CollectionItemListResponse( - data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], - stoken=new_stoken, - done=True, # we always return all the items, so it's always done + return MsgpackResponse( + CollectionItemListResponse( + data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], + stoken=new_stoken, + done=True, # we always return all the items, so it's always done + ) ) @@ -575,7 +579,9 @@ def item_transaction( stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user), ): - return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks) + return MsgpackResponse( + item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks) + ) @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) @@ -586,7 +592,9 @@ def item_batch( stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user), ): - return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks) + return MsgpackResponse( + item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks) + ) # Chunks @@ -611,7 +619,12 @@ async def chunk_update( collection: models.Collection = Depends(get_collection), ): # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) - content_file = ContentFile(await request.body()) + if isinstance(request, MsgpackRequest): + body = await request.raw_body() + else: + body = await request.body() + + content_file = ContentFile(body) try: await chunk_save(chunk_uid, collection, content_file) except IntegrityError: diff --git a/etebase_server/fastapi/routers/invitation.py b/etebase_server/fastapi/routers/invitation.py index b5d841b..43a182f 100644 --- a/etebase_server/fastapi/routers/invitation.py +++ b/etebase_server/fastapi/routers/invitation.py @@ -10,7 +10,7 @@ from etebase_server.myauth.models import UserType, get_typed_user_model from ..db_hack import django_db_cleanup_decorator from ..exceptions import HttpError, PermissionDenied -from ..msgpack import MsgpackRoute +from ..msgpack import MsgpackResponse, MsgpackRoute from ..utils import ( PERMISSIONS_READ, PERMISSIONS_READWRITE, @@ -34,7 +34,7 @@ class UserInfoOut(BaseModel): pubkey: bytes class Config: - from_attributes= True + from_attributes = True @classmethod def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut": @@ -121,7 +121,7 @@ def list_common( iterator = ret_data[-1].uid if len(result) > 0 else None return InvitationListResponse( - data=ret_data, + data=[CollectionInvitationOut.from_orm(x) for x in ret_data], iterator=iterator, done=done, ) @@ -133,7 +133,7 @@ def incoming_list( limit: int = 50, queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): - return list_common(queryset, iterator, limit) + return MsgpackResponse(list_common(queryset, iterator, limit)) @invitation_incoming_router.get( @@ -144,7 +144,7 @@ def incoming_get( queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): obj = get_object_or_404(queryset, uid=invitation_uid) - return CollectionInvitationOut.from_orm(obj) + return MsgpackResponse(CollectionInvitationOut.from_orm(obj)) @invitation_incoming_router.delete( @@ -219,7 +219,7 @@ def outgoing_list( limit: int = 50, queryset: InvitationQuerySet = Depends(get_outgoing_queryset), ): - return list_common(queryset, iterator, limit) + return MsgpackResponse(list_common(queryset, iterator, limit)) @invitation_outgoing_router.delete( @@ -242,4 +242,4 @@ def outgoing_fetch_user_profile( kwargs = get_user_username_email_kwargs(username) user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) - return UserInfoOut.from_orm(user_info) + return MsgpackResponse(UserInfoOut.from_orm(user_info)) diff --git a/etebase_server/fastapi/routers/member.py b/etebase_server/fastapi/routers/member.py index e913fbf..71da4c7 100644 --- a/etebase_server/fastapi/routers/member.py +++ b/etebase_server/fastapi/routers/member.py @@ -8,7 +8,7 @@ from etebase_server.django import models from etebase_server.myauth.models import UserType, get_typed_user_model from ..db_hack import django_db_cleanup_decorator -from ..msgpack import MsgpackRoute +from ..msgpack import MsgpackResponse, MsgpackRoute from ..stoken_handler import filter_by_stoken_and_limit from ..utils import PERMISSIONS_READ, PERMISSIONS_READWRITE, BaseModel, get_object_or_404, permission_responses from .authentication import get_authenticated_user @@ -66,10 +66,12 @@ def member_list( ) new_stoken = new_stoken_obj and new_stoken_obj.uid - return MemberListResponse( - data=[CollectionMemberOut.from_orm(item) for item in result], - iterator=new_stoken, - done=done, + return MsgpackResponse( + MemberListResponse( + data=[CollectionMemberOut.from_orm(item) for item in result], + iterator=new_stoken, + done=done, + ) ) From 138d99dd7ffc1e10cbe1274936c073402b56e18b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 20:27:13 -0400 Subject: [PATCH 11/13] Update code to adjust to most recent python/fastapi. --- etebase_server/fastapi/sendfile/utils.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/etebase_server/fastapi/sendfile/utils.py b/etebase_server/fastapi/sendfile/utils.py index c68559f..22e8266 100644 --- a/etebase_server/fastapi/sendfile/utils.py +++ b/etebase_server/fastapi/sendfile/utils.py @@ -1,4 +1,5 @@ import logging +import os from functools import lru_cache from importlib import import_module from pathlib import Path, PurePath @@ -32,9 +33,7 @@ def _convert_file_to_url(path): path_obj = PurePath(path) relpath = path_obj.relative_to(path_root) - # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an - # already instantiated Path object - url = relpath._flavour.pathmod.normpath(str(url_root / relpath)) + url = os.path.normpath(str(url_root / relpath)) return quote(str(url)) @@ -48,9 +47,7 @@ def _sanitize_path(filepath): filepath_obj = Path(filepath) # get absolute path - # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an - # already instantiated Path object - filepath_abs = Path(filepath_obj._flavour.pathmod.normpath(str(path_root / filepath_obj))) + filepath_abs = Path(os.path.normpath(str(path_root / filepath_obj))) # if filepath_abs is not relative to path_root, relative_to throws an error try: From a27ce2f4d0ece74392b5632a7da400a79ecd903e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 20:41:01 -0400 Subject: [PATCH 12/13] Also handle 422 as msgpack. --- etebase_server/fastapi/main.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/etebase_server/fastapi/main.py b/etebase_server/fastapi/main.py index 9e80dba..5d87a84 100644 --- a/etebase_server/fastapi/main.py +++ b/etebase_server/fastapi/main.py @@ -1,7 +1,9 @@ from django.conf import settings # Not at the top of the file because we first need to setup django -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, status +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware from fastapi.staticfiles import StaticFiles @@ -73,6 +75,13 @@ def create_application(prefix="", middlewares=[]): async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + @app.exception_handler(RequestValidationError) + async def validation_exception_handler(request: Request, exc: RequestValidationError): + return MsgpackResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content=jsonable_encoder({"detail": exc.errors()}), + ) + app.mount(settings.STATIC_URL, StaticFiles(directory=settings.STATIC_ROOT), name="static") return app From d7075c01696c9b9aff30ac8bc9caafcbdf356da0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 8 Jun 2024 20:49:13 -0400 Subject: [PATCH 13/13] Mark optional field as optional. --- etebase_server/fastapi/routers/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_server/fastapi/routers/collection.py b/etebase_server/fastapi/routers/collection.py index 9fe74f1..2de35a8 100644 --- a/etebase_server/fastapi/routers/collection.py +++ b/etebase_server/fastapi/routers/collection.py @@ -152,7 +152,7 @@ class CollectionItemRevisionListResponse(BaseModel): class CollectionItemBulkGetIn(BaseModel): uid: str - etag: t.Optional[str] + etag: t.Optional[str] = None class ItemDepIn(BaseModel):