Merge PR #184: Upgrade FastAPI and Pydantic to most recent versions

etebase_server/django/__init__.py

@@ -1 +1,3 @@
 from .app_settings_inner import app_settings
+
+__all__ = ["app_settings"]

etebase_server/django/app_settings_inner.py

@@ -34,11 +34,11 @@ class AppSettings:
         return getattr(settings, self.prefix + name, dflt)
 
     @cached_property
-    def REDIS_URI(self) -> t.Optional[str]:  # pylint: disable=invalid-name
+    def REDIS_URI(self) -> t.Optional[str]:  # noqa: N802
         return self._setting("REDIS_URI", None)
 
     @cached_property
-    def API_PERMISSIONS_READ(self):  # pylint: disable=invalid-name
+    def API_PERMISSIONS_READ(self):  # noqa: N802
         perms = self._setting("API_PERMISSIONS_READ", tuple())
         ret = []
         for perm in perms:
@@ -46,7 +46,7 @@ class AppSettings:
         return ret
 
     @cached_property
-    def API_PERMISSIONS_WRITE(self):  # pylint: disable=invalid-name
+    def API_PERMISSIONS_WRITE(self):  # noqa: N802
         perms = self._setting("API_PERMISSIONS_WRITE", tuple())
         ret = []
         for perm in perms:
@@ -54,35 +54,35 @@ class AppSettings:
         return ret
 
     @cached_property
-    def GET_USER_QUERYSET_FUNC(self):  # pylint: disable=invalid-name
+    def GET_USER_QUERYSET_FUNC(self):  # noqa: N802
         get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None)
         if get_user_queryset is not None:
             return self.import_from_str(get_user_queryset)
         return None
 
     @cached_property
-    def CREATE_USER_FUNC(self):  # pylint: disable=invalid-name
+    def CREATE_USER_FUNC(self):  # noqa: N802
         func = self._setting("CREATE_USER_FUNC", None)
         if func is not None:
             return self.import_from_str(func)
         return None
 
     @cached_property
-    def DASHBOARD_URL_FUNC(self):  # pylint: disable=invalid-name
+    def DASHBOARD_URL_FUNC(self):  # noqa: N802
         func = self._setting("DASHBOARD_URL_FUNC", None)
         if func is not None:
             return self.import_from_str(func)
         return None
 
     @cached_property
-    def CHUNK_PATH_FUNC(self):  # pylint: disable=invalid-name
+    def CHUNK_PATH_FUNC(self):  # noqa: N802
         func = self._setting("CHUNK_PATH_FUNC", None)
         if func is not None:
             return self.import_from_str(func)
         return None
 
     @cached_property
-    def CHALLENGE_VALID_SECONDS(self):  # pylint: disable=invalid-name
+    def CHALLENGE_VALID_SECONDS(self):  # noqa: N802
         return self._setting("CHALLENGE_VALID_SECONDS", 60)
 
 

etebase_server/django/models.py

@@ -15,22 +15,21 @@
 import typing as t
 from pathlib import Path
 
-from django.db import models, transaction
 from django.conf import settings
 from django.core.validators import RegexValidator
-from django.db.models import Max, Value as V
+from django.db import models, transaction
+from django.db.models import Max, Value as Val
 from django.db.models.functions import Coalesce, Greatest
-from django.utils.functional import cached_property
 from django.utils.crypto import get_random_string
+from django.utils.functional import cached_property
 
 from . import app_settings
 
-
 UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID")
 
 
 def stoken_annotation_builder(stoken_id_fields: t.List[str]):
-    aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields]
+    aggr_fields = [Coalesce(Max(field), Val(0)) for field in stoken_id_fields]
     return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0]
 
 

etebase_server/django/token_auth/models.py

@@ -1,6 +1,7 @@
 from django.db import models
 from django.utils import timezone
 from django.utils.crypto import get_random_string
+
 from etebase_server.myauth.models import get_typed_user_model
 
 User = get_typed_user_model()
@@ -15,7 +16,6 @@ def get_default_expiry():
 
 
 class AuthToken(models.Model):
-
     key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key)
     user = models.ForeignKey(User, null=False, blank=False, related_name="auth_token_set", on_delete=models.CASCADE)
     created = models.DateTimeField(auto_now_add=True)

etebase_server/django/utils.py

@@ -1,13 +1,13 @@
 import typing as t
 from dataclasses import dataclass
 
-from django.db.models import QuerySet
 from django.core.exceptions import PermissionDenied
+from django.db.models import QuerySet
+
 from etebase_server.myauth.models import UserType, get_typed_user_model
 
 from . import app_settings
 
-
 User = get_typed_user_model()
 
 

etebase_server/fastapi/db_hack.py

@@ -2,6 +2,7 @@
 FIXME: this whole function is a hack around the django db limitations due to how db connections are cached and cleaned.
 Essentially django assumes there's the django request dispatcher to automatically clean up after the ORM.
 """
+
 import typing as t
 from functools import wraps
 

etebase_server/fastapi/dependencies.py

@@ -1,18 +1,17 @@
 import dataclasses
 
+from django.db.models import QuerySet
+from django.utils import timezone
 from fastapi import Depends
 from fastapi.security import APIKeyHeader
 
-from django.utils import timezone
-from django.db.models import QuerySet
-
 from etebase_server.django import models
 from etebase_server.django.token_auth.models import AuthToken, get_default_expiry
 from etebase_server.myauth.models import UserType, get_typed_user_model
+
+from .db_hack import django_db_cleanup_decorator
 from .exceptions import AuthenticationFailed
 from .utils import get_object_or_404
-from .db_hack import django_db_cleanup_decorator
-
 
 User = get_typed_user_model()
 token_scheme = APIKeyHeader(name="Authorization")

etebase_server/fastapi/exceptions.py

@@ -1,8 +1,8 @@
-from fastapi import status, HTTPException
 import typing as t
 
-from pydantic import BaseModel
 from django.core.exceptions import ValidationError as DjangoValidationError
+from fastapi import HTTPException, status
+from pydantic import BaseModel
 
 
 class HttpErrorField(BaseModel):
@@ -11,7 +11,7 @@ class HttpErrorField(BaseModel):
     detail: str
 
     class Config:
-        orm_mode = True
+        from_attributes = True
 
 
 class HttpErrorOut(BaseModel):
@@ -20,7 +20,7 @@ class HttpErrorOut(BaseModel):
     errors: t.Optional[t.List[HttpErrorField]]
 
     class Config:
-        orm_mode = True
+        from_attributes = True
 
 
 class CustomHttpException(HTTPException):

etebase_server/fastapi/main.py

@@ -1,19 +1,19 @@
 from django.conf import settings
 
 # Not at the top of the file because we first need to setup django
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, Request, status
+from fastapi.encoders import jsonable_encoder
+from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.trustedhost import TrustedHostMiddleware
 from fastapi.staticfiles import StaticFiles
 
-from etebase_server.django import app_settings
-
 from .exceptions import CustomHttpException
 from .msgpack import MsgpackResponse
 from .routers.authentication import authentication_router
 from .routers.collection import collection_router, item_router
-from .routers.member import member_router
 from .routers.invitation import invitation_incoming_router, invitation_outgoing_router
+from .routers.member import member_router
 from .routers.websocket import websocket_router
 
 
@@ -24,12 +24,12 @@ def create_application(prefix="", middlewares=[]):
         externalDocs={
             "url": "https://docs.etebase.com",
             "description": "Docs about the API specifications and clients.",
-        }
+        },
         # FIXME: version="2.5.0",
     )
-    VERSION = "v1"
-    BASE_PATH = f"{prefix}/api/{VERSION}"
-    COLLECTION_UID_MARKER = "{collection_uid}"
+    VERSION = "v1"  # noqa: N806
+    BASE_PATH = f"{prefix}/api/{VERSION}"  # noqa: N806
+    COLLECTION_UID_MARKER = "{collection_uid}"  # noqa: N806
     app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"])
     app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"])
     app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"])
@@ -75,6 +75,13 @@ def create_application(prefix="", middlewares=[]):
     async def custom_exception_handler(request: Request, exc: CustomHttpException):
         return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict)
 
+    @app.exception_handler(RequestValidationError)
+    async def validation_exception_handler(request: Request, exc: RequestValidationError):
+        return MsgpackResponse(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            content=jsonable_encoder({"detail": exc.errors()}),
+        )
+
     app.mount(settings.STATIC_URL, StaticFiles(directory=settings.STATIC_ROOT), name="static")
 
     return app

etebase_server/fastapi/msgpack.py

@@ -5,16 +5,19 @@ from pydantic import BaseModel
 from starlette.requests import Request
 from starlette.responses import Response
 
-from .utils import msgpack_encode, msgpack_decode
 from .db_hack import django_db_cleanup_decorator
+from .utils import msgpack_decode, msgpack_encode
 
 
 class MsgpackRequest(Request):
     media_type = "application/msgpack"
 
+    async def raw_body(self) -> bytes:
+        return await super().body()
+
     async def body(self) -> bytes:
         if not hasattr(self, "_json"):
-            body = await super().body()
+            body = await self.raw_body()
             self._json = msgpack_decode(body)
         return self._json
 
@@ -27,7 +30,7 @@ class MsgpackResponse(Response):
             return b""
 
         if isinstance(content, BaseModel):
-            content = content.dict()
+            content = content.model_dump()
         return msgpack_encode(content)
 
 
@@ -48,7 +51,7 @@ class MsgpackRoute(APIRoute):
             status_code=self.status_code,
             # use custom response class or fallback on default self.response_class
             response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class),
-            response_field=self.secure_cloned_response_field,
+            response_field=self.response_field,
             response_model_include=self.response_model_include,
             response_model_exclude=self.response_model_exclude,
             response_model_by_alias=self.response_model_by_alias,
@@ -60,14 +63,14 @@ class MsgpackRoute(APIRoute):
 
     def get_route_handler(self) -> t.Callable:
         async def custom_route_handler(request: Request) -> Response:
-
             content_type = request.headers.get("Content-Type")
-            try:
-                request_cls = self.REQUESTS_CLASSES[content_type]
-                request = request_cls(request.scope, request.receive)
-            except KeyError:
-                # nothing registered to handle content_type, process given requests as-is
-                pass
+            if content_type is not None:
+                try:
+                    request_cls = self.REQUESTS_CLASSES[content_type]
+                    request = request_cls(request.scope, request.receive)
+                except KeyError:
+                    # nothing registered to handle content_type, process given requests as-is
+                    pass
 
             accept = request.headers.get("Accept")
             route_handler = self._get_media_type_route_handler(accept)

etebase_server/fastapi/redis.py

@@ -1,4 +1,5 @@
 import typing as t
+
 from redis import asyncio as aioredis
 
 from etebase_server.django import app_settings

etebase_server/fastapi/routers/authentication.py

@@ -1,5 +1,4 @@
 import typing as t
-from typing_extensions import Literal
 from datetime import datetime
 
 import nacl
@@ -8,22 +7,24 @@ import nacl.hash
 import nacl.secret
 import nacl.signing
 from django.conf import settings
-from django.contrib.auth import user_logged_out, user_logged_in
+from django.contrib.auth import user_logged_in, user_logged_out
 from django.core import exceptions as django_exceptions
 from django.db import transaction
 from django.utils.functional import cached_property
-from fastapi import APIRouter, Depends, status, Request
+from fastapi import APIRouter, Depends, Request, status
+from typing_extensions import Literal
 
 from etebase_server.django import app_settings, models
-from etebase_server.django.token_auth.models import AuthToken
 from etebase_server.django.models import UserInfo
 from etebase_server.django.signals import user_signed_up
-from etebase_server.django.utils import create_user, get_user_queryset, CallbackContext
+from etebase_server.django.token_auth.models import AuthToken
+from etebase_server.django.utils import CallbackContext, create_user, get_user_queryset
 from etebase_server.myauth.models import UserType, get_typed_user_model
-from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError
-from ..msgpack import MsgpackRoute
-from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode, get_user_username_email_kwargs
+
 from ..dependencies import AuthData, get_auth_data, get_authenticated_user
+from ..exceptions import AuthenticationFailed, HttpError, transform_validation_error
+from ..msgpack import MsgpackResponse, MsgpackRoute
+from ..utils import BaseModel, get_user_username_email_kwargs, msgpack_decode, msgpack_encode, permission_responses
 
 User = get_typed_user_model()
 authentication_router = APIRouter(route_class=MsgpackRoute)
@@ -75,7 +76,7 @@ class LoginOut(BaseModel):
 
 class Authentication(BaseModel):
     class Config:
-        keep_untouched = (cached_property,)
+        ignored_types = (cached_property,)
 
     response: bytes
     signature: bytes
@@ -187,7 +188,7 @@ def login_challenge(user: UserType = Depends(get_login_user)):
         "userId": user.id,
     }
     challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder))
-    return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version)
+    return MsgpackResponse(LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version))
 
 
 @authentication_router.post("/login/", response_model=LoginOut)
@@ -197,7 +198,7 @@ def login(data: Login, request: Request):
     validate_login_request(data.response_data, data, user, "login", host)
     ret = LoginOut.from_orm(user)
     user_logged_in.send(sender=user.__class__, request=None, user=user)
-    return ret
+    return MsgpackResponse(ret)
 
 
 @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses)
@@ -222,7 +223,7 @@ def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_u
     ret = {
         "url": get_dashboard_url(CallbackContext(request.path_params, user=user)),
     }
-    return ret
+    return MsgpackResponse(ret)
 
 
 def signup_save(data: SignupIn, request: Request) -> UserType:
@@ -260,4 +261,4 @@ def signup(data: SignupIn, request: Request):
     user = signup_save(data, request)
     ret = LoginOut.from_orm(user)
     user_signed_up.send(sender=user.__class__, request=None, user=user)
-    return ret
+    return MsgpackResponse(ret)

etebase_server/fastapi/routers/collection.py

@@ -3,33 +3,34 @@ import typing as t
 from asgiref.sync import sync_to_async
 from django.core import exceptions as django_exceptions
 from django.core.files.base import ContentFile
-from django.db import transaction, IntegrityError
+from django.db import IntegrityError, transaction
 from django.db.models import Q, QuerySet
-from fastapi import APIRouter, Depends, status, Request, BackgroundTasks
+from fastapi import APIRouter, BackgroundTasks, Depends, Request, status
 
 from etebase_server.django import models
 from etebase_server.myauth.models import UserType
-from .authentication import get_authenticated_user
-from .websocket import get_ticket, TicketRequest, TicketOut
-from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError
-from ..msgpack import MsgpackRoute
-from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken
+
+from ..db_hack import django_db_cleanup_decorator
+from ..dependencies import get_collection, get_collection_queryset, get_item_queryset
+from ..exceptions import HttpError, PermissionDenied, ValidationError, transform_validation_error
+from ..msgpack import MsgpackRequest, MsgpackResponse, MsgpackRoute
+from ..redis import redisw
+from ..sendfile import sendfile
+from ..stoken_handler import filter_by_stoken, filter_by_stoken_and_limit, get_queryset_stoken, get_stoken_obj
 from ..utils import (
-    get_object_or_404,
+    PERMISSIONS_READ,
+    PERMISSIONS_READWRITE,
+    BaseModel,
     Context,
     Prefetch,
     PrefetchQuery,
+    get_object_or_404,
     is_collection_admin,
     msgpack_encode,
-    BaseModel,
     permission_responses,
-    PERMISSIONS_READ,
-    PERMISSIONS_READWRITE,
 )
-from ..dependencies import get_collection_queryset, get_item_queryset, get_collection
-from ..sendfile import sendfile
-from ..redis import redisw
-from ..db_hack import django_db_cleanup_decorator
+from .authentication import get_authenticated_user
+from .websocket import TicketOut, TicketRequest, get_ticket
 
 collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
 item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@@ -51,7 +52,7 @@ class CollectionItemRevisionInOut(BaseModel):
     chunks: t.List[ChunkType]
 
     class Config:
-        orm_mode = True
+        from_attributes = True
 
     @classmethod
     def from_orm_context(
@@ -77,7 +78,7 @@ class CollectionItemCommon(BaseModel):
 
 class CollectionItemOut(CollectionItemCommon):
     class Config:
-        orm_mode = True
+        from_attributes = True
 
     @classmethod
     def from_orm_context(
@@ -134,7 +135,7 @@ class CollectionListResponse(BaseModel):
     stoken: t.Optional[str]
     done: bool
 
-    removedMemberships: t.Optional[t.List[RemovedMembershipOut]]
+    removedMemberships: t.Optional[t.List[RemovedMembershipOut]] = None
 
 
 class CollectionItemListResponse(BaseModel):
@@ -151,7 +152,7 @@ class CollectionItemRevisionListResponse(BaseModel):
 
 class CollectionItemBulkGetIn(BaseModel):
     uid: str
-    etag: t.Optional[str]
+    etag: t.Optional[str] = None
 
 
 class ItemDepIn(BaseModel):
@@ -274,7 +275,7 @@ def list_multi(
         Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True)
     )
 
-    return collection_list_common(queryset, user, stoken, limit, prefetch)
+    return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
 
 
 @collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ)
@@ -285,7 +286,7 @@ def collection_list(
     user: UserType = Depends(get_authenticated_user),
     queryset: CollectionQuerySet = Depends(get_collection_queryset),
 ):
-    return collection_list_common(queryset, user, stoken, limit, prefetch)
+    return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
 
 
 def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut):
@@ -364,7 +365,7 @@ def collection_get(
     user: UserType = Depends(get_authenticated_user),
     prefetch: Prefetch = PrefetchQuery,
 ):
-    return CollectionOut.from_orm_context(obj, Context(user, prefetch))
+    return MsgpackResponse(CollectionOut.from_orm_context(obj, Context(user, prefetch)))
 
 
 def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool):
@@ -373,7 +374,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val
     revision_data = item_model.content
     uid = item_model.uid
 
-    Model = models.CollectionItem
+    Model = models.CollectionItem  # noqa: N806
 
     with transaction.atomic():
         instance, created = Model.objects.get_or_create(
@@ -417,7 +418,7 @@ def item_get(
     prefetch: Prefetch = PrefetchQuery,
 ):
     obj = queryset.get(uid=item_uid)
-    return CollectionItemOut.from_orm_context(obj, Context(user, prefetch))
+    return MsgpackResponse(CollectionItemOut.from_orm_context(obj, Context(user, prefetch)))
 
 
 def item_list_common(
@@ -449,7 +450,7 @@ def item_list(
         queryset = queryset.filter(parent__isnull=True)
 
     response = item_list_common(queryset, user, stoken, limit, prefetch)
-    return response
+    return MsgpackResponse(response)
 
 
 @item_router.post("/item/subscription-ticket/", response_model=TicketOut, dependencies=PERMISSIONS_READ)
@@ -458,7 +459,7 @@ async def item_list_subscription_ticket(
     user: UserType = Depends(get_authenticated_user),
 ):
     """Get an authentication ticket that can be used with the websocket endpoint"""
-    return await get_ticket(TicketRequest(collection=collection.uid), user)
+    return MsgpackResponse(await get_ticket(TicketRequest(collection=collection.uid), user))
 
 
 def item_bulk_common(
@@ -526,10 +527,12 @@ def item_revisions(
     ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result]
     iterator = ret_data[-1].uid if len(result) > 0 else None
 
-    return CollectionItemRevisionListResponse(
-        data=ret_data,
-        iterator=iterator,
-        done=done,
+    return MsgpackResponse(
+        CollectionItemRevisionListResponse(
+            data=ret_data,
+            iterator=iterator,
+            done=done,
+        )
     )
 
 
@@ -559,10 +562,12 @@ def fetch_updates(
     new_stoken = new_stoken or stoken_rev_uid
 
     context = Context(user, prefetch)
-    return CollectionItemListResponse(
-        data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
-        stoken=new_stoken,
-        done=True,  # we always return all the items, so it's always done
+    return MsgpackResponse(
+        CollectionItemListResponse(
+            data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
+            stoken=new_stoken,
+            done=True,  # we always return all the items, so it's always done
+        )
     )
 
 
@@ -574,7 +579,9 @@ def item_transaction(
     stoken: t.Optional[str] = None,
     user: UserType = Depends(get_authenticated_user),
 ):
-    return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
+    return MsgpackResponse(
+        item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
+    )
 
 
 @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE])
@@ -585,7 +592,9 @@ def item_batch(
     stoken: t.Optional[str] = None,
     user: UserType = Depends(get_authenticated_user),
 ):
-    return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
+    return MsgpackResponse(
+        item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
+    )
 
 
 # Chunks
@@ -610,7 +619,12 @@ async def chunk_update(
     collection: models.Collection = Depends(get_collection),
 ):
     # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid)
-    content_file = ContentFile(await request.body())
+    if isinstance(request, MsgpackRequest):
+        body = await request.raw_body()
+    else:
+        body = await request.body()
+
+    content_file = ContentFile(body)
     try:
         await chunk_save(chunk_uid, collection, content_file)
     except IntegrityError:

etebase_server/fastapi/routers/invitation.py

@@ -1,26 +1,27 @@
 import typing as t
 
-from django.db import transaction, IntegrityError
+from django.db import IntegrityError, transaction
 from django.db.models import QuerySet
-from fastapi import APIRouter, Depends, status, Request
+from fastapi import APIRouter, Depends, Request, status
 
 from etebase_server.django import models
-from etebase_server.django.utils import get_user_queryset, CallbackContext
+from etebase_server.django.utils import CallbackContext, get_user_queryset
 from etebase_server.myauth.models import UserType, get_typed_user_model
-from .authentication import get_authenticated_user
+
+from ..db_hack import django_db_cleanup_decorator
 from ..exceptions import HttpError, PermissionDenied
-from ..msgpack import MsgpackRoute
+from ..msgpack import MsgpackResponse, MsgpackRoute
 from ..utils import (
-    get_object_or_404,
-    get_user_username_email_kwargs,
-    Context,
-    is_collection_admin,
-    BaseModel,
-    permission_responses,
     PERMISSIONS_READ,
     PERMISSIONS_READWRITE,
+    BaseModel,
+    Context,
+    get_object_or_404,
+    get_user_username_email_kwargs,
+    is_collection_admin,
+    permission_responses,
 )
-from ..db_hack import django_db_cleanup_decorator
+from .authentication import get_authenticated_user
 
 User = get_typed_user_model()
 invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@@ -33,7 +34,7 @@ class UserInfoOut(BaseModel):
     pubkey: bytes
 
     class Config:
-        orm_mode = True
+        from_attributes = True
 
     @classmethod
     def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut":
@@ -66,7 +67,7 @@ class CollectionInvitationOut(CollectionInvitationCommon):
     fromPubkey: bytes
 
     class Config:
-        orm_mode = True
+        from_attributes = True
 
     @classmethod
     def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut":
@@ -120,7 +121,7 @@ def list_common(
     iterator = ret_data[-1].uid if len(result) > 0 else None
 
     return InvitationListResponse(
-        data=ret_data,
+        data=[CollectionInvitationOut.from_orm(x) for x in ret_data],
         iterator=iterator,
         done=done,
     )
@@ -132,7 +133,7 @@ def incoming_list(
     limit: int = 50,
     queryset: InvitationQuerySet = Depends(get_incoming_queryset),
 ):
-    return list_common(queryset, iterator, limit)
+    return MsgpackResponse(list_common(queryset, iterator, limit))
 
 
 @invitation_incoming_router.get(
@@ -143,7 +144,7 @@ def incoming_get(
     queryset: InvitationQuerySet = Depends(get_incoming_queryset),
 ):
     obj = get_object_or_404(queryset, uid=invitation_uid)
-    return CollectionInvitationOut.from_orm(obj)
+    return MsgpackResponse(CollectionInvitationOut.from_orm(obj))
 
 
 @invitation_incoming_router.delete(
@@ -218,7 +219,7 @@ def outgoing_list(
     limit: int = 50,
     queryset: InvitationQuerySet = Depends(get_outgoing_queryset),
 ):
-    return list_common(queryset, iterator, limit)
+    return MsgpackResponse(list_common(queryset, iterator, limit))
 
 
 @invitation_outgoing_router.delete(
@@ -241,4 +242,4 @@ def outgoing_fetch_user_profile(
     kwargs = get_user_username_email_kwargs(username)
     user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs)
     user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user)
-    return UserInfoOut.from_orm(user_info)
+    return MsgpackResponse(UserInfoOut.from_orm(user_info))

etebase_server/fastapi/routers/member.py

@@ -6,12 +6,12 @@ from fastapi import APIRouter, Depends, status
 
 from etebase_server.django import models
 from etebase_server.myauth.models import UserType, get_typed_user_model
-from .authentication import get_authenticated_user
-from ..msgpack import MsgpackRoute
-from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE
-from ..stoken_handler import filter_by_stoken_and_limit
-from ..db_hack import django_db_cleanup_decorator
 
+from ..db_hack import django_db_cleanup_decorator
+from ..msgpack import MsgpackResponse, MsgpackRoute
+from ..stoken_handler import filter_by_stoken_and_limit
+from ..utils import PERMISSIONS_READ, PERMISSIONS_READWRITE, BaseModel, get_object_or_404, permission_responses
+from .authentication import get_authenticated_user
 from .collection import get_collection, verify_collection_admin
 
 User = get_typed_user_model()
@@ -39,7 +39,7 @@ class CollectionMemberOut(BaseModel):
     accessLevel: models.AccessLevels
 
     class Config:
-        orm_mode = True
+        from_attributes = True
 
     @classmethod
     def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut":
@@ -66,10 +66,12 @@ def member_list(
     )
     new_stoken = new_stoken_obj and new_stoken_obj.uid
 
-    return MemberListResponse(
-        data=[CollectionMemberOut.from_orm(item) for item in result],
-        iterator=new_stoken,
-        done=done,
+    return MsgpackResponse(
+        MemberListResponse(
+            data=[CollectionMemberOut.from_orm(item) for item in result],
+            iterator=new_stoken,
+            done=done,
+        )
     )
 
 

etebase_server/fastapi/routers/test_reset_view.py

@@ -3,12 +3,13 @@ from django.db import transaction
 from django.shortcuts import get_object_or_404
 from fastapi import APIRouter, Request, status
 
-from etebase_server.django.utils import get_user_queryset, CallbackContext
-from .authentication import SignupIn, signup_save
-from ..msgpack import MsgpackRoute
-from ..exceptions import HttpError
+from etebase_server.django.utils import CallbackContext, get_user_queryset
 from etebase_server.myauth.models import get_typed_user_model
 
+from ..exceptions import HttpError
+from ..msgpack import MsgpackRoute
+from .authentication import SignupIn, signup_save
+
 test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"])
 User = get_typed_user_model()
 

etebase_server/fastapi/routers/websocket.py

@@ -1,13 +1,13 @@
 import asyncio
 import typing as t
 
-from redis import asyncio as aioredis
-from redis.exceptions import ConnectionError
+import nacl.encoding
+import nacl.utils
 from asgiref.sync import sync_to_async
 from django.db.models import QuerySet
 from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status
-import nacl.encoding
-import nacl.utils
+from redis import asyncio as aioredis
+from redis.exceptions import ConnectionError
 
 from etebase_server.django import models
 from etebase_server.django.utils import CallbackContext, get_user_queryset
@@ -19,7 +19,6 @@ from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode
 from ..redis import redisw
 from ..utils import BaseModel, permission_responses
 
-
 User = get_typed_user_model()
 websocket_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
 CollectionQuerySet = QuerySet[models.Collection]

etebase_server/fastapi/sendfile/utils.py

@@ -1,14 +1,15 @@
+import logging
+import os
 from functools import lru_cache
 from importlib import import_module
 from pathlib import Path, PurePath
 from urllib.parse import quote
-import logging
-
-from fastapi import status
-from ..exceptions import HttpError
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
+from fastapi import status
+
+from ..exceptions import HttpError
 
 logger = logging.getLogger(__name__)
 
@@ -32,9 +33,7 @@ def _convert_file_to_url(path):
     path_obj = PurePath(path)
 
     relpath = path_obj.relative_to(path_root)
-    # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
-    # already instantiated Path object
-    url = relpath._flavour.pathmod.normpath(str(url_root / relpath))
+    url = os.path.normpath(str(url_root / relpath))
 
     return quote(str(url))
 
@@ -48,9 +47,7 @@ def _sanitize_path(filepath):
     filepath_obj = Path(filepath)
 
     # get absolute path
-    # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
-    # already instantiated Path object
-    filepath_abs = Path(filepath_obj._flavour.pathmod.normpath(str(path_root / filepath_obj)))
+    filepath_abs = Path(os.path.normpath(str(path_root / filepath_obj)))
 
     # if filepath_abs is not relative to path_root, relative_to throws an error
     try:

etebase_server/fastapi/stoken_handler.py

@@ -47,7 +47,6 @@ def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]:
 def filter_by_stoken_and_limit(
     stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation
 ) -> t.Tuple[list, t.Optional[Stoken], bool]:
-
     queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation)
 
     result = list(queryset[: limit + 1])

etebase_server/fastapi/utils.py

@@ -1,14 +1,13 @@
+import base64
 import dataclasses
 import typing as t
-from typing_extensions import Literal
+
 import msgpack
-import base64
-
-from fastapi import status, Query, Depends
-from pydantic import BaseModel as PyBaseModel
-
-from django.db.models import Model, QuerySet
 from django.core.exceptions import ObjectDoesNotExist
+from django.db.models import Model, QuerySet
+from fastapi import Depends, Query, status
+from pydantic import BaseModel as PyBaseModel
+from typing_extensions import Literal
 
 from etebase_server.django import app_settings
 from etebase_server.django.models import AccessLevels
@@ -26,10 +25,7 @@ T = t.TypeVar("T", bound=Model, covariant=True)
 
 
 class BaseModel(PyBaseModel):
-    class Config:
-        json_encoders = {
-            bytes: lambda x: x,
-        }
+    pass
 
 
 @dataclasses.dataclass

etebase_server/myauth/admin.py

@@ -1,12 +1,21 @@
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin as DjangoUserAdmin
-from .models import User
+
 from .forms import AdminUserCreationForm
+from .models import User
 
 
 class UserAdmin(DjangoUserAdmin):
     add_form = AdminUserCreationForm
-    add_fieldsets = ((None, {"classes": ("wide",), "fields": ("username",),}),)
+    add_fieldsets = (
+        (
+            None,
+            {
+                "classes": ("wide",),
+                "fields": ("username",),
+            },
+        ),
+    )
 
 
 admin.site.register(User, UserAdmin)

etebase_server/myauth/forms.py

@@ -1,5 +1,6 @@
 from django import forms
 from django.contrib.auth.forms import UsernameField
+
 from etebase_server.myauth.models import get_typed_user_model
 
 User = get_typed_user_model()

etebase_server/myauth/ldap.py

@@ -1,15 +1,15 @@
 import logging
 
-from django.utils import timezone
+import ldap
 from django.conf import settings
 from django.core.exceptions import PermissionDenied as DjangoPermissionDenied
-from etebase_server.django.utils import CallbackContext
-from etebase_server.myauth.models import get_typed_user_model, UserType
-from etebase_server.fastapi.dependencies import get_authenticated_user
-from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied
+from django.utils import timezone
 from fastapi import Depends
 
-import ldap
+from etebase_server.django.utils import CallbackContext
+from etebase_server.fastapi.dependencies import get_authenticated_user
+from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied
+from etebase_server.myauth.models import UserType, get_typed_user_model
 
 User = get_typed_user_model()
 

etebase_server/myauth/models.py

@@ -1,21 +1,13 @@
 import typing as t
 
 from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager
-from django.core import validators
+from django.contrib.auth.validators import UnicodeUsernameValidator
 from django.db import models
-from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _
 
 
-@deconstructible
-class UnicodeUsernameValidator(validators.RegexValidator):
-    regex = r"^[\w.-]+\Z"
-    message = _("Enter a valid username. This value may contain only letters, " "numbers, and ./-/_ characters.")
-    flags = 0
-
-
 class UserManager(DjangoUserManager):
-    def get_by_natural_key(self, username: str):
+    def get_by_natural_key(self, username: t.Optional[str]):
         return self.get(**{self.model.USERNAME_FIELD + "__iexact": username})
 
 

etebase_server/myauth/tests.py

@@ -1,3 +1 @@
-from django.test import TestCase
-
 # Create your tests here.

etebase_server/myauth/views.py

@@ -1,3 +1 @@
-from django.shortcuts import render
-
 # Create your views here.

etebase_server/settings.py

@@ -10,8 +10,9 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.0/ref/settings/
 """
 
-import os
 import configparser
+import os
+
 from .utils import get_secret_from_file
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -44,7 +45,7 @@ DATABASES = {
     }
 }
 
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 # Application definition
 
@@ -200,7 +201,7 @@ if "DJANGO_MEDIA_ROOT" in os.environ:
 
 # Make an `etebase_server_settings` module available to override settings.
 try:
-    from etebase_server_settings import *
+    from etebase_server_settings import *  # noqa: F403
 except ImportError:
     pass
 

etebase_server/urls.py

@@ -1,11 +1,6 @@
-import os
-
-from django.conf import settings
 from django.contrib import admin
 from django.urls import path
 from django.views.generic import TemplateView
-from django.views.static import serve
-from django.contrib.staticfiles import finders
 
 urlpatterns = [
     path("admin/", admin.site.urls),

etebase_server/utils.py

@@ -12,10 +12,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-from django.core.management import utils
 import os
 import stat
 
+from django.core.management import utils
+
 
 def get_secret_from_file(path):
     try:

manage.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Django's command-line utility for administrative tasks."""
+
 import os
 import sys
 

pyproject.toml

@@ -4,3 +4,25 @@ line-length = 120
 [build-system]
 requires = ["setuptools>=42"]
 build-backend = "setuptools.build_meta"
+
+[tool.ruff]
+line-length = 120
+exclude = [
+    ".git",
+    ".git-rewrite",
+    ".mypy_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".venv",
+    "build",
+    "dist",
+    "node_modules",
+    "migrations", # Alembic migrations
+]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "T20", "W"]
+ignore = ["E203", "E501", "E711", "E712", "N803", "N815", "N818", "T201"]
+
+[tool.ruff.lint.isort]
+combine-as-imports = true

requirements-dev.txt

@@ -1,66 +1,53 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile --output-file=requirements-dev.txt requirements.in/development.txt
 #
-asgiref==3.5.2
-    # via django
-black==22.10.0
-    # via -r requirements.in/development.txt
-build==0.9.0
+asgiref==3.8.1
+    # via
+    #   django
+    #   django-stubs
+build==1.2.1
     # via pip-tools
-click==8.1.3
+click==8.1.7
+    # via pip-tools
+coverage==7.5.3
+    # via -r requirements.in/development.txt
+django==5.0.6
     # via
-    #   black
+    #   django-stubs
+    #   django-stubs-ext
+django-stubs==5.0.2
+    # via -r requirements.in/development.txt
+django-stubs-ext==5.0.2
+    # via django-stubs
+mypy==1.10.0
+    # via -r requirements.in/development.txt
+mypy-extensions==1.0.0
+    # via mypy
+packaging==24.0
+    # via build
+pip-tools==7.4.1
+    # via -r requirements.in/development.txt
+pyproject-hooks==1.1.0
+    # via
+    #   build
     #   pip-tools
-coverage==6.5.0
+pywatchman==2.0.0
     # via -r requirements.in/development.txt
-django==3.2.16
-    # via
-    #   -r requirements.in/development.txt
-    #   django-stubs
-    #   django-stubs-ext
-django-stubs==1.13.0
+ruff==0.4.8
     # via -r requirements.in/development.txt
-django-stubs-ext==0.7.0
-    # via django-stubs
-mypy==0.991
-    # via django-stubs
-mypy-extensions==0.4.3
-    # via
-    #   black
-    #   mypy
-packaging==21.3
-    # via build
-pathspec==0.10.2
-    # via black
-pep517==0.13.0
-    # via build
-pip-tools==6.11.0
-    # via -r requirements.in/development.txt
-platformdirs==2.6.0
-    # via black
-pyparsing==3.0.9
-    # via packaging
-pytz==2022.6
+sqlparse==0.5.0
     # via django
-pywatchman==1.4.1
-    # via -r requirements.in/development.txt
-sqlparse==0.4.3
-    # via django
-tomli==2.0.1
+types-pyyaml==6.0.12.20240311
     # via django-stubs
-types-pytz==2022.6.0.1
-    # via django-stubs
-types-pyyaml==6.0.12.2
-    # via django-stubs
-typing-extensions==4.4.0
+typing-extensions==4.12.2
     # via
     #   django-stubs
     #   django-stubs-ext
     #   mypy
-wheel==0.38.4
+wheel==0.43.0
     # via pip-tools
 
 # The following packages are considered to be unsafe in a requirements file:

requirements.in/base.txt

@@ -1,7 +1,8 @@
 django>=4.0,<5.0
 msgpack
 pynacl
-fastapi
+fastapi>=0.104
+pydantic>=2.0.0
 typing_extensions
 uvicorn[standard]
 aiofiles

requirements.in/development.txt

@@ -1,6 +1,6 @@
 coverage
 pip-tools
 pywatchman
-black
+ruff
+mypy
 django-stubs
-django<4.0

requirements.txt

@@ -4,59 +4,114 @@
 #
 #    pip-compile --output-file=requirements.txt requirements.in/base.txt
 #
-aiofiles==22.1.0
+aiofiles==23.2.1
     # via -r requirements.in/base.txt
-anyio==3.6.2
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.4.0
     # via
+    #   httpx
     #   starlette
     #   watchfiles
-asgiref==3.5.2
+asgiref==3.8.1
     # via django
-async-timeout==4.0.2
-    # via redis
-cffi==1.15.1
+certifi==2024.6.2
+    # via
+    #   httpcore
+    #   httpx
+cffi==1.16.0
     # via pynacl
-click==8.1.3
-    # via uvicorn
-django==4.1.13
+click==8.1.7
+    # via
+    #   typer
+    #   uvicorn
+django==4.2.13
     # via -r requirements.in/base.txt
-fastapi==0.88.0
+dnspython==2.6.1
+    # via email-validator
+email-validator==2.1.1
+    # via fastapi
+fastapi==0.111.0
     # via -r requirements.in/base.txt
+fastapi-cli==0.0.4
+    # via fastapi
 h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+httpcore==1.0.5
+    # via httpx
+httptools==0.6.1
     # via uvicorn
-httptools==0.5.0
-    # via uvicorn
-idna==3.4
-    # via anyio
-msgpack==1.0.4
+httpx==0.27.0
+    # via fastapi
+idna==3.7
+    # via
+    #   anyio
+    #   email-validator
+    #   httpx
+jinja2==3.1.4
+    # via fastapi
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+msgpack==1.0.8
     # via -r requirements.in/base.txt
-pycparser==2.21
+orjson==3.10.3
+    # via fastapi
+pycparser==2.22
     # via cffi
-pydantic==1.10.2
-    # via fastapi
-pynacl==1.5.0
-    # via -r requirements.in/base.txt
-python-dotenv==0.21.0
-    # via uvicorn
-pyyaml==6.0.1
-    # via uvicorn
-redis==4.4.0
-    # via -r requirements.in/base.txt
-sniffio==1.3.0
-    # via anyio
-sqlparse==0.4.3
-    # via django
-starlette==0.22.0
-    # via fastapi
-typing-extensions==4.4.0
+pydantic==2.7.3
     # via
     #   -r requirements.in/base.txt
-    #   pydantic
-uvicorn[standard]==0.20.0
+    #   fastapi
+pydantic-core==2.18.4
+    # via pydantic
+pygments==2.18.0
+    # via rich
+pynacl==1.5.0
     # via -r requirements.in/base.txt
-uvloop==0.17.0
+python-dotenv==1.0.1
     # via uvicorn
-watchfiles==0.18.1
+python-multipart==0.0.9
+    # via fastapi
+pyyaml==6.0.1
     # via uvicorn
-websockets==10.4
+redis==5.1.0b6
+    # via -r requirements.in/base.txt
+rich==13.7.1
+    # via typer
+shellingham==1.5.4
+    # via typer
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+sqlparse==0.5.0
+    # via django
+starlette==0.37.2
+    # via fastapi
+typer==0.12.3
+    # via fastapi-cli
+typing-extensions==4.12.2
+    # via
+    #   -r requirements.in/base.txt
+    #   fastapi
+    #   pydantic
+    #   pydantic-core
+    #   typer
+ujson==5.10.0
+    # via fastapi
+uvicorn[standard]==0.30.1
+    # via
+    #   -r requirements.in/base.txt
+    #   fastapi
+uvloop==0.19.0
+    # via uvicorn
+watchfiles==0.22.0
+    # via uvicorn
+websockets==12.0
     # via uvicorn