diff --git a/etebase_server/django/__init__.py b/etebase_server/django/__init__.py index 99ee8b6..d321c95 100644 --- a/etebase_server/django/__init__.py +++ b/etebase_server/django/__init__.py @@ -1 +1,3 @@ from .app_settings_inner import app_settings + +__all__ = ["app_settings"] diff --git a/etebase_server/django/app_settings_inner.py b/etebase_server/django/app_settings_inner.py index 41fd910..9adf120 100644 --- a/etebase_server/django/app_settings_inner.py +++ b/etebase_server/django/app_settings_inner.py @@ -34,11 +34,11 @@ class AppSettings: return getattr(settings, self.prefix + name, dflt) @cached_property - def REDIS_URI(self) -> t.Optional[str]: # pylint: disable=invalid-name + def REDIS_URI(self) -> t.Optional[str]: # noqa: N802 return self._setting("REDIS_URI", None) @cached_property - def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name + def API_PERMISSIONS_READ(self): # noqa: N802 perms = self._setting("API_PERMISSIONS_READ", tuple()) ret = [] for perm in perms: @@ -46,7 +46,7 @@ class AppSettings: return ret @cached_property - def API_PERMISSIONS_WRITE(self): # pylint: disable=invalid-name + def API_PERMISSIONS_WRITE(self): # noqa: N802 perms = self._setting("API_PERMISSIONS_WRITE", tuple()) ret = [] for perm in perms: @@ -54,35 +54,35 @@ class AppSettings: return ret @cached_property - def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name + def GET_USER_QUERYSET_FUNC(self): # noqa: N802 get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None) if get_user_queryset is not None: return self.import_from_str(get_user_queryset) return None @cached_property - def CREATE_USER_FUNC(self): # pylint: disable=invalid-name + def CREATE_USER_FUNC(self): # noqa: N802 func = self._setting("CREATE_USER_FUNC", None) if func is not None: return self.import_from_str(func) return None @cached_property - def DASHBOARD_URL_FUNC(self): # pylint: disable=invalid-name + def DASHBOARD_URL_FUNC(self): # noqa: N802 func = self._setting("DASHBOARD_URL_FUNC", None) if func is not None: return self.import_from_str(func) return None @cached_property - def CHUNK_PATH_FUNC(self): # pylint: disable=invalid-name + def CHUNK_PATH_FUNC(self): # noqa: N802 func = self._setting("CHUNK_PATH_FUNC", None) if func is not None: return self.import_from_str(func) return None @cached_property - def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name + def CHALLENGE_VALID_SECONDS(self): # noqa: N802 return self._setting("CHALLENGE_VALID_SECONDS", 60) diff --git a/etebase_server/django/models.py b/etebase_server/django/models.py index 336e3b8..4e6e5bd 100644 --- a/etebase_server/django/models.py +++ b/etebase_server/django/models.py @@ -15,22 +15,21 @@ import typing as t from pathlib import Path -from django.db import models, transaction from django.conf import settings from django.core.validators import RegexValidator -from django.db.models import Max, Value as V +from django.db import models, transaction +from django.db.models import Max, Value as Val from django.db.models.functions import Coalesce, Greatest -from django.utils.functional import cached_property from django.utils.crypto import get_random_string +from django.utils.functional import cached_property from . import app_settings - UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID") def stoken_annotation_builder(stoken_id_fields: t.List[str]): - aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields] + aggr_fields = [Coalesce(Max(field), Val(0)) for field in stoken_id_fields] return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] diff --git a/etebase_server/django/token_auth/models.py b/etebase_server/django/token_auth/models.py index de2ffc1..6cb143f 100644 --- a/etebase_server/django/token_auth/models.py +++ b/etebase_server/django/token_auth/models.py @@ -1,6 +1,7 @@ from django.db import models from django.utils import timezone from django.utils.crypto import get_random_string + from etebase_server.myauth.models import get_typed_user_model User = get_typed_user_model() @@ -15,7 +16,6 @@ def get_default_expiry(): class AuthToken(models.Model): - key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key) user = models.ForeignKey(User, null=False, blank=False, related_name="auth_token_set", on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) diff --git a/etebase_server/django/utils.py b/etebase_server/django/utils.py index d4aca72..1cf0f27 100644 --- a/etebase_server/django/utils.py +++ b/etebase_server/django/utils.py @@ -1,13 +1,13 @@ import typing as t from dataclasses import dataclass -from django.db.models import QuerySet from django.core.exceptions import PermissionDenied +from django.db.models import QuerySet + from etebase_server.myauth.models import UserType, get_typed_user_model from . import app_settings - User = get_typed_user_model() diff --git a/etebase_server/fastapi/db_hack.py b/etebase_server/fastapi/db_hack.py index 24d5824..979f1d4 100644 --- a/etebase_server/fastapi/db_hack.py +++ b/etebase_server/fastapi/db_hack.py @@ -2,6 +2,7 @@ FIXME: this whole function is a hack around the django db limitations due to how db connections are cached and cleaned. Essentially django assumes there's the django request dispatcher to automatically clean up after the ORM. """ + import typing as t from functools import wraps diff --git a/etebase_server/fastapi/dependencies.py b/etebase_server/fastapi/dependencies.py index b4d5cf4..a54ac97 100644 --- a/etebase_server/fastapi/dependencies.py +++ b/etebase_server/fastapi/dependencies.py @@ -1,18 +1,17 @@ import dataclasses +from django.db.models import QuerySet +from django.utils import timezone from fastapi import Depends from fastapi.security import APIKeyHeader -from django.utils import timezone -from django.db.models import QuerySet - from etebase_server.django import models from etebase_server.django.token_auth.models import AuthToken, get_default_expiry from etebase_server.myauth.models import UserType, get_typed_user_model + +from .db_hack import django_db_cleanup_decorator from .exceptions import AuthenticationFailed from .utils import get_object_or_404 -from .db_hack import django_db_cleanup_decorator - User = get_typed_user_model() token_scheme = APIKeyHeader(name="Authorization") diff --git a/etebase_server/fastapi/exceptions.py b/etebase_server/fastapi/exceptions.py index 1a98fcb..2fbb9a1 100644 --- a/etebase_server/fastapi/exceptions.py +++ b/etebase_server/fastapi/exceptions.py @@ -1,8 +1,8 @@ -from fastapi import status, HTTPException import typing as t -from pydantic import BaseModel from django.core.exceptions import ValidationError as DjangoValidationError +from fastapi import HTTPException, status +from pydantic import BaseModel class HttpErrorField(BaseModel): @@ -11,7 +11,7 @@ class HttpErrorField(BaseModel): detail: str class Config: - orm_mode = True + from_attributes = True class HttpErrorOut(BaseModel): @@ -20,7 +20,7 @@ class HttpErrorOut(BaseModel): errors: t.Optional[t.List[HttpErrorField]] class Config: - orm_mode = True + from_attributes = True class CustomHttpException(HTTPException): diff --git a/etebase_server/fastapi/main.py b/etebase_server/fastapi/main.py index e4abd6c..5d87a84 100644 --- a/etebase_server/fastapi/main.py +++ b/etebase_server/fastapi/main.py @@ -1,19 +1,19 @@ from django.conf import settings # Not at the top of the file because we first need to setup django -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, status +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware from fastapi.staticfiles import StaticFiles -from etebase_server.django import app_settings - from .exceptions import CustomHttpException from .msgpack import MsgpackResponse from .routers.authentication import authentication_router from .routers.collection import collection_router, item_router -from .routers.member import member_router from .routers.invitation import invitation_incoming_router, invitation_outgoing_router +from .routers.member import member_router from .routers.websocket import websocket_router @@ -24,12 +24,12 @@ def create_application(prefix="", middlewares=[]): externalDocs={ "url": "https://docs.etebase.com", "description": "Docs about the API specifications and clients.", - } + }, # FIXME: version="2.5.0", ) - VERSION = "v1" - BASE_PATH = f"{prefix}/api/{VERSION}" - COLLECTION_UID_MARKER = "{collection_uid}" + VERSION = "v1" # noqa: N806 + BASE_PATH = f"{prefix}/api/{VERSION}" # noqa: N806 + COLLECTION_UID_MARKER = "{collection_uid}" # noqa: N806 app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) @@ -75,6 +75,13 @@ def create_application(prefix="", middlewares=[]): async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + @app.exception_handler(RequestValidationError) + async def validation_exception_handler(request: Request, exc: RequestValidationError): + return MsgpackResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content=jsonable_encoder({"detail": exc.errors()}), + ) + app.mount(settings.STATIC_URL, StaticFiles(directory=settings.STATIC_ROOT), name="static") return app diff --git a/etebase_server/fastapi/msgpack.py b/etebase_server/fastapi/msgpack.py index 3871d15..9820852 100644 --- a/etebase_server/fastapi/msgpack.py +++ b/etebase_server/fastapi/msgpack.py @@ -5,16 +5,19 @@ from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response -from .utils import msgpack_encode, msgpack_decode from .db_hack import django_db_cleanup_decorator +from .utils import msgpack_decode, msgpack_encode class MsgpackRequest(Request): media_type = "application/msgpack" + async def raw_body(self) -> bytes: + return await super().body() + async def body(self) -> bytes: if not hasattr(self, "_json"): - body = await super().body() + body = await self.raw_body() self._json = msgpack_decode(body) return self._json @@ -27,7 +30,7 @@ class MsgpackResponse(Response): return b"" if isinstance(content, BaseModel): - content = content.dict() + content = content.model_dump() return msgpack_encode(content) @@ -48,7 +51,7 @@ class MsgpackRoute(APIRoute): status_code=self.status_code, # use custom response class or fallback on default self.response_class response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class), - response_field=self.secure_cloned_response_field, + response_field=self.response_field, response_model_include=self.response_model_include, response_model_exclude=self.response_model_exclude, response_model_by_alias=self.response_model_by_alias, @@ -60,14 +63,14 @@ class MsgpackRoute(APIRoute): def get_route_handler(self) -> t.Callable: async def custom_route_handler(request: Request) -> Response: - content_type = request.headers.get("Content-Type") - try: - request_cls = self.REQUESTS_CLASSES[content_type] - request = request_cls(request.scope, request.receive) - except KeyError: - # nothing registered to handle content_type, process given requests as-is - pass + if content_type is not None: + try: + request_cls = self.REQUESTS_CLASSES[content_type] + request = request_cls(request.scope, request.receive) + except KeyError: + # nothing registered to handle content_type, process given requests as-is + pass accept = request.headers.get("Accept") route_handler = self._get_media_type_route_handler(accept) diff --git a/etebase_server/fastapi/redis.py b/etebase_server/fastapi/redis.py index b4f7a04..8e9f14c 100644 --- a/etebase_server/fastapi/redis.py +++ b/etebase_server/fastapi/redis.py @@ -1,4 +1,5 @@ import typing as t + from redis import asyncio as aioredis from etebase_server.django import app_settings diff --git a/etebase_server/fastapi/routers/authentication.py b/etebase_server/fastapi/routers/authentication.py index d771a5c..533b0eb 100644 --- a/etebase_server/fastapi/routers/authentication.py +++ b/etebase_server/fastapi/routers/authentication.py @@ -1,5 +1,4 @@ import typing as t -from typing_extensions import Literal from datetime import datetime import nacl @@ -8,22 +7,24 @@ import nacl.hash import nacl.secret import nacl.signing from django.conf import settings -from django.contrib.auth import user_logged_out, user_logged_in +from django.contrib.auth import user_logged_in, user_logged_out from django.core import exceptions as django_exceptions from django.db import transaction from django.utils.functional import cached_property -from fastapi import APIRouter, Depends, status, Request +from fastapi import APIRouter, Depends, Request, status +from typing_extensions import Literal from etebase_server.django import app_settings, models -from etebase_server.django.token_auth.models import AuthToken from etebase_server.django.models import UserInfo from etebase_server.django.signals import user_signed_up -from etebase_server.django.utils import create_user, get_user_queryset, CallbackContext +from etebase_server.django.token_auth.models import AuthToken +from etebase_server.django.utils import CallbackContext, create_user, get_user_queryset from etebase_server.myauth.models import UserType, get_typed_user_model -from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError -from ..msgpack import MsgpackRoute -from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode, get_user_username_email_kwargs + from ..dependencies import AuthData, get_auth_data, get_authenticated_user +from ..exceptions import AuthenticationFailed, HttpError, transform_validation_error +from ..msgpack import MsgpackResponse, MsgpackRoute +from ..utils import BaseModel, get_user_username_email_kwargs, msgpack_decode, msgpack_encode, permission_responses User = get_typed_user_model() authentication_router = APIRouter(route_class=MsgpackRoute) @@ -75,7 +76,7 @@ class LoginOut(BaseModel): class Authentication(BaseModel): class Config: - keep_untouched = (cached_property,) + ignored_types = (cached_property,) response: bytes signature: bytes @@ -187,7 +188,7 @@ def login_challenge(user: UserType = Depends(get_login_user)): "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version) + return MsgpackResponse(LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version)) @authentication_router.post("/login/", response_model=LoginOut) @@ -197,7 +198,7 @@ def login(data: Login, request: Request): validate_login_request(data.response_data, data, user, "login", host) ret = LoginOut.from_orm(user) user_logged_in.send(sender=user.__class__, request=None, user=user) - return ret + return MsgpackResponse(ret) @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) @@ -222,7 +223,7 @@ def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_u ret = { "url": get_dashboard_url(CallbackContext(request.path_params, user=user)), } - return ret + return MsgpackResponse(ret) def signup_save(data: SignupIn, request: Request) -> UserType: @@ -260,4 +261,4 @@ def signup(data: SignupIn, request: Request): user = signup_save(data, request) ret = LoginOut.from_orm(user) user_signed_up.send(sender=user.__class__, request=None, user=user) - return ret + return MsgpackResponse(ret) diff --git a/etebase_server/fastapi/routers/collection.py b/etebase_server/fastapi/routers/collection.py index 97dbbf4..2de35a8 100644 --- a/etebase_server/fastapi/routers/collection.py +++ b/etebase_server/fastapi/routers/collection.py @@ -3,33 +3,34 @@ import typing as t from asgiref.sync import sync_to_async from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from django.db.models import Q, QuerySet -from fastapi import APIRouter, Depends, status, Request, BackgroundTasks +from fastapi import APIRouter, BackgroundTasks, Depends, Request, status from etebase_server.django import models from etebase_server.myauth.models import UserType -from .authentication import get_authenticated_user -from .websocket import get_ticket, TicketRequest, TicketOut -from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError -from ..msgpack import MsgpackRoute -from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken + +from ..db_hack import django_db_cleanup_decorator +from ..dependencies import get_collection, get_collection_queryset, get_item_queryset +from ..exceptions import HttpError, PermissionDenied, ValidationError, transform_validation_error +from ..msgpack import MsgpackRequest, MsgpackResponse, MsgpackRoute +from ..redis import redisw +from ..sendfile import sendfile +from ..stoken_handler import filter_by_stoken, filter_by_stoken_and_limit, get_queryset_stoken, get_stoken_obj from ..utils import ( - get_object_or_404, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, + BaseModel, Context, Prefetch, PrefetchQuery, + get_object_or_404, is_collection_admin, msgpack_encode, - BaseModel, permission_responses, - PERMISSIONS_READ, - PERMISSIONS_READWRITE, ) -from ..dependencies import get_collection_queryset, get_item_queryset, get_collection -from ..sendfile import sendfile -from ..redis import redisw -from ..db_hack import django_db_cleanup_decorator +from .authentication import get_authenticated_user +from .websocket import TicketOut, TicketRequest, get_ticket collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -51,7 +52,7 @@ class CollectionItemRevisionInOut(BaseModel): chunks: t.List[ChunkType] class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm_context( @@ -77,7 +78,7 @@ class CollectionItemCommon(BaseModel): class CollectionItemOut(CollectionItemCommon): class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm_context( @@ -134,7 +135,7 @@ class CollectionListResponse(BaseModel): stoken: t.Optional[str] done: bool - removedMemberships: t.Optional[t.List[RemovedMembershipOut]] + removedMemberships: t.Optional[t.List[RemovedMembershipOut]] = None class CollectionItemListResponse(BaseModel): @@ -151,7 +152,7 @@ class CollectionItemRevisionListResponse(BaseModel): class CollectionItemBulkGetIn(BaseModel): uid: str - etag: t.Optional[str] + etag: t.Optional[str] = None class ItemDepIn(BaseModel): @@ -274,7 +275,7 @@ def list_multi( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - return collection_list_common(queryset, user, stoken, limit, prefetch) + return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch)) @collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ) @@ -285,7 +286,7 @@ def collection_list( user: UserType = Depends(get_authenticated_user), queryset: CollectionQuerySet = Depends(get_collection_queryset), ): - return collection_list_common(queryset, user, stoken, limit, prefetch) + return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch)) def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): @@ -364,7 +365,7 @@ def collection_get( user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - return CollectionOut.from_orm_context(obj, Context(user, prefetch)) + return MsgpackResponse(CollectionOut.from_orm_context(obj, Context(user, prefetch))) def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool): @@ -373,7 +374,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val revision_data = item_model.content uid = item_model.uid - Model = models.CollectionItem + Model = models.CollectionItem # noqa: N806 with transaction.atomic(): instance, created = Model.objects.get_or_create( @@ -417,7 +418,7 @@ def item_get( prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) - return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) + return MsgpackResponse(CollectionItemOut.from_orm_context(obj, Context(user, prefetch))) def item_list_common( @@ -449,7 +450,7 @@ def item_list( queryset = queryset.filter(parent__isnull=True) response = item_list_common(queryset, user, stoken, limit, prefetch) - return response + return MsgpackResponse(response) @item_router.post("/item/subscription-ticket/", response_model=TicketOut, dependencies=PERMISSIONS_READ) @@ -458,7 +459,7 @@ async def item_list_subscription_ticket( user: UserType = Depends(get_authenticated_user), ): """Get an authentication ticket that can be used with the websocket endpoint""" - return await get_ticket(TicketRequest(collection=collection.uid), user) + return MsgpackResponse(await get_ticket(TicketRequest(collection=collection.uid), user)) def item_bulk_common( @@ -526,10 +527,12 @@ def item_revisions( ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result] iterator = ret_data[-1].uid if len(result) > 0 else None - return CollectionItemRevisionListResponse( - data=ret_data, - iterator=iterator, - done=done, + return MsgpackResponse( + CollectionItemRevisionListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) ) @@ -559,10 +562,12 @@ def fetch_updates( new_stoken = new_stoken or stoken_rev_uid context = Context(user, prefetch) - return CollectionItemListResponse( - data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], - stoken=new_stoken, - done=True, # we always return all the items, so it's always done + return MsgpackResponse( + CollectionItemListResponse( + data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], + stoken=new_stoken, + done=True, # we always return all the items, so it's always done + ) ) @@ -574,7 +579,9 @@ def item_transaction( stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user), ): - return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks) + return MsgpackResponse( + item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks) + ) @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) @@ -585,7 +592,9 @@ def item_batch( stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user), ): - return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks) + return MsgpackResponse( + item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks) + ) # Chunks @@ -610,7 +619,12 @@ async def chunk_update( collection: models.Collection = Depends(get_collection), ): # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) - content_file = ContentFile(await request.body()) + if isinstance(request, MsgpackRequest): + body = await request.raw_body() + else: + body = await request.body() + + content_file = ContentFile(body) try: await chunk_save(chunk_uid, collection, content_file) except IntegrityError: diff --git a/etebase_server/fastapi/routers/invitation.py b/etebase_server/fastapi/routers/invitation.py index adb51c6..43a182f 100644 --- a/etebase_server/fastapi/routers/invitation.py +++ b/etebase_server/fastapi/routers/invitation.py @@ -1,26 +1,27 @@ import typing as t -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Request +from fastapi import APIRouter, Depends, Request, status from etebase_server.django import models -from etebase_server.django.utils import get_user_queryset, CallbackContext +from etebase_server.django.utils import CallbackContext, get_user_queryset from etebase_server.myauth.models import UserType, get_typed_user_model -from .authentication import get_authenticated_user + +from ..db_hack import django_db_cleanup_decorator from ..exceptions import HttpError, PermissionDenied -from ..msgpack import MsgpackRoute +from ..msgpack import MsgpackResponse, MsgpackRoute from ..utils import ( - get_object_or_404, - get_user_username_email_kwargs, - Context, - is_collection_admin, - BaseModel, - permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE, + BaseModel, + Context, + get_object_or_404, + get_user_username_email_kwargs, + is_collection_admin, + permission_responses, ) -from ..db_hack import django_db_cleanup_decorator +from .authentication import get_authenticated_user User = get_typed_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -33,7 +34,7 @@ class UserInfoOut(BaseModel): pubkey: bytes class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut": @@ -66,7 +67,7 @@ class CollectionInvitationOut(CollectionInvitationCommon): fromPubkey: bytes class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut": @@ -120,7 +121,7 @@ def list_common( iterator = ret_data[-1].uid if len(result) > 0 else None return InvitationListResponse( - data=ret_data, + data=[CollectionInvitationOut.from_orm(x) for x in ret_data], iterator=iterator, done=done, ) @@ -132,7 +133,7 @@ def incoming_list( limit: int = 50, queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): - return list_common(queryset, iterator, limit) + return MsgpackResponse(list_common(queryset, iterator, limit)) @invitation_incoming_router.get( @@ -143,7 +144,7 @@ def incoming_get( queryset: InvitationQuerySet = Depends(get_incoming_queryset), ): obj = get_object_or_404(queryset, uid=invitation_uid) - return CollectionInvitationOut.from_orm(obj) + return MsgpackResponse(CollectionInvitationOut.from_orm(obj)) @invitation_incoming_router.delete( @@ -218,7 +219,7 @@ def outgoing_list( limit: int = 50, queryset: InvitationQuerySet = Depends(get_outgoing_queryset), ): - return list_common(queryset, iterator, limit) + return MsgpackResponse(list_common(queryset, iterator, limit)) @invitation_outgoing_router.delete( @@ -241,4 +242,4 @@ def outgoing_fetch_user_profile( kwargs = get_user_username_email_kwargs(username) user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) - return UserInfoOut.from_orm(user_info) + return MsgpackResponse(UserInfoOut.from_orm(user_info)) diff --git a/etebase_server/fastapi/routers/member.py b/etebase_server/fastapi/routers/member.py index 123357b..71da4c7 100644 --- a/etebase_server/fastapi/routers/member.py +++ b/etebase_server/fastapi/routers/member.py @@ -6,12 +6,12 @@ from fastapi import APIRouter, Depends, status from etebase_server.django import models from etebase_server.myauth.models import UserType, get_typed_user_model -from .authentication import get_authenticated_user -from ..msgpack import MsgpackRoute -from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE -from ..stoken_handler import filter_by_stoken_and_limit -from ..db_hack import django_db_cleanup_decorator +from ..db_hack import django_db_cleanup_decorator +from ..msgpack import MsgpackResponse, MsgpackRoute +from ..stoken_handler import filter_by_stoken_and_limit +from ..utils import PERMISSIONS_READ, PERMISSIONS_READWRITE, BaseModel, get_object_or_404, permission_responses +from .authentication import get_authenticated_user from .collection import get_collection, verify_collection_admin User = get_typed_user_model() @@ -39,7 +39,7 @@ class CollectionMemberOut(BaseModel): accessLevel: models.AccessLevels class Config: - orm_mode = True + from_attributes = True @classmethod def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut": @@ -66,10 +66,12 @@ def member_list( ) new_stoken = new_stoken_obj and new_stoken_obj.uid - return MemberListResponse( - data=[CollectionMemberOut.from_orm(item) for item in result], - iterator=new_stoken, - done=done, + return MsgpackResponse( + MemberListResponse( + data=[CollectionMemberOut.from_orm(item) for item in result], + iterator=new_stoken, + done=done, + ) ) diff --git a/etebase_server/fastapi/routers/test_reset_view.py b/etebase_server/fastapi/routers/test_reset_view.py index 7895697..1ec60e2 100644 --- a/etebase_server/fastapi/routers/test_reset_view.py +++ b/etebase_server/fastapi/routers/test_reset_view.py @@ -3,12 +3,13 @@ from django.db import transaction from django.shortcuts import get_object_or_404 from fastapi import APIRouter, Request, status -from etebase_server.django.utils import get_user_queryset, CallbackContext -from .authentication import SignupIn, signup_save -from ..msgpack import MsgpackRoute -from ..exceptions import HttpError +from etebase_server.django.utils import CallbackContext, get_user_queryset from etebase_server.myauth.models import get_typed_user_model +from ..exceptions import HttpError +from ..msgpack import MsgpackRoute +from .authentication import SignupIn, signup_save + test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) User = get_typed_user_model() diff --git a/etebase_server/fastapi/routers/websocket.py b/etebase_server/fastapi/routers/websocket.py index 2caf7c6..a434dbd 100644 --- a/etebase_server/fastapi/routers/websocket.py +++ b/etebase_server/fastapi/routers/websocket.py @@ -1,13 +1,13 @@ import asyncio import typing as t -from redis import asyncio as aioredis -from redis.exceptions import ConnectionError +import nacl.encoding +import nacl.utils from asgiref.sync import sync_to_async from django.db.models import QuerySet from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status -import nacl.encoding -import nacl.utils +from redis import asyncio as aioredis +from redis.exceptions import ConnectionError from etebase_server.django import models from etebase_server.django.utils import CallbackContext, get_user_queryset @@ -19,7 +19,6 @@ from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode from ..redis import redisw from ..utils import BaseModel, permission_responses - User = get_typed_user_model() websocket_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) CollectionQuerySet = QuerySet[models.Collection] diff --git a/etebase_server/fastapi/sendfile/utils.py b/etebase_server/fastapi/sendfile/utils.py index c35d6df..22e8266 100644 --- a/etebase_server/fastapi/sendfile/utils.py +++ b/etebase_server/fastapi/sendfile/utils.py @@ -1,14 +1,15 @@ +import logging +import os from functools import lru_cache from importlib import import_module from pathlib import Path, PurePath from urllib.parse import quote -import logging - -from fastapi import status -from ..exceptions import HttpError from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from fastapi import status + +from ..exceptions import HttpError logger = logging.getLogger(__name__) @@ -32,9 +33,7 @@ def _convert_file_to_url(path): path_obj = PurePath(path) relpath = path_obj.relative_to(path_root) - # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an - # already instantiated Path object - url = relpath._flavour.pathmod.normpath(str(url_root / relpath)) + url = os.path.normpath(str(url_root / relpath)) return quote(str(url)) @@ -48,9 +47,7 @@ def _sanitize_path(filepath): filepath_obj = Path(filepath) # get absolute path - # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an - # already instantiated Path object - filepath_abs = Path(filepath_obj._flavour.pathmod.normpath(str(path_root / filepath_obj))) + filepath_abs = Path(os.path.normpath(str(path_root / filepath_obj))) # if filepath_abs is not relative to path_root, relative_to throws an error try: diff --git a/etebase_server/fastapi/stoken_handler.py b/etebase_server/fastapi/stoken_handler.py index b4c7eab..be5646f 100644 --- a/etebase_server/fastapi/stoken_handler.py +++ b/etebase_server/fastapi/stoken_handler.py @@ -47,7 +47,6 @@ def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]: def filter_by_stoken_and_limit( stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation ) -> t.Tuple[list, t.Optional[Stoken], bool]: - queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation) result = list(queryset[: limit + 1]) diff --git a/etebase_server/fastapi/utils.py b/etebase_server/fastapi/utils.py index 334633c..0b28875 100644 --- a/etebase_server/fastapi/utils.py +++ b/etebase_server/fastapi/utils.py @@ -1,14 +1,13 @@ +import base64 import dataclasses import typing as t -from typing_extensions import Literal + import msgpack -import base64 - -from fastapi import status, Query, Depends -from pydantic import BaseModel as PyBaseModel - -from django.db.models import Model, QuerySet from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Model, QuerySet +from fastapi import Depends, Query, status +from pydantic import BaseModel as PyBaseModel +from typing_extensions import Literal from etebase_server.django import app_settings from etebase_server.django.models import AccessLevels @@ -26,10 +25,7 @@ T = t.TypeVar("T", bound=Model, covariant=True) class BaseModel(PyBaseModel): - class Config: - json_encoders = { - bytes: lambda x: x, - } + pass @dataclasses.dataclass diff --git a/etebase_server/myauth/admin.py b/etebase_server/myauth/admin.py index 1f4b767..8da68b4 100644 --- a/etebase_server/myauth/admin.py +++ b/etebase_server/myauth/admin.py @@ -1,12 +1,21 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin as DjangoUserAdmin -from .models import User + from .forms import AdminUserCreationForm +from .models import User class UserAdmin(DjangoUserAdmin): add_form = AdminUserCreationForm - add_fieldsets = ((None, {"classes": ("wide",), "fields": ("username",),}),) + add_fieldsets = ( + ( + None, + { + "classes": ("wide",), + "fields": ("username",), + }, + ), + ) admin.site.register(User, UserAdmin) diff --git a/etebase_server/myauth/forms.py b/etebase_server/myauth/forms.py index 7681835..b745832 100644 --- a/etebase_server/myauth/forms.py +++ b/etebase_server/myauth/forms.py @@ -1,5 +1,6 @@ from django import forms from django.contrib.auth.forms import UsernameField + from etebase_server.myauth.models import get_typed_user_model User = get_typed_user_model() diff --git a/etebase_server/myauth/ldap.py b/etebase_server/myauth/ldap.py index 00de7a6..65a511d 100644 --- a/etebase_server/myauth/ldap.py +++ b/etebase_server/myauth/ldap.py @@ -1,15 +1,15 @@ import logging -from django.utils import timezone +import ldap from django.conf import settings from django.core.exceptions import PermissionDenied as DjangoPermissionDenied -from etebase_server.django.utils import CallbackContext -from etebase_server.myauth.models import get_typed_user_model, UserType -from etebase_server.fastapi.dependencies import get_authenticated_user -from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied +from django.utils import timezone from fastapi import Depends -import ldap +from etebase_server.django.utils import CallbackContext +from etebase_server.fastapi.dependencies import get_authenticated_user +from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied +from etebase_server.myauth.models import UserType, get_typed_user_model User = get_typed_user_model() diff --git a/etebase_server/myauth/models.py b/etebase_server/myauth/models.py index 89b94b4..c862a46 100644 --- a/etebase_server/myauth/models.py +++ b/etebase_server/myauth/models.py @@ -1,21 +1,13 @@ import typing as t from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager -from django.core import validators +from django.contrib.auth.validators import UnicodeUsernameValidator from django.db import models -from django.utils.deconstruct import deconstructible from django.utils.translation import gettext_lazy as _ -@deconstructible -class UnicodeUsernameValidator(validators.RegexValidator): - regex = r"^[\w.-]+\Z" - message = _("Enter a valid username. This value may contain only letters, " "numbers, and ./-/_ characters.") - flags = 0 - - class UserManager(DjangoUserManager): - def get_by_natural_key(self, username: str): + def get_by_natural_key(self, username: t.Optional[str]): return self.get(**{self.model.USERNAME_FIELD + "__iexact": username}) diff --git a/etebase_server/myauth/tests.py b/etebase_server/myauth/tests.py index 7ce503c..a39b155 100644 --- a/etebase_server/myauth/tests.py +++ b/etebase_server/myauth/tests.py @@ -1,3 +1 @@ -from django.test import TestCase - # Create your tests here. diff --git a/etebase_server/myauth/views.py b/etebase_server/myauth/views.py index 91ea44a..60f00ef 100644 --- a/etebase_server/myauth/views.py +++ b/etebase_server/myauth/views.py @@ -1,3 +1 @@ -from django.shortcuts import render - # Create your views here. diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 1f9bf3e..fc486fe 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -10,8 +10,9 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.0/ref/settings/ """ -import os import configparser +import os + from .utils import get_secret_from_file # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -44,7 +45,7 @@ DATABASES = { } } -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # Application definition @@ -200,7 +201,7 @@ if "DJANGO_MEDIA_ROOT" in os.environ: # Make an `etebase_server_settings` module available to override settings. try: - from etebase_server_settings import * + from etebase_server_settings import * # noqa: F403 except ImportError: pass diff --git a/etebase_server/urls.py b/etebase_server/urls.py index 574ddf7..17ebc0e 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,11 +1,6 @@ -import os - -from django.conf import settings from django.contrib import admin from django.urls import path from django.views.generic import TemplateView -from django.views.static import serve -from django.contrib.staticfiles import finders urlpatterns = [ path("admin/", admin.site.urls), diff --git a/etebase_server/utils.py b/etebase_server/utils.py index 9f56457..1ccbe1e 100644 --- a/etebase_server/utils.py +++ b/etebase_server/utils.py @@ -12,10 +12,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from django.core.management import utils import os import stat +from django.core.management import utils + def get_secret_from_file(path): try: diff --git a/manage.py b/manage.py index a3a8741..f8ee1ea 100755 --- a/manage.py +++ b/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Django's command-line utility for administrative tasks.""" + import os import sys diff --git a/pyproject.toml b/pyproject.toml index b622800..22072d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,25 @@ line-length = 120 [build-system] requires = ["setuptools>=42"] build-backend = "setuptools.build_meta" + +[tool.ruff] +line-length = 120 +exclude = [ + ".git", + ".git-rewrite", + ".mypy_cache", + ".pytype", + ".ruff_cache", + ".venv", + "build", + "dist", + "node_modules", + "migrations", # Alembic migrations +] + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "T20", "W"] +ignore = ["E203", "E501", "E711", "E712", "N803", "N815", "N818", "T201"] + +[tool.ruff.lint.isort] +combine-as-imports = true diff --git a/requirements-dev.txt b/requirements-dev.txt index ae24fb4..c48fb6b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,66 +1,53 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements-dev.txt requirements.in/development.txt # -asgiref==3.5.2 - # via django -black==22.10.0 - # via -r requirements.in/development.txt -build==0.9.0 +asgiref==3.8.1 + # via + # django + # django-stubs +build==1.2.1 # via pip-tools -click==8.1.3 +click==8.1.7 + # via pip-tools +coverage==7.5.3 + # via -r requirements.in/development.txt +django==5.0.6 # via - # black + # django-stubs + # django-stubs-ext +django-stubs==5.0.2 + # via -r requirements.in/development.txt +django-stubs-ext==5.0.2 + # via django-stubs +mypy==1.10.0 + # via -r requirements.in/development.txt +mypy-extensions==1.0.0 + # via mypy +packaging==24.0 + # via build +pip-tools==7.4.1 + # via -r requirements.in/development.txt +pyproject-hooks==1.1.0 + # via + # build # pip-tools -coverage==6.5.0 +pywatchman==2.0.0 # via -r requirements.in/development.txt -django==3.2.16 - # via - # -r requirements.in/development.txt - # django-stubs - # django-stubs-ext -django-stubs==1.13.0 +ruff==0.4.8 # via -r requirements.in/development.txt -django-stubs-ext==0.7.0 - # via django-stubs -mypy==0.991 - # via django-stubs -mypy-extensions==0.4.3 - # via - # black - # mypy -packaging==21.3 - # via build -pathspec==0.10.2 - # via black -pep517==0.13.0 - # via build -pip-tools==6.11.0 - # via -r requirements.in/development.txt -platformdirs==2.6.0 - # via black -pyparsing==3.0.9 - # via packaging -pytz==2022.6 +sqlparse==0.5.0 # via django -pywatchman==1.4.1 - # via -r requirements.in/development.txt -sqlparse==0.4.3 - # via django -tomli==2.0.1 +types-pyyaml==6.0.12.20240311 # via django-stubs -types-pytz==2022.6.0.1 - # via django-stubs -types-pyyaml==6.0.12.2 - # via django-stubs -typing-extensions==4.4.0 +typing-extensions==4.12.2 # via # django-stubs # django-stubs-ext # mypy -wheel==0.38.4 +wheel==0.43.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 6842b02..37de777 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -1,7 +1,8 @@ django>=4.0,<5.0 msgpack pynacl -fastapi +fastapi>=0.104 +pydantic>=2.0.0 typing_extensions uvicorn[standard] aiofiles diff --git a/requirements.in/development.txt b/requirements.in/development.txt index bf565de..488d088 100644 --- a/requirements.in/development.txt +++ b/requirements.in/development.txt @@ -1,6 +1,6 @@ coverage pip-tools pywatchman -black +ruff +mypy django-stubs -django<4.0 diff --git a/requirements.txt b/requirements.txt index 62b6344..a0c480a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,59 +4,114 @@ # # pip-compile --output-file=requirements.txt requirements.in/base.txt # -aiofiles==22.1.0 +aiofiles==23.2.1 # via -r requirements.in/base.txt -anyio==3.6.2 +annotated-types==0.7.0 + # via pydantic +anyio==4.4.0 # via + # httpx # starlette # watchfiles -asgiref==3.5.2 +asgiref==3.8.1 # via django -async-timeout==4.0.2 - # via redis -cffi==1.15.1 +certifi==2024.6.2 + # via + # httpcore + # httpx +cffi==1.16.0 # via pynacl -click==8.1.3 - # via uvicorn -django==4.1.13 +click==8.1.7 + # via + # typer + # uvicorn +django==4.2.13 # via -r requirements.in/base.txt -fastapi==0.88.0 +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via fastapi +fastapi==0.111.0 # via -r requirements.in/base.txt +fastapi-cli==0.0.4 + # via fastapi h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx +httptools==0.6.1 # via uvicorn -httptools==0.5.0 - # via uvicorn -idna==3.4 - # via anyio -msgpack==1.0.4 +httpx==0.27.0 + # via fastapi +idna==3.7 + # via + # anyio + # email-validator + # httpx +jinja2==3.1.4 + # via fastapi +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +msgpack==1.0.8 # via -r requirements.in/base.txt -pycparser==2.21 +orjson==3.10.3 + # via fastapi +pycparser==2.22 # via cffi -pydantic==1.10.2 - # via fastapi -pynacl==1.5.0 - # via -r requirements.in/base.txt -python-dotenv==0.21.0 - # via uvicorn -pyyaml==6.0.1 - # via uvicorn -redis==4.4.0 - # via -r requirements.in/base.txt -sniffio==1.3.0 - # via anyio -sqlparse==0.4.3 - # via django -starlette==0.22.0 - # via fastapi -typing-extensions==4.4.0 +pydantic==2.7.3 # via # -r requirements.in/base.txt - # pydantic -uvicorn[standard]==0.20.0 + # fastapi +pydantic-core==2.18.4 + # via pydantic +pygments==2.18.0 + # via rich +pynacl==1.5.0 # via -r requirements.in/base.txt -uvloop==0.17.0 +python-dotenv==1.0.1 # via uvicorn -watchfiles==0.18.1 +python-multipart==0.0.9 + # via fastapi +pyyaml==6.0.1 # via uvicorn -websockets==10.4 +redis==5.1.0b6 + # via -r requirements.in/base.txt +rich==13.7.1 + # via typer +shellingham==1.5.4 + # via typer +sniffio==1.3.1 + # via + # anyio + # httpx +sqlparse==0.5.0 + # via django +starlette==0.37.2 + # via fastapi +typer==0.12.3 + # via fastapi-cli +typing-extensions==4.12.2 + # via + # -r requirements.in/base.txt + # fastapi + # pydantic + # pydantic-core + # typer +ujson==5.10.0 + # via fastapi +uvicorn[standard]==0.30.1 + # via + # -r requirements.in/base.txt + # fastapi +uvloop==0.19.0 + # via uvicorn +watchfiles==0.22.0 + # via uvicorn +websockets==12.0 # via uvicorn