ChangeLog.md

@@ -1,33 +1,5 @@
 # Changelog
 
-## Version 0.14.2
-- Fix issue with some requests failing in some scenarios with the JS client.
-  - The JS client was omitting optional fields which were accidentally made to be required. It happened because pydantic v2 changed the behavior in a few ways (for the better fwiw) and we missed a few places when upgrading.
-
-## Version 0.14.1
-- Fix issue with serializing non utf8 422 errors
-- Appease django warnings about default auto field
-
-## Version 0.14.0
-- Upgrade to latest FastAPI and Pydantic for compatibility with Python 3.12
-
-## Version 0.13.1
-- Fix requirements definition for django.
-
-## Version 0.13.0
-- Update Django dependency to 4 (LTS)
-
-## Version 0.12.0
-- Optimize some database queries fixing performance degradation with large installations.
-- Ensure environment variables always override config
-
-## Version 0.11.0
-- Update deps for Python 3.11
-
-## Version 0.10.0
-- Replace the deprecated aioredis with redis-py
-- Optimize how we fetch the current (latest) revision of items
-
 ## Version 0.9.1
 - Update pinned Django version (only matters if using `requirements.txt`).
 

README.md

@@ -63,12 +63,6 @@ Now you can initialise our django app.
 ./manage.py migrate
 ```
 
-Create static files:
-
-```
-./manage.py collectstatic
-```
-
 And you are done! You can now run the debug server just to see everything works as expected by running:
 
 ```
@@ -184,4 +178,3 @@ Become a financial contributor and help us sustain our community!
 [![ilovept](https://github.com/ilovept.png?size=40)](https://github.com/ilovept)
 [![ryanleesipes](https://github.com/ryanleesipes.png?size=40)](https://github.com/ryanleesipes)
 [![DanielG](https://github.com/DanielG.png?size=40)](https://github.com/DanielG)
-[![Kanaye](https://github.com/Kanaye.png?size=40)](https://github.com/Kanaye)

docker/test-server/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.0-alpine
+FROM python:3.9.0-alpine
 
 ARG ETESYNC_VERSION
 

etebase_server/django/__init__.py

@@ -1,3 +1 @@
 from .app_settings_inner import app_settings
-
-__all__ = ["app_settings"]

etebase_server/django/app_settings_inner.py

@@ -34,11 +34,11 @@ class AppSettings:
         return getattr(settings, self.prefix + name, dflt)
 
     @cached_property
-    def REDIS_URI(self) -> t.Optional[str]:  # noqa: N802
+    def REDIS_URI(self) -> t.Optional[str]:  # pylint: disable=invalid-name
         return self._setting("REDIS_URI", None)
 
     @cached_property
-    def API_PERMISSIONS_READ(self):  # noqa: N802
+    def API_PERMISSIONS_READ(self):  # pylint: disable=invalid-name
         perms = self._setting("API_PERMISSIONS_READ", tuple())
         ret = []
         for perm in perms:
@@ -46,7 +46,7 @@ class AppSettings:
         return ret
 
     @cached_property
-    def API_PERMISSIONS_WRITE(self):  # noqa: N802
+    def API_PERMISSIONS_WRITE(self):  # pylint: disable=invalid-name
         perms = self._setting("API_PERMISSIONS_WRITE", tuple())
         ret = []
         for perm in perms:
@@ -54,35 +54,35 @@ class AppSettings:
         return ret
 
     @cached_property
-    def GET_USER_QUERYSET_FUNC(self):  # noqa: N802
+    def GET_USER_QUERYSET_FUNC(self):  # pylint: disable=invalid-name
         get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None)
         if get_user_queryset is not None:
             return self.import_from_str(get_user_queryset)
         return None
 
     @cached_property
-    def CREATE_USER_FUNC(self):  # noqa: N802
+    def CREATE_USER_FUNC(self):  # pylint: disable=invalid-name
         func = self._setting("CREATE_USER_FUNC", None)
         if func is not None:
             return self.import_from_str(func)
         return None
 
     @cached_property
-    def DASHBOARD_URL_FUNC(self):  # noqa: N802
+    def DASHBOARD_URL_FUNC(self):  # pylint: disable=invalid-name
         func = self._setting("DASHBOARD_URL_FUNC", None)
         if func is not None:
             return self.import_from_str(func)
         return None
 
     @cached_property
-    def CHUNK_PATH_FUNC(self):  # noqa: N802
+    def CHUNK_PATH_FUNC(self):  # pylint: disable=invalid-name
         func = self._setting("CHUNK_PATH_FUNC", None)
         if func is not None:
             return self.import_from_str(func)
         return None
 
     @cached_property
-    def CHALLENGE_VALID_SECONDS(self):  # noqa: N802
+    def CHALLENGE_VALID_SECONDS(self):  # pylint: disable=invalid-name
         return self._setting("CHALLENGE_VALID_SECONDS", 60)
 
 

etebase_server/django/models.py

@@ -15,21 +15,22 @@
 import typing as t
 from pathlib import Path
 
+from django.db import models, transaction
 from django.conf import settings
 from django.core.validators import RegexValidator
-from django.db import models, transaction
+from django.db.models import Max, Value as V
-from django.db.models import Max, Value as Val
 from django.db.models.functions import Coalesce, Greatest
-from django.utils.crypto import get_random_string
 from django.utils.functional import cached_property
+from django.utils.crypto import get_random_string
 
 from . import app_settings
 
+
 UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID")
 
 
 def stoken_annotation_builder(stoken_id_fields: t.List[str]):
-    aggr_fields = [Coalesce(Max(field), Val(0)) for field in stoken_id_fields]
+    aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields]
     return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0]
 
 
@@ -65,7 +66,7 @@ class Collection(models.Model):
     @cached_property
     def stoken(self) -> str:
         stoken_id = (
-            self.__class__.objects.filter(main_item=self.main_item, items__revisions__current=True)
+            self.__class__.objects.filter(main_item=self.main_item)
             .annotate(max_stoken=self.stoken_annotation)
             .values("max_stoken")
             .first()["max_stoken"]
@@ -95,7 +96,7 @@ class CollectionItem(models.Model):
 
     @cached_property
     def content(self) -> "CollectionItemRevision":
-        return self.revisions.filter(current=True)[0]
+        return self.revisions.get(current=True)
 
     @property
     def etag(self) -> str:

etebase_server/django/token_auth/models.py

@@ -1,7 +1,6 @@
 from django.db import models
 from django.utils import timezone
 from django.utils.crypto import get_random_string
-
 from etebase_server.myauth.models import get_typed_user_model
 
 User = get_typed_user_model()
@@ -16,6 +15,7 @@ def get_default_expiry():
 
 
 class AuthToken(models.Model):
+
     key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key)
     user = models.ForeignKey(User, null=False, blank=False, related_name="auth_token_set", on_delete=models.CASCADE)
     created = models.DateTimeField(auto_now_add=True)

etebase_server/django/utils.py

@@ -1,13 +1,13 @@
 import typing as t
 from dataclasses import dataclass
 
-from django.core.exceptions import PermissionDenied
 from django.db.models import QuerySet
-
+from django.core.exceptions import PermissionDenied
 from etebase_server.myauth.models import UserType, get_typed_user_model
 
 from . import app_settings
 
+
 User = get_typed_user_model()
 
 

etebase_server/fastapi/db_hack.py

@@ -2,7 +2,6 @@
 FIXME: this whole function is a hack around the django db limitations due to how db connections are cached and cleaned.
 Essentially django assumes there's the django request dispatcher to automatically clean up after the ORM.
 """
-
 import typing as t
 from functools import wraps
 

etebase_server/fastapi/dependencies.py

@@ -1,17 +1,18 @@
 import dataclasses
 
-from django.db.models import QuerySet
-from django.utils import timezone
 from fastapi import Depends
 from fastapi.security import APIKeyHeader
 
+from django.utils import timezone
+from django.db.models import QuerySet
+
 from etebase_server.django import models
 from etebase_server.django.token_auth.models import AuthToken, get_default_expiry
 from etebase_server.myauth.models import UserType, get_typed_user_model
-
-from .db_hack import django_db_cleanup_decorator
 from .exceptions import AuthenticationFailed
 from .utils import get_object_or_404
+from .db_hack import django_db_cleanup_decorator
+
 
 User = get_typed_user_model()
 token_scheme = APIKeyHeader(name="Authorization")

etebase_server/fastapi/exceptions.py

@@ -1,8 +1,8 @@
+from fastapi import status, HTTPException
 import typing as t
 
-from django.core.exceptions import ValidationError as DjangoValidationError
-from fastapi import HTTPException, status
 from pydantic import BaseModel
+from django.core.exceptions import ValidationError as DjangoValidationError
 
 
 class HttpErrorField(BaseModel):
@@ -11,7 +11,7 @@ class HttpErrorField(BaseModel):
     detail: str
 
     class Config:
-        from_attributes = True
+        orm_mode = True
 
 
 class HttpErrorOut(BaseModel):
@@ -20,7 +20,7 @@ class HttpErrorOut(BaseModel):
     errors: t.Optional[t.List[HttpErrorField]]
 
     class Config:
-        from_attributes = True
+        orm_mode = True
 
 
 class CustomHttpException(HTTPException):

etebase_server/fastapi/main.py

@@ -1,20 +1,19 @@
-import typing as t
-
 from django.conf import settings
 
 # Not at the top of the file because we first need to setup django
-from fastapi import FastAPI, Request, status
+from fastapi import FastAPI, Request
-from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.trustedhost import TrustedHostMiddleware
 from fastapi.staticfiles import StaticFiles
 
+from etebase_server.django import app_settings
+
 from .exceptions import CustomHttpException
 from .msgpack import MsgpackResponse
 from .routers.authentication import authentication_router
 from .routers.collection import collection_router, item_router
-from .routers.invitation import invitation_incoming_router, invitation_outgoing_router
 from .routers.member import member_router
+from .routers.invitation import invitation_incoming_router, invitation_outgoing_router
 from .routers.websocket import websocket_router
 
 
@@ -25,12 +24,12 @@ def create_application(prefix="", middlewares=[]):
         externalDocs={
             "url": "https://docs.etebase.com",
             "description": "Docs about the API specifications and clients.",
-        },
+        }
         # FIXME: version="2.5.0",
     )
-    VERSION = "v1"  # noqa: N806
+    VERSION = "v1"
-    BASE_PATH = f"{prefix}/api/{VERSION}"  # noqa: N806
+    BASE_PATH = f"{prefix}/api/{VERSION}"
-    COLLECTION_UID_MARKER = "{collection_uid}"  # noqa: N806
+    COLLECTION_UID_MARKER = "{collection_uid}"
     app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"])
     app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"])
     app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"])
@@ -76,16 +75,6 @@ def create_application(prefix="", middlewares=[]):
     async def custom_exception_handler(request: Request, exc: CustomHttpException):
         return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict)
 
-    @app.exception_handler(RequestValidationError)
-    async def validation_exception_handler(request: Request, exc: RequestValidationError):
-        from pydantic import TypeAdapter
-
-        errors = TypeAdapter(t.Dict[str, t.Any])
-        return MsgpackResponse(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            content=errors.dump_python({"detail": exc.errors()}),
-        )
-
     app.mount(settings.STATIC_URL, StaticFiles(directory=settings.STATIC_ROOT), name="static")
 
     return app

etebase_server/fastapi/msgpack.py

@@ -5,19 +5,16 @@ from pydantic import BaseModel
 from starlette.requests import Request
 from starlette.responses import Response
 
+from .utils import msgpack_encode, msgpack_decode
 from .db_hack import django_db_cleanup_decorator
-from .utils import msgpack_decode, msgpack_encode
 
 
 class MsgpackRequest(Request):
     media_type = "application/msgpack"
 
-    async def raw_body(self) -> bytes:
-        return await super().body()
-
     async def body(self) -> bytes:
         if not hasattr(self, "_json"):
-            body = await self.raw_body()
+            body = await super().body()
             self._json = msgpack_decode(body)
         return self._json
 
@@ -30,7 +27,7 @@ class MsgpackResponse(Response):
             return b""
 
         if isinstance(content, BaseModel):
-            content = content.model_dump()
+            content = content.dict()
         return msgpack_encode(content)
 
 
@@ -51,7 +48,7 @@ class MsgpackRoute(APIRoute):
             status_code=self.status_code,
             # use custom response class or fallback on default self.response_class
             response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class),
-            response_field=self.response_field,
+            response_field=self.secure_cloned_response_field,
             response_model_include=self.response_model_include,
             response_model_exclude=self.response_model_exclude,
             response_model_by_alias=self.response_model_by_alias,
@@ -63,14 +60,14 @@ class MsgpackRoute(APIRoute):
 
     def get_route_handler(self) -> t.Callable:
         async def custom_route_handler(request: Request) -> Response:
+
             content_type = request.headers.get("Content-Type")
-            if content_type is not None:
+            try:
-                try:
+                request_cls = self.REQUESTS_CLASSES[content_type]
-                    request_cls = self.REQUESTS_CLASSES[content_type]
+                request = request_cls(request.scope, request.receive)
-                    request = request_cls(request.scope, request.receive)
+            except KeyError:
-                except KeyError:
+                # nothing registered to handle content_type, process given requests as-is
-                    # nothing registered to handle content_type, process given requests as-is
+                pass
-                    pass
 
             accept = request.headers.get("Accept")
             route_handler = self._get_media_type_route_handler(accept)

etebase_server/fastapi/redis.py

@@ -1,6 +1,5 @@
 import typing as t
-
+import aioredis
-from redis import asyncio as aioredis
 
 from etebase_server.django import app_settings
 
@@ -13,11 +12,12 @@ class RedisWrapper:
 
     async def setup(self):
         if self.redis_uri is not None:
-            self.redis = await aioredis.from_url(self.redis_uri)
+            self.redis = await aioredis.create_redis_pool(self.redis_uri)
 
     async def close(self):
         if hasattr(self, "redis"):
-            await self.redis.close()
+            self.redis.close()
+            await self.redis.wait_closed()
 
     @property
     def is_active(self):

etebase_server/fastapi/routers/authentication.py

@@ -1,4 +1,5 @@
 import typing as t
+from typing_extensions import Literal
 from datetime import datetime
 
 import nacl
@@ -7,24 +8,22 @@ import nacl.hash
 import nacl.secret
 import nacl.signing
 from django.conf import settings
-from django.contrib.auth import user_logged_in, user_logged_out
+from django.contrib.auth import user_logged_out, user_logged_in
 from django.core import exceptions as django_exceptions
 from django.db import transaction
 from django.utils.functional import cached_property
-from fastapi import APIRouter, Depends, Request, status
+from fastapi import APIRouter, Depends, status, Request
-from typing_extensions import Literal
 
 from etebase_server.django import app_settings, models
+from etebase_server.django.token_auth.models import AuthToken
 from etebase_server.django.models import UserInfo
 from etebase_server.django.signals import user_signed_up
-from etebase_server.django.token_auth.models import AuthToken
+from etebase_server.django.utils import create_user, get_user_queryset, CallbackContext
-from etebase_server.django.utils import CallbackContext, create_user, get_user_queryset
 from etebase_server.myauth.models import UserType, get_typed_user_model
-
+from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError
+from ..msgpack import MsgpackRoute
+from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode, get_user_username_email_kwargs
 from ..dependencies import AuthData, get_auth_data, get_authenticated_user
-from ..exceptions import AuthenticationFailed, HttpError, transform_validation_error
-from ..msgpack import MsgpackResponse, MsgpackRoute
-from ..utils import BaseModel, get_user_username_email_kwargs, msgpack_decode, msgpack_encode, permission_responses
 
 User = get_typed_user_model()
 authentication_router = APIRouter(route_class=MsgpackRoute)
@@ -76,7 +75,7 @@ class LoginOut(BaseModel):
 
 class Authentication(BaseModel):
     class Config:
-        ignored_types = (cached_property,)
+        keep_untouched = (cached_property,)
 
     response: bytes
     signature: bytes
@@ -188,7 +187,7 @@ def login_challenge(user: UserType = Depends(get_login_user)):
         "userId": user.id,
     }
     challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder))
-    return MsgpackResponse(LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version))
+    return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version)
 
 
 @authentication_router.post("/login/", response_model=LoginOut)
@@ -198,7 +197,7 @@ def login(data: Login, request: Request):
     validate_login_request(data.response_data, data, user, "login", host)
     ret = LoginOut.from_orm(user)
     user_logged_in.send(sender=user.__class__, request=None, user=user)
-    return MsgpackResponse(ret)
+    return ret
 
 
 @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses)
@@ -223,7 +222,7 @@ def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_u
     ret = {
         "url": get_dashboard_url(CallbackContext(request.path_params, user=user)),
     }
-    return MsgpackResponse(ret)
+    return ret
 
 
 def signup_save(data: SignupIn, request: Request) -> UserType:
@@ -261,4 +260,4 @@ def signup(data: SignupIn, request: Request):
     user = signup_save(data, request)
     ret = LoginOut.from_orm(user)
     user_signed_up.send(sender=user.__class__, request=None, user=user)
-    return MsgpackResponse(ret)
+    return ret

etebase_server/fastapi/routers/collection.py

@@ -3,34 +3,33 @@ import typing as t
 from asgiref.sync import sync_to_async
 from django.core import exceptions as django_exceptions
 from django.core.files.base import ContentFile
-from django.db import IntegrityError, transaction
+from django.db import transaction, IntegrityError
 from django.db.models import Q, QuerySet
-from fastapi import APIRouter, BackgroundTasks, Depends, Request, status
+from fastapi import APIRouter, Depends, status, Request, BackgroundTasks
 
 from etebase_server.django import models
 from etebase_server.myauth.models import UserType
-
+from .authentication import get_authenticated_user
-from ..db_hack import django_db_cleanup_decorator
+from .websocket import get_ticket, TicketRequest, TicketOut
-from ..dependencies import get_collection, get_collection_queryset, get_item_queryset
+from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError
-from ..exceptions import HttpError, PermissionDenied, ValidationError, transform_validation_error
+from ..msgpack import MsgpackRoute
-from ..msgpack import MsgpackRequest, MsgpackResponse, MsgpackRoute
+from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken
-from ..redis import redisw
-from ..sendfile import sendfile
-from ..stoken_handler import filter_by_stoken, filter_by_stoken_and_limit, get_queryset_stoken, get_stoken_obj
 from ..utils import (
-    PERMISSIONS_READ,
+    get_object_or_404,
-    PERMISSIONS_READWRITE,
-    BaseModel,
     Context,
     Prefetch,
     PrefetchQuery,
-    get_object_or_404,
     is_collection_admin,
     msgpack_encode,
+    BaseModel,
     permission_responses,
+    PERMISSIONS_READ,
+    PERMISSIONS_READWRITE,
 )
-from .authentication import get_authenticated_user
+from ..dependencies import get_collection_queryset, get_item_queryset, get_collection
-from .websocket import TicketOut, TicketRequest, get_ticket
+from ..sendfile import sendfile
+from ..redis import redisw
+from ..db_hack import django_db_cleanup_decorator
 
 collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
 item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@@ -52,7 +51,7 @@ class CollectionItemRevisionInOut(BaseModel):
     chunks: t.List[ChunkType]
 
     class Config:
-        from_attributes = True
+        orm_mode = True
 
     @classmethod
     def from_orm_context(
@@ -72,13 +71,13 @@ class CollectionItemRevisionInOut(BaseModel):
 class CollectionItemCommon(BaseModel):
     uid: str
     version: int
-    encryptionKey: t.Optional[bytes] = None
+    encryptionKey: t.Optional[bytes]
     content: CollectionItemRevisionInOut
 
 
 class CollectionItemOut(CollectionItemCommon):
     class Config:
-        from_attributes = True
+        orm_mode = True
 
     @classmethod
     def from_orm_context(
@@ -93,12 +92,12 @@ class CollectionItemOut(CollectionItemCommon):
 
 
 class CollectionItemIn(CollectionItemCommon):
-    etag: t.Optional[str] = None
+    etag: t.Optional[str]
 
 
 class CollectionCommon(BaseModel):
     # FIXME: remove optional once we finish collection-type-migration
-    collectionType: t.Optional[bytes] = None
+    collectionType: t.Optional[bytes]
     collectionKey: bytes
 
 
@@ -132,27 +131,27 @@ class RemovedMembershipOut(BaseModel):
 
 class CollectionListResponse(BaseModel):
     data: t.List[CollectionOut]
-    stoken: t.Optional[str] = None
+    stoken: t.Optional[str]
     done: bool
 
-    removedMemberships: t.Optional[t.List[RemovedMembershipOut]] = None
+    removedMemberships: t.Optional[t.List[RemovedMembershipOut]]
 
 
 class CollectionItemListResponse(BaseModel):
     data: t.List[CollectionItemOut]
-    stoken: t.Optional[str] = None
+    stoken: t.Optional[str]
     done: bool
 
 
 class CollectionItemRevisionListResponse(BaseModel):
     data: t.List[CollectionItemRevisionInOut]
-    iterator: t.Optional[str] = None
+    iterator: t.Optional[str]
     done: bool
 
 
 class CollectionItemBulkGetIn(BaseModel):
     uid: str
-    etag: t.Optional[str] = None
+    etag: t.Optional[str]
 
 
 class ItemDepIn(BaseModel):
@@ -173,7 +172,7 @@ class ItemDepIn(BaseModel):
 
 class ItemBatchIn(BaseModel):
     items: t.List[CollectionItemIn]
-    deps: t.Optional[t.List[ItemDepIn]] = None
+    deps: t.Optional[t.List[ItemDepIn]]
 
     def validate_db(self):
         if self.deps is not None:
@@ -209,7 +208,7 @@ def collection_list_common(
     prefetch: Prefetch,
 ) -> CollectionListResponse:
     result, new_stoken_obj, done = filter_by_stoken_and_limit(
-        stoken, limit, queryset.filter(items__revisions__current=True), models.Collection.stoken_annotation
+        stoken, limit, queryset, models.Collection.stoken_annotation
     )
     new_stoken = new_stoken_obj and new_stoken_obj.uid
     context = Context(user, prefetch)
@@ -275,7 +274,7 @@ def list_multi(
         Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True)
     )
 
-    return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
+    return collection_list_common(queryset, user, stoken, limit, prefetch)
 
 
 @collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ)
@@ -286,7 +285,7 @@ def collection_list(
     user: UserType = Depends(get_authenticated_user),
     queryset: CollectionQuerySet = Depends(get_collection_queryset),
 ):
-    return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
+    return collection_list_common(queryset, user, stoken, limit, prefetch)
 
 
 def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut):
@@ -342,10 +341,7 @@ def _create(data: CollectionIn, user: UserType):
         # TODO
         process_revisions_for_item(main_item, data.item.content)
 
-        try:
+        collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user)
-            collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user)
-        except IntegrityError:
-            raise ValidationError("bad_collection_type", "collectionType is null")
 
         models.CollectionMember(
             collection=instance,
@@ -368,7 +364,7 @@ def collection_get(
     user: UserType = Depends(get_authenticated_user),
     prefetch: Prefetch = PrefetchQuery,
 ):
-    return MsgpackResponse(CollectionOut.from_orm_context(obj, Context(user, prefetch)))
+    return CollectionOut.from_orm_context(obj, Context(user, prefetch))
 
 
 def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool):
@@ -377,7 +373,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val
     revision_data = item_model.content
     uid = item_model.uid
 
-    Model = models.CollectionItem  # noqa: N806
+    Model = models.CollectionItem
 
     with transaction.atomic():
         instance, created = Model.objects.get_or_create(
@@ -400,7 +396,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val
         if not created:
             # We don't have to use select_for_update here because the unique constraint on current guards against
             # the race condition. But it's a good idea because it'll lock and wait rather than fail.
-            current_revision = instance.revisions.filter(current=True).select_for_update()[0]
+            current_revision = instance.revisions.filter(current=True).select_for_update().first()
             assert current_revision is not None
             current_revision.current = None
             current_revision.save()
@@ -421,7 +417,7 @@ def item_get(
     prefetch: Prefetch = PrefetchQuery,
 ):
     obj = queryset.get(uid=item_uid)
-    return MsgpackResponse(CollectionItemOut.from_orm_context(obj, Context(user, prefetch)))
+    return CollectionItemOut.from_orm_context(obj, Context(user, prefetch))
 
 
 def item_list_common(
@@ -432,7 +428,7 @@ def item_list_common(
     prefetch: Prefetch,
 ) -> CollectionItemListResponse:
     result, new_stoken_obj, done = filter_by_stoken_and_limit(
-        stoken, limit, queryset.filter(revisions__current=True), models.CollectionItem.stoken_annotation
+        stoken, limit, queryset, models.CollectionItem.stoken_annotation
     )
     new_stoken = new_stoken_obj and new_stoken_obj.uid
     context = Context(user, prefetch)
@@ -453,7 +449,7 @@ def item_list(
         queryset = queryset.filter(parent__isnull=True)
 
     response = item_list_common(queryset, user, stoken, limit, prefetch)
-    return MsgpackResponse(response)
+    return response
 
 
 @item_router.post("/item/subscription-ticket/", response_model=TicketOut, dependencies=PERMISSIONS_READ)
@@ -462,7 +458,7 @@ async def item_list_subscription_ticket(
     user: UserType = Depends(get_authenticated_user),
 ):
     """Get an authentication ticket that can be used with the websocket endpoint"""
-    return MsgpackResponse(await get_ticket(TicketRequest(collection=collection.uid), user))
+    return await get_ticket(TicketRequest(collection=collection.uid), user)
 
 
 def item_bulk_common(
@@ -530,12 +526,10 @@ def item_revisions(
     ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result]
     iterator = ret_data[-1].uid if len(result) > 0 else None
 
-    return MsgpackResponse(
+    return CollectionItemRevisionListResponse(
-        CollectionItemRevisionListResponse(
+        data=ret_data,
-            data=ret_data,
+        iterator=iterator,
-            iterator=iterator,
+        done=done,
-            done=done,
-        )
     )
 
 
@@ -565,12 +559,10 @@ def fetch_updates(
     new_stoken = new_stoken or stoken_rev_uid
 
     context = Context(user, prefetch)
-    return MsgpackResponse(
+    return CollectionItemListResponse(
-        CollectionItemListResponse(
+        data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
-            data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
+        stoken=new_stoken,
-            stoken=new_stoken,
+        done=True,  # we always return all the items, so it's always done
-            done=True,  # we always return all the items, so it's always done
-        )
     )
 
 
@@ -582,9 +574,7 @@ def item_transaction(
     stoken: t.Optional[str] = None,
     user: UserType = Depends(get_authenticated_user),
 ):
-    return MsgpackResponse(
+    return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
-        item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
-    )
 
 
 @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE])
@@ -595,9 +585,7 @@ def item_batch(
     stoken: t.Optional[str] = None,
     user: UserType = Depends(get_authenticated_user),
 ):
-    return MsgpackResponse(
+    return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
-        item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
-    )
 
 
 # Chunks
@@ -622,12 +610,7 @@ async def chunk_update(
     collection: models.Collection = Depends(get_collection),
 ):
     # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid)
-    if isinstance(request, MsgpackRequest):
+    content_file = ContentFile(await request.body())
-        body = await request.raw_body()
-    else:
-        body = await request.body()
-
-    content_file = ContentFile(body)
     try:
         await chunk_save(chunk_uid, collection, content_file)
     except IntegrityError:

etebase_server/fastapi/routers/invitation.py

@@ -1,27 +1,26 @@
 import typing as t
 
-from django.db import IntegrityError, transaction
+from django.db import transaction, IntegrityError
 from django.db.models import QuerySet
-from fastapi import APIRouter, Depends, Request, status
+from fastapi import APIRouter, Depends, status, Request
 
 from etebase_server.django import models
-from etebase_server.django.utils import CallbackContext, get_user_queryset
+from etebase_server.django.utils import get_user_queryset, CallbackContext
 from etebase_server.myauth.models import UserType, get_typed_user_model
-
+from .authentication import get_authenticated_user
-from ..db_hack import django_db_cleanup_decorator
 from ..exceptions import HttpError, PermissionDenied
-from ..msgpack import MsgpackResponse, MsgpackRoute
+from ..msgpack import MsgpackRoute
 from ..utils import (
-    PERMISSIONS_READ,
-    PERMISSIONS_READWRITE,
-    BaseModel,
-    Context,
     get_object_or_404,
     get_user_username_email_kwargs,
+    Context,
     is_collection_admin,
+    BaseModel,
     permission_responses,
+    PERMISSIONS_READ,
+    PERMISSIONS_READWRITE,
 )
-from .authentication import get_authenticated_user
+from ..db_hack import django_db_cleanup_decorator
 
 User = get_typed_user_model()
 invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@@ -34,7 +33,7 @@ class UserInfoOut(BaseModel):
     pubkey: bytes
 
     class Config:
-        from_attributes = True
+        orm_mode = True
 
     @classmethod
     def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut":
@@ -67,7 +66,7 @@ class CollectionInvitationOut(CollectionInvitationCommon):
     fromPubkey: bytes
 
     class Config:
-        from_attributes = True
+        orm_mode = True
 
     @classmethod
     def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut":
@@ -85,7 +84,7 @@ class CollectionInvitationOut(CollectionInvitationCommon):
 
 class InvitationListResponse(BaseModel):
     data: t.List[CollectionInvitationOut]
-    iterator: t.Optional[str] = None
+    iterator: t.Optional[str]
     done: bool
 
 
@@ -121,7 +120,7 @@ def list_common(
     iterator = ret_data[-1].uid if len(result) > 0 else None
 
     return InvitationListResponse(
-        data=[CollectionInvitationOut.from_orm(x) for x in ret_data],
+        data=ret_data,
         iterator=iterator,
         done=done,
     )
@@ -133,7 +132,7 @@ def incoming_list(
     limit: int = 50,
     queryset: InvitationQuerySet = Depends(get_incoming_queryset),
 ):
-    return MsgpackResponse(list_common(queryset, iterator, limit))
+    return list_common(queryset, iterator, limit)
 
 
 @invitation_incoming_router.get(
@@ -144,7 +143,7 @@ def incoming_get(
     queryset: InvitationQuerySet = Depends(get_incoming_queryset),
 ):
     obj = get_object_or_404(queryset, uid=invitation_uid)
-    return MsgpackResponse(CollectionInvitationOut.from_orm(obj))
+    return CollectionInvitationOut.from_orm(obj)
 
 
 @invitation_incoming_router.delete(
@@ -219,7 +218,7 @@ def outgoing_list(
     limit: int = 50,
     queryset: InvitationQuerySet = Depends(get_outgoing_queryset),
 ):
-    return MsgpackResponse(list_common(queryset, iterator, limit))
+    return list_common(queryset, iterator, limit)
 
 
 @invitation_outgoing_router.delete(
@@ -242,4 +241,4 @@ def outgoing_fetch_user_profile(
     kwargs = get_user_username_email_kwargs(username)
     user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs)
     user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user)
-    return MsgpackResponse(UserInfoOut.from_orm(user_info))
+    return UserInfoOut.from_orm(user_info)

etebase_server/fastapi/routers/member.py

@@ -6,12 +6,12 @@ from fastapi import APIRouter, Depends, status
 
 from etebase_server.django import models
 from etebase_server.myauth.models import UserType, get_typed_user_model
-
-from ..db_hack import django_db_cleanup_decorator
-from ..msgpack import MsgpackResponse, MsgpackRoute
-from ..stoken_handler import filter_by_stoken_and_limit
-from ..utils import PERMISSIONS_READ, PERMISSIONS_READWRITE, BaseModel, get_object_or_404, permission_responses
 from .authentication import get_authenticated_user
+from ..msgpack import MsgpackRoute
+from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE
+from ..stoken_handler import filter_by_stoken_and_limit
+from ..db_hack import django_db_cleanup_decorator
+
 from .collection import get_collection, verify_collection_admin
 
 User = get_typed_user_model()
@@ -39,7 +39,7 @@ class CollectionMemberOut(BaseModel):
     accessLevel: models.AccessLevels
 
     class Config:
-        from_attributes = True
+        orm_mode = True
 
     @classmethod
     def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut":
@@ -48,7 +48,7 @@ class CollectionMemberOut(BaseModel):
 
 class MemberListResponse(BaseModel):
     data: t.List[CollectionMemberOut]
-    iterator: t.Optional[str] = None
+    iterator: t.Optional[str]
     done: bool
 
 
@@ -66,12 +66,10 @@ def member_list(
     )
     new_stoken = new_stoken_obj and new_stoken_obj.uid
 
-    return MsgpackResponse(
+    return MemberListResponse(
-        MemberListResponse(
+        data=[CollectionMemberOut.from_orm(item) for item in result],
-            data=[CollectionMemberOut.from_orm(item) for item in result],
+        iterator=new_stoken,
-            iterator=new_stoken,
+        done=done,
-            done=done,
-        )
     )
 
 

etebase_server/fastapi/routers/test_reset_view.py

@@ -3,12 +3,11 @@ from django.db import transaction
 from django.shortcuts import get_object_or_404
 from fastapi import APIRouter, Request, status
 
-from etebase_server.django.utils import CallbackContext, get_user_queryset
+from etebase_server.django.utils import get_user_queryset, CallbackContext
-from etebase_server.myauth.models import get_typed_user_model
-
-from ..exceptions import HttpError
-from ..msgpack import MsgpackRoute
 from .authentication import SignupIn, signup_save
+from ..msgpack import MsgpackRoute
+from ..exceptions import HttpError
+from etebase_server.myauth.models import get_typed_user_model
 
 test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"])
 User = get_typed_user_model()

etebase_server/fastapi/routers/websocket.py

@@ -1,13 +1,12 @@
 import asyncio
 import typing as t
 
-import nacl.encoding
+import aioredis
-import nacl.utils
 from asgiref.sync import sync_to_async
 from django.db.models import QuerySet
 from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status
-from redis import asyncio as aioredis
+import nacl.encoding
-from redis.exceptions import ConnectionError
+import nacl.utils
 
 from etebase_server.django import models
 from etebase_server.django.utils import CallbackContext, get_user_queryset
@@ -19,6 +18,7 @@ from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode
 from ..redis import redisw
 from ..utils import BaseModel, permission_responses
 
+
 User = get_typed_user_model()
 websocket_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
 CollectionQuerySet = QuerySet[models.Collection]
@@ -51,7 +51,7 @@ async def get_ticket(
     uid = nacl.encoding.URLSafeBase64Encoder.encode(nacl.utils.random(32))
     ticket_model = TicketInner(user=user.id, req=ticket_request)
     ticket_raw = msgpack_encode(ticket_model.dict())
-    await redisw.redis.set(uid, ticket_raw, ex=TICKET_VALIDITY_SECONDS * 1000)
+    await redisw.redis.set(uid, ticket_raw, expire=TICKET_VALIDITY_SECONDS * 1000)
     return TicketOut(ticket=uid)
 
 
@@ -103,9 +103,9 @@ async def send_item_updates(
 
 async def redis_connector(websocket: WebSocket, ticket_model: TicketInner, user: UserType, stoken: t.Optional[str]):
     async def producer_handler(r: aioredis.Redis, ws: WebSocket):
-        pubsub = r.pubsub()
         channel_name = f"col.{ticket_model.req.collection}"
-        await pubsub.subscribe(channel_name)
+        (channel,) = await r.psubscribe(channel_name)
+        assert isinstance(channel, aioredis.Channel)
 
         # Send missing items if we are not up to date
         queryset: QuerySet[models.Collection] = get_collection_queryset(user)
@@ -117,29 +117,26 @@ async def redis_connector(websocket: WebSocket, ticket_model: TicketInner, user:
             return
         await send_item_updates(websocket, collection, user, stoken)
 
-        async def handle_message():
-            msg = await pubsub.get_message(ignore_subscribe_messages=True, timeout=20)
-            message_raw = t.cast(t.Optional[t.Tuple[str, bytes]], msg)
-            if message_raw:
-                _, message = message_raw
-                await ws.send_bytes(message)
-
         try:
             while True:
                 # We wait on the websocket so we fail if web sockets fail or get data
                 receive = asyncio.create_task(websocket.receive())
                 done, pending = await asyncio.wait(
-                    {receive, handle_message()},
+                    {receive, channel.wait_message()}, return_when=asyncio.FIRST_COMPLETED
-                    return_when=asyncio.FIRST_COMPLETED,
                 )
                 for task in pending:
                     task.cancel()
                 if receive in done:
-                    # Web socket should never receive any data
+                    # Web socket should never receieve any data
                     await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
                     return
 
-        except ConnectionError:
+                message_raw = t.cast(t.Optional[t.Tuple[str, bytes]], await channel.get())
+                if message_raw:
+                    _, message = message_raw
+                    await ws.send_bytes(message)
+
+        except aioredis.errors.ConnectionClosedError:
             await websocket.close(code=status.WS_1012_SERVICE_RESTART)
         except WebSocketDisconnect:
             pass

etebase_server/fastapi/sendfile/utils.py

@@ -1,15 +1,14 @@
-import logging
-import os
 from functools import lru_cache
 from importlib import import_module
 from pathlib import Path, PurePath
 from urllib.parse import quote
+import logging
+
+from fastapi import status
+from ..exceptions import HttpError
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
-from fastapi import status
-
-from ..exceptions import HttpError
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +32,9 @@ def _convert_file_to_url(path):
     path_obj = PurePath(path)
 
     relpath = path_obj.relative_to(path_root)
-    url = os.path.normpath(str(url_root / relpath))
+    # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
+    # already instantiated Path object
+    url = relpath._flavour.pathmod.normpath(str(url_root / relpath))
 
     return quote(str(url))
 
@@ -47,7 +48,9 @@ def _sanitize_path(filepath):
     filepath_obj = Path(filepath)
 
     # get absolute path
-    filepath_abs = Path(os.path.normpath(str(path_root / filepath_obj)))
+    # Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
+    # already instantiated Path object
+    filepath_abs = Path(filepath_obj._flavour.pathmod.normpath(str(path_root / filepath_obj)))
 
     # if filepath_abs is not relative to path_root, relative_to throws an error
     try:

etebase_server/fastapi/stoken_handler.py

@@ -47,6 +47,7 @@ def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]:
 def filter_by_stoken_and_limit(
     stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation
 ) -> t.Tuple[list, t.Optional[Stoken], bool]:
+
     queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation)
 
     result = list(queryset[: limit + 1])

etebase_server/fastapi/utils.py

@@ -1,13 +1,14 @@
-import base64
 import dataclasses
 import typing as t
-
-import msgpack
-from django.core.exceptions import ObjectDoesNotExist
-from django.db.models import Model, QuerySet
-from fastapi import Depends, Query, status
-from pydantic import BaseModel as PyBaseModel
 from typing_extensions import Literal
+import msgpack
+import base64
+
+from fastapi import status, Query, Depends
+from pydantic import BaseModel as PyBaseModel
+
+from django.db.models import Model, QuerySet
+from django.core.exceptions import ObjectDoesNotExist
 
 from etebase_server.django import app_settings
 from etebase_server.django.models import AccessLevels
@@ -25,7 +26,10 @@ T = t.TypeVar("T", bound=Model, covariant=True)
 
 
 class BaseModel(PyBaseModel):
-    pass
+    class Config:
+        json_encoders = {
+            bytes: lambda x: x,
+        }
 
 
 @dataclasses.dataclass

etebase_server/myauth/admin.py

@@ -1,21 +1,12 @@
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin as DjangoUserAdmin
-
-from .forms import AdminUserCreationForm
 from .models import User
+from .forms import AdminUserCreationForm
 
 
 class UserAdmin(DjangoUserAdmin):
     add_form = AdminUserCreationForm
-    add_fieldsets = (
+    add_fieldsets = ((None, {"classes": ("wide",), "fields": ("username",),}),)
-        (
-            None,
-            {
-                "classes": ("wide",),
-                "fields": ("username",),
-            },
-        ),
-    )
 
 
 admin.site.register(User, UserAdmin)

etebase_server/myauth/forms.py

@@ -1,6 +1,5 @@
 from django import forms
 from django.contrib.auth.forms import UsernameField
-
 from etebase_server.myauth.models import get_typed_user_model
 
 User = get_typed_user_model()

etebase_server/myauth/ldap.py

@@ -1,15 +1,15 @@
 import logging
 
-import ldap
+from django.utils import timezone
 from django.conf import settings
 from django.core.exceptions import PermissionDenied as DjangoPermissionDenied
-from django.utils import timezone
-from fastapi import Depends
-
 from etebase_server.django.utils import CallbackContext
+from etebase_server.myauth.models import get_typed_user_model, UserType
 from etebase_server.fastapi.dependencies import get_authenticated_user
 from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied
-from etebase_server.myauth.models import UserType, get_typed_user_model
+from fastapi import Depends
+
+import ldap
 
 User = get_typed_user_model()
 
@@ -54,7 +54,7 @@ class LDAPConnection:
         try:
             self.__ldap_connection.simple_bind_s(ldap_setting("BIND_DN", ""), password)
         except ldap.LDAPError as err:
-            logging.error(f"LDAP Error occurring during bind: {err.desc}")
+            logging.error(f"LDAP Error occuring during bind: {err.desc}")
 
     def __is_cache_valid(self, username):
         """Returns True if the cache entry is still valid. Returns False otherwise."""
@@ -85,7 +85,7 @@ class LDAPConnection:
             # we may expect ldap.NO_RESULTS_RETURNED, but not any other error
             return False
         except ldap.LDAPError as err:
-            logging.error(f"Error occurred while performing an LDAP query: {err.desc}")
+            logging.error(f"Error occured while performing an LDAP query: {err.desc}")
             return False
 
         if len(result) == 1:

etebase_server/myauth/models.py

@@ -1,13 +1,21 @@
 import typing as t
 
 from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager
-from django.contrib.auth.validators import UnicodeUsernameValidator
+from django.core import validators
 from django.db import models
+from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _
 
 
+@deconstructible
+class UnicodeUsernameValidator(validators.RegexValidator):
+    regex = r"^[\w.-]+\Z"
+    message = _("Enter a valid username. This value may contain only letters, " "numbers, and ./-/_ characters.")
+    flags = 0
+
+
 class UserManager(DjangoUserManager):
-    def get_by_natural_key(self, username: t.Optional[str]):
+    def get_by_natural_key(self, username: str):
         return self.get(**{self.model.USERNAME_FIELD + "__iexact": username})
 
 

etebase_server/myauth/tests.py

@@ -1 +1,3 @@
+from django.test import TestCase
+
 # Create your tests here.

etebase_server/myauth/views.py

@@ -1 +1,3 @@
+from django.shortcuts import render
+
 # Create your views here.

etebase_server/settings.py

@@ -10,9 +10,8 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.0/ref/settings/
 """
 
-import configparser
 import os
-
+import configparser
 from .utils import get_secret_from_file
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -21,7 +20,6 @@ BASE_DIR = os.path.dirname(SOURCE_DIR)
 
 AUTH_USER_MODEL = "myauth.User"
 
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
@@ -46,7 +44,7 @@ DATABASES = {
     }
 }
 
-DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 # Application definition
 
@@ -128,12 +126,13 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 
-STATIC_ROOT = os.path.join(BASE_DIR, "static")
 STATIC_URL = "/static/"
+STATIC_ROOT = os.environ.get("DJANGO_STATIC_ROOT", os.path.join(BASE_DIR, "static"))
 
-MEDIA_ROOT = os.path.join(BASE_DIR, "media")
+MEDIA_ROOT = os.environ.get("DJANGO_MEDIA_ROOT", os.path.join(BASE_DIR, "media"))
 MEDIA_URL = "/user-media/"
 
+
 # Define where to find configuration files
 config_locations = [
     os.environ.get("ETEBASE_EASY_CONFIG_PATH", ""),
@@ -164,8 +163,6 @@ if any(os.path.isfile(x) for x in config_locations):
 
     if "allowed_hosts" in config:
         ALLOWED_HOSTS = [y for x, y in config.items("allowed_hosts")]
-        CSRF_TRUSTED_ORIGINS = ["https://" + y for x, y in config.items("allowed_hosts")] + \
-                               ["http://" + y for x, y in config.items("allowed_hosts")]
 
     if "database" in config:
         DATABASES = {"default": {x.upper(): y for x, y in config.items("database")}}
@@ -196,15 +193,9 @@ if any(os.path.isfile(x) for x in config_locations):
 SENDFILE_BACKEND = "etebase_server.fastapi.sendfile.backends.simple"
 SENDFILE_ROOT = MEDIA_ROOT
 
-if "DJANGO_STATIC_ROOT" in os.environ:
-    STATIC_ROOT = os.environ["DJANGO_STATIC_ROOT"]
-
-if "DJANGO_MEDIA_ROOT" in os.environ:
-    MEDIA_ROOT = os.environ["DJANGO_MEDIA_ROOT"]
-
 # Make an `etebase_server_settings` module available to override settings.
 try:
-    from etebase_server_settings import *  # noqa: F403
+    from etebase_server_settings import *
 except ImportError:
     pass
 

etebase_server/urls.py

@@ -1,6 +1,11 @@
+import os
+
+from django.conf import settings
 from django.contrib import admin
 from django.urls import path
 from django.views.generic import TemplateView
+from django.views.static import serve
+from django.contrib.staticfiles import finders
 
 urlpatterns = [
     path("admin/", admin.site.urls),

etebase_server/utils.py

@@ -12,11 +12,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
+from django.core.management import utils
 import os
 import stat
 
-from django.core.management import utils
-
 
 def get_secret_from_file(path):
     try:

manage.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 """Django's command-line utility for administrative tasks."""
-
 import os
 import sys
 

pyproject.toml

@@ -4,25 +4,3 @@ line-length = 120
 [build-system]
 requires = ["setuptools>=42"]
 build-backend = "setuptools.build_meta"
-
-[tool.ruff]
-line-length = 120
-exclude = [
-    ".git",
-    ".git-rewrite",
-    ".mypy_cache",
-    ".pytype",
-    ".ruff_cache",
-    ".venv",
-    "build",
-    "dist",
-    "node_modules",
-    "migrations", # Alembic migrations
-]
-
-[tool.ruff.lint]
-select = ["E", "F", "I", "N", "T20", "W"]
-ignore = ["E203", "E501", "E711", "E712", "N803", "N815", "N818", "T201"]
-
-[tool.ruff.lint.isort]
-combine-as-imports = true

requirements-dev.txt

@@ -1,53 +1,65 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with python 3.10
-# by the following command:
+# To update, run:
 #
 #    pip-compile --output-file=requirements-dev.txt requirements.in/development.txt
 #
-asgiref==3.8.1
+asgiref==3.5.0
-    # via
+    # via django
-    #   django
+black==22.1.0
-    #   django-stubs
-build==1.2.1
-    # via pip-tools
-click==8.1.7
-    # via pip-tools
-coverage==7.5.3
     # via -r requirements.in/development.txt
-django==5.0.6
+click==8.0.4
     # via
+    #   black
+    #   pip-tools
+coverage==6.3.2
+    # via -r requirements.in/development.txt
+django==3.2.13
+    # via
+    #   -r requirements.in/development.txt
     #   django-stubs
     #   django-stubs-ext
-django-stubs==5.0.2
+django-stubs==1.9.0
     # via -r requirements.in/development.txt
-django-stubs-ext==5.0.2
+django-stubs-ext==0.3.1
     # via django-stubs
-mypy==1.10.0
+mypy==0.941
-    # via -r requirements.in/development.txt
+    # via django-stubs
-mypy-extensions==1.0.0
+mypy-extensions==0.4.3
-    # via mypy
-packaging==24.0
-    # via build
-pip-tools==7.4.1
-    # via -r requirements.in/development.txt
-pyproject-hooks==1.1.0
     # via
-    #   build
+    #   black
-    #   pip-tools
+    #   mypy
-pywatchman==2.0.0
+pathspec==0.9.0
+    # via black
+pep517==0.12.0
+    # via pip-tools
+pip-tools==6.5.1
     # via -r requirements.in/development.txt
-ruff==0.4.8
+platformdirs==2.5.1
-    # via -r requirements.in/development.txt
+    # via black
-sqlparse==0.5.0
+pytz==2022.1
     # via django
-types-pyyaml==6.0.12.20240311
+pywatchman==1.4.1
+    # via -r requirements.in/development.txt
+sqlparse==0.4.2
+    # via django
+toml==0.10.2
     # via django-stubs
-typing-extensions==4.12.2
+tomli==2.0.1
+    # via
+    #   black
+    #   mypy
+    #   pep517
+types-pytz==2021.3.6
+    # via django-stubs
+types-pyyaml==6.0.5
+    # via django-stubs
+typing-extensions==4.1.1
     # via
     #   django-stubs
     #   django-stubs-ext
     #   mypy
-wheel==0.43.0
+wheel==0.37.1
     # via pip-tools
 
 # The following packages are considered to be unsafe in a requirements file:

requirements.in/base.txt

@@ -1,9 +1,8 @@
-django>=4.0,<5.0
+django<4.0
 msgpack
 pynacl
-fastapi>=0.104
+fastapi
-pydantic>=2.0.0
 typing_extensions
 uvicorn[standard]
 aiofiles
-redis>=4.2.0rc1
+aioredis

requirements.in/development.txt

@@ -1,6 +1,6 @@
 coverage
 pip-tools
 pywatchman
-ruff
+black
-mypy
 django-stubs
+django<4.0

requirements.txt

@@ -1,117 +1,67 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with python 3.10
-# by the following command:
+# To update, run:
 #
 #    pip-compile --output-file=requirements.txt requirements.in/base.txt
 #
-aiofiles==23.2.1
+aiofiles==0.8.0
     # via -r requirements.in/base.txt
-annotated-types==0.7.0
+aioredis==2.0.1
-    # via pydantic
+    # via -r requirements.in/base.txt
-anyio==4.4.0
+anyio==3.5.0
     # via
-    #   httpx
     #   starlette
-    #   watchfiles
+    #   watchgod
-asgiref==3.8.1
+asgiref==3.5.0
-    # via django
-certifi==2024.6.2
     # via
-    #   httpcore
+    #   django
-    #   httpx
+    #   uvicorn
-cffi==1.16.0
+async-timeout==4.0.2
+    # via aioredis
+cffi==1.15.0
     # via pynacl
-click==8.1.7
+click==8.0.4
-    # via
-    #   typer
-    #   uvicorn
-django==4.2.14
-    # via -r requirements.in/base.txt
-dnspython==2.6.1
-    # via email-validator
-email-validator==2.1.1
-    # via fastapi
-fastapi==0.111.0
-    # via -r requirements.in/base.txt
-fastapi-cli==0.0.4
-    # via fastapi
-h11==0.14.0
-    # via
-    #   httpcore
-    #   uvicorn
-httpcore==1.0.5
-    # via httpx
-httptools==0.6.1
     # via uvicorn
-httpx==0.27.0
+django==3.2.13
-    # via fastapi
-idna==3.7
-    # via
-    #   anyio
-    #   email-validator
-    #   httpx
-jinja2==3.1.4
-    # via fastapi
-markdown-it-py==3.0.0
-    # via rich
-markupsafe==2.1.5
-    # via jinja2
-mdurl==0.1.2
-    # via markdown-it-py
-msgpack==1.0.8
     # via -r requirements.in/base.txt
-orjson==3.10.3
+fastapi==0.75.0
-    # via fastapi
+    # via -r requirements.in/base.txt
-pycparser==2.22
+h11==0.13.0
+    # via uvicorn
+httptools==0.4.0
+    # via uvicorn
+idna==3.3
+    # via anyio
+msgpack==1.0.3
+    # via -r requirements.in/base.txt
+pycparser==2.21
     # via cffi
-pydantic==2.7.3
+pydantic==1.9.0
-    # via
+    # via fastapi
-    #   -r requirements.in/base.txt
-    #   fastapi
-pydantic-core==2.18.4
-    # via pydantic
-pygments==2.18.0
-    # via rich
 pynacl==1.5.0
     # via -r requirements.in/base.txt
-python-dotenv==1.0.1
+python-dotenv==0.19.2
     # via uvicorn
-python-multipart==0.0.9
+pytz==2022.1
-    # via fastapi
-pyyaml==6.0.1
-    # via uvicorn
-redis==5.1.0b6
-    # via -r requirements.in/base.txt
-rich==13.7.1
-    # via typer
-shellingham==1.5.4
-    # via typer
-sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
-sqlparse==0.5.0
     # via django
-starlette==0.37.2
+pyyaml==6.0
+    # via uvicorn
+sniffio==1.2.0
+    # via anyio
+sqlparse==0.4.2
+    # via django
+starlette==0.17.1
     # via fastapi
-typer==0.12.3
+typing-extensions==4.1.1
-    # via fastapi-cli
-typing-extensions==4.12.2
     # via
     #   -r requirements.in/base.txt
-    #   fastapi
+    #   aioredis
     #   pydantic
-    #   pydantic-core
+uvicorn[standard]==0.17.6
-    #   typer
+    # via -r requirements.in/base.txt
-ujson==5.10.0
+uvloop==0.16.0
-    # via fastapi
-uvicorn[standard]==0.30.1
-    # via
-    #   -r requirements.in/base.txt
-    #   fastapi
-uvloop==0.19.0
     # via uvicorn
-watchfiles==0.22.0
+watchgod==0.8.1
     # via uvicorn
-websockets==12.0
+websockets==10.2
     # via uvicorn

setup.py

@@ -1,20 +1,20 @@
 from setuptools import find_packages, setup
 
 setup(
-    name="etebase_server",
+    name='etebase_server',
-    version="0.14.2",
+    version='0.9.1',
-    description="An Etebase (EteSync 2.0) server",
+    description='An Etebase (EteSync 2.0) server',
-    url="https://www.etebase.com/",
+    url='https://www.etebase.com/',
     classifiers=[
-        "Development Status :: 5 - Production/Stable",
+        'Development Status :: 5 - Production/Stable',
-        "Framework :: Django",
+        'Framework :: Django',
-        "Framework :: FastAPI",
+        'Framework :: FastAPI',
-        "Intended Audience :: System Administrators",
+        'Intended Audience :: System Administrators',
-        "License :: OSI Approved :: GNU Affero General Public License v3",
+        'License :: OSI Approved :: GNU Affero General Public License v3',
     ],
-    packages=find_packages(include=["etebase_server", "etebase_server.*"]),
+    packages=find_packages(include=['etebase_server', 'etebase_server.*']),
-    install_requires=list(open("requirements.in/base.txt")),
+    install_requires=list(open('requirements.in/base.txt')),
     package_data={
-        "etebase_server": ["templates/*"],
+        'etebase_server': ['templates/*'],
-    },
+    }
 )