1
0
mirror of https://github.com/etesync/server synced 2025-04-24 20:29:05 +00:00

Compare commits

..

No commits in common. "master" and "v0.13.0" have entirely different histories.

37 changed files with 275 additions and 382 deletions

View File

@ -1,19 +1,5 @@
# Changelog
## Version 0.14.2
- Fix issue with some requests failing in some scenarios with the JS client.
- The JS client was omitting optional fields which were accidentally made to be required. It happened because pydantic v2 changed the behavior in a few ways (for the better fwiw) and we missed a few places when upgrading.
## Version 0.14.1
- Fix issue with serializing non utf8 422 errors
- Appease django warnings about default auto field
## Version 0.14.0
- Upgrade to latest FastAPI and Pydantic for compatibility with Python 3.12
## Version 0.13.1
- Fix requirements definition for django.
## Version 0.13.0
- Update Django dependency to 4 (LTS)

View File

@ -1,3 +1 @@
from .app_settings_inner import app_settings
__all__ = ["app_settings"]

View File

@ -34,11 +34,11 @@ class AppSettings:
return getattr(settings, self.prefix + name, dflt)
@cached_property
def REDIS_URI(self) -> t.Optional[str]: # noqa: N802
def REDIS_URI(self) -> t.Optional[str]: # pylint: disable=invalid-name
return self._setting("REDIS_URI", None)
@cached_property
def API_PERMISSIONS_READ(self): # noqa: N802
def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name
perms = self._setting("API_PERMISSIONS_READ", tuple())
ret = []
for perm in perms:
@ -46,7 +46,7 @@ class AppSettings:
return ret
@cached_property
def API_PERMISSIONS_WRITE(self): # noqa: N802
def API_PERMISSIONS_WRITE(self): # pylint: disable=invalid-name
perms = self._setting("API_PERMISSIONS_WRITE", tuple())
ret = []
for perm in perms:
@ -54,35 +54,35 @@ class AppSettings:
return ret
@cached_property
def GET_USER_QUERYSET_FUNC(self): # noqa: N802
def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name
get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None)
if get_user_queryset is not None:
return self.import_from_str(get_user_queryset)
return None
@cached_property
def CREATE_USER_FUNC(self): # noqa: N802
def CREATE_USER_FUNC(self): # pylint: disable=invalid-name
func = self._setting("CREATE_USER_FUNC", None)
if func is not None:
return self.import_from_str(func)
return None
@cached_property
def DASHBOARD_URL_FUNC(self): # noqa: N802
def DASHBOARD_URL_FUNC(self): # pylint: disable=invalid-name
func = self._setting("DASHBOARD_URL_FUNC", None)
if func is not None:
return self.import_from_str(func)
return None
@cached_property
def CHUNK_PATH_FUNC(self): # noqa: N802
def CHUNK_PATH_FUNC(self): # pylint: disable=invalid-name
func = self._setting("CHUNK_PATH_FUNC", None)
if func is not None:
return self.import_from_str(func)
return None
@cached_property
def CHALLENGE_VALID_SECONDS(self): # noqa: N802
def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name
return self._setting("CHALLENGE_VALID_SECONDS", 60)

View File

@ -15,21 +15,22 @@
import typing as t
from pathlib import Path
from django.db import models, transaction
from django.conf import settings
from django.core.validators import RegexValidator
from django.db import models, transaction
from django.db.models import Max, Value as Val
from django.db.models import Max, Value as V
from django.db.models.functions import Coalesce, Greatest
from django.utils.crypto import get_random_string
from django.utils.functional import cached_property
from django.utils.crypto import get_random_string
from . import app_settings
UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID")
def stoken_annotation_builder(stoken_id_fields: t.List[str]):
aggr_fields = [Coalesce(Max(field), Val(0)) for field in stoken_id_fields]
aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields]
return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0]

View File

@ -1,7 +1,6 @@
from django.db import models
from django.utils import timezone
from django.utils.crypto import get_random_string
from etebase_server.myauth.models import get_typed_user_model
User = get_typed_user_model()
@ -16,6 +15,7 @@ def get_default_expiry():
class AuthToken(models.Model):
key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key)
user = models.ForeignKey(User, null=False, blank=False, related_name="auth_token_set", on_delete=models.CASCADE)
created = models.DateTimeField(auto_now_add=True)

View File

@ -1,13 +1,13 @@
import typing as t
from dataclasses import dataclass
from django.core.exceptions import PermissionDenied
from django.db.models import QuerySet
from django.core.exceptions import PermissionDenied
from etebase_server.myauth.models import UserType, get_typed_user_model
from . import app_settings
User = get_typed_user_model()

View File

@ -2,7 +2,6 @@
FIXME: this whole function is a hack around the django db limitations due to how db connections are cached and cleaned.
Essentially django assumes there's the django request dispatcher to automatically clean up after the ORM.
"""
import typing as t
from functools import wraps

View File

@ -1,17 +1,18 @@
import dataclasses
from django.db.models import QuerySet
from django.utils import timezone
from fastapi import Depends
from fastapi.security import APIKeyHeader
from django.utils import timezone
from django.db.models import QuerySet
from etebase_server.django import models
from etebase_server.django.token_auth.models import AuthToken, get_default_expiry
from etebase_server.myauth.models import UserType, get_typed_user_model
from .db_hack import django_db_cleanup_decorator
from .exceptions import AuthenticationFailed
from .utils import get_object_or_404
from .db_hack import django_db_cleanup_decorator
User = get_typed_user_model()
token_scheme = APIKeyHeader(name="Authorization")

View File

@ -1,8 +1,8 @@
from fastapi import status, HTTPException
import typing as t
from django.core.exceptions import ValidationError as DjangoValidationError
from fastapi import HTTPException, status
from pydantic import BaseModel
from django.core.exceptions import ValidationError as DjangoValidationError
class HttpErrorField(BaseModel):
@ -11,7 +11,7 @@ class HttpErrorField(BaseModel):
detail: str
class Config:
from_attributes = True
orm_mode = True
class HttpErrorOut(BaseModel):
@ -20,7 +20,7 @@ class HttpErrorOut(BaseModel):
errors: t.Optional[t.List[HttpErrorField]]
class Config:
from_attributes = True
orm_mode = True
class CustomHttpException(HTTPException):

View File

@ -1,20 +1,19 @@
import typing as t
from django.conf import settings
# Not at the top of the file because we first need to setup django
from fastapi import FastAPI, Request, status
from fastapi.exceptions import RequestValidationError
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.trustedhost import TrustedHostMiddleware
from fastapi.staticfiles import StaticFiles
from etebase_server.django import app_settings
from .exceptions import CustomHttpException
from .msgpack import MsgpackResponse
from .routers.authentication import authentication_router
from .routers.collection import collection_router, item_router
from .routers.invitation import invitation_incoming_router, invitation_outgoing_router
from .routers.member import member_router
from .routers.invitation import invitation_incoming_router, invitation_outgoing_router
from .routers.websocket import websocket_router
@ -25,12 +24,12 @@ def create_application(prefix="", middlewares=[]):
externalDocs={
"url": "https://docs.etebase.com",
"description": "Docs about the API specifications and clients.",
},
}
# FIXME: version="2.5.0",
)
VERSION = "v1" # noqa: N806
BASE_PATH = f"{prefix}/api/{VERSION}" # noqa: N806
COLLECTION_UID_MARKER = "{collection_uid}" # noqa: N806
VERSION = "v1"
BASE_PATH = f"{prefix}/api/{VERSION}"
COLLECTION_UID_MARKER = "{collection_uid}"
app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"])
app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"])
app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"])
@ -76,16 +75,6 @@ def create_application(prefix="", middlewares=[]):
async def custom_exception_handler(request: Request, exc: CustomHttpException):
return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict)
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
from pydantic import TypeAdapter
errors = TypeAdapter(t.Dict[str, t.Any])
return MsgpackResponse(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
content=errors.dump_python({"detail": exc.errors()}),
)
app.mount(settings.STATIC_URL, StaticFiles(directory=settings.STATIC_ROOT), name="static")
return app

View File

@ -5,19 +5,16 @@ from pydantic import BaseModel
from starlette.requests import Request
from starlette.responses import Response
from .utils import msgpack_encode, msgpack_decode
from .db_hack import django_db_cleanup_decorator
from .utils import msgpack_decode, msgpack_encode
class MsgpackRequest(Request):
media_type = "application/msgpack"
async def raw_body(self) -> bytes:
return await super().body()
async def body(self) -> bytes:
if not hasattr(self, "_json"):
body = await self.raw_body()
body = await super().body()
self._json = msgpack_decode(body)
return self._json
@ -30,7 +27,7 @@ class MsgpackResponse(Response):
return b""
if isinstance(content, BaseModel):
content = content.model_dump()
content = content.dict()
return msgpack_encode(content)
@ -51,7 +48,7 @@ class MsgpackRoute(APIRoute):
status_code=self.status_code,
# use custom response class or fallback on default self.response_class
response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class),
response_field=self.response_field,
response_field=self.secure_cloned_response_field,
response_model_include=self.response_model_include,
response_model_exclude=self.response_model_exclude,
response_model_by_alias=self.response_model_by_alias,
@ -63,14 +60,14 @@ class MsgpackRoute(APIRoute):
def get_route_handler(self) -> t.Callable:
async def custom_route_handler(request: Request) -> Response:
content_type = request.headers.get("Content-Type")
if content_type is not None:
try:
request_cls = self.REQUESTS_CLASSES[content_type]
request = request_cls(request.scope, request.receive)
except KeyError:
# nothing registered to handle content_type, process given requests as-is
pass
try:
request_cls = self.REQUESTS_CLASSES[content_type]
request = request_cls(request.scope, request.receive)
except KeyError:
# nothing registered to handle content_type, process given requests as-is
pass
accept = request.headers.get("Accept")
route_handler = self._get_media_type_route_handler(accept)

View File

@ -1,5 +1,4 @@
import typing as t
from redis import asyncio as aioredis
from etebase_server.django import app_settings

View File

@ -1,4 +1,5 @@
import typing as t
from typing_extensions import Literal
from datetime import datetime
import nacl
@ -7,24 +8,22 @@ import nacl.hash
import nacl.secret
import nacl.signing
from django.conf import settings
from django.contrib.auth import user_logged_in, user_logged_out
from django.contrib.auth import user_logged_out, user_logged_in
from django.core import exceptions as django_exceptions
from django.db import transaction
from django.utils.functional import cached_property
from fastapi import APIRouter, Depends, Request, status
from typing_extensions import Literal
from fastapi import APIRouter, Depends, status, Request
from etebase_server.django import app_settings, models
from etebase_server.django.token_auth.models import AuthToken
from etebase_server.django.models import UserInfo
from etebase_server.django.signals import user_signed_up
from etebase_server.django.token_auth.models import AuthToken
from etebase_server.django.utils import CallbackContext, create_user, get_user_queryset
from etebase_server.django.utils import create_user, get_user_queryset, CallbackContext
from etebase_server.myauth.models import UserType, get_typed_user_model
from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError
from ..msgpack import MsgpackRoute
from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode, get_user_username_email_kwargs
from ..dependencies import AuthData, get_auth_data, get_authenticated_user
from ..exceptions import AuthenticationFailed, HttpError, transform_validation_error
from ..msgpack import MsgpackResponse, MsgpackRoute
from ..utils import BaseModel, get_user_username_email_kwargs, msgpack_decode, msgpack_encode, permission_responses
User = get_typed_user_model()
authentication_router = APIRouter(route_class=MsgpackRoute)
@ -76,7 +75,7 @@ class LoginOut(BaseModel):
class Authentication(BaseModel):
class Config:
ignored_types = (cached_property,)
keep_untouched = (cached_property,)
response: bytes
signature: bytes
@ -188,7 +187,7 @@ def login_challenge(user: UserType = Depends(get_login_user)):
"userId": user.id,
}
challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder))
return MsgpackResponse(LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version))
return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version)
@authentication_router.post("/login/", response_model=LoginOut)
@ -198,7 +197,7 @@ def login(data: Login, request: Request):
validate_login_request(data.response_data, data, user, "login", host)
ret = LoginOut.from_orm(user)
user_logged_in.send(sender=user.__class__, request=None, user=user)
return MsgpackResponse(ret)
return ret
@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses)
@ -223,7 +222,7 @@ def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_u
ret = {
"url": get_dashboard_url(CallbackContext(request.path_params, user=user)),
}
return MsgpackResponse(ret)
return ret
def signup_save(data: SignupIn, request: Request) -> UserType:
@ -261,4 +260,4 @@ def signup(data: SignupIn, request: Request):
user = signup_save(data, request)
ret = LoginOut.from_orm(user)
user_signed_up.send(sender=user.__class__, request=None, user=user)
return MsgpackResponse(ret)
return ret

View File

@ -3,34 +3,33 @@ import typing as t
from asgiref.sync import sync_to_async
from django.core import exceptions as django_exceptions
from django.core.files.base import ContentFile
from django.db import IntegrityError, transaction
from django.db import transaction, IntegrityError
from django.db.models import Q, QuerySet
from fastapi import APIRouter, BackgroundTasks, Depends, Request, status
from fastapi import APIRouter, Depends, status, Request, BackgroundTasks
from etebase_server.django import models
from etebase_server.myauth.models import UserType
from ..db_hack import django_db_cleanup_decorator
from ..dependencies import get_collection, get_collection_queryset, get_item_queryset
from ..exceptions import HttpError, PermissionDenied, ValidationError, transform_validation_error
from ..msgpack import MsgpackRequest, MsgpackResponse, MsgpackRoute
from ..redis import redisw
from ..sendfile import sendfile
from ..stoken_handler import filter_by_stoken, filter_by_stoken_and_limit, get_queryset_stoken, get_stoken_obj
from .authentication import get_authenticated_user
from .websocket import get_ticket, TicketRequest, TicketOut
from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError
from ..msgpack import MsgpackRoute
from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken
from ..utils import (
PERMISSIONS_READ,
PERMISSIONS_READWRITE,
BaseModel,
get_object_or_404,
Context,
Prefetch,
PrefetchQuery,
get_object_or_404,
is_collection_admin,
msgpack_encode,
BaseModel,
permission_responses,
PERMISSIONS_READ,
PERMISSIONS_READWRITE,
)
from .authentication import get_authenticated_user
from .websocket import TicketOut, TicketRequest, get_ticket
from ..dependencies import get_collection_queryset, get_item_queryset, get_collection
from ..sendfile import sendfile
from ..redis import redisw
from ..db_hack import django_db_cleanup_decorator
collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@ -52,7 +51,7 @@ class CollectionItemRevisionInOut(BaseModel):
chunks: t.List[ChunkType]
class Config:
from_attributes = True
orm_mode = True
@classmethod
def from_orm_context(
@ -72,13 +71,13 @@ class CollectionItemRevisionInOut(BaseModel):
class CollectionItemCommon(BaseModel):
uid: str
version: int
encryptionKey: t.Optional[bytes] = None
encryptionKey: t.Optional[bytes]
content: CollectionItemRevisionInOut
class CollectionItemOut(CollectionItemCommon):
class Config:
from_attributes = True
orm_mode = True
@classmethod
def from_orm_context(
@ -93,12 +92,12 @@ class CollectionItemOut(CollectionItemCommon):
class CollectionItemIn(CollectionItemCommon):
etag: t.Optional[str] = None
etag: t.Optional[str]
class CollectionCommon(BaseModel):
# FIXME: remove optional once we finish collection-type-migration
collectionType: t.Optional[bytes] = None
collectionType: t.Optional[bytes]
collectionKey: bytes
@ -132,27 +131,27 @@ class RemovedMembershipOut(BaseModel):
class CollectionListResponse(BaseModel):
data: t.List[CollectionOut]
stoken: t.Optional[str] = None
stoken: t.Optional[str]
done: bool
removedMemberships: t.Optional[t.List[RemovedMembershipOut]] = None
removedMemberships: t.Optional[t.List[RemovedMembershipOut]]
class CollectionItemListResponse(BaseModel):
data: t.List[CollectionItemOut]
stoken: t.Optional[str] = None
stoken: t.Optional[str]
done: bool
class CollectionItemRevisionListResponse(BaseModel):
data: t.List[CollectionItemRevisionInOut]
iterator: t.Optional[str] = None
iterator: t.Optional[str]
done: bool
class CollectionItemBulkGetIn(BaseModel):
uid: str
etag: t.Optional[str] = None
etag: t.Optional[str]
class ItemDepIn(BaseModel):
@ -173,7 +172,7 @@ class ItemDepIn(BaseModel):
class ItemBatchIn(BaseModel):
items: t.List[CollectionItemIn]
deps: t.Optional[t.List[ItemDepIn]] = None
deps: t.Optional[t.List[ItemDepIn]]
def validate_db(self):
if self.deps is not None:
@ -275,7 +274,7 @@ def list_multi(
Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True)
)
return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
return collection_list_common(queryset, user, stoken, limit, prefetch)
@collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ)
@ -286,7 +285,7 @@ def collection_list(
user: UserType = Depends(get_authenticated_user),
queryset: CollectionQuerySet = Depends(get_collection_queryset),
):
return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
return collection_list_common(queryset, user, stoken, limit, prefetch)
def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut):
@ -342,10 +341,7 @@ def _create(data: CollectionIn, user: UserType):
# TODO
process_revisions_for_item(main_item, data.item.content)
try:
collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user)
except IntegrityError:
raise ValidationError("bad_collection_type", "collectionType is null")
collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user)
models.CollectionMember(
collection=instance,
@ -368,7 +364,7 @@ def collection_get(
user: UserType = Depends(get_authenticated_user),
prefetch: Prefetch = PrefetchQuery,
):
return MsgpackResponse(CollectionOut.from_orm_context(obj, Context(user, prefetch)))
return CollectionOut.from_orm_context(obj, Context(user, prefetch))
def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool):
@ -377,7 +373,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val
revision_data = item_model.content
uid = item_model.uid
Model = models.CollectionItem # noqa: N806
Model = models.CollectionItem
with transaction.atomic():
instance, created = Model.objects.get_or_create(
@ -421,7 +417,7 @@ def item_get(
prefetch: Prefetch = PrefetchQuery,
):
obj = queryset.get(uid=item_uid)
return MsgpackResponse(CollectionItemOut.from_orm_context(obj, Context(user, prefetch)))
return CollectionItemOut.from_orm_context(obj, Context(user, prefetch))
def item_list_common(
@ -453,7 +449,7 @@ def item_list(
queryset = queryset.filter(parent__isnull=True)
response = item_list_common(queryset, user, stoken, limit, prefetch)
return MsgpackResponse(response)
return response
@item_router.post("/item/subscription-ticket/", response_model=TicketOut, dependencies=PERMISSIONS_READ)
@ -462,7 +458,7 @@ async def item_list_subscription_ticket(
user: UserType = Depends(get_authenticated_user),
):
"""Get an authentication ticket that can be used with the websocket endpoint"""
return MsgpackResponse(await get_ticket(TicketRequest(collection=collection.uid), user))
return await get_ticket(TicketRequest(collection=collection.uid), user)
def item_bulk_common(
@ -530,12 +526,10 @@ def item_revisions(
ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result]
iterator = ret_data[-1].uid if len(result) > 0 else None
return MsgpackResponse(
CollectionItemRevisionListResponse(
data=ret_data,
iterator=iterator,
done=done,
)
return CollectionItemRevisionListResponse(
data=ret_data,
iterator=iterator,
done=done,
)
@ -565,12 +559,10 @@ def fetch_updates(
new_stoken = new_stoken or stoken_rev_uid
context = Context(user, prefetch)
return MsgpackResponse(
CollectionItemListResponse(
data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
stoken=new_stoken,
done=True, # we always return all the items, so it's always done
)
return CollectionItemListResponse(
data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
stoken=new_stoken,
done=True, # we always return all the items, so it's always done
)
@ -582,9 +574,7 @@ def item_transaction(
stoken: t.Optional[str] = None,
user: UserType = Depends(get_authenticated_user),
):
return MsgpackResponse(
item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
)
return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
@item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE])
@ -595,9 +585,7 @@ def item_batch(
stoken: t.Optional[str] = None,
user: UserType = Depends(get_authenticated_user),
):
return MsgpackResponse(
item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
)
return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
# Chunks
@ -622,12 +610,7 @@ async def chunk_update(
collection: models.Collection = Depends(get_collection),
):
# IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid)
if isinstance(request, MsgpackRequest):
body = await request.raw_body()
else:
body = await request.body()
content_file = ContentFile(body)
content_file = ContentFile(await request.body())
try:
await chunk_save(chunk_uid, collection, content_file)
except IntegrityError:

View File

@ -1,27 +1,26 @@
import typing as t
from django.db import IntegrityError, transaction
from django.db import transaction, IntegrityError
from django.db.models import QuerySet
from fastapi import APIRouter, Depends, Request, status
from fastapi import APIRouter, Depends, status, Request
from etebase_server.django import models
from etebase_server.django.utils import CallbackContext, get_user_queryset
from etebase_server.django.utils import get_user_queryset, CallbackContext
from etebase_server.myauth.models import UserType, get_typed_user_model
from ..db_hack import django_db_cleanup_decorator
from .authentication import get_authenticated_user
from ..exceptions import HttpError, PermissionDenied
from ..msgpack import MsgpackResponse, MsgpackRoute
from ..msgpack import MsgpackRoute
from ..utils import (
PERMISSIONS_READ,
PERMISSIONS_READWRITE,
BaseModel,
Context,
get_object_or_404,
get_user_username_email_kwargs,
Context,
is_collection_admin,
BaseModel,
permission_responses,
PERMISSIONS_READ,
PERMISSIONS_READWRITE,
)
from .authentication import get_authenticated_user
from ..db_hack import django_db_cleanup_decorator
User = get_typed_user_model()
invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@ -34,7 +33,7 @@ class UserInfoOut(BaseModel):
pubkey: bytes
class Config:
from_attributes = True
orm_mode = True
@classmethod
def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut":
@ -67,7 +66,7 @@ class CollectionInvitationOut(CollectionInvitationCommon):
fromPubkey: bytes
class Config:
from_attributes = True
orm_mode = True
@classmethod
def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut":
@ -85,7 +84,7 @@ class CollectionInvitationOut(CollectionInvitationCommon):
class InvitationListResponse(BaseModel):
data: t.List[CollectionInvitationOut]
iterator: t.Optional[str] = None
iterator: t.Optional[str]
done: bool
@ -121,7 +120,7 @@ def list_common(
iterator = ret_data[-1].uid if len(result) > 0 else None
return InvitationListResponse(
data=[CollectionInvitationOut.from_orm(x) for x in ret_data],
data=ret_data,
iterator=iterator,
done=done,
)
@ -133,7 +132,7 @@ def incoming_list(
limit: int = 50,
queryset: InvitationQuerySet = Depends(get_incoming_queryset),
):
return MsgpackResponse(list_common(queryset, iterator, limit))
return list_common(queryset, iterator, limit)
@invitation_incoming_router.get(
@ -144,7 +143,7 @@ def incoming_get(
queryset: InvitationQuerySet = Depends(get_incoming_queryset),
):
obj = get_object_or_404(queryset, uid=invitation_uid)
return MsgpackResponse(CollectionInvitationOut.from_orm(obj))
return CollectionInvitationOut.from_orm(obj)
@invitation_incoming_router.delete(
@ -219,7 +218,7 @@ def outgoing_list(
limit: int = 50,
queryset: InvitationQuerySet = Depends(get_outgoing_queryset),
):
return MsgpackResponse(list_common(queryset, iterator, limit))
return list_common(queryset, iterator, limit)
@invitation_outgoing_router.delete(
@ -242,4 +241,4 @@ def outgoing_fetch_user_profile(
kwargs = get_user_username_email_kwargs(username)
user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs)
user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user)
return MsgpackResponse(UserInfoOut.from_orm(user_info))
return UserInfoOut.from_orm(user_info)

View File

@ -6,12 +6,12 @@ from fastapi import APIRouter, Depends, status
from etebase_server.django import models
from etebase_server.myauth.models import UserType, get_typed_user_model
from ..db_hack import django_db_cleanup_decorator
from ..msgpack import MsgpackResponse, MsgpackRoute
from ..stoken_handler import filter_by_stoken_and_limit
from ..utils import PERMISSIONS_READ, PERMISSIONS_READWRITE, BaseModel, get_object_or_404, permission_responses
from .authentication import get_authenticated_user
from ..msgpack import MsgpackRoute
from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE
from ..stoken_handler import filter_by_stoken_and_limit
from ..db_hack import django_db_cleanup_decorator
from .collection import get_collection, verify_collection_admin
User = get_typed_user_model()
@ -39,7 +39,7 @@ class CollectionMemberOut(BaseModel):
accessLevel: models.AccessLevels
class Config:
from_attributes = True
orm_mode = True
@classmethod
def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut":
@ -48,7 +48,7 @@ class CollectionMemberOut(BaseModel):
class MemberListResponse(BaseModel):
data: t.List[CollectionMemberOut]
iterator: t.Optional[str] = None
iterator: t.Optional[str]
done: bool
@ -66,12 +66,10 @@ def member_list(
)
new_stoken = new_stoken_obj and new_stoken_obj.uid
return MsgpackResponse(
MemberListResponse(
data=[CollectionMemberOut.from_orm(item) for item in result],
iterator=new_stoken,
done=done,
)
return MemberListResponse(
data=[CollectionMemberOut.from_orm(item) for item in result],
iterator=new_stoken,
done=done,
)

View File

@ -3,12 +3,11 @@ from django.db import transaction
from django.shortcuts import get_object_or_404
from fastapi import APIRouter, Request, status
from etebase_server.django.utils import CallbackContext, get_user_queryset
from etebase_server.myauth.models import get_typed_user_model
from ..exceptions import HttpError
from ..msgpack import MsgpackRoute
from etebase_server.django.utils import get_user_queryset, CallbackContext
from .authentication import SignupIn, signup_save
from ..msgpack import MsgpackRoute
from ..exceptions import HttpError
from etebase_server.myauth.models import get_typed_user_model
test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"])
User = get_typed_user_model()

View File

@ -1,13 +1,13 @@
import asyncio
import typing as t
import nacl.encoding
import nacl.utils
from redis import asyncio as aioredis
from redis.exceptions import ConnectionError
from asgiref.sync import sync_to_async
from django.db.models import QuerySet
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status
from redis import asyncio as aioredis
from redis.exceptions import ConnectionError
import nacl.encoding
import nacl.utils
from etebase_server.django import models
from etebase_server.django.utils import CallbackContext, get_user_queryset
@ -19,6 +19,7 @@ from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode
from ..redis import redisw
from ..utils import BaseModel, permission_responses
User = get_typed_user_model()
websocket_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
CollectionQuerySet = QuerySet[models.Collection]

View File

@ -1,15 +1,14 @@
import logging
import os
from functools import lru_cache
from importlib import import_module
from pathlib import Path, PurePath
from urllib.parse import quote
import logging
from fastapi import status
from ..exceptions import HttpError
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from fastapi import status
from ..exceptions import HttpError
logger = logging.getLogger(__name__)
@ -33,7 +32,9 @@ def _convert_file_to_url(path):
path_obj = PurePath(path)
relpath = path_obj.relative_to(path_root)
url = os.path.normpath(str(url_root / relpath))
# Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
# already instantiated Path object
url = relpath._flavour.pathmod.normpath(str(url_root / relpath))
return quote(str(url))
@ -47,7 +48,9 @@ def _sanitize_path(filepath):
filepath_obj = Path(filepath)
# get absolute path
filepath_abs = Path(os.path.normpath(str(path_root / filepath_obj)))
# Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
# already instantiated Path object
filepath_abs = Path(filepath_obj._flavour.pathmod.normpath(str(path_root / filepath_obj)))
# if filepath_abs is not relative to path_root, relative_to throws an error
try:

View File

@ -47,6 +47,7 @@ def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]:
def filter_by_stoken_and_limit(
stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation
) -> t.Tuple[list, t.Optional[Stoken], bool]:
queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation)
result = list(queryset[: limit + 1])

View File

@ -1,13 +1,14 @@
import base64
import dataclasses
import typing as t
import msgpack
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Model, QuerySet
from fastapi import Depends, Query, status
from pydantic import BaseModel as PyBaseModel
from typing_extensions import Literal
import msgpack
import base64
from fastapi import status, Query, Depends
from pydantic import BaseModel as PyBaseModel
from django.db.models import Model, QuerySet
from django.core.exceptions import ObjectDoesNotExist
from etebase_server.django import app_settings
from etebase_server.django.models import AccessLevels
@ -25,7 +26,10 @@ T = t.TypeVar("T", bound=Model, covariant=True)
class BaseModel(PyBaseModel):
pass
class Config:
json_encoders = {
bytes: lambda x: x,
}
@dataclasses.dataclass

View File

@ -1,21 +1,12 @@
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin as DjangoUserAdmin
from .forms import AdminUserCreationForm
from .models import User
from .forms import AdminUserCreationForm
class UserAdmin(DjangoUserAdmin):
add_form = AdminUserCreationForm
add_fieldsets = (
(
None,
{
"classes": ("wide",),
"fields": ("username",),
},
),
)
add_fieldsets = ((None, {"classes": ("wide",), "fields": ("username",),}),)
admin.site.register(User, UserAdmin)

View File

@ -1,6 +1,5 @@
from django import forms
from django.contrib.auth.forms import UsernameField
from etebase_server.myauth.models import get_typed_user_model
User = get_typed_user_model()

View File

@ -1,15 +1,15 @@
import logging
import ldap
from django.utils import timezone
from django.conf import settings
from django.core.exceptions import PermissionDenied as DjangoPermissionDenied
from django.utils import timezone
from fastapi import Depends
from etebase_server.django.utils import CallbackContext
from etebase_server.myauth.models import get_typed_user_model, UserType
from etebase_server.fastapi.dependencies import get_authenticated_user
from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied
from etebase_server.myauth.models import UserType, get_typed_user_model
from fastapi import Depends
import ldap
User = get_typed_user_model()

View File

@ -1,13 +1,21 @@
import typing as t
from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager
from django.contrib.auth.validators import UnicodeUsernameValidator
from django.core import validators
from django.db import models
from django.utils.deconstruct import deconstructible
from django.utils.translation import gettext_lazy as _
@deconstructible
class UnicodeUsernameValidator(validators.RegexValidator):
regex = r"^[\w.-]+\Z"
message = _("Enter a valid username. This value may contain only letters, " "numbers, and ./-/_ characters.")
flags = 0
class UserManager(DjangoUserManager):
def get_by_natural_key(self, username: t.Optional[str]):
def get_by_natural_key(self, username: str):
return self.get(**{self.model.USERNAME_FIELD + "__iexact": username})

View File

@ -1 +1,3 @@
from django.test import TestCase
# Create your tests here.

View File

@ -1 +1,3 @@
from django.shortcuts import render
# Create your views here.

View File

@ -10,9 +10,8 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.0/ref/settings/
"""
import configparser
import os
import configparser
from .utils import get_secret_from_file
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@ -21,7 +20,6 @@ BASE_DIR = os.path.dirname(SOURCE_DIR)
AUTH_USER_MODEL = "myauth.User"
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
@ -46,7 +44,7 @@ DATABASES = {
}
}
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
# Application definition
@ -164,8 +162,6 @@ if any(os.path.isfile(x) for x in config_locations):
if "allowed_hosts" in config:
ALLOWED_HOSTS = [y for x, y in config.items("allowed_hosts")]
CSRF_TRUSTED_ORIGINS = ["https://" + y for x, y in config.items("allowed_hosts")] + \
["http://" + y for x, y in config.items("allowed_hosts")]
if "database" in config:
DATABASES = {"default": {x.upper(): y for x, y in config.items("database")}}
@ -204,7 +200,7 @@ if "DJANGO_MEDIA_ROOT" in os.environ:
# Make an `etebase_server_settings` module available to override settings.
try:
from etebase_server_settings import * # noqa: F403
from etebase_server_settings import *
except ImportError:
pass

View File

@ -1,6 +1,11 @@
import os
from django.conf import settings
from django.contrib import admin
from django.urls import path
from django.views.generic import TemplateView
from django.views.static import serve
from django.contrib.staticfiles import finders
urlpatterns = [
path("admin/", admin.site.urls),

View File

@ -12,11 +12,10 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.core.management import utils
import os
import stat
from django.core.management import utils
def get_secret_from_file(path):
try:

View File

@ -1,6 +1,5 @@
#!/usr/bin/env python3
"""Django's command-line utility for administrative tasks."""
import os
import sys

View File

@ -4,25 +4,3 @@ line-length = 120
[build-system]
requires = ["setuptools>=42"]
build-backend = "setuptools.build_meta"
[tool.ruff]
line-length = 120
exclude = [
".git",
".git-rewrite",
".mypy_cache",
".pytype",
".ruff_cache",
".venv",
"build",
"dist",
"node_modules",
"migrations", # Alembic migrations
]
[tool.ruff.lint]
select = ["E", "F", "I", "N", "T20", "W"]
ignore = ["E203", "E501", "E711", "E712", "N803", "N815", "N818", "T201"]
[tool.ruff.lint.isort]
combine-as-imports = true

View File

@ -1,53 +1,66 @@
#
# This file is autogenerated by pip-compile with Python 3.12
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --output-file=requirements-dev.txt requirements.in/development.txt
#
asgiref==3.8.1
# via
# django
# django-stubs
build==1.2.1
# via pip-tools
click==8.1.7
# via pip-tools
coverage==7.5.3
asgiref==3.5.2
# via django
black==22.10.0
# via -r requirements.in/development.txt
django==5.0.6
build==0.9.0
# via pip-tools
click==8.1.3
# via
# black
# pip-tools
coverage==6.5.0
# via -r requirements.in/development.txt
django==3.2.16
# via
# -r requirements.in/development.txt
# django-stubs
# django-stubs-ext
django-stubs==5.0.2
django-stubs==1.13.0
# via -r requirements.in/development.txt
django-stubs-ext==5.0.2
django-stubs-ext==0.7.0
# via django-stubs
mypy==1.10.0
# via -r requirements.in/development.txt
mypy-extensions==1.0.0
# via mypy
packaging==24.0
# via build
pip-tools==7.4.1
# via -r requirements.in/development.txt
pyproject-hooks==1.1.0
mypy==0.991
# via django-stubs
mypy-extensions==0.4.3
# via
# build
# pip-tools
pywatchman==2.0.0
# black
# mypy
packaging==21.3
# via build
pathspec==0.10.2
# via black
pep517==0.13.0
# via build
pip-tools==6.11.0
# via -r requirements.in/development.txt
ruff==0.4.8
# via -r requirements.in/development.txt
sqlparse==0.5.0
platformdirs==2.6.0
# via black
pyparsing==3.0.9
# via packaging
pytz==2022.6
# via django
types-pyyaml==6.0.12.20240311
pywatchman==1.4.1
# via -r requirements.in/development.txt
sqlparse==0.4.3
# via django
tomli==2.0.1
# via django-stubs
typing-extensions==4.12.2
types-pytz==2022.6.0.1
# via django-stubs
types-pyyaml==6.0.12.2
# via django-stubs
typing-extensions==4.4.0
# via
# django-stubs
# django-stubs-ext
# mypy
wheel==0.43.0
wheel==0.38.4
# via pip-tools
# The following packages are considered to be unsafe in a requirements file:

View File

@ -1,8 +1,7 @@
django>=4.0,<5.0
django>=4.0<5.0
msgpack
pynacl
fastapi>=0.104
pydantic>=2.0.0
fastapi
typing_extensions
uvicorn[standard]
aiofiles

View File

@ -1,6 +1,6 @@
coverage
pip-tools
pywatchman
ruff
mypy
black
django-stubs
django<4.0

View File

@ -4,114 +4,59 @@
#
# pip-compile --output-file=requirements.txt requirements.in/base.txt
#
aiofiles==23.2.1
aiofiles==22.1.0
# via -r requirements.in/base.txt
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
anyio==3.6.2
# via
# httpx
# starlette
# watchfiles
asgiref==3.8.1
asgiref==3.5.2
# via django
certifi==2024.6.2
# via
# httpcore
# httpx
cffi==1.16.0
async-timeout==4.0.2
# via redis
cffi==1.15.1
# via pynacl
click==8.1.7
# via
# typer
# uvicorn
django==4.2.14
# via -r requirements.in/base.txt
dnspython==2.6.1
# via email-validator
email-validator==2.1.1
# via fastapi
fastapi==0.111.0
# via -r requirements.in/base.txt
fastapi-cli==0.0.4
# via fastapi
h11==0.14.0
# via
# httpcore
# uvicorn
httpcore==1.0.5
# via httpx
httptools==0.6.1
click==8.1.3
# via uvicorn
httpx==0.27.0
# via fastapi
idna==3.7
# via
# anyio
# email-validator
# httpx
jinja2==3.1.4
# via fastapi
markdown-it-py==3.0.0
# via rich
markupsafe==2.1.5
# via jinja2
mdurl==0.1.2
# via markdown-it-py
msgpack==1.0.8
django==4.1.13
# via -r requirements.in/base.txt
orjson==3.10.3
# via fastapi
pycparser==2.22
fastapi==0.88.0
# via -r requirements.in/base.txt
h11==0.14.0
# via uvicorn
httptools==0.5.0
# via uvicorn
idna==3.4
# via anyio
msgpack==1.0.4
# via -r requirements.in/base.txt
pycparser==2.21
# via cffi
pydantic==2.7.3
# via
# -r requirements.in/base.txt
# fastapi
pydantic-core==2.18.4
# via pydantic
pygments==2.18.0
# via rich
pydantic==1.10.2
# via fastapi
pynacl==1.5.0
# via -r requirements.in/base.txt
python-dotenv==1.0.1
python-dotenv==0.21.0
# via uvicorn
python-multipart==0.0.9
# via fastapi
pyyaml==6.0.1
# via uvicorn
redis==5.1.0b6
redis==4.4.0
# via -r requirements.in/base.txt
rich==13.7.1
# via typer
shellingham==1.5.4
# via typer
sniffio==1.3.1
# via
# anyio
# httpx
sqlparse==0.5.0
sniffio==1.3.0
# via anyio
sqlparse==0.4.3
# via django
starlette==0.37.2
starlette==0.22.0
# via fastapi
typer==0.12.3
# via fastapi-cli
typing-extensions==4.12.2
typing-extensions==4.4.0
# via
# -r requirements.in/base.txt
# fastapi
# pydantic
# pydantic-core
# typer
ujson==5.10.0
# via fastapi
uvicorn[standard]==0.30.1
# via
# -r requirements.in/base.txt
# fastapi
uvloop==0.19.0
uvicorn[standard]==0.20.0
# via -r requirements.in/base.txt
uvloop==0.17.0
# via uvicorn
watchfiles==0.22.0
watchfiles==0.18.1
# via uvicorn
websockets==12.0
websockets==10.4
# via uvicorn

View File

@ -2,7 +2,7 @@ from setuptools import find_packages, setup
setup(
name="etebase_server",
version="0.14.2",
version="0.13.0",
description="An Etebase (EteSync 2.0) server",
url="https://www.etebase.com/",
classifiers=[