1
0
mirror of https://github.com/etesync/server synced 2025-04-10 21:45:47 +00:00

Compare commits

...

26 Commits

Author SHA1 Message Date
Tom Hacohen
6ac5412104 Bump django version. 2024-07-12 08:44:19 -04:00
Tom Hacohen
cc54a136f1 Fix server error when passing null collection types. 2024-06-29 14:55:48 -04:00
Tom Hacohen
386c01d89e Bump version and update changelog. 2024-06-13 14:02:33 -04:00
Tom Hacohen
f35b4b94e5 Make more fields truely optional (behavior change in pydantic v2). 2024-06-13 13:50:38 -04:00
Craeckie
19aba5345a
added CSRF_TRUSTED_ORIGINS (#183)
Since some recent upgrade, I'm not able to login to the admin page of etesync (`/admin/login/`), because the CSRF check fails.


After adding `CSRF_TRUSTED_ORIGINS = ['https://my-domain.com']`, it works.
According to the [docs](https://docs.djangoproject.com/en/4.2/ref/settings/#csrf-trusted-origins), this setting is required in addition to `ALLOWED_HOSTS`.
2024-06-09 23:27:49 +03:00
Tom Hacohen
2deb59e932 Bump version and update changelog. 2024-06-09 11:21:24 -04:00
Tom Hacohen
c9b8f43845 Set default auto field to appease django warning. 2024-06-09 11:15:41 -04:00
Tom Hacohen
f1c072bd0a Fix issue with how we render binary 422 errors
json_encoder assumes it's not binary, but our error messages may include
some binary stuff.
2024-06-08 22:54:02 -04:00
Tom Hacohen
043dc972ae Bump version and update changelog. 2024-06-08 21:52:36 -04:00
Tom Hacohen
8f588af665
Merge PR #184: Upgrade FastAPI and Pydantic to most recent versions 2024-06-08 20:50:42 -04:00
Tom Hacohen
d7075c0169 Mark optional field as optional. 2024-06-08 20:49:13 -04:00
Tom Hacohen
a27ce2f4d0 Also handle 422 as msgpack. 2024-06-08 20:41:01 -04:00
Tom Hacohen
138d99dd7f Update code to adjust to most recent python/fastapi. 2024-06-08 20:27:33 -04:00
Tom Hacohen
0be14a7b0e Fixes for fastapi. 2024-06-08 20:17:02 -04:00
Tom Hacohen
57e676baa1 Adjust to fastapi changes. 2024-06-08 19:52:10 -04:00
Tom Hacohen
b9f2cea951 more 2024-06-08 19:29:56 -04:00
Tom Hacohen
0d9c9f153d Type fix. 2024-06-08 19:29:56 -04:00
Tom Hacohen
49eeeefef5 Make fastapi 0.104 and pydantic 2.0 min requirements 2024-06-08 19:29:56 -04:00
Tom Hacohen
fb9cc701d0 Adjust pydantic code to v2. 2024-06-08 18:10:34 -04:00
Tom Hacohen
0cdab19308 Fix rust complaints. 2024-06-08 18:04:59 -04:00
Tom Hacohen
79d28586c5 Run ruff format. 2024-06-08 17:51:44 -04:00
Tom Hacohen
df0d1596e2 Upgrade dev deps and add ruff. 2024-06-08 17:49:40 -04:00
Tom Hacohen
dae6f17355 Upgrade dependencies. 2024-06-08 17:45:45 -04:00
Tom Hacohen
45d2696e21 Bump version and update changelog. 2024-06-04 08:20:11 -04:00
Tom Hacohen
e6750d7a78
Merge pull request #182 from pyrox0/master
Fix django version specifier
2024-06-03 14:15:37 -04:00
Pyrox
b3e09e77bf
Fix django version specifier
This errors out when building. I've patched this in Nixpkgs, but an
upstream patch is much better for others.
2024-06-03 14:03:31 -04:00
37 changed files with 392 additions and 285 deletions

View File

@ -1,5 +1,19 @@
# Changelog
## Version 0.14.2
- Fix issue with some requests failing in some scenarios with the JS client.
- The JS client was omitting optional fields which were accidentally made to be required. It happened because pydantic v2 changed the behavior in a few ways (for the better fwiw) and we missed a few places when upgrading.
## Version 0.14.1
- Fix issue with serializing non utf8 422 errors
- Appease django warnings about default auto field
## Version 0.14.0
- Upgrade to latest FastAPI and Pydantic for compatibility with Python 3.12
## Version 0.13.1
- Fix requirements definition for django.
## Version 0.13.0
- Update Django dependency to 4 (LTS)

View File

@ -1 +1,3 @@
from .app_settings_inner import app_settings
__all__ = ["app_settings"]

View File

@ -34,11 +34,11 @@ class AppSettings:
return getattr(settings, self.prefix + name, dflt)
@cached_property
def REDIS_URI(self) -> t.Optional[str]: # pylint: disable=invalid-name
def REDIS_URI(self) -> t.Optional[str]: # noqa: N802
return self._setting("REDIS_URI", None)
@cached_property
def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name
def API_PERMISSIONS_READ(self): # noqa: N802
perms = self._setting("API_PERMISSIONS_READ", tuple())
ret = []
for perm in perms:
@ -46,7 +46,7 @@ class AppSettings:
return ret
@cached_property
def API_PERMISSIONS_WRITE(self): # pylint: disable=invalid-name
def API_PERMISSIONS_WRITE(self): # noqa: N802
perms = self._setting("API_PERMISSIONS_WRITE", tuple())
ret = []
for perm in perms:
@ -54,35 +54,35 @@ class AppSettings:
return ret
@cached_property
def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name
def GET_USER_QUERYSET_FUNC(self): # noqa: N802
get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None)
if get_user_queryset is not None:
return self.import_from_str(get_user_queryset)
return None
@cached_property
def CREATE_USER_FUNC(self): # pylint: disable=invalid-name
def CREATE_USER_FUNC(self): # noqa: N802
func = self._setting("CREATE_USER_FUNC", None)
if func is not None:
return self.import_from_str(func)
return None
@cached_property
def DASHBOARD_URL_FUNC(self): # pylint: disable=invalid-name
def DASHBOARD_URL_FUNC(self): # noqa: N802
func = self._setting("DASHBOARD_URL_FUNC", None)
if func is not None:
return self.import_from_str(func)
return None
@cached_property
def CHUNK_PATH_FUNC(self): # pylint: disable=invalid-name
def CHUNK_PATH_FUNC(self): # noqa: N802
func = self._setting("CHUNK_PATH_FUNC", None)
if func is not None:
return self.import_from_str(func)
return None
@cached_property
def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name
def CHALLENGE_VALID_SECONDS(self): # noqa: N802
return self._setting("CHALLENGE_VALID_SECONDS", 60)

View File

@ -15,22 +15,21 @@
import typing as t
from pathlib import Path
from django.db import models, transaction
from django.conf import settings
from django.core.validators import RegexValidator
from django.db.models import Max, Value as V
from django.db import models, transaction
from django.db.models import Max, Value as Val
from django.db.models.functions import Coalesce, Greatest
from django.utils.functional import cached_property
from django.utils.crypto import get_random_string
from django.utils.functional import cached_property
from . import app_settings
UidValidator = RegexValidator(regex=r"^[a-zA-Z0-9\-_]{20,}$", message="Not a valid UID")
def stoken_annotation_builder(stoken_id_fields: t.List[str]):
aggr_fields = [Coalesce(Max(field), V(0)) for field in stoken_id_fields]
aggr_fields = [Coalesce(Max(field), Val(0)) for field in stoken_id_fields]
return Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0]

View File

@ -1,6 +1,7 @@
from django.db import models
from django.utils import timezone
from django.utils.crypto import get_random_string
from etebase_server.myauth.models import get_typed_user_model
User = get_typed_user_model()
@ -15,7 +16,6 @@ def get_default_expiry():
class AuthToken(models.Model):
key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key)
user = models.ForeignKey(User, null=False, blank=False, related_name="auth_token_set", on_delete=models.CASCADE)
created = models.DateTimeField(auto_now_add=True)

View File

@ -1,13 +1,13 @@
import typing as t
from dataclasses import dataclass
from django.db.models import QuerySet
from django.core.exceptions import PermissionDenied
from django.db.models import QuerySet
from etebase_server.myauth.models import UserType, get_typed_user_model
from . import app_settings
User = get_typed_user_model()

View File

@ -2,6 +2,7 @@
FIXME: this whole function is a hack around the django db limitations due to how db connections are cached and cleaned.
Essentially django assumes there's the django request dispatcher to automatically clean up after the ORM.
"""
import typing as t
from functools import wraps

View File

@ -1,18 +1,17 @@
import dataclasses
from django.db.models import QuerySet
from django.utils import timezone
from fastapi import Depends
from fastapi.security import APIKeyHeader
from django.utils import timezone
from django.db.models import QuerySet
from etebase_server.django import models
from etebase_server.django.token_auth.models import AuthToken, get_default_expiry
from etebase_server.myauth.models import UserType, get_typed_user_model
from .db_hack import django_db_cleanup_decorator
from .exceptions import AuthenticationFailed
from .utils import get_object_or_404
from .db_hack import django_db_cleanup_decorator
User = get_typed_user_model()
token_scheme = APIKeyHeader(name="Authorization")

View File

@ -1,8 +1,8 @@
from fastapi import status, HTTPException
import typing as t
from pydantic import BaseModel
from django.core.exceptions import ValidationError as DjangoValidationError
from fastapi import HTTPException, status
from pydantic import BaseModel
class HttpErrorField(BaseModel):
@ -11,7 +11,7 @@ class HttpErrorField(BaseModel):
detail: str
class Config:
orm_mode = True
from_attributes = True
class HttpErrorOut(BaseModel):
@ -20,7 +20,7 @@ class HttpErrorOut(BaseModel):
errors: t.Optional[t.List[HttpErrorField]]
class Config:
orm_mode = True
from_attributes = True
class CustomHttpException(HTTPException):

View File

@ -1,19 +1,20 @@
import typing as t
from django.conf import settings
# Not at the top of the file because we first need to setup django
from fastapi import FastAPI, Request
from fastapi import FastAPI, Request, status
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.trustedhost import TrustedHostMiddleware
from fastapi.staticfiles import StaticFiles
from etebase_server.django import app_settings
from .exceptions import CustomHttpException
from .msgpack import MsgpackResponse
from .routers.authentication import authentication_router
from .routers.collection import collection_router, item_router
from .routers.member import member_router
from .routers.invitation import invitation_incoming_router, invitation_outgoing_router
from .routers.member import member_router
from .routers.websocket import websocket_router
@ -24,12 +25,12 @@ def create_application(prefix="", middlewares=[]):
externalDocs={
"url": "https://docs.etebase.com",
"description": "Docs about the API specifications and clients.",
}
},
# FIXME: version="2.5.0",
)
VERSION = "v1"
BASE_PATH = f"{prefix}/api/{VERSION}"
COLLECTION_UID_MARKER = "{collection_uid}"
VERSION = "v1" # noqa: N806
BASE_PATH = f"{prefix}/api/{VERSION}" # noqa: N806
COLLECTION_UID_MARKER = "{collection_uid}" # noqa: N806
app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"])
app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"])
app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"])
@ -75,6 +76,16 @@ def create_application(prefix="", middlewares=[]):
async def custom_exception_handler(request: Request, exc: CustomHttpException):
return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict)
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
from pydantic import TypeAdapter
errors = TypeAdapter(t.Dict[str, t.Any])
return MsgpackResponse(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
content=errors.dump_python({"detail": exc.errors()}),
)
app.mount(settings.STATIC_URL, StaticFiles(directory=settings.STATIC_ROOT), name="static")
return app

View File

@ -5,16 +5,19 @@ from pydantic import BaseModel
from starlette.requests import Request
from starlette.responses import Response
from .utils import msgpack_encode, msgpack_decode
from .db_hack import django_db_cleanup_decorator
from .utils import msgpack_decode, msgpack_encode
class MsgpackRequest(Request):
media_type = "application/msgpack"
async def raw_body(self) -> bytes:
return await super().body()
async def body(self) -> bytes:
if not hasattr(self, "_json"):
body = await super().body()
body = await self.raw_body()
self._json = msgpack_decode(body)
return self._json
@ -27,7 +30,7 @@ class MsgpackResponse(Response):
return b""
if isinstance(content, BaseModel):
content = content.dict()
content = content.model_dump()
return msgpack_encode(content)
@ -48,7 +51,7 @@ class MsgpackRoute(APIRoute):
status_code=self.status_code,
# use custom response class or fallback on default self.response_class
response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class),
response_field=self.secure_cloned_response_field,
response_field=self.response_field,
response_model_include=self.response_model_include,
response_model_exclude=self.response_model_exclude,
response_model_by_alias=self.response_model_by_alias,
@ -60,14 +63,14 @@ class MsgpackRoute(APIRoute):
def get_route_handler(self) -> t.Callable:
async def custom_route_handler(request: Request) -> Response:
content_type = request.headers.get("Content-Type")
try:
request_cls = self.REQUESTS_CLASSES[content_type]
request = request_cls(request.scope, request.receive)
except KeyError:
# nothing registered to handle content_type, process given requests as-is
pass
if content_type is not None:
try:
request_cls = self.REQUESTS_CLASSES[content_type]
request = request_cls(request.scope, request.receive)
except KeyError:
# nothing registered to handle content_type, process given requests as-is
pass
accept = request.headers.get("Accept")
route_handler = self._get_media_type_route_handler(accept)

View File

@ -1,4 +1,5 @@
import typing as t
from redis import asyncio as aioredis
from etebase_server.django import app_settings

View File

@ -1,5 +1,4 @@
import typing as t
from typing_extensions import Literal
from datetime import datetime
import nacl
@ -8,22 +7,24 @@ import nacl.hash
import nacl.secret
import nacl.signing
from django.conf import settings
from django.contrib.auth import user_logged_out, user_logged_in
from django.contrib.auth import user_logged_in, user_logged_out
from django.core import exceptions as django_exceptions
from django.db import transaction
from django.utils.functional import cached_property
from fastapi import APIRouter, Depends, status, Request
from fastapi import APIRouter, Depends, Request, status
from typing_extensions import Literal
from etebase_server.django import app_settings, models
from etebase_server.django.token_auth.models import AuthToken
from etebase_server.django.models import UserInfo
from etebase_server.django.signals import user_signed_up
from etebase_server.django.utils import create_user, get_user_queryset, CallbackContext
from etebase_server.django.token_auth.models import AuthToken
from etebase_server.django.utils import CallbackContext, create_user, get_user_queryset
from etebase_server.myauth.models import UserType, get_typed_user_model
from ..exceptions import AuthenticationFailed, transform_validation_error, HttpError
from ..msgpack import MsgpackRoute
from ..utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode, get_user_username_email_kwargs
from ..dependencies import AuthData, get_auth_data, get_authenticated_user
from ..exceptions import AuthenticationFailed, HttpError, transform_validation_error
from ..msgpack import MsgpackResponse, MsgpackRoute
from ..utils import BaseModel, get_user_username_email_kwargs, msgpack_decode, msgpack_encode, permission_responses
User = get_typed_user_model()
authentication_router = APIRouter(route_class=MsgpackRoute)
@ -75,7 +76,7 @@ class LoginOut(BaseModel):
class Authentication(BaseModel):
class Config:
keep_untouched = (cached_property,)
ignored_types = (cached_property,)
response: bytes
signature: bytes
@ -187,7 +188,7 @@ def login_challenge(user: UserType = Depends(get_login_user)):
"userId": user.id,
}
challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder))
return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version)
return MsgpackResponse(LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version))
@authentication_router.post("/login/", response_model=LoginOut)
@ -197,7 +198,7 @@ def login(data: Login, request: Request):
validate_login_request(data.response_data, data, user, "login", host)
ret = LoginOut.from_orm(user)
user_logged_in.send(sender=user.__class__, request=None, user=user)
return ret
return MsgpackResponse(ret)
@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses)
@ -222,7 +223,7 @@ def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_u
ret = {
"url": get_dashboard_url(CallbackContext(request.path_params, user=user)),
}
return ret
return MsgpackResponse(ret)
def signup_save(data: SignupIn, request: Request) -> UserType:
@ -260,4 +261,4 @@ def signup(data: SignupIn, request: Request):
user = signup_save(data, request)
ret = LoginOut.from_orm(user)
user_signed_up.send(sender=user.__class__, request=None, user=user)
return ret
return MsgpackResponse(ret)

View File

@ -3,33 +3,34 @@ import typing as t
from asgiref.sync import sync_to_async
from django.core import exceptions as django_exceptions
from django.core.files.base import ContentFile
from django.db import transaction, IntegrityError
from django.db import IntegrityError, transaction
from django.db.models import Q, QuerySet
from fastapi import APIRouter, Depends, status, Request, BackgroundTasks
from fastapi import APIRouter, BackgroundTasks, Depends, Request, status
from etebase_server.django import models
from etebase_server.myauth.models import UserType
from .authentication import get_authenticated_user
from .websocket import get_ticket, TicketRequest, TicketOut
from ..exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError
from ..msgpack import MsgpackRoute
from ..stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken
from ..db_hack import django_db_cleanup_decorator
from ..dependencies import get_collection, get_collection_queryset, get_item_queryset
from ..exceptions import HttpError, PermissionDenied, ValidationError, transform_validation_error
from ..msgpack import MsgpackRequest, MsgpackResponse, MsgpackRoute
from ..redis import redisw
from ..sendfile import sendfile
from ..stoken_handler import filter_by_stoken, filter_by_stoken_and_limit, get_queryset_stoken, get_stoken_obj
from ..utils import (
get_object_or_404,
PERMISSIONS_READ,
PERMISSIONS_READWRITE,
BaseModel,
Context,
Prefetch,
PrefetchQuery,
get_object_or_404,
is_collection_admin,
msgpack_encode,
BaseModel,
permission_responses,
PERMISSIONS_READ,
PERMISSIONS_READWRITE,
)
from ..dependencies import get_collection_queryset, get_item_queryset, get_collection
from ..sendfile import sendfile
from ..redis import redisw
from ..db_hack import django_db_cleanup_decorator
from .authentication import get_authenticated_user
from .websocket import TicketOut, TicketRequest, get_ticket
collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@ -51,7 +52,7 @@ class CollectionItemRevisionInOut(BaseModel):
chunks: t.List[ChunkType]
class Config:
orm_mode = True
from_attributes = True
@classmethod
def from_orm_context(
@ -71,13 +72,13 @@ class CollectionItemRevisionInOut(BaseModel):
class CollectionItemCommon(BaseModel):
uid: str
version: int
encryptionKey: t.Optional[bytes]
encryptionKey: t.Optional[bytes] = None
content: CollectionItemRevisionInOut
class CollectionItemOut(CollectionItemCommon):
class Config:
orm_mode = True
from_attributes = True
@classmethod
def from_orm_context(
@ -92,12 +93,12 @@ class CollectionItemOut(CollectionItemCommon):
class CollectionItemIn(CollectionItemCommon):
etag: t.Optional[str]
etag: t.Optional[str] = None
class CollectionCommon(BaseModel):
# FIXME: remove optional once we finish collection-type-migration
collectionType: t.Optional[bytes]
collectionType: t.Optional[bytes] = None
collectionKey: bytes
@ -131,27 +132,27 @@ class RemovedMembershipOut(BaseModel):
class CollectionListResponse(BaseModel):
data: t.List[CollectionOut]
stoken: t.Optional[str]
stoken: t.Optional[str] = None
done: bool
removedMemberships: t.Optional[t.List[RemovedMembershipOut]]
removedMemberships: t.Optional[t.List[RemovedMembershipOut]] = None
class CollectionItemListResponse(BaseModel):
data: t.List[CollectionItemOut]
stoken: t.Optional[str]
stoken: t.Optional[str] = None
done: bool
class CollectionItemRevisionListResponse(BaseModel):
data: t.List[CollectionItemRevisionInOut]
iterator: t.Optional[str]
iterator: t.Optional[str] = None
done: bool
class CollectionItemBulkGetIn(BaseModel):
uid: str
etag: t.Optional[str]
etag: t.Optional[str] = None
class ItemDepIn(BaseModel):
@ -172,7 +173,7 @@ class ItemDepIn(BaseModel):
class ItemBatchIn(BaseModel):
items: t.List[CollectionItemIn]
deps: t.Optional[t.List[ItemDepIn]]
deps: t.Optional[t.List[ItemDepIn]] = None
def validate_db(self):
if self.deps is not None:
@ -274,7 +275,7 @@ def list_multi(
Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True)
)
return collection_list_common(queryset, user, stoken, limit, prefetch)
return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
@collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ)
@ -285,7 +286,7 @@ def collection_list(
user: UserType = Depends(get_authenticated_user),
queryset: CollectionQuerySet = Depends(get_collection_queryset),
):
return collection_list_common(queryset, user, stoken, limit, prefetch)
return MsgpackResponse(collection_list_common(queryset, user, stoken, limit, prefetch))
def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut):
@ -341,7 +342,10 @@ def _create(data: CollectionIn, user: UserType):
# TODO
process_revisions_for_item(main_item, data.item.content)
collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user)
try:
collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user)
except IntegrityError:
raise ValidationError("bad_collection_type", "collectionType is null")
models.CollectionMember(
collection=instance,
@ -364,7 +368,7 @@ def collection_get(
user: UserType = Depends(get_authenticated_user),
prefetch: Prefetch = PrefetchQuery,
):
return CollectionOut.from_orm_context(obj, Context(user, prefetch))
return MsgpackResponse(CollectionOut.from_orm_context(obj, Context(user, prefetch)))
def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool):
@ -373,7 +377,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val
revision_data = item_model.content
uid = item_model.uid
Model = models.CollectionItem
Model = models.CollectionItem # noqa: N806
with transaction.atomic():
instance, created = Model.objects.get_or_create(
@ -417,7 +421,7 @@ def item_get(
prefetch: Prefetch = PrefetchQuery,
):
obj = queryset.get(uid=item_uid)
return CollectionItemOut.from_orm_context(obj, Context(user, prefetch))
return MsgpackResponse(CollectionItemOut.from_orm_context(obj, Context(user, prefetch)))
def item_list_common(
@ -449,7 +453,7 @@ def item_list(
queryset = queryset.filter(parent__isnull=True)
response = item_list_common(queryset, user, stoken, limit, prefetch)
return response
return MsgpackResponse(response)
@item_router.post("/item/subscription-ticket/", response_model=TicketOut, dependencies=PERMISSIONS_READ)
@ -458,7 +462,7 @@ async def item_list_subscription_ticket(
user: UserType = Depends(get_authenticated_user),
):
"""Get an authentication ticket that can be used with the websocket endpoint"""
return await get_ticket(TicketRequest(collection=collection.uid), user)
return MsgpackResponse(await get_ticket(TicketRequest(collection=collection.uid), user))
def item_bulk_common(
@ -526,10 +530,12 @@ def item_revisions(
ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result]
iterator = ret_data[-1].uid if len(result) > 0 else None
return CollectionItemRevisionListResponse(
data=ret_data,
iterator=iterator,
done=done,
return MsgpackResponse(
CollectionItemRevisionListResponse(
data=ret_data,
iterator=iterator,
done=done,
)
)
@ -559,10 +565,12 @@ def fetch_updates(
new_stoken = new_stoken or stoken_rev_uid
context = Context(user, prefetch)
return CollectionItemListResponse(
data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
stoken=new_stoken,
done=True, # we always return all the items, so it's always done
return MsgpackResponse(
CollectionItemListResponse(
data=[CollectionItemOut.from_orm_context(item, context) for item in queryset],
stoken=new_stoken,
done=True, # we always return all the items, so it's always done
)
)
@ -574,7 +582,9 @@ def item_transaction(
stoken: t.Optional[str] = None,
user: UserType = Depends(get_authenticated_user),
):
return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
return MsgpackResponse(
item_bulk_common(data, user, stoken, collection_uid, validate_etag=True, background_tasks=background_tasks)
)
@item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE])
@ -585,7 +595,9 @@ def item_batch(
stoken: t.Optional[str] = None,
user: UserType = Depends(get_authenticated_user),
):
return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
return MsgpackResponse(
item_bulk_common(data, user, stoken, collection_uid, validate_etag=False, background_tasks=background_tasks)
)
# Chunks
@ -610,7 +622,12 @@ async def chunk_update(
collection: models.Collection = Depends(get_collection),
):
# IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid)
content_file = ContentFile(await request.body())
if isinstance(request, MsgpackRequest):
body = await request.raw_body()
else:
body = await request.body()
content_file = ContentFile(body)
try:
await chunk_save(chunk_uid, collection, content_file)
except IntegrityError:

View File

@ -1,26 +1,27 @@
import typing as t
from django.db import transaction, IntegrityError
from django.db import IntegrityError, transaction
from django.db.models import QuerySet
from fastapi import APIRouter, Depends, status, Request
from fastapi import APIRouter, Depends, Request, status
from etebase_server.django import models
from etebase_server.django.utils import get_user_queryset, CallbackContext
from etebase_server.django.utils import CallbackContext, get_user_queryset
from etebase_server.myauth.models import UserType, get_typed_user_model
from .authentication import get_authenticated_user
from ..db_hack import django_db_cleanup_decorator
from ..exceptions import HttpError, PermissionDenied
from ..msgpack import MsgpackRoute
from ..msgpack import MsgpackResponse, MsgpackRoute
from ..utils import (
get_object_or_404,
get_user_username_email_kwargs,
Context,
is_collection_admin,
BaseModel,
permission_responses,
PERMISSIONS_READ,
PERMISSIONS_READWRITE,
BaseModel,
Context,
get_object_or_404,
get_user_username_email_kwargs,
is_collection_admin,
permission_responses,
)
from ..db_hack import django_db_cleanup_decorator
from .authentication import get_authenticated_user
User = get_typed_user_model()
invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
@ -33,7 +34,7 @@ class UserInfoOut(BaseModel):
pubkey: bytes
class Config:
orm_mode = True
from_attributes = True
@classmethod
def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut":
@ -66,7 +67,7 @@ class CollectionInvitationOut(CollectionInvitationCommon):
fromPubkey: bytes
class Config:
orm_mode = True
from_attributes = True
@classmethod
def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut":
@ -84,7 +85,7 @@ class CollectionInvitationOut(CollectionInvitationCommon):
class InvitationListResponse(BaseModel):
data: t.List[CollectionInvitationOut]
iterator: t.Optional[str]
iterator: t.Optional[str] = None
done: bool
@ -120,7 +121,7 @@ def list_common(
iterator = ret_data[-1].uid if len(result) > 0 else None
return InvitationListResponse(
data=ret_data,
data=[CollectionInvitationOut.from_orm(x) for x in ret_data],
iterator=iterator,
done=done,
)
@ -132,7 +133,7 @@ def incoming_list(
limit: int = 50,
queryset: InvitationQuerySet = Depends(get_incoming_queryset),
):
return list_common(queryset, iterator, limit)
return MsgpackResponse(list_common(queryset, iterator, limit))
@invitation_incoming_router.get(
@ -143,7 +144,7 @@ def incoming_get(
queryset: InvitationQuerySet = Depends(get_incoming_queryset),
):
obj = get_object_or_404(queryset, uid=invitation_uid)
return CollectionInvitationOut.from_orm(obj)
return MsgpackResponse(CollectionInvitationOut.from_orm(obj))
@invitation_incoming_router.delete(
@ -218,7 +219,7 @@ def outgoing_list(
limit: int = 50,
queryset: InvitationQuerySet = Depends(get_outgoing_queryset),
):
return list_common(queryset, iterator, limit)
return MsgpackResponse(list_common(queryset, iterator, limit))
@invitation_outgoing_router.delete(
@ -241,4 +242,4 @@ def outgoing_fetch_user_profile(
kwargs = get_user_username_email_kwargs(username)
user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs)
user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user)
return UserInfoOut.from_orm(user_info)
return MsgpackResponse(UserInfoOut.from_orm(user_info))

View File

@ -6,12 +6,12 @@ from fastapi import APIRouter, Depends, status
from etebase_server.django import models
from etebase_server.myauth.models import UserType, get_typed_user_model
from .authentication import get_authenticated_user
from ..msgpack import MsgpackRoute
from ..utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE
from ..stoken_handler import filter_by_stoken_and_limit
from ..db_hack import django_db_cleanup_decorator
from ..db_hack import django_db_cleanup_decorator
from ..msgpack import MsgpackResponse, MsgpackRoute
from ..stoken_handler import filter_by_stoken_and_limit
from ..utils import PERMISSIONS_READ, PERMISSIONS_READWRITE, BaseModel, get_object_or_404, permission_responses
from .authentication import get_authenticated_user
from .collection import get_collection, verify_collection_admin
User = get_typed_user_model()
@ -39,7 +39,7 @@ class CollectionMemberOut(BaseModel):
accessLevel: models.AccessLevels
class Config:
orm_mode = True
from_attributes = True
@classmethod
def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut":
@ -48,7 +48,7 @@ class CollectionMemberOut(BaseModel):
class MemberListResponse(BaseModel):
data: t.List[CollectionMemberOut]
iterator: t.Optional[str]
iterator: t.Optional[str] = None
done: bool
@ -66,10 +66,12 @@ def member_list(
)
new_stoken = new_stoken_obj and new_stoken_obj.uid
return MemberListResponse(
data=[CollectionMemberOut.from_orm(item) for item in result],
iterator=new_stoken,
done=done,
return MsgpackResponse(
MemberListResponse(
data=[CollectionMemberOut.from_orm(item) for item in result],
iterator=new_stoken,
done=done,
)
)

View File

@ -3,12 +3,13 @@ from django.db import transaction
from django.shortcuts import get_object_or_404
from fastapi import APIRouter, Request, status
from etebase_server.django.utils import get_user_queryset, CallbackContext
from .authentication import SignupIn, signup_save
from ..msgpack import MsgpackRoute
from ..exceptions import HttpError
from etebase_server.django.utils import CallbackContext, get_user_queryset
from etebase_server.myauth.models import get_typed_user_model
from ..exceptions import HttpError
from ..msgpack import MsgpackRoute
from .authentication import SignupIn, signup_save
test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"])
User = get_typed_user_model()

View File

@ -1,13 +1,13 @@
import asyncio
import typing as t
from redis import asyncio as aioredis
from redis.exceptions import ConnectionError
import nacl.encoding
import nacl.utils
from asgiref.sync import sync_to_async
from django.db.models import QuerySet
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, status
import nacl.encoding
import nacl.utils
from redis import asyncio as aioredis
from redis.exceptions import ConnectionError
from etebase_server.django import models
from etebase_server.django.utils import CallbackContext, get_user_queryset
@ -19,7 +19,6 @@ from ..msgpack import MsgpackRoute, msgpack_decode, msgpack_encode
from ..redis import redisw
from ..utils import BaseModel, permission_responses
User = get_typed_user_model()
websocket_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses)
CollectionQuerySet = QuerySet[models.Collection]

View File

@ -1,14 +1,15 @@
import logging
import os
from functools import lru_cache
from importlib import import_module
from pathlib import Path, PurePath
from urllib.parse import quote
import logging
from fastapi import status
from ..exceptions import HttpError
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from fastapi import status
from ..exceptions import HttpError
logger = logging.getLogger(__name__)
@ -32,9 +33,7 @@ def _convert_file_to_url(path):
path_obj = PurePath(path)
relpath = path_obj.relative_to(path_root)
# Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
# already instantiated Path object
url = relpath._flavour.pathmod.normpath(str(url_root / relpath))
url = os.path.normpath(str(url_root / relpath))
return quote(str(url))
@ -48,9 +47,7 @@ def _sanitize_path(filepath):
filepath_obj = Path(filepath)
# get absolute path
# Python 3.5: Path.resolve() has no `strict` kwarg, so use pathmod from an
# already instantiated Path object
filepath_abs = Path(filepath_obj._flavour.pathmod.normpath(str(path_root / filepath_obj)))
filepath_abs = Path(os.path.normpath(str(path_root / filepath_obj)))
# if filepath_abs is not relative to path_root, relative_to throws an error
try:

View File

@ -47,7 +47,6 @@ def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]:
def filter_by_stoken_and_limit(
stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation
) -> t.Tuple[list, t.Optional[Stoken], bool]:
queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation)
result = list(queryset[: limit + 1])

View File

@ -1,14 +1,13 @@
import base64
import dataclasses
import typing as t
from typing_extensions import Literal
import msgpack
import base64
from fastapi import status, Query, Depends
from pydantic import BaseModel as PyBaseModel
from django.db.models import Model, QuerySet
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Model, QuerySet
from fastapi import Depends, Query, status
from pydantic import BaseModel as PyBaseModel
from typing_extensions import Literal
from etebase_server.django import app_settings
from etebase_server.django.models import AccessLevels
@ -26,10 +25,7 @@ T = t.TypeVar("T", bound=Model, covariant=True)
class BaseModel(PyBaseModel):
class Config:
json_encoders = {
bytes: lambda x: x,
}
pass
@dataclasses.dataclass

View File

@ -1,12 +1,21 @@
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin as DjangoUserAdmin
from .models import User
from .forms import AdminUserCreationForm
from .models import User
class UserAdmin(DjangoUserAdmin):
add_form = AdminUserCreationForm
add_fieldsets = ((None, {"classes": ("wide",), "fields": ("username",),}),)
add_fieldsets = (
(
None,
{
"classes": ("wide",),
"fields": ("username",),
},
),
)
admin.site.register(User, UserAdmin)

View File

@ -1,5 +1,6 @@
from django import forms
from django.contrib.auth.forms import UsernameField
from etebase_server.myauth.models import get_typed_user_model
User = get_typed_user_model()

View File

@ -1,15 +1,15 @@
import logging
from django.utils import timezone
import ldap
from django.conf import settings
from django.core.exceptions import PermissionDenied as DjangoPermissionDenied
from etebase_server.django.utils import CallbackContext
from etebase_server.myauth.models import get_typed_user_model, UserType
from etebase_server.fastapi.dependencies import get_authenticated_user
from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied
from django.utils import timezone
from fastapi import Depends
import ldap
from etebase_server.django.utils import CallbackContext
from etebase_server.fastapi.dependencies import get_authenticated_user
from etebase_server.fastapi.exceptions import PermissionDenied as FastAPIPermissionDenied
from etebase_server.myauth.models import UserType, get_typed_user_model
User = get_typed_user_model()

View File

@ -1,21 +1,13 @@
import typing as t
from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager
from django.core import validators
from django.contrib.auth.validators import UnicodeUsernameValidator
from django.db import models
from django.utils.deconstruct import deconstructible
from django.utils.translation import gettext_lazy as _
@deconstructible
class UnicodeUsernameValidator(validators.RegexValidator):
regex = r"^[\w.-]+\Z"
message = _("Enter a valid username. This value may contain only letters, " "numbers, and ./-/_ characters.")
flags = 0
class UserManager(DjangoUserManager):
def get_by_natural_key(self, username: str):
def get_by_natural_key(self, username: t.Optional[str]):
return self.get(**{self.model.USERNAME_FIELD + "__iexact": username})

View File

@ -1,3 +1 @@
from django.test import TestCase
# Create your tests here.

View File

@ -1,3 +1 @@
from django.shortcuts import render
# Create your views here.

View File

@ -10,8 +10,9 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.0/ref/settings/
"""
import os
import configparser
import os
from .utils import get_secret_from_file
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@ -20,6 +21,7 @@ BASE_DIR = os.path.dirname(SOURCE_DIR)
AUTH_USER_MODEL = "myauth.User"
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
@ -44,7 +46,7 @@ DATABASES = {
}
}
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
# Application definition
@ -162,6 +164,8 @@ if any(os.path.isfile(x) for x in config_locations):
if "allowed_hosts" in config:
ALLOWED_HOSTS = [y for x, y in config.items("allowed_hosts")]
CSRF_TRUSTED_ORIGINS = ["https://" + y for x, y in config.items("allowed_hosts")] + \
["http://" + y for x, y in config.items("allowed_hosts")]
if "database" in config:
DATABASES = {"default": {x.upper(): y for x, y in config.items("database")}}
@ -200,7 +204,7 @@ if "DJANGO_MEDIA_ROOT" in os.environ:
# Make an `etebase_server_settings` module available to override settings.
try:
from etebase_server_settings import *
from etebase_server_settings import * # noqa: F403
except ImportError:
pass

View File

@ -1,11 +1,6 @@
import os
from django.conf import settings
from django.contrib import admin
from django.urls import path
from django.views.generic import TemplateView
from django.views.static import serve
from django.contrib.staticfiles import finders
urlpatterns = [
path("admin/", admin.site.urls),

View File

@ -12,10 +12,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.core.management import utils
import os
import stat
from django.core.management import utils
def get_secret_from_file(path):
try:

View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Django's command-line utility for administrative tasks."""
import os
import sys

View File

@ -4,3 +4,25 @@ line-length = 120
[build-system]
requires = ["setuptools>=42"]
build-backend = "setuptools.build_meta"
[tool.ruff]
line-length = 120
exclude = [
".git",
".git-rewrite",
".mypy_cache",
".pytype",
".ruff_cache",
".venv",
"build",
"dist",
"node_modules",
"migrations", # Alembic migrations
]
[tool.ruff.lint]
select = ["E", "F", "I", "N", "T20", "W"]
ignore = ["E203", "E501", "E711", "E712", "N803", "N815", "N818", "T201"]
[tool.ruff.lint.isort]
combine-as-imports = true

View File

@ -1,66 +1,53 @@
#
# This file is autogenerated by pip-compile with Python 3.11
# This file is autogenerated by pip-compile with Python 3.12
# by the following command:
#
# pip-compile --output-file=requirements-dev.txt requirements.in/development.txt
#
asgiref==3.5.2
# via django
black==22.10.0
# via -r requirements.in/development.txt
build==0.9.0
asgiref==3.8.1
# via
# django
# django-stubs
build==1.2.1
# via pip-tools
click==8.1.3
click==8.1.7
# via pip-tools
coverage==7.5.3
# via -r requirements.in/development.txt
django==5.0.6
# via
# black
# django-stubs
# django-stubs-ext
django-stubs==5.0.2
# via -r requirements.in/development.txt
django-stubs-ext==5.0.2
# via django-stubs
mypy==1.10.0
# via -r requirements.in/development.txt
mypy-extensions==1.0.0
# via mypy
packaging==24.0
# via build
pip-tools==7.4.1
# via -r requirements.in/development.txt
pyproject-hooks==1.1.0
# via
# build
# pip-tools
coverage==6.5.0
pywatchman==2.0.0
# via -r requirements.in/development.txt
django==3.2.16
# via
# -r requirements.in/development.txt
# django-stubs
# django-stubs-ext
django-stubs==1.13.0
ruff==0.4.8
# via -r requirements.in/development.txt
django-stubs-ext==0.7.0
# via django-stubs
mypy==0.991
# via django-stubs
mypy-extensions==0.4.3
# via
# black
# mypy
packaging==21.3
# via build
pathspec==0.10.2
# via black
pep517==0.13.0
# via build
pip-tools==6.11.0
# via -r requirements.in/development.txt
platformdirs==2.6.0
# via black
pyparsing==3.0.9
# via packaging
pytz==2022.6
sqlparse==0.5.0
# via django
pywatchman==1.4.1
# via -r requirements.in/development.txt
sqlparse==0.4.3
# via django
tomli==2.0.1
types-pyyaml==6.0.12.20240311
# via django-stubs
types-pytz==2022.6.0.1
# via django-stubs
types-pyyaml==6.0.12.2
# via django-stubs
typing-extensions==4.4.0
typing-extensions==4.12.2
# via
# django-stubs
# django-stubs-ext
# mypy
wheel==0.38.4
wheel==0.43.0
# via pip-tools
# The following packages are considered to be unsafe in a requirements file:

View File

@ -1,7 +1,8 @@
django>=4.0<5.0
django>=4.0,<5.0
msgpack
pynacl
fastapi
fastapi>=0.104
pydantic>=2.0.0
typing_extensions
uvicorn[standard]
aiofiles

View File

@ -1,6 +1,6 @@
coverage
pip-tools
pywatchman
black
ruff
mypy
django-stubs
django<4.0

View File

@ -4,59 +4,114 @@
#
# pip-compile --output-file=requirements.txt requirements.in/base.txt
#
aiofiles==22.1.0
aiofiles==23.2.1
# via -r requirements.in/base.txt
anyio==3.6.2
annotated-types==0.7.0
# via pydantic
anyio==4.4.0
# via
# httpx
# starlette
# watchfiles
asgiref==3.5.2
asgiref==3.8.1
# via django
async-timeout==4.0.2
# via redis
cffi==1.15.1
certifi==2024.6.2
# via
# httpcore
# httpx
cffi==1.16.0
# via pynacl
click==8.1.3
# via uvicorn
django==4.1.13
click==8.1.7
# via
# typer
# uvicorn
django==4.2.14
# via -r requirements.in/base.txt
fastapi==0.88.0
dnspython==2.6.1
# via email-validator
email-validator==2.1.1
# via fastapi
fastapi==0.111.0
# via -r requirements.in/base.txt
fastapi-cli==0.0.4
# via fastapi
h11==0.14.0
# via
# httpcore
# uvicorn
httpcore==1.0.5
# via httpx
httptools==0.6.1
# via uvicorn
httptools==0.5.0
# via uvicorn
idna==3.4
# via anyio
msgpack==1.0.4
httpx==0.27.0
# via fastapi
idna==3.7
# via
# anyio
# email-validator
# httpx
jinja2==3.1.4
# via fastapi
markdown-it-py==3.0.0
# via rich
markupsafe==2.1.5
# via jinja2
mdurl==0.1.2
# via markdown-it-py
msgpack==1.0.8
# via -r requirements.in/base.txt
pycparser==2.21
orjson==3.10.3
# via fastapi
pycparser==2.22
# via cffi
pydantic==1.10.2
# via fastapi
pynacl==1.5.0
# via -r requirements.in/base.txt
python-dotenv==0.21.0
# via uvicorn
pyyaml==6.0.1
# via uvicorn
redis==4.4.0
# via -r requirements.in/base.txt
sniffio==1.3.0
# via anyio
sqlparse==0.4.3
# via django
starlette==0.22.0
# via fastapi
typing-extensions==4.4.0
pydantic==2.7.3
# via
# -r requirements.in/base.txt
# pydantic
uvicorn[standard]==0.20.0
# fastapi
pydantic-core==2.18.4
# via pydantic
pygments==2.18.0
# via rich
pynacl==1.5.0
# via -r requirements.in/base.txt
uvloop==0.17.0
python-dotenv==1.0.1
# via uvicorn
watchfiles==0.18.1
python-multipart==0.0.9
# via fastapi
pyyaml==6.0.1
# via uvicorn
websockets==10.4
redis==5.1.0b6
# via -r requirements.in/base.txt
rich==13.7.1
# via typer
shellingham==1.5.4
# via typer
sniffio==1.3.1
# via
# anyio
# httpx
sqlparse==0.5.0
# via django
starlette==0.37.2
# via fastapi
typer==0.12.3
# via fastapi-cli
typing-extensions==4.12.2
# via
# -r requirements.in/base.txt
# fastapi
# pydantic
# pydantic-core
# typer
ujson==5.10.0
# via fastapi
uvicorn[standard]==0.30.1
# via
# -r requirements.in/base.txt
# fastapi
uvloop==0.19.0
# via uvicorn
watchfiles==0.22.0
# via uvicorn
websockets==12.0
# via uvicorn

View File

@ -2,7 +2,7 @@ from setuptools import find_packages, setup
setup(
name="etebase_server",
version="0.13.0",
version="0.14.2",
description="An Etebase (EteSync 2.0) server",
url="https://www.etebase.com/",
classifiers=[