diff --git a/README.md b/README.md index 3e0bd53..1787a2f 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Now you can initialise our django app. And you are done! You can now run the debug server just to see everything works as expected by running: ``` -./manage.py runserver 0.0.0.0:8000 +uvicorn etebase_server.asgi:application --port 8000 ``` Using the debug server in production is not recommended, so please read the following section for a proper deployment. diff --git a/django_etebase/admin.py b/django_etebase/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/django_etebase/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index 7c93f5f..90225a6 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -32,22 +32,16 @@ class AppSettings: return getattr(settings, self.prefix + name, dflt) @cached_property - def API_PERMISSIONS(self): # pylint: disable=invalid-name - perms = self._setting("API_PERMISSIONS", ("rest_framework.permissions.IsAuthenticated",)) + def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name + perms = self._setting("API_PERMISSIONS_READ", tuple()) ret = [] for perm in perms: ret.append(self.import_from_str(perm)) return ret @cached_property - def API_AUTHENTICATORS(self): # pylint: disable=invalid-name - perms = self._setting( - "API_AUTHENTICATORS", - ( - "rest_framework.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", - ), - ) + def API_PERMISSIONS_WRITE(self): # pylint: disable=invalid-name + perms = self._setting("API_PERMISSIONS_WRITE", tuple()) ret = [] for perm in perms: ret.append(self.import_from_str(perm)) diff --git a/django_etebase/drf_msgpack/apps.py b/django_etebase/drf_msgpack/apps.py deleted file mode 100644 index 22ea2c1..0000000 --- a/django_etebase/drf_msgpack/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class DrfMsgpackConfig(AppConfig): - name = "drf_msgpack" diff --git a/django_etebase/drf_msgpack/parsers.py b/django_etebase/drf_msgpack/parsers.py deleted file mode 100644 index 0504a76..0000000 --- a/django_etebase/drf_msgpack/parsers.py +++ /dev/null @@ -1,14 +0,0 @@ -import msgpack - -from rest_framework.parsers import BaseParser -from rest_framework.exceptions import ParseError - - -class MessagePackParser(BaseParser): - media_type = "application/msgpack" - - def parse(self, stream, media_type=None, parser_context=None): - try: - return msgpack.unpackb(stream.read(), raw=False) - except Exception as exc: - raise ParseError("MessagePack parse error - %s" % str(exc)) diff --git a/django_etebase/drf_msgpack/renderers.py b/django_etebase/drf_msgpack/renderers.py deleted file mode 100644 index 35a4afa..0000000 --- a/django_etebase/drf_msgpack/renderers.py +++ /dev/null @@ -1,15 +0,0 @@ -import msgpack - -from rest_framework.renderers import BaseRenderer - - -class MessagePackRenderer(BaseRenderer): - media_type = "application/msgpack" - format = "msgpack" - render_style = "binary" - charset = None - - def render(self, data, media_type=None, renderer_context=None): - if data is None: - return b"" - return msgpack.packb(data, use_bin_type=True) diff --git a/django_etebase/drf_msgpack/views.py b/django_etebase/drf_msgpack/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/django_etebase/drf_msgpack/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/django_etebase/exceptions.py b/django_etebase/exceptions.py deleted file mode 100644 index 437a71c..0000000 --- a/django_etebase/exceptions.py +++ /dev/null @@ -1,12 +0,0 @@ -from rest_framework import serializers, status - - -class EtebaseValidationError(serializers.ValidationError): - def __init__(self, code, detail, status_code=status.HTTP_400_BAD_REQUEST): - super().__init__( - { - "code": code, - "detail": detail, - } - ) - self.status_code = status_code diff --git a/django_etebase/parsers.py b/django_etebase/parsers.py deleted file mode 100644 index ed1e713..0000000 --- a/django_etebase/parsers.py +++ /dev/null @@ -1,14 +0,0 @@ -from rest_framework.parsers import FileUploadParser - - -class ChunkUploadParser(FileUploadParser): - """ - Parser for chunk upload data. - """ - - def get_filename(self, stream, media_type, parser_context): - """ - Detects the uploaded file name. - """ - view = parser_context["view"] - return parser_context["kwargs"][view.lookup_field] diff --git a/django_etebase/permissions.py b/django_etebase/permissions.py deleted file mode 100644 index 3c77d06..0000000 --- a/django_etebase/permissions.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from rest_framework import permissions -from django_etebase.models import Collection, AccessLevels - - -def is_collection_admin(collection, user): - member = collection.members.filter(user=user).first() - return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) - - -class IsCollectionAdmin(permissions.BasePermission): - """ - Custom permission to only allow owners of a collection to view it - """ - - message = { - "detail": "Only collection admins can perform this operation.", - "code": "admin_access_required", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs["collection_uid"] - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - return is_collection_admin(collection, request.user) - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True - - -class IsCollectionAdminOrReadOnly(permissions.BasePermission): - """ - Custom permission to only allow owners of a collection to edit it - """ - - message = { - "detail": "Only collection admins can edit collections.", - "code": "admin_access_required", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs.get("collection_uid", None) - - # Allow creating new collections - if collection_uid is None: - return True - - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - if request.method in permissions.SAFE_METHODS: - return True - - return is_collection_admin(collection, request.user) - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True - - -class HasWriteAccessOrReadOnly(permissions.BasePermission): - """ - Custom permission to restrict write - """ - - message = { - "detail": "You need write access to write to this collection", - "code": "no_write_access", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs["collection_uid"] - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - if request.method in permissions.SAFE_METHODS: - return True - else: - member = collection.members.get(user=request.user) - return member.accessLevel != AccessLevels.READ_ONLY - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True diff --git a/django_etebase/renderers.py b/django_etebase/renderers.py deleted file mode 100644 index 0d359d3..0000000 --- a/django_etebase/renderers.py +++ /dev/null @@ -1,19 +0,0 @@ -from rest_framework.utils.encoders import JSONEncoder as DRFJSONEncoder -from rest_framework.renderers import JSONRenderer as DRFJSONRenderer - -from .serializers import b64encode - - -class JSONEncoder(DRFJSONEncoder): - def default(self, obj): - if isinstance(obj, bytes) or isinstance(obj, memoryview): - return b64encode(obj) - return super().default(obj) - - -class JSONRenderer(DRFJSONRenderer): - """ - Renderer which serializes to JSON with support for our base64 - """ - - encoder_class = JSONEncoder diff --git a/django_etebase/sendfile/backends/__init__.py b/django_etebase/sendfile/backends/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/sendfile/backends/development.py b/django_etebase/sendfile/backends/development.py deleted file mode 100644 index d321932..0000000 --- a/django_etebase/sendfile/backends/development.py +++ /dev/null @@ -1,17 +0,0 @@ -import os.path - -from django.views.static import serve - - -def sendfile(request, filename, **kwargs): - """ - Send file using Django dev static file server. - - .. warning:: - - Do not use in production. This is only to be used when developing and - is provided for convenience only - """ - dirname = os.path.dirname(filename) - basename = os.path.basename(filename) - return serve(request, basename, dirname) diff --git a/django_etebase/sendfile/backends/mod_wsgi.py b/django_etebase/sendfile/backends/mod_wsgi.py deleted file mode 100644 index 07ba3f1..0000000 --- a/django_etebase/sendfile/backends/mod_wsgi.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import absolute_import - -from django.http import HttpResponse - -from ..utils import _convert_file_to_url - - -def sendfile(request, filename, **kwargs): - response = HttpResponse() - response['Location'] = _convert_file_to_url(filename) - # need to destroy get_host() to stop django - # rewriting our location to include http, so that - # mod_wsgi is able to do the internal redirect - request.get_host = lambda: '' - request.build_absolute_uri = lambda location: location - - return response diff --git a/django_etebase/sendfile/backends/nginx.py b/django_etebase/sendfile/backends/nginx.py deleted file mode 100644 index 8764309..0000000 --- a/django_etebase/sendfile/backends/nginx.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import absolute_import - -from django.http import HttpResponse - -from ..utils import _convert_file_to_url - - -def sendfile(request, filename, **kwargs): - response = HttpResponse() - response['X-Accel-Redirect'] = _convert_file_to_url(filename) - - return response diff --git a/django_etebase/sendfile/backends/simple.py b/django_etebase/sendfile/backends/simple.py deleted file mode 100644 index 0549b20..0000000 --- a/django_etebase/sendfile/backends/simple.py +++ /dev/null @@ -1,60 +0,0 @@ -from email.utils import mktime_tz, parsedate_tz -import re - -from django.core.files.base import File -from django.http import HttpResponse, HttpResponseNotModified -from django.utils.http import http_date - - -def sendfile(request, filepath, **kwargs): - '''Use the SENDFILE_ROOT value composed with the path arrived as argument - to build an absolute path with which resolve and return the file contents. - - If the path points to a file out of the root directory (should cover both - situations with '..' and symlinks) then a 404 is raised. - ''' - statobj = filepath.stat() - - # Respect the If-Modified-Since header. - if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'), - statobj.st_mtime, statobj.st_size): - return HttpResponseNotModified() - - with File(filepath.open('rb')) as f: - response = HttpResponse(f.chunks()) - - response["Last-Modified"] = http_date(statobj.st_mtime) - return response - - -def was_modified_since(header=None, mtime=0, size=0): - """ - Was something modified since the user last downloaded it? - - header - This is the value of the If-Modified-Since header. If this is None, - I'll just return True. - - mtime - This is the modification time of the item we're talking about. - - size - This is the size of the item we're talking about. - """ - try: - if header is None: - raise ValueError - matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, - re.IGNORECASE) - header_date = parsedate_tz(matches.group(1)) - if header_date is None: - raise ValueError - header_mtime = mktime_tz(header_date) - header_len = matches.group(3) - if header_len and int(header_len) != size: - raise ValueError - if mtime > header_mtime: - raise ValueError - except (AttributeError, ValueError, OverflowError): - return True - return False diff --git a/django_etebase/sendfile/backends/xsendfile.py b/django_etebase/sendfile/backends/xsendfile.py deleted file mode 100644 index 74993ee..0000000 --- a/django_etebase/sendfile/backends/xsendfile.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.http import HttpResponse - - -def sendfile(request, filename, **kwargs): - filename = str(filename) - response = HttpResponse() - response['X-Sendfile'] = filename - - return response diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py deleted file mode 100644 index 26ac5a7..0000000 --- a/django_etebase/serializers.py +++ /dev/null @@ -1,598 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import base64 - -from django.core.files.base import ContentFile -from django.core import exceptions as django_exceptions -from django.contrib.auth import get_user_model -from django.db import IntegrityError, transaction -from rest_framework import serializers, status -from . import models -from .utils import get_user_queryset, create_user, CallbackContext - -from .exceptions import EtebaseValidationError - -User = get_user_model() - - -def process_revisions_for_item(item, revision_data): - chunks_objs = [] - chunks = revision_data.pop("chunks_relation") - - revision = models.CollectionItemRevision(**revision_data, item=item) - revision.validate_unique() # Verify there aren't any validation issues - - for chunk in chunks: - uid = chunk[0] - chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() - content = chunk[1] if len(chunk) > 1 else None - # If the chunk already exists we assume it's fine. Otherwise, we upload it. - if chunk_obj is None: - if content is not None: - chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) - chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) - chunk_obj.save() - else: - raise EtebaseValidationError("chunk_no_content", "Tried to create a new chunk without content") - - chunks_objs.append(chunk_obj) - - stoken = models.Stoken.objects.create() - revision.stoken = stoken - revision.save() - - for chunk in chunks_objs: - models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) - return revision - - -def b64encode(value): - return base64.urlsafe_b64encode(value).decode("ascii").strip("=") - - -def b64decode(data): - data += "=" * ((4 - len(data) % 4) % 4) - return base64.urlsafe_b64decode(data) - - -def b64decode_or_bytes(data): - if isinstance(data, bytes): - return data - else: - return b64decode(data) - - -class BinaryBase64Field(serializers.Field): - def to_representation(self, value): - return value - - def to_internal_value(self, data): - return b64decode_or_bytes(data) - - -class CollectionEncryptionKeyField(BinaryBase64Field): - def get_attribute(self, instance): - request = self.context.get("request", None) - if request is not None: - return instance.members.get(user=request.user).encryptionKey - return None - - -class CollectionTypeField(BinaryBase64Field): - def get_attribute(self, instance): - request = self.context.get("request", None) - if request is not None: - collection_type = instance.members.get(user=request.user).collectionType - return collection_type and collection_type.uid - return None - - -class UserSlugRelatedField(serializers.SlugRelatedField): - def get_queryset(self): - view = self.context.get("view", None) - return get_user_queryset(super().get_queryset(), context=CallbackContext(view.kwargs)) - - def __init__(self, **kwargs): - super().__init__(slug_field=User.USERNAME_FIELD, **kwargs) - - def to_internal_value(self, data): - return super().to_internal_value(data.lower()) - - -class ChunksField(serializers.RelatedField): - def to_representation(self, obj): - obj = obj.chunk - if self.context.get("prefetch") == "auto": - with open(obj.chunkFile.path, "rb") as f: - return (obj.uid, f.read()) - else: - return (obj.uid,) - - def to_internal_value(self, data): - content = data[1] if len(data) > 1 else None - if data[0] is None: - raise EtebaseValidationError("no_null", "null is not allowed") - return (data[0], b64decode_or_bytes(content) if content is not None else None) - - -class BetterErrorsMixin: - @property - def errors(self): - nice = [] - errors = super().errors - for error_type in errors: - if error_type == "non_field_errors": - nice.extend(self.flatten_errors(None, errors[error_type])) - else: - nice.extend(self.flatten_errors(error_type, errors[error_type])) - if nice: - return {"code": "field_errors", "detail": "Field validations failed.", "errors": nice} - return {} - - def flatten_errors(self, field_name, errors): - ret = [] - if isinstance(errors, dict): - for error_key in errors: - error = errors[error_key] - ret.extend(self.flatten_errors("{}.{}".format(field_name, error_key), error)) - else: - for error in errors: - if getattr(error, "messages", None): - message = error.messages[0] - else: - message = str(error) - ret.append( - { - "field": field_name, - "code": error.code, - "detail": message, - } - ) - return ret - - def transform_validation_error(self, prefix, err): - if hasattr(err, "error_dict"): - errors = self.flatten_errors(prefix, err.error_dict) - elif not hasattr(err, "message"): - errors = self.flatten_errors(prefix, err.error_list) - else: - raise EtebaseValidationError(err.code, err.message) - - raise serializers.ValidationError( - { - "code": "field_errors", - "detail": "Field validations failed.", - "errors": errors, - } - ) - - -class CollectionItemChunkSerializer(BetterErrorsMixin, serializers.ModelSerializer): - class Meta: - model = models.CollectionItemChunk - fields = ("uid", "chunkFile") - - -class CollectionItemRevisionSerializer(BetterErrorsMixin, serializers.ModelSerializer): - chunks = ChunksField( - source="chunks_relation", - queryset=models.RevisionChunkRelation.objects.all(), - style={"base_template": "input.html"}, - many=True, - ) - meta = BinaryBase64Field() - - class Meta: - model = models.CollectionItemRevision - fields = ("chunks", "meta", "uid", "deleted") - extra_kwargs = { - "uid": {"validators": []}, # We deal with it in the serializers - } - - -class CollectionItemSerializer(BetterErrorsMixin, serializers.ModelSerializer): - encryptionKey = BinaryBase64Field(required=False, default=None, allow_null=True) - etag = serializers.CharField(allow_null=True, write_only=True) - content = CollectionItemRevisionSerializer(many=False) - - class Meta: - model = models.CollectionItem - fields = ("uid", "version", "encryptionKey", "content", "etag") - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - validate_etag = self.context.get("validate_etag", False) - etag = validated_data.pop("etag") - revision_data = validated_data.pop("content") - uid = validated_data.pop("uid") - - Model = self.__class__.Meta.model - - with transaction.atomic(): - instance, created = Model.objects.get_or_create(uid=uid, defaults=validated_data) - cur_etag = instance.etag if not created else None - - # If we are trying to update an up to date item, abort early and consider it a success - if cur_etag == revision_data.get("uid"): - return instance - - if validate_etag and cur_etag != etag: - raise EtebaseValidationError( - "wrong_etag", - "Wrong etag. Expected {} got {}".format(cur_etag, etag), - status_code=status.HTTP_409_CONFLICT, - ) - - if not created: - # We don't have to use select_for_update here because the unique constraint on current guards against - # the race condition. But it's a good idea because it'll lock and wait rather than fail. - current_revision = instance.revisions.filter(current=True).select_for_update().first() - - # If we are just re-uploading the same revision, consider it a succes and return. - if current_revision.uid == revision_data.get("uid"): - return instance - - current_revision.current = None - current_revision.save() - - try: - process_revisions_for_item(instance, revision_data) - except django_exceptions.ValidationError as e: - self.transform_validation_error("content", e) - - return instance - - def update(self, instance, validated_data): - # We never update, we always update in the create method - raise NotImplementedError() - - -class CollectionItemDepSerializer(BetterErrorsMixin, serializers.ModelSerializer): - etag = serializers.CharField() - - class Meta: - model = models.CollectionItem - fields = ("uid", "etag") - - def validate(self, data): - item = self.__class__.Meta.model.objects.get(uid=data["uid"]) - etag = data["etag"] - if item.etag != etag: - raise EtebaseValidationError( - "wrong_etag", - "Wrong etag. Expected {} got {}".format(item.etag, etag), - status_code=status.HTTP_409_CONFLICT, - ) - - return data - - -class CollectionItemBulkGetSerializer(BetterErrorsMixin, serializers.ModelSerializer): - etag = serializers.CharField(required=False) - - class Meta: - model = models.CollectionItem - fields = ("uid", "etag") - - -class CollectionListMultiSerializer(BetterErrorsMixin, serializers.Serializer): - collectionTypes = serializers.ListField(child=BinaryBase64Field()) - - -class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): - collectionKey = CollectionEncryptionKeyField() - collectionType = CollectionTypeField() - accessLevel = serializers.SerializerMethodField("get_access_level_from_context") - stoken = serializers.CharField(read_only=True) - - item = CollectionItemSerializer(many=False, source="main_item") - - class Meta: - model = models.Collection - fields = ("item", "accessLevel", "collectionKey", "collectionType", "stoken") - - def get_access_level_from_context(self, obj): - request = self.context.get("request", None) - if request is not None: - return obj.members.get(user=request.user).accessLevel - return None - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - collection_key = validated_data.pop("collectionKey") - collection_type = validated_data.pop("collectionType") - - user = validated_data.get("owner") - main_item_data = validated_data.pop("main_item") - uid = main_item_data.get("uid") - etag = main_item_data.pop("etag") - revision_data = main_item_data.pop("content") - - instance = self.__class__.Meta.model(uid=uid, **validated_data) - - with transaction.atomic(): - if etag is not None: - raise EtebaseValidationError("bad_etag", "etag is not null") - - try: - instance.validate_unique() - except django_exceptions.ValidationError: - raise EtebaseValidationError( - "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT - ) - instance.save() - - main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) - - instance.main_item = main_item - instance.save() - - process_revisions_for_item(main_item, revision_data) - - collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) - - models.CollectionMember( - collection=instance, - stoken=models.Stoken.objects.create(), - user=user, - accessLevel=models.AccessLevels.ADMIN, - encryptionKey=collection_key, - collectionType=collection_type_obj, - ).save() - - return instance - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField( - source="user", - read_only=True, - style={"base_template": "input.html"}, - ) - - class Meta: - model = models.CollectionMember - fields = ("username", "accessLevel") - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - with transaction.atomic(): - # We only allow updating accessLevel - access_level = validated_data.pop("accessLevel") - if instance.accessLevel != access_level: - instance.stoken = models.Stoken.objects.create() - instance.accessLevel = access_level - instance.save() - - return instance - - -class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField( - source="user", - queryset=User.objects, - style={"base_template": "input.html"}, - ) - collection = serializers.CharField(source="collection.uid") - fromUsername = serializers.CharField(source="fromMember.user.username", read_only=True) - fromPubkey = BinaryBase64Field(source="fromMember.user.userinfo.pubkey", read_only=True) - signedEncryptionKey = BinaryBase64Field() - - class Meta: - model = models.CollectionInvitation - fields = ( - "username", - "uid", - "collection", - "signedEncryptionKey", - "accessLevel", - "fromUsername", - "fromPubkey", - "version", - ) - - def validate_user(self, value): - request = self.context["request"] - - if request.user.username == value.lower(): - raise EtebaseValidationError("no_self_invite", "Inviting yourself is not allowed") - return value - - def create(self, validated_data): - request = self.context["request"] - collection = validated_data.pop("collection") - - member = collection.members.get(user=request.user) - - with transaction.atomic(): - try: - return type(self).Meta.model.objects.create(**validated_data, fromMember=member) - except IntegrityError: - raise EtebaseValidationError("invitation_exists", "Invitation already exists") - - def update(self, instance, validated_data): - with transaction.atomic(): - instance.accessLevel = validated_data.pop("accessLevel") - instance.signedEncryptionKey = validated_data.pop("signedEncryptionKey") - instance.save() - - return instance - - -class InvitationAcceptSerializer(BetterErrorsMixin, serializers.Serializer): - collectionType = BinaryBase64Field() - encryptionKey = BinaryBase64Field() - - def create(self, validated_data): - - with transaction.atomic(): - invitation = self.context["invitation"] - encryption_key = validated_data.get("encryptionKey") - collection_type = validated_data.pop("collectionType") - - user = invitation.user - collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) - - member = models.CollectionMember.objects.create( - collection=invitation.collection, - stoken=models.Stoken.objects.create(), - user=user, - accessLevel=invitation.accessLevel, - encryptionKey=encryption_key, - collectionType=collection_type_obj, - ) - - models.CollectionMemberRemoved.objects.filter( - user=invitation.user, collection=invitation.collection - ).delete() - - invitation.delete() - - return member - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class UserSerializer(BetterErrorsMixin, serializers.ModelSerializer): - pubkey = BinaryBase64Field(source="userinfo.pubkey") - encryptedContent = BinaryBase64Field(source="userinfo.encryptedContent") - - class Meta: - model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, "pubkey", "encryptedContent") - - -class UserInfoPubkeySerializer(BetterErrorsMixin, serializers.ModelSerializer): - pubkey = BinaryBase64Field() - - class Meta: - model = models.UserInfo - fields = ("pubkey",) - - -class UserSignupSerializer(BetterErrorsMixin, serializers.ModelSerializer): - class Meta: - model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) - extra_kwargs = { - "username": {"validators": []}, # We specifically validate in SignupSerializer - } - - -class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): - """Used both for creating new accounts and setting up existing ones for the first time. - When setting up existing ones the email is ignored." - """ - - user = UserSignupSerializer(many=False) - salt = BinaryBase64Field() - loginPubkey = BinaryBase64Field() - pubkey = BinaryBase64Field() - encryptedContent = BinaryBase64Field() - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - user_data = validated_data.pop("user") - - with transaction.atomic(): - view = self.context.get("view", None) - try: - user_queryset = get_user_queryset(User.objects.all(), context=CallbackContext(view.kwargs)) - instance = user_queryset.get(**{User.USERNAME_FIELD: user_data["username"].lower()}) - except User.DoesNotExist: - # Create the user and save the casing the user chose as the first name - try: - instance = create_user( - **user_data, - password=None, - first_name=user_data["username"], - context=CallbackContext(view.kwargs) - ) - instance.full_clean() - except EtebaseValidationError as e: - raise e - except django_exceptions.ValidationError as e: - self.transform_validation_error("user", e) - except Exception as e: - raise EtebaseValidationError("generic", str(e)) - - if hasattr(instance, "userinfo"): - raise EtebaseValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) - - models.UserInfo.objects.create(**validated_data, owner=instance) - - return instance - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginChallengeSerializer(BetterErrorsMixin, serializers.Serializer): - username = serializers.CharField(required=True) - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginSerializer(BetterErrorsMixin, serializers.Serializer): - response = BinaryBase64Field() - signature = BinaryBase64Field() - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): - challenge = BinaryBase64Field() - host = serializers.CharField() - action = serializers.CharField() - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationChangePasswordInnerSerializer(AuthenticationLoginInnerSerializer): - loginPubkey = BinaryBase64Field() - encryptedContent = BinaryBase64Field() - - class Meta: - model = models.UserInfo - fields = ("loginPubkey", "encryptedContent") - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - with transaction.atomic(): - instance.loginPubkey = validated_data.pop("loginPubkey") - instance.encryptedContent = validated_data.pop("encryptedContent") - instance.save() - - return instance diff --git a/django_etebase/tests.py b/django_etebase/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/django_etebase/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/django_etebase/token_auth/admin.py b/django_etebase/token_auth/admin.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/token_auth/authentication.py b/django_etebase/token_auth/authentication.py deleted file mode 100644 index 7e84956..0000000 --- a/django_etebase/token_auth/authentication.py +++ /dev/null @@ -1,46 +0,0 @@ -from django.utils import timezone -from django.utils.translation import gettext_lazy as _ - -from rest_framework import exceptions -from rest_framework.authentication import TokenAuthentication as DRFTokenAuthentication - -from .models import AuthToken, get_default_expiry - - -AUTO_REFRESH = True -MIN_REFRESH_INTERVAL = 60 - - -class TokenAuthentication(DRFTokenAuthentication): - keyword = "Token" - model = AuthToken - - def authenticate_credentials(self, key): - msg = _("Invalid token.") - model = self.get_model() - try: - token = model.objects.select_related("user").get(key=key) - except model.DoesNotExist: - raise exceptions.AuthenticationFailed(msg) - - if not token.user.is_active: - raise exceptions.AuthenticationFailed(_("User inactive or deleted.")) - - if token.expiry is not None: - if token.expiry < timezone.now(): - token.delete() - raise exceptions.AuthenticationFailed(msg) - - if AUTO_REFRESH: - self.renew_token(token) - - return (token.user, token) - - def renew_token(self, auth_token): - current_expiry = auth_token.expiry - new_expiry = get_default_expiry() - # Throttle refreshing of token to avoid db writes - delta = (new_expiry - current_expiry).total_seconds() - if delta > MIN_REFRESH_INTERVAL: - auth_token.expiry = new_expiry - auth_token.save(update_fields=("expiry",)) diff --git a/django_etebase/token_auth/models.py b/django_etebase/token_auth/models.py index ac1efff..dd5ae87 100644 --- a/django_etebase/token_auth/models.py +++ b/django_etebase/token_auth/models.py @@ -1,9 +1,9 @@ -from django.contrib.auth import get_user_model from django.db import models from django.utils import timezone from django.utils.crypto import get_random_string +from myauth.models import get_typed_user_model -User = get_user_model() +User = get_typed_user_model() def generate_key(): diff --git a/django_etebase/urls.py b/django_etebase/urls.py deleted file mode 100644 index 01797c1..0000000 --- a/django_etebase/urls.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.conf import settings -from django.conf.urls import include -from django.urls import path - -from rest_framework_nested import routers - -from django_etebase import views - -router = routers.DefaultRouter() -router.register(r"collection", views.CollectionViewSet) -router.register(r"authentication", views.AuthenticationViewSet, basename="authentication") -router.register(r"invitation/incoming", views.InvitationIncomingViewSet, basename="invitation_incoming") -router.register(r"invitation/outgoing", views.InvitationOutgoingViewSet, basename="invitation_outgoing") - -collections_router = routers.NestedSimpleRouter(router, r"collection", lookup="collection") -collections_router.register(r"item", views.CollectionItemViewSet, basename="collection_item") -collections_router.register(r"member", views.CollectionMemberViewSet, basename="collection_member") - -item_router = routers.NestedSimpleRouter(collections_router, r"item", lookup="collection_item") -item_router.register(r"chunk", views.CollectionItemChunkViewSet, basename="collection_items_chunk") - -if settings.DEBUG: - router.register(r"test/authentication", views.TestAuthenticationViewSet, basename="test_authentication") - -app_name = "django_etebase" -urlpatterns = [ - path("v1/", include(router.urls)), - path("v1/", include(collections_router.urls)), - path("v1/", include(item_router.urls)), -] diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 1c8654b..d812ae3 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -1,13 +1,13 @@ import typing as t from dataclasses import dataclass -from django.contrib.auth import get_user_model from django.core.exceptions import PermissionDenied +from myauth.models import UserType, get_typed_user_model from . import app_settings -User = get_user_model() +User = get_typed_user_model() @dataclass @@ -15,6 +15,7 @@ class CallbackContext: """Class for passing extra context to callbacks""" url_kwargs: t.Dict[str, t.Any] + user: t.Optional[UserType] = None def get_user_queryset(queryset, context: CallbackContext): @@ -27,7 +28,7 @@ def get_user_queryset(queryset, context: CallbackContext): def create_user(context: CallbackContext, *args, **kwargs): custom_func = app_settings.CREATE_USER_FUNC if custom_func is not None: - return custom_func(*args, **kwargs) + return custom_func(context, *args, **kwargs) return User.objects.create_user(*args, **kwargs) diff --git a/django_etebase/views.py b/django_etebase/views.py deleted file mode 100644 index 5a03aa4..0000000 --- a/django_etebase/views.py +++ /dev/null @@ -1,861 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import msgpack - -from django.conf import settings -from django.contrib.auth import get_user_model, user_logged_in, user_logged_out -from django.core.exceptions import PermissionDenied -from django.db import transaction, IntegrityError -from django.db.models import Q -from django.http import HttpResponseBadRequest, HttpResponse, Http404 -from django.shortcuts import get_object_or_404 - -from rest_framework import status -from rest_framework import viewsets -from rest_framework.decorators import action as action_decorator -from rest_framework.response import Response -from rest_framework.parsers import JSONParser, FormParser, MultiPartParser -from rest_framework.renderers import BrowsableAPIRenderer -from rest_framework.exceptions import AuthenticationFailed -from rest_framework.permissions import IsAuthenticated - -import nacl.encoding -import nacl.signing -import nacl.secret -import nacl.hash - -from .sendfile import sendfile -from .token_auth.models import AuthToken - -from .drf_msgpack.parsers import MessagePackParser -from .drf_msgpack.renderers import MessagePackRenderer - -from . import app_settings, permissions -from .renderers import JSONRenderer -from .models import ( - Collection, - CollectionItem, - CollectionItemRevision, - CollectionMember, - CollectionMemberRemoved, - CollectionInvitation, - Stoken, - UserInfo, -) -from .serializers import ( - AuthenticationChangePasswordInnerSerializer, - AuthenticationSignupSerializer, - AuthenticationLoginChallengeSerializer, - AuthenticationLoginSerializer, - AuthenticationLoginInnerSerializer, - CollectionSerializer, - CollectionItemSerializer, - CollectionItemBulkGetSerializer, - CollectionItemDepSerializer, - CollectionItemRevisionSerializer, - CollectionItemChunkSerializer, - CollectionListMultiSerializer, - CollectionMemberSerializer, - CollectionInvitationSerializer, - InvitationAcceptSerializer, - UserInfoPubkeySerializer, - UserSerializer, -) -from .utils import get_user_queryset, CallbackContext -from .exceptions import EtebaseValidationError -from .parsers import ChunkUploadParser -from .signals import user_signed_up - -User = get_user_model() - - -def msgpack_encode(content): - return msgpack.packb(content, use_bin_type=True) - - -def msgpack_decode(content): - return msgpack.unpackb(content, raw=False) - - -class BaseViewSet(viewsets.ModelViewSet): - authentication_classes = tuple(app_settings.API_AUTHENTICATORS) - permission_classes = tuple(app_settings.API_PERMISSIONS) - renderer_classes = [JSONRenderer, MessagePackRenderer] + ([BrowsableAPIRenderer] if settings.DEBUG else []) - parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] - stoken_annotation = None - - def get_serializer_class(self): - serializer_class = self.serializer_class - - if self.request.method == "PUT": - serializer_class = getattr(self, "serializer_update_class", serializer_class) - - return serializer_class - - def get_collection_queryset(self, queryset=Collection.objects): - user = self.request.user - return queryset.filter(members__user=user) - - def get_stoken_obj_id(self, request): - return request.GET.get("stoken", None) - - def get_stoken_obj(self, request): - stoken = self.get_stoken_obj_id(request) - - if stoken is not None: - try: - return Stoken.objects.get(uid=stoken) - except Stoken.DoesNotExist: - raise EtebaseValidationError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) - - return None - - def filter_by_stoken(self, request, queryset): - stoken_rev = self.get_stoken_obj(request) - - queryset = queryset.annotate(max_stoken=self.stoken_annotation).order_by("max_stoken") - - if stoken_rev is not None: - queryset = queryset.filter(max_stoken__gt=stoken_rev.id) - - return queryset, stoken_rev - - def get_queryset_stoken(self, queryset): - maxid = -1 - for row in queryset: - rowmaxid = getattr(row, "max_stoken") or -1 - maxid = max(maxid, rowmaxid) - new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) - - return new_stoken or None - - def filter_by_stoken_and_limit(self, request, queryset): - limit = int(request.GET.get("limit", 50)) - - queryset, stoken_rev = self.filter_by_stoken(request, queryset) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - new_stoken_obj = self.get_queryset_stoken(result) or stoken_rev - - return result, new_stoken_obj, done - - # Change how our list works by default - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - serializer = self.get_serializer(queryset, many=True) - - ret = { - "data": serializer.data, - "done": True, # we always return all the items, so it's always done - } - - return Response(ret) - - -class CollectionViewSet(BaseViewSet): - allowed_methods = ["GET", "POST"] - permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdminOrReadOnly,) - queryset = Collection.objects.all() - serializer_class = CollectionSerializer - lookup_field = "uid" - lookup_url_kwarg = "uid" - stoken_annotation = Collection.stoken_annotation - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - return self.get_collection_queryset(queryset) - - def get_serializer_context(self): - context = super().get_serializer_context() - prefetch = self.request.query_params.get("prefetch", "auto") - context.update({"request": self.request, "prefetch": prefetch}) - return context - - def destroy(self, request, uid=None, *args, **kwargs): - # FIXME: implement - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - serializer.save(owner=self.request.user) - - return Response({}, status=status.HTTP_201_CREATED) - - def list(self, request, *args, **kwargs): - queryset = self.get_queryset() - return self.list_common(request, queryset, *args, **kwargs) - - @action_decorator(detail=False, methods=["POST"]) - def list_multi(self, request, *args, **kwargs): - serializer = CollectionListMultiSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - - collection_types = serializer.validated_data["collectionTypes"] - - queryset = self.get_queryset() - # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") - queryset = queryset.filter( - Q(members__collectionType__uid__in=collection_types) | Q(members__collectionType__isnull=True) - ) - - return self.list_common(request, queryset, *args, **kwargs) - - def list_common(self, request, queryset, *args, **kwargs): - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": done, - } - - stoken_obj = self.get_stoken_obj(request) - if stoken_obj is not None: - # FIXME: honour limit? (the limit should be combined for data and this because of stoken) - remed_qs = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) - if not ret["done"]: - # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken - # can point to the most recent collection change rather than most recent removed membership. - remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) - - remed = remed_qs.values_list("collection__uid", flat=True) - if len(remed) > 0: - ret["removedMemberships"] = [{"uid": x} for x in remed] - - return Response(ret) - - -class CollectionItemViewSet(BaseViewSet): - allowed_methods = ["GET", "POST", "PUT"] - permission_classes = BaseViewSet.permission_classes + (permissions.HasWriteAccessOrReadOnly,) - queryset = CollectionItem.objects.all() - serializer_class = CollectionItemSerializer - lookup_field = "uid" - stoken_annotation = CollectionItem.stoken_annotation - - def get_queryset(self): - collection_uid = self.kwargs["collection_uid"] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = type(self).queryset.filter(collection__pk=collection.pk, revisions__current=True) - - return queryset - - def get_serializer_context(self): - context = super().get_serializer_context() - prefetch = self.request.query_params.get("prefetch", "auto") - context.update({"request": self.request, "prefetch": prefetch}) - return context - - def create(self, request, collection_uid=None, *args, **kwargs): - # We create using batch and transaction - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def destroy(self, request, collection_uid=None, uid=None, *args, **kwargs): - # We can't have destroy because we need to get data from the user (in the body) such as hmac. - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, collection_uid=None, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, collection_uid=None, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - - if not self.request.query_params.get("withCollection", False): - queryset = queryset.filter(parent__isnull=True) - - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": done, - } - return Response(ret) - - @action_decorator(detail=True, methods=["GET"]) - def revision(self, request, collection_uid=None, uid=None, *args, **kwargs): - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - item = get_object_or_404(col.items, uid=uid) - - limit = int(request.GET.get("limit", 50)) - iterator = request.GET.get("iterator", None) - - queryset = item.revisions.order_by("-id") - - if iterator is not None: - iterator = get_object_or_404(queryset, uid=iterator) - queryset = queryset.filter(id__lt=iterator.id) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - serializer = CollectionItemRevisionSerializer(result, context=self.get_serializer_context(), many=True) - - iterator = serializer.data[-1]["uid"] if len(result) > 0 else None - - ret = { - "data": serializer.data, - "iterator": iterator, - "done": done, - } - return Response(ret) - - # FIXME: rename to something consistent with what the clients have - maybe list_updates? - @action_decorator(detail=False, methods=["POST"]) - def fetch_updates(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - - serializer = CollectionItemBulkGetSerializer(data=request.data, many=True) - serializer.is_valid(raise_exception=True) - # FIXME: make configurable? - item_limit = 200 - - if len(serializer.validated_data) > item_limit: - content = {"code": "too_many_items", "detail": "Request has too many items. Limit: {}".format(item_limit)} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - queryset, stoken_rev = self.filter_by_stoken(request, queryset) - - uids, etags = zip(*[(item["uid"], item.get("etag")) for item in serializer.validated_data]) - revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) - queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) - - new_stoken_obj = self.get_queryset_stoken(queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - stoken = stoken_rev and getattr(stoken_rev, "uid", None) - new_stoken = new_stoken or stoken - - serializer = self.get_serializer(queryset, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": True, # we always return all the items, so it's always done - } - return Response(ret) - - @action_decorator(detail=False, methods=["POST"]) - def batch(self, request, collection_uid=None, *args, **kwargs): - return self.transaction(request, collection_uid, validate_etag=False) - - @action_decorator(detail=False, methods=["POST"]) - def transaction(self, request, collection_uid=None, validate_etag=True, *args, **kwargs): - stoken = request.GET.get("stoken", None) - with transaction.atomic(): # We need this for locking on the collection object - collection_object = get_object_or_404( - self.get_collection_queryset(Collection.objects).select_for_update(), # Lock writes on the collection - uid=collection_uid, - ) - - if stoken is not None and stoken != collection_object.stoken: - content = {"code": "stale_stoken", "detail": "Stoken is too old"} - return Response(content, status=status.HTTP_409_CONFLICT) - - items = request.data.get("items") - deps = request.data.get("deps", None) - # FIXME: It should just be one serializer - context = self.get_serializer_context() - context.update({"validate_etag": validate_etag}) - serializer = self.get_serializer_class()(data=items, context=context, many=True) - deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) - - ser_valid = serializer.is_valid() - deps_ser_valid = deps is None or deps_serializer.is_valid() - if ser_valid and deps_ser_valid: - items = serializer.save(collection=collection_object) - - ret = {} - return Response(ret, status=status.HTTP_200_OK) - - return Response( - { - "items": serializer.errors, - "deps": deps_serializer.errors if deps is not None else [], - }, - status=status.HTTP_409_CONFLICT, - ) - - -class CollectionItemChunkViewSet(viewsets.ViewSet): - allowed_methods = ["GET", "PUT"] - authentication_classes = BaseViewSet.authentication_classes - permission_classes = BaseViewSet.permission_classes - renderer_classes = BaseViewSet.renderer_classes - parser_classes = (ChunkUploadParser,) - serializer_class = CollectionItemChunkSerializer - lookup_field = "uid" - - def get_serializer_class(self): - return self.serializer_class - - def get_collection_queryset(self, queryset=Collection.objects): - user = self.request.user - return queryset.filter(members__user=user) - - def update(self, request, *args, collection_uid=None, collection_item_uid=None, uid=None, **kwargs): - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) - # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) - - data = { - "uid": uid, - "chunkFile": request.data["file"], - } - - serializer = self.get_serializer_class()(data=data) - serializer.is_valid(raise_exception=True) - try: - serializer.save(collection=col) - except IntegrityError: - return Response( - {"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT - ) - - return Response({}, status=status.HTTP_201_CREATED) - - @action_decorator(detail=True, methods=["GET"]) - def download(self, request, collection_uid=None, collection_item_uid=None, uid=None, *args, **kwargs): - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) - chunk = get_object_or_404(col.chunks, uid=uid) - - filename = chunk.chunkFile.path - return sendfile(request, filename) - - -class CollectionMemberViewSet(BaseViewSet): - allowed_methods = ["GET", "PUT", "DELETE"] - our_base_permission_classes = BaseViewSet.permission_classes - permission_classes = our_base_permission_classes + (permissions.IsCollectionAdmin,) - queryset = CollectionMember.objects.all() - serializer_class = CollectionMemberSerializer - lookup_field = f"user__{User.USERNAME_FIELD}__iexact" - lookup_url_kwarg = "username" - stoken_annotation = CollectionMember.stoken_annotation - - # FIXME: need to make sure that there's always an admin, and maybe also don't let an owner remove adm access - # (if we want to transfer, we need to do that specifically) - - def get_queryset(self, queryset=None): - collection_uid = self.kwargs["collection_uid"] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(collection=collection) - - # We override this method because we expect the stoken to be called iterator - def get_stoken_obj_id(self, request): - return request.GET.get("iterator", None) - - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset().order_by("id") - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "iterator": new_stoken, # Here we call it an iterator, it's only stoken for collection/items - "done": done, - } - - return Response(ret) - - def create(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - # FIXME: block leaving if we are the last admins - should be deleted / assigned in this case depending if there - # are other memebers. - def perform_destroy(self, instance): - instance.revoke() - - @action_decorator(detail=False, methods=["POST"], permission_classes=our_base_permission_classes) - def leave(self, request, collection_uid=None, *args, **kwargs): - collection_uid = self.kwargs["collection_uid"] - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - - member = col.members.get(user=request.user) - self.perform_destroy(member) - - return Response({}) - - -class InvitationBaseViewSet(BaseViewSet): - queryset = CollectionInvitation.objects.all() - serializer_class = CollectionInvitationSerializer - lookup_field = "uid" - lookup_url_kwarg = "invitation_uid" - - def list(self, request, collection_uid=None, *args, **kwargs): - limit = int(request.GET.get("limit", 50)) - iterator = request.GET.get("iterator", None) - - queryset = self.get_queryset().order_by("id") - - if iterator is not None: - iterator = get_object_or_404(queryset, uid=iterator) - queryset = queryset.filter(id__gt=iterator.id) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - serializer = self.get_serializer(result, many=True) - - iterator = serializer.data[-1]["uid"] if len(result) > 0 else None - - ret = { - "data": serializer.data, - "iterator": iterator, - "done": done, - } - - return Response(ret) - - -class InvitationOutgoingViewSet(InvitationBaseViewSet): - allowed_methods = ["GET", "POST", "PUT", "DELETE"] - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(fromMember__user=self.request.user) - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - collection_uid = serializer.validated_data.get("collection", {}).get("uid") - - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - - if request.user == serializer.validated_data.get("user"): - content = {"code": "self_invite", "detail": "Inviting yourself is invalid"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - if not permissions.is_collection_admin(collection, request.user): - raise PermissionDenied( - {"code": "admin_access_required", "detail": "User is not an admin of this collection"} - ) - - serializer.save(collection=collection) - - return Response({}, status=status.HTTP_201_CREATED) - - @action_decorator(detail=False, allowed_methods=["GET"], methods=["GET"]) - def fetch_user_profile(self, request, *args, **kwargs): - username = request.GET.get("username") - kwargs = {User.USERNAME_FIELD: username.lower()} - user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)), **kwargs) - user_info = get_object_or_404(UserInfo.objects.all(), owner=user) - serializer = UserInfoPubkeySerializer(user_info) - return Response(serializer.data) - - -class InvitationIncomingViewSet(InvitationBaseViewSet): - allowed_methods = ["GET", "DELETE"] - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(user=self.request.user) - - @action_decorator(detail=True, allowed_methods=["POST"], methods=["POST"]) - def accept(self, request, invitation_uid=None, *args, **kwargs): - invitation = get_object_or_404(self.get_queryset(), uid=invitation_uid) - context = self.get_serializer_context() - context.update({"invitation": invitation}) - - serializer = InvitationAcceptSerializer(data=request.data, context=context) - serializer.is_valid(raise_exception=True) - serializer.save() - return Response(status=status.HTTP_201_CREATED) - - -class AuthenticationViewSet(viewsets.ViewSet): - allowed_methods = ["POST"] - authentication_classes = BaseViewSet.authentication_classes - renderer_classes = BaseViewSet.renderer_classes - parser_classes = BaseViewSet.parser_classes - - def get_encryption_key(self, salt): - key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) - return nacl.hash.blake2b( - b"", - key=key, - salt=salt[: nacl.hash.BLAKE2B_SALTBYTES], - person=b"etebase-auth", - encoder=nacl.encoding.RawEncoder, - ) - - def get_queryset(self): - return get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) - - def get_serializer_context(self): - return {"request": self.request, "format": self.format_kwarg, "view": self} - - def login_response_data(self, user): - return { - "token": AuthToken.objects.create(user=user).key, - "user": UserSerializer(user).data, - } - - def list(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - @action_decorator(detail=False, methods=["POST"]) - def signup(self, request, *args, **kwargs): - serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) - serializer.is_valid(raise_exception=True) - user = serializer.save() - - user_signed_up.send(sender=user.__class__, request=request, user=user) - - data = self.login_response_data(user) - return Response(data, status=status.HTTP_201_CREATED) - - def get_login_user(self, username): - kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} - try: - user = self.get_queryset().get(**kwargs) - if not hasattr(user, "userinfo"): - raise AuthenticationFailed({"code": "user_not_init", "detail": "User not properly init"}) - return user - except User.DoesNotExist: - raise AuthenticationFailed({"code": "user_not_found", "detail": "User not found"}) - - def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): - from datetime import datetime - - username = validated_data.get("username") - user = self.get_login_user(username) - host = validated_data["host"] - challenge = validated_data["challenge"] - action = validated_data["action"] - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = msgpack_decode(box.decrypt(challenge)) - now = int(datetime.now().timestamp()) - if action != expected_action: - content = {"code": "wrong_action", "detail": 'Expected "{}" but got something else'.format(expected_action)} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: - content = {"code": "challenge_expired", "detail": "Login challange has expired"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif challenge_data["userId"] != user.id: - content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif not settings.DEBUG and host.split(":", 1)[0] != request.get_host().split(":", 1)[0]: - detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) - content = {"code": "wrong_host", "detail": detail} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) - - try: - verify_key.verify(response_raw, signature) - except nacl.exceptions.BadSignatureError: - return Response( - {"code": "login_bad_signature", "detail": "Wrong password for user."}, - status=status.HTTP_401_UNAUTHORIZED, - ) - - return None - - @action_decorator(detail=False, methods=["GET"]) - def is_etebase(self, request, *args, **kwargs): - return Response({}, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"]) - def login_challenge(self, request, *args, **kwargs): - from datetime import datetime - - serializer = AuthenticationLoginChallengeSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - username = serializer.validated_data.get("username") - user = self.get_login_user(username) - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = { - "timestamp": int(datetime.now().timestamp()), - "userId": user.id, - } - challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) - - ret = { - "salt": salt, - "challenge": challenge, - "version": user.userinfo.version, - } - return Response(ret, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"]) - def login(self, request, *args, **kwargs): - outer_serializer = AuthenticationLoginSerializer(data=request.data) - outer_serializer.is_valid(raise_exception=True) - - response_raw = outer_serializer.validated_data["response"] - response = msgpack_decode(response_raw) - signature = outer_serializer.validated_data["signature"] - - context = {"host": request.get_host()} - serializer = AuthenticationLoginInnerSerializer(data=response, context=context) - serializer.is_valid(raise_exception=True) - - bad_login_response = self.validate_login_request( - request, serializer.validated_data, response_raw, signature, "login" - ) - if bad_login_response is not None: - return bad_login_response - - username = serializer.validated_data.get("username") - user = self.get_login_user(username) - - data = self.login_response_data(user) - - user_logged_in.send(sender=user.__class__, request=request, user=user) - - return Response(data, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"], permission_classes=[IsAuthenticated]) - def logout(self, request, *args, **kwargs): - request.auth.delete() - user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) - return Response(status=status.HTTP_204_NO_CONTENT) - - @action_decorator(detail=False, methods=["POST"], permission_classes=BaseViewSet.permission_classes) - def change_password(self, request, *args, **kwargs): - outer_serializer = AuthenticationLoginSerializer(data=request.data) - outer_serializer.is_valid(raise_exception=True) - - response_raw = outer_serializer.validated_data["response"] - response = msgpack_decode(response_raw) - signature = outer_serializer.validated_data["signature"] - - context = {"host": request.get_host()} - serializer = AuthenticationChangePasswordInnerSerializer(request.user.userinfo, data=response, context=context) - serializer.is_valid(raise_exception=True) - - bad_login_response = self.validate_login_request( - request, serializer.validated_data, response_raw, signature, "changePassword" - ) - if bad_login_response is not None: - return bad_login_response - - serializer.save() - - return Response({}, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"], permission_classes=[IsAuthenticated]) - def dashboard_url(self, request, *args, **kwargs): - get_dashboard_url = app_settings.DASHBOARD_URL_FUNC - if get_dashboard_url is None: - raise EtebaseValidationError( - "not_supported", "This server doesn't have a user dashboard.", status_code=status.HTTP_400_BAD_REQUEST - ) - - ret = { - "url": get_dashboard_url(request, *args, **kwargs), - } - return Response(ret) - - -class TestAuthenticationViewSet(viewsets.ViewSet): - allowed_methods = ["POST"] - renderer_classes = BaseViewSet.renderer_classes - parser_classes = BaseViewSet.parser_classes - - def get_serializer_context(self): - return {"request": self.request, "format": self.format_kwarg, "view": self} - - def list(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - @action_decorator(detail=False, methods=["POST"]) - def reset(self, request, *args, **kwargs): - # Only run when in DEBUG mode! It's only used for tests - if not settings.DEBUG: - return HttpResponseBadRequest("Only allowed in debug mode.") - - with transaction.atomic(): - user_queryset = get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) - user = get_object_or_404(user_queryset, username=request.data.get("user").get("username")) - - # Only allow test users for extra safety - if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): - return HttpResponseBadRequest("Endpoint not allowed for user.") - - if hasattr(user, "userinfo"): - user.userinfo.delete() - - serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) - serializer.is_valid(raise_exception=True) - serializer.save() - - # Delete all of the journal data for this user for a clear test env - user.collection_set.all().delete() - user.collectionmember_set.all().delete() - user.incoming_invitations.all().delete() - - # FIXME: also delete chunk files!!! - - return HttpResponse() diff --git a/django_etebase/drf_msgpack/__init__.py b/etebase_fastapi/__init__.py similarity index 100% rename from django_etebase/drf_msgpack/__init__.py rename to etebase_fastapi/__init__.py diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py new file mode 100644 index 0000000..064d2da --- /dev/null +++ b/etebase_fastapi/authentication.py @@ -0,0 +1,267 @@ +import typing as t +from datetime import datetime +from functools import cached_property + +import nacl +import nacl.encoding +import nacl.hash +import nacl.secret +import nacl.signing +from asgiref.sync import sync_to_async +from django.conf import settings +from django.contrib.auth import user_logged_out, user_logged_in +from django.core import exceptions as django_exceptions +from django.db import transaction +from fastapi import APIRouter, Depends, status, Request + +from django_etebase import app_settings, models +from django_etebase.token_auth.models import AuthToken +from django_etebase.models import UserInfo +from django_etebase.signals import user_signed_up +from django_etebase.utils import create_user, get_user_queryset, CallbackContext +from myauth.models import UserType, get_typed_user_model +from .exceptions import AuthenticationFailed, transform_validation_error, HttpError +from .msgpack import MsgpackRoute +from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode +from .dependencies import AuthData, get_auth_data, get_authenticated_user + +User = get_typed_user_model() +authentication_router = APIRouter(route_class=MsgpackRoute) + + +class LoginChallengeIn(BaseModel): + username: str + + +class LoginChallengeOut(BaseModel): + salt: bytes + challenge: bytes + version: int + + +class LoginResponse(BaseModel): + username: str + challenge: bytes + host: str + action: t.Literal["login", "changePassword"] + + +class UserOut(BaseModel): + username: str + email: str + pubkey: bytes + encryptedContent: bytes + + @classmethod + def from_orm(cls: t.Type["UserOut"], obj: UserType) -> "UserOut": + return cls( + username=obj.username, + email=obj.email, + pubkey=bytes(obj.userinfo.pubkey), + encryptedContent=bytes(obj.userinfo.encryptedContent), + ) + + +class LoginOut(BaseModel): + token: str + user: UserOut + + @classmethod + def from_orm(cls: t.Type["LoginOut"], obj: UserType) -> "LoginOut": + token = AuthToken.objects.create(user=obj).key + user = UserOut.from_orm(obj) + return cls(token=token, user=user) + + +class Authentication(BaseModel): + class Config: + keep_untouched = (cached_property,) + + response: bytes + signature: bytes + + +class Login(Authentication): + @cached_property + def response_data(self) -> LoginResponse: + return LoginResponse(**msgpack_decode(self.response)) + + +class ChangePasswordResponse(LoginResponse): + loginPubkey: bytes + encryptedContent: bytes + + +class ChangePassword(Authentication): + @cached_property + def response_data(self) -> ChangePasswordResponse: + return ChangePasswordResponse(**msgpack_decode(self.response)) + + +class UserSignup(BaseModel): + username: str + email: str + + +class SignupIn(BaseModel): + user: UserSignup + salt: bytes + loginPubkey: bytes + pubkey: bytes + encryptedContent: bytes + + +@sync_to_async +def __get_login_user(username: str) -> UserType: + kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} + try: + user = User.objects.get(**kwargs) + if not hasattr(user, "userinfo"): + raise AuthenticationFailed(code="user_not_init", detail="User not properly init") + return user + except User.DoesNotExist: + raise AuthenticationFailed(code="user_not_found", detail="User not found") + + +async def get_login_user(challenge: LoginChallengeIn) -> UserType: + user = await __get_login_user(challenge.username) + return user + + +def get_encryption_key(salt): + key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) + return nacl.hash.blake2b( + b"", + key=key, + salt=salt[: nacl.hash.BLAKE2B_SALTBYTES], + person=b"etebase-auth", + encoder=nacl.encoding.RawEncoder, + ) + + +def save_changed_password(data: ChangePassword, user: UserType): + response_data = data.response_data + user_info: UserInfo = user.userinfo + user_info.loginPubkey = response_data.loginPubkey + user_info.encryptedContent = response_data.encryptedContent + user_info.save() + + +@sync_to_async +def validate_login_request( + validated_data: LoginResponse, + challenge_sent_to_user: Authentication, + user: UserType, + expected_action: str, + host_from_request: str, +): + enc_key = get_encryption_key(bytes(user.userinfo.salt)) + box = nacl.secret.SecretBox(enc_key) + challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) + now = int(datetime.now().timestamp()) + if validated_data.action != expected_action: + raise HttpError("wrong_action", f'Expected "{expected_action}" but got something else') + elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: + raise HttpError("challenge_expired", "Login challenge has expired") + elif challenge_data["userId"] != user.id: + raise HttpError("wrong_user", "This challenge is for the wrong user") + elif not settings.DEBUG and validated_data.host.split(":", 1)[0] != host_from_request: + raise HttpError( + "wrong_host", f'Found wrong host name. Got: "{validated_data.host}" expected: "{host_from_request}"' + ) + verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) + try: + verify_key.verify(challenge_sent_to_user.response, challenge_sent_to_user.signature) + except nacl.exceptions.BadSignatureError: + raise HttpError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) + + +@authentication_router.get("/is_etebase/") +async def is_etebase(): + pass + + +@authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) +def login_challenge(user: UserType = Depends(get_login_user)): + salt = bytes(user.userinfo.salt) + enc_key = get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + challenge_data = { + "timestamp": int(datetime.now().timestamp()), + "userId": user.id, + } + challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) + return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version) + + +@authentication_router.post("/login/", response_model=LoginOut) +async def login(data: Login, request: Request): + user = await get_login_user(LoginChallengeIn(username=data.response_data.username)) + host = request.headers.get("Host") + await validate_login_request(data.response_data, data, user, "login", host) + data = await sync_to_async(LoginOut.from_orm)(user) + await sync_to_async(user_logged_in.send)(sender=user.__class__, request=None, user=user) + return data + + +@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) +def logout(auth_data: AuthData = Depends(get_auth_data)): + auth_data.token.delete() + user_logged_out.send(sender=auth_data.user.__class__, request=None, user=auth_data.user) + + +@authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) +async def change_password(data: ChangePassword, request: Request, user: UserType = Depends(get_authenticated_user)): + host = request.headers.get("Host") + await validate_login_request(data.response_data, data, user, "changePassword", host) + await sync_to_async(save_changed_password)(data, user) + + +@authentication_router.post("/dashboard_url/", responses=permission_responses) +def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_user)): + get_dashboard_url = app_settings.DASHBOARD_URL_FUNC + if get_dashboard_url is None: + raise HttpError("not_supported", "This server doesn't have a user dashboard.") + + ret = { + "url": get_dashboard_url(CallbackContext(request.path_params, user=user)), + } + return ret + + +def signup_save(data: SignupIn, request: Request) -> UserType: + user_data = data.user + with transaction.atomic(): + try: + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) + instance = user_queryset.get(**{User.USERNAME_FIELD: user_data.username.lower()}) + except User.DoesNotExist: + # Create the user and save the casing the user chose as the first name + try: + instance = create_user( + CallbackContext(request.path_params), + **user_data.dict(), + password=None, + first_name=user_data.username, + ) + instance.full_clean() + except HttpError as e: + raise e + except django_exceptions.ValidationError as e: + transform_validation_error("user", e) + except Exception as e: + raise HttpError("generic", str(e)) + + if hasattr(instance, "userinfo"): + raise HttpError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) + + models.UserInfo.objects.create(**data.dict(exclude={"user"}), owner=instance) + return instance + + +@authentication_router.post("/signup/", response_model=LoginOut, status_code=status.HTTP_201_CREATED) +def signup(data: SignupIn, request: Request): + user = signup_save(data, request) + ret = LoginOut.from_orm(user) + user_signed_up.send(sender=user.__class__, request=None, user=user) + return ret diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py new file mode 100644 index 0000000..9e25b38 --- /dev/null +++ b/etebase_fastapi/collection.py @@ -0,0 +1,589 @@ +import typing as t + +from asgiref.sync import sync_to_async +from django.core import exceptions as django_exceptions +from django.core.files.base import ContentFile +from django.db import transaction, IntegrityError +from django.db.models import Q, QuerySet +from fastapi import APIRouter, Depends, status, Request + +from django_etebase import models +from myauth.models import UserType, get_typed_user_model +from .authentication import get_authenticated_user +from .exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError +from .msgpack import MsgpackRoute +from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken +from .utils import ( + get_object_or_404, + Context, + Prefetch, + PrefetchQuery, + is_collection_admin, + BaseModel, + permission_responses, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, +) +from .dependencies import get_collection_queryset, get_item_queryset, get_collection +from .sendfile import sendfile + +User = get_typed_user_model +collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) + + +class ListMulti(BaseModel): + collectionTypes: t.List[bytes] + + +ChunkType = t.Tuple[str, t.Optional[bytes]] + + +class CollectionItemRevisionInOut(BaseModel): + uid: str + meta: bytes + deleted: bool + chunks: t.List[ChunkType] + + class Config: + orm_mode = True + + @classmethod + def from_orm_context( + cls: t.Type["CollectionItemRevisionInOut"], obj: models.CollectionItemRevision, context: Context + ) -> "CollectionItemRevisionInOut": + chunks: t.List[ChunkType] = [] + for chunk_relation in obj.chunks_relation.all(): + chunk_obj = chunk_relation.chunk + if context.prefetch == "auto": + with open(chunk_obj.chunkFile.path, "rb") as f: + chunks.append((chunk_obj.uid, f.read())) + else: + chunks.append((chunk_obj.uid, None)) + return cls(uid=obj.uid, meta=bytes(obj.meta), deleted=obj.deleted, chunks=chunks) + + +class CollectionItemCommon(BaseModel): + uid: str + version: int + encryptionKey: t.Optional[bytes] + content: CollectionItemRevisionInOut + + +class CollectionItemOut(CollectionItemCommon): + class Config: + orm_mode = True + + @classmethod + def from_orm_context( + cls: t.Type["CollectionItemOut"], obj: models.CollectionItem, context: Context + ) -> "CollectionItemOut": + return cls( + uid=obj.uid, + version=obj.version, + encryptionKey=obj.encryptionKey, + content=CollectionItemRevisionInOut.from_orm_context(obj.content, context), + ) + + +class CollectionItemIn(CollectionItemCommon): + etag: t.Optional[str] + + +class CollectionCommon(BaseModel): + # FIXME: remove optional once we finish collection-type-migration + collectionType: t.Optional[bytes] + collectionKey: bytes + + +class CollectionOut(CollectionCommon): + accessLevel: models.AccessLevels + stoken: str + item: CollectionItemOut + + @classmethod + def from_orm_context(cls: t.Type["CollectionOut"], obj: models.Collection, context: Context) -> "CollectionOut": + member: models.CollectionMember = obj.members.get(user=context.user) + collection_type = member.collectionType + ret = cls( + collectionType=collection_type and bytes(collection_type.uid), + collectionKey=bytes(member.encryptionKey), + accessLevel=member.accessLevel, + stoken=obj.stoken, + item=CollectionItemOut.from_orm_context(obj.main_item, context), + ) + return ret + + +class CollectionIn(CollectionCommon): + item: CollectionItemIn + + +class RemovedMembershipOut(BaseModel): + uid: str + + +class CollectionListResponse(BaseModel): + data: t.List[CollectionOut] + stoken: t.Optional[str] + done: bool + + removedMemberships: t.Optional[t.List[RemovedMembershipOut]] + + +class CollectionItemListResponse(BaseModel): + data: t.List[CollectionItemOut] + stoken: t.Optional[str] + done: bool + + +class CollectionItemRevisionListResponse(BaseModel): + data: t.List[CollectionItemRevisionInOut] + iterator: t.Optional[str] + done: bool + + +class CollectionItemBulkGetIn(BaseModel): + uid: str + etag: t.Optional[str] + + +class ItemDepIn(BaseModel): + uid: str + etag: str + + def validate_db(self): + item = models.CollectionItem.objects.get(uid=self.uid) + etag = self.etag + if item.etag != etag: + raise ValidationError( + "wrong_etag", + "Wrong etag. Expected {} got {}".format(item.etag, etag), + status_code=status.HTTP_409_CONFLICT, + field=self.uid, + ) + + +class ItemBatchIn(BaseModel): + items: t.List[CollectionItemIn] + deps: t.Optional[t.List[ItemDepIn]] + + def validate_db(self): + if self.deps is not None: + errors: t.List[HttpError] = [] + for dep in self.deps: + try: + dep.validate_db() + except ValidationError as e: + errors.append(e) + if len(errors) > 0: + raise ValidationError( + code="dep_failed", + detail="Dependencies failed to validate", + errors=errors, + status_code=status.HTTP_409_CONFLICT, + ) + + +@sync_to_async +def collection_list_common( + queryset: QuerySet, + user: UserType, + stoken: t.Optional[str], + limit: int, + prefetch: Prefetch, +) -> CollectionListResponse: + result, new_stoken_obj, done = filter_by_stoken_and_limit( + stoken, limit, queryset, models.Collection.stoken_annotation + ) + new_stoken = new_stoken_obj and new_stoken_obj.uid + context = Context(user, prefetch) + data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] + + ret = CollectionListResponse(data=data, stoken=new_stoken, done=done) + + stoken_obj = get_stoken_obj(stoken) + if stoken_obj is not None: + # FIXME: honour limit? (the limit should be combined for data and this because of stoken) + remed_qs = models.CollectionMemberRemoved.objects.filter(user=user, stoken__id__gt=stoken_obj.id) + if not done and new_stoken_obj is not None: + # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken + # can point to the most recent collection change rather than most recent removed membership. + remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) + + remed = remed_qs.values_list("collection__uid", flat=True) + if len(remed) > 0: + ret.removedMemberships = [RemovedMembershipOut(uid=x) for x in remed] + + return ret + + +# permissions + + +def verify_collection_admin( + collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) +): + if not is_collection_admin(collection, user): + raise PermissionDenied("admin_access_required", "Only collection admins can perform this operation.") + + +def has_write_access( + collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) +): + member = collection.members.get(user=user) + if member.accessLevel == models.AccessLevels.READ_ONLY: + raise PermissionDenied("no_write_access", "You need write access to write to this collection") + + +# paths + + +@collection_router.post( + "/list_multi/", + response_model=CollectionListResponse, + response_model_exclude_unset=True, + dependencies=PERMISSIONS_READ, +) +async def list_multi( + data: ListMulti, + stoken: t.Optional[str] = None, + limit: int = 50, + queryset: QuerySet = Depends(get_collection_queryset), + user: UserType = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, +): + # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") + queryset = queryset.filter( + Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) + ) + + return await collection_list_common(queryset, user, stoken, limit, prefetch) + + +@collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ) +async def collection_list( + stoken: t.Optional[str] = None, + limit: int = 50, + prefetch: Prefetch = PrefetchQuery, + user: UserType = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_collection_queryset), +): + return await collection_list_common(queryset, user, stoken, limit, prefetch) + + +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): + chunks_objs = [] + + revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) + revision.validate_unique() # Verify there aren't any validation issues + + for chunk in revision_data.chunks: + uid = chunk[0] + chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() + content = chunk[1] if len(chunk) > 1 else None + # If the chunk already exists we assume it's fine. Otherwise, we upload it. + if chunk_obj is None: + if content is not None: + chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) + chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) + chunk_obj.save() + else: + raise ValidationError("chunk_no_content", "Tried to create a new chunk without content") + + chunks_objs.append(chunk_obj) + + stoken = models.Stoken.objects.create() + revision.stoken = stoken + revision.save() + + for chunk in chunks_objs: + models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) + return revision + + +def _create(data: CollectionIn, user: UserType): + with transaction.atomic(): + if data.item.etag is not None: + raise ValidationError("bad_etag", "etag is not null") + instance = models.Collection(uid=data.item.uid, owner=user) + try: + instance.validate_unique() + except django_exceptions.ValidationError: + raise ValidationError( + "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT + ) + instance.save() + + main_item = models.CollectionItem.objects.create( + uid=data.item.uid, version=data.item.version, collection=instance + ) + + instance.main_item = main_item + instance.save() + + # TODO + process_revisions_for_item(main_item, data.item.content) + + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user) + + models.CollectionMember( + collection=instance, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=models.AccessLevels.ADMIN, + encryptionKey=data.collectionKey, + collectionType=collection_type_obj, + ).save() + + +@collection_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) +async def create(data: CollectionIn, user: UserType = Depends(get_authenticated_user)): + await sync_to_async(_create)(data, user) + + +@collection_router.get("/{collection_uid}/", response_model=CollectionOut, dependencies=PERMISSIONS_READ) +def collection_get( + obj: models.Collection = Depends(get_collection), + user: UserType = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, +): + return CollectionOut.from_orm_context(obj, Context(user, prefetch)) + + +def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool): + """Function that's called when this serializer creates an item""" + etag = item_model.etag + revision_data = item_model.content + uid = item_model.uid + + Model = models.CollectionItem + + with transaction.atomic(): + instance, created = Model.objects.get_or_create( + uid=uid, collection=collection, defaults=item_model.dict(exclude={"uid", "etag", "content"}) + ) + cur_etag = instance.etag if not created else None + + # If we are trying to update an up to date item, abort early and consider it a success + if cur_etag == revision_data.uid: + return instance + + if validate_etag and cur_etag != etag: + raise ValidationError( + "wrong_etag", + "Wrong etag. Expected {} got {}".format(cur_etag, etag), + status_code=status.HTTP_409_CONFLICT, + field=uid, + ) + + if not created: + # We don't have to use select_for_update here because the unique constraint on current guards against + # the race condition. But it's a good idea because it'll lock and wait rather than fail. + current_revision = instance.revisions.filter(current=True).select_for_update().first() + current_revision.current = None + current_revision.save() + + try: + process_revisions_for_item(instance, revision_data) + except django_exceptions.ValidationError as e: + transform_validation_error("content", e) + + return instance + + +@item_router.get("/item/{item_uid}/", response_model=CollectionItemOut, dependencies=PERMISSIONS_READ) +def item_get( + item_uid: str, + queryset: QuerySet = Depends(get_item_queryset), + user: UserType = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, +): + obj = queryset.get(uid=item_uid) + return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) + + +@sync_to_async +def item_list_common( + queryset: QuerySet, + user: UserType, + stoken: t.Optional[str], + limit: int, + prefetch: Prefetch, +) -> CollectionItemListResponse: + result, new_stoken_obj, done = filter_by_stoken_and_limit( + stoken, limit, queryset, models.CollectionItem.stoken_annotation + ) + new_stoken = new_stoken_obj and new_stoken_obj.uid + context = Context(user, prefetch) + data: t.List[CollectionItemOut] = [CollectionItemOut.from_orm_context(item, context) for item in result] + return CollectionItemListResponse(data=data, stoken=new_stoken, done=done) + + +@item_router.get("/item/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) +async def item_list( + queryset: QuerySet = Depends(get_item_queryset), + stoken: t.Optional[str] = None, + limit: int = 50, + prefetch: Prefetch = PrefetchQuery, + withCollection: bool = False, + user: UserType = Depends(get_authenticated_user), +): + if not withCollection: + queryset = queryset.filter(parent__isnull=True) + + response = await item_list_common(queryset, user, stoken, limit, prefetch) + return response + + +def item_bulk_common(data: ItemBatchIn, user: UserType, stoken: t.Optional[str], uid: str, validate_etag: bool): + queryset = get_collection_queryset(user) + with transaction.atomic(): # We need this for locking the collection object + collection_object = queryset.select_for_update().get(uid=uid) + + if stoken is not None and stoken != collection_object.stoken: + raise HttpError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) + + data.validate_db() + + errors: t.List[HttpError] = [] + for item in data.items: + try: + item_create(item, collection_object, validate_etag) + except ValidationError as e: + errors.append(e) + + if len(errors) > 0: + raise ValidationError( + code="item_failed", + detail="Items failed to validate", + errors=errors, + status_code=status.HTTP_409_CONFLICT, + ) + + +@item_router.get( + "/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse, dependencies=PERMISSIONS_READ +) +def item_revisions( + item_uid: str, + limit: int = 50, + iterator: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: UserType = Depends(get_authenticated_user), + items: QuerySet = Depends(get_item_queryset), +): + item = get_object_or_404(items, uid=item_uid) + + queryset = item.revisions.order_by("-id") + + if iterator is not None: + iterator_obj = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__lt=iterator_obj.id) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + context = Context(user, prefetch) + ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result] + iterator = ret_data[-1].uid if len(result) > 0 else None + + return CollectionItemRevisionListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) + + +@item_router.post("/item/fetch_updates/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) +def fetch_updates( + data: t.List[CollectionItemBulkGetIn], + stoken: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: UserType = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_item_queryset), +): + # FIXME: make configurable? + item_limit = 200 + + if len(data) > item_limit: + raise HttpError("too_many_items", "Request has too many items.", status_code=status.HTTP_400_BAD_REQUEST) + + queryset, stoken_rev = filter_by_stoken(stoken, queryset, models.CollectionItem.stoken_annotation) + + uids, etags = zip(*[(item.uid, item.etag) for item in data]) + revs = models.CollectionItemRevision.objects.filter(uid__in=etags, current=True) + queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) + + new_stoken_obj = get_queryset_stoken(queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + stoken = stoken_rev and getattr(stoken_rev, "uid", None) + new_stoken = new_stoken or stoken + + context = Context(user, prefetch) + return CollectionItemListResponse( + data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], + stoken=new_stoken, + done=True, # we always return all the items, so it's always done + ) + + +@item_router.post("/item/transaction/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) +def item_transaction( + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) +): + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) + + +@item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) +def item_batch( + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) +): + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) + + +# Chunks + + +@sync_to_async +def chunk_save(chunk_uid: str, collection: models.Collection, content_file: ContentFile): + chunk_obj = models.CollectionItemChunk(uid=chunk_uid, collection=collection) + chunk_obj.chunkFile.save("IGNORED", content_file) + chunk_obj.save() + return chunk_obj + + +@item_router.put( + "/item/{item_uid}/chunk/{chunk_uid}/", + dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE], + status_code=status.HTTP_201_CREATED, +) +async def chunk_update( + request: Request, + chunk_uid: str, + collection: models.Collection = Depends(get_collection), +): + # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) + content_file = ContentFile(await request.body()) + try: + await chunk_save(chunk_uid, collection, content_file) + except IntegrityError: + raise HttpError("chunk_exists", "Chunk already exists.", status_code=status.HTTP_409_CONFLICT) + + +@item_router.get( + "/item/{item_uid}/chunk/{chunk_uid}/download/", + dependencies=PERMISSIONS_READ, +) +def chunk_download( + chunk_uid: str, + collection: models.Collection = Depends(get_collection), +): + chunk = get_object_or_404(collection.chunks, uid=chunk_uid) + + filename = chunk.chunkFile.path + return sendfile(filename) diff --git a/etebase_fastapi/dependencies.py b/etebase_fastapi/dependencies.py new file mode 100644 index 0000000..fb9cec5 --- /dev/null +++ b/etebase_fastapi/dependencies.py @@ -0,0 +1,82 @@ +import dataclasses + +from fastapi import Depends +from fastapi.security import APIKeyHeader + +from django.utils import timezone +from django.db.models import QuerySet + +from django_etebase import models +from django_etebase.token_auth.models import AuthToken, get_default_expiry +from myauth.models import UserType, get_typed_user_model +from .exceptions import AuthenticationFailed +from .utils import get_object_or_404 + + +User = get_typed_user_model() +token_scheme = APIKeyHeader(name="Authorization") +AUTO_REFRESH = True +MIN_REFRESH_INTERVAL = 60 + + +@dataclasses.dataclass(frozen=True) +class AuthData: + user: UserType + token: AuthToken + + +def __renew_token(auth_token: AuthToken): + current_expiry = auth_token.expiry + new_expiry = get_default_expiry() + # Throttle refreshing of token to avoid db writes + delta = (new_expiry - current_expiry).total_seconds() + if delta > MIN_REFRESH_INTERVAL: + auth_token.expiry = new_expiry + auth_token.save(update_fields=("expiry",)) + + +def __get_authenticated_user(api_token: str): + api_token = api_token.split()[1] + try: + token: AuthToken = AuthToken.objects.select_related("user").get(key=api_token) + except AuthToken.DoesNotExist: + raise AuthenticationFailed(detail="Invalid token.") + if not token.user.is_active: + raise AuthenticationFailed(detail="User inactive or deleted.") + + if token.expiry is not None: + if token.expiry < timezone.now(): + token.delete() + raise AuthenticationFailed(detail="Invalid token.") + + if AUTO_REFRESH: + __renew_token(token) + + return token.user, token + + +def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: + user, token = __get_authenticated_user(api_token) + return AuthData(user, token) + + +def get_authenticated_user(api_token: str = Depends(token_scheme)) -> UserType: + user, _ = __get_authenticated_user(api_token) + return user + + +def get_collection_queryset(user: UserType = Depends(get_authenticated_user)) -> QuerySet: + default_queryset: QuerySet = models.Collection.objects.all() + return default_queryset.filter(members__user=user) + + +def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: + return get_object_or_404(queryset, uid=collection_uid) + + +def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: + default_item_queryset: QuerySet = models.CollectionItem.objects.all() + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') + queryset = default_item_queryset.filter(collection__pk=collection.pk, revisions__current=True) + + return queryset diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py new file mode 100644 index 0000000..72a3faf --- /dev/null +++ b/etebase_fastapi/exceptions.py @@ -0,0 +1,118 @@ +from fastapi import status +import typing as t + +from pydantic import BaseModel + + +class HttpErrorField(BaseModel): + field: str + code: str + detail: str + + class Config: + orm_mode = True + + +class HttpErrorOut(BaseModel): + code: str + detail: str + errors: t.Optional[t.List[HttpErrorField]] + + class Config: + orm_mode = True + + +class CustomHttpException(Exception): + def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): + self.status_code = status_code + self.code = code + self.detail = detail + + @property + def as_dict(self) -> dict: + return {"code": self.code, "detail": self.detail} + + +class AuthenticationFailed(CustomHttpException): + def __init__( + self, + code="authentication_failed", + detail: str = "Incorrect authentication credentials.", + status_code: int = status.HTTP_401_UNAUTHORIZED, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + +class NotAuthenticated(CustomHttpException): + def __init__( + self, + code="not_authenticated", + detail: str = "Authentication credentials were not provided.", + status_code: int = status.HTTP_401_UNAUTHORIZED, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + +class PermissionDenied(CustomHttpException): + def __init__( + self, + code="permission_denied", + detail: str = "You do not have permission to perform this action.", + status_code: int = status.HTTP_403_FORBIDDEN, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + +class HttpError(CustomHttpException): + def __init__( + self, + code: str, + detail: str, + status_code: int = status.HTTP_400_BAD_REQUEST, + errors: t.Optional[t.List["HttpError"]] = None, + ): + self.errors = errors + super().__init__(code=code, detail=detail, status_code=status_code) + + @property + def as_dict(self) -> dict: + return HttpErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() + + +class ValidationError(HttpError): + def __init__( + self, + code: str, + detail: str, + status_code: int = status.HTTP_400_BAD_REQUEST, + errors: t.Optional[t.List["HttpError"]] = None, + field: t.Optional[str] = None, + ): + self.field = field + super().__init__(code=code, detail=detail, errors=errors, status_code=status_code) + + +def flatten_errors(field_name, errors) -> t.List[HttpError]: + ret = [] + if isinstance(errors, dict): + for error_key in errors: + error = errors[error_key] + ret.extend(flatten_errors("{}.{}".format(field_name, error_key), error)) + else: + for error in errors: + if error.messages: + message = error.messages[0] + else: + message = str(error) + ret.append(dict(code=error.code, detail=message, field=field_name)) + return ret + + +def transform_validation_error(prefix, err): + if hasattr(err, "error_dict"): + errors = flatten_errors(prefix, err.error_dict) + elif not hasattr(err, "message"): + errors = flatten_errors(prefix, err.error_list) + else: + raise HttpError(err.code, err.message) + raise HttpError(code="field_errors", detail="Field validations failed.", errors=errors) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py new file mode 100644 index 0000000..eb9f549 --- /dev/null +++ b/etebase_fastapi/invitation.py @@ -0,0 +1,240 @@ +import typing as t + +from django.db import transaction, IntegrityError +from django.db.models import QuerySet +from fastapi import APIRouter, Depends, status, Request + +from django_etebase import models +from django_etebase.utils import get_user_queryset, CallbackContext +from myauth.models import UserType, get_typed_user_model +from .authentication import get_authenticated_user +from .exceptions import HttpError, PermissionDenied +from .msgpack import MsgpackRoute +from .utils import ( + get_object_or_404, + Context, + is_collection_admin, + BaseModel, + permission_responses, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, +) + +User = get_typed_user_model() +invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +default_queryset: QuerySet = models.CollectionInvitation.objects.all() + + +class UserInfoOut(BaseModel): + pubkey: bytes + + class Config: + orm_mode = True + + @classmethod + def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut": + return cls(pubkey=bytes(obj.pubkey)) + + +class CollectionInvitationAcceptIn(BaseModel): + collectionType: bytes + encryptionKey: bytes + + +class CollectionInvitationCommon(BaseModel): + uid: str + version: int + accessLevel: models.AccessLevels + username: str + collection: str + signedEncryptionKey: bytes + + +class CollectionInvitationIn(CollectionInvitationCommon): + def validate_db(self, context: Context): + user = context.user + if user is not None and (user.username == self.username.lower()): + raise HttpError("no_self_invite", "Inviting yourself is not allowed") + + +class CollectionInvitationOut(CollectionInvitationCommon): + fromUsername: str + fromPubkey: bytes + + class Config: + orm_mode = True + + @classmethod + def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut": + return cls( + uid=obj.uid, + version=obj.version, + accessLevel=obj.accessLevel, + username=obj.user.username, + collection=obj.collection.uid, + fromUsername=obj.fromMember.user.username, + fromPubkey=bytes(obj.fromMember.user.userinfo.pubkey), + signedEncryptionKey=bytes(obj.signedEncryptionKey), + ) + + +class InvitationListResponse(BaseModel): + data: t.List[CollectionInvitationOut] + iterator: t.Optional[str] + done: bool + + +def get_incoming_queryset(user: UserType = Depends(get_authenticated_user)): + return default_queryset.filter(user=user) + + +def get_outgoing_queryset(user: UserType = Depends(get_authenticated_user)): + return default_queryset.filter(fromMember__user=user) + + +def list_common( + queryset: QuerySet, + iterator: t.Optional[str], + limit: int, +) -> InvitationListResponse: + queryset = queryset.order_by("id") + + if iterator is not None: + iterator_obj = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__gt=iterator_obj.id) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + ret_data = result + iterator = ret_data[-1].uid if len(result) > 0 else None + + return InvitationListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) + + +@invitation_incoming_router.get("/", response_model=InvitationListResponse, dependencies=PERMISSIONS_READ) +def incoming_list( + iterator: t.Optional[str] = None, + limit: int = 50, + queryset: QuerySet = Depends(get_incoming_queryset), +): + return list_common(queryset, iterator, limit) + + +@invitation_incoming_router.get( + "/{invitation_uid}/", response_model=CollectionInvitationOut, dependencies=PERMISSIONS_READ +) +def incoming_get( + invitation_uid: str, + queryset: QuerySet = Depends(get_incoming_queryset), +): + obj = get_object_or_404(queryset, uid=invitation_uid) + return CollectionInvitationOut.from_orm(obj) + + +@invitation_incoming_router.delete( + "/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READWRITE +) +def incoming_delete( + invitation_uid: str, + queryset: QuerySet = Depends(get_incoming_queryset), +): + obj = get_object_or_404(queryset, uid=invitation_uid) + obj.delete() + + +@invitation_incoming_router.post( + "/{invitation_uid}/accept/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE +) +def incoming_accept( + invitation_uid: str, + data: CollectionInvitationAcceptIn, + queryset: QuerySet = Depends(get_incoming_queryset), +): + invitation = get_object_or_404(queryset, uid=invitation_uid) + + with transaction.atomic(): + user = invitation.user + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user) + + models.CollectionMember.objects.create( + collection=invitation.collection, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=invitation.accessLevel, + encryptionKey=data.encryptionKey, + collectionType=collection_type_obj, + ) + + models.CollectionMemberRemoved.objects.filter(user=invitation.user, collection=invitation.collection).delete() + + invitation.delete() + + +@invitation_outgoing_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) +def outgoing_create( + data: CollectionInvitationIn, + request: Request, + user: UserType = Depends(get_authenticated_user), +): + collection = get_object_or_404(models.Collection.objects, uid=data.collection) + to_user = get_object_or_404( + get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), username=data.username + ) + + context = Context(user, None) + data.validate_db(context) + + if not is_collection_admin(collection, user): + raise PermissionDenied("admin_access_required", "User is not an admin of this collection") + + member = collection.members.get(user=user) + + with transaction.atomic(): + try: + ret = models.CollectionInvitation.objects.create( + **data.dict(exclude={"collection", "username"}), user=to_user, fromMember=member + ) + except IntegrityError: + raise HttpError("invitation_exists", "Invitation already exists") + + +@invitation_outgoing_router.get("/", response_model=InvitationListResponse, dependencies=PERMISSIONS_READ) +def outgoing_list( + iterator: t.Optional[str] = None, + limit: int = 50, + queryset: QuerySet = Depends(get_outgoing_queryset), +): + return list_common(queryset, iterator, limit) + + +@invitation_outgoing_router.delete( + "/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READWRITE +) +def outgoing_delete( + invitation_uid: str, + queryset: QuerySet = Depends(get_outgoing_queryset), +): + obj = get_object_or_404(queryset, uid=invitation_uid) + obj.delete() + + +@invitation_outgoing_router.get("/fetch_user_profile/", response_model=UserInfoOut, dependencies=PERMISSIONS_READ) +def outgoing_fetch_user_profile( + username: str, + request: Request, + user: UserType = Depends(get_authenticated_user), +): + kwargs = {User.USERNAME_FIELD: username.lower()} + user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) + user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) + return UserInfoOut.from_orm(user_info) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py new file mode 100644 index 0000000..534798e --- /dev/null +++ b/etebase_fastapi/main.py @@ -0,0 +1,61 @@ +from django.conf import settings + +# Not at the top of the file because we first need to setup django +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.trustedhost import TrustedHostMiddleware + +from .exceptions import CustomHttpException +from .authentication import authentication_router +from .collection import collection_router, item_router +from .member import member_router +from .invitation import invitation_incoming_router, invitation_outgoing_router +from .msgpack import MsgpackResponse + + +def create_application(prefix="", middlewares=[]): + app = FastAPI( + title="Etebase", + description="The Etebase server API documentation", + externalDocs={ + "url": "https://docs.etebase.com", + "description": "Docs about the API specifications and clients.", + } + # FIXME: version="2.5.0", + ) + VERSION = "v1" + BASE_PATH = f"{prefix}/api/{VERSION}" + COLLECTION_UID_MARKER = "{collection_uid}" + app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) + app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) + app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) + app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) + app.include_router( + invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"] + ) + app.include_router( + invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"] + ) + + if settings.DEBUG: + from etebase_fastapi.test_reset_view import test_reset_view_router + + app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") + + app.add_middleware( + CORSMiddleware, + allow_origin_regex="https?://.*", + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) + + for middleware in middlewares: + app.add_middleware(middleware) + + @app.exception_handler(CustomHttpException) + async def custom_exception_handler(request: Request, exc: CustomHttpException): + return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + + return app diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py new file mode 100644 index 0000000..22977ac --- /dev/null +++ b/etebase_fastapi/member.py @@ -0,0 +1,105 @@ +import typing as t + +from django.db import transaction +from django.db.models import QuerySet +from fastapi import APIRouter, Depends, status + +from django_etebase import models +from myauth.models import UserType, get_typed_user_model +from .authentication import get_authenticated_user +from .msgpack import MsgpackRoute +from .utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE +from .stoken_handler import filter_by_stoken_and_limit + +from .collection import get_collection, verify_collection_admin + +User = get_typed_user_model() +member_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +default_queryset: QuerySet = models.CollectionMember.objects.all() + + +def get_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: + return default_queryset.filter(collection=collection) + + +def get_member(username: str, queryset: QuerySet = Depends(get_queryset)) -> QuerySet: + return get_object_or_404(queryset, user__username__iexact=username) + + +class CollectionMemberModifyAccessLevelIn(BaseModel): + accessLevel: models.AccessLevels + + +class CollectionMemberOut(BaseModel): + username: str + accessLevel: models.AccessLevels + + class Config: + orm_mode = True + + @classmethod + def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut": + return cls(username=obj.user.username, accessLevel=obj.accessLevel) + + +class MemberListResponse(BaseModel): + data: t.List[CollectionMemberOut] + iterator: t.Optional[str] + done: bool + + +@member_router.get( + "/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READ] +) +def member_list( + iterator: t.Optional[str] = None, + limit: int = 50, + queryset: QuerySet = Depends(get_queryset), +): + queryset = queryset.order_by("id") + result, new_stoken_obj, done = filter_by_stoken_and_limit( + iterator, limit, queryset, models.CollectionMember.stoken_annotation + ) + new_stoken = new_stoken_obj and new_stoken_obj.uid + + return MemberListResponse( + data=[CollectionMemberOut.from_orm(item) for item in result], + iterator=new_stoken, + done=done, + ) + + +@member_router.delete( + "/member/{username}/", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READWRITE], +) +def member_delete( + obj: models.CollectionMember = Depends(get_member), +): + obj.revoke() + + +@member_router.patch( + "/member/{username}/", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READWRITE], +) +def member_patch( + data: CollectionMemberModifyAccessLevelIn, + instance: models.CollectionMember = Depends(get_member), +): + with transaction.atomic(): + # We only allow updating accessLevel + if instance.accessLevel != data.accessLevel: + instance.stoken = models.Stoken.objects.create() + instance.accessLevel = data.accessLevel + instance.save() + + +@member_router.post("/member/leave/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READ) +def member_leave( + user: UserType = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection) +): + obj = get_object_or_404(collection.members, user=user) + obj.revoke() diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py new file mode 100644 index 0000000..915e783 --- /dev/null +++ b/etebase_fastapi/msgpack.py @@ -0,0 +1,71 @@ +import typing as t +import msgpack +from fastapi.routing import APIRoute, get_request_handler +from pydantic import BaseModel +from starlette.requests import Request +from starlette.responses import Response + + +class MsgpackRequest(Request): + media_type = "application/msgpack" + + async def json(self) -> bytes: + if not hasattr(self, "_json"): + body = await super().body() + self._json = msgpack.unpackb(body, raw=False) + return self._json + + +class MsgpackResponse(Response): + media_type = "application/msgpack" + + def render(self, content: t.Optional[t.Any]) -> bytes: + if content is None: + return b"" + + if isinstance(content, BaseModel): + content = content.dict() + ret = msgpack.packb(content, use_bin_type=True) + assert ret is not None + return ret + + +class MsgpackRoute(APIRoute): + # keep track of content-type -> request classes + REQUESTS_CLASSES = {MsgpackRequest.media_type: MsgpackRequest} + # keep track of content-type -> response classes + ROUTES_HANDLERS_CLASSES = {MsgpackResponse.media_type: MsgpackResponse} + + def _get_media_type_route_handler(self, media_type): + return get_request_handler( + dependant=self.dependant, + body_field=self.body_field, + status_code=self.status_code, + # use custom response class or fallback on default self.response_class + response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class), + response_field=self.secure_cloned_response_field, + response_model_include=self.response_model_include, + response_model_exclude=self.response_model_exclude, + response_model_by_alias=self.response_model_by_alias, + response_model_exclude_unset=self.response_model_exclude_unset, + response_model_exclude_defaults=self.response_model_exclude_defaults, + response_model_exclude_none=self.response_model_exclude_none, + dependency_overrides_provider=self.dependency_overrides_provider, + ) + + def get_route_handler(self) -> t.Callable: + async def custom_route_handler(request: Request) -> Response: + + content_type = request.headers.get("Content-Type") + try: + request_cls = self.REQUESTS_CLASSES[content_type] + request = request_cls(request.scope, request.receive) + except KeyError: + # nothing registered to handle content_type, process given requests as-is + pass + + accept = request.headers.get("Accept") + route_handler = self._get_media_type_route_handler(accept) + return await route_handler(request) + + return custom_route_handler diff --git a/django_etebase/sendfile/LICENSE b/etebase_fastapi/sendfile/LICENSE similarity index 100% rename from django_etebase/sendfile/LICENSE rename to etebase_fastapi/sendfile/LICENSE diff --git a/django_etebase/sendfile/README.md b/etebase_fastapi/sendfile/README.md similarity index 100% rename from django_etebase/sendfile/README.md rename to etebase_fastapi/sendfile/README.md diff --git a/django_etebase/sendfile/__init__.py b/etebase_fastapi/sendfile/__init__.py similarity index 100% rename from django_etebase/sendfile/__init__.py rename to etebase_fastapi/sendfile/__init__.py diff --git a/django_etebase/drf_msgpack/migrations/__init__.py b/etebase_fastapi/sendfile/backends/__init__.py similarity index 100% rename from django_etebase/drf_msgpack/migrations/__init__.py rename to etebase_fastapi/sendfile/backends/__init__.py diff --git a/etebase_fastapi/sendfile/backends/mod_wsgi.py b/etebase_fastapi/sendfile/backends/mod_wsgi.py new file mode 100644 index 0000000..b8fc6c0 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/mod_wsgi.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from fastapi import Response + +from ..utils import _convert_file_to_url + + +def sendfile(filename, **kwargs): + return Response(headers={"Location": _convert_file_to_url(filename)}) diff --git a/etebase_fastapi/sendfile/backends/nginx.py b/etebase_fastapi/sendfile/backends/nginx.py new file mode 100644 index 0000000..b22e0d0 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/nginx.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from fastapi import Response + +from ..utils import _convert_file_to_url + + +def sendfile(filename, **kwargs): + return Response(headers={"X-Accel-Redirect": _convert_file_to_url(filename)}) diff --git a/etebase_fastapi/sendfile/backends/simple.py b/etebase_fastapi/sendfile/backends/simple.py new file mode 100644 index 0000000..f3a3548 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/simple.py @@ -0,0 +1,12 @@ +from fastapi.responses import FileResponse + + +def sendfile(filename, mimetype, **kwargs): + """Use the SENDFILE_ROOT value composed with the path arrived as argument + to build an absolute path with which resolve and return the file contents. + + If the path points to a file out of the root directory (should cover both + situations with '..' and symlinks) then a 404 is raised. + """ + + return FileResponse(filename, media_type=mimetype) diff --git a/etebase_fastapi/sendfile/backends/xsendfile.py b/etebase_fastapi/sendfile/backends/xsendfile.py new file mode 100644 index 0000000..530f6a1 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/xsendfile.py @@ -0,0 +1,6 @@ +from fastapi import Response + + +def sendfile(filename, **kwargs): + filename = str(filename) + return Response(headers={"X-Sendfile": filename}) diff --git a/django_etebase/sendfile/utils.py b/etebase_fastapi/sendfile/utils.py similarity index 81% rename from django_etebase/sendfile/utils.py rename to etebase_fastapi/sendfile/utils.py index 97c06d7..7c8b1f2 100644 --- a/django_etebase/sendfile/utils.py +++ b/etebase_fastapi/sendfile/utils.py @@ -4,9 +4,11 @@ from pathlib import Path, PurePath from urllib.parse import quote import logging +from fastapi import status +from ..exceptions import HttpError + from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.http import Http404 logger = logging.getLogger(__name__) @@ -54,12 +56,12 @@ def _sanitize_path(filepath): try: filepath_abs.relative_to(path_root) except ValueError: - raise Http404("{} wrt {} is impossible".format(filepath_abs, path_root)) + raise HttpError("generic", "{} wrt {} is impossible".format(filepath_abs, path_root), status_code=status.HTTP_404_NOT_FOUND) return filepath_abs -def sendfile(request, filename, mimetype="application/octet-stream", encoding=None): +def sendfile(filename, mimetype="application/octet-stream", encoding=None): """ Create a response to send file using backend configured in ``SENDFILE_BACKEND`` @@ -75,11 +77,10 @@ def sendfile(request, filename, mimetype="application/octet-stream", encoding=No _sendfile = _get_sendfile() if not filepath_obj.exists(): - raise Http404('"%s" does not exist' % filepath_obj) + raise HttpError("does_not_exist", '"%s" does not exist' % filepath_obj, status_code=status.HTTP_404_NOT_FOUND) - response = _sendfile(request, filepath_obj, mimetype=mimetype) + response = _sendfile(filepath_obj, mimetype=mimetype) - response["Content-length"] = filepath_obj.stat().st_size - response["Content-Type"] = mimetype + response.headers["Content-Type"] = mimetype return response diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py new file mode 100644 index 0000000..76d348a --- /dev/null +++ b/etebase_fastapi/stoken_handler.py @@ -0,0 +1,62 @@ +import typing as t + +from django.db.models import QuerySet +from fastapi import status + +from django_etebase.models import Stoken + +from .exceptions import HttpError + +# TODO missing stoken_annotation type +StokenAnnotation = t.Any + + +def get_stoken_obj(stoken: t.Optional[str]) -> t.Optional[Stoken]: + if stoken is not None: + try: + return Stoken.objects.get(uid=stoken) + except Stoken.DoesNotExist: + raise HttpError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) + + return None + + +def filter_by_stoken( + stoken: t.Optional[str], queryset: QuerySet, stoken_annotation: StokenAnnotation +) -> t.Tuple[QuerySet, t.Optional[Stoken]]: + stoken_rev = get_stoken_obj(stoken) + + queryset = queryset.annotate(max_stoken=stoken_annotation).order_by("max_stoken") + + if stoken_rev is not None: + queryset = queryset.filter(max_stoken__gt=stoken_rev.id) + + return queryset, stoken_rev + + +def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]: + maxid = -1 + for row in queryset: + rowmaxid = getattr(row, "max_stoken") or -1 + maxid = max(maxid, rowmaxid) + new_stoken = Stoken.objects.get(id=maxid) if (maxid >= 0) else None + + return new_stoken or None + + +def filter_by_stoken_and_limit( + stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation +) -> t.Tuple[list, t.Optional[Stoken], bool]: + + queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + new_stoken_obj = get_queryset_stoken(result) or stoken_rev + + return result, new_stoken_obj, done diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py new file mode 100644 index 0000000..e328875 --- /dev/null +++ b/etebase_fastapi/test_reset_view.py @@ -0,0 +1,38 @@ +from django.conf import settings +from django.db import transaction +from django.shortcuts import get_object_or_404 +from fastapi import APIRouter, Request, status + +from django_etebase.utils import get_user_queryset, CallbackContext +from etebase_fastapi.authentication import SignupIn, signup_save +from etebase_fastapi.msgpack import MsgpackRoute +from etebase_fastapi.exceptions import HttpError +from myauth.models import get_typed_user_model + +test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) +User = get_typed_user_model() + + +@test_reset_view_router.post("/reset/", status_code=status.HTTP_204_NO_CONTENT) +def reset(data: SignupIn, request: Request): + # Only run when in DEBUG mode! It's only used for tests + if not settings.DEBUG: + raise HttpError(code="generic", detail="Only allowed in debug mode.") + + with transaction.atomic(): + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) + user = get_object_or_404(user_queryset, username=data.user.username) + # Only allow test users for extra safety + if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): + raise HttpError(code="generic", detail="Endpoint not allowed for user.") + + if hasattr(user, "userinfo"): + user.userinfo.delete() + + signup_save(data, request) + # Delete all of the journal data for this user for a clear test env + user.collection_set.all().delete() + user.collectionmember_set.all().delete() + user.incoming_invitations.all().delete() + + # FIXME: also delete chunk files!!! diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py new file mode 100644 index 0000000..c91c3ec --- /dev/null +++ b/etebase_fastapi/utils.py @@ -0,0 +1,74 @@ +import dataclasses +import typing as t +import msgpack +import base64 + +from fastapi import status, Query, Depends +from pydantic import BaseModel as PyBaseModel + +from django.db.models import QuerySet +from django.core.exceptions import ObjectDoesNotExist + +from django_etebase import app_settings +from django_etebase.models import AccessLevels +from myauth.models import UserType, get_typed_user_model + +from .exceptions import HttpError, HttpErrorOut + +User = get_typed_user_model() + +Prefetch = t.Literal["auto", "medium"] +PrefetchQuery = Query(default="auto") + + +class BaseModel(PyBaseModel): + class Config: + json_encoders = { + bytes: lambda x: x, + } + + +@dataclasses.dataclass +class Context: + user: t.Optional[UserType] + prefetch: t.Optional[Prefetch] + + +def get_object_or_404(queryset: QuerySet, **kwargs): + try: + return queryset.get(**kwargs) + except ObjectDoesNotExist as e: + raise HttpError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) + + +def is_collection_admin(collection, user): + member = collection.members.filter(user=user).first() + return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) + + +def msgpack_encode(content): + return msgpack.packb(content, use_bin_type=True) + + +def msgpack_decode(content): + return msgpack.unpackb(content, raw=False) + + +def b64encode(value): + return base64.urlsafe_b64encode(value).decode("ascii").strip("=") + + +def b64decode(data): + data += "=" * ((4 - len(data) % 4) % 4) + return base64.urlsafe_b64decode(data) + + +PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] +PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] + + +response_model_dict = {"model": HttpErrorOut} +permission_responses: t.Dict[t.Union[int, str], t.Dict[str, t.Any]] = { + 401: response_model_dict, + 403: response_model_dict, +} diff --git a/etebase_server/asgi.py b/etebase_server/asgi.py index 0bf63ec..25dbf77 100644 --- a/etebase_server/asgi.py +++ b/etebase_server/asgi.py @@ -1,16 +1,19 @@ -""" -ASGI config for etebase_server project. - -It exposes the ASGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ -""" - import os from django.core.asgi import get_asgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") +django_application = get_asgi_application() + + +def create_application(): + from etebase_fastapi.main import create_application + + app = create_application() + + app.mount("/", django_application) + + return app + -application = get_asgi_application() +application = create_application() diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 325dca9..5d57ec0 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -53,8 +53,6 @@ INSTALLED_APPS = [ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "corsheaders", - "rest_framework", "myauth.apps.MyauthConfig", "django_etebase.apps.DjangoEtebaseConfig", "django_etebase.token_auth.apps.TokenAuthConfig", @@ -63,7 +61,6 @@ INSTALLED_APPS = [ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", - "corsheaders.middleware.CorsMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", @@ -124,9 +121,6 @@ USE_L10N = True USE_TZ = True -# Cors -CORS_ORIGIN_ALLOW_ALL = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ @@ -166,11 +160,6 @@ if any(os.path.isfile(x) for x in config_locations): if "database" in config: DATABASES = {"default": {x.upper(): y for x, y in config.items("database")}} -ETEBASE_API_PERMISSIONS = ("rest_framework.permissions.IsAuthenticated",) -ETEBASE_API_AUTHENTICATORS = ( - "django_etebase.token_auth.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", -) ETEBASE_CREATE_USER_FUNC = "django_etebase.utils.create_user_blocked" # Efficient file streaming (for large files) diff --git a/etebase_server/urls.py b/etebase_server/urls.py index f285977..7cf5a60 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,16 +1,25 @@ +import os + from django.conf import settings -from django.conf.urls import include, url +from django.conf.urls import url from django.contrib import admin -from django.urls import path +from django.urls import path, re_path from django.views.generic import TemplateView +from django.views.static import serve +from django.contrib.staticfiles import finders urlpatterns = [ - url(r"^api/", include("django_etebase.urls")), url(r"^admin/", admin.site.urls), path("", TemplateView.as_view(template_name="success.html")), ] if settings.DEBUG: - urlpatterns += [ - url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), - ] + + def serve_static(request, path): + filename = finders.find(path) + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + + return serve(request, basename, dirname) + + urlpatterns += [re_path(r"^static/(?P.*)$", serve_static)] diff --git a/etebase_server/wsgi.py b/etebase_server/wsgi.py deleted file mode 100644 index 908f88c..0000000 --- a/etebase_server/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for etebase_server project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") - -application = get_wsgi_application() diff --git a/myauth/forms.py b/myauth/forms.py index 7aacb9b..fc2be74 100644 --- a/myauth/forms.py +++ b/myauth/forms.py @@ -1,8 +1,8 @@ from django import forms -from django.contrib.auth import get_user_model from django.contrib.auth.forms import UsernameField +from myauth.models import get_typed_user_model -User = get_user_model() +User = get_typed_user_model() class AdminUserCreationForm(forms.ModelForm): diff --git a/myauth/models.py b/myauth/models.py index d6585a8..5bc4af7 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -1,3 +1,5 @@ +import typing as t + from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager from django.core import validators from django.db import models @@ -28,9 +30,21 @@ class User(AbstractUser): unique=True, help_text=_("Required. 150 characters or fewer. Letters, digits and ./-/_ only."), validators=[username_validator], - error_messages={"unique": _("A user with that username already exists."),}, + error_messages={ + "unique": _("A user with that username already exists."), + }, ) @classmethod def normalize_username(cls, username): return super().normalize_username(username).lower() + + +UserType = t.Type[User] + + +def get_typed_user_model() -> UserType: + from django.contrib.auth import get_user_model + + ret: t.Any = get_user_model() + return ret diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..15a8b60 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements-dev.txt requirements.in/development.txt +# +appdirs==1.4.4 # via black +asgiref==3.3.1 # via django +black==20.8b1 # via -r requirements.in/development.txt +click==7.1.2 # via black, pip-tools +coverage==5.3.1 # via -r requirements.in/development.txt +django-stubs==1.7.0 # via -r requirements.in/development.txt +django==3.1.4 # via django-stubs +mypy-extensions==0.4.3 # via black, mypy +mypy==0.790 # via django-stubs +pathspec==0.8.1 # via black +pip-tools==5.4.0 # via -r requirements.in/development.txt +pytz==2020.5 # via django +pywatchman==1.4.1 # via -r requirements.in/development.txt +regex==2020.11.13 # via black +six==1.15.0 # via pip-tools +sqlparse==0.4.1 # via django +toml==0.10.2 # via black +typed-ast==1.4.1 # via black, mypy +typing-extensions==3.7.4.3 # via black, django-stubs, mypy + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 7d5bf7e..fee4a56 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -1,7 +1,5 @@ django -django-cors-headers -djangorestframework -drf-nested-routers msgpack -psycopg2-binary pynacl +fastapi +uvicorn diff --git a/requirements.in/development.txt b/requirements.in/development.txt index a956471..fb281d3 100644 --- a/requirements.in/development.txt +++ b/requirements.in/development.txt @@ -1,4 +1,5 @@ coverage pip-tools pywatchman -black \ No newline at end of file +black +django-stubs diff --git a/requirements.txt b/requirements.txt index f6c8ed4..cfce456 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,16 +4,18 @@ # # pip-compile --output-file=requirements.txt requirements.in/base.txt # -asgiref==3.2.10 # via django -cffi==1.14.0 # via pynacl -django-cors-headers==3.2.1 # via -r requirements.in/base.txt -django==3.1.1 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers -djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers -drf-nested-routers==0.91 # via -r requirements.in/base.txt -msgpack==1.0.0 # via -r requirements.in/base.txt -psycopg2-binary==2.8.4 # via -r requirements.in/base.txt +asgiref==3.3.1 # via django +cffi==1.14.4 # via pynacl +click==7.1.2 # via uvicorn +django==3.1.4 # via -r requirements.in/base.txt +fastapi==0.63.0 # via -r requirements.in/base.txt +h11==0.11.0 # via uvicorn +msgpack==1.0.2 # via -r requirements.in/base.txt pycparser==2.20 # via cffi -pynacl==1.3.0 # via -r requirements.in/base.txt -pytz==2019.3 # via django -six==1.14.0 # via pynacl -sqlparse==0.3.0 # via django +pydantic==1.7.3 # via fastapi +pynacl==1.4.0 # via -r requirements.in/base.txt +pytz==2020.4 # via django +six==1.15.0 # via pynacl +sqlparse==0.4.1 # via django +starlette==0.13.6 # via fastapi +uvicorn==0.13.2 # via -r requirements.in/base.txt