From 228522d0191f4718dfedc5fd3576c02b8a5b3e0e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 19 Feb 2020 14:54:35 +0200 Subject: [PATCH 001/251] Add requirements. --- requirements.in/base.txt | 11 +++++++++++ requirements.in/development.txt | 2 ++ requirements.txt | 30 ++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 requirements.in/base.txt create mode 100644 requirements.in/development.txt create mode 100644 requirements.txt diff --git a/requirements.in/base.txt b/requirements.in/base.txt new file mode 100644 index 0000000..a7d1734 --- /dev/null +++ b/requirements.in/base.txt @@ -0,0 +1,11 @@ +django +django-allauth +django-anymail +django-appconf +django-cors-headers +django-debug-toolbar +django-fullurl +django-ipware +djangorestframework +drf-nested-routers +psycopg2-binary diff --git a/requirements.in/development.txt b/requirements.in/development.txt new file mode 100644 index 0000000..30fb558 --- /dev/null +++ b/requirements.in/development.txt @@ -0,0 +1,2 @@ +coverage +pip-tools diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..34ca428 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements.txt requirements.in/base.txt +# +asgiref==3.2.3 # via django +certifi==2019.11.28 # via requests +chardet==3.0.4 # via requests +defusedxml==0.6.0 # via python3-openid +django-allauth==0.41.0 +django-anymail==7.0.0 +django-appconf==1.0.3 +django-cors-headers==3.2.1 +django-debug-toolbar==2.2 +django-fullurl==1.0 +django-ipware==2.1.0 +django==3.0.3 +djangorestframework==3.11.0 +drf-nested-routers==0.91 +idna==2.8 # via requests +oauthlib==3.1.0 # via requests-oauthlib +psycopg2-binary==2.8.4 +python3-openid==3.1.0 # via django-allauth +pytz==2019.3 # via django +requests-oauthlib==1.3.0 # via django-allauth +requests==2.22.0 # via django-allauth, django-anymail, requests-oauthlib +six==1.14.0 # via django-anymail, django-appconf +sqlparse==0.3.0 # via django, django-debug-toolbar +urllib3==1.25.8 # via requests From 703a5ae36aaaeea69885b716a1072c239b69014d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 19 Feb 2020 14:55:56 +0200 Subject: [PATCH 002/251] Create new django project. --- django_etesync/__init__.py | 0 django_etesync/admin.py | 3 +++ django_etesync/apps.py | 5 +++++ django_etesync/migrations/__init__.py | 0 django_etesync/models.py | 3 +++ django_etesync/tests.py | 3 +++ django_etesync/views.py | 3 +++ 7 files changed, 17 insertions(+) create mode 100644 django_etesync/__init__.py create mode 100644 django_etesync/admin.py create mode 100644 django_etesync/apps.py create mode 100644 django_etesync/migrations/__init__.py create mode 100644 django_etesync/models.py create mode 100644 django_etesync/tests.py create mode 100644 django_etesync/views.py diff --git a/django_etesync/__init__.py b/django_etesync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etesync/admin.py b/django_etesync/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/django_etesync/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/django_etesync/apps.py b/django_etesync/apps.py new file mode 100644 index 0000000..adb8f96 --- /dev/null +++ b/django_etesync/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class DjangoEtesyncConfig(AppConfig): + name = 'django_etesync' diff --git a/django_etesync/migrations/__init__.py b/django_etesync/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etesync/models.py b/django_etesync/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/django_etesync/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/django_etesync/tests.py b/django_etesync/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/django_etesync/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/django_etesync/views.py b/django_etesync/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/django_etesync/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 818bb8d70f5d576f0a0d756ba8688527a2ff3dd4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 19 Feb 2020 20:53:43 +0200 Subject: [PATCH 003/251] Create the django_etesync app. --- django_etesync/__init__.py | 1 + django_etesync/app_settings.py | 52 ++++++++ django_etesync/migrations/0001_initial.py | 67 +++++++++++ django_etesync/models.py | 77 +++++++++++- django_etesync/paginators.py | 36 ++++++ django_etesync/permissions.py | 65 ++++++++++ django_etesync/serializers.py | 88 ++++++++++++++ django_etesync/views.py | 140 +++++++++++++++++++++- 8 files changed, 523 insertions(+), 3 deletions(-) create mode 100644 django_etesync/app_settings.py create mode 100644 django_etesync/migrations/0001_initial.py create mode 100644 django_etesync/paginators.py create mode 100644 django_etesync/permissions.py create mode 100644 django_etesync/serializers.py diff --git a/django_etesync/__init__.py b/django_etesync/__init__.py index e69de29..426fefd 100644 --- a/django_etesync/__init__.py +++ b/django_etesync/__init__.py @@ -0,0 +1 @@ +from .app_settings import app_settings diff --git a/django_etesync/app_settings.py b/django_etesync/app_settings.py new file mode 100644 index 0000000..6a04b4e --- /dev/null +++ b/django_etesync/app_settings.py @@ -0,0 +1,52 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys # noqa + + +class AppSettings: + def __init__(self, prefix): + self.prefix = prefix + + def import_from_str(self, name): + from importlib import import_module + + p, m = name.rsplit('.', 1) + + mod = import_module(p) + return getattr(mod, m) + + def _setting(self, name, dflt): + from django.conf import settings + return getattr(settings, self.prefix + name, dflt) + + @property + def API_PERMISSIONS(self): + perms = self._setting("API_PERMISSIONS", ('rest_framework.permissions.IsAuthenticated', )) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + + @property + def API_AUTHENTICATORS(self): + perms = self._setting("API_AUTHENTICATORS", ('rest_framework.authentication.TokenAuthentication', + 'rest_framework.authentication.SessionAuthentication')) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + + +app_settings = AppSettings('ETESYNC_') diff --git a/django_etesync/migrations/0001_initial.py b/django_etesync/migrations/0001_initial.py new file mode 100644 index 0000000..6fa724b --- /dev/null +++ b/django_etesync/migrations/0001_initial.py @@ -0,0 +1,67 @@ +# Generated by Django 3.0.3 on 2020-02-19 15:33 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Collection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-fA-F0-9\\-_=]{44}')])), + ('version', models.PositiveSmallIntegerField()), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('uid', 'owner')}, + }, + ), + migrations.CreateModel( + name='CollectionItem', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-fA-F0-9\\-_=]{44}')])), + ('version', models.PositiveSmallIntegerField()), + ('encryptionKey', models.BinaryField(editable=True)), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etesync.Collection')), + ], + options={ + 'unique_together': {('uid', 'collection')}, + }, + ), + migrations.CreateModel( + name='CollectionItemSnapshot', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('current', models.BooleanField(default=True)), + ('chunkHmac', models.CharField(max_length=50)), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etesync.CollectionItem')), + ], + options={ + 'unique_together': {('item', 'current')}, + }, + ), + migrations.CreateModel( + name='CollectionItemChunk', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-fA-F0-9\\-_=]{44}')])), + ('order', models.CharField(max_length=100)), + ('itemSnapshot', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_etesync.CollectionItemSnapshot')), + ], + options={ + 'ordering': ['order'], + }, + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 71a8362..9e578ba 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -1,3 +1,78 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + from django.db import models +from django.conf import settings +from django.core.validators import RegexValidator +from django.utils.functional import cached_property + + +UidValidator = RegexValidator(regex=r'[a-zA-Z0-9\-_=]{44}', message='Not a valid UID. Expected a 256bit base64url.') + + +class Collection(models.Model): + uid = models.CharField(db_index=True, blank=False, null=False, + max_length=44, validators=[UidValidator]) + version = models.PositiveSmallIntegerField() + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + + class Meta: + unique_together = ('uid', 'owner') + + def __str__(self): + return self.uid + + +class CollectionItem(models.Model): + uid = models.CharField(db_index=True, blank=False, null=False, + max_length=44, validators=[UidValidator]) + version = models.PositiveSmallIntegerField() + encryptionKey = models.BinaryField(editable=True, blank=False, null=False) + collection = models.ForeignKey(Collection, on_delete=models.CASCADE) + + class Meta: + unique_together = ('uid', 'collection') + + @cached_property + def content(self): + return self.snapshots.get(current=True) + + def __str__(self): + return self.uid + + +class CollectionItemSnapshot(models.Model): + item = models.ForeignKey(CollectionItem, related_name='snapshots', on_delete=models.CASCADE) + current = models.BooleanField(default=True) + chunkHmac = models.CharField(max_length=50, blank=False, null=False) + + class Meta: + unique_together = ('item', 'current') + + def __str__(self): + return "{}, current={}".format(self.item.uid, self.current) + + +class CollectionItemChunk(models.Model): + uid = models.CharField(db_index=True, blank=False, null=False, + max_length=44, validators=[UidValidator]) + itemSnapshot = models.ForeignKey(CollectionItemSnapshot, related_name='chunks', null=True, on_delete=models.SET_NULL) + order = models.CharField(max_length=100, blank=False, null=False) + + class Meta: + # unique_together = ('itemSnapshot', 'order') # Currently off because we set the item snapshot to null on deletion + ordering = ['order'] -# Create your models here. + def __str__(self): + return self.uid diff --git a/django_etesync/paginators.py b/django_etesync/paginators.py new file mode 100644 index 0000000..6d55599 --- /dev/null +++ b/django_etesync/paginators.py @@ -0,0 +1,36 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from rest_framework import pagination +from rest_framework.response import Response + + +class LinkHeaderPagination(pagination.LimitOffsetPagination): + def get_paginated_response(self, data): + next_url = self.get_next_link() + previous_url = self.get_previous_link() + + if next_url is not None and previous_url is not None: + link = '<{next_url}>; rel="next", <{previous_url}>; rel="prev"' + elif next_url is not None: + link = '<{next_url}>; rel="next"' + elif previous_url is not None: + link = '<{previous_url}>; rel="prev"' + else: + link = '' + + link = link.format(next_url=next_url, previous_url=previous_url) + headers = {'Link': link} if link else {} + + return Response(data, headers=headers) diff --git a/django_etesync/permissions.py b/django_etesync/permissions.py new file mode 100644 index 0000000..f553930 --- /dev/null +++ b/django_etesync/permissions.py @@ -0,0 +1,65 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from rest_framework import permissions +from journal.models import Journal, JournalMember + + +class IsOwnerOrReadOnly(permissions.BasePermission): + """ + Custom permission to only allow owners of an object to edit it. + """ + + def has_object_permission(self, request, view, obj): + if request.method in permissions.SAFE_METHODS: + return True + + return obj.owner == request.user + + +class IsJournalOwner(permissions.BasePermission): + """ + Custom permission to only allow owners of a journal to view it + """ + + def has_permission(self, request, view): + journal_uid = view.kwargs['journal_uid'] + try: + journal = view.get_journal_queryset().get(uid=journal_uid) + return journal.owner == request.user + except Journal.DoesNotExist: + # If the journal does not exist, we want to 404 later, not permission denied. + return True + + +class IsMemberReadOnly(permissions.BasePermission): + """ + Custom permission to make a journal read only if a read only member + """ + + def has_permission(self, request, view): + if request.method in permissions.SAFE_METHODS: + return True + + journal_uid = view.kwargs['journal_uid'] + try: + journal = view.get_journal_queryset().get(uid=journal_uid) + member = journal.members.get(user=request.user) + return not member.readOnly + except Journal.DoesNotExist: + # If the journal does not exist, we want to 404 later, not permission denied. + return True + except JournalMember.DoesNotExist: + # Not being a member means we are the owner. + return True diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py new file mode 100644 index 0000000..d4f259f --- /dev/null +++ b/django_etesync/serializers.py @@ -0,0 +1,88 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import base64 + +from django.contrib.auth import get_user_model +from rest_framework import serializers +from . import models + +User = get_user_model() + + +class BinaryBase64Field(serializers.Field): + def to_representation(self, value): + return base64.b64encode(value).decode('ascii') + + def to_internal_value(self, data): + return base64.b64decode(data) + + +class CollectionSerializer(serializers.ModelSerializer): + owner = serializers.SlugRelatedField( + slug_field=User.USERNAME_FIELD, + read_only=True + ) + encryptionKey = serializers.SerializerMethodField('get_key_from_context') + permissions = serializers.SerializerMethodField('get_permission_from_context') + ctag = serializers.SerializerMethodField('get_ctag') + + class Meta: + model = models.Collection + fields = ('uid', 'version', 'owner', 'encryptionKey', 'permissions', 'ctag') + + def get_key_from_context(self, obj): + request = self.context.get('request', None) + if request is not None: + return 'FIXME' + return None + + def get_permission_from_context(self, obj): + request = self.context.get('request', None) + if request is not None: + return 'FIXME' + return 'readOnly' + + def get_ctag(self, obj): + return 'FIXME' + + +class CollectionItemChunkSerializer(serializers.ModelSerializer): + class Meta: + model = models.CollectionItemChunk + fields = ('uid', ) + + +class CollectionItemSnapshotSerializer(serializers.ModelSerializer): + chunks = serializers.SlugRelatedField( + slug_field='uid', + queryset=models.CollectionItemChunk, + many=True + ) + + class Meta: + model = models.CollectionItemSnapshot + fields = ('chunks', 'chunkHmac') + + +class CollectionItemSerializer(serializers.ModelSerializer): + encryptionKey = BinaryBase64Field() + content = CollectionItemSnapshotSerializer( + read_only=True, + many=False + ) + + class Meta: + model = models.CollectionItem + fields = ('uid', 'version', 'encryptionKey', 'content') diff --git a/django_etesync/views.py b/django_etesync/views.py index 91ea44a..4d99dad 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -1,3 +1,139 @@ -from django.shortcuts import render +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . -# Create your views here. +from django.conf import settings +from django.contrib.auth import login, get_user_model +from django.db import IntegrityError, transaction +from django.db.models import Q +from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpResponseBadRequest, HttpResponse, Http404 +from django.shortcuts import get_object_or_404 +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + +from rest_framework import status +from rest_framework import viewsets +from rest_framework.response import Response + +from . import app_settings, paginators +from .models import Collection, CollectionItem, CollectionItemSnapshot, CollectionItemChunk +from .serializers import ( + CollectionSerializer, + CollectionItemSerializer, + CollectionItemSnapshotSerializer, + CollectionItemChunkSerializer + ) + + +User = get_user_model() + + +class BaseViewSet(viewsets.ModelViewSet): + authentication_classes = tuple(app_settings.API_AUTHENTICATORS) + permission_classes = tuple(app_settings.API_PERMISSIONS) + + def get_serializer_class(self): + serializer_class = self.serializer_class + + if self.request.method == 'PUT': + serializer_class = getattr(self, 'serializer_update_class', serializer_class) + + return serializer_class + + def get_collection_queryset(self, queryset=Collection.objects): + return queryset.all() + + +class CollectionViewSet(BaseViewSet): + allowed_methods = ['GET', 'POST', 'DELETE'] + permission_classes = BaseViewSet.permission_classes + queryset = Collection.objects.all() + serializer_class = CollectionSerializer + lookup_field = 'uid' + + def get_queryset(self): + queryset = type(self).queryset + return self.get_collection_queryset(queryset) + + def destroy(self, request, uid=None): + # FIXME: implement + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def create(self, request, *args, **kwargs): + serializer = self.serializer_class(data=request.data) + if serializer.is_valid(): + try: + with transaction.atomic(): + serializer.save(owner=self.request.user) + except IntegrityError: + content = {'code': 'integrity_error'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + return Response({}, status=status.HTTP_201_CREATED) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + def list(self, request): + queryset = self.get_queryset() + + serializer = self.serializer_class(queryset, context={'request': request}, many=True) + return Response(serializer.data) + + +class CollectionItemViewSet(BaseViewSet): + allowed_methods = ['GET', 'POST'] + permission_classes = BaseViewSet.permission_classes + queryset = CollectionItem.objects.all() + serializer_class = CollectionItemSerializer + pagination_class = paginators.LinkHeaderPagination + lookup_field = 'uid' + + def get_queryset(self): + collection_uid = self.kwargs['collection_uid'] + try: + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + except Collection.DoesNotExist: + raise Http404("Collection does not exist") + # XXX Potentially add this for performance: .prefetch_related('snapshots__chunks') + queryset = type(self).queryset.filter(collection__pk=collection.pk) + + return queryset + + def create(self, request, collection_uid=None): + collection_object = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + + many = isinstance(request.data, list) + serializer = self.serializer_class(data=request.data, many=many) + if serializer.is_valid(): + try: + serializer.save(collection=collection_object) + except IntegrityError: + content = {'code': 'integrity_error'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + return Response({}, status=status.HTTP_201_CREATED) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + def destroy(self, request, collection_uid=None, uid=None): + # FIXME: implement + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def update(self, request, collection_uid=None, uid=None): + # FIXME: implement, or should it be implemented elsewhere? + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def partial_update(self, request, collection_uid=None, uid=None): + # FIXME: implement, or should it be implemented elsewhere? + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) From 0a3bb6f4bb47b38313b5e4b8ce7ca278eebf5b2f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 12:02:59 +0200 Subject: [PATCH 004/251] Merge item snapshot and item to be one model. --- .../migrations/0002_auto_20200220_0943.py | 53 +++++++++++++++++++ .../migrations/0003_collectionitem_current.py | 18 +++++++ .../migrations/0004_auto_20200220_1029.py | 18 +++++++ django_etesync/models.py | 32 ++++++----- django_etesync/serializers.py | 28 +++++----- django_etesync/views.py | 4 +- 6 files changed, 121 insertions(+), 32 deletions(-) create mode 100644 django_etesync/migrations/0002_auto_20200220_0943.py create mode 100644 django_etesync/migrations/0003_collectionitem_current.py create mode 100644 django_etesync/migrations/0004_auto_20200220_1029.py diff --git a/django_etesync/migrations/0002_auto_20200220_0943.py b/django_etesync/migrations/0002_auto_20200220_0943.py new file mode 100644 index 0000000..c150a11 --- /dev/null +++ b/django_etesync/migrations/0002_auto_20200220_0943.py @@ -0,0 +1,53 @@ +# Generated by Django 3.0.3 on 2020-02-20 09:43 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitemchunk', + name='itemSnapshot', + ), + migrations.AddField( + model_name='collectionitem', + name='hmac', + field=models.CharField(default='', max_length=50), + preserve_default=False, + ), + migrations.AddField( + model_name='collectionitemchunk', + name='items', + field=models.ManyToManyField(related_name='chunks', to='django_etesync.CollectionItem'), + ), + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{44}')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='collection', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='django_etesync.Collection'), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{44}')]), + ), + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{44}')]), + ), + migrations.DeleteModel( + name='CollectionItemSnapshot', + ), + ] diff --git a/django_etesync/migrations/0003_collectionitem_current.py b/django_etesync/migrations/0003_collectionitem_current.py new file mode 100644 index 0000000..2ffbf54 --- /dev/null +++ b/django_etesync/migrations/0003_collectionitem_current.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-02-20 09:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0002_auto_20200220_0943'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitem', + name='current', + field=models.BooleanField(default=True), + ), + ] diff --git a/django_etesync/migrations/0004_auto_20200220_1029.py b/django_etesync/migrations/0004_auto_20200220_1029.py new file mode 100644 index 0000000..1ea337a --- /dev/null +++ b/django_etesync/migrations/0004_auto_20200220_1029.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-02-20 10:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0003_collectionitem_current'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitem', + name='current', + field=models.BooleanField(db_index=True, default=True), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 9e578ba..f773495 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -12,6 +12,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from pathlib import Path + from django.db import models from django.conf import settings from django.core.validators import RegexValidator @@ -33,45 +35,41 @@ class Collection(models.Model): def __str__(self): return self.uid + @cached_property + def current_items(self): + return self.items.filter(current=True) + class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) version = models.PositiveSmallIntegerField() encryptionKey = models.BinaryField(editable=True, blank=False, null=False) - collection = models.ForeignKey(Collection, on_delete=models.CASCADE) + collection = models.ForeignKey(Collection, related_name='items', on_delete=models.CASCADE) + hmac = models.CharField(max_length=50, blank=False, null=False) + current = models.BooleanField(db_index=True, default=True) class Meta: unique_together = ('uid', 'collection') - @cached_property - def content(self): - return self.snapshots.get(current=True) - def __str__(self): return self.uid -class CollectionItemSnapshot(models.Model): - item = models.ForeignKey(CollectionItem, related_name='snapshots', on_delete=models.CASCADE) - current = models.BooleanField(default=True) - chunkHmac = models.CharField(max_length=50, blank=False, null=False) - - class Meta: - unique_together = ('item', 'current') - - def __str__(self): - return "{}, current={}".format(self.item.uid, self.current) +def chunk_directory_path(instance, filename): + col = instance.itemSnapshot.item.collection + user_id = col.owner.id + return Path('user_{}'.format(user_id), col.uid, instance.uid) class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) - itemSnapshot = models.ForeignKey(CollectionItemSnapshot, related_name='chunks', null=True, on_delete=models.SET_NULL) + items = models.ManyToManyField(CollectionItem, related_name='chunks') order = models.CharField(max_length=100, blank=False, null=False) + # We probably just want to implement this manually because we can have more than one pointing to a file. chunkFile = models.FileField(upload_to=chunk_directory_path) class Meta: - # unique_together = ('itemSnapshot', 'order') # Currently off because we set the item snapshot to null on deletion ordering = ['order'] def __str__(self): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index d4f259f..15034c2 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -64,25 +64,27 @@ class CollectionItemChunkSerializer(serializers.ModelSerializer): fields = ('uid', ) -class CollectionItemSnapshotSerializer(serializers.ModelSerializer): +class CollectionItemSerializer(serializers.ModelSerializer): + encryptionKey = BinaryBase64Field() chunks = serializers.SlugRelatedField( slug_field='uid', - queryset=models.CollectionItemChunk, + queryset=models.CollectionItemChunk.objects.all(), many=True ) class Meta: - model = models.CollectionItemSnapshot - fields = ('chunks', 'chunkHmac') + model = models.CollectionItem + fields = ('uid', 'version', 'encryptionKey', 'chunks', 'hmac') -class CollectionItemSerializer(serializers.ModelSerializer): - encryptionKey = BinaryBase64Field() - content = CollectionItemSnapshotSerializer( - read_only=True, - many=False - ) +class CollectionItemInlineSerializer(CollectionItemSerializer): + chunksData = serializers.SerializerMethodField('get_inline_chunks_from_context') - class Meta: - model = models.CollectionItem - fields = ('uid', 'version', 'encryptionKey', 'content') + class Meta(CollectionItemSerializer.Meta): + fields = CollectionItemSerializer.Meta.fields + ('chunksData', ) + + def get_inline_chunks_from_context(self, obj): + request = self.context.get('request', None) + if request is not None: + return ['SomeInlineData', 'Somemoredata'] + return 'readOnly' diff --git a/django_etesync/views.py b/django_etesync/views.py index 4d99dad..8724ae2 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -27,11 +27,11 @@ from rest_framework import viewsets from rest_framework.response import Response from . import app_settings, paginators -from .models import Collection, CollectionItem, CollectionItemSnapshot, CollectionItemChunk +from .models import Collection, CollectionItem, CollectionItemChunk from .serializers import ( CollectionSerializer, CollectionItemSerializer, - CollectionItemSnapshotSerializer, + CollectionItemInlineSerializer, CollectionItemChunkSerializer ) From 4075f775e76acbaf047d131e15c6eefe35b315de Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 12:30:20 +0200 Subject: [PATCH 005/251] Implement prefer-inline for fetching items. --- django_etesync/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 8724ae2..0c28974 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -99,6 +99,12 @@ class CollectionItemViewSet(BaseViewSet): pagination_class = paginators.LinkHeaderPagination lookup_field = 'uid' + def get_serializer_class(self): + if self.request.method == 'GET' and self.request.query_params.get('prefer_inline'): + return CollectionItemInlineSerializer + + return super().get_serializer_class() + def get_queryset(self): collection_uid = self.kwargs['collection_uid'] try: From 67fb714ddba00af185f43cdadc47069c7c559958 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 13:56:16 +0200 Subject: [PATCH 006/251] More progress. --- .../migrations/0005_auto_20200220_1123.py | 29 +++++++++++ .../migrations/0006_auto_20200220_1137.py | 49 +++++++++++++++++++ .../migrations/0007_auto_20200220_1144.py | 28 +++++++++++ django_etesync/models.py | 40 ++++++++++----- django_etesync/serializers.py | 24 ++++++--- django_etesync/views.py | 38 ++++++++++++++ 6 files changed, 189 insertions(+), 19 deletions(-) create mode 100644 django_etesync/migrations/0005_auto_20200220_1123.py create mode 100644 django_etesync/migrations/0006_auto_20200220_1137.py create mode 100644 django_etesync/migrations/0007_auto_20200220_1144.py diff --git a/django_etesync/migrations/0005_auto_20200220_1123.py b/django_etesync/migrations/0005_auto_20200220_1123.py new file mode 100644 index 0000000..88c9ea6 --- /dev/null +++ b/django_etesync/migrations/0005_auto_20200220_1123.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.3 on 2020-02-20 11:23 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0004_auto_20200220_1029'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitemchunk', + name='items', + ), + migrations.AddField( + model_name='collectionitem', + name='chunks', + field=models.ManyToManyField(related_name='items', to='django_etesync.CollectionItemChunk'), + ), + migrations.AddField( + model_name='collectionitemchunk', + name='collection', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etesync.Collection'), + preserve_default=False, + ), + ] diff --git a/django_etesync/migrations/0006_auto_20200220_1137.py b/django_etesync/migrations/0006_auto_20200220_1137.py new file mode 100644 index 0000000..efc421e --- /dev/null +++ b/django_etesync/migrations/0006_auto_20200220_1137.py @@ -0,0 +1,49 @@ +# Generated by Django 3.0.3 on 2020-02-20 11:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0005_auto_20200220_1123'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitem', + name='chunks', + ), + migrations.RemoveField( + model_name='collectionitem', + name='current', + ), + migrations.RemoveField( + model_name='collectionitem', + name='encryptionKey', + ), + migrations.RemoveField( + model_name='collectionitem', + name='hmac', + ), + migrations.RemoveField( + model_name='collectionitem', + name='version', + ), + migrations.CreateModel( + name='CollectionItemSnapshot', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('version', models.PositiveSmallIntegerField()), + ('encryptionKey', models.BinaryField(editable=True)), + ('hmac', models.CharField(max_length=50)), + ('current', models.BooleanField(db_index=True, default=True)), + ('chunks', models.ManyToManyField(related_name='items', to='django_etesync.CollectionItemChunk')), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='snapshots', to='django_etesync.CollectionItem')), + ], + options={ + 'unique_together': {('item', 'current')}, + }, + ), + ] diff --git a/django_etesync/migrations/0007_auto_20200220_1144.py b/django_etesync/migrations/0007_auto_20200220_1144.py new file mode 100644 index 0000000..3ebf55b --- /dev/null +++ b/django_etesync/migrations/0007_auto_20200220_1144.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-02-20 11:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0006_auto_20200220_1137'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemchunk', + name='item', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etesync.CollectionItem'), + preserve_default=False, + ), + migrations.AlterUniqueTogether( + name='collectionitemchunk', + unique_together={('item', 'order')}, + ), + migrations.RemoveField( + model_name='collectionitemchunk', + name='collection', + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index f773495..1bd2090 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -35,19 +35,18 @@ class Collection(models.Model): def __str__(self): return self.uid - @cached_property - def current_items(self): - return self.items.filter(current=True) + +def chunk_directory_path(instance, filename): + col = instance.itemSnapshot.item.collection + user_id = col.owner.id + return Path('user_{}'.format(user_id), col.uid, instance.uid) + class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) - version = models.PositiveSmallIntegerField() - encryptionKey = models.BinaryField(editable=True, blank=False, null=False) collection = models.ForeignKey(Collection, related_name='items', on_delete=models.CASCADE) - hmac = models.CharField(max_length=50, blank=False, null=False) - current = models.BooleanField(db_index=True, default=True) class Meta: unique_together = ('uid', 'collection') @@ -55,22 +54,37 @@ class CollectionItem(models.Model): def __str__(self): return self.uid - -def chunk_directory_path(instance, filename): - col = instance.itemSnapshot.item.collection - user_id = col.owner.id - return Path('user_{}'.format(user_id), col.uid, instance.uid) + @cached_property + def content(self): + return self.snapshots.get(current=True) class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) - items = models.ManyToManyField(CollectionItem, related_name='chunks') + item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) order = models.CharField(max_length=100, blank=False, null=False) # We probably just want to implement this manually because we can have more than one pointing to a file. chunkFile = models.FileField(upload_to=chunk_directory_path) class Meta: + unique_together = ('item', 'order') ordering = ['order'] def __str__(self): return self.uid + + +class CollectionItemSnapshot(models.Model): + version = models.PositiveSmallIntegerField() + encryptionKey = models.BinaryField(editable=True, blank=False, null=False) + item = models.ForeignKey(CollectionItem, related_name='snapshots', on_delete=models.CASCADE) + chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') + hmac = models.CharField(max_length=50, blank=False, null=False) + current = models.BooleanField(db_index=True, default=True) + + class Meta: + unique_together = ('item', 'current') + + def __str__(self): + return '{} {} current={}'.format(self.item.uid, self.id, self.current) + diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 15034c2..f86d5e1 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -64,7 +64,7 @@ class CollectionItemChunkSerializer(serializers.ModelSerializer): fields = ('uid', ) -class CollectionItemSerializer(serializers.ModelSerializer): +class CollectionItemSnapshotSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() chunks = serializers.SlugRelatedField( slug_field='uid', @@ -73,18 +73,30 @@ class CollectionItemSerializer(serializers.ModelSerializer): ) class Meta: - model = models.CollectionItem - fields = ('uid', 'version', 'encryptionKey', 'chunks', 'hmac') + model = models.CollectionItemSnapshot + fields = ('version', 'encryptionKey', 'chunks', 'hmac') -class CollectionItemInlineSerializer(CollectionItemSerializer): +class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotSerializer): chunksData = serializers.SerializerMethodField('get_inline_chunks_from_context') - class Meta(CollectionItemSerializer.Meta): - fields = CollectionItemSerializer.Meta.fields + ('chunksData', ) + class Meta(CollectionItemSnapshotSerializer.Meta): + fields = CollectionItemSnapshotSerializer.Meta.fields + ('chunksData', ) def get_inline_chunks_from_context(self, obj): request = self.context.get('request', None) if request is not None: return ['SomeInlineData', 'Somemoredata'] return 'readOnly' + + +class CollectionItemSerializer(serializers.ModelSerializer): + content = CollectionItemSnapshotSerializer(read_only=True, many=False) + + class Meta: + model = models.CollectionItem + fields = ('uid', 'content') + + +class CollectionItemInlineSerializer(CollectionItemSerializer): + content = CollectionItemSnapshotInlineSerializer(read_only=True, many=False) diff --git a/django_etesync/views.py b/django_etesync/views.py index 0c28974..57136a4 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -24,6 +24,7 @@ from django.views.decorators.http import require_POST from rest_framework import status from rest_framework import viewsets +from rest_framework import parsers from rest_framework.response import Response from . import app_settings, paginators @@ -143,3 +144,40 @@ class CollectionItemViewSet(BaseViewSet): def partial_update(self, request, collection_uid=None, uid=None): # FIXME: implement, or should it be implemented elsewhere? return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + +class CollectionItemChunkViewSet(viewsets.ViewSet): + allowed_methods = ['GET', 'POST'] + authentication_classes = BaseViewSet.authentication_classes + permission_classes = BaseViewSet.permission_classes + parser_classes = (parsers.MultiPartParser, ) + lookup_field = 'uid' + + def create(self, request, collection_uid=None): + # FIXME: we are potentially not getting the correct queryset + collection_object = Collection.objects.get(uid=collection_uid) + + many = isinstance(request.data, list) + serializer = self.serializer_class(data=request.data, many=many) + if serializer.is_valid(): + try: + serializer.save(collection=collection_object) + except IntegrityError: + content = {'code': 'integrity_error'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + return Response({}, status=status.HTTP_201_CREATED) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + def destroy(self, request, collection_uid=None, uid=None): + # FIXME: implement + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def update(self, request, collection_uid=None, uid=None): + # FIXME: implement, or should it be implemented elsewhere? + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def partial_update(self, request, collection_uid=None, uid=None): + # FIXME: implement, or should it be implemented elsewhere? + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) From 0c44f738fdff2f16064d782cd15a2a3e3b1021a7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 14:42:35 +0200 Subject: [PATCH 007/251] More progress - support chunk uploading. --- .../0008_collectionitemchunk_chunkfile.py | 20 ++++++++++ .../migrations/0009_auto_20200220_1220.py | 19 +++++++++ django_etesync/models.py | 16 ++++---- django_etesync/serializers.py | 40 ++++++++++++++----- django_etesync/views.py | 25 ++++-------- 5 files changed, 83 insertions(+), 37 deletions(-) create mode 100644 django_etesync/migrations/0008_collectionitemchunk_chunkfile.py create mode 100644 django_etesync/migrations/0009_auto_20200220_1220.py diff --git a/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py b/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py new file mode 100644 index 0000000..68bb27c --- /dev/null +++ b/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.3 on 2020-02-20 12:16 + +from django.db import migrations, models +import django_etesync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0007_auto_20200220_1144'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemchunk', + name='chunkFile', + field=models.FileField(default='', upload_to=django_etesync.models.chunk_directory_path), + preserve_default=False, + ), + ] diff --git a/django_etesync/migrations/0009_auto_20200220_1220.py b/django_etesync/migrations/0009_auto_20200220_1220.py new file mode 100644 index 0000000..71e1539 --- /dev/null +++ b/django_etesync/migrations/0009_auto_20200220_1220.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-02-20 12:20 + +from django.db import migrations, models +import django_etesync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0008_collectionitemchunk_chunkfile'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='chunkFile', + field=models.FileField(max_length=150, upload_to=django_etesync.models.chunk_directory_path), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 1bd2090..0cefbee 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -36,12 +36,6 @@ class Collection(models.Model): return self.uid -def chunk_directory_path(instance, filename): - col = instance.itemSnapshot.item.collection - user_id = col.owner.id - return Path('user_{}'.format(user_id), col.uid, instance.uid) - - class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, @@ -59,12 +53,19 @@ class CollectionItem(models.Model): return self.snapshots.get(current=True) +def chunk_directory_path(instance, filename): + item = instance.item + col = item.collection + user_id = col.owner.id + return Path('user_{}'.format(user_id), col.uid, item.uid, instance.uid) + + class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) order = models.CharField(max_length=100, blank=False, null=False) - # We probably just want to implement this manually because we can have more than one pointing to a file. chunkFile = models.FileField(upload_to=chunk_directory_path) + chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150) class Meta: unique_together = ('item', 'order') @@ -87,4 +88,3 @@ class CollectionItemSnapshot(models.Model): def __str__(self): return '{} {} current={}'.format(self.item.uid, self.id, self.current) - diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index f86d5e1..a4d6fbe 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -61,10 +61,10 @@ class CollectionSerializer(serializers.ModelSerializer): class CollectionItemChunkSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemChunk - fields = ('uid', ) + fields = ('uid', 'chunkFile') -class CollectionItemSnapshotSerializer(serializers.ModelSerializer): +class CollectionItemSnapshotBaseSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() chunks = serializers.SlugRelatedField( slug_field='uid', @@ -77,18 +77,36 @@ class CollectionItemSnapshotSerializer(serializers.ModelSerializer): fields = ('version', 'encryptionKey', 'chunks', 'hmac') -class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotSerializer): - chunksData = serializers.SerializerMethodField('get_inline_chunks_from_context') +class CollectionItemSnapshotSerializer(CollectionItemSnapshotBaseSerializer): + chunksUrls = serializers.SerializerMethodField('get_chunks_urls') - class Meta(CollectionItemSnapshotSerializer.Meta): - fields = CollectionItemSnapshotSerializer.Meta.fields + ('chunksData', ) + class Meta(CollectionItemSnapshotBaseSerializer.Meta): + fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksUrls', ) - def get_inline_chunks_from_context(self, obj): - request = self.context.get('request', None) - if request is not None: - return ['SomeInlineData', 'Somemoredata'] - return 'readOnly' + # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still save it under the user. + # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot. + # E.g. chunkBaseUrl: "/media/bla/bla/" or chunkBaseUrl: "https://media.etesync.com/bla/bla" + def get_chunks_urls(self, obj): + ret = [] + for chunk in obj.chunks.all(): + ret.append(chunk.chunkFile.url) + + return ret + + +class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotBaseSerializer): + chunksData = serializers.SerializerMethodField('get_chunks_data') + + class Meta(CollectionItemSnapshotBaseSerializer.Meta): + fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksData', ) + + def get_chunks_data(self, obj): + ret = [] + for chunk in obj.chunks.all(): + with open(chunk.chunkFile.path, 'rb') as f: + ret.append(base64.b64encode(f.read()).decode('ascii')) + return ret class CollectionItemSerializer(serializers.ModelSerializer): content = CollectionItemSnapshotSerializer(read_only=True, many=False) diff --git a/django_etesync/views.py b/django_etesync/views.py index 57136a4..5caa452 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -148,20 +148,21 @@ class CollectionItemViewSet(BaseViewSet): class CollectionItemChunkViewSet(viewsets.ViewSet): allowed_methods = ['GET', 'POST'] + parser_classes = (parsers.MultiPartParser, ) authentication_classes = BaseViewSet.authentication_classes permission_classes = BaseViewSet.permission_classes - parser_classes = (parsers.MultiPartParser, ) + serializer_class = CollectionItemChunkSerializer lookup_field = 'uid' - def create(self, request, collection_uid=None): + def create(self, request, collection_uid=None, collection_item_uid=None): # FIXME: we are potentially not getting the correct queryset - collection_object = Collection.objects.get(uid=collection_uid) + col = get_object_or_404(Collection.objects, uid=collection_uid) + col_it = get_object_or_404(col.items, uid=collection_item_uid) - many = isinstance(request.data, list) - serializer = self.serializer_class(data=request.data, many=many) + serializer = self.serializer_class(data=request.data) if serializer.is_valid(): try: - serializer.save(collection=collection_object) + serializer.save(item=col_it, order='abc') except IntegrityError: content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) @@ -169,15 +170,3 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return Response({}, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - def destroy(self, request, collection_uid=None, uid=None): - # FIXME: implement - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, collection_uid=None, uid=None): - # FIXME: implement, or should it be implemented elsewhere? - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, collection_uid=None, uid=None): - # FIXME: implement, or should it be implemented elsewhere? - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) From d57ed0341712f47b69abf9dc870a68c40f900f2e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 14:48:19 +0200 Subject: [PATCH 008/251] Make sure we don't upload the same file twice. --- .../migrations/0010_auto_20200220_1248.py | 19 +++++++++++++++++++ django_etesync/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0010_auto_20200220_1248.py diff --git a/django_etesync/migrations/0010_auto_20200220_1248.py b/django_etesync/migrations/0010_auto_20200220_1248.py new file mode 100644 index 0000000..0c08ed0 --- /dev/null +++ b/django_etesync/migrations/0010_auto_20200220_1248.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-02-20 12:48 + +from django.db import migrations, models +import django_etesync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0009_auto_20200220_1220'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='chunkFile', + field=models.FileField(max_length=150, unique=True, upload_to=django_etesync.models.chunk_directory_path), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 0cefbee..37c0dc1 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -65,7 +65,7 @@ class CollectionItemChunk(models.Model): max_length=44, validators=[UidValidator]) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) order = models.CharField(max_length=100, blank=False, null=False) - chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150) + chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) class Meta: unique_together = ('item', 'order') From b17e944dd27e733463f8ed92ecadedf573eca79d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 15:39:52 +0200 Subject: [PATCH 009/251] Make it possible to download the chunk from the rest API. --- django_etesync/views.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 5caa452..9d816ec 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -25,6 +25,7 @@ from django.views.decorators.http import require_POST from rest_framework import status from rest_framework import viewsets from rest_framework import parsers +from rest_framework.decorators import action as action_decorator from rest_framework.response import Response from . import app_settings, paginators @@ -170,3 +171,19 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return Response({}, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + @action_decorator(detail=True, methods=['GET']) + def download(self, request, collection_uid=None, collection_item_uid=None, uid=None): + import os + from django.views.static import serve + + col = get_object_or_404(Collection.objects, uid=collection_uid) + col_it = get_object_or_404(col.items, uid=collection_item_uid) + chunk = get_object_or_404(col_it.chunks, uid=uid) + + filename = chunk.chunkFile.path + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + + # FIXME: DO NOT USE! Use django-send file or etc instead. + return serve(request, basename, dirname) From 24cb6ed6ee9d7931503f18352819313931739274 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 16:35:20 +0200 Subject: [PATCH 010/251] Also serve an item's snapshots. --- django_etesync/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 9d816ec..f44f79a 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -34,6 +34,8 @@ from .serializers import ( CollectionSerializer, CollectionItemSerializer, CollectionItemInlineSerializer, + CollectionItemSnapshotSerializer, + CollectionItemSnapshotInlineSerializer, CollectionItemChunkSerializer ) @@ -146,6 +148,14 @@ class CollectionItemViewSet(BaseViewSet): # FIXME: implement, or should it be implemented elsewhere? return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + @action_decorator(detail=True, methods=['GET']) + def snapshots(self, request, collection_uid=None, uid=None): + col = get_object_or_404(Collection.objects, uid=collection_uid) + col_it = get_object_or_404(col.items, uid=uid) + + serializer = CollectionItemSnapshotSerializer(col_it.snapshots, many=True) + return Response(serializer.data) + class CollectionItemChunkViewSet(viewsets.ViewSet): allowed_methods = ['GET', 'POST'] From 0a40a04d3bc0ec765f6be36b46bcfd55056207fb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 17:33:34 +0200 Subject: [PATCH 011/251] Chunk view: unify how we get the wanted collection queryset. --- django_etesync/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index f44f79a..1e97b4f 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -165,9 +165,11 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): serializer_class = CollectionItemChunkSerializer lookup_field = 'uid' + def get_collection_queryset(self, queryset=Collection.objects): + return queryset.all() + def create(self, request, collection_uid=None, collection_item_uid=None): - # FIXME: we are potentially not getting the correct queryset - col = get_object_or_404(Collection.objects, uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) col_it = get_object_or_404(col.items, uid=collection_item_uid) serializer = self.serializer_class(data=request.data) @@ -187,7 +189,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): import os from django.views.static import serve - col = get_object_or_404(Collection.objects, uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) col_it = get_object_or_404(col.items, uid=collection_item_uid) chunk = get_object_or_404(col_it.chunks, uid=uid) From c3fc00b9d8a7092fd2a329bfd6433381285de4f4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 17:34:51 +0200 Subject: [PATCH 012/251] Add a FIXME. --- django_etesync/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 1e97b4f..e7a6f87 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -175,6 +175,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): serializer = self.serializer_class(data=request.data) if serializer.is_valid(): try: + # FIXME: actually generate the correct order value. Or alternatively have it null at first and only set it when ommitting to a snapshot serializer.save(item=col_it, order='abc') except IntegrityError: content = {'code': 'integrity_error'} From 052483d38c7776ac0b74c9cde73ecd5b82001222 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 18:41:07 +0200 Subject: [PATCH 013/251] Serve snapshots newest to oldest. --- django_etesync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index e7a6f87..8e21e3c 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -153,7 +153,7 @@ class CollectionItemViewSet(BaseViewSet): col = get_object_or_404(Collection.objects, uid=collection_uid) col_it = get_object_or_404(col.items, uid=uid) - serializer = CollectionItemSnapshotSerializer(col_it.snapshots, many=True) + serializer = CollectionItemSnapshotSerializer(col_it.snapshots.order_by('-id'), many=True) return Response(serializer.data) From cc00391504889799fae3ca9670e53978e445443b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 22:41:39 +0200 Subject: [PATCH 014/251] Rename Snapshot to Revision --- .../migrations/0011_auto_20200220_2037.py | 17 ++++++++++++++++ .../migrations/0012_auto_20200220_2038.py | 19 ++++++++++++++++++ django_etesync/models.py | 6 +++--- django_etesync/serializers.py | 20 +++++++++---------- django_etesync/views.py | 10 +++++----- 5 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 django_etesync/migrations/0011_auto_20200220_2037.py create mode 100644 django_etesync/migrations/0012_auto_20200220_2038.py diff --git a/django_etesync/migrations/0011_auto_20200220_2037.py b/django_etesync/migrations/0011_auto_20200220_2037.py new file mode 100644 index 0000000..2d79074 --- /dev/null +++ b/django_etesync/migrations/0011_auto_20200220_2037.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-02-20 20:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0010_auto_20200220_1248'), + ] + + operations = [ + migrations.RenameModel( + old_name='CollectionItemSnapshot', + new_name='CollectionItemRevision', + ), + ] diff --git a/django_etesync/migrations/0012_auto_20200220_2038.py b/django_etesync/migrations/0012_auto_20200220_2038.py new file mode 100644 index 0000000..2657973 --- /dev/null +++ b/django_etesync/migrations/0012_auto_20200220_2038.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-02-20 20:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0011_auto_20200220_2037'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemrevision', + name='item', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='django_etesync.CollectionItem'), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 37c0dc1..0de3f90 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -50,7 +50,7 @@ class CollectionItem(models.Model): @cached_property def content(self): - return self.snapshots.get(current=True) + return self.revisions.get(current=True) def chunk_directory_path(instance, filename): @@ -75,10 +75,10 @@ class CollectionItemChunk(models.Model): return self.uid -class CollectionItemSnapshot(models.Model): +class CollectionItemRevision(models.Model): version = models.PositiveSmallIntegerField() encryptionKey = models.BinaryField(editable=True, blank=False, null=False) - item = models.ForeignKey(CollectionItem, related_name='snapshots', on_delete=models.CASCADE) + item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') hmac = models.CharField(max_length=50, blank=False, null=False) current = models.BooleanField(db_index=True, default=True) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index a4d6fbe..de49c98 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -64,7 +64,7 @@ class CollectionItemChunkSerializer(serializers.ModelSerializer): fields = ('uid', 'chunkFile') -class CollectionItemSnapshotBaseSerializer(serializers.ModelSerializer): +class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() chunks = serializers.SlugRelatedField( slug_field='uid', @@ -73,15 +73,15 @@ class CollectionItemSnapshotBaseSerializer(serializers.ModelSerializer): ) class Meta: - model = models.CollectionItemSnapshot + model = models.CollectionItemRevision fields = ('version', 'encryptionKey', 'chunks', 'hmac') -class CollectionItemSnapshotSerializer(CollectionItemSnapshotBaseSerializer): +class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): chunksUrls = serializers.SerializerMethodField('get_chunks_urls') - class Meta(CollectionItemSnapshotBaseSerializer.Meta): - fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksUrls', ) + class Meta(CollectionItemRevisionBaseSerializer.Meta): + fields = CollectionItemRevisionBaseSerializer.Meta.fields + ('chunksUrls', ) # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still save it under the user. # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot. @@ -94,11 +94,11 @@ class CollectionItemSnapshotSerializer(CollectionItemSnapshotBaseSerializer): return ret -class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotBaseSerializer): +class CollectionItemRevisionInlineSerializer(CollectionItemRevisionBaseSerializer): chunksData = serializers.SerializerMethodField('get_chunks_data') - class Meta(CollectionItemSnapshotBaseSerializer.Meta): - fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksData', ) + class Meta(CollectionItemRevisionBaseSerializer.Meta): + fields = CollectionItemRevisionBaseSerializer.Meta.fields + ('chunksData', ) def get_chunks_data(self, obj): ret = [] @@ -109,7 +109,7 @@ class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotBaseSerialize return ret class CollectionItemSerializer(serializers.ModelSerializer): - content = CollectionItemSnapshotSerializer(read_only=True, many=False) + content = CollectionItemRevisionSerializer(read_only=True, many=False) class Meta: model = models.CollectionItem @@ -117,4 +117,4 @@ class CollectionItemSerializer(serializers.ModelSerializer): class CollectionItemInlineSerializer(CollectionItemSerializer): - content = CollectionItemSnapshotInlineSerializer(read_only=True, many=False) + content = CollectionItemRevisionInlineSerializer(read_only=True, many=False) diff --git a/django_etesync/views.py b/django_etesync/views.py index 8e21e3c..d8604cc 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -34,8 +34,8 @@ from .serializers import ( CollectionSerializer, CollectionItemSerializer, CollectionItemInlineSerializer, - CollectionItemSnapshotSerializer, - CollectionItemSnapshotInlineSerializer, + CollectionItemRevisionSerializer, + CollectionItemRevisionInlineSerializer, CollectionItemChunkSerializer ) @@ -115,7 +115,7 @@ class CollectionItemViewSet(BaseViewSet): collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) except Collection.DoesNotExist: raise Http404("Collection does not exist") - # XXX Potentially add this for performance: .prefetch_related('snapshots__chunks') + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = type(self).queryset.filter(collection__pk=collection.pk) return queryset @@ -149,11 +149,11 @@ class CollectionItemViewSet(BaseViewSet): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) @action_decorator(detail=True, methods=['GET']) - def snapshots(self, request, collection_uid=None, uid=None): + def revision(self, request, collection_uid=None, uid=None): col = get_object_or_404(Collection.objects, uid=collection_uid) col_it = get_object_or_404(col.items, uid=uid) - serializer = CollectionItemSnapshotSerializer(col_it.snapshots.order_by('-id'), many=True) + serializer = CollectionItemRevisionSerializer(col_it.revisions.order_by('-id'), many=True) return Response(serializer.data) From d6df94facf6c5015ef2373c2739c9bd3c4fe09c2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 14:20:23 +0200 Subject: [PATCH 015/251] Item create: 404 if collection isn't found. It doesn't actually change anything beacuse it 404s in the collection getting, but still, good to have this here too. --- django_etesync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index d8604cc..4c744bd 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -121,7 +121,7 @@ class CollectionItemViewSet(BaseViewSet): return queryset def create(self, request, collection_uid=None): - collection_object = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) many = isinstance(request.data, list) serializer = self.serializer_class(data=request.data, many=many) From 358c59f6d75d90892732d8ff5860feaa6dbf3bb5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 14:20:52 +0200 Subject: [PATCH 016/251] Item: add bulk_get and a note about bulk creating. --- django_etesync/views.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 4c744bd..9150db0 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -123,6 +123,8 @@ class CollectionItemViewSet(BaseViewSet): def create(self, request, collection_uid=None): collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) + # FIXME: change this to also support bulk update, or have another endpoint for that. + # See https://www.django-rest-framework.org/api-guide/serializers/#customizing-multiple-update many = isinstance(request.data, list) serializer = self.serializer_class(data=request.data, many=many) if serializer.is_valid(): @@ -156,6 +158,16 @@ class CollectionItemViewSet(BaseViewSet): serializer = CollectionItemRevisionSerializer(col_it.revisions.order_by('-id'), many=True) return Response(serializer.data) + @action_decorator(detail=False, methods=['POST']) + def bulk_get(self, request, collection_uid=None): + queryset = self.get_queryset() + + if isinstance(request.data, list): + queryset = queryset.filter(uid__in=request.data) + + serializer = self.get_serializer_class()(queryset, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + class CollectionItemChunkViewSet(viewsets.ViewSet): allowed_methods = ['GET', 'POST'] From 0beaaf5bf95fc6b7411d92a10d168b9d68f798db Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 14:21:14 +0200 Subject: [PATCH 017/251] lint: fix many pylint warnings. --- django_etesync/serializers.py | 4 +++- django_etesync/views.py | 15 +++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index de49c98..0e5b355 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -83,7 +83,8 @@ class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): class Meta(CollectionItemRevisionBaseSerializer.Meta): fields = CollectionItemRevisionBaseSerializer.Meta.fields + ('chunksUrls', ) - # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still save it under the user. + # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still + # save it under the user. # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot. # E.g. chunkBaseUrl: "/media/bla/bla/" or chunkBaseUrl: "https://media.etesync.com/bla/bla" def get_chunks_urls(self, obj): @@ -108,6 +109,7 @@ class CollectionItemRevisionInlineSerializer(CollectionItemRevisionBaseSerialize return ret + class CollectionItemSerializer(serializers.ModelSerializer): content = CollectionItemRevisionSerializer(read_only=True, many=False) diff --git a/django_etesync/views.py b/django_etesync/views.py index 9150db0..2e91261 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -12,15 +12,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from django.conf import settings -from django.contrib.auth import login, get_user_model +from django.contrib.auth import get_user_model from django.db import IntegrityError, transaction -from django.db.models import Q -from django.core.exceptions import ObjectDoesNotExist -from django.http import HttpResponseBadRequest, HttpResponse, Http404 +from django.http import Http404 from django.shortcuts import get_object_or_404 -from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_POST from rest_framework import status from rest_framework import viewsets @@ -29,13 +24,12 @@ from rest_framework.decorators import action as action_decorator from rest_framework.response import Response from . import app_settings, paginators -from .models import Collection, CollectionItem, CollectionItemChunk +from .models import Collection, CollectionItem from .serializers import ( CollectionSerializer, CollectionItemSerializer, CollectionItemInlineSerializer, CollectionItemRevisionSerializer, - CollectionItemRevisionInlineSerializer, CollectionItemChunkSerializer ) @@ -187,7 +181,8 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): serializer = self.serializer_class(data=request.data) if serializer.is_valid(): try: - # FIXME: actually generate the correct order value. Or alternatively have it null at first and only set it when ommitting to a snapshot + # FIXME: actually generate the correct order value. Or alternatively have it null at first and only + # set it when ommitting to a snapshot serializer.save(item=col_it, order='abc') except IntegrityError: content = {'code': 'integrity_error'} From 727cd3e5faf45af298de747823820d7fda619158 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 14:32:25 +0200 Subject: [PATCH 018/251] pylint: fix more warnings. --- django_etesync/app_settings.py | 12 +++++------- django_etesync/models.py | 1 - 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/django_etesync/app_settings.py b/django_etesync/app_settings.py index 6a04b4e..a0a0d99 100644 --- a/django_etesync/app_settings.py +++ b/django_etesync/app_settings.py @@ -12,8 +12,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import sys # noqa - class AppSettings: def __init__(self, prefix): @@ -22,17 +20,17 @@ class AppSettings: def import_from_str(self, name): from importlib import import_module - p, m = name.rsplit('.', 1) + path, prop = name.rsplit('.', 1) - mod = import_module(p) - return getattr(mod, m) + mod = import_module(path) + return getattr(mod, prop) def _setting(self, name, dflt): from django.conf import settings return getattr(settings, self.prefix + name, dflt) @property - def API_PERMISSIONS(self): + def API_PERMISSIONS(self): # pylint: disable=invalid-name perms = self._setting("API_PERMISSIONS", ('rest_framework.permissions.IsAuthenticated', )) ret = [] for perm in perms: @@ -40,7 +38,7 @@ class AppSettings: return ret @property - def API_AUTHENTICATORS(self): + def API_AUTHENTICATORS(self): # pylint: disable=invalid-name perms = self._setting("API_AUTHENTICATORS", ('rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.SessionAuthentication')) ret = [] diff --git a/django_etesync/models.py b/django_etesync/models.py index 0de3f90..18743fd 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -36,7 +36,6 @@ class Collection(models.Model): return self.uid - class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) From 4054a2f78cad318b2d45022e07529c7f7b2dac21 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 15:53:25 +0200 Subject: [PATCH 019/251] Implement item update and deletion. Deletion is essentially an update with "isDeletion" set to True. --- ...0013_collectionitemrevision_is_deletion.py | 18 +++++++++++++++ .../migrations/0014_auto_20200226_1322.py | 18 +++++++++++++++ .../migrations/0015_auto_20200226_1349.py | 18 +++++++++++++++ django_etesync/models.py | 3 ++- django_etesync/serializers.py | 22 +++++++++++++++++-- django_etesync/views.py | 8 ++----- 6 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 django_etesync/migrations/0013_collectionitemrevision_is_deletion.py create mode 100644 django_etesync/migrations/0014_auto_20200226_1322.py create mode 100644 django_etesync/migrations/0015_auto_20200226_1349.py diff --git a/django_etesync/migrations/0013_collectionitemrevision_is_deletion.py b/django_etesync/migrations/0013_collectionitemrevision_is_deletion.py new file mode 100644 index 0000000..27f4953 --- /dev/null +++ b/django_etesync/migrations/0013_collectionitemrevision_is_deletion.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-02-26 13:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0012_auto_20200220_2038'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemrevision', + name='is_deletion', + field=models.BooleanField(default=False), + ), + ] diff --git a/django_etesync/migrations/0014_auto_20200226_1322.py b/django_etesync/migrations/0014_auto_20200226_1322.py new file mode 100644 index 0000000..1937015 --- /dev/null +++ b/django_etesync/migrations/0014_auto_20200226_1322.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-02-26 13:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0013_collectionitemrevision_is_deletion'), + ] + + operations = [ + migrations.RenameField( + model_name='collectionitemrevision', + old_name='is_deletion', + new_name='isDeletion', + ), + ] diff --git a/django_etesync/migrations/0015_auto_20200226_1349.py b/django_etesync/migrations/0015_auto_20200226_1349.py new file mode 100644 index 0000000..896619d --- /dev/null +++ b/django_etesync/migrations/0015_auto_20200226_1349.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-02-26 13:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0014_auto_20200226_1322'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemrevision', + name='current', + field=models.BooleanField(blank=True, db_index=True, default=True, null=True), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 18743fd..dda081f 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -80,7 +80,8 @@ class CollectionItemRevision(models.Model): item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') hmac = models.CharField(max_length=50, blank=False, null=False) - current = models.BooleanField(db_index=True, default=True) + current = models.BooleanField(db_index=True, default=True, blank=True, null=True) + isDeletion = models.BooleanField(default=False) class Meta: unique_together = ('item', 'current') diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 0e5b355..b3f7254 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -15,6 +15,7 @@ import base64 from django.contrib.auth import get_user_model +from django.db import transaction from rest_framework import serializers from . import models @@ -74,7 +75,7 @@ class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemRevision - fields = ('version', 'encryptionKey', 'chunks', 'hmac') + fields = ('version', 'encryptionKey', 'chunks', 'hmac', 'isDeletion') class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): @@ -111,12 +112,29 @@ class CollectionItemRevisionInlineSerializer(CollectionItemRevisionBaseSerialize class CollectionItemSerializer(serializers.ModelSerializer): - content = CollectionItemRevisionSerializer(read_only=True, many=False) + content = CollectionItemRevisionSerializer(many=False) class Meta: model = models.CollectionItem fields = ('uid', 'content') + def update(self, instance, validated_data): + """Function that's called when this serializer is meant to update an item""" + revision_data = validated_data.pop('content') + + with transaction.atomic(): + # We don't have to use select_for_update here because the unique constraint on current guards against + # the race condition. But it's a good idea because it'll lock and wait rather than fail. + current_revision = instance.revisions.filter(current=True).select_for_update().first() + current_revision.current = None + current_revision.save() + + chunks = revision_data.pop('chunks') + revision = models.CollectionItemRevision.objects.create(**revision_data, item=instance) + revision.chunks.set(chunks) + + return instance + class CollectionItemInlineSerializer(CollectionItemSerializer): content = CollectionItemRevisionInlineSerializer(read_only=True, many=False) diff --git a/django_etesync/views.py b/django_etesync/views.py index 2e91261..b90b62e 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -90,7 +90,7 @@ class CollectionViewSet(BaseViewSet): class CollectionItemViewSet(BaseViewSet): - allowed_methods = ['GET', 'POST'] + allowed_methods = ['GET', 'POST', 'PUT'] permission_classes = BaseViewSet.permission_classes queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer @@ -133,11 +133,7 @@ class CollectionItemViewSet(BaseViewSet): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) def destroy(self, request, collection_uid=None, uid=None): - # FIXME: implement - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, collection_uid=None, uid=None): - # FIXME: implement, or should it be implemented elsewhere? + # We can't have destroy because we need to get data from the user (in the body) such as hmac. return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def partial_update(self, request, collection_uid=None, uid=None): From 452a8f1e7effef4db934d0513b5529d46f44ec81 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 16:07:55 +0200 Subject: [PATCH 020/251] Implement item creation. --- django_etesync/serializers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index b3f7254..a576b6d 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -118,6 +118,20 @@ class CollectionItemSerializer(serializers.ModelSerializer): model = models.CollectionItem fields = ('uid', 'content') + def create(self, validated_data): + """Function that's called when this serializer creates an item""" + revision_data = validated_data.pop('content') + instance = self.__class__.Meta.model(**validated_data) + + with transaction.atomic(): + instance.save() + + chunks = revision_data.pop('chunks') + revision = models.CollectionItemRevision.objects.create(**revision_data, item=instance) + revision.chunks.set(chunks) + + return instance + def update(self, instance, validated_data): """Function that's called when this serializer is meant to update an item""" revision_data = validated_data.pop('content') From f4cb7cb74f2252ae46268dc3195048f24bc2cdbd Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 16:42:49 +0200 Subject: [PATCH 021/251] Collection item list: limit only to non-deleted by default. --- django_etesync/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index b90b62e..b1bb18b 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -110,7 +110,9 @@ class CollectionItemViewSet(BaseViewSet): except Collection.DoesNotExist: raise Http404("Collection does not exist") # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = type(self).queryset.filter(collection__pk=collection.pk) + queryset = type(self).queryset.filter(collection__pk=collection.pk, + revisions__current=True, + revisions__isDeletion=False) return queryset From f1bfb0a9a057aab754b93dab32e0f442e6c6d474 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 16:46:41 +0200 Subject: [PATCH 022/251] Model uid validator: fix off-by-1 error with the uid. 256bit is actually 43 base64 chars, not 44. --- .../migrations/0016_auto_20200226_1446.py | 29 +++++++++++++++++++ django_etesync/models.py | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0016_auto_20200226_1446.py diff --git a/django_etesync/migrations/0016_auto_20200226_1446.py b/django_etesync/migrations/0016_auto_20200226_1446.py new file mode 100644 index 0000000..2929cbf --- /dev/null +++ b/django_etesync/migrations/0016_auto_20200226_1446.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.3 on 2020-02-26 14:46 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0015_auto_20200226_1349'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{43}')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{43}')]), + ), + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{43}')]), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index dda081f..4577efb 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -20,7 +20,7 @@ from django.core.validators import RegexValidator from django.utils.functional import cached_property -UidValidator = RegexValidator(regex=r'[a-zA-Z0-9\-_=]{44}', message='Not a valid UID. Expected a 256bit base64url.') +UidValidator = RegexValidator(regex=r'[a-zA-Z0-9\-_=]{43}', message='Not a valid UID. Expected a 256bit base64url.') class Collection(models.Model): From 0ee00e1a9fff8e3b7a2fd0855b3b12665614a7e8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 16:55:47 +0200 Subject: [PATCH 023/251] Collection item: rename isDeletion to deleted --- .../migrations/0017_auto_20200226_1455.py | 18 ++++++++++++++++++ django_etesync/models.py | 2 +- django_etesync/serializers.py | 2 +- django_etesync/views.py | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 django_etesync/migrations/0017_auto_20200226_1455.py diff --git a/django_etesync/migrations/0017_auto_20200226_1455.py b/django_etesync/migrations/0017_auto_20200226_1455.py new file mode 100644 index 0000000..2148123 --- /dev/null +++ b/django_etesync/migrations/0017_auto_20200226_1455.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-02-26 14:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0016_auto_20200226_1446'), + ] + + operations = [ + migrations.RenameField( + model_name='collectionitemrevision', + old_name='isDeletion', + new_name='deleted', + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 4577efb..1792031 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -81,7 +81,7 @@ class CollectionItemRevision(models.Model): chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') hmac = models.CharField(max_length=50, blank=False, null=False) current = models.BooleanField(db_index=True, default=True, blank=True, null=True) - isDeletion = models.BooleanField(default=False) + deleted = models.BooleanField(default=False) class Meta: unique_together = ('item', 'current') diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index a576b6d..8736a02 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -75,7 +75,7 @@ class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemRevision - fields = ('version', 'encryptionKey', 'chunks', 'hmac', 'isDeletion') + fields = ('version', 'encryptionKey', 'chunks', 'hmac', 'deleted') class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): diff --git a/django_etesync/views.py b/django_etesync/views.py index b1bb18b..3d27e72 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -112,7 +112,7 @@ class CollectionItemViewSet(BaseViewSet): # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = type(self).queryset.filter(collection__pk=collection.pk, revisions__current=True, - revisions__isDeletion=False) + revisions__deleted=False) return queryset From e0d593a9b6b8df6497f0a9c0fc222961a32a960c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 20:04:26 +0200 Subject: [PATCH 024/251] Collection Item Revision: dissalow blank for the current field. --- .../migrations/0018_auto_20200226_1803.py | 18 ++++++++++++++++++ django_etesync/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0018_auto_20200226_1803.py diff --git a/django_etesync/migrations/0018_auto_20200226_1803.py b/django_etesync/migrations/0018_auto_20200226_1803.py new file mode 100644 index 0000000..ae9200d --- /dev/null +++ b/django_etesync/migrations/0018_auto_20200226_1803.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-02-26 18:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0017_auto_20200226_1455'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemrevision', + name='current', + field=models.BooleanField(db_index=True, default=True, null=True), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 1792031..b11c36c 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -80,7 +80,7 @@ class CollectionItemRevision(models.Model): item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') hmac = models.CharField(max_length=50, blank=False, null=False) - current = models.BooleanField(db_index=True, default=True, blank=True, null=True) + current = models.BooleanField(db_index=True, default=True, null=True) deleted = models.BooleanField(default=False) class Meta: From be11e3e0e6e198460165a6466dd0a21c3c125956 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 20:38:07 +0200 Subject: [PATCH 025/251] Collection: implement collection membership. --- .../migrations/0019_collectionmember.py | 29 +++++++++++++++++++ django_etesync/models.py | 22 ++++++++++++++ django_etesync/serializers.py | 16 ++++------ 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 django_etesync/migrations/0019_collectionmember.py diff --git a/django_etesync/migrations/0019_collectionmember.py b/django_etesync/migrations/0019_collectionmember.py new file mode 100644 index 0000000..142e945 --- /dev/null +++ b/django_etesync/migrations/0019_collectionmember.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.3 on 2020-02-26 18:33 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('django_etesync', '0018_auto_20200226_1803'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionMember', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('encryptionKey', models.BinaryField(editable=True)), + ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='django_etesync.Collection')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'collection')}, + }, + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index b11c36c..3be7b06 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -88,3 +88,25 @@ class CollectionItemRevision(models.Model): def __str__(self): return '{} {} current={}'.format(self.item.uid, self.id, self.current) + + +class CollectionMember(models.Model): + class AccessLevels(models.TextChoices): + ADMIN = 'adm' + READ_WRITE = 'rw' + READ_ONLY = 'ro' + + collection = models.ForeignKey(Collection, related_name='members', on_delete=models.CASCADE) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + encryptionKey = models.BinaryField(editable=True, blank=False, null=False) + accessLevel = models.CharField( + max_length=3, + choices=AccessLevels.choices, + default=AccessLevels.READ_ONLY, + ) + + class Meta: + unique_together = ('user', 'collection') + + def __str__(self): + return '{} {}'.format(self.collection.uid, self.user) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 8736a02..c194243 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -31,29 +31,25 @@ class BinaryBase64Field(serializers.Field): class CollectionSerializer(serializers.ModelSerializer): - owner = serializers.SlugRelatedField( - slug_field=User.USERNAME_FIELD, - read_only=True - ) encryptionKey = serializers.SerializerMethodField('get_key_from_context') - permissions = serializers.SerializerMethodField('get_permission_from_context') + accessLevel = serializers.SerializerMethodField('get_access_level_from_context') ctag = serializers.SerializerMethodField('get_ctag') class Meta: model = models.Collection - fields = ('uid', 'version', 'owner', 'encryptionKey', 'permissions', 'ctag') + fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'ctag') def get_key_from_context(self, obj): request = self.context.get('request', None) if request is not None: - return 'FIXME' + return obj.members.get(user=request.user).encryptionKey return None - def get_permission_from_context(self, obj): + def get_access_level_from_context(self, obj): request = self.context.get('request', None) if request is not None: - return 'FIXME' - return 'readOnly' + return obj.members.get(user=request.user).accessLevel + return None def get_ctag(self, obj): return 'FIXME' From 3eb79e0a04444f3b3ec5b3e4110a1b9910299c81 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 20:42:28 +0200 Subject: [PATCH 026/251] Create collection member when creating collection. --- django_etesync/views.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 3d27e72..60e79a6 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -24,7 +24,7 @@ from rest_framework.decorators import action as action_decorator from rest_framework.response import Response from . import app_settings, paginators -from .models import Collection, CollectionItem +from .models import Collection, CollectionItem, CollectionMember from .serializers import ( CollectionSerializer, CollectionItemSerializer, @@ -73,7 +73,12 @@ class CollectionViewSet(BaseViewSet): if serializer.is_valid(): try: with transaction.atomic(): - serializer.save(owner=self.request.user) + col = serializer.save(owner=self.request.user) + CollectionMember(collection=col, + user=self.request.user, + accessLevel=CollectionMember.AccessLevels.ADMIN, + encryptionKey=serializer.validated_data['encryptionKey'] + ).save() except IntegrityError: content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) From 7a38e26872a9d0094e056ebe67862be3867187a0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 20:54:00 +0200 Subject: [PATCH 027/251] Collection: fix issue with encryptionKey not being base64 encoded. --- django_etesync/serializers.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index c194243..5a57900 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -30,8 +30,16 @@ class BinaryBase64Field(serializers.Field): return base64.b64decode(data) +class CollectionEncryptionKeyField(BinaryBase64Field): + def get_attribute(self, instance): + request = self.context.get('request', None) + if request is not None: + return instance.members.get(user=request.user).encryptionKey + return None + + class CollectionSerializer(serializers.ModelSerializer): - encryptionKey = serializers.SerializerMethodField('get_key_from_context') + encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') ctag = serializers.SerializerMethodField('get_ctag') @@ -39,12 +47,6 @@ class CollectionSerializer(serializers.ModelSerializer): model = models.Collection fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'ctag') - def get_key_from_context(self, obj): - request = self.context.get('request', None) - if request is not None: - return obj.members.get(user=request.user).encryptionKey - return None - def get_access_level_from_context(self, obj): request = self.context.get('request', None) if request is not None: From 771d2d013deb5891389a6901b3ce4635d73d1f3a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 21:11:29 +0200 Subject: [PATCH 028/251] Fix creation of collection membership when creating collections. --- django_etesync/serializers.py | 16 ++++++++++++++++ django_etesync/views.py | 12 +++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 5a57900..4d4558b 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -56,6 +56,22 @@ class CollectionSerializer(serializers.ModelSerializer): def get_ctag(self, obj): return 'FIXME' + def create(self, validated_data): + """Function that's called when this serializer creates an item""" + encryption_key = validated_data.pop('encryptionKey') + instance = self.__class__.Meta.model(**validated_data) + + print(validated_data) + with transaction.atomic(): + instance.save() + models.CollectionMember(collection=instance, + user=validated_data.get('owner'), + accessLevel=models.CollectionMember.AccessLevels.ADMIN, + encryptionKey=encryption_key, + ).save() + + return instance + class CollectionItemChunkSerializer(serializers.ModelSerializer): class Meta: diff --git a/django_etesync/views.py b/django_etesync/views.py index 60e79a6..9bdb244 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -13,7 +13,7 @@ # along with this program. If not, see . from django.contrib.auth import get_user_model -from django.db import IntegrityError, transaction +from django.db import IntegrityError from django.http import Http404 from django.shortcuts import get_object_or_404 @@ -24,7 +24,7 @@ from rest_framework.decorators import action as action_decorator from rest_framework.response import Response from . import app_settings, paginators -from .models import Collection, CollectionItem, CollectionMember +from .models import Collection, CollectionItem from .serializers import ( CollectionSerializer, CollectionItemSerializer, @@ -72,13 +72,7 @@ class CollectionViewSet(BaseViewSet): serializer = self.serializer_class(data=request.data) if serializer.is_valid(): try: - with transaction.atomic(): - col = serializer.save(owner=self.request.user) - CollectionMember(collection=col, - user=self.request.user, - accessLevel=CollectionMember.AccessLevels.ADMIN, - encryptionKey=serializer.validated_data['encryptionKey'] - ).save() + serializer.save(owner=self.request.user) except IntegrityError: content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) From c74ed50bd5029aebc31841912ca573af03b97aa2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 21:13:33 +0200 Subject: [PATCH 029/251] Collection: filter queryset only to collections for which the user has access to. --- django_etesync/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 9bdb244..8de7313 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -50,7 +50,8 @@ class BaseViewSet(viewsets.ModelViewSet): return serializer_class def get_collection_queryset(self, queryset=Collection.objects): - return queryset.all() + user = self.request.user + return queryset.filter(members__user=user) class CollectionViewSet(BaseViewSet): @@ -143,7 +144,7 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=True, methods=['GET']) def revision(self, request, collection_uid=None, uid=None): - col = get_object_or_404(Collection.objects, uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) col_it = get_object_or_404(col.items, uid=uid) serializer = CollectionItemRevisionSerializer(col_it.revisions.order_by('-id'), many=True) @@ -169,7 +170,8 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): lookup_field = 'uid' def get_collection_queryset(self, queryset=Collection.objects): - return queryset.all() + user = self.request.user + return queryset.filter(members__user=user) def create(self, request, collection_uid=None, collection_item_uid=None): col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) From 5ceaa9fb1ab288488b343ba0585644bdc5cdc510 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Feb 2020 21:22:58 +0200 Subject: [PATCH 030/251] Collection: calculate a value for ctag in the meanwhile. --- django_etesync/serializers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 4d4558b..f3e0fbd 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -54,7 +54,13 @@ class CollectionSerializer(serializers.ModelSerializer): return None def get_ctag(self, obj): - return 'FIXME' + # FIXME: we need to have something that's more privacy friendly + last_revision = models.CollectionItemRevision.objects.filter(item__collection=obj).last() + if last_revision is None: + # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. + return None + + return str(last_revision.id) def create(self, validated_data): """Function that's called when this serializer creates an item""" From 3beb7ac4bbf335d2b4f69568828e8d38eceade29 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 10 Mar 2020 16:27:57 +0200 Subject: [PATCH 031/251] Requirements: add pywatchman for more efficient watching. --- requirements.in/development.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.in/development.txt b/requirements.in/development.txt index 30fb558..c752bfb 100644 --- a/requirements.in/development.txt +++ b/requirements.in/development.txt @@ -1,2 +1,3 @@ coverage pip-tools +pywatchman From d587f8185bf046cd6d0bf86dd4da26dbe44b4b3f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 10 Mar 2020 16:40:42 +0200 Subject: [PATCH 032/251] Uids: change uids to be much shorter and base62 for non-chunks. --- .../migrations/0020_auto_20200310_1438.py | 29 +++++++++++++++++++ .../migrations/0021_auto_20200310_1439.py | 24 +++++++++++++++ django_etesync/models.py | 5 ++-- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 django_etesync/migrations/0020_auto_20200310_1438.py create mode 100644 django_etesync/migrations/0021_auto_20200310_1439.py diff --git a/django_etesync/migrations/0020_auto_20200310_1438.py b/django_etesync/migrations/0020_auto_20200310_1438.py new file mode 100644 index 0000000..6949145 --- /dev/null +++ b/django_etesync/migrations/0020_auto_20200310_1438.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.3 on 2020-03-10 14:38 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0019_collectionmember'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]{24}')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]{24}')]), + ), + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}=?$')]), + ), + ] diff --git a/django_etesync/migrations/0021_auto_20200310_1439.py b/django_etesync/migrations/0021_auto_20200310_1439.py new file mode 100644 index 0000000..3f1341e --- /dev/null +++ b/django_etesync/migrations/0021_auto_20200310_1439.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.3 on 2020-03-10 14:39 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0020_auto_20200310_1438'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 3be7b06..7eebd1a 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -20,7 +20,8 @@ from django.core.validators import RegexValidator from django.utils.functional import cached_property -UidValidator = RegexValidator(regex=r'[a-zA-Z0-9\-_=]{43}', message='Not a valid UID. Expected a 256bit base64url.') +Base64Url256BitValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{43}=?$', message='Expected a 256bit base64url.') +UidValidator = RegexValidator(regex=r'[a-zA-Z0-9]', message='Not a valid UID') class Collection(models.Model): @@ -61,7 +62,7 @@ def chunk_directory_path(instance, filename): class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, - max_length=44, validators=[UidValidator]) + max_length=44, validators=[Base64Url256BitValidator]) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) order = models.CharField(max_length=100, blank=False, null=False) chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) From dfbfa01bc5e7ecb45f771bfbc14cc90e8e85e8b6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 10 Mar 2020 17:49:23 +0200 Subject: [PATCH 033/251] CollectionItem: move version and encryption key to the item itself. --- .../migrations/0022_auto_20200310_1547.py | 33 +++++++++++++++++++ django_etesync/models.py | 4 +-- django_etesync/serializers.py | 9 ++--- 3 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 django_etesync/migrations/0022_auto_20200310_1547.py diff --git a/django_etesync/migrations/0022_auto_20200310_1547.py b/django_etesync/migrations/0022_auto_20200310_1547.py new file mode 100644 index 0000000..cbd0ee7 --- /dev/null +++ b/django_etesync/migrations/0022_auto_20200310_1547.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.3 on 2020-03-10 15:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0021_auto_20200310_1439'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitemrevision', + name='encryptionKey', + ), + migrations.RemoveField( + model_name='collectionitemrevision', + name='version', + ), + migrations.AddField( + model_name='collectionitem', + name='encryptionKey', + field=models.BinaryField(default=b'aoesnutheounth', editable=True), + preserve_default=False, + ), + migrations.AddField( + model_name='collectionitem', + name='version', + field=models.PositiveSmallIntegerField(default=1), + preserve_default=False, + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 7eebd1a..d996769 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -41,6 +41,8 @@ class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) collection = models.ForeignKey(Collection, related_name='items', on_delete=models.CASCADE) + version = models.PositiveSmallIntegerField() + encryptionKey = models.BinaryField(editable=True, blank=False, null=False) class Meta: unique_together = ('uid', 'collection') @@ -76,8 +78,6 @@ class CollectionItemChunk(models.Model): class CollectionItemRevision(models.Model): - version = models.PositiveSmallIntegerField() - encryptionKey = models.BinaryField(editable=True, blank=False, null=False) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') hmac = models.CharField(max_length=50, blank=False, null=False) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index f3e0fbd..c24239b 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -54,7 +54,8 @@ class CollectionSerializer(serializers.ModelSerializer): return None def get_ctag(self, obj): - # FIXME: we need to have something that's more privacy friendly + # FIXME: we need to have something that's more privacy friendly. Can probably just generate a uid per revision + # on revision creation (on the server) and just use that. last_revision = models.CollectionItemRevision.objects.filter(item__collection=obj).last() if last_revision is None: # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. @@ -86,7 +87,6 @@ class CollectionItemChunkSerializer(serializers.ModelSerializer): class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): - encryptionKey = BinaryBase64Field() chunks = serializers.SlugRelatedField( slug_field='uid', queryset=models.CollectionItemChunk.objects.all(), @@ -95,7 +95,7 @@ class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemRevision - fields = ('version', 'encryptionKey', 'chunks', 'hmac', 'deleted') + fields = ('chunks', 'hmac', 'deleted') class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): @@ -132,11 +132,12 @@ class CollectionItemRevisionInlineSerializer(CollectionItemRevisionBaseSerialize class CollectionItemSerializer(serializers.ModelSerializer): + encryptionKey = BinaryBase64Field() content = CollectionItemRevisionSerializer(many=False) class Meta: model = models.CollectionItem - fields = ('uid', 'content') + fields = ('uid', 'version', 'encryptionKey', 'content') def create(self, validated_data): """Function that's called when this serializer creates an item""" From 23edc29bb81c1ab6671fb43f57c19b1a28543767 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 10 Mar 2020 17:56:24 +0200 Subject: [PATCH 034/251] Chunks: order based on item too so items are clustered together. --- .../migrations/0023_auto_20200310_1556.py | 17 +++++++++++++++++ django_etesync/models.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0023_auto_20200310_1556.py diff --git a/django_etesync/migrations/0023_auto_20200310_1556.py b/django_etesync/migrations/0023_auto_20200310_1556.py new file mode 100644 index 0000000..e2a9b80 --- /dev/null +++ b/django_etesync/migrations/0023_auto_20200310_1556.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-03-10 15:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0022_auto_20200310_1547'), + ] + + operations = [ + migrations.AlterModelOptions( + name='collectionitemchunk', + options={'ordering': ('item', 'order')}, + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index d996769..df25867 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -71,7 +71,7 @@ class CollectionItemChunk(models.Model): class Meta: unique_together = ('item', 'order') - ordering = ['order'] + ordering = ('item', 'order') def __str__(self): return self.uid From f8a94eeb04bec54e92c753aeff11dbed51ae7e6d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 12 Mar 2020 15:52:36 +0200 Subject: [PATCH 035/251] Revision: add a proper uid for revisions (which we also use for sync tag). --- .../0024_collectionitemrevision_uid.py | 19 +++++++++++++++++++ .../migrations/0025_auto_20200312_1350.py | 19 +++++++++++++++++++ django_etesync/models.py | 4 +++- django_etesync/serializers.py | 15 ++++++++++----- 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 django_etesync/migrations/0024_collectionitemrevision_uid.py create mode 100644 django_etesync/migrations/0025_auto_20200312_1350.py diff --git a/django_etesync/migrations/0024_collectionitemrevision_uid.py b/django_etesync/migrations/0024_collectionitemrevision_uid.py new file mode 100644 index 0000000..6134c89 --- /dev/null +++ b/django_etesync/migrations/0024_collectionitemrevision_uid.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-12 13:41 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0023_auto_20200310_1556'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemrevision', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), + ), + ] diff --git a/django_etesync/migrations/0025_auto_20200312_1350.py b/django_etesync/migrations/0025_auto_20200312_1350.py new file mode 100644 index 0000000..b54aeff --- /dev/null +++ b/django_etesync/migrations/0025_auto_20200312_1350.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-12 13:50 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0024_collectionitemrevision_uid'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemrevision', + name='uid', + field=models.CharField(db_index=True, max_length=44, unique=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index df25867..8079c28 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -78,6 +78,8 @@ class CollectionItemChunk(models.Model): class CollectionItemRevision(models.Model): + uid = models.CharField(db_index=True, unique=True, blank=False, null=False, + max_length=44, validators=[UidValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') hmac = models.CharField(max_length=50, blank=False, null=False) @@ -88,7 +90,7 @@ class CollectionItemRevision(models.Model): unique_together = ('item', 'current') def __str__(self): - return '{} {} current={}'.format(self.item.uid, self.id, self.current) + return '{} {} current={}'.format(self.uid, self.item.uid, self.current) class CollectionMember(models.Model): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index c24239b..c6d3a86 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -16,12 +16,17 @@ import base64 from django.contrib.auth import get_user_model from django.db import transaction +from django.utils.crypto import get_random_string from rest_framework import serializers from . import models User = get_user_model() +def generate_rev_uid(length=32): + return get_random_string(length) + + class BinaryBase64Field(serializers.Field): def to_representation(self, value): return base64.b64encode(value).decode('ascii') @@ -54,14 +59,12 @@ class CollectionSerializer(serializers.ModelSerializer): return None def get_ctag(self, obj): - # FIXME: we need to have something that's more privacy friendly. Can probably just generate a uid per revision - # on revision creation (on the server) and just use that. last_revision = models.CollectionItemRevision.objects.filter(item__collection=obj).last() if last_revision is None: # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. return None - return str(last_revision.id) + return last_revision.uid def create(self, validated_data): """Function that's called when this serializer creates an item""" @@ -148,7 +151,8 @@ class CollectionItemSerializer(serializers.ModelSerializer): instance.save() chunks = revision_data.pop('chunks') - revision = models.CollectionItemRevision.objects.create(**revision_data, item=instance) + revision = models.CollectionItemRevision.objects.create(**revision_data, uid=generate_rev_uid(), + item=instance) revision.chunks.set(chunks) return instance @@ -165,7 +169,8 @@ class CollectionItemSerializer(serializers.ModelSerializer): current_revision.save() chunks = revision_data.pop('chunks') - revision = models.CollectionItemRevision.objects.create(**revision_data, item=instance) + revision = models.CollectionItemRevision.objects.create(**revision_data, uid=generate_rev_uid(), + item=instance) revision.chunks.set(chunks) return instance From d1df6db8b16d3a544568b344ca70fffaa2e2c994 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 12 Mar 2020 16:02:00 +0200 Subject: [PATCH 036/251] Revision: add metadata field. --- .../0026_collectionitemrevision_meta.py | 18 ++++++++++++++++++ django_etesync/models.py | 1 + 2 files changed, 19 insertions(+) create mode 100644 django_etesync/migrations/0026_collectionitemrevision_meta.py diff --git a/django_etesync/migrations/0026_collectionitemrevision_meta.py b/django_etesync/migrations/0026_collectionitemrevision_meta.py new file mode 100644 index 0000000..8056e61 --- /dev/null +++ b/django_etesync/migrations/0026_collectionitemrevision_meta.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-12 14:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0025_auto_20200312_1350'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemrevision', + name='meta', + field=models.BinaryField(blank=True, editable=True, null=True), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 8079c28..02a173c 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -81,6 +81,7 @@ class CollectionItemRevision(models.Model): uid = models.CharField(db_index=True, unique=True, blank=False, null=False, max_length=44, validators=[UidValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) + meta = models.BinaryField(editable=True, blank=True, null=True) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') hmac = models.CharField(max_length=50, blank=False, null=False) current = models.BooleanField(db_index=True, default=True, null=True) From c56cbb3f8209bd79c4ac8e0273c54e43a41f064e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 12 Mar 2020 16:06:15 +0200 Subject: [PATCH 037/251] Remove debug print. --- django_etesync/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index c6d3a86..2c12592 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -71,7 +71,6 @@ class CollectionSerializer(serializers.ModelSerializer): encryption_key = validated_data.pop('encryptionKey') instance = self.__class__.Meta.model(**validated_data) - print(validated_data) with transaction.atomic(): instance.save() models.CollectionMember(collection=instance, From 66e5062461398b0cbec6f4fa28b73f92577fe73e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 12 Mar 2020 21:02:27 +0200 Subject: [PATCH 038/251] Collection: add content support. --- .../migrations/0027_collection_mainitem.py | 19 ++++ .../migrations/0028_auto_20200312_1819.py | 24 +++++ .../migrations/0029_auto_20200312_1849.py | 31 +++++++ .../migrations/0030_auto_20200312_1859.py | 19 ++++ django_etesync/models.py | 11 ++- django_etesync/serializers.py | 87 +++++++++++-------- 6 files changed, 154 insertions(+), 37 deletions(-) create mode 100644 django_etesync/migrations/0027_collection_mainitem.py create mode 100644 django_etesync/migrations/0028_auto_20200312_1819.py create mode 100644 django_etesync/migrations/0029_auto_20200312_1849.py create mode 100644 django_etesync/migrations/0030_auto_20200312_1859.py diff --git a/django_etesync/migrations/0027_collection_mainitem.py b/django_etesync/migrations/0027_collection_mainitem.py new file mode 100644 index 0000000..b420d8f --- /dev/null +++ b/django_etesync/migrations/0027_collection_mainitem.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-12 14:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0026_collectionitemrevision_meta'), + ] + + operations = [ + migrations.AddField( + model_name='collection', + name='mainItem', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='of_collection', to='django_etesync.CollectionItem'), + ), + ] diff --git a/django_etesync/migrations/0028_auto_20200312_1819.py b/django_etesync/migrations/0028_auto_20200312_1819.py new file mode 100644 index 0000000..6d76499 --- /dev/null +++ b/django_etesync/migrations/0028_auto_20200312_1819.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.3 on 2020-03-12 18:19 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0027_collection_mainitem'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitem', + name='encryptionKey', + field=models.BinaryField(editable=True, null=True), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=44, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), + ), + ] diff --git a/django_etesync/migrations/0029_auto_20200312_1849.py b/django_etesync/migrations/0029_auto_20200312_1849.py new file mode 100644 index 0000000..165b405 --- /dev/null +++ b/django_etesync/migrations/0029_auto_20200312_1849.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.3 on 2020-03-12 18:49 + +from django.db import migrations +from django_etesync.serializers import generate_rev_uid + + +def add_collection_main_item(apps, schema_editor): + Collection = apps.get_model('django_etesync', 'Collection') + CollectionItem = apps.get_model('django_etesync', 'CollectionItem') + CollectionItemRevision = apps.get_model('django_etesync', 'CollectionItemRevision') + + for col in Collection.objects.all(): + main_item = CollectionItem.objects.create(uid=None, encryptionKey=None, version=col.version, collection=col) + col.mainItem = main_item + col.save() + + CollectionItemRevision.objects.create( + uid=generate_rev_uid(), + hmac='hmac-hash', + item=main_item) + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0028_auto_20200312_1819'), + ] + + operations = [ + migrations.RunPython(add_collection_main_item), + ] diff --git a/django_etesync/migrations/0030_auto_20200312_1859.py b/django_etesync/migrations/0030_auto_20200312_1859.py new file mode 100644 index 0000000..fe8050a --- /dev/null +++ b/django_etesync/migrations/0030_auto_20200312_1859.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-12 18:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0029_auto_20200312_1849'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='mainItem', + field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='of_collection', to='django_etesync.CollectionItem'), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 02a173c..6c30b00 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -29,6 +29,7 @@ class Collection(models.Model): max_length=44, validators=[UidValidator]) version = models.PositiveSmallIntegerField() owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + mainItem = models.OneToOneField('CollectionItem', related_name='of_collection', on_delete=models.PROTECT) class Meta: unique_together = ('uid', 'owner') @@ -36,19 +37,23 @@ class Collection(models.Model): def __str__(self): return self.uid + @cached_property + def content(self): + return self.mainItem.content + class CollectionItem(models.Model): - uid = models.CharField(db_index=True, blank=False, null=False, + uid = models.CharField(db_index=True, blank=False, null=True, max_length=44, validators=[UidValidator]) collection = models.ForeignKey(Collection, related_name='items', on_delete=models.CASCADE) version = models.PositiveSmallIntegerField() - encryptionKey = models.BinaryField(editable=True, blank=False, null=False) + encryptionKey = models.BinaryField(editable=True, blank=False, null=True) class Meta: unique_together = ('uid', 'collection') def __str__(self): - return self.uid + return '{} {}'.format(self.uid, self.collection.uid) @cached_property def content(self): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 2c12592..7a2d3fd 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -43,44 +43,13 @@ class CollectionEncryptionKeyField(BinaryBase64Field): return None -class CollectionSerializer(serializers.ModelSerializer): - encryptionKey = CollectionEncryptionKeyField() - accessLevel = serializers.SerializerMethodField('get_access_level_from_context') - ctag = serializers.SerializerMethodField('get_ctag') - - class Meta: - model = models.Collection - fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'ctag') - - def get_access_level_from_context(self, obj): +class CollectionContentField(BinaryBase64Field): + def get_attribute(self, instance): request = self.context.get('request', None) if request is not None: - return obj.members.get(user=request.user).accessLevel + return instance.members.get(user=request.user).encryptionKey return None - def get_ctag(self, obj): - last_revision = models.CollectionItemRevision.objects.filter(item__collection=obj).last() - if last_revision is None: - # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. - return None - - return last_revision.uid - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - encryption_key = validated_data.pop('encryptionKey') - instance = self.__class__.Meta.model(**validated_data) - - with transaction.atomic(): - instance.save() - models.CollectionMember(collection=instance, - user=validated_data.get('owner'), - accessLevel=models.CollectionMember.AccessLevels.ADMIN, - encryptionKey=encryption_key, - ).save() - - return instance - class CollectionItemChunkSerializer(serializers.ModelSerializer): class Meta: @@ -177,3 +146,53 @@ class CollectionItemSerializer(serializers.ModelSerializer): class CollectionItemInlineSerializer(CollectionItemSerializer): content = CollectionItemRevisionInlineSerializer(read_only=True, many=False) + + +class CollectionSerializer(serializers.ModelSerializer): + encryptionKey = CollectionEncryptionKeyField() + accessLevel = serializers.SerializerMethodField('get_access_level_from_context') + ctag = serializers.SerializerMethodField('get_ctag') + content = CollectionItemRevisionSerializer(many=False) + + class Meta: + model = models.Collection + fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'ctag') + + def get_access_level_from_context(self, obj): + request = self.context.get('request', None) + if request is not None: + return obj.members.get(user=request.user).accessLevel + return None + + def get_ctag(self, obj): + last_revision = models.CollectionItemRevision.objects.filter(item__collection=obj).last() + if last_revision is None: + # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. + return None + + return last_revision.uid + + def create(self, validated_data): + """Function that's called when this serializer creates an item""" + revision_data = validated_data.pop('content') + encryption_key = validated_data.pop('encryptionKey') + instance = self.__class__.Meta.model(**validated_data) + + with transaction.atomic(): + main_item = models.CollectionItem.objects.create( + uid=None, encryptionKey=None, version=instance.version, collection=instance) + instance.mainItem = main_item + + chunks = revision_data.pop('chunks') + revision = models.CollectionItemRevision.objects.create(**revision_data, uid=generate_rev_uid(), + item=main_item) + revision.chunks.set(chunks) + + instance.save() + models.CollectionMember(collection=instance, + user=validated_data.get('owner'), + accessLevel=models.CollectionMember.AccessLevels.ADMIN, + encryptionKey=encryption_key, + ).save() + + return instance From 86b6a449176bbfeda363d25b467777cb3013729a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 17 Mar 2020 17:10:53 +0200 Subject: [PATCH 039/251] We use base64url without padding. --- .../migrations/0031_auto_20200317_1509.py | 19 +++++++++++++++++++ django_etesync/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0031_auto_20200317_1509.py diff --git a/django_etesync/migrations/0031_auto_20200317_1509.py b/django_etesync/migrations/0031_auto_20200317_1509.py new file mode 100644 index 0000000..7166781 --- /dev/null +++ b/django_etesync/migrations/0031_auto_20200317_1509.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-17 15:09 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0030_auto_20200312_1859'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')]), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 6c30b00..8df5d44 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -20,7 +20,7 @@ from django.core.validators import RegexValidator from django.utils.functional import cached_property -Base64Url256BitValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{43}=?$', message='Expected a 256bit base64url.') +Base64Url256BitValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{43}$', message='Expected a 256bit base64url.') UidValidator = RegexValidator(regex=r'[a-zA-Z0-9]', message='Not a valid UID') From ab86a912cd8d0ecb3dbe071a9ad53a5a984e626a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 17 Mar 2020 17:13:47 +0200 Subject: [PATCH 040/251] Revision uid is now the hmac. --- .../migrations/0032_auto_20200317_1513.py | 23 +++++++++++++++++++ django_etesync/models.py | 3 +-- django_etesync/serializers.py | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 django_etesync/migrations/0032_auto_20200317_1513.py diff --git a/django_etesync/migrations/0032_auto_20200317_1513.py b/django_etesync/migrations/0032_auto_20200317_1513.py new file mode 100644 index 0000000..0546711 --- /dev/null +++ b/django_etesync/migrations/0032_auto_20200317_1513.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-03-17 15:13 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0031_auto_20200317_1509'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitemrevision', + name='hmac', + ), + migrations.AlterField( + model_name='collectionitemrevision', + name='uid', + field=models.CharField(db_index=True, max_length=44, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')]), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 8df5d44..586c105 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -84,11 +84,10 @@ class CollectionItemChunk(models.Model): class CollectionItemRevision(models.Model): uid = models.CharField(db_index=True, unique=True, blank=False, null=False, - max_length=44, validators=[UidValidator]) + max_length=44, validators=[Base64Url256BitValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) meta = models.BinaryField(editable=True, blank=True, null=True) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') - hmac = models.CharField(max_length=50, blank=False, null=False) current = models.BooleanField(db_index=True, default=True, null=True) deleted = models.BooleanField(default=False) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 7a2d3fd..33a09ac 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -66,7 +66,7 @@ class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemRevision - fields = ('chunks', 'hmac', 'deleted') + fields = ('chunks', 'uid', 'deleted') class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): From 80ef568397bf790fae6f7a77311c23cff184f1a3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 17 Mar 2020 22:10:33 +0200 Subject: [PATCH 041/251] Make meta not-null. --- .../migrations/0033_auto_20200317_2010.py | 19 +++++++++++++++++++ django_etesync/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0033_auto_20200317_2010.py diff --git a/django_etesync/migrations/0033_auto_20200317_2010.py b/django_etesync/migrations/0033_auto_20200317_2010.py new file mode 100644 index 0000000..7a42b38 --- /dev/null +++ b/django_etesync/migrations/0033_auto_20200317_2010.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-17 20:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0032_auto_20200317_1513'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemrevision', + name='meta', + field=models.BinaryField(default=b'', editable=True), + preserve_default=False, + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 586c105..ca3dea9 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -86,7 +86,7 @@ class CollectionItemRevision(models.Model): uid = models.CharField(db_index=True, unique=True, blank=False, null=False, max_length=44, validators=[Base64Url256BitValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) - meta = models.BinaryField(editable=True, blank=True, null=True) + meta = models.BinaryField(editable=True, blank=False, null=False) chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') current = models.BooleanField(db_index=True, default=True, null=True) deleted = models.BooleanField(default=False) From 2ac0b55de971dba99e2c0355687a4298761a36e9 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 17 Mar 2020 22:11:18 +0200 Subject: [PATCH 042/251] Revision: expose meta. --- django_etesync/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 33a09ac..127d88c 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -66,7 +66,7 @@ class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemRevision - fields = ('chunks', 'uid', 'deleted') + fields = ('chunks', 'meta', 'uid', 'deleted') class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): From 9b13404ce74c8953432759e83395a1bffcf4e42d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 14 Apr 2020 16:21:51 +0300 Subject: [PATCH 043/251] Add a reset view for tests. --- django_etesync/views.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 8de7313..a4b8108 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -12,9 +12,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from django.conf import settings from django.contrib.auth import get_user_model from django.db import IntegrityError -from django.http import Http404 +from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 from rest_framework import status @@ -206,3 +207,26 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): # FIXME: DO NOT USE! Use django-send file or etc instead. return serve(request, basename, dirname) + + +class ResetViewSet(BaseViewSet): + allowed_methods = ['POST'] + + def post(self, request, *args, **kwargs): + # Only run when in DEBUG mode! It's only used for tests + if not settings.DEBUG: + return HttpResponseBadRequest("Only allowed in debug mode.") + + # Only allow local users, for extra safety + if not getattr(request.user, User.USERNAME_FIELD).endswith('@localhost'): + return HttpResponseBadRequest("Endpoint not allowed for user.") + + # Delete all of the journal data for this user for a clear test env + request.user.collection_set.all().delete() + + # FIXME: also delete chunk files!!! + + return HttpResponse() + + +reset = ResetViewSet.as_view({'post': 'post'}) From cf06534d6d9ea70f9fa1183a5c6887364e20529e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 14 Apr 2020 18:29:56 +0300 Subject: [PATCH 044/251] Serializers: handle our variant of b64 (no padding, urlsafe). --- django_etesync/serializers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 127d88c..37b119e 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -29,10 +29,11 @@ def generate_rev_uid(length=32): class BinaryBase64Field(serializers.Field): def to_representation(self, value): - return base64.b64encode(value).decode('ascii') + return base64.urlsafe_b64encode(value).decode('ascii') def to_internal_value(self, data): - return base64.b64decode(data) + data += "=" * ((4 - len(data) % 4) % 4) + return base64.urlsafe_b64decode(data) class CollectionEncryptionKeyField(BinaryBase64Field): From a97bb969e78114982f0bb82010148fcf9b71312b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 14 Apr 2020 18:30:07 +0300 Subject: [PATCH 045/251] Make meta a binary base64 field too. --- django_etesync/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 37b119e..f0bb1c3 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -64,6 +64,7 @@ class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): queryset=models.CollectionItemChunk.objects.all(), many=True ) + meta = BinaryBase64Field() class Meta: model = models.CollectionItemRevision From 5dfa2ac8cbab2f0aa62158ff0f80e55726f7c898 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 14:33:38 +0300 Subject: [PATCH 046/251] Make chunks use the same b64 encoding we use elsewhere. --- django_etesync/serializers.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index f0bb1c3..2310c8d 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -27,13 +27,21 @@ def generate_rev_uid(length=32): return get_random_string(length) +def b64encode(value): + return base64.urlsafe_b64encode(value).decode('ascii') + + +def b64decode(data): + data += "=" * ((4 - len(data) % 4) % 4) + return base64.urlsafe_b64decode(data) + + class BinaryBase64Field(serializers.Field): def to_representation(self, value): - return base64.urlsafe_b64encode(value).decode('ascii') + return b64encode(value) def to_internal_value(self, data): - data += "=" * ((4 - len(data) % 4) % 4) - return base64.urlsafe_b64decode(data) + return b64decode(data) class CollectionEncryptionKeyField(BinaryBase64Field): @@ -99,7 +107,7 @@ class CollectionItemRevisionInlineSerializer(CollectionItemRevisionBaseSerialize ret = [] for chunk in obj.chunks.all(): with open(chunk.chunkFile.path, 'rb') as f: - ret.append(base64.b64encode(f.read()).decode('ascii')) + ret.append(b64encode(f.read())) return ret From 963dc3c62df8457a64492b15aeba03b71e97009e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 15:23:07 +0300 Subject: [PATCH 047/251] Cleanup how we handle inline serializers. --- django_etesync/serializers.py | 42 ++++++++++++++++++----------------- django_etesync/views.py | 23 +++++++++++-------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 2310c8d..6a495bf 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -66,44 +66,40 @@ class CollectionItemChunkSerializer(serializers.ModelSerializer): fields = ('uid', 'chunkFile') -class CollectionItemRevisionBaseSerializer(serializers.ModelSerializer): +class CollectionItemRevisionSerializer(serializers.ModelSerializer): chunks = serializers.SlugRelatedField( slug_field='uid', queryset=models.CollectionItemChunk.objects.all(), many=True ) + chunksUrls = serializers.SerializerMethodField('get_chunks_urls') + chunksData = serializers.SerializerMethodField('get_chunks_data') meta = BinaryBase64Field() class Meta: model = models.CollectionItemRevision - fields = ('chunks', 'meta', 'uid', 'deleted') - - -class CollectionItemRevisionSerializer(CollectionItemRevisionBaseSerializer): - chunksUrls = serializers.SerializerMethodField('get_chunks_urls') - - class Meta(CollectionItemRevisionBaseSerializer.Meta): - fields = CollectionItemRevisionBaseSerializer.Meta.fields + ('chunksUrls', ) + fields = ('chunks', 'meta', 'uid', 'deleted', 'chunksUrls', 'chunksData') # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still # save it under the user. # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot. # E.g. chunkBaseUrl: "/media/bla/bla/" or chunkBaseUrl: "https://media.etesync.com/bla/bla" def get_chunks_urls(self, obj): + prefer_inline = self.context.get('prefer_inline', False) + if prefer_inline: + return None + ret = [] for chunk in obj.chunks.all(): ret.append(chunk.chunkFile.url) return ret - -class CollectionItemRevisionInlineSerializer(CollectionItemRevisionBaseSerializer): - chunksData = serializers.SerializerMethodField('get_chunks_data') - - class Meta(CollectionItemRevisionBaseSerializer.Meta): - fields = CollectionItemRevisionBaseSerializer.Meta.fields + ('chunksData', ) - def get_chunks_data(self, obj): + prefer_inline = self.context.get('prefer_inline', False) + if not prefer_inline: + return None + ret = [] for chunk in obj.chunks.all(): with open(chunk.chunkFile.path, 'rb') as f: @@ -111,6 +107,16 @@ class CollectionItemRevisionInlineSerializer(CollectionItemRevisionBaseSerialize return ret + def to_representation(self, instance): + ret = super().to_representation(instance) + prefer_inline = self.context.get('prefer_inline', False) + if prefer_inline: + ret.pop('chunksUrls') + else: + ret.pop('chunksData') + + return ret + class CollectionItemSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() @@ -154,10 +160,6 @@ class CollectionItemSerializer(serializers.ModelSerializer): return instance -class CollectionItemInlineSerializer(CollectionItemSerializer): - content = CollectionItemRevisionInlineSerializer(read_only=True, many=False) - - class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') diff --git a/django_etesync/views.py b/django_etesync/views.py index a4b8108..37de4bb 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -29,7 +29,6 @@ from .models import Collection, CollectionItem from .serializers import ( CollectionSerializer, CollectionItemSerializer, - CollectionItemInlineSerializer, CollectionItemRevisionSerializer, CollectionItemChunkSerializer ) @@ -66,12 +65,18 @@ class CollectionViewSet(BaseViewSet): queryset = type(self).queryset return self.get_collection_queryset(queryset) + def get_serializer_context(self): + context = super().get_serializer_context() + prefer_inline = self.request.method == 'GET' and 'prefer_inline' in self.request.query_params + context.update({'request': self.request, 'prefer_inline': prefer_inline}) + return context + def destroy(self, request, uid=None): # FIXME: implement return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def create(self, request, *args, **kwargs): - serializer = self.serializer_class(data=request.data) + serializer = self.serializer_class(data=request.data, context=self.get_serializer_context()) if serializer.is_valid(): try: serializer.save(owner=self.request.user) @@ -86,7 +91,7 @@ class CollectionViewSet(BaseViewSet): def list(self, request): queryset = self.get_queryset() - serializer = self.serializer_class(queryset, context={'request': request}, many=True) + serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) return Response(serializer.data) @@ -98,12 +103,6 @@ class CollectionItemViewSet(BaseViewSet): pagination_class = paginators.LinkHeaderPagination lookup_field = 'uid' - def get_serializer_class(self): - if self.request.method == 'GET' and self.request.query_params.get('prefer_inline'): - return CollectionItemInlineSerializer - - return super().get_serializer_class() - def get_queryset(self): collection_uid = self.kwargs['collection_uid'] try: @@ -117,6 +116,12 @@ class CollectionItemViewSet(BaseViewSet): return queryset + def get_serializer_context(self): + context = super().get_serializer_context() + prefer_inline = self.request.method == 'GET' and 'prefer_inline' in self.request.query_params + context.update({'request': self.request, 'prefer_inline': prefer_inline}) + return context + def create(self, request, collection_uid=None): collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) From c589d06cbe1283e42d6c951530d547c32ce1e730 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 16:00:06 +0300 Subject: [PATCH 048/251] Collection: lax the restrictions on mainItem. --- .../migrations/0034_auto_20200415_1248.py | 19 +++++++++++++++++++ .../migrations/0035_auto_20200415_1259.py | 19 +++++++++++++++++++ django_etesync/models.py | 2 +- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0034_auto_20200415_1248.py create mode 100644 django_etesync/migrations/0035_auto_20200415_1259.py diff --git a/django_etesync/migrations/0034_auto_20200415_1248.py b/django_etesync/migrations/0034_auto_20200415_1248.py new file mode 100644 index 0000000..1156676 --- /dev/null +++ b/django_etesync/migrations/0034_auto_20200415_1248.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-04-15 12:48 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0033_auto_20200317_2010'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='mainItem', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='of_collection', to='django_etesync.CollectionItem'), + ), + ] diff --git a/django_etesync/migrations/0035_auto_20200415_1259.py b/django_etesync/migrations/0035_auto_20200415_1259.py new file mode 100644 index 0000000..d558e31 --- /dev/null +++ b/django_etesync/migrations/0035_auto_20200415_1259.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-04-15 12:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0034_auto_20200415_1248'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='mainItem', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='of_collection', to='django_etesync.CollectionItem'), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index ca3dea9..65f130a 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -29,7 +29,7 @@ class Collection(models.Model): max_length=44, validators=[UidValidator]) version = models.PositiveSmallIntegerField() owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - mainItem = models.OneToOneField('CollectionItem', related_name='of_collection', on_delete=models.PROTECT) + mainItem = models.OneToOneField('CollectionItem', related_name='of_collection', null=True, on_delete=models.SET_NULL) class Meta: unique_together = ('uid', 'owner') From 6711cfcf493a0e5dc65ae96703b4ea5ac59266fc Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 16:27:03 +0300 Subject: [PATCH 049/251] Change chunks to be just one field. --- django_etesync/serializers.py | 57 +++++++++-------------------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 6a495bf..5946293 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -60,6 +60,19 @@ class CollectionContentField(BinaryBase64Field): return None +class ChunksField(serializers.RelatedField): + def to_representation(self, obj): + prefer_inline = self.context.get('prefer_inline', False) + if prefer_inline: + with open(obj.chunkFile.path, 'rb') as f: + return (obj.uid, b64encode(f.read())) + else: + return (obj.uid, ) + + def to_internal_value(self, data): + return (data[0], b64decode(data[1])) + + class CollectionItemChunkSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemChunk @@ -67,55 +80,15 @@ class CollectionItemChunkSerializer(serializers.ModelSerializer): class CollectionItemRevisionSerializer(serializers.ModelSerializer): - chunks = serializers.SlugRelatedField( - slug_field='uid', + chunks = ChunksField( queryset=models.CollectionItemChunk.objects.all(), many=True ) - chunksUrls = serializers.SerializerMethodField('get_chunks_urls') - chunksData = serializers.SerializerMethodField('get_chunks_data') meta = BinaryBase64Field() class Meta: model = models.CollectionItemRevision - fields = ('chunks', 'meta', 'uid', 'deleted', 'chunksUrls', 'chunksData') - - # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still - # save it under the user. - # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot. - # E.g. chunkBaseUrl: "/media/bla/bla/" or chunkBaseUrl: "https://media.etesync.com/bla/bla" - def get_chunks_urls(self, obj): - prefer_inline = self.context.get('prefer_inline', False) - if prefer_inline: - return None - - ret = [] - for chunk in obj.chunks.all(): - ret.append(chunk.chunkFile.url) - - return ret - - def get_chunks_data(self, obj): - prefer_inline = self.context.get('prefer_inline', False) - if not prefer_inline: - return None - - ret = [] - for chunk in obj.chunks.all(): - with open(chunk.chunkFile.path, 'rb') as f: - ret.append(b64encode(f.read())) - - return ret - - def to_representation(self, instance): - ret = super().to_representation(instance) - prefer_inline = self.context.get('prefer_inline', False) - if prefer_inline: - ret.pop('chunksUrls') - else: - ret.pop('chunksData') - - return ret + fields = ('chunks', 'meta', 'uid', 'deleted') class CollectionItemSerializer(serializers.ModelSerializer): From 2e018dfe7672e6fd4813c410f0f0e386b392ead8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 16:47:31 +0300 Subject: [PATCH 050/251] Rename prefer_inline to inline. --- django_etesync/serializers.py | 4 ++-- django_etesync/views.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 5946293..bc04bc0 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -62,8 +62,8 @@ class CollectionContentField(BinaryBase64Field): class ChunksField(serializers.RelatedField): def to_representation(self, obj): - prefer_inline = self.context.get('prefer_inline', False) - if prefer_inline: + inline = self.context.get('inline', False) + if inline: with open(obj.chunkFile.path, 'rb') as f: return (obj.uid, b64encode(f.read())) else: diff --git a/django_etesync/views.py b/django_etesync/views.py index 37de4bb..1c9da78 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -67,8 +67,8 @@ class CollectionViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - prefer_inline = self.request.method == 'GET' and 'prefer_inline' in self.request.query_params - context.update({'request': self.request, 'prefer_inline': prefer_inline}) + inline = self.request.method == 'GET' and 'inline' in self.request.query_params + context.update({'request': self.request, 'inline': inline}) return context def destroy(self, request, uid=None): @@ -118,8 +118,8 @@ class CollectionItemViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - prefer_inline = self.request.method == 'GET' and 'prefer_inline' in self.request.query_params - context.update({'request': self.request, 'prefer_inline': prefer_inline}) + inline = self.request.method == 'GET' and 'inline' in self.request.query_params + context.update({'request': self.request, 'inline': inline}) return context def create(self, request, collection_uid=None): From 3db204e4bb52163e3bc2ddd2063fd858fb9ec3b0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 16:50:47 +0300 Subject: [PATCH 051/251] b64: don't add redundant padding. --- django_etesync/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index bc04bc0..1b87a86 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -28,7 +28,7 @@ def generate_rev_uid(length=32): def b64encode(value): - return base64.urlsafe_b64encode(value).decode('ascii') + return base64.urlsafe_b64encode(value).decode('ascii').strip('=') def b64decode(data): From 6dfa2360c090e6f0768b63146d108dae91d23293 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 16:52:36 +0300 Subject: [PATCH 052/251] Chunk: fix for a collection's main item. --- django_etesync/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index 65f130a..fdf8041 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -64,7 +64,8 @@ def chunk_directory_path(instance, filename): item = instance.item col = item.collection user_id = col.owner.id - return Path('user_{}'.format(user_id), col.uid, item.uid, instance.uid) + item_uid = item.uid or 'main' + return Path('user_{}'.format(user_id), col.uid, item_uid, instance.uid) class CollectionItemChunk(models.Model): From ab9d66fcc0da829ce3bed28d1d436ea48206a168 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 16:53:31 +0300 Subject: [PATCH 053/251] Implement collection creation. --- django_etesync/serializers.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 1b87a86..4d8a206 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -14,6 +14,7 @@ import base64 +from django.core.files.base import ContentFile from django.contrib.auth import get_user_model from django.db import transaction from django.utils.crypto import get_random_string @@ -164,14 +165,28 @@ class CollectionSerializer(serializers.ModelSerializer): instance = self.__class__.Meta.model(**validated_data) with transaction.atomic(): + instance.save() main_item = models.CollectionItem.objects.create( uid=None, encryptionKey=None, version=instance.version, collection=instance) instance.mainItem = main_item + chunks_ids = [] chunks = revision_data.pop('chunks') - revision = models.CollectionItemRevision.objects.create(**revision_data, uid=generate_rev_uid(), - item=main_item) - revision.chunks.set(chunks) + for chunk in chunks: + uid = chunk[0] + if len(chunk) > 1: + content = chunk[1] + # FIXME: fix order! + chunk = models.CollectionItemChunk(uid=uid, item=main_item, order='abc') + chunk.chunkFile.save('IGNORED', ContentFile(content)) + chunk.save() + chunks_ids.append(chunk.id) + else: + chunk = models.CollectionItemChunk.objects.get(uid=uid) + chunks_ids.append(chunk.id) + + revision = models.CollectionItemRevision.objects.create(**revision_data, item=main_item) + revision.chunks.set(chunks_ids) instance.save() models.CollectionMember(collection=instance, From 7a0a00c738f9795324e7610102fef468a0e63ef8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 16:59:30 +0300 Subject: [PATCH 054/251] Unify how we handle revision creation. --- django_etesync/serializers.py | 49 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 4d8a206..60c7560 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -28,6 +28,27 @@ def generate_rev_uid(length=32): return get_random_string(length) +def process_revisions_for_item(item, revision_data): + chunks_ids = [] + chunks = revision_data.pop('chunks') + for chunk in chunks: + uid = chunk[0] + if len(chunk) > 1: + content = chunk[1] + # FIXME: fix order! + chunk = models.CollectionItemChunk(uid=uid, item=item, order='abc') + chunk.chunkFile.save('IGNORED', ContentFile(content)) + chunk.save() + chunks_ids.append(chunk.id) + else: + chunk = models.CollectionItemChunk.objects.get(uid=uid) + chunks_ids.append(chunk.id) + + revision = models.CollectionItemRevision.objects.create(**revision_data, item=item) + revision.chunks.set(chunks_ids) + return revision + + def b64encode(value): return base64.urlsafe_b64encode(value).decode('ascii').strip('=') @@ -108,10 +129,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): with transaction.atomic(): instance.save() - chunks = revision_data.pop('chunks') - revision = models.CollectionItemRevision.objects.create(**revision_data, uid=generate_rev_uid(), - item=instance) - revision.chunks.set(chunks) + process_revisions_for_item(instance, revision_data) return instance @@ -126,10 +144,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): current_revision.current = None current_revision.save() - chunks = revision_data.pop('chunks') - revision = models.CollectionItemRevision.objects.create(**revision_data, uid=generate_rev_uid(), - item=instance) - revision.chunks.set(chunks) + process_revisions_for_item(instance, revision_data) return instance @@ -170,23 +185,7 @@ class CollectionSerializer(serializers.ModelSerializer): uid=None, encryptionKey=None, version=instance.version, collection=instance) instance.mainItem = main_item - chunks_ids = [] - chunks = revision_data.pop('chunks') - for chunk in chunks: - uid = chunk[0] - if len(chunk) > 1: - content = chunk[1] - # FIXME: fix order! - chunk = models.CollectionItemChunk(uid=uid, item=main_item, order='abc') - chunk.chunkFile.save('IGNORED', ContentFile(content)) - chunk.save() - chunks_ids.append(chunk.id) - else: - chunk = models.CollectionItemChunk.objects.get(uid=uid) - chunks_ids.append(chunk.id) - - revision = models.CollectionItemRevision.objects.create(**revision_data, item=main_item) - revision.chunks.set(chunks_ids) + process_revisions_for_item(main_item, revision_data) instance.save() models.CollectionMember(collection=instance, From 62a7496b66b83284fd0dad4d59042149c99720bf Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 17:35:51 +0300 Subject: [PATCH 055/251] Change how we handle chunk ordering (and relation). --- .../migrations/0036_auto_20200415_1420.py | 37 +++++++++++++++++++ .../migrations/0037_auto_20200415_1421.py | 23 ++++++++++++ ...38_remove_collectionitemrevision_chunks.py | 17 +++++++++ django_etesync/models.py | 14 ++++--- django_etesync/serializers.py | 18 +++++---- django_etesync/views.py | 4 +- 6 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 django_etesync/migrations/0036_auto_20200415_1420.py create mode 100644 django_etesync/migrations/0037_auto_20200415_1421.py create mode 100644 django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py diff --git a/django_etesync/migrations/0036_auto_20200415_1420.py b/django_etesync/migrations/0036_auto_20200415_1420.py new file mode 100644 index 0000000..a7b8003 --- /dev/null +++ b/django_etesync/migrations/0036_auto_20200415_1420.py @@ -0,0 +1,37 @@ +# Generated by Django 3.0.3 on 2020-04-15 14:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0035_auto_20200415_1259'), + ] + + operations = [ + migrations.AlterModelOptions( + name='collectionitemchunk', + options={}, + ), + migrations.AlterUniqueTogether( + name='collectionitemchunk', + unique_together=set(), + ), + migrations.CreateModel( + name='RevisionChunkRelation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('chunk', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions_relation', to='django_etesync.CollectionItemChunk')), + ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks_relation', to='django_etesync.CollectionItemRevision')), + ], + options={ + 'ordering': ('id',), + }, + ), + migrations.RemoveField( + model_name='collectionitemchunk', + name='order', + ), + ] diff --git a/django_etesync/migrations/0037_auto_20200415_1421.py b/django_etesync/migrations/0037_auto_20200415_1421.py new file mode 100644 index 0000000..d1a47db --- /dev/null +++ b/django_etesync/migrations/0037_auto_20200415_1421.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-04-15 14:21 + +from django.db import migrations + + +def change_chunk_relation(apps, schema_editor): + CollectionItemRevision = apps.get_model('django_etesync', 'CollectionItemRevision') + RevisionChunkRelation = apps.get_model('django_etesync', 'RevisionChunkRelation') + + for revision in CollectionItemRevision.objects.all(): + for chunk in revision.chunks.all(): + RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0036_auto_20200415_1420'), + ] + + operations = [ + migrations.RunPython(change_chunk_relation), + ] diff --git a/django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py b/django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py new file mode 100644 index 0000000..6e35b86 --- /dev/null +++ b/django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-04-15 14:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0037_auto_20200415_1421'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitemrevision', + name='chunks', + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index fdf8041..449c743 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -72,13 +72,8 @@ class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[Base64Url256BitValidator]) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) - order = models.CharField(max_length=100, blank=False, null=False) chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) - class Meta: - unique_together = ('item', 'order') - ordering = ('item', 'order') - def __str__(self): return self.uid @@ -88,7 +83,6 @@ class CollectionItemRevision(models.Model): max_length=44, validators=[Base64Url256BitValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) meta = models.BinaryField(editable=True, blank=False, null=False) - chunks = models.ManyToManyField(CollectionItemChunk, related_name='items') current = models.BooleanField(db_index=True, default=True, null=True) deleted = models.BooleanField(default=False) @@ -99,6 +93,14 @@ class CollectionItemRevision(models.Model): return '{} {} current={}'.format(self.uid, self.item.uid, self.current) +class RevisionChunkRelation(models.Model): + chunk = models.ForeignKey(CollectionItemChunk, related_name='revisions_relation', on_delete=models.CASCADE) + revision = models.ForeignKey(CollectionItemRevision, related_name='chunks_relation', on_delete=models.CASCADE) + + class Meta: + ordering = ('id', ) + + class CollectionMember(models.Model): class AccessLevels(models.TextChoices): ADMIN = 'adm' diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 60c7560..f7f4b71 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -29,23 +29,23 @@ def generate_rev_uid(length=32): def process_revisions_for_item(item, revision_data): - chunks_ids = [] - chunks = revision_data.pop('chunks') + chunks_objs = [] + chunks = revision_data.pop('chunks_relation') for chunk in chunks: uid = chunk[0] if len(chunk) > 1: content = chunk[1] - # FIXME: fix order! - chunk = models.CollectionItemChunk(uid=uid, item=item, order='abc') + chunk = models.CollectionItemChunk(uid=uid, item=item) chunk.chunkFile.save('IGNORED', ContentFile(content)) chunk.save() - chunks_ids.append(chunk.id) + chunks_objs.append(chunk) else: chunk = models.CollectionItemChunk.objects.get(uid=uid) - chunks_ids.append(chunk.id) + chunks_objs.append(chunk) revision = models.CollectionItemRevision.objects.create(**revision_data, item=item) - revision.chunks.set(chunks_ids) + for chunk in chunks_objs: + models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) return revision @@ -84,6 +84,7 @@ class CollectionContentField(BinaryBase64Field): class ChunksField(serializers.RelatedField): def to_representation(self, obj): + obj = obj.chunk inline = self.context.get('inline', False) if inline: with open(obj.chunkFile.path, 'rb') as f: @@ -103,7 +104,8 @@ class CollectionItemChunkSerializer(serializers.ModelSerializer): class CollectionItemRevisionSerializer(serializers.ModelSerializer): chunks = ChunksField( - queryset=models.CollectionItemChunk.objects.all(), + source='chunks_relation', + queryset=models.RevisionChunkRelation.objects.all(), many=True ) meta = BinaryBase64Field() diff --git a/django_etesync/views.py b/django_etesync/views.py index 1c9da78..c4c8479 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -186,9 +186,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): serializer = self.serializer_class(data=request.data) if serializer.is_valid(): try: - # FIXME: actually generate the correct order value. Or alternatively have it null at first and only - # set it when ommitting to a snapshot - serializer.save(item=col_it, order='abc') + serializer.save(item=col_it) except IntegrityError: content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) From 0fbc5c104c0359be3c37747b10fadbdd167f8591 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 15 Apr 2020 17:54:39 +0300 Subject: [PATCH 056/251] Implement collection updating. --- django_etesync/serializers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index f7f4b71..d4b809f 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -197,3 +197,19 @@ class CollectionSerializer(serializers.ModelSerializer): ).save() return instance + + def update(self, instance, validated_data): + """Function that's called when this serializer is meant to update an item""" + revision_data = validated_data.pop('content') + + with transaction.atomic(): + main_item = instance.mainItem + # We don't have to use select_for_update here because the unique constraint on current guards against + # the race condition. But it's a good idea because it'll lock and wait rather than fail. + current_revision = main_item.revisions.filter(current=True).select_for_update().first() + current_revision.current = None + current_revision.save() + + process_revisions_for_item(main_item, revision_data) + + return instance From a72543f6c9ae10f8f9bff8bd439dbbef7159703d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 11:28:49 +0300 Subject: [PATCH 057/251] Collection remove the redundant mainItem model attr. --- .../0039_remove_collection_mainitem.py | 17 +++++++++++++++++ django_etesync/models.py | 7 +++++-- django_etesync/serializers.py | 4 +--- 3 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 django_etesync/migrations/0039_remove_collection_mainitem.py diff --git a/django_etesync/migrations/0039_remove_collection_mainitem.py b/django_etesync/migrations/0039_remove_collection_mainitem.py new file mode 100644 index 0000000..1822bc7 --- /dev/null +++ b/django_etesync/migrations/0039_remove_collection_mainitem.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-04-16 08:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0038_remove_collectionitemrevision_chunks'), + ] + + operations = [ + migrations.RemoveField( + model_name='collection', + name='mainItem', + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 449c743..7315ee8 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -29,7 +29,6 @@ class Collection(models.Model): max_length=44, validators=[UidValidator]) version = models.PositiveSmallIntegerField() owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - mainItem = models.OneToOneField('CollectionItem', related_name='of_collection', null=True, on_delete=models.SET_NULL) class Meta: unique_together = ('uid', 'owner') @@ -37,9 +36,13 @@ class Collection(models.Model): def __str__(self): return self.uid + @cached_property + def main_item(self): + return self.items.get(uid=None) + @cached_property def content(self): - return self.mainItem.content + return self.main_item.content class CollectionItem(models.Model): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index d4b809f..b2d6683 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -185,11 +185,9 @@ class CollectionSerializer(serializers.ModelSerializer): instance.save() main_item = models.CollectionItem.objects.create( uid=None, encryptionKey=None, version=instance.version, collection=instance) - instance.mainItem = main_item process_revisions_for_item(main_item, revision_data) - instance.save() models.CollectionMember(collection=instance, user=validated_data.get('owner'), accessLevel=models.CollectionMember.AccessLevels.ADMIN, @@ -203,7 +201,7 @@ class CollectionSerializer(serializers.ModelSerializer): revision_data = validated_data.pop('content') with transaction.atomic(): - main_item = instance.mainItem + main_item = instance.main_item # We don't have to use select_for_update here because the unique constraint on current guards against # the race condition. But it's a good idea because it'll lock and wait rather than fail. current_revision = main_item.revisions.filter(current=True).select_for_update().first() From ca7d7dfd1287447f3215cf78e05277ae3b373882 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 11:35:58 +0300 Subject: [PATCH 058/251] Allow passing inline to not only GET requests. --- django_etesync/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index c4c8479..6c2faf3 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -67,7 +67,7 @@ class CollectionViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - inline = self.request.method == 'GET' and 'inline' in self.request.query_params + inline = 'inline' in self.request.query_params context.update({'request': self.request, 'inline': inline}) return context @@ -118,7 +118,7 @@ class CollectionItemViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - inline = self.request.method == 'GET' and 'inline' in self.request.query_params + inline = 'inline' in self.request.query_params context.update({'request': self.request, 'inline': inline}) return context From 1f97d1dbf7ccb50c6b693042d5958055751e51b3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 12:56:42 +0300 Subject: [PATCH 059/251] Remove redundant gen_rev_uid. --- django_etesync/migrations/0029_auto_20200312_1849.py | 6 +++++- django_etesync/serializers.py | 5 ----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/django_etesync/migrations/0029_auto_20200312_1849.py b/django_etesync/migrations/0029_auto_20200312_1849.py index 165b405..568aaa2 100644 --- a/django_etesync/migrations/0029_auto_20200312_1849.py +++ b/django_etesync/migrations/0029_auto_20200312_1849.py @@ -1,7 +1,11 @@ # Generated by Django 3.0.3 on 2020-03-12 18:49 from django.db import migrations -from django_etesync.serializers import generate_rev_uid +from django.utils.crypto import get_random_string + + +def generate_rev_uid(length=32): + return get_random_string(length) def add_collection_main_item(apps, schema_editor): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index b2d6683..4d1bfa5 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -17,17 +17,12 @@ import base64 from django.core.files.base import ContentFile from django.contrib.auth import get_user_model from django.db import transaction -from django.utils.crypto import get_random_string from rest_framework import serializers from . import models User = get_user_model() -def generate_rev_uid(length=32): - return get_random_string(length) - - def process_revisions_for_item(item, revision_data): chunks_objs = [] chunks = revision_data.pop('chunks_relation') From edaa7b0f0533c7658ffadd91cd28dc02a83eb44e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 15:35:44 +0300 Subject: [PATCH 060/251] Rename ctag to stoken. --- django_etesync/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 4d1bfa5..03dd0bb 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -149,12 +149,12 @@ class CollectionItemSerializer(serializers.ModelSerializer): class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') - ctag = serializers.SerializerMethodField('get_ctag') + stoken = serializers.SerializerMethodField('get_stoken') content = CollectionItemRevisionSerializer(many=False) class Meta: model = models.Collection - fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'ctag') + fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'stoken') def get_access_level_from_context(self, obj): request = self.context.get('request', None) @@ -162,7 +162,7 @@ class CollectionSerializer(serializers.ModelSerializer): return obj.members.get(user=request.user).accessLevel return None - def get_ctag(self, obj): + def get_stoken(self, obj): last_revision = models.CollectionItemRevision.objects.filter(item__collection=obj).last() if last_revision is None: # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. From c5af5fd4e6b2ee26b9658c9c95a9b9b5294d2d48 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 16:33:16 +0300 Subject: [PATCH 061/251] Collection: move stoken to the model. --- django_etesync/models.py | 9 +++++++++ django_etesync/serializers.py | 10 +--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index 7315ee8..c6ec571 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -44,6 +44,15 @@ class Collection(models.Model): def content(self): return self.main_item.content + @cached_property + def stoken(self): + last_revision = CollectionItemRevision.objects.filter(item__collection=self).last() + if last_revision is None: + # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. + return None + + return last_revision.uid + class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=True, diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 03dd0bb..eb50d76 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -149,7 +149,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') - stoken = serializers.SerializerMethodField('get_stoken') + stoken = serializers.CharField(read_only=True) content = CollectionItemRevisionSerializer(many=False) class Meta: @@ -162,14 +162,6 @@ class CollectionSerializer(serializers.ModelSerializer): return obj.members.get(user=request.user).accessLevel return None - def get_stoken(self, obj): - last_revision = models.CollectionItemRevision.objects.filter(item__collection=obj).last() - if last_revision is None: - # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. - return None - - return last_revision.uid - def create(self, validated_data): """Function that's called when this serializer creates an item""" revision_data = validated_data.pop('content') From 687bf9924b80f10e852d69eaa78c291c3dd99763 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 16:37:26 +0300 Subject: [PATCH 062/251] API: change how pagination and stoken are done --- django_etesync/paginators.py | 36 ---------------------------- django_etesync/views.py | 46 +++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 42 deletions(-) delete mode 100644 django_etesync/paginators.py diff --git a/django_etesync/paginators.py b/django_etesync/paginators.py deleted file mode 100644 index 6d55599..0000000 --- a/django_etesync/paginators.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from rest_framework import pagination -from rest_framework.response import Response - - -class LinkHeaderPagination(pagination.LimitOffsetPagination): - def get_paginated_response(self, data): - next_url = self.get_next_link() - previous_url = self.get_previous_link() - - if next_url is not None and previous_url is not None: - link = '<{next_url}>; rel="next", <{previous_url}>; rel="prev"' - elif next_url is not None: - link = '<{next_url}>; rel="next"' - elif previous_url is not None: - link = '<{previous_url}>; rel="prev"' - else: - link = '' - - link = link.format(next_url=next_url, previous_url=previous_url) - headers = {'Link': link} if link else {} - - return Response(data, headers=headers) diff --git a/django_etesync/views.py b/django_etesync/views.py index 6c2faf3..df22068 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -24,8 +24,8 @@ from rest_framework import parsers from rest_framework.decorators import action as action_decorator from rest_framework.response import Response -from . import app_settings, paginators -from .models import Collection, CollectionItem +from . import app_settings +from .models import Collection, CollectionItem, CollectionItemRevision from .serializers import ( CollectionSerializer, CollectionItemSerializer, @@ -61,8 +61,9 @@ class CollectionViewSet(BaseViewSet): serializer_class = CollectionSerializer lookup_field = 'uid' - def get_queryset(self): - queryset = type(self).queryset + def get_queryset(self, queryset=None): + if queryset is None: + queryset = type(self).queryset return self.get_collection_queryset(queryset) def get_serializer_context(self): @@ -89,10 +90,24 @@ class CollectionViewSet(BaseViewSet): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) def list(self, request): + stoken = request.GET.get('stoken', None) + limit = int(request.GET.get('limit', 50)) + queryset = self.get_queryset() + if stoken is not None: + last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) + queryset = queryset.filter(items__revisions__id__gt=last_rev.id) + + queryset = queryset[:limit] + serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) - return Response(serializer.data) + + new_stoken = serializer.data[-1]['stoken'] if len(serializer.data) > 0 else stoken + ret = { + 'data': serializer.data, + } + return Response(ret, headers={'X-EteSync-SToken': new_stoken}) class CollectionItemViewSet(BaseViewSet): @@ -100,7 +115,6 @@ class CollectionItemViewSet(BaseViewSet): permission_classes = BaseViewSet.permission_classes queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer - pagination_class = paginators.LinkHeaderPagination lookup_field = 'uid' def get_queryset(self): @@ -148,6 +162,26 @@ class CollectionItemViewSet(BaseViewSet): # FIXME: implement, or should it be implemented elsewhere? return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + def list(self, request, collection_uid=None): + stoken = request.GET.get('stoken', None) + limit = int(request.GET.get('limit', 50)) + + queryset = self.get_queryset() + + if stoken is not None: + last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) + queryset = queryset.filter(revisions__id__gt=last_rev.id) + + queryset = queryset[:limit] + + serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) + + new_stoken = serializer.data[-1]['content']['uid'] if len(serializer.data) > 0 else stoken + ret = { + 'data': serializer.data, + } + return Response(ret, headers={'X-EteSync-SToken': new_stoken}) + @action_decorator(detail=True, methods=['GET']) def revision(self, request, collection_uid=None, uid=None): col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) From 19b93265d788e9a2c4f5661de7d15a13e21c9c93 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 16:43:21 +0300 Subject: [PATCH 063/251] Add a comment. --- django_etesync/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index df22068..e8bd6de 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -184,6 +184,7 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=True, methods=['GET']) def revision(self, request, collection_uid=None, uid=None): + # FIXME: need pagination support col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) col_it = get_object_or_404(col.items, uid=uid) From d134934f8c190f9b11129ae847c8857906369da2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 17:12:51 +0300 Subject: [PATCH 064/251] Bulk_get: implement stoken, limit and inline --- django_etesync/views.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index e8bd6de..d9b6283 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -164,18 +164,14 @@ class CollectionItemViewSet(BaseViewSet): def list(self, request, collection_uid=None): stoken = request.GET.get('stoken', None) - limit = int(request.GET.get('limit', 50)) queryset = self.get_queryset() - - if stoken is not None: - last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) - queryset = queryset.filter(revisions__id__gt=last_rev.id) - - queryset = queryset[:limit] + queryset = self.filter_by_stoken_and_limit(request, queryset) serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) + queryset = self.filter_by_stoken_and_limit(request, queryset) + new_stoken = serializer.data[-1]['content']['uid'] if len(serializer.data) > 0 else stoken ret = { 'data': serializer.data, @@ -193,13 +189,31 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def bulk_get(self, request, collection_uid=None): + stoken = request.GET.get('stoken', None) queryset = self.get_queryset() if isinstance(request.data, list): queryset = queryset.filter(uid__in=request.data) - serializer = self.get_serializer_class()(queryset, many=True) - return Response(serializer.data, status=status.HTTP_200_OK) + queryset = self.filter_by_stoken_and_limit(request, queryset) + + serializer = self.get_serializer_class()(queryset, context=self.get_serializer_context(), many=True) + + new_stoken = serializer.data[-1]['content']['uid'] if len(serializer.data) > 0 else stoken + ret = { + 'data': serializer.data, + } + return Response(ret, headers={'X-EteSync-SToken': new_stoken}) + + def filter_by_stoken_and_limit(self, request, queryset): + stoken = request.GET.get('stoken', None) + limit = int(request.GET.get('limit', 50)) + + if stoken is not None: + last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) + queryset = queryset.filter(revisions__id__gt=last_rev.id) + + return queryset[:limit] class CollectionItemChunkViewSet(viewsets.ViewSet): From f23815d46d348a95401aeeeca485d105b68a7c01 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 17:14:03 +0300 Subject: [PATCH 065/251] Fix calculation of stoken. --- django_etesync/views.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index d9b6283..53e6adb 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -15,6 +15,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import IntegrityError +from django.db.models import Max from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -103,7 +104,9 @@ class CollectionViewSet(BaseViewSet): serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) - new_stoken = serializer.data[-1]['stoken'] if len(serializer.data) > 0 else stoken + new_stoken_id = queryset.aggregate(stoken_id=Max('items__revisions__id'))['stoken_id'] + new_stoken = CollectionItemRevision.objects.get(id=new_stoken_id).uid if new_stoken_id is not None else stoken + ret = { 'data': serializer.data, } @@ -163,16 +166,11 @@ class CollectionItemViewSet(BaseViewSet): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def list(self, request, collection_uid=None): - stoken = request.GET.get('stoken', None) - queryset = self.get_queryset() - queryset = self.filter_by_stoken_and_limit(request, queryset) + queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) - queryset = self.filter_by_stoken_and_limit(request, queryset) - - new_stoken = serializer.data[-1]['content']['uid'] if len(serializer.data) > 0 else stoken ret = { 'data': serializer.data, } @@ -189,17 +187,15 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def bulk_get(self, request, collection_uid=None): - stoken = request.GET.get('stoken', None) queryset = self.get_queryset() if isinstance(request.data, list): queryset = queryset.filter(uid__in=request.data) - queryset = self.filter_by_stoken_and_limit(request, queryset) + queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) serializer = self.get_serializer_class()(queryset, context=self.get_serializer_context(), many=True) - new_stoken = serializer.data[-1]['content']['uid'] if len(serializer.data) > 0 else stoken ret = { 'data': serializer.data, } @@ -213,7 +209,10 @@ class CollectionItemViewSet(BaseViewSet): last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) queryset = queryset.filter(revisions__id__gt=last_rev.id) - return queryset[:limit] + new_stoken_id = queryset.aggregate(stoken_id=Max('revisions__id'))['stoken_id'] + new_stoken = CollectionItemRevision.objects.get(id=new_stoken_id).uid if new_stoken_id is not None else stoken + + return queryset[:limit], new_stoken class CollectionItemChunkViewSet(viewsets.ViewSet): From 9f0f00a59496c57c84aa8cf553150f4fb6f92f37 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 17:36:06 +0300 Subject: [PATCH 066/251] Unify the stoken filtering and calculation. --- django_etesync/views.py | 45 ++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 53e6adb..25f2d2a 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -41,6 +41,7 @@ User = get_user_model() class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) + stoken_id_field = None def get_serializer_class(self): serializer_class = self.serializer_class @@ -54,6 +55,22 @@ class BaseViewSet(viewsets.ModelViewSet): user = self.request.user return queryset.filter(members__user=user) + def filter_by_stoken_and_limit(self, request, queryset): + stoken = request.GET.get('stoken', None) + limit = int(request.GET.get('limit', 50)) + + stoken_id_field = self.stoken_id_field + '__id' + + if stoken is not None: + last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) + filter_by = {stoken_id_field + '__gt': last_rev.id} + queryset = queryset.filter(**filter_by) + + new_stoken_id = queryset.aggregate(stoken_id=Max(stoken_id_field))['stoken_id'] + new_stoken = CollectionItemRevision.objects.get(id=new_stoken_id).uid if new_stoken_id is not None else stoken + + return queryset[:limit], new_stoken + class CollectionViewSet(BaseViewSet): allowed_methods = ['GET', 'POST', 'DELETE'] @@ -61,6 +78,7 @@ class CollectionViewSet(BaseViewSet): queryset = Collection.objects.all() serializer_class = CollectionSerializer lookup_field = 'uid' + stoken_id_field = 'items__revisions' def get_queryset(self, queryset=None): if queryset is None: @@ -91,22 +109,11 @@ class CollectionViewSet(BaseViewSet): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) def list(self, request): - stoken = request.GET.get('stoken', None) - limit = int(request.GET.get('limit', 50)) - queryset = self.get_queryset() - - if stoken is not None: - last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) - queryset = queryset.filter(items__revisions__id__gt=last_rev.id) - - queryset = queryset[:limit] + queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) - new_stoken_id = queryset.aggregate(stoken_id=Max('items__revisions__id'))['stoken_id'] - new_stoken = CollectionItemRevision.objects.get(id=new_stoken_id).uid if new_stoken_id is not None else stoken - ret = { 'data': serializer.data, } @@ -119,6 +126,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer lookup_field = 'uid' + stoken_id_field = 'revisions' def get_queryset(self): collection_uid = self.kwargs['collection_uid'] @@ -201,19 +209,6 @@ class CollectionItemViewSet(BaseViewSet): } return Response(ret, headers={'X-EteSync-SToken': new_stoken}) - def filter_by_stoken_and_limit(self, request, queryset): - stoken = request.GET.get('stoken', None) - limit = int(request.GET.get('limit', 50)) - - if stoken is not None: - last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) - queryset = queryset.filter(revisions__id__gt=last_rev.id) - - new_stoken_id = queryset.aggregate(stoken_id=Max('revisions__id'))['stoken_id'] - new_stoken = CollectionItemRevision.objects.get(id=new_stoken_id).uid if new_stoken_id is not None else stoken - - return queryset[:limit], new_stoken - class CollectionItemChunkViewSet(viewsets.ViewSet): allowed_methods = ['GET', 'POST'] From af2787195554cfd624ab343b05f50004997709a2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Apr 2020 17:38:07 +0300 Subject: [PATCH 067/251] Revision: change the shape of the list response. --- django_etesync/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 25f2d2a..7d6234f 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -191,7 +191,10 @@ class CollectionItemViewSet(BaseViewSet): col_it = get_object_or_404(col.items, uid=uid) serializer = CollectionItemRevisionSerializer(col_it.revisions.order_by('-id'), many=True) - return Response(serializer.data) + ret = { + 'data': serializer.data, + } + return Response(ret) @action_decorator(detail=False, methods=['POST']) def bulk_get(self, request, collection_uid=None): From d66d0640dcfbebbcc6d375e356878e93e53503fc Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 19 Apr 2020 15:13:09 +0300 Subject: [PATCH 068/251] Collection: disallow partial updates. --- django_etesync/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 7d6234f..b9cd878 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -95,6 +95,9 @@ class CollectionViewSet(BaseViewSet): # FIXME: implement return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + def partial_update(self, request, uid=None): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + def create(self, request, *args, **kwargs): serializer = self.serializer_class(data=request.data, context=self.get_serializer_context()) if serializer.is_valid(): From df0f7d134d5b07894762d07190b62019bf84a55e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 19 Apr 2020 17:32:40 +0300 Subject: [PATCH 069/251] Collection items: add a transaction endpoint. --- django_etesync/models.py | 4 ++++ django_etesync/views.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index c6ec571..05b81b3 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -71,6 +71,10 @@ class CollectionItem(models.Model): def content(self): return self.revisions.get(current=True) + @cached_property + def stoken(self): + return self.content.uid + def chunk_directory_path(instance, filename): item = instance.item diff --git a/django_etesync/views.py b/django_etesync/views.py index b9cd878..87c8010 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -14,7 +14,7 @@ from django.conf import settings from django.contrib.auth import get_user_model -from django.db import IntegrityError +from django.db import transaction, IntegrityError from django.db.models import Max from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -215,6 +215,35 @@ class CollectionItemViewSet(BaseViewSet): } return Response(ret, headers={'X-EteSync-SToken': new_stoken}) + @action_decorator(detail=False, methods=['POST']) + def transaction(self, request, collection_uid=None): + collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) + + items = request.data.get('items') + # FIXME: deps should actually be just pairs of uid and stoken + deps = request.data.get('deps', None) + serializer = self.get_serializer_class()(data=items, context=self.get_serializer_context(), many=True) + deps_serializer = self.get_serializer_class()(data=deps, context=self.get_serializer_context(), many=True) + if serializer.is_valid() and (deps is None or deps_serializer.is_valid()): + try: + with transaction.atomic(): + collections = serializer.save(collection=collection_object) + except IntegrityError: + content = {'code': 'integrity_error'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + ret = { + "data": [collection.stoken for collection in collections], + } + return Response(ret, status=status.HTTP_201_CREATED) + + return Response( + { + "items": serializer.errors, + "deps": deps_serializer.errors if deps is not None else [], + }, + status=status.HTTP_400_BAD_REQUEST) + class CollectionItemChunkViewSet(viewsets.ViewSet): allowed_methods = ['GET', 'POST'] From 6b0a40e9dd87a35f716186904b36b263c0155d98 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 13 May 2020 16:01:49 +0300 Subject: [PATCH 070/251] Set custom user model and reset migrations. --- django_etesync/migrations/0001_initial.py | 52 +++++++++++++----- .../migrations/0002_auto_20200220_0943.py | 53 ------------------- .../migrations/0003_collectionitem_current.py | 18 ------- .../migrations/0004_auto_20200220_1029.py | 18 ------- .../migrations/0005_auto_20200220_1123.py | 29 ---------- .../migrations/0006_auto_20200220_1137.py | 49 ----------------- .../migrations/0007_auto_20200220_1144.py | 28 ---------- .../0008_collectionitemchunk_chunkfile.py | 20 ------- .../migrations/0009_auto_20200220_1220.py | 19 ------- .../migrations/0010_auto_20200220_1248.py | 19 ------- .../migrations/0011_auto_20200220_2037.py | 17 ------ .../migrations/0012_auto_20200220_2038.py | 19 ------- ...0013_collectionitemrevision_is_deletion.py | 18 ------- .../migrations/0014_auto_20200226_1322.py | 18 ------- .../migrations/0015_auto_20200226_1349.py | 18 ------- .../migrations/0016_auto_20200226_1446.py | 29 ---------- .../migrations/0017_auto_20200226_1455.py | 18 ------- .../migrations/0018_auto_20200226_1803.py | 18 ------- .../migrations/0019_collectionmember.py | 29 ---------- .../migrations/0020_auto_20200310_1438.py | 29 ---------- .../migrations/0021_auto_20200310_1439.py | 24 --------- .../migrations/0022_auto_20200310_1547.py | 33 ------------ .../migrations/0023_auto_20200310_1556.py | 17 ------ .../0024_collectionitemrevision_uid.py | 19 ------- .../migrations/0025_auto_20200312_1350.py | 19 ------- .../0026_collectionitemrevision_meta.py | 18 ------- .../migrations/0027_collection_mainitem.py | 19 ------- .../migrations/0028_auto_20200312_1819.py | 24 --------- .../migrations/0029_auto_20200312_1849.py | 35 ------------ .../migrations/0030_auto_20200312_1859.py | 19 ------- .../migrations/0031_auto_20200317_1509.py | 19 ------- .../migrations/0032_auto_20200317_1513.py | 23 -------- .../migrations/0033_auto_20200317_2010.py | 19 ------- .../migrations/0034_auto_20200415_1248.py | 19 ------- .../migrations/0035_auto_20200415_1259.py | 19 ------- .../migrations/0036_auto_20200415_1420.py | 37 ------------- .../migrations/0037_auto_20200415_1421.py | 23 -------- ...38_remove_collectionitemrevision_chunks.py | 17 ------ .../0039_remove_collection_mainitem.py | 17 ------ myauth/__init__.py | 0 myauth/admin.py | 5 ++ myauth/apps.py | 5 ++ myauth/migrations/0001_initial.py | 44 +++++++++++++++ myauth/migrations/__init__.py | 0 myauth/models.py | 5 ++ myauth/tests.py | 3 ++ myauth/views.py | 3 ++ 47 files changed, 103 insertions(+), 900 deletions(-) delete mode 100644 django_etesync/migrations/0002_auto_20200220_0943.py delete mode 100644 django_etesync/migrations/0003_collectionitem_current.py delete mode 100644 django_etesync/migrations/0004_auto_20200220_1029.py delete mode 100644 django_etesync/migrations/0005_auto_20200220_1123.py delete mode 100644 django_etesync/migrations/0006_auto_20200220_1137.py delete mode 100644 django_etesync/migrations/0007_auto_20200220_1144.py delete mode 100644 django_etesync/migrations/0008_collectionitemchunk_chunkfile.py delete mode 100644 django_etesync/migrations/0009_auto_20200220_1220.py delete mode 100644 django_etesync/migrations/0010_auto_20200220_1248.py delete mode 100644 django_etesync/migrations/0011_auto_20200220_2037.py delete mode 100644 django_etesync/migrations/0012_auto_20200220_2038.py delete mode 100644 django_etesync/migrations/0013_collectionitemrevision_is_deletion.py delete mode 100644 django_etesync/migrations/0014_auto_20200226_1322.py delete mode 100644 django_etesync/migrations/0015_auto_20200226_1349.py delete mode 100644 django_etesync/migrations/0016_auto_20200226_1446.py delete mode 100644 django_etesync/migrations/0017_auto_20200226_1455.py delete mode 100644 django_etesync/migrations/0018_auto_20200226_1803.py delete mode 100644 django_etesync/migrations/0019_collectionmember.py delete mode 100644 django_etesync/migrations/0020_auto_20200310_1438.py delete mode 100644 django_etesync/migrations/0021_auto_20200310_1439.py delete mode 100644 django_etesync/migrations/0022_auto_20200310_1547.py delete mode 100644 django_etesync/migrations/0023_auto_20200310_1556.py delete mode 100644 django_etesync/migrations/0024_collectionitemrevision_uid.py delete mode 100644 django_etesync/migrations/0025_auto_20200312_1350.py delete mode 100644 django_etesync/migrations/0026_collectionitemrevision_meta.py delete mode 100644 django_etesync/migrations/0027_collection_mainitem.py delete mode 100644 django_etesync/migrations/0028_auto_20200312_1819.py delete mode 100644 django_etesync/migrations/0029_auto_20200312_1849.py delete mode 100644 django_etesync/migrations/0030_auto_20200312_1859.py delete mode 100644 django_etesync/migrations/0031_auto_20200317_1509.py delete mode 100644 django_etesync/migrations/0032_auto_20200317_1513.py delete mode 100644 django_etesync/migrations/0033_auto_20200317_2010.py delete mode 100644 django_etesync/migrations/0034_auto_20200415_1248.py delete mode 100644 django_etesync/migrations/0035_auto_20200415_1259.py delete mode 100644 django_etesync/migrations/0036_auto_20200415_1420.py delete mode 100644 django_etesync/migrations/0037_auto_20200415_1421.py delete mode 100644 django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py delete mode 100644 django_etesync/migrations/0039_remove_collection_mainitem.py create mode 100644 myauth/__init__.py create mode 100644 myauth/admin.py create mode 100644 myauth/apps.py create mode 100644 myauth/migrations/0001_initial.py create mode 100644 myauth/migrations/__init__.py create mode 100644 myauth/models.py create mode 100644 myauth/tests.py create mode 100644 myauth/views.py diff --git a/django_etesync/migrations/0001_initial.py b/django_etesync/migrations/0001_initial.py index 6fa724b..dc3a2ff 100644 --- a/django_etesync/migrations/0001_initial.py +++ b/django_etesync/migrations/0001_initial.py @@ -1,9 +1,10 @@ -# Generated by Django 3.0.3 on 2020-02-19 15:33 +# Generated by Django 3.0.3 on 2020-05-13 13:01 from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion +import django_etesync.models class Migration(migrations.Migration): @@ -19,7 +20,7 @@ class Migration(migrations.Migration): name='Collection', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-fA-F0-9\\-_=]{44}')])), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')])), ('version', models.PositiveSmallIntegerField()), ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], @@ -31,37 +32,60 @@ class Migration(migrations.Migration): name='CollectionItem', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-fA-F0-9\\-_=]{44}')])), + ('uid', models.CharField(db_index=True, max_length=44, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')])), ('version', models.PositiveSmallIntegerField()), - ('encryptionKey', models.BinaryField(editable=True)), - ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etesync.Collection')), + ('encryptionKey', models.BinaryField(editable=True, null=True)), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='django_etesync.Collection')), ], options={ 'unique_together': {('uid', 'collection')}, }, ), migrations.CreateModel( - name='CollectionItemSnapshot', + name='CollectionItemChunk', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), + ('chunkFile', models.FileField(max_length=150, unique=True, upload_to=django_etesync.models.chunk_directory_path)), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etesync.CollectionItem')), + ], + ), + migrations.CreateModel( + name='CollectionItemRevision', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('current', models.BooleanField(default=True)), - ('chunkHmac', models.CharField(max_length=50)), - ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etesync.CollectionItem')), + ('uid', models.CharField(db_index=True, max_length=44, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), + ('meta', models.BinaryField(editable=True)), + ('current', models.BooleanField(db_index=True, default=True, null=True)), + ('deleted', models.BooleanField(default=False)), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='django_etesync.CollectionItem')), ], options={ 'unique_together': {('item', 'current')}, }, ), migrations.CreateModel( - name='CollectionItemChunk', + name='RevisionChunkRelation', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-fA-F0-9\\-_=]{44}')])), - ('order', models.CharField(max_length=100)), - ('itemSnapshot', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_etesync.CollectionItemSnapshot')), + ('chunk', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions_relation', to='django_etesync.CollectionItemChunk')), + ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks_relation', to='django_etesync.CollectionItemRevision')), + ], + options={ + 'ordering': ('id',), + }, + ), + migrations.CreateModel( + name='CollectionMember', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('encryptionKey', models.BinaryField(editable=True)), + ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='django_etesync.Collection')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], options={ - 'ordering': ['order'], + 'unique_together': {('user', 'collection')}, }, ), ] diff --git a/django_etesync/migrations/0002_auto_20200220_0943.py b/django_etesync/migrations/0002_auto_20200220_0943.py deleted file mode 100644 index c150a11..0000000 --- a/django_etesync/migrations/0002_auto_20200220_0943.py +++ /dev/null @@ -1,53 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 09:43 - -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='collectionitemchunk', - name='itemSnapshot', - ), - migrations.AddField( - model_name='collectionitem', - name='hmac', - field=models.CharField(default='', max_length=50), - preserve_default=False, - ), - migrations.AddField( - model_name='collectionitemchunk', - name='items', - field=models.ManyToManyField(related_name='chunks', to='django_etesync.CollectionItem'), - ), - migrations.AlterField( - model_name='collection', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{44}')]), - ), - migrations.AlterField( - model_name='collectionitem', - name='collection', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='django_etesync.Collection'), - ), - migrations.AlterField( - model_name='collectionitem', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{44}')]), - ), - migrations.AlterField( - model_name='collectionitemchunk', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{44}')]), - ), - migrations.DeleteModel( - name='CollectionItemSnapshot', - ), - ] diff --git a/django_etesync/migrations/0003_collectionitem_current.py b/django_etesync/migrations/0003_collectionitem_current.py deleted file mode 100644 index 2ffbf54..0000000 --- a/django_etesync/migrations/0003_collectionitem_current.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 09:47 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0002_auto_20200220_0943'), - ] - - operations = [ - migrations.AddField( - model_name='collectionitem', - name='current', - field=models.BooleanField(default=True), - ), - ] diff --git a/django_etesync/migrations/0004_auto_20200220_1029.py b/django_etesync/migrations/0004_auto_20200220_1029.py deleted file mode 100644 index 1ea337a..0000000 --- a/django_etesync/migrations/0004_auto_20200220_1029.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 10:29 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0003_collectionitem_current'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitem', - name='current', - field=models.BooleanField(db_index=True, default=True), - ), - ] diff --git a/django_etesync/migrations/0005_auto_20200220_1123.py b/django_etesync/migrations/0005_auto_20200220_1123.py deleted file mode 100644 index 88c9ea6..0000000 --- a/django_etesync/migrations/0005_auto_20200220_1123.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 11:23 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0004_auto_20200220_1029'), - ] - - operations = [ - migrations.RemoveField( - model_name='collectionitemchunk', - name='items', - ), - migrations.AddField( - model_name='collectionitem', - name='chunks', - field=models.ManyToManyField(related_name='items', to='django_etesync.CollectionItemChunk'), - ), - migrations.AddField( - model_name='collectionitemchunk', - name='collection', - field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etesync.Collection'), - preserve_default=False, - ), - ] diff --git a/django_etesync/migrations/0006_auto_20200220_1137.py b/django_etesync/migrations/0006_auto_20200220_1137.py deleted file mode 100644 index efc421e..0000000 --- a/django_etesync/migrations/0006_auto_20200220_1137.py +++ /dev/null @@ -1,49 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 11:37 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0005_auto_20200220_1123'), - ] - - operations = [ - migrations.RemoveField( - model_name='collectionitem', - name='chunks', - ), - migrations.RemoveField( - model_name='collectionitem', - name='current', - ), - migrations.RemoveField( - model_name='collectionitem', - name='encryptionKey', - ), - migrations.RemoveField( - model_name='collectionitem', - name='hmac', - ), - migrations.RemoveField( - model_name='collectionitem', - name='version', - ), - migrations.CreateModel( - name='CollectionItemSnapshot', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('version', models.PositiveSmallIntegerField()), - ('encryptionKey', models.BinaryField(editable=True)), - ('hmac', models.CharField(max_length=50)), - ('current', models.BooleanField(db_index=True, default=True)), - ('chunks', models.ManyToManyField(related_name='items', to='django_etesync.CollectionItemChunk')), - ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='snapshots', to='django_etesync.CollectionItem')), - ], - options={ - 'unique_together': {('item', 'current')}, - }, - ), - ] diff --git a/django_etesync/migrations/0007_auto_20200220_1144.py b/django_etesync/migrations/0007_auto_20200220_1144.py deleted file mode 100644 index 3ebf55b..0000000 --- a/django_etesync/migrations/0007_auto_20200220_1144.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 11:44 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0006_auto_20200220_1137'), - ] - - operations = [ - migrations.AddField( - model_name='collectionitemchunk', - name='item', - field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etesync.CollectionItem'), - preserve_default=False, - ), - migrations.AlterUniqueTogether( - name='collectionitemchunk', - unique_together={('item', 'order')}, - ), - migrations.RemoveField( - model_name='collectionitemchunk', - name='collection', - ), - ] diff --git a/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py b/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py deleted file mode 100644 index 68bb27c..0000000 --- a/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 12:16 - -from django.db import migrations, models -import django_etesync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0007_auto_20200220_1144'), - ] - - operations = [ - migrations.AddField( - model_name='collectionitemchunk', - name='chunkFile', - field=models.FileField(default='', upload_to=django_etesync.models.chunk_directory_path), - preserve_default=False, - ), - ] diff --git a/django_etesync/migrations/0009_auto_20200220_1220.py b/django_etesync/migrations/0009_auto_20200220_1220.py deleted file mode 100644 index 71e1539..0000000 --- a/django_etesync/migrations/0009_auto_20200220_1220.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 12:20 - -from django.db import migrations, models -import django_etesync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0008_collectionitemchunk_chunkfile'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemchunk', - name='chunkFile', - field=models.FileField(max_length=150, upload_to=django_etesync.models.chunk_directory_path), - ), - ] diff --git a/django_etesync/migrations/0010_auto_20200220_1248.py b/django_etesync/migrations/0010_auto_20200220_1248.py deleted file mode 100644 index 0c08ed0..0000000 --- a/django_etesync/migrations/0010_auto_20200220_1248.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 12:48 - -from django.db import migrations, models -import django_etesync.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0009_auto_20200220_1220'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemchunk', - name='chunkFile', - field=models.FileField(max_length=150, unique=True, upload_to=django_etesync.models.chunk_directory_path), - ), - ] diff --git a/django_etesync/migrations/0011_auto_20200220_2037.py b/django_etesync/migrations/0011_auto_20200220_2037.py deleted file mode 100644 index 2d79074..0000000 --- a/django_etesync/migrations/0011_auto_20200220_2037.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 20:37 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0010_auto_20200220_1248'), - ] - - operations = [ - migrations.RenameModel( - old_name='CollectionItemSnapshot', - new_name='CollectionItemRevision', - ), - ] diff --git a/django_etesync/migrations/0012_auto_20200220_2038.py b/django_etesync/migrations/0012_auto_20200220_2038.py deleted file mode 100644 index 2657973..0000000 --- a/django_etesync/migrations/0012_auto_20200220_2038.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-20 20:38 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0011_auto_20200220_2037'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemrevision', - name='item', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='django_etesync.CollectionItem'), - ), - ] diff --git a/django_etesync/migrations/0013_collectionitemrevision_is_deletion.py b/django_etesync/migrations/0013_collectionitemrevision_is_deletion.py deleted file mode 100644 index 27f4953..0000000 --- a/django_etesync/migrations/0013_collectionitemrevision_is_deletion.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-26 13:17 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0012_auto_20200220_2038'), - ] - - operations = [ - migrations.AddField( - model_name='collectionitemrevision', - name='is_deletion', - field=models.BooleanField(default=False), - ), - ] diff --git a/django_etesync/migrations/0014_auto_20200226_1322.py b/django_etesync/migrations/0014_auto_20200226_1322.py deleted file mode 100644 index 1937015..0000000 --- a/django_etesync/migrations/0014_auto_20200226_1322.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-26 13:22 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0013_collectionitemrevision_is_deletion'), - ] - - operations = [ - migrations.RenameField( - model_name='collectionitemrevision', - old_name='is_deletion', - new_name='isDeletion', - ), - ] diff --git a/django_etesync/migrations/0015_auto_20200226_1349.py b/django_etesync/migrations/0015_auto_20200226_1349.py deleted file mode 100644 index 896619d..0000000 --- a/django_etesync/migrations/0015_auto_20200226_1349.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-26 13:49 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0014_auto_20200226_1322'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemrevision', - name='current', - field=models.BooleanField(blank=True, db_index=True, default=True, null=True), - ), - ] diff --git a/django_etesync/migrations/0016_auto_20200226_1446.py b/django_etesync/migrations/0016_auto_20200226_1446.py deleted file mode 100644 index 2929cbf..0000000 --- a/django_etesync/migrations/0016_auto_20200226_1446.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-26 14:46 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0015_auto_20200226_1349'), - ] - - operations = [ - migrations.AlterField( - model_name='collection', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{43}')]), - ), - migrations.AlterField( - model_name='collectionitem', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{43}')]), - ), - migrations.AlterField( - model_name='collectionitemchunk', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID. Expected a 256bit base64url.', regex='[a-zA-Z0-9\\-_=]{43}')]), - ), - ] diff --git a/django_etesync/migrations/0017_auto_20200226_1455.py b/django_etesync/migrations/0017_auto_20200226_1455.py deleted file mode 100644 index 2148123..0000000 --- a/django_etesync/migrations/0017_auto_20200226_1455.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-26 14:55 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0016_auto_20200226_1446'), - ] - - operations = [ - migrations.RenameField( - model_name='collectionitemrevision', - old_name='isDeletion', - new_name='deleted', - ), - ] diff --git a/django_etesync/migrations/0018_auto_20200226_1803.py b/django_etesync/migrations/0018_auto_20200226_1803.py deleted file mode 100644 index ae9200d..0000000 --- a/django_etesync/migrations/0018_auto_20200226_1803.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-26 18:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0017_auto_20200226_1455'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemrevision', - name='current', - field=models.BooleanField(db_index=True, default=True, null=True), - ), - ] diff --git a/django_etesync/migrations/0019_collectionmember.py b/django_etesync/migrations/0019_collectionmember.py deleted file mode 100644 index 142e945..0000000 --- a/django_etesync/migrations/0019_collectionmember.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.0.3 on 2020-02-26 18:33 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('django_etesync', '0018_auto_20200226_1803'), - ] - - operations = [ - migrations.CreateModel( - name='CollectionMember', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('encryptionKey', models.BinaryField(editable=True)), - ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), - ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='django_etesync.Collection')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'unique_together': {('user', 'collection')}, - }, - ), - ] diff --git a/django_etesync/migrations/0020_auto_20200310_1438.py b/django_etesync/migrations/0020_auto_20200310_1438.py deleted file mode 100644 index 6949145..0000000 --- a/django_etesync/migrations/0020_auto_20200310_1438.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-10 14:38 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0019_collectionmember'), - ] - - operations = [ - migrations.AlterField( - model_name='collection', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]{24}')]), - ), - migrations.AlterField( - model_name='collectionitem', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]{24}')]), - ), - migrations.AlterField( - model_name='collectionitemchunk', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}=?$')]), - ), - ] diff --git a/django_etesync/migrations/0021_auto_20200310_1439.py b/django_etesync/migrations/0021_auto_20200310_1439.py deleted file mode 100644 index 3f1341e..0000000 --- a/django_etesync/migrations/0021_auto_20200310_1439.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-10 14:39 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0020_auto_20200310_1438'), - ] - - operations = [ - migrations.AlterField( - model_name='collection', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), - ), - migrations.AlterField( - model_name='collectionitem', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), - ), - ] diff --git a/django_etesync/migrations/0022_auto_20200310_1547.py b/django_etesync/migrations/0022_auto_20200310_1547.py deleted file mode 100644 index cbd0ee7..0000000 --- a/django_etesync/migrations/0022_auto_20200310_1547.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-10 15:47 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0021_auto_20200310_1439'), - ] - - operations = [ - migrations.RemoveField( - model_name='collectionitemrevision', - name='encryptionKey', - ), - migrations.RemoveField( - model_name='collectionitemrevision', - name='version', - ), - migrations.AddField( - model_name='collectionitem', - name='encryptionKey', - field=models.BinaryField(default=b'aoesnutheounth', editable=True), - preserve_default=False, - ), - migrations.AddField( - model_name='collectionitem', - name='version', - field=models.PositiveSmallIntegerField(default=1), - preserve_default=False, - ), - ] diff --git a/django_etesync/migrations/0023_auto_20200310_1556.py b/django_etesync/migrations/0023_auto_20200310_1556.py deleted file mode 100644 index e2a9b80..0000000 --- a/django_etesync/migrations/0023_auto_20200310_1556.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-10 15:56 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0022_auto_20200310_1547'), - ] - - operations = [ - migrations.AlterModelOptions( - name='collectionitemchunk', - options={'ordering': ('item', 'order')}, - ), - ] diff --git a/django_etesync/migrations/0024_collectionitemrevision_uid.py b/django_etesync/migrations/0024_collectionitemrevision_uid.py deleted file mode 100644 index 6134c89..0000000 --- a/django_etesync/migrations/0024_collectionitemrevision_uid.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-12 13:41 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0023_auto_20200310_1556'), - ] - - operations = [ - migrations.AddField( - model_name='collectionitemrevision', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), - ), - ] diff --git a/django_etesync/migrations/0025_auto_20200312_1350.py b/django_etesync/migrations/0025_auto_20200312_1350.py deleted file mode 100644 index b54aeff..0000000 --- a/django_etesync/migrations/0025_auto_20200312_1350.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-12 13:50 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0024_collectionitemrevision_uid'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemrevision', - name='uid', - field=models.CharField(db_index=True, max_length=44, unique=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), - ), - ] diff --git a/django_etesync/migrations/0026_collectionitemrevision_meta.py b/django_etesync/migrations/0026_collectionitemrevision_meta.py deleted file mode 100644 index 8056e61..0000000 --- a/django_etesync/migrations/0026_collectionitemrevision_meta.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-12 14:01 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0025_auto_20200312_1350'), - ] - - operations = [ - migrations.AddField( - model_name='collectionitemrevision', - name='meta', - field=models.BinaryField(blank=True, editable=True, null=True), - ), - ] diff --git a/django_etesync/migrations/0027_collection_mainitem.py b/django_etesync/migrations/0027_collection_mainitem.py deleted file mode 100644 index b420d8f..0000000 --- a/django_etesync/migrations/0027_collection_mainitem.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-12 14:14 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0026_collectionitemrevision_meta'), - ] - - operations = [ - migrations.AddField( - model_name='collection', - name='mainItem', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='of_collection', to='django_etesync.CollectionItem'), - ), - ] diff --git a/django_etesync/migrations/0028_auto_20200312_1819.py b/django_etesync/migrations/0028_auto_20200312_1819.py deleted file mode 100644 index 6d76499..0000000 --- a/django_etesync/migrations/0028_auto_20200312_1819.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-12 18:19 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0027_collection_mainitem'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitem', - name='encryptionKey', - field=models.BinaryField(editable=True, null=True), - ), - migrations.AlterField( - model_name='collectionitem', - name='uid', - field=models.CharField(db_index=True, max_length=44, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')]), - ), - ] diff --git a/django_etesync/migrations/0029_auto_20200312_1849.py b/django_etesync/migrations/0029_auto_20200312_1849.py deleted file mode 100644 index 568aaa2..0000000 --- a/django_etesync/migrations/0029_auto_20200312_1849.py +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-12 18:49 - -from django.db import migrations -from django.utils.crypto import get_random_string - - -def generate_rev_uid(length=32): - return get_random_string(length) - - -def add_collection_main_item(apps, schema_editor): - Collection = apps.get_model('django_etesync', 'Collection') - CollectionItem = apps.get_model('django_etesync', 'CollectionItem') - CollectionItemRevision = apps.get_model('django_etesync', 'CollectionItemRevision') - - for col in Collection.objects.all(): - main_item = CollectionItem.objects.create(uid=None, encryptionKey=None, version=col.version, collection=col) - col.mainItem = main_item - col.save() - - CollectionItemRevision.objects.create( - uid=generate_rev_uid(), - hmac='hmac-hash', - item=main_item) - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0028_auto_20200312_1819'), - ] - - operations = [ - migrations.RunPython(add_collection_main_item), - ] diff --git a/django_etesync/migrations/0030_auto_20200312_1859.py b/django_etesync/migrations/0030_auto_20200312_1859.py deleted file mode 100644 index fe8050a..0000000 --- a/django_etesync/migrations/0030_auto_20200312_1859.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-12 18:59 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0029_auto_20200312_1849'), - ] - - operations = [ - migrations.AlterField( - model_name='collection', - name='mainItem', - field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='of_collection', to='django_etesync.CollectionItem'), - ), - ] diff --git a/django_etesync/migrations/0031_auto_20200317_1509.py b/django_etesync/migrations/0031_auto_20200317_1509.py deleted file mode 100644 index 7166781..0000000 --- a/django_etesync/migrations/0031_auto_20200317_1509.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-17 15:09 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0030_auto_20200312_1859'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemchunk', - name='uid', - field=models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')]), - ), - ] diff --git a/django_etesync/migrations/0032_auto_20200317_1513.py b/django_etesync/migrations/0032_auto_20200317_1513.py deleted file mode 100644 index 0546711..0000000 --- a/django_etesync/migrations/0032_auto_20200317_1513.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-17 15:13 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0031_auto_20200317_1509'), - ] - - operations = [ - migrations.RemoveField( - model_name='collectionitemrevision', - name='hmac', - ), - migrations.AlterField( - model_name='collectionitemrevision', - name='uid', - field=models.CharField(db_index=True, max_length=44, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')]), - ), - ] diff --git a/django_etesync/migrations/0033_auto_20200317_2010.py b/django_etesync/migrations/0033_auto_20200317_2010.py deleted file mode 100644 index 7a42b38..0000000 --- a/django_etesync/migrations/0033_auto_20200317_2010.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-03-17 20:10 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0032_auto_20200317_1513'), - ] - - operations = [ - migrations.AlterField( - model_name='collectionitemrevision', - name='meta', - field=models.BinaryField(default=b'', editable=True), - preserve_default=False, - ), - ] diff --git a/django_etesync/migrations/0034_auto_20200415_1248.py b/django_etesync/migrations/0034_auto_20200415_1248.py deleted file mode 100644 index 1156676..0000000 --- a/django_etesync/migrations/0034_auto_20200415_1248.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-04-15 12:48 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0033_auto_20200317_2010'), - ] - - operations = [ - migrations.AlterField( - model_name='collection', - name='mainItem', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='of_collection', to='django_etesync.CollectionItem'), - ), - ] diff --git a/django_etesync/migrations/0035_auto_20200415_1259.py b/django_etesync/migrations/0035_auto_20200415_1259.py deleted file mode 100644 index d558e31..0000000 --- a/django_etesync/migrations/0035_auto_20200415_1259.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 3.0.3 on 2020-04-15 12:59 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0034_auto_20200415_1248'), - ] - - operations = [ - migrations.AlterField( - model_name='collection', - name='mainItem', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='of_collection', to='django_etesync.CollectionItem'), - ), - ] diff --git a/django_etesync/migrations/0036_auto_20200415_1420.py b/django_etesync/migrations/0036_auto_20200415_1420.py deleted file mode 100644 index a7b8003..0000000 --- a/django_etesync/migrations/0036_auto_20200415_1420.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generated by Django 3.0.3 on 2020-04-15 14:20 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0035_auto_20200415_1259'), - ] - - operations = [ - migrations.AlterModelOptions( - name='collectionitemchunk', - options={}, - ), - migrations.AlterUniqueTogether( - name='collectionitemchunk', - unique_together=set(), - ), - migrations.CreateModel( - name='RevisionChunkRelation', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('chunk', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions_relation', to='django_etesync.CollectionItemChunk')), - ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks_relation', to='django_etesync.CollectionItemRevision')), - ], - options={ - 'ordering': ('id',), - }, - ), - migrations.RemoveField( - model_name='collectionitemchunk', - name='order', - ), - ] diff --git a/django_etesync/migrations/0037_auto_20200415_1421.py b/django_etesync/migrations/0037_auto_20200415_1421.py deleted file mode 100644 index d1a47db..0000000 --- a/django_etesync/migrations/0037_auto_20200415_1421.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.0.3 on 2020-04-15 14:21 - -from django.db import migrations - - -def change_chunk_relation(apps, schema_editor): - CollectionItemRevision = apps.get_model('django_etesync', 'CollectionItemRevision') - RevisionChunkRelation = apps.get_model('django_etesync', 'RevisionChunkRelation') - - for revision in CollectionItemRevision.objects.all(): - for chunk in revision.chunks.all(): - RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0036_auto_20200415_1420'), - ] - - operations = [ - migrations.RunPython(change_chunk_relation), - ] diff --git a/django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py b/django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py deleted file mode 100644 index 6e35b86..0000000 --- a/django_etesync/migrations/0038_remove_collectionitemrevision_chunks.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.0.3 on 2020-04-15 14:34 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0037_auto_20200415_1421'), - ] - - operations = [ - migrations.RemoveField( - model_name='collectionitemrevision', - name='chunks', - ), - ] diff --git a/django_etesync/migrations/0039_remove_collection_mainitem.py b/django_etesync/migrations/0039_remove_collection_mainitem.py deleted file mode 100644 index 1822bc7..0000000 --- a/django_etesync/migrations/0039_remove_collection_mainitem.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.0.3 on 2020-04-16 08:28 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_etesync', '0038_remove_collectionitemrevision_chunks'), - ] - - operations = [ - migrations.RemoveField( - model_name='collection', - name='mainItem', - ), - ] diff --git a/myauth/__init__.py b/myauth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myauth/admin.py b/myauth/admin.py new file mode 100644 index 0000000..f91be8f --- /dev/null +++ b/myauth/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin +from .models import User + +admin.site.register(User, UserAdmin) diff --git a/myauth/apps.py b/myauth/apps.py new file mode 100644 index 0000000..611e83d --- /dev/null +++ b/myauth/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MyauthConfig(AppConfig): + name = 'myauth' diff --git a/myauth/migrations/0001_initial.py b/myauth/migrations/0001_initial.py new file mode 100644 index 0000000..1f81e95 --- /dev/null +++ b/myauth/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 3.0.3 on 2020-05-13 13:00 + +import django.contrib.auth.models +import django.contrib.auth.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0011_update_proxy_permissions'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/myauth/migrations/__init__.py b/myauth/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myauth/models.py b/myauth/models.py new file mode 100644 index 0000000..3d30525 --- /dev/null +++ b/myauth/models.py @@ -0,0 +1,5 @@ +from django.contrib.auth.models import AbstractUser + + +class User(AbstractUser): + pass diff --git a/myauth/tests.py b/myauth/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/myauth/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/myauth/views.py b/myauth/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/myauth/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 32a8b9c90dc798d20c4e7faaee6fc39d220675ed Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 14 May 2020 13:43:49 +0300 Subject: [PATCH 071/251] Implement a ZKPP login flow. --- django_etesync/app_settings.py | 4 + django_etesync/migrations/0002_userinfo.py | 25 +++++ django_etesync/models.py | 10 ++ django_etesync/serializers.py | 64 ++++++++++++ django_etesync/views.py | 116 +++++++++++++++++++++ requirements.in/base.txt | 1 + requirements.txt | 27 ++--- 7 files changed, 235 insertions(+), 12 deletions(-) create mode 100644 django_etesync/migrations/0002_userinfo.py diff --git a/django_etesync/app_settings.py b/django_etesync/app_settings.py index a0a0d99..89b38f7 100644 --- a/django_etesync/app_settings.py +++ b/django_etesync/app_settings.py @@ -46,5 +46,9 @@ class AppSettings: ret.append(self.import_from_str(perm)) return ret + @property + def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name + return self._setting("CHALLENGE_VALID_SECONDS", 60) + app_settings = AppSettings('ETESYNC_') diff --git a/django_etesync/migrations/0002_userinfo.py b/django_etesync/migrations/0002_userinfo.py new file mode 100644 index 0000000..ad7018a --- /dev/null +++ b/django_etesync/migrations/0002_userinfo.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-05-14 09:51 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('myauth', '0001_initial'), + ('django_etesync', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='UserInfo', + fields=[ + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to=settings.AUTH_USER_MODEL)), + ('version', models.PositiveSmallIntegerField(default=1)), + ('pubkey', models.BinaryField(editable=True)), + ('salt', models.BinaryField(editable=True)), + ], + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 05b81b3..0d0f3dc 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -137,3 +137,13 @@ class CollectionMember(models.Model): def __str__(self): return '{} {}'.format(self.collection.uid, self.user) + + +class UserInfo(models.Model): + owner = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, primary_key=True) + version = models.PositiveSmallIntegerField(default=1) + pubkey = models.BinaryField(editable=True, blank=False, null=False) + salt = models.BinaryField(editable=True, blank=False, null=False) + + def __str__(self): + return "UserInfo<{}>".format(self.owner) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index eb50d76..3497d78 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -198,3 +198,67 @@ class CollectionSerializer(serializers.ModelSerializer): process_revisions_for_item(main_item, revision_data) return instance + + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) + + +class AuthenticationSignupSerializer(serializers.Serializer): + user = UserSerializer(many=False) + salt = BinaryBase64Field() + pubkey = BinaryBase64Field() + + def create(self, validated_data): + """Function that's called when this serializer creates an item""" + salt = validated_data.pop('salt') + pubkey = validated_data.pop('pubkey') + + with transaction.atomic(): + instance = UserSerializer.Meta.model.objects.create(**validated_data) + instance.set_unusable_password() + + models.UserInfo.objects.create(salt=salt, pubkey=pubkey, owner=instance) + + return instance + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class AuthenticationLoginChallengeSerializer(serializers.Serializer): + username = serializers.CharField(required=False) + email = serializers.EmailField(required=False) + + def validate(self, data): + if not data.get('email') and not data.get('username'): + raise serializers.ValidationError('Either email or username must be set') + return data + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class AuthenticationLoginSerializer(AuthenticationLoginChallengeSerializer): + challenge = BinaryBase64Field() + host = serializers.CharField() + signature = BinaryBase64Field() + + def validate(self, data): + host = self.context.get('host', None) + if data['host'] != host: + raise serializers.ValidationError( + 'Found wrong host name. Got: "{}" expected: "{}"'.format(data['host'], host)) + + return super().validate(data) + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + raise NotImplementedError() diff --git a/django_etesync/views.py b/django_etesync/views.py index 87c8010..c628ab0 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -12,6 +12,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import json + from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction, IntegrityError @@ -24,10 +26,20 @@ from rest_framework import viewsets from rest_framework import parsers from rest_framework.decorators import action as action_decorator from rest_framework.response import Response +from rest_framework.authtoken.models import Token + +import nacl.encoding +import nacl.signing +import nacl.secret +import nacl.hash from . import app_settings from .models import Collection, CollectionItem, CollectionItemRevision from .serializers import ( + b64encode, + AuthenticationSignupSerializer, + AuthenticationLoginChallengeSerializer, + AuthenticationLoginSerializer, CollectionSerializer, CollectionItemSerializer, CollectionItemRevisionSerializer, @@ -290,6 +302,110 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return serve(request, basename, dirname) +class AuthenticationViewSet(viewsets.ViewSet): + allowed_methods = ['POST'] + + def get_encryption_key(self, salt): + key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) + return nacl.hash.blake2b(b'', key=key, salt=salt, person=b'etesync-auth', encoder=nacl.encoding.RawEncoder) + + def get_queryset(self): + return User.objects.all() + + def list(self, request): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + @action_decorator(detail=False, methods=['POST']) + def signup(self, request): + serializer = AuthenticationSignupSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + + return Response({}, status=status.HTTP_201_CREATED) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + def get_login_user(self, serializer): + username = serializer.validated_data.get('username') + email = serializer.validated_data.get('email') + if username: + kwargs = {User.USERNAME_FIELD: username} + user = get_object_or_404(self.get_queryset(), **kwargs) + elif email: + kwargs = {User.EMAIL_FIELD: email} + user = get_object_or_404(self.get_queryset(), **kwargs) + + return user + + @action_decorator(detail=False, methods=['POST']) + def login_challenge(self, request): + from datetime import datetime + + serializer = AuthenticationLoginChallengeSerializer(data=request.data) + if serializer.is_valid(): + user = self.get_login_user(serializer) + + salt = user.userinfo.salt + enc_key = self.get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + + challenge_data = { + "timestamp": int(datetime.now().timestamp()), + "userId": user.id, + } + challenge = box.encrypt(json.dumps( + challenge_data, separators=(',', ':')).encode(), encoder=nacl.encoding.RawEncoder) + + ret = { + "salt": b64encode(salt), + "challenge": b64encode(challenge), + } + return Response(ret, status=status.HTTP_200_OK) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + @action_decorator(detail=False, methods=['POST']) + def login(self, request): + from datetime import datetime + + serializer = AuthenticationLoginSerializer( + data=request.data, context={'host': request.get_host()}) + if serializer.is_valid(): + user = self.get_login_user(serializer) + challenge = serializer.validated_data['challenge'] + signature = serializer.validated_data['signature'] + + salt = user.userinfo.salt + enc_key = self.get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + + challenge_data = json.loads(box.decrypt(challenge).decode()) + now = int(datetime.now().timestamp()) + if now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: + content = {'code': 'challenge_expired', 'detail': 'Login challange has expired'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif challenge_data['userId'] != user.id: + content = {'code': 'wrong_user', 'detail': 'This challenge is for the wrong user'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + host_hash = nacl.hash.blake2b( + serializer.validated_data['host'].encode(), encoder=nacl.encoding.RawEncoder) + verify_key = nacl.signing.VerifyKey(user.userinfo.pubkey, encoder=nacl.encoding.RawEncoder) + verify_key.verify(challenge + host_hash, signature) + + data = { + 'token': Token.objects.get_or_create(user=user)[0].key, + } + return Response(data, status=status.HTTP_200_OK) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + @action_decorator(detail=False, methods=['POST']) + def logout(self, request): + # FIXME: expire the token - we need better token handling - using knox? Something else? + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + class ResetViewSet(BaseViewSet): allowed_methods = ['POST'] diff --git a/requirements.in/base.txt b/requirements.in/base.txt index a7d1734..e6d6379 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -9,3 +9,4 @@ django-ipware djangorestframework drf-nested-routers psycopg2-binary +pynacl diff --git a/requirements.txt b/requirements.txt index 34ca428..dcc37cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,25 +6,28 @@ # asgiref==3.2.3 # via django certifi==2019.11.28 # via requests +cffi==1.14.0 # via pynacl chardet==3.0.4 # via requests defusedxml==0.6.0 # via python3-openid -django-allauth==0.41.0 -django-anymail==7.0.0 -django-appconf==1.0.3 -django-cors-headers==3.2.1 -django-debug-toolbar==2.2 -django-fullurl==1.0 -django-ipware==2.1.0 -django==3.0.3 -djangorestframework==3.11.0 -drf-nested-routers==0.91 +django-allauth==0.41.0 # via -r requirements.in/base.txt +django-anymail==7.0.0 # via -r requirements.in/base.txt +django-appconf==1.0.3 # via -r requirements.in/base.txt +django-cors-headers==3.2.1 # via -r requirements.in/base.txt +django-debug-toolbar==2.2 # via -r requirements.in/base.txt +django-fullurl==1.0 # via -r requirements.in/base.txt +django-ipware==2.1.0 # via -r requirements.in/base.txt +django==3.0.3 # via -r requirements.in/base.txt, django-allauth, django-anymail, django-appconf, django-cors-headers, django-debug-toolbar, django-fullurl, djangorestframework, drf-nested-routers +djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers +drf-nested-routers==0.91 # via -r requirements.in/base.txt idna==2.8 # via requests oauthlib==3.1.0 # via requests-oauthlib -psycopg2-binary==2.8.4 +psycopg2-binary==2.8.4 # via -r requirements.in/base.txt +pycparser==2.20 # via cffi +pynacl==1.3.0 # via -r requirements.in/base.txt python3-openid==3.1.0 # via django-allauth pytz==2019.3 # via django requests-oauthlib==1.3.0 # via django-allauth requests==2.22.0 # via django-allauth, django-anymail, requests-oauthlib -six==1.14.0 # via django-anymail, django-appconf +six==1.14.0 # via django-anymail, django-appconf, pynacl sqlparse==0.3.0 # via django, django-debug-toolbar urllib3==1.25.8 # via requests From 93a0e41f03a937722da7d50b9006384f52b64815 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 14 May 2020 15:42:42 +0300 Subject: [PATCH 072/251] Change login flow to better verify all relevant fields. --- django_etesync/serializers.py | 20 ++++++----- django_etesync/views.py | 66 ++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 3497d78..fabc7cb 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -244,18 +244,20 @@ class AuthenticationLoginChallengeSerializer(serializers.Serializer): raise NotImplementedError() -class AuthenticationLoginSerializer(AuthenticationLoginChallengeSerializer): - challenge = BinaryBase64Field() - host = serializers.CharField() +class AuthenticationLoginSerializer(serializers.Serializer): + response = BinaryBase64Field() signature = BinaryBase64Field() - def validate(self, data): - host = self.context.get('host', None) - if data['host'] != host: - raise serializers.ValidationError( - 'Found wrong host name. Got: "{}" expected: "{}"'.format(data['host'], host)) + def create(self, validated_data): + raise NotImplementedError() - return super().validate(data) + def update(self, instance, validated_data): + raise NotImplementedError() + + +class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): + challenge = BinaryBase64Field() + host = serializers.CharField() def create(self, validated_data): raise NotImplementedError() diff --git a/django_etesync/views.py b/django_etesync/views.py index c628ab0..cb52ca5 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -40,6 +40,7 @@ from .serializers import ( AuthenticationSignupSerializer, AuthenticationLoginChallengeSerializer, AuthenticationLoginSerializer, + AuthenticationLoginInnerSerializer, CollectionSerializer, CollectionItemSerializer, CollectionItemRevisionSerializer, @@ -368,35 +369,42 @@ class AuthenticationViewSet(viewsets.ViewSet): def login(self, request): from datetime import datetime - serializer = AuthenticationLoginSerializer( - data=request.data, context={'host': request.get_host()}) - if serializer.is_valid(): - user = self.get_login_user(serializer) - challenge = serializer.validated_data['challenge'] - signature = serializer.validated_data['signature'] - - salt = user.userinfo.salt - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = json.loads(box.decrypt(challenge).decode()) - now = int(datetime.now().timestamp()) - if now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: - content = {'code': 'challenge_expired', 'detail': 'Login challange has expired'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif challenge_data['userId'] != user.id: - content = {'code': 'wrong_user', 'detail': 'This challenge is for the wrong user'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - host_hash = nacl.hash.blake2b( - serializer.validated_data['host'].encode(), encoder=nacl.encoding.RawEncoder) - verify_key = nacl.signing.VerifyKey(user.userinfo.pubkey, encoder=nacl.encoding.RawEncoder) - verify_key.verify(challenge + host_hash, signature) - - data = { - 'token': Token.objects.get_or_create(user=user)[0].key, - } - return Response(data, status=status.HTTP_200_OK) + outer_serializer = AuthenticationLoginSerializer(data=request.data) + if outer_serializer.is_valid(): + response_raw = outer_serializer.validated_data['response'] + response = json.loads(response_raw.decode()) + signature = outer_serializer.validated_data['signature'] + + serializer = AuthenticationLoginInnerSerializer(data=response, context={'host': request.get_host()}) + if serializer.is_valid(): + user = self.get_login_user(serializer) + host = serializer.validated_data['host'] + challenge = serializer.validated_data['challenge'] + + salt = user.userinfo.salt + enc_key = self.get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + + challenge_data = json.loads(box.decrypt(challenge).decode()) + now = int(datetime.now().timestamp()) + if now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: + content = {'code': 'challenge_expired', 'detail': 'Login challange has expired'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif challenge_data['userId'] != user.id: + content = {'code': 'wrong_user', 'detail': 'This challenge is for the wrong user'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif host != request.get_host(): + detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) + content = {'code': 'wrong_host', 'detail': detail} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + verify_key = nacl.signing.VerifyKey(user.userinfo.pubkey, encoder=nacl.encoding.RawEncoder) + verify_key.verify(response_raw, signature) + + data = { + 'token': Token.objects.get_or_create(user=user)[0].key, + } + return Response(data, status=status.HTTP_200_OK) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) From e9e77945a6fb1723a1bae51a4eeeb865670bcf74 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 14 May 2020 17:19:18 +0300 Subject: [PATCH 073/251] Return token and user on signup/login. --- django_etesync/serializers.py | 10 ++++++++-- django_etesync/views.py | 19 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index fabc7cb..7173d73 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -206,8 +206,14 @@ class UserSerializer(serializers.ModelSerializer): fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) +class UserQuerySerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) + + class AuthenticationSignupSerializer(serializers.Serializer): - user = UserSerializer(many=False) + user = UserQuerySerializer(many=False) salt = BinaryBase64Field() pubkey = BinaryBase64Field() @@ -217,7 +223,7 @@ class AuthenticationSignupSerializer(serializers.Serializer): pubkey = validated_data.pop('pubkey') with transaction.atomic(): - instance = UserSerializer.Meta.model.objects.create(**validated_data) + instance = User.objects.create(**validated_data) instance.set_unusable_password() models.UserInfo.objects.create(salt=salt, pubkey=pubkey, owner=instance) diff --git a/django_etesync/views.py b/django_etesync/views.py index cb52ca5..eaf0e35 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -44,7 +44,8 @@ from .serializers import ( CollectionSerializer, CollectionItemSerializer, CollectionItemRevisionSerializer, - CollectionItemChunkSerializer + CollectionItemChunkSerializer, + UserSerializer, ) @@ -313,6 +314,12 @@ class AuthenticationViewSet(viewsets.ViewSet): def get_queryset(self): return User.objects.all() + def login_response_data(self, user): + return { + 'token': Token.objects.get_or_create(user=user)[0].key, + 'user': UserSerializer(user).data, + } + def list(self, request): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) @@ -320,9 +327,10 @@ class AuthenticationViewSet(viewsets.ViewSet): def signup(self, request): serializer = AuthenticationSignupSerializer(data=request.data) if serializer.is_valid(): - serializer.save() + user = serializer.save() - return Response({}, status=status.HTTP_201_CREATED) + data = self.login_response_data(user) + return Response(data, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -360,6 +368,7 @@ class AuthenticationViewSet(viewsets.ViewSet): ret = { "salt": b64encode(salt), "challenge": b64encode(challenge), + "version": user.userinfo.version, } return Response(ret, status=status.HTTP_200_OK) @@ -401,9 +410,7 @@ class AuthenticationViewSet(viewsets.ViewSet): verify_key = nacl.signing.VerifyKey(user.userinfo.pubkey, encoder=nacl.encoding.RawEncoder) verify_key.verify(response_raw, signature) - data = { - 'token': Token.objects.get_or_create(user=user)[0].key, - } + data = self.login_response_data(user) return Response(data, status=status.HTTP_200_OK) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) From 4083be8e8cb1412f8fff33062b660a67e9187f28 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 15 May 2020 11:01:56 +0300 Subject: [PATCH 074/251] Username: disallow @ in usernames. --- myauth/migrations/0002_auto_20200515_0801.py | 19 ++++++++++++++ myauth/models.py | 27 +++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 myauth/migrations/0002_auto_20200515_0801.py diff --git a/myauth/migrations/0002_auto_20200515_0801.py b/myauth/migrations/0002_auto_20200515_0801.py new file mode 100644 index 0000000..3ce02b2 --- /dev/null +++ b/myauth/migrations/0002_auto_20200515_0801.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-05-15 08:01 + +from django.db import migrations, models +import myauth.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('myauth', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='username', + field=models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and ./+/-/_ only.', max_length=150, unique=True, validators=[myauth.models.UnicodeUsernameValidator()], verbose_name='username'), + ), + ] diff --git a/myauth/models.py b/myauth/models.py index 3d30525..4afc27c 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -1,5 +1,30 @@ from django.contrib.auth.models import AbstractUser +from django.core import validators +from django.db import models +from django.utils.deconstruct import deconstructible +from django.utils.translation import gettext_lazy as _ + + +@deconstructible +class UnicodeUsernameValidator(validators.RegexValidator): + regex = r'^[\w.+-]+\Z' + message = _( + 'Enter a valid username. This value may contain only letters, ' + 'numbers, and ./+/-/_ characters.' + ) + flags = 0 class User(AbstractUser): - pass + username_validator = UnicodeUsernameValidator() + + username = models.CharField( + _('username'), + max_length=150, + unique=True, + help_text=_('Required. 150 characters or fewer. Letters, digits and ./+/-/_ only.'), + validators=[username_validator], + error_messages={ + 'unique': _("A user with that username already exists."), + }, + ) From f438d0e947a777ca949b904d7b3793b629f2d59d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 15 May 2020 12:44:10 +0300 Subject: [PATCH 075/251] Trim salt when creating the challenge. --- django_etesync/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index eaf0e35..1a7c5bc 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -309,7 +309,8 @@ class AuthenticationViewSet(viewsets.ViewSet): def get_encryption_key(self, salt): key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) - return nacl.hash.blake2b(b'', key=key, salt=salt, person=b'etesync-auth', encoder=nacl.encoding.RawEncoder) + return nacl.hash.blake2b(b'', key=key, salt=salt[:nacl.hash.BLAKE2B_SALTBYTES], person=b'etesync-auth', + encoder=nacl.encoding.RawEncoder) def get_queryset(self): return User.objects.all() From 48ebbfb3223b6fc28eb00ce0c63cf1e58406b4cd Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 15 May 2020 12:51:05 +0300 Subject: [PATCH 076/251] Disable host verification for debug mode. Was causing issues with mitm proxy and etc which was a pain. --- django_etesync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 1a7c5bc..894f06d 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -403,7 +403,7 @@ class AuthenticationViewSet(viewsets.ViewSet): elif challenge_data['userId'] != user.id: content = {'code': 'wrong_user', 'detail': 'This challenge is for the wrong user'} return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif host != request.get_host(): + elif not settings.DEBUG and host != request.get_host(): detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) content = {'code': 'wrong_host', 'detail': detail} return Response(content, status=status.HTTP_400_BAD_REQUEST) From 644539bd6834f0c2934b21f644a1394c3fabc2d0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 15 May 2020 12:44:25 +0300 Subject: [PATCH 077/251] Reset view: adjust reset view path and class. --- django_etesync/views.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 894f06d..dd110db 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -422,16 +422,22 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response({}, status=status.HTTP_400_BAD_REQUEST) -class ResetViewSet(BaseViewSet): +class TestAuthenticationViewSet(viewsets.ViewSet): + authentication_classes = BaseViewSet.authentication_classes + permission_classes = BaseViewSet.permission_classes allowed_methods = ['POST'] - def post(self, request, *args, **kwargs): + def list(self, request): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + @action_decorator(detail=False, methods=['POST']) + def reset(self, request, *args, **kwargs): # Only run when in DEBUG mode! It's only used for tests if not settings.DEBUG: return HttpResponseBadRequest("Only allowed in debug mode.") # Only allow local users, for extra safety - if not getattr(request.user, User.USERNAME_FIELD).endswith('@localhost'): + if not getattr(request.user, User.EMAIL_FIELD).endswith('@localhost'): return HttpResponseBadRequest("Endpoint not allowed for user.") # Delete all of the journal data for this user for a clear test env @@ -440,6 +446,3 @@ class ResetViewSet(BaseViewSet): # FIXME: also delete chunk files!!! return HttpResponse() - - -reset = ResetViewSet.as_view({'post': 'post'}) From bced00dc8aa1df3ecb1c734e91f6249ea65b762d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 15 May 2020 13:03:04 +0300 Subject: [PATCH 078/251] Enable logout for now so client tests pass. --- django_etesync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index dd110db..db8550b 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -419,7 +419,7 @@ class AuthenticationViewSet(viewsets.ViewSet): @action_decorator(detail=False, methods=['POST']) def logout(self, request): # FIXME: expire the token - we need better token handling - using knox? Something else? - return Response({}, status=status.HTTP_400_BAD_REQUEST) + return Response({}, status=status.HTTP_200_OK) class TestAuthenticationViewSet(viewsets.ViewSet): From bd1d11fe5f81d8e2d6f2b0bc55d3e17d291154f6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 18 May 2020 16:13:48 +0300 Subject: [PATCH 079/251] Fix signup and let signup to an empty account. --- django_etesync/serializers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 7173d73..88d237f 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -219,12 +219,17 @@ class AuthenticationSignupSerializer(serializers.Serializer): def create(self, validated_data): """Function that's called when this serializer creates an item""" + user_data = validated_data.pop('user') salt = validated_data.pop('salt') pubkey = validated_data.pop('pubkey') with transaction.atomic(): - instance = User.objects.create(**validated_data) + instance = User.objects.get_or_create(**user_data) + if hasattr(instance, 'userinfo'): + raise serializers.ValidationError('User already exists') + instance.set_unusable_password() + # FIXME: send email verification models.UserInfo.objects.create(salt=salt, pubkey=pubkey, owner=instance) From 00a80740caf29dbede57e5c1bd88b59ef69736d4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 10:48:39 +0300 Subject: [PATCH 080/251] Collection/item create/update require stoken. --- django_etesync/serializers.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 88d237f..a5755d5 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -112,18 +112,23 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): class CollectionItemSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() + stoken = serializers.CharField(allow_null=True) content = CollectionItemRevisionSerializer(many=False) class Meta: model = models.CollectionItem - fields = ('uid', 'version', 'encryptionKey', 'content') + fields = ('uid', 'version', 'encryptionKey', 'content', 'stoken') def create(self, validated_data): """Function that's called when this serializer creates an item""" + stoken = validated_data.pop('stoken') revision_data = validated_data.pop('content') instance = self.__class__.Meta.model(**validated_data) with transaction.atomic(): + if stoken is not None: + raise serializers.ValidationError('Stoken is not None') + instance.save() process_revisions_for_item(instance, revision_data) @@ -132,9 +137,13 @@ class CollectionItemSerializer(serializers.ModelSerializer): def update(self, instance, validated_data): """Function that's called when this serializer is meant to update an item""" + stoken = validated_data.pop('stoken') revision_data = validated_data.pop('content') with transaction.atomic(): + if stoken != instance.stoken: + raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(instance.stoken, stoken)) + # We don't have to use select_for_update here because the unique constraint on current guards against # the race condition. But it's a good idea because it'll lock and wait rather than fail. current_revision = instance.revisions.filter(current=True).select_for_update().first() @@ -149,7 +158,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') - stoken = serializers.CharField(read_only=True) + stoken = serializers.CharField(allow_null=True) content = CollectionItemRevisionSerializer(many=False) class Meta: @@ -164,11 +173,15 @@ class CollectionSerializer(serializers.ModelSerializer): def create(self, validated_data): """Function that's called when this serializer creates an item""" + stoken = validated_data.pop('stoken') revision_data = validated_data.pop('content') encryption_key = validated_data.pop('encryptionKey') instance = self.__class__.Meta.model(**validated_data) with transaction.atomic(): + if stoken is not None: + raise serializers.ValidationError('Stoken is not None') + instance.save() main_item = models.CollectionItem.objects.create( uid=None, encryptionKey=None, version=instance.version, collection=instance) @@ -185,9 +198,13 @@ class CollectionSerializer(serializers.ModelSerializer): def update(self, instance, validated_data): """Function that's called when this serializer is meant to update an item""" + stoken = validated_data.pop('stoken') revision_data = validated_data.pop('content') with transaction.atomic(): + if stoken != instance.stoken: + raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(instance.stoken, stoken)) + main_item = instance.main_item # We don't have to use select_for_update here because the unique constraint on current guards against # the race condition. But it's a good idea because it'll lock and wait rather than fail. From 775f438e611d36ea81231500f6c593b67dcb2dc3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 11:20:02 +0300 Subject: [PATCH 081/251] Change deps to be pairs of uid/stoken. --- django_etesync/serializers.py | 17 +++++++++++++++++ django_etesync/views.py | 5 +++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index a5755d5..71ddcb3 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -155,6 +155,23 @@ class CollectionItemSerializer(serializers.ModelSerializer): return instance +class CollectionItemDepSerializer(serializers.ModelSerializer): + stoken = serializers.CharField() + + class Meta: + model = models.CollectionItem + fields = ('uid', 'stoken') + + def validate(self, data): + for item_data in data: + item = self.__class__.Meta.model.objects.get(uid=item_data['uid']) + stoken = item_data['stoken'] + if item.stoken != stoken: + raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(item.stoken, stoken)) + + return data + + class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') diff --git a/django_etesync/views.py b/django_etesync/views.py index db8550b..28dc601 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -43,6 +43,7 @@ from .serializers import ( AuthenticationLoginInnerSerializer, CollectionSerializer, CollectionItemSerializer, + CollectionItemDepSerializer, CollectionItemRevisionSerializer, CollectionItemChunkSerializer, UserSerializer, @@ -234,15 +235,15 @@ class CollectionItemViewSet(BaseViewSet): collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) items = request.data.get('items') - # FIXME: deps should actually be just pairs of uid and stoken deps = request.data.get('deps', None) serializer = self.get_serializer_class()(data=items, context=self.get_serializer_context(), many=True) - deps_serializer = self.get_serializer_class()(data=deps, context=self.get_serializer_context(), many=True) + deps_serializer = CollectionItemDepSerializer(data=deps, context=self.get_serializer_context(), many=True) if serializer.is_valid() and (deps is None or deps_serializer.is_valid()): try: with transaction.atomic(): collections = serializer.save(collection=collection_object) except IntegrityError: + # FIXME: should return the items with a bad token (including deps) so we don't have to fetch them after content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) From 306e7dcd113477a65220db20e2e0ca92afef673a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 11:44:20 +0300 Subject: [PATCH 082/251] Item deps: fix. --- django_etesync/serializers.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 71ddcb3..3f7c28c 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -163,11 +163,10 @@ class CollectionItemDepSerializer(serializers.ModelSerializer): fields = ('uid', 'stoken') def validate(self, data): - for item_data in data: - item = self.__class__.Meta.model.objects.get(uid=item_data['uid']) - stoken = item_data['stoken'] - if item.stoken != stoken: - raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(item.stoken, stoken)) + item = self.__class__.Meta.model.objects.get(uid=data['uid']) + stoken = data['stoken'] + if item.stoken != stoken: + raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(item.stoken, stoken)) return data From 23dcbc1f9ea60c689e6a27acb31ba7657cdfa07a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 12:57:18 +0300 Subject: [PATCH 083/251] CollectionItem: always run both serializers when serializing. --- django_etesync/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 28dc601..034eaed 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -236,9 +236,13 @@ class CollectionItemViewSet(BaseViewSet): items = request.data.get('items') deps = request.data.get('deps', None) + # FIXME: It should just be one serializer serializer = self.get_serializer_class()(data=items, context=self.get_serializer_context(), many=True) deps_serializer = CollectionItemDepSerializer(data=deps, context=self.get_serializer_context(), many=True) - if serializer.is_valid() and (deps is None or deps_serializer.is_valid()): + + ser_valid = serializer.is_valid() + deps_ser_valid = (deps is None or deps_serializer.is_valid()) + if ser_valid and deps_ser_valid: try: with transaction.atomic(): collections = serializer.save(collection=collection_object) From 4c7e30eca579d32eb4092b09c760483eb0d6ca3a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 13:00:54 +0300 Subject: [PATCH 084/251] CollectionItem: implement both update and create. --- django_etesync/serializers.py | 37 +++++++++++++++-------------------- django_etesync/views.py | 4 +++- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 3f7c28c..5fd6ea8 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -123,36 +123,31 @@ class CollectionItemSerializer(serializers.ModelSerializer): """Function that's called when this serializer creates an item""" stoken = validated_data.pop('stoken') revision_data = validated_data.pop('content') - instance = self.__class__.Meta.model(**validated_data) + uid = validated_data.pop('uid') + + Model = self.__class__.Meta.model with transaction.atomic(): - if stoken is not None: - raise serializers.ValidationError('Stoken is not None') + instance, created = Model.objects.get_or_create(uid=uid, defaults=validated_data) + cur_stoken = instance.stoken if not created else None - instance.save() + if cur_stoken != stoken: + raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(cur_stoken, stoken)) + + if not created: + # We don't have to use select_for_update here because the unique constraint on current guards against + # the race condition. But it's a good idea because it'll lock and wait rather than fail. + current_revision = instance.revisions.filter(current=True).select_for_update().first() + current_revision.current = None + current_revision.save() process_revisions_for_item(instance, revision_data) return instance def update(self, instance, validated_data): - """Function that's called when this serializer is meant to update an item""" - stoken = validated_data.pop('stoken') - revision_data = validated_data.pop('content') - - with transaction.atomic(): - if stoken != instance.stoken: - raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(instance.stoken, stoken)) - - # We don't have to use select_for_update here because the unique constraint on current guards against - # the race condition. But it's a good idea because it'll lock and wait rather than fail. - current_revision = instance.revisions.filter(current=True).select_for_update().first() - current_revision.current = None - current_revision.save() - - process_revisions_for_item(instance, revision_data) - - return instance + # We never update, we always update in the create method + raise NotImplementedError() class CollectionItemDepSerializer(serializers.ModelSerializer): diff --git a/django_etesync/views.py b/django_etesync/views.py index 034eaed..24b62fa 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -187,8 +187,10 @@ class CollectionItemViewSet(BaseViewSet): # We can't have destroy because we need to get data from the user (in the body) such as hmac. return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + def update(self, request, collection_uid=None, uid=None): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + def partial_update(self, request, collection_uid=None, uid=None): - # FIXME: implement, or should it be implemented elsewhere? return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def list(self, request, collection_uid=None): From f7c66eaadb0842386548ee6a9910e2e228094931 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 13:10:50 +0300 Subject: [PATCH 085/251] CollectionItem: add a batch endpoint for batch operations. --- django_etesync/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 24b62fa..c905a54 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -232,6 +232,11 @@ class CollectionItemViewSet(BaseViewSet): } return Response(ret, headers={'X-EteSync-SToken': new_stoken}) + @action_decorator(detail=False, methods=['POST']) + def batch(self, request, collection_uid=None): + # FIXME: different to transaction slightly + return self.transaction(request, collection_uid) + @action_decorator(detail=False, methods=['POST']) def transaction(self, request, collection_uid=None): collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) From eeaea6e6aba90e54fa84bb4b5fcc27e43e526cf4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 13:19:25 +0300 Subject: [PATCH 086/251] Transaction: return 200 rather than 201. --- django_etesync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index c905a54..c56878b 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -261,7 +261,7 @@ class CollectionItemViewSet(BaseViewSet): ret = { "data": [collection.stoken for collection in collections], } - return Response(ret, status=status.HTTP_201_CREATED) + return Response(ret, status=status.HTTP_200_OK) return Response( { From ae4aafcf96f955464c32c6c9fb418f8668937cb3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 14:54:44 +0300 Subject: [PATCH 087/251] Transaction: make it possible to pass a global stoken to block by. --- django_etesync/views.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index c56878b..ad57d28 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -70,16 +70,27 @@ class BaseViewSet(viewsets.ModelViewSet): user = self.request.user return queryset.filter(members__user=user) - def filter_by_stoken_and_limit(self, request, queryset): + def get_stoken_rev(self, request): stoken = request.GET.get('stoken', None) + + if stoken is not None: + return get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) + + return None + + def filter_by_stoken_and_limit(self, request, queryset): limit = int(request.GET.get('limit', 50)) stoken_id_field = self.stoken_id_field + '__id' - if stoken is not None: - last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) + stoken_rev = self.get_stoken_rev(request) + if stoken_rev is not None: + last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken_rev.uid) filter_by = {stoken_id_field + '__gt': last_rev.id} queryset = queryset.filter(**filter_by) + stoken = stoken_rev.uid + else: + stoken = None new_stoken_id = queryset.aggregate(stoken_id=Max(stoken_id_field))['stoken_id'] new_stoken = CollectionItemRevision.objects.get(id=new_stoken_id).uid if new_stoken_id is not None else stoken @@ -239,8 +250,13 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def transaction(self, request, collection_uid=None): + stoken = request.GET.get('stoken', None) collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) + if stoken is not None and stoken != collection_object.stoken: + content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + items = request.data.get('items') deps = request.data.get('deps', None) # FIXME: It should just be one serializer From e851fb98776ddb3025111cbaa315d64c025fbb2b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 15:28:20 +0300 Subject: [PATCH 088/251] Views: fix wrong items name. --- django_etesync/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index ad57d28..9d5d938 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -268,14 +268,14 @@ class CollectionItemViewSet(BaseViewSet): if ser_valid and deps_ser_valid: try: with transaction.atomic(): - collections = serializer.save(collection=collection_object) + items = serializer.save(collection=collection_object) except IntegrityError: # FIXME: should return the items with a bad token (including deps) so we don't have to fetch them after content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) ret = { - "data": [collection.stoken for collection in collections], + "data": [item.stoken for item in items], } return Response(ret, status=status.HTTP_200_OK) From b6571c93f63d6ebb48194b66ff82dda34ca8e9dc Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 15:33:10 +0300 Subject: [PATCH 089/251] Collection: fix stoken and add cstoken for the collection token. --- django_etesync/models.py | 4 +++ django_etesync/serializers.py | 3 +- django_etesync/views.py | 54 +++++++++++++++++------------------ 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index 0d0f3dc..29c7e57 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -46,6 +46,10 @@ class Collection(models.Model): @cached_property def stoken(self): + return self.main_item.stoken + + @cached_property + def cstoken(self): last_revision = CollectionItemRevision.objects.filter(item__collection=self).last() if last_revision is None: # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 5fd6ea8..e5a9e4f 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -169,12 +169,13 @@ class CollectionItemDepSerializer(serializers.ModelSerializer): class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') + cstoken = serializers.CharField(read_only=True) stoken = serializers.CharField(allow_null=True) content = CollectionItemRevisionSerializer(many=False) class Meta: model = models.Collection - fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'stoken') + fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'cstoken', 'stoken') def get_access_level_from_context(self, obj): request = self.context.get('request', None) diff --git a/django_etesync/views.py b/django_etesync/views.py index 9d5d938..081bd37 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -56,7 +56,7 @@ User = get_user_model() class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) - stoken_id_field = None + cstoken_id_field = None def get_serializer_class(self): serializer_class = self.serializer_class @@ -70,32 +70,32 @@ class BaseViewSet(viewsets.ModelViewSet): user = self.request.user return queryset.filter(members__user=user) - def get_stoken_rev(self, request): - stoken = request.GET.get('stoken', None) + def get_cstoken_rev(self, request): + cstoken = request.GET.get('cstoken', None) - if stoken is not None: - return get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken) + if cstoken is not None: + return get_object_or_404(CollectionItemRevision.objects.all(), uid=cstoken) return None - def filter_by_stoken_and_limit(self, request, queryset): + def filter_by_cstoken_and_limit(self, request, queryset): limit = int(request.GET.get('limit', 50)) - stoken_id_field = self.stoken_id_field + '__id' + cstoken_id_field = self.cstoken_id_field + '__id' - stoken_rev = self.get_stoken_rev(request) - if stoken_rev is not None: - last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=stoken_rev.uid) - filter_by = {stoken_id_field + '__gt': last_rev.id} + cstoken_rev = self.get_cstoken_rev(request) + if cstoken_rev is not None: + last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=cstoken_rev.uid) + filter_by = {cstoken_id_field + '__gt': last_rev.id} queryset = queryset.filter(**filter_by) - stoken = stoken_rev.uid + cstoken = cstoken_rev.uid else: - stoken = None + cstoken = None - new_stoken_id = queryset.aggregate(stoken_id=Max(stoken_id_field))['stoken_id'] - new_stoken = CollectionItemRevision.objects.get(id=new_stoken_id).uid if new_stoken_id is not None else stoken + new_cstoken_id = queryset.aggregate(cstoken_id=Max(cstoken_id_field))['cstoken_id'] + new_cstoken = CollectionItemRevision.objects.get(id=new_cstoken_id).uid if new_cstoken_id is not None else cstoken - return queryset[:limit], new_stoken + return queryset[:limit], new_cstoken class CollectionViewSet(BaseViewSet): @@ -104,7 +104,7 @@ class CollectionViewSet(BaseViewSet): queryset = Collection.objects.all() serializer_class = CollectionSerializer lookup_field = 'uid' - stoken_id_field = 'items__revisions' + cstoken_id_field = 'items__revisions' def get_queryset(self, queryset=None): if queryset is None: @@ -139,14 +139,14 @@ class CollectionViewSet(BaseViewSet): def list(self, request): queryset = self.get_queryset() - queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) + queryset, new_cstoken = self.filter_by_cstoken_and_limit(request, queryset) serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) ret = { 'data': serializer.data, } - return Response(ret, headers={'X-EteSync-SToken': new_stoken}) + return Response(ret, headers={'X-EteSync-SToken': new_cstoken}) class CollectionItemViewSet(BaseViewSet): @@ -155,7 +155,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer lookup_field = 'uid' - stoken_id_field = 'revisions' + cstoken_id_field = 'revisions' def get_queryset(self): collection_uid = self.kwargs['collection_uid'] @@ -206,14 +206,14 @@ class CollectionItemViewSet(BaseViewSet): def list(self, request, collection_uid=None): queryset = self.get_queryset() - queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) + queryset, new_cstoken = self.filter_by_cstoken_and_limit(request, queryset) serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) ret = { 'data': serializer.data, } - return Response(ret, headers={'X-EteSync-SToken': new_stoken}) + return Response(ret, headers={'X-EteSync-SToken': new_cstoken}) @action_decorator(detail=True, methods=['GET']) def revision(self, request, collection_uid=None, uid=None): @@ -234,14 +234,14 @@ class CollectionItemViewSet(BaseViewSet): if isinstance(request.data, list): queryset = queryset.filter(uid__in=request.data) - queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) + queryset, new_cstoken = self.filter_by_cstoken_and_limit(request, queryset) serializer = self.get_serializer_class()(queryset, context=self.get_serializer_context(), many=True) ret = { 'data': serializer.data, } - return Response(ret, headers={'X-EteSync-SToken': new_stoken}) + return Response(ret, headers={'X-EteSync-SToken': new_cstoken}) @action_decorator(detail=False, methods=['POST']) def batch(self, request, collection_uid=None): @@ -250,11 +250,11 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def transaction(self, request, collection_uid=None): - stoken = request.GET.get('stoken', None) + cstoken = request.GET.get('cstoken', None) collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - if stoken is not None and stoken != collection_object.stoken: - content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} + if cstoken is not None and cstoken != collection_object.cstoken: + content = {'code': 'stale_cstoken', 'detail': 'CSToken is too old'} return Response(content, status=status.HTTP_400_BAD_REQUEST) items = request.data.get('items') From c63210fe77ff0c3d839174e32b4fe8f91818f590 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 16:16:40 +0300 Subject: [PATCH 090/251] CollectionItem: implement batch updating. --- django_etesync/serializers.py | 3 ++- django_etesync/views.py | 38 +++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index e5a9e4f..773a8a2 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -121,6 +121,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): def create(self, validated_data): """Function that's called when this serializer creates an item""" + validate_stoken = self.context.get('validate_stoken', False) stoken = validated_data.pop('stoken') revision_data = validated_data.pop('content') uid = validated_data.pop('uid') @@ -131,7 +132,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): instance, created = Model.objects.get_or_create(uid=uid, defaults=validated_data) cur_stoken = instance.stoken if not created else None - if cur_stoken != stoken: + if validate_stoken and cur_stoken != stoken: raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(cur_stoken, stoken)) if not created: diff --git a/django_etesync/views.py b/django_etesync/views.py index 081bd37..ab5c0c7 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -245,8 +245,36 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def batch(self, request, collection_uid=None): - # FIXME: different to transaction slightly - return self.transaction(request, collection_uid) + cstoken = request.GET.get('cstoken', None) + collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) + + if cstoken is not None and cstoken != collection_object.cstoken: + content = {'code': 'stale_cstoken', 'detail': 'CSToken is too old'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + items = request.data.get('items') + context = self.get_serializer_context() + serializer = self.get_serializer_class()(data=items, context=context, many=True) + + if serializer.is_valid(): + try: + with transaction.atomic(): + items = serializer.save(collection=collection_object) + except IntegrityError: + # FIXME: should return the items with a bad token (including deps) so we don't have to fetch them after + content = {'code': 'integrity_error'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + ret = { + "data": [item.stoken for item in items], + } + return Response(ret, status=status.HTTP_200_OK) + + return Response( + { + "items": serializer.errors, + }, + status=status.HTTP_400_BAD_REQUEST) @action_decorator(detail=False, methods=['POST']) def transaction(self, request, collection_uid=None): @@ -260,8 +288,10 @@ class CollectionItemViewSet(BaseViewSet): items = request.data.get('items') deps = request.data.get('deps', None) # FIXME: It should just be one serializer - serializer = self.get_serializer_class()(data=items, context=self.get_serializer_context(), many=True) - deps_serializer = CollectionItemDepSerializer(data=deps, context=self.get_serializer_context(), many=True) + context = self.get_serializer_context() + context.update({'validate_stoken': True}) + serializer = self.get_serializer_class()(data=items, context=context, many=True) + deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) ser_valid = serializer.is_valid() deps_ser_valid = (deps is None or deps_serializer.is_valid()) From 9bbb7ef3d75a88ab294187341d7febdd7c4c1fd8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 17:29:54 +0300 Subject: [PATCH 091/251] Fix filter by cstoken function to not fetch twice. --- django_etesync/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index ab5c0c7..aa3c148 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -85,8 +85,7 @@ class BaseViewSet(viewsets.ModelViewSet): cstoken_rev = self.get_cstoken_rev(request) if cstoken_rev is not None: - last_rev = get_object_or_404(CollectionItemRevision.objects.all(), uid=cstoken_rev.uid) - filter_by = {cstoken_id_field + '__gt': last_rev.id} + filter_by = {cstoken_id_field + '__gt': cstoken_rev.id} queryset = queryset.filter(**filter_by) cstoken = cstoken_rev.uid else: From aaee8f5e381f688dd37c8bccc467a37917ad1dc0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 17:41:27 +0300 Subject: [PATCH 092/251] Fix new_cstoken getting for list functions. We were getting the general cstoken, and were not honouring our limit. --- django_etesync/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index aa3c148..5a4101c 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -91,10 +91,11 @@ class BaseViewSet(viewsets.ModelViewSet): else: cstoken = None + queryset = queryset[:limit] new_cstoken_id = queryset.aggregate(cstoken_id=Max(cstoken_id_field))['cstoken_id'] new_cstoken = CollectionItemRevision.objects.get(id=new_cstoken_id).uid if new_cstoken_id is not None else cstoken - return queryset[:limit], new_cstoken + return queryset, new_cstoken class CollectionViewSet(BaseViewSet): From c30cc2f229b5e79f7b8d8e7684c8d52a8115f3bb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 19 May 2020 17:57:51 +0300 Subject: [PATCH 093/251] Improve and rename bulk_get to filter by item stokens too + cstoken changes Also change how we return cstokens --- django_etesync/serializers.py | 8 ++++ django_etesync/views.py | 71 +++++++++++++++++++++++++---------- 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 773a8a2..6413ab1 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -167,6 +167,14 @@ class CollectionItemDepSerializer(serializers.ModelSerializer): return data +class CollectionItemBulkGetSerializer(serializers.ModelSerializer): + stoken = serializers.CharField(required=False) + + class Meta: + model = models.CollectionItem + fields = ('uid', 'stoken') + + class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') diff --git a/django_etesync/views.py b/django_etesync/views.py index 5a4101c..d100d7c 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -43,6 +43,7 @@ from .serializers import ( AuthenticationLoginInnerSerializer, CollectionSerializer, CollectionItemSerializer, + CollectionItemBulkGetSerializer, CollectionItemDepSerializer, CollectionItemRevisionSerializer, CollectionItemChunkSerializer, @@ -78,22 +79,33 @@ class BaseViewSet(viewsets.ModelViewSet): return None - def filter_by_cstoken_and_limit(self, request, queryset): - limit = int(request.GET.get('limit', 50)) - + def filter_by_cstoken(self, request, queryset): cstoken_id_field = self.cstoken_id_field + '__id' cstoken_rev = self.get_cstoken_rev(request) if cstoken_rev is not None: filter_by = {cstoken_id_field + '__gt': cstoken_rev.id} queryset = queryset.filter(**filter_by) - cstoken = cstoken_rev.uid - else: - cstoken = None - queryset = queryset[:limit] + return queryset, cstoken_rev + + def get_queryset_cstoken(self, queryset): + cstoken_id_field = self.cstoken_id_field + '__id' + new_cstoken_id = queryset.aggregate(cstoken_id=Max(cstoken_id_field))['cstoken_id'] - new_cstoken = CollectionItemRevision.objects.get(id=new_cstoken_id).uid if new_cstoken_id is not None else cstoken + new_cstoken = new_cstoken_id and CollectionItemRevision.objects.get(id=new_cstoken_id).uid + + return queryset, new_cstoken + + def filter_by_cstoken_and_limit(self, request, queryset): + limit = int(request.GET.get('limit', 50)) + + queryset, cstoken_rev = self.filter_by_cstoken(request, queryset) + cstoken = cstoken_rev.uid if cstoken_rev is not None else None + + queryset = queryset[:limit] + queryset, new_cstoken = self.get_queryset_cstoken(queryset) + new_cstoken = new_cstoken or cstoken return queryset, new_cstoken @@ -145,8 +157,9 @@ class CollectionViewSet(BaseViewSet): ret = { 'data': serializer.data, + 'cstoken': new_cstoken, } - return Response(ret, headers={'X-EteSync-SToken': new_cstoken}) + return Response(ret) class CollectionItemViewSet(BaseViewSet): @@ -212,8 +225,9 @@ class CollectionItemViewSet(BaseViewSet): ret = { 'data': serializer.data, + 'cstoken': new_cstoken, } - return Response(ret, headers={'X-EteSync-SToken': new_cstoken}) + return Response(ret) @action_decorator(detail=True, methods=['GET']) def revision(self, request, collection_uid=None, uid=None): @@ -227,21 +241,40 @@ class CollectionItemViewSet(BaseViewSet): } return Response(ret) + # FIXME: rename to something consistent with what the clients have - maybe list_updates? @action_decorator(detail=False, methods=['POST']) - def bulk_get(self, request, collection_uid=None): + def fetch_updates(self, request, collection_uid=None): queryset = self.get_queryset() - if isinstance(request.data, list): - queryset = queryset.filter(uid__in=request.data) + serializer = CollectionItemBulkGetSerializer(data=request.data, many=True) + if serializer.is_valid(): + # FIXME: make configurable? + item_limit = 200 - queryset, new_cstoken = self.filter_by_cstoken_and_limit(request, queryset) + if len(serializer.validated_data) > item_limit: + content = {'code': 'too_many_items', + 'detail': 'Request has too many items. Limit: {}'. format(item_limit)} + return Response(content, status=status.HTTP_400_BAD_REQUEST) - serializer = self.get_serializer_class()(queryset, context=self.get_serializer_context(), many=True) + queryset, cstoken_rev = self.filter_by_cstoken(request, queryset) - ret = { - 'data': serializer.data, - } - return Response(ret, headers={'X-EteSync-SToken': new_cstoken}) + uids, stokens = zip(*[(item['uid'], item.get('stoken')) for item in serializer.validated_data]) + rev_ids = CollectionItemRevision.objects.filter(uid__in=stokens, current=True).values_list('id', flat=True) + queryset = queryset.filter(uid__in=uids).exclude(revisions__id__in=rev_ids) + + queryset, new_cstoken = self.get_queryset_cstoken(queryset) + cstoken = cstoken_rev and cstoken_rev.uid + new_cstoken = new_cstoken or cstoken + + serializer = self.get_serializer_class()(queryset, context=self.get_serializer_context(), many=True) + + ret = { + 'data': serializer.data, + 'cstoken': new_cstoken, + } + return Response(ret) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @action_decorator(detail=False, methods=['POST']) def batch(self, request, collection_uid=None): From 4ca74bc69ba6616d5edf99ef30d36a66c6b07010 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 20 May 2020 13:47:06 +0300 Subject: [PATCH 094/251] Permissions: start from scratch and add IsCollectionAdmin permission. --- django_etesync/permissions.py | 53 ++++++++--------------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/django_etesync/permissions.py b/django_etesync/permissions.py index f553930..29806c6 100644 --- a/django_etesync/permissions.py +++ b/django_etesync/permissions.py @@ -13,53 +13,22 @@ # along with this program. If not, see . from rest_framework import permissions -from journal.models import Journal, JournalMember +from django_etesync.models import Collection, AccessLevels -class IsOwnerOrReadOnly(permissions.BasePermission): +class IsCollectionAdmin(permissions.BasePermission): """ - Custom permission to only allow owners of an object to edit it. - """ - - def has_object_permission(self, request, view, obj): - if request.method in permissions.SAFE_METHODS: - return True - - return obj.owner == request.user - - -class IsJournalOwner(permissions.BasePermission): - """ - Custom permission to only allow owners of a journal to view it - """ - - def has_permission(self, request, view): - journal_uid = view.kwargs['journal_uid'] - try: - journal = view.get_journal_queryset().get(uid=journal_uid) - return journal.owner == request.user - except Journal.DoesNotExist: - # If the journal does not exist, we want to 404 later, not permission denied. - return True - - -class IsMemberReadOnly(permissions.BasePermission): - """ - Custom permission to make a journal read only if a read only member + Custom permission to only allow owners of a collection to view it """ + message = 'Only collection admins can perform this operation.' + code = 'admin_access_required' def has_permission(self, request, view): - if request.method in permissions.SAFE_METHODS: - return True - - journal_uid = view.kwargs['journal_uid'] + collection_uid = view.kwargs['collection_uid'] try: - journal = view.get_journal_queryset().get(uid=journal_uid) - member = journal.members.get(user=request.user) - return not member.readOnly - except Journal.DoesNotExist: - # If the journal does not exist, we want to 404 later, not permission denied. - return True - except JournalMember.DoesNotExist: - # Not being a member means we are the owner. + collection = view.get_collection_queryset().get(uid=collection_uid) + member = collection.members.filter(user=request.user).first() + return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) + except Collection.DoesNotExist: + # If the collection does not exist, we want to 404 later, not permission denied. return True From edd88427b0fc739a8610f1e6135680f1627f7c47 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 20 May 2020 13:48:46 +0300 Subject: [PATCH 095/251] Add a viewset to control collection membership. --- django_etesync/models.py | 11 ++++++----- django_etesync/serializers.py | 26 +++++++++++++++++++++++++- django_etesync/views.py | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index 29c7e57..36851a1 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -121,12 +121,13 @@ class RevisionChunkRelation(models.Model): ordering = ('id', ) -class CollectionMember(models.Model): - class AccessLevels(models.TextChoices): - ADMIN = 'adm' - READ_WRITE = 'rw' - READ_ONLY = 'ro' +class AccessLevels(models.TextChoices): + ADMIN = 'adm' + READ_WRITE = 'rw' + READ_ONLY = 'ro' + +class CollectionMember(models.Model): collection = models.ForeignKey(Collection, related_name='members', on_delete=models.CASCADE) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) encryptionKey = models.BinaryField(editable=True, blank=False, null=False) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 6413ab1..06bb7a9 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -211,7 +211,7 @@ class CollectionSerializer(serializers.ModelSerializer): models.CollectionMember(collection=instance, user=validated_data.get('owner'), - accessLevel=models.CollectionMember.AccessLevels.ADMIN, + accessLevel=models.AccessLevels.ADMIN, encryptionKey=encryption_key, ).save() @@ -238,6 +238,30 @@ class CollectionSerializer(serializers.ModelSerializer): return instance +class CollectionMemberSerializer(serializers.ModelSerializer): + username = serializers.SlugRelatedField( + source='user', + slug_field=User.USERNAME_FIELD, + queryset=User.objects + ) + encryptionKey = BinaryBase64Field() + + class Meta: + model = models.CollectionMember + fields = ('username', 'encryptionKey', 'accessLevel') + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + with transaction.atomic(): + # We only allow updating accessLevel + instance.accessLevel = validated_data.pop('accessLevel') + instance.save() + + return instance + + class UserSerializer(serializers.ModelSerializer): class Meta: model = User diff --git a/django_etesync/views.py b/django_etesync/views.py index d100d7c..5703410 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -33,8 +33,8 @@ import nacl.signing import nacl.secret import nacl.hash -from . import app_settings -from .models import Collection, CollectionItem, CollectionItemRevision +from . import app_settings, permissions +from .models import Collection, CollectionItem, CollectionItemRevision, CollectionMember from .serializers import ( b64encode, AuthenticationSignupSerializer, @@ -47,6 +47,7 @@ from .serializers import ( CollectionItemDepSerializer, CollectionItemRevisionSerializer, CollectionItemChunkSerializer, + CollectionMemberSerializer, UserSerializer, ) @@ -395,6 +396,33 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return serve(request, basename, dirname) +class CollectionMemberViewSet(BaseViewSet): + allowed_methods = ['GET', 'PUT', 'DELETE'] + permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdmin, ) + queryset = CollectionMember.objects.all() + serializer_class = CollectionMemberSerializer + lookup_field = 'user__' + User.USERNAME_FIELD + lookup_url_kwarg = 'username' + + # FIXME: need to make sure that there's always an admin, and maybe also don't let an owner remove adm access + # (if we want to transfer, we need to do that specifically) + + def get_queryset(self, queryset=None): + collection_uid = self.kwargs['collection_uid'] + try: + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + except Collection.DoesNotExist: + raise Http404('Collection does not exist') + + if queryset is None: + queryset = type(self).queryset + + return queryset.filter(collection=collection) + + def create(self, request): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + class AuthenticationViewSet(viewsets.ViewSet): allowed_methods = ['POST'] From 8d1c02dcb99944e38dd69e79f2ae088871811990 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 20 May 2020 14:30:09 +0300 Subject: [PATCH 096/251] Collection invitation: implement creating and manipulating collections invitations. --- .../migrations/0003_collectioninvitation.py | 31 +++++++++++++++ .../0004_collectioninvitation_version.py | 18 +++++++++ django_etesync/models.py | 22 +++++++++++ django_etesync/serializers.py | 39 +++++++++++++++++++ django_etesync/views.py | 36 ++++++++++++++++- 5 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0003_collectioninvitation.py create mode 100644 django_etesync/migrations/0004_collectioninvitation_version.py diff --git a/django_etesync/migrations/0003_collectioninvitation.py b/django_etesync/migrations/0003_collectioninvitation.py new file mode 100644 index 0000000..3880a63 --- /dev/null +++ b/django_etesync/migrations/0003_collectioninvitation.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.3 on 2020-05-20 11:03 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('django_etesync', '0002_userinfo'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionInvitation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), + ('signedEncryptionKey', models.BinaryField()), + ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), + ('fromMember', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etesync.CollectionMember')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='incoming_invitations', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'fromMember')}, + }, + ), + ] diff --git a/django_etesync/migrations/0004_collectioninvitation_version.py b/django_etesync/migrations/0004_collectioninvitation_version.py new file mode 100644 index 0000000..3fbaed9 --- /dev/null +++ b/django_etesync/migrations/0004_collectioninvitation_version.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-05-21 14:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0003_collectioninvitation'), + ] + + operations = [ + migrations.AddField( + model_name='collectioninvitation', + name='version', + field=models.PositiveSmallIntegerField(default=1), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 36851a1..e0a79f6 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -144,6 +144,28 @@ class CollectionMember(models.Model): return '{} {}'.format(self.collection.uid, self.user) +class CollectionInvitation(models.Model): + uid = models.CharField(db_index=True, blank=False, null=False, + max_length=44, validators=[Base64Url256BitValidator]) + version = models.PositiveSmallIntegerField(default=1) + fromMember = models.ForeignKey(CollectionMember, on_delete=models.CASCADE) + # FIXME: make sure to delete all invitations for the same collection once one is accepted + + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='incoming_invitations', on_delete=models.CASCADE) + signedEncryptionKey = models.BinaryField(editable=False, blank=False, null=False) + accessLevel = models.CharField( + max_length=3, + choices=AccessLevels.choices, + default=AccessLevels.READ_ONLY, + ) + + class Meta: + unique_together = ('user', 'fromMember') + + def __str__(self): + return '{} {}'.format(self.fromMember.collection.uid, self.user) + + class UserInfo(models.Model): owner = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, primary_key=True) version = models.PositiveSmallIntegerField(default=1) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 06bb7a9..54b8d9c 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -262,6 +262,45 @@ class CollectionMemberSerializer(serializers.ModelSerializer): return instance +class CollectionInvitationSerializer(serializers.ModelSerializer): + username = serializers.SlugRelatedField( + source='user', + slug_field=User.USERNAME_FIELD, + queryset=User.objects + ) + collection = serializers.SlugRelatedField( + source='fromMember__collection', + slug_field='uid', + read_only=True, + ) + fromPubkey = BinaryBase64Field(source='fromMember__user__userinfo__pubkey', read_only=True) + signedEncryptionKey = BinaryBase64Field() + + class Meta: + model = models.CollectionInvitation + fields = ('username', 'uid', 'collection', 'signedEncryptionKey', 'accessLevel', 'fromPubkey', 'version') + + def create(self, validated_data): + collection = self.context['collection'] + request = self.context['request'] + + if request.user == validated_data.get('user'): + raise serializers.ValidationError('Inviting yourself is not allowed') + + member = collection.members.get(user=request.user) + + with transaction.atomic(): + return type(self).Meta.model.objects.create(**validated_data, fromMember=member) + + def update(self, instance, validated_data): + with transaction.atomic(): + instance.accessLevel = validated_data.pop('accessLevel') + instance.signedEncryptionKey = validated_data.pop('signedEncryptionKey') + instance.save() + + return instance + + class UserSerializer(serializers.ModelSerializer): class Meta: model = User diff --git a/django_etesync/views.py b/django_etesync/views.py index 5703410..ffb9503 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -34,7 +34,7 @@ import nacl.secret import nacl.hash from . import app_settings, permissions -from .models import Collection, CollectionItem, CollectionItemRevision, CollectionMember +from .models import Collection, CollectionItem, CollectionItemRevision, CollectionMember, CollectionInvitation from .serializers import ( b64encode, AuthenticationSignupSerializer, @@ -48,6 +48,7 @@ from .serializers import ( CollectionItemRevisionSerializer, CollectionItemChunkSerializer, CollectionMemberSerializer, + CollectionInvitationSerializer, UserSerializer, ) @@ -423,6 +424,38 @@ class CollectionMemberViewSet(BaseViewSet): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) +class CollectionInvitationViewSet(BaseViewSet): + allowed_methods = ['GET', 'POST', 'PUT', 'DELETE'] + permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdmin, ) + queryset = CollectionInvitation.objects.all() + serializer_class = CollectionInvitationSerializer + lookup_field = 'uid' + lookup_url_kwarg = 'invitation_uid' + + def get_serializer_context(self): + context = super().get_serializer_context() + collection_uid = self.kwargs['collection_uid'] + try: + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + except Collection.DoesNotExist: + raise Http404('Collection does not exist') + + context.update({'request': self.request, 'collection': collection}) + return context + + def get_queryset(self, queryset=None): + collection_uid = self.kwargs['collection_uid'] + try: + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + except Collection.DoesNotExist: + raise Http404('Collection does not exist') + + if queryset is None: + queryset = type(self).queryset + + return queryset.filter(fromMember__collection=collection) + + class AuthenticationViewSet(viewsets.ViewSet): allowed_methods = ['POST'] @@ -561,6 +594,7 @@ class TestAuthenticationViewSet(viewsets.ViewSet): # Delete all of the journal data for this user for a clear test env request.user.collection_set.all().delete() + request.user.incoming_invitations.all().delete() # FIXME: also delete chunk files!!! From 47e1eec122eec8c3cdba484ba098f3327e9551fe Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 20 May 2020 15:15:24 +0300 Subject: [PATCH 097/251] Incoming invitations: implement incoming invitations and accepting them --- django_etesync/models.py | 5 +++++ django_etesync/serializers.py | 38 +++++++++++++++++++++++++++++------ django_etesync/views.py | 26 ++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index e0a79f6..cbfa269 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -150,6 +150,7 @@ class CollectionInvitation(models.Model): version = models.PositiveSmallIntegerField(default=1) fromMember = models.ForeignKey(CollectionMember, on_delete=models.CASCADE) # FIXME: make sure to delete all invitations for the same collection once one is accepted + # Make sure to not allow invitations if already a member user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='incoming_invitations', on_delete=models.CASCADE) signedEncryptionKey = models.BinaryField(editable=False, blank=False, null=False) @@ -165,6 +166,10 @@ class CollectionInvitation(models.Model): def __str__(self): return '{} {}'.format(self.fromMember.collection.uid, self.user) + @cached_property + def collection(self): + return self.fromMember.collection + class UserInfo(models.Model): owner = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, primary_key=True) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 54b8d9c..185eb6e 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -268,18 +268,20 @@ class CollectionInvitationSerializer(serializers.ModelSerializer): slug_field=User.USERNAME_FIELD, queryset=User.objects ) - collection = serializers.SlugRelatedField( - source='fromMember__collection', - slug_field='uid', - read_only=True, - ) - fromPubkey = BinaryBase64Field(source='fromMember__user__userinfo__pubkey', read_only=True) + collection = serializers.SerializerMethodField('get_collection') + fromPubkey = serializers.SerializerMethodField('get_from_pubkey') signedEncryptionKey = BinaryBase64Field() class Meta: model = models.CollectionInvitation fields = ('username', 'uid', 'collection', 'signedEncryptionKey', 'accessLevel', 'fromPubkey', 'version') + def get_collection(self, obj): + return obj.collection.uid + + def get_from_pubkey(self, obj): + return b64encode(obj.fromMember.user.userinfo.pubkey) + def create(self, validated_data): collection = self.context['collection'] request = self.context['request'] @@ -301,6 +303,30 @@ class CollectionInvitationSerializer(serializers.ModelSerializer): return instance +class InvitationAcceptSerializer(serializers.Serializer): + encryptionKey = BinaryBase64Field() + + def create(self, validated_data): + + with transaction.atomic(): + invitation = self.context['invitation'] + encryption_key = validated_data.get('encryptionKey') + + member = models.CollectionMember.objects.create( + collection=invitation.collection, + user=invitation.user, + accessLevel=invitation.accessLevel, + encryptionKey=encryption_key, + ) + + invitation.delete() + + return member + + def update(self, instance, validated_data): + raise NotImplementedError() + + class UserSerializer(serializers.ModelSerializer): class Meta: model = User diff --git a/django_etesync/views.py b/django_etesync/views.py index ffb9503..a1d6d09 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -49,6 +49,7 @@ from .serializers import ( CollectionItemChunkSerializer, CollectionMemberSerializer, CollectionInvitationSerializer, + InvitationAcceptSerializer, UserSerializer, ) @@ -456,6 +457,31 @@ class CollectionInvitationViewSet(BaseViewSet): return queryset.filter(fromMember__collection=collection) +class InvitationIncomingViewSet(BaseViewSet): + allowed_methods = ['GET', 'DELETE'] + queryset = CollectionInvitation.objects.all() + serializer_class = CollectionInvitationSerializer + lookup_field = 'uid' + lookup_url_kwarg = 'invitation_uid' + + def get_queryset(self, queryset=None): + if queryset is None: + queryset = type(self).queryset + + return queryset.filter(user=self.request.user) + + @action_decorator(detail=True, allowed_methods=['POST'], methods=['POST']) + def accept(self, request, invitation_uid=None): + invitation = get_object_or_404(self.get_queryset(), uid=invitation_uid) + context = self.get_serializer_context() + context.update({'invitation': invitation}) + + serializer = InvitationAcceptSerializer(data=request.data, context=context) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(status=status.HTTP_201_CREATED) + + class AuthenticationViewSet(viewsets.ViewSet): allowed_methods = ['POST'] From 40b7edcb84794ab9eb4a70eb4cf55a9b6985816e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 24 May 2020 15:20:55 +0300 Subject: [PATCH 098/251] Add a way to fetch a user's pubkey. --- django_etesync/serializers.py | 8 ++++++++ django_etesync/views.py | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 185eb6e..1cc4a7e 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -339,6 +339,14 @@ class UserQuerySerializer(serializers.ModelSerializer): fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) +class UserInfoPubkeySerializer(serializers.ModelSerializer): + pubkey = BinaryBase64Field() + + class Meta: + model = models.UserInfo + fields = ('pubkey', ) + + class AuthenticationSignupSerializer(serializers.Serializer): user = UserQuerySerializer(many=False) salt = BinaryBase64Field() diff --git a/django_etesync/views.py b/django_etesync/views.py index a1d6d09..7130eae 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -34,7 +34,7 @@ import nacl.secret import nacl.hash from . import app_settings, permissions -from .models import Collection, CollectionItem, CollectionItemRevision, CollectionMember, CollectionInvitation +from .models import Collection, CollectionItem, CollectionItemRevision, CollectionMember, CollectionInvitation, UserInfo from .serializers import ( b64encode, AuthenticationSignupSerializer, @@ -50,6 +50,7 @@ from .serializers import ( CollectionMemberSerializer, CollectionInvitationSerializer, InvitationAcceptSerializer, + UserInfoPubkeySerializer, UserSerializer, ) @@ -456,6 +457,14 @@ class CollectionInvitationViewSet(BaseViewSet): return queryset.filter(fromMember__collection=collection) + @action_decorator(detail=False, allowed_methods=['GET'], methods=['GET']) + def fetch_user_profile(self, request, collection_uid=None): + username = request.GET.get('username') + kwargs = {'owner__' + User.USERNAME_FIELD: username} + user_info = get_object_or_404(UserInfo.objects.all(), **kwargs) + serializer = UserInfoPubkeySerializer(user_info) + return Response(serializer.data) + class InvitationIncomingViewSet(BaseViewSet): allowed_methods = ['GET', 'DELETE'] From 7f7d223b9b10cdc7dc2418befe060ae5deb83f30 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 24 May 2020 17:22:43 +0300 Subject: [PATCH 099/251] Fix indentation error. --- django_etesync/serializers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 1cc4a7e..21ce6a4 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -313,11 +313,11 @@ class InvitationAcceptSerializer(serializers.Serializer): encryption_key = validated_data.get('encryptionKey') member = models.CollectionMember.objects.create( - collection=invitation.collection, - user=invitation.user, - accessLevel=invitation.accessLevel, - encryptionKey=encryption_key, - ) + collection=invitation.collection, + user=invitation.user, + accessLevel=invitation.accessLevel, + encryptionKey=encryption_key, + ) invitation.delete() From 118dbea4e34bbdfc06fec9b6b42d219a10369ce0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 24 May 2020 17:52:09 +0300 Subject: [PATCH 100/251] InvitationSerializer: fix user validator. --- django_etesync/serializers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 21ce6a4..95dbc1f 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -282,13 +282,16 @@ class CollectionInvitationSerializer(serializers.ModelSerializer): def get_from_pubkey(self, obj): return b64encode(obj.fromMember.user.userinfo.pubkey) - def create(self, validated_data): - collection = self.context['collection'] + def validate_user(self, value): request = self.context['request'] - if request.user == validated_data.get('user'): + if request.user == value: raise serializers.ValidationError('Inviting yourself is not allowed') + def create(self, validated_data): + collection = self.context['collection'] + request = self.context['request'] + member = collection.members.get(user=request.user) with transaction.atomic(): From a965a76c36081958ee9f3a495ac295e0240f9045 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 24 May 2020 18:19:22 +0300 Subject: [PATCH 101/251] Invitation: move outgoing invitations to invite/outgoing. --- django_etesync/permissions.py | 8 +++++-- django_etesync/serializers.py | 15 +++++-------- django_etesync/views.py | 42 +++++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/django_etesync/permissions.py b/django_etesync/permissions.py index 29806c6..c371743 100644 --- a/django_etesync/permissions.py +++ b/django_etesync/permissions.py @@ -16,6 +16,11 @@ from rest_framework import permissions from django_etesync.models import Collection, AccessLevels +def is_collection_admin(collection, user): + member = collection.members.filter(user=user).first() + return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) + + class IsCollectionAdmin(permissions.BasePermission): """ Custom permission to only allow owners of a collection to view it @@ -27,8 +32,7 @@ class IsCollectionAdmin(permissions.BasePermission): collection_uid = view.kwargs['collection_uid'] try: collection = view.get_collection_queryset().get(uid=collection_uid) - member = collection.members.filter(user=request.user).first() - return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) + return is_collection_admin(collection, request.user) except Collection.DoesNotExist: # If the collection does not exist, we want to 404 later, not permission denied. return True diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 95dbc1f..29068fa 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -268,29 +268,24 @@ class CollectionInvitationSerializer(serializers.ModelSerializer): slug_field=User.USERNAME_FIELD, queryset=User.objects ) - collection = serializers.SerializerMethodField('get_collection') - fromPubkey = serializers.SerializerMethodField('get_from_pubkey') + collection = serializers.CharField(source='collection.uid') + fromPubkey = BinaryBase64Field(source='fromMember.user.userinfo.pubkey', read_only=True) signedEncryptionKey = BinaryBase64Field() class Meta: model = models.CollectionInvitation fields = ('username', 'uid', 'collection', 'signedEncryptionKey', 'accessLevel', 'fromPubkey', 'version') - def get_collection(self, obj): - return obj.collection.uid - - def get_from_pubkey(self, obj): - return b64encode(obj.fromMember.user.userinfo.pubkey) - def validate_user(self, value): request = self.context['request'] - if request.user == value: + if request.user == value.lower(): raise serializers.ValidationError('Inviting yourself is not allowed') + return value def create(self, validated_data): - collection = self.context['collection'] request = self.context['request'] + collection = validated_data.pop('collection') member = collection.members.get(user=request.user) diff --git a/django_etesync/views.py b/django_etesync/views.py index 7130eae..bde22e4 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -16,6 +16,7 @@ import json from django.conf import settings from django.contrib.auth import get_user_model +from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError from django.db.models import Max from django.http import HttpResponseBadRequest, HttpResponse, Http404 @@ -426,9 +427,9 @@ class CollectionMemberViewSet(BaseViewSet): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) -class CollectionInvitationViewSet(BaseViewSet): +class InvitationOutgoingViewSet(BaseViewSet): allowed_methods = ['GET', 'POST', 'PUT', 'DELETE'] - permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdmin, ) + permission_classes = BaseViewSet.permission_classes queryset = CollectionInvitation.objects.all() serializer_class = CollectionInvitationSerializer lookup_field = 'uid' @@ -436,29 +437,36 @@ class CollectionInvitationViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - collection_uid = self.kwargs['collection_uid'] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404('Collection does not exist') - - context.update({'request': self.request, 'collection': collection}) + context.update({'request': self.request}) return context def get_queryset(self, queryset=None): - collection_uid = self.kwargs['collection_uid'] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404('Collection does not exist') - if queryset is None: queryset = type(self).queryset - return queryset.filter(fromMember__collection=collection) + return queryset.filter(fromMember__user=self.request.user) + + def create(self, request, *args, **kwargs): + serializer = self.serializer_class(data=request.data, context=self.get_serializer_context()) + if serializer.is_valid(): + collection_uid = serializer.validated_data.get('collection', {}).get('uid') + + try: + collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + except Collection.DoesNotExist: + raise Http404('Collection does not exist') + + if not permissions.is_collection_admin(collection, request.user): + raise PermissionDenied('User is not an admin of this collection') + + serializer.save(collection=collection) + + return Response({}, status=status.HTTP_201_CREATED) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @action_decorator(detail=False, allowed_methods=['GET'], methods=['GET']) - def fetch_user_profile(self, request, collection_uid=None): + def fetch_user_profile(self, request): username = request.GET.get('username') kwargs = {'owner__' + User.USERNAME_FIELD: username} user_info = get_object_or_404(UserInfo.objects.all(), **kwargs) From 2412c295de6a60393573ed65160fcf8a1f793096 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 13:17:35 +0300 Subject: [PATCH 102/251] Signup: fix bug making signup not to work. --- django_etesync/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 29068fa..650b981 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -357,7 +357,7 @@ class AuthenticationSignupSerializer(serializers.Serializer): pubkey = validated_data.pop('pubkey') with transaction.atomic(): - instance = User.objects.get_or_create(**user_data) + instance, _ = User.objects.get_or_create(**user_data) if hasattr(instance, 'userinfo'): raise serializers.ValidationError('User already exists') From 863c405802ecb5c8cdc394c94b76b2a673864cb1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 13:23:45 +0300 Subject: [PATCH 103/251] Rename pubkey to loginPubkey because we'll soon have another pubkey. This breaks sharing because we no longer have a normal pubkey. This will be fixed in the next commit. --- .../migrations/0005_auto_20200526_1021.py | 18 ++++++++++++++++++ django_etesync/models.py | 2 +- django_etesync/serializers.py | 6 ++---- django_etesync/views.py | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 django_etesync/migrations/0005_auto_20200526_1021.py diff --git a/django_etesync/migrations/0005_auto_20200526_1021.py b/django_etesync/migrations/0005_auto_20200526_1021.py new file mode 100644 index 0000000..470556b --- /dev/null +++ b/django_etesync/migrations/0005_auto_20200526_1021.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-05-26 10:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0004_collectioninvitation_version'), + ] + + operations = [ + migrations.RenameField( + model_name='userinfo', + old_name='pubkey', + new_name='loginPubkey', + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index cbfa269..62c3868 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -174,7 +174,7 @@ class CollectionInvitation(models.Model): class UserInfo(models.Model): owner = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, primary_key=True) version = models.PositiveSmallIntegerField(default=1) - pubkey = models.BinaryField(editable=True, blank=False, null=False) + loginPubkey = models.BinaryField(editable=True, blank=False, null=False) salt = models.BinaryField(editable=True, blank=False, null=False) def __str__(self): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 650b981..5772000 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -348,13 +348,11 @@ class UserInfoPubkeySerializer(serializers.ModelSerializer): class AuthenticationSignupSerializer(serializers.Serializer): user = UserQuerySerializer(many=False) salt = BinaryBase64Field() - pubkey = BinaryBase64Field() + loginPubkey = BinaryBase64Field() def create(self, validated_data): """Function that's called when this serializer creates an item""" user_data = validated_data.pop('user') - salt = validated_data.pop('salt') - pubkey = validated_data.pop('pubkey') with transaction.atomic(): instance, _ = User.objects.get_or_create(**user_data) @@ -364,7 +362,7 @@ class AuthenticationSignupSerializer(serializers.Serializer): instance.set_unusable_password() # FIXME: send email verification - models.UserInfo.objects.create(salt=salt, pubkey=pubkey, owner=instance) + models.UserInfo.objects.create(**validated_data, owner=instance) return instance diff --git a/django_etesync/views.py b/django_etesync/views.py index bde22e4..916c2a2 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -603,7 +603,7 @@ class AuthenticationViewSet(viewsets.ViewSet): content = {'code': 'wrong_host', 'detail': detail} return Response(content, status=status.HTTP_400_BAD_REQUEST) - verify_key = nacl.signing.VerifyKey(user.userinfo.pubkey, encoder=nacl.encoding.RawEncoder) + verify_key = nacl.signing.VerifyKey(user.userinfo.loginPubkey, encoder=nacl.encoding.RawEncoder) verify_key.verify(response_raw, signature) data = self.login_response_data(user) From e94e2f9d70224301f34b388578c68b951e74a72e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 13:44:40 +0300 Subject: [PATCH 104/251] Add a separate pubkey/privatekey for sharing. It's separated from the login one so that encryption key and identity can be rotated separately. --- .../migrations/0006_auto_20200526_1040.py | 25 +++++++++++++++++++ django_etesync/models.py | 2 ++ django_etesync/serializers.py | 7 +++++- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0006_auto_20200526_1040.py diff --git a/django_etesync/migrations/0006_auto_20200526_1040.py b/django_etesync/migrations/0006_auto_20200526_1040.py new file mode 100644 index 0000000..84e8fa3 --- /dev/null +++ b/django_etesync/migrations/0006_auto_20200526_1040.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-05-26 10:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0005_auto_20200526_1021'), + ] + + operations = [ + migrations.AddField( + model_name='userinfo', + name='encryptedSeckey', + field=models.BinaryField(default=b'', editable=True), + preserve_default=False, + ), + migrations.AddField( + model_name='userinfo', + name='pubkey', + field=models.BinaryField(default=b'', editable=True), + preserve_default=False, + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 62c3868..edfb18d 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -175,6 +175,8 @@ class UserInfo(models.Model): owner = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, primary_key=True) version = models.PositiveSmallIntegerField(default=1) loginPubkey = models.BinaryField(editable=True, blank=False, null=False) + pubkey = models.BinaryField(editable=True, blank=False, null=False) + encryptedSeckey = models.BinaryField(editable=True, blank=False, null=False) salt = models.BinaryField(editable=True, blank=False, null=False) def __str__(self): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 5772000..bff20ad 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -326,9 +326,12 @@ class InvitationAcceptSerializer(serializers.Serializer): class UserSerializer(serializers.ModelSerializer): + pubkey = BinaryBase64Field(source='userinfo.pubkey') + encryptedSeckey = BinaryBase64Field(source='userinfo.encryptedSeckey') + class Meta: model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) + fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, 'pubkey', 'encryptedSeckey') class UserQuerySerializer(serializers.ModelSerializer): @@ -349,6 +352,8 @@ class AuthenticationSignupSerializer(serializers.Serializer): user = UserQuerySerializer(many=False) salt = BinaryBase64Field() loginPubkey = BinaryBase64Field() + pubkey = BinaryBase64Field() + encryptedSeckey = BinaryBase64Field() def create(self, validated_data): """Function that's called when this serializer creates an item""" From 10b9d33ffe95c4ab68ee8a784c1517a3327a6fcf Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 16:13:18 +0300 Subject: [PATCH 105/251] UidValidator: fix to actually validate. --- django_etesync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index edfb18d..98f57f2 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -21,7 +21,7 @@ from django.utils.functional import cached_property Base64Url256BitValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{43}$', message='Expected a 256bit base64url.') -UidValidator = RegexValidator(regex=r'[a-zA-Z0-9]', message='Not a valid UID') +UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9]*$', message='Not a valid UID') class Collection(models.Model): From fce844bfc3c7078167b2e34edf1b0b9696197807 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 16:26:57 +0300 Subject: [PATCH 106/251] Uid: Change how validation is done. --- .../migrations/0007_auto_20200526_1336.py | 39 +++++++++++++++++++ django_etesync/models.py | 12 +++--- 2 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 django_etesync/migrations/0007_auto_20200526_1336.py diff --git a/django_etesync/migrations/0007_auto_20200526_1336.py b/django_etesync/migrations/0007_auto_20200526_1336.py new file mode 100644 index 0000000..37e31ac --- /dev/null +++ b/django_etesync/migrations/0007_auto_20200526_1336.py @@ -0,0 +1,39 @@ +# Generated by Django 3.0.3 on 2020-05-26 13:36 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0006_auto_20200526_1040'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9]*$')]), + ), + migrations.AlterField( + model_name='collectioninvitation', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9]*$')]), + ), + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')]), + ), + migrations.AlterField( + model_name='collectionitemrevision', + name='uid', + field=models.CharField(db_index=True, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')]), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 98f57f2..3a286ce 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -20,13 +20,13 @@ from django.core.validators import RegexValidator from django.utils.functional import cached_property -Base64Url256BitValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{43}$', message='Expected a 256bit base64url.') +Base64Url256BitlValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{42,43}$', message='Expected a base64url.') UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9]*$', message='Not a valid UID') class Collection(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, - max_length=44, validators=[UidValidator]) + max_length=43, validators=[UidValidator]) version = models.PositiveSmallIntegerField() owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) @@ -60,7 +60,7 @@ class Collection(models.Model): class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=True, - max_length=44, validators=[UidValidator]) + max_length=43, validators=[UidValidator]) collection = models.ForeignKey(Collection, related_name='items', on_delete=models.CASCADE) version = models.PositiveSmallIntegerField() encryptionKey = models.BinaryField(editable=True, blank=False, null=True) @@ -90,7 +90,7 @@ def chunk_directory_path(instance, filename): class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, - max_length=44, validators=[Base64Url256BitValidator]) + max_length=43, validators=[Base64Url256BitlValidator]) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) @@ -100,7 +100,7 @@ class CollectionItemChunk(models.Model): class CollectionItemRevision(models.Model): uid = models.CharField(db_index=True, unique=True, blank=False, null=False, - max_length=44, validators=[Base64Url256BitValidator]) + max_length=43, validators=[Base64Url256BitlValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) meta = models.BinaryField(editable=True, blank=False, null=False) current = models.BooleanField(db_index=True, default=True, null=True) @@ -146,7 +146,7 @@ class CollectionMember(models.Model): class CollectionInvitation(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, - max_length=44, validators=[Base64Url256BitValidator]) + max_length=43, validators=[Base64Url256BitlValidator]) version = models.PositiveSmallIntegerField(default=1) fromMember = models.ForeignKey(CollectionMember, on_delete=models.CASCADE) # FIXME: make sure to delete all invitations for the same collection once one is accepted From 3cdb7783fe6ac2cfcc9840c6e4e6d9c39dd4f917 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 18:14:39 +0300 Subject: [PATCH 107/251] Make sure to always return fresh stokens. --- django_etesync/models.py | 4 ++-- django_etesync/views.py | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index 3a286ce..ba6665c 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -44,7 +44,7 @@ class Collection(models.Model): def content(self): return self.main_item.content - @cached_property + @property def stoken(self): return self.main_item.stoken @@ -75,7 +75,7 @@ class CollectionItem(models.Model): def content(self): return self.revisions.get(current=True) - @cached_property + @property def stoken(self): return self.content.uid diff --git a/django_etesync/views.py b/django_etesync/views.py index 916c2a2..7f07f2e 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -59,6 +59,24 @@ from .serializers import ( User = get_user_model() +def get_fresh_stoken(obj): + try: + del obj.main_item + except AttributeError: + pass + + return obj.stoken + + +def get_fresh_item_stoken(obj): + try: + del obj.content + except AttributeError: + pass + + return obj.stoken + + class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) @@ -141,16 +159,24 @@ class CollectionViewSet(BaseViewSet): def partial_update(self, request, uid=None): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + def update(self, request, *args, **kwargs): + instance = self.get_object() + serializer = self.get_serializer(instance, data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) + + return Response({'stoken': get_fresh_stoken(instance)}) + def create(self, request, *args, **kwargs): serializer = self.serializer_class(data=request.data, context=self.get_serializer_context()) if serializer.is_valid(): try: - serializer.save(owner=self.request.user) + instance = serializer.save(owner=self.request.user) except IntegrityError: content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) - return Response({}, status=status.HTTP_201_CREATED) + return Response({'stoken': get_fresh_stoken(instance)}, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -304,7 +330,7 @@ class CollectionItemViewSet(BaseViewSet): return Response(content, status=status.HTTP_400_BAD_REQUEST) ret = { - "data": [item.stoken for item in items], + "data": [get_fresh_item_stoken(item) for item in items], } return Response(ret, status=status.HTTP_200_OK) @@ -343,7 +369,7 @@ class CollectionItemViewSet(BaseViewSet): return Response(content, status=status.HTTP_400_BAD_REQUEST) ret = { - "data": [item.stoken for item in items], + "data": [get_fresh_item_stoken(item) for item in items], } return Response(ret, status=status.HTTP_200_OK) From 2a39f3538e2d846d2f3051100ea672b248f27b64 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 18:52:44 +0300 Subject: [PATCH 108/251] Change to standalone stoken objects (+ small optimisation). Makes it possible to now generate Stokens as we need so we can add them to non-revision objects, for example, membership changes. We also slightly improved how we filter by revs. --- django_etesync/models.py | 15 +++++++++++++-- django_etesync/serializers.py | 4 +++- django_etesync/views.py | 26 +++++++++++++++++--------- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index ba6665c..3767b3a 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -18,6 +18,7 @@ from django.db import models from django.conf import settings from django.core.validators import RegexValidator from django.utils.functional import cached_property +from django.utils.crypto import get_random_string Base64Url256BitlValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{42,43}$', message='Expected a base64url.') @@ -55,7 +56,7 @@ class Collection(models.Model): # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. return None - return last_revision.uid + return last_revision.stoken.uid class CollectionItem(models.Model): @@ -77,7 +78,7 @@ class CollectionItem(models.Model): @property def stoken(self): - return self.content.uid + return self.content.stoken.uid def chunk_directory_path(instance, filename): @@ -98,7 +99,17 @@ class CollectionItemChunk(models.Model): return self.uid +def generate_stoken_uid(): + return get_random_string(32, allowed_chars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_') + + +class Stoken(models.Model): + uid = models.CharField(db_index=True, unique=True, blank=False, null=False, default=generate_stoken_uid, + max_length=43, validators=[Base64Url256BitlValidator]) + + class CollectionItemRevision(models.Model): + stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT) uid = models.CharField(db_index=True, unique=True, blank=False, null=False, max_length=43, validators=[Base64Url256BitlValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index bff20ad..d08fcfd 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -38,7 +38,9 @@ def process_revisions_for_item(item, revision_data): chunk = models.CollectionItemChunk.objects.get(uid=uid) chunks_objs.append(chunk) - revision = models.CollectionItemRevision.objects.create(**revision_data, item=item) + stoken = models.Stoken.objects.create() + + revision = models.CollectionItemRevision.objects.create(**revision_data, item=item, stoken=stoken) for chunk in chunks_objs: models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) return revision diff --git a/django_etesync/views.py b/django_etesync/views.py index 7f07f2e..49b8110 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -35,7 +35,15 @@ import nacl.secret import nacl.hash from . import app_settings, permissions -from .models import Collection, CollectionItem, CollectionItemRevision, CollectionMember, CollectionInvitation, UserInfo +from .models import ( + Collection, + CollectionItem, + CollectionItemRevision, + CollectionMember, + CollectionInvitation, + Stoken, + UserInfo, + ) from .serializers import ( b64encode, AuthenticationSignupSerializer, @@ -94,18 +102,18 @@ class BaseViewSet(viewsets.ModelViewSet): user = self.request.user return queryset.filter(members__user=user) - def get_cstoken_rev(self, request): + def get_cstoken_obj(self, request): cstoken = request.GET.get('cstoken', None) if cstoken is not None: - return get_object_or_404(CollectionItemRevision.objects.all(), uid=cstoken) + return get_object_or_404(Stoken.objects.all(), uid=cstoken) return None def filter_by_cstoken(self, request, queryset): cstoken_id_field = self.cstoken_id_field + '__id' - cstoken_rev = self.get_cstoken_rev(request) + cstoken_rev = self.get_cstoken_obj(request) if cstoken_rev is not None: filter_by = {cstoken_id_field + '__gt': cstoken_rev.id} queryset = queryset.filter(**filter_by) @@ -116,7 +124,7 @@ class BaseViewSet(viewsets.ModelViewSet): cstoken_id_field = self.cstoken_id_field + '__id' new_cstoken_id = queryset.aggregate(cstoken_id=Max(cstoken_id_field))['cstoken_id'] - new_cstoken = new_cstoken_id and CollectionItemRevision.objects.get(id=new_cstoken_id).uid + new_cstoken = new_cstoken_id and Stoken.objects.get(id=new_cstoken_id).uid return queryset, new_cstoken @@ -139,7 +147,7 @@ class CollectionViewSet(BaseViewSet): queryset = Collection.objects.all() serializer_class = CollectionSerializer lookup_field = 'uid' - cstoken_id_field = 'items__revisions' + cstoken_id_field = 'items__revisions__stoken' def get_queryset(self, queryset=None): if queryset is None: @@ -199,7 +207,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer lookup_field = 'uid' - cstoken_id_field = 'revisions' + cstoken_id_field = 'revisions__stoken' def get_queryset(self): collection_uid = self.kwargs['collection_uid'] @@ -290,8 +298,8 @@ class CollectionItemViewSet(BaseViewSet): queryset, cstoken_rev = self.filter_by_cstoken(request, queryset) uids, stokens = zip(*[(item['uid'], item.get('stoken')) for item in serializer.validated_data]) - rev_ids = CollectionItemRevision.objects.filter(uid__in=stokens, current=True).values_list('id', flat=True) - queryset = queryset.filter(uid__in=uids).exclude(revisions__id__in=rev_ids) + revs = CollectionItemRevision.objects.filter(stoken__uid__in=stokens, current=True) + queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) queryset, new_cstoken = self.get_queryset_cstoken(queryset) cstoken = cstoken_rev and cstoken_rev.uid From 6e7ad92a12365d7c2c32cfd2f7add657464c9b6f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 26 May 2020 18:52:44 +0300 Subject: [PATCH 109/251] Add missing migrations forgotten in the previous commit Missing from: 73f4ff765c7713c9aa48dec2bfc4c3c1c0c7e9f3 --- .../migrations/0008_auto_20200526_1535.py | 28 +++++++++++++++++++ .../migrations/0009_auto_20200526_1535.py | 23 +++++++++++++++ .../migrations/0010_auto_20200526_1539.py | 19 +++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 django_etesync/migrations/0008_auto_20200526_1535.py create mode 100644 django_etesync/migrations/0009_auto_20200526_1535.py create mode 100644 django_etesync/migrations/0010_auto_20200526_1539.py diff --git a/django_etesync/migrations/0008_auto_20200526_1535.py b/django_etesync/migrations/0008_auto_20200526_1535.py new file mode 100644 index 0000000..e544bdd --- /dev/null +++ b/django_etesync/migrations/0008_auto_20200526_1535.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-05-26 15:35 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django_etesync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0007_auto_20200526_1336'), + ] + + operations = [ + migrations.CreateModel( + name='Stoken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, default=django_etesync.models.generate_stoken_uid, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')])), + ], + ), + migrations.AddField( + model_name='collectionitemrevision', + name='stoken', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken'), + ), + ] diff --git a/django_etesync/migrations/0009_auto_20200526_1535.py b/django_etesync/migrations/0009_auto_20200526_1535.py new file mode 100644 index 0000000..53100b3 --- /dev/null +++ b/django_etesync/migrations/0009_auto_20200526_1535.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-05-26 15:35 + +from django.db import migrations + + +def create_stokens(apps, schema_editor): + Stoken = apps.get_model('django_etesync', 'Stoken') + CollectionItemRevision = apps.get_model('django_etesync', 'CollectionItemRevision') + + for rev in CollectionItemRevision.objects.all(): + rev.stoken = Stoken.objects.create() + rev.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0008_auto_20200526_1535'), + ] + + operations = [ + migrations.RunPython(create_stokens), + ] diff --git a/django_etesync/migrations/0010_auto_20200526_1539.py b/django_etesync/migrations/0010_auto_20200526_1539.py new file mode 100644 index 0000000..c894fd2 --- /dev/null +++ b/django_etesync/migrations/0010_auto_20200526_1539.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-05-26 15:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0009_auto_20200526_1535'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemrevision', + name='stoken', + field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken'), + ), + ] From 8eee280bbb41f114b6082e3798803d9c36028451 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 09:39:31 +0300 Subject: [PATCH 110/251] Split cstoken and stoken to be different concepts The stokens are really just integrity checks for items, and are really just tied to what revision we expected to have first what we have. So we will rename stoken to lastRev or something, and have them completely separate. A partial revert of e22a49f982046e875d4e1c5007a91353527d7a0f --- django_etesync/models.py | 6 +++--- django_etesync/views.py | 28 ++++------------------------ 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index 3767b3a..6c50ea2 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -41,13 +41,13 @@ class Collection(models.Model): def main_item(self): return self.items.get(uid=None) - @cached_property + @property def content(self): return self.main_item.content @property def stoken(self): - return self.main_item.stoken + return self.content.uid @cached_property def cstoken(self): @@ -78,7 +78,7 @@ class CollectionItem(models.Model): @property def stoken(self): - return self.content.stoken.uid + return self.content.uid def chunk_directory_path(instance, filename): diff --git a/django_etesync/views.py b/django_etesync/views.py index 49b8110..2341484 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -67,24 +67,6 @@ from .serializers import ( User = get_user_model() -def get_fresh_stoken(obj): - try: - del obj.main_item - except AttributeError: - pass - - return obj.stoken - - -def get_fresh_item_stoken(obj): - try: - del obj.content - except AttributeError: - pass - - return obj.stoken - - class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) @@ -173,18 +155,18 @@ class CollectionViewSet(BaseViewSet): serializer.is_valid(raise_exception=True) self.perform_update(serializer) - return Response({'stoken': get_fresh_stoken(instance)}) + return Response({}) def create(self, request, *args, **kwargs): serializer = self.serializer_class(data=request.data, context=self.get_serializer_context()) if serializer.is_valid(): try: - instance = serializer.save(owner=self.request.user) + serializer.save(owner=self.request.user) except IntegrityError: content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) - return Response({'stoken': get_fresh_stoken(instance)}, status=status.HTTP_201_CREATED) + return Response({}, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -298,7 +280,7 @@ class CollectionItemViewSet(BaseViewSet): queryset, cstoken_rev = self.filter_by_cstoken(request, queryset) uids, stokens = zip(*[(item['uid'], item.get('stoken')) for item in serializer.validated_data]) - revs = CollectionItemRevision.objects.filter(stoken__uid__in=stokens, current=True) + revs = CollectionItemRevision.objects.filter(uid__in=stokens, current=True) queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) queryset, new_cstoken = self.get_queryset_cstoken(queryset) @@ -338,7 +320,6 @@ class CollectionItemViewSet(BaseViewSet): return Response(content, status=status.HTTP_400_BAD_REQUEST) ret = { - "data": [get_fresh_item_stoken(item) for item in items], } return Response(ret, status=status.HTTP_200_OK) @@ -377,7 +358,6 @@ class CollectionItemViewSet(BaseViewSet): return Response(content, status=status.HTTP_400_BAD_REQUEST) ret = { - "data": [get_fresh_item_stoken(item) for item in items], } return Response(ret, status=status.HTTP_200_OK) From 9c63f8d6746661542982aff7147df8129cfbc1e7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 10:09:45 +0300 Subject: [PATCH 111/251] Rename stoken to etag and cstoken to stoken. This conforms better with what people know from HTTP and properly differentiates from CSToken which is now renamed to stoken. --- django_etesync/models.py | 6 +-- django_etesync/serializers.py | 44 +++++++++--------- django_etesync/views.py | 84 +++++++++++++++++------------------ 3 files changed, 67 insertions(+), 67 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index 6c50ea2..c9c95a1 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -46,11 +46,11 @@ class Collection(models.Model): return self.main_item.content @property - def stoken(self): + def etag(self): return self.content.uid @cached_property - def cstoken(self): + def stoken(self): last_revision = CollectionItemRevision.objects.filter(item__collection=self).last() if last_revision is None: # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. @@ -77,7 +77,7 @@ class CollectionItem(models.Model): return self.revisions.get(current=True) @property - def stoken(self): + def etag(self): return self.content.uid diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index d08fcfd..75baba0 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -114,17 +114,17 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): class CollectionItemSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() - stoken = serializers.CharField(allow_null=True) + etag = serializers.CharField(allow_null=True) content = CollectionItemRevisionSerializer(many=False) class Meta: model = models.CollectionItem - fields = ('uid', 'version', 'encryptionKey', 'content', 'stoken') + fields = ('uid', 'version', 'encryptionKey', 'content', 'etag') def create(self, validated_data): """Function that's called when this serializer creates an item""" - validate_stoken = self.context.get('validate_stoken', False) - stoken = validated_data.pop('stoken') + validate_etag = self.context.get('validate_etag', False) + etag = validated_data.pop('etag') revision_data = validated_data.pop('content') uid = validated_data.pop('uid') @@ -132,10 +132,10 @@ class CollectionItemSerializer(serializers.ModelSerializer): with transaction.atomic(): instance, created = Model.objects.get_or_create(uid=uid, defaults=validated_data) - cur_stoken = instance.stoken if not created else None + cur_etag = instance.etag if not created else None - if validate_stoken and cur_stoken != stoken: - raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(cur_stoken, stoken)) + if validate_etag and cur_etag != etag: + raise serializers.ValidationError('Wrong etag. Expected {} got {}'.format(cur_etag, etag)) if not created: # We don't have to use select_for_update here because the unique constraint on current guards against @@ -154,39 +154,39 @@ class CollectionItemSerializer(serializers.ModelSerializer): class CollectionItemDepSerializer(serializers.ModelSerializer): - stoken = serializers.CharField() + etag = serializers.CharField() class Meta: model = models.CollectionItem - fields = ('uid', 'stoken') + fields = ('uid', 'etag') def validate(self, data): item = self.__class__.Meta.model.objects.get(uid=data['uid']) - stoken = data['stoken'] - if item.stoken != stoken: - raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(item.stoken, stoken)) + etag = data['etag'] + if item.etag != etag: + raise serializers.ValidationError('Wrong etag. Expected {} got {}'.format(item.etag, etag)) return data class CollectionItemBulkGetSerializer(serializers.ModelSerializer): - stoken = serializers.CharField(required=False) + etag = serializers.CharField(required=False) class Meta: model = models.CollectionItem - fields = ('uid', 'stoken') + fields = ('uid', 'etag') class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') - cstoken = serializers.CharField(read_only=True) - stoken = serializers.CharField(allow_null=True) + stoken = serializers.CharField(read_only=True) + etag = serializers.CharField(allow_null=True) content = CollectionItemRevisionSerializer(many=False) class Meta: model = models.Collection - fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'cstoken', 'stoken') + fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'stoken', 'etag') def get_access_level_from_context(self, obj): request = self.context.get('request', None) @@ -196,13 +196,13 @@ class CollectionSerializer(serializers.ModelSerializer): def create(self, validated_data): """Function that's called when this serializer creates an item""" - stoken = validated_data.pop('stoken') + etag = validated_data.pop('etag') revision_data = validated_data.pop('content') encryption_key = validated_data.pop('encryptionKey') instance = self.__class__.Meta.model(**validated_data) with transaction.atomic(): - if stoken is not None: + if etag is not None: raise serializers.ValidationError('Stoken is not None') instance.save() @@ -221,12 +221,12 @@ class CollectionSerializer(serializers.ModelSerializer): def update(self, instance, validated_data): """Function that's called when this serializer is meant to update an item""" - stoken = validated_data.pop('stoken') + etag = validated_data.pop('etag') revision_data = validated_data.pop('content') with transaction.atomic(): - if stoken != instance.stoken: - raise serializers.ValidationError('Wrong stoken. Expected {} got {}'.format(instance.stoken, stoken)) + if etag != instance.etag: + raise serializers.ValidationError('Wrong etag. Expected {} got {}'.format(instance.etag, etag)) main_item = instance.main_item # We don't have to use select_for_update here because the unique constraint on current guards against diff --git a/django_etesync/views.py b/django_etesync/views.py index 2341484..e66bb82 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -70,7 +70,7 @@ User = get_user_model() class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) - cstoken_id_field = None + stoken_id_field = None def get_serializer_class(self): serializer_class = self.serializer_class @@ -84,43 +84,43 @@ class BaseViewSet(viewsets.ModelViewSet): user = self.request.user return queryset.filter(members__user=user) - def get_cstoken_obj(self, request): - cstoken = request.GET.get('cstoken', None) + def get_stoken_obj(self, request): + stoken = request.GET.get('stoken', None) - if cstoken is not None: - return get_object_or_404(Stoken.objects.all(), uid=cstoken) + if stoken is not None: + return get_object_or_404(Stoken.objects.all(), uid=stoken) return None - def filter_by_cstoken(self, request, queryset): - cstoken_id_field = self.cstoken_id_field + '__id' + def filter_by_stoken(self, request, queryset): + stoken_id_field = self.stoken_id_field + '__id' - cstoken_rev = self.get_cstoken_obj(request) - if cstoken_rev is not None: - filter_by = {cstoken_id_field + '__gt': cstoken_rev.id} + stoken_rev = self.get_stoken_obj(request) + if stoken_rev is not None: + filter_by = {stoken_id_field + '__gt': stoken_rev.id} queryset = queryset.filter(**filter_by) - return queryset, cstoken_rev + return queryset, stoken_rev - def get_queryset_cstoken(self, queryset): - cstoken_id_field = self.cstoken_id_field + '__id' + def get_queryset_stoken(self, queryset): + stoken_id_field = self.stoken_id_field + '__id' - new_cstoken_id = queryset.aggregate(cstoken_id=Max(cstoken_id_field))['cstoken_id'] - new_cstoken = new_cstoken_id and Stoken.objects.get(id=new_cstoken_id).uid + new_stoken_id = queryset.aggregate(stoken_id=Max(stoken_id_field))['stoken_id'] + new_stoken = new_stoken_id and Stoken.objects.get(id=new_stoken_id).uid - return queryset, new_cstoken + return queryset, new_stoken - def filter_by_cstoken_and_limit(self, request, queryset): + def filter_by_stoken_and_limit(self, request, queryset): limit = int(request.GET.get('limit', 50)) - queryset, cstoken_rev = self.filter_by_cstoken(request, queryset) - cstoken = cstoken_rev.uid if cstoken_rev is not None else None + queryset, stoken_rev = self.filter_by_stoken(request, queryset) + stoken = stoken_rev.uid if stoken_rev is not None else None queryset = queryset[:limit] - queryset, new_cstoken = self.get_queryset_cstoken(queryset) - new_cstoken = new_cstoken or cstoken + queryset, new_stoken = self.get_queryset_stoken(queryset) + new_stoken = new_stoken or stoken - return queryset, new_cstoken + return queryset, new_stoken class CollectionViewSet(BaseViewSet): @@ -129,7 +129,7 @@ class CollectionViewSet(BaseViewSet): queryset = Collection.objects.all() serializer_class = CollectionSerializer lookup_field = 'uid' - cstoken_id_field = 'items__revisions__stoken' + stoken_id_field = 'items__revisions__stoken' def get_queryset(self, queryset=None): if queryset is None: @@ -172,13 +172,13 @@ class CollectionViewSet(BaseViewSet): def list(self, request): queryset = self.get_queryset() - queryset, new_cstoken = self.filter_by_cstoken_and_limit(request, queryset) + queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) ret = { 'data': serializer.data, - 'cstoken': new_cstoken, + 'stoken': new_stoken, } return Response(ret) @@ -189,7 +189,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer lookup_field = 'uid' - cstoken_id_field = 'revisions__stoken' + stoken_id_field = 'revisions__stoken' def get_queryset(self): collection_uid = self.kwargs['collection_uid'] @@ -240,13 +240,13 @@ class CollectionItemViewSet(BaseViewSet): def list(self, request, collection_uid=None): queryset = self.get_queryset() - queryset, new_cstoken = self.filter_by_cstoken_and_limit(request, queryset) + queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) ret = { 'data': serializer.data, - 'cstoken': new_cstoken, + 'stoken': new_stoken, } return Response(ret) @@ -277,21 +277,21 @@ class CollectionItemViewSet(BaseViewSet): 'detail': 'Request has too many items. Limit: {}'. format(item_limit)} return Response(content, status=status.HTTP_400_BAD_REQUEST) - queryset, cstoken_rev = self.filter_by_cstoken(request, queryset) + queryset, stoken_rev = self.filter_by_stoken(request, queryset) - uids, stokens = zip(*[(item['uid'], item.get('stoken')) for item in serializer.validated_data]) - revs = CollectionItemRevision.objects.filter(uid__in=stokens, current=True) + uids, etags = zip(*[(item['uid'], item.get('etag')) for item in serializer.validated_data]) + revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) - queryset, new_cstoken = self.get_queryset_cstoken(queryset) - cstoken = cstoken_rev and cstoken_rev.uid - new_cstoken = new_cstoken or cstoken + queryset, new_stoken = self.get_queryset_stoken(queryset) + stoken = stoken_rev and stoken_rev.uid + new_stoken = new_stoken or stoken serializer = self.get_serializer_class()(queryset, context=self.get_serializer_context(), many=True) ret = { 'data': serializer.data, - 'cstoken': new_cstoken, + 'stoken': new_stoken, } return Response(ret) @@ -299,11 +299,11 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def batch(self, request, collection_uid=None): - cstoken = request.GET.get('cstoken', None) + stoken = request.GET.get('stoken', None) collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - if cstoken is not None and cstoken != collection_object.cstoken: - content = {'code': 'stale_cstoken', 'detail': 'CSToken is too old'} + if stoken is not None and stoken != collection_object.stoken: + content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} return Response(content, status=status.HTTP_400_BAD_REQUEST) items = request.data.get('items') @@ -331,18 +331,18 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def transaction(self, request, collection_uid=None): - cstoken = request.GET.get('cstoken', None) + stoken = request.GET.get('stoken', None) collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - if cstoken is not None and cstoken != collection_object.cstoken: - content = {'code': 'stale_cstoken', 'detail': 'CSToken is too old'} + if stoken is not None and stoken != collection_object.stoken: + content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} return Response(content, status=status.HTTP_400_BAD_REQUEST) items = request.data.get('items') deps = request.data.get('deps', None) # FIXME: It should just be one serializer context = self.get_serializer_context() - context.update({'validate_stoken': True}) + context.update({'validate_etag': True}) serializer = self.get_serializer_class()(data=items, context=context, many=True) deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) From 91aadb65656cbc0676ccdfea9b1b8f7c7d316140 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 10:16:55 +0300 Subject: [PATCH 112/251] Make etag write-only. --- django_etesync/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 75baba0..f78803a 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -114,7 +114,7 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): class CollectionItemSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() - etag = serializers.CharField(allow_null=True) + etag = serializers.CharField(allow_null=True, write_only=True) content = CollectionItemRevisionSerializer(many=False) class Meta: @@ -181,7 +181,7 @@ class CollectionSerializer(serializers.ModelSerializer): encryptionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') stoken = serializers.CharField(read_only=True) - etag = serializers.CharField(allow_null=True) + etag = serializers.CharField(allow_null=True, write_only=True) content = CollectionItemRevisionSerializer(many=False) class Meta: From 1f18f4e50bc228f165afacba163e73b339755c7d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 10:52:27 +0300 Subject: [PATCH 113/251] CollectionMember: add stokens when we create/change the member. --- .../0011_collectionmember_stoken.py | 19 +++++++++++++++ .../migrations/0012_auto_20200527_0743.py | 23 +++++++++++++++++++ django_etesync/models.py | 1 + django_etesync/serializers.py | 9 ++++++-- 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 django_etesync/migrations/0011_collectionmember_stoken.py create mode 100644 django_etesync/migrations/0012_auto_20200527_0743.py diff --git a/django_etesync/migrations/0011_collectionmember_stoken.py b/django_etesync/migrations/0011_collectionmember_stoken.py new file mode 100644 index 0000000..6b79bf6 --- /dev/null +++ b/django_etesync/migrations/0011_collectionmember_stoken.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-05-27 07:43 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0010_auto_20200526_1539'), + ] + + operations = [ + migrations.AddField( + model_name='collectionmember', + name='stoken', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken'), + ), + ] diff --git a/django_etesync/migrations/0012_auto_20200527_0743.py b/django_etesync/migrations/0012_auto_20200527_0743.py new file mode 100644 index 0000000..28b8745 --- /dev/null +++ b/django_etesync/migrations/0012_auto_20200527_0743.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-05-27 07:43 + +from django.db import migrations + + +def create_stokens(apps, schema_editor): + Stoken = apps.get_model('django_etesync', 'Stoken') + CollectionMember = apps.get_model('django_etesync', 'CollectionMember') + + for member in CollectionMember.objects.all(): + member.stoken = Stoken.objects.create() + member.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0011_collectionmember_stoken'), + ] + + operations = [ + migrations.RunPython(create_stokens), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index c9c95a1..ca68209 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -139,6 +139,7 @@ class AccessLevels(models.TextChoices): class CollectionMember(models.Model): + stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT, null=True) collection = models.ForeignKey(Collection, related_name='members', on_delete=models.CASCADE) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) encryptionKey = models.BinaryField(editable=True, blank=False, null=False) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index f78803a..7079742 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -212,6 +212,7 @@ class CollectionSerializer(serializers.ModelSerializer): process_revisions_for_item(main_item, revision_data) models.CollectionMember(collection=instance, + stoken=models.Stoken.objects.create(), user=validated_data.get('owner'), accessLevel=models.AccessLevels.ADMIN, encryptionKey=encryption_key, @@ -258,8 +259,11 @@ class CollectionMemberSerializer(serializers.ModelSerializer): def update(self, instance, validated_data): with transaction.atomic(): # We only allow updating accessLevel - instance.accessLevel = validated_data.pop('accessLevel') - instance.save() + access_level = validated_data.pop('accessLevel') + if instance.accessLevel != access_level: + instance.stoken = models.Stoken.objects.create() + instance.accessLevel = access_level + instance.save() return instance @@ -314,6 +318,7 @@ class InvitationAcceptSerializer(serializers.Serializer): member = models.CollectionMember.objects.create( collection=invitation.collection, + stoken=models.Stoken.objects.create(), user=invitation.user, accessLevel=invitation.accessLevel, encryptionKey=encryption_key, From d93a5d3f06ff9f3f140e96fc080a6a655906e84f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 12:13:54 +0300 Subject: [PATCH 114/251] Collections: use the member stokens for filtering based on stoken While at it, also generalised the stoken handling to be generic and extendible. --- django_etesync/models.py | 13 ++++++++----- django_etesync/views.py | 25 +++++++++++++------------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/django_etesync/models.py b/django_etesync/models.py index ca68209..91746ba 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -17,6 +17,7 @@ from pathlib import Path from django.db import models from django.conf import settings from django.core.validators import RegexValidator +from django.db.models import Q from django.utils.functional import cached_property from django.utils.crypto import get_random_string @@ -51,12 +52,14 @@ class Collection(models.Model): @cached_property def stoken(self): - last_revision = CollectionItemRevision.objects.filter(item__collection=self).last() - if last_revision is None: - # FIXME: what is the etag for None? Though if we use the revision for collection it should be shared anyway. - return None + stoken = Stoken.objects.filter( + Q(collectionitemrevision__item__collection=self) | Q(collectionmember__collection=self) + ).order_by('id').last() - return last_revision.stoken.uid + if stoken is None: + raise Exception('stoken is None. Should never happen') + + return stoken.uid class CollectionItem(models.Model): diff --git a/django_etesync/views.py b/django_etesync/views.py index e66bb82..0f448fa 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -13,12 +13,14 @@ # along with this program. If not, see . import json +from functools import reduce from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError -from django.db.models import Max +from django.db.models import Max, Q +from django.db.models.functions import Greatest from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -70,7 +72,7 @@ User = get_user_model() class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) - stoken_id_field = None + stoken_id_fields = None def get_serializer_class(self): serializer_class = self.serializer_class @@ -93,20 +95,19 @@ class BaseViewSet(viewsets.ModelViewSet): return None def filter_by_stoken(self, request, queryset): - stoken_id_field = self.stoken_id_field + '__id' - stoken_rev = self.get_stoken_obj(request) if stoken_rev is not None: - filter_by = {stoken_id_field + '__gt': stoken_rev.id} - queryset = queryset.filter(**filter_by) + filter_by_map = map(lambda x: Q(**{x + '__gt': stoken_rev.id}), self.stoken_id_fields) + filter_by = reduce(lambda x, y: x | y, filter_by_map) + queryset = queryset.filter(filter_by).distinct() return queryset, stoken_rev def get_queryset_stoken(self, queryset): - stoken_id_field = self.stoken_id_field + '__id' - - new_stoken_id = queryset.aggregate(stoken_id=Max(stoken_id_field))['stoken_id'] - new_stoken = new_stoken_id and Stoken.objects.get(id=new_stoken_id).uid + aggr_fields = {x: Max(x) for x in self.stoken_id_fields} + aggr = queryset.aggregate(**aggr_fields) + maxid = max(map(lambda x: x or -1, aggr.values())) + new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid).uid return queryset, new_stoken @@ -129,7 +130,7 @@ class CollectionViewSet(BaseViewSet): queryset = Collection.objects.all() serializer_class = CollectionSerializer lookup_field = 'uid' - stoken_id_field = 'items__revisions__stoken' + stoken_id_fields = ['items__revisions__stoken__id', 'members__stoken__id'] def get_queryset(self, queryset=None): if queryset is None: @@ -189,7 +190,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer lookup_field = 'uid' - stoken_id_field = 'revisions__stoken' + stoken_id_fields = ['revisions__stoken__id'] def get_queryset(self): collection_uid = self.kwargs['collection_uid'] From 6e7fd5d0dd03b783cfcf99a1960a79d440988cf7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 16:03:16 +0300 Subject: [PATCH 115/251] Collection membership: implement leaving/revoking access. --- .../0013_collectionmemberremoved.py | 28 +++++++++++++++++++ django_etesync/models.py | 25 ++++++++++++++++- django_etesync/serializers.py | 6 ++-- django_etesync/views.py | 28 ++++++++++++++++++- 4 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 django_etesync/migrations/0013_collectionmemberremoved.py diff --git a/django_etesync/migrations/0013_collectionmemberremoved.py b/django_etesync/migrations/0013_collectionmemberremoved.py new file mode 100644 index 0000000..e796232 --- /dev/null +++ b/django_etesync/migrations/0013_collectionmemberremoved.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-05-27 11:29 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('django_etesync', '0012_auto_20200527_0743'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionMemberRemoved', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='removed_members', to='django_etesync.Collection')), + ('stoken', models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'collection')}, + }, + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 91746ba..7fbea61 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -14,7 +14,7 @@ from pathlib import Path -from django.db import models +from django.db import models, transaction from django.conf import settings from django.core.validators import RegexValidator from django.db.models import Q @@ -158,6 +158,29 @@ class CollectionMember(models.Model): def __str__(self): return '{} {}'.format(self.collection.uid, self.user) + def revoke(self): + with transaction.atomic(): + CollectionMemberRemoved.objects.update_or_create( + collection=self.collection, user=self.user, + defaults={ + 'stoken': Stoken.objects.create(), + }, + ) + + self.delete() + + +class CollectionMemberRemoved(models.Model): + stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT, null=True) + collection = models.ForeignKey(Collection, related_name='removed_members', on_delete=models.CASCADE) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + + class Meta: + unique_together = ('user', 'collection') + + def __str__(self): + return '{} {}'.format(self.collection.uid, self.user) + class CollectionInvitation(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 7079742..c5734d2 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -247,11 +247,10 @@ class CollectionMemberSerializer(serializers.ModelSerializer): slug_field=User.USERNAME_FIELD, queryset=User.objects ) - encryptionKey = BinaryBase64Field() class Meta: model = models.CollectionMember - fields = ('username', 'encryptionKey', 'accessLevel') + fields = ('username', 'accessLevel') def create(self, validated_data): raise NotImplementedError() @@ -324,6 +323,9 @@ class InvitationAcceptSerializer(serializers.Serializer): encryptionKey=encryption_key, ) + models.CollectionMemberRemoved.objects.filter( + user=invitation.user, collection=invitation.collection).delete() + invitation.delete() return member diff --git a/django_etesync/views.py b/django_etesync/views.py index 0f448fa..fd91aed 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -42,6 +42,7 @@ from .models import ( CollectionItem, CollectionItemRevision, CollectionMember, + CollectionMemberRemoved, CollectionInvitation, Stoken, UserInfo, @@ -181,6 +182,15 @@ class CollectionViewSet(BaseViewSet): 'data': serializer.data, 'stoken': new_stoken, } + + stoken_obj = self.get_stoken_obj(request) + if stoken_obj is not None: + # FIXME: honour limit? (the limit should be combined for data and this because of stoken) + remed = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) \ + .values_list('collection__uid', flat=True) + if len(remed) > 0: + ret['removedMemberships'] = [{'uid': x} for x in remed] + return Response(ret) @@ -417,7 +427,8 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): class CollectionMemberViewSet(BaseViewSet): allowed_methods = ['GET', 'PUT', 'DELETE'] - permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdmin, ) + our_base_permission_classes = BaseViewSet.permission_classes + permission_classes = our_base_permission_classes + (permissions.IsCollectionAdmin, ) queryset = CollectionMember.objects.all() serializer_class = CollectionMemberSerializer lookup_field = 'user__' + User.USERNAME_FIELD @@ -441,6 +452,21 @@ class CollectionMemberViewSet(BaseViewSet): def create(self, request): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + # FIXME: block leaving if we are the last admins - should be deleted / assigned in this case depending if there + # are other memebers. + def perform_destroy(self, instance): + instance.revoke() + + @action_decorator(detail=False, methods=['POST'], permission_classes=our_base_permission_classes) + def leave(self, request, collection_uid=None): + collection_uid = self.kwargs['collection_uid'] + col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) + + member = col.members.get(user=request.user) + self.perform_destroy(member) + + return Response({}) + class InvitationOutgoingViewSet(BaseViewSet): allowed_methods = ['GET', 'POST', 'PUT', 'DELETE'] From e159bf971bc0aea1aea38635a907cdf80ed9293f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 16:40:08 +0300 Subject: [PATCH 116/251] Collection/item viewsets: enforce access. --- django_etesync/permissions.py | 46 +++++++++++++++++++++++++++++++++++ django_etesync/views.py | 4 +-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/django_etesync/permissions.py b/django_etesync/permissions.py index c371743..611977b 100644 --- a/django_etesync/permissions.py +++ b/django_etesync/permissions.py @@ -36,3 +36,49 @@ class IsCollectionAdmin(permissions.BasePermission): except Collection.DoesNotExist: # If the collection does not exist, we want to 404 later, not permission denied. return True + + +class IsCollectionAdminOrReadOnly(permissions.BasePermission): + """ + Custom permission to only allow owners of a collection to edit it + """ + message = 'Only collection admins can edit collections.' + code = 'admin_access_required' + + def has_permission(self, request, view): + collection_uid = view.kwargs.get('collection_uid', None) + + # Allow creating new collections + if collection_uid is None: + return True + + try: + collection = view.get_collection_queryset().get(uid=collection_uid) + if request.method in permissions.SAFE_METHODS: + return True + + return is_collection_admin(collection, request.user) + except Collection.DoesNotExist: + # If the collection does not exist, we want to 404 later, not permission denied. + return True + + +class HasWriteAccessOrReadOnly(permissions.BasePermission): + """ + Custom permission to restrict write + """ + message = 'You need write access to write to this collection' + code = 'no_write_access' + + def has_permission(self, request, view): + collection_uid = view.kwargs['collection_uid'] + try: + collection = view.get_collection_queryset().get(uid=collection_uid) + if request.method in permissions.SAFE_METHODS: + return True + else: + member = collection.members.get(user=request.user) + return member.accessLevel != AccessLevels.READ_ONLY + except Collection.DoesNotExist: + # If the collection does not exist, we want to 404 later, not permission denied. + return True diff --git a/django_etesync/views.py b/django_etesync/views.py index fd91aed..148081a 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -127,7 +127,7 @@ class BaseViewSet(viewsets.ModelViewSet): class CollectionViewSet(BaseViewSet): allowed_methods = ['GET', 'POST', 'DELETE'] - permission_classes = BaseViewSet.permission_classes + permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdminOrReadOnly, ) queryset = Collection.objects.all() serializer_class = CollectionSerializer lookup_field = 'uid' @@ -196,7 +196,7 @@ class CollectionViewSet(BaseViewSet): class CollectionItemViewSet(BaseViewSet): allowed_methods = ['GET', 'POST', 'PUT'] - permission_classes = BaseViewSet.permission_classes + permission_classes = BaseViewSet.permission_classes + (permissions.HasWriteAccessOrReadOnly, ) queryset = CollectionItem.objects.all() serializer_class = CollectionItemSerializer lookup_field = 'uid' From f6960bb8cb198743830720825de72a1820ee28da Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 16:51:12 +0300 Subject: [PATCH 117/251] CollectionMember: fix collection list to return data in the right format. --- django_etesync/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index 148081a..38919a8 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -449,6 +449,16 @@ class CollectionMemberViewSet(BaseViewSet): return queryset.filter(collection=collection) + def list(self, request, collection_uid=None): + queryset = self.get_queryset() + serializer = self.get_serializer(queryset, many=True) + + ret = { + 'data': serializer.data, + } + + return Response(ret) + def create(self, request): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) From 6c31b8fb3033f1cd9f559c43cef0752b63f6a21b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 16:59:24 +0300 Subject: [PATCH 118/251] CollectionItemView: disallow normal item creation People should only use transaction/batch --- django_etesync/views.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 38919a8..1ba5fd1 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -222,22 +222,8 @@ class CollectionItemViewSet(BaseViewSet): return context def create(self, request, collection_uid=None): - collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - - # FIXME: change this to also support bulk update, or have another endpoint for that. - # See https://www.django-rest-framework.org/api-guide/serializers/#customizing-multiple-update - many = isinstance(request.data, list) - serializer = self.serializer_class(data=request.data, many=many) - if serializer.is_valid(): - try: - serializer.save(collection=collection_object) - except IntegrityError: - content = {'code': 'integrity_error'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - return Response({}, status=status.HTTP_201_CREATED) - - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + # We create using batch and transaction + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def destroy(self, request, collection_uid=None, uid=None): # We can't have destroy because we need to get data from the user (in the body) such as hmac. From 9f2140ffaca5d8a49e941ce1aa0f760e76e9a4df Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 17:00:33 +0300 Subject: [PATCH 119/251] Change serializer fetching to the more drf way of doing it. Also fix the ItemChunk serializer. --- django_etesync/views.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 1ba5fd1..31f0550 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -20,7 +20,6 @@ from django.contrib.auth import get_user_model from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError from django.db.models import Max, Q -from django.db.models.functions import Greatest from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -160,7 +159,7 @@ class CollectionViewSet(BaseViewSet): return Response({}) def create(self, request, *args, **kwargs): - serializer = self.serializer_class(data=request.data, context=self.get_serializer_context()) + serializer = self.get_serializer(data=request.data) if serializer.is_valid(): try: serializer.save(owner=self.request.user) @@ -176,7 +175,7 @@ class CollectionViewSet(BaseViewSet): queryset = self.get_queryset() queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) - serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) + serializer = self.get_serializer(queryset, many=True) ret = { 'data': serializer.data, @@ -239,7 +238,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = self.get_queryset() queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) - serializer = self.serializer_class(queryset, context=self.get_serializer_context(), many=True) + serializer = self.get_serializer(queryset, many=True) ret = { 'data': serializer.data, @@ -284,7 +283,7 @@ class CollectionItemViewSet(BaseViewSet): stoken = stoken_rev and stoken_rev.uid new_stoken = new_stoken or stoken - serializer = self.get_serializer_class()(queryset, context=self.get_serializer_context(), many=True) + serializer = self.get_serializer(queryset, many=True) ret = { 'data': serializer.data, @@ -304,8 +303,7 @@ class CollectionItemViewSet(BaseViewSet): return Response(content, status=status.HTTP_400_BAD_REQUEST) items = request.data.get('items') - context = self.get_serializer_context() - serializer = self.get_serializer_class()(data=items, context=context, many=True) + serializer = self.get_serializer(data=items, many=True) if serializer.is_valid(): try: @@ -374,6 +372,9 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): serializer_class = CollectionItemChunkSerializer lookup_field = 'uid' + def get_serializer_class(self): + return self.serializer_class + def get_collection_queryset(self, queryset=Collection.objects): user = self.request.user return queryset.filter(members__user=user) @@ -382,7 +383,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) col_it = get_object_or_404(col.items, uid=collection_item_uid) - serializer = self.serializer_class(data=request.data) + serializer = self.get_serializer_class()(data=request.data) if serializer.is_valid(): try: serializer.save(item=col_it) @@ -484,7 +485,7 @@ class InvitationOutgoingViewSet(BaseViewSet): return queryset.filter(fromMember__user=self.request.user) def create(self, request, *args, **kwargs): - serializer = self.serializer_class(data=request.data, context=self.get_serializer_context()) + serializer = self.get_serializer(data=request.data) if serializer.is_valid(): collection_uid = serializer.validated_data.get('collection', {}).get('uid') From 89b47c67b7d98f2fd9ead10d308d0d46018b4b0f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 17:06:22 +0300 Subject: [PATCH 120/251] Removed redundant get_serializer_context. This is already provided by default in drf. --- django_etesync/views.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 31f0550..dfc0274 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -473,11 +473,6 @@ class InvitationOutgoingViewSet(BaseViewSet): lookup_field = 'uid' lookup_url_kwarg = 'invitation_uid' - def get_serializer_context(self): - context = super().get_serializer_context() - context.update({'request': self.request}) - return context - def get_queryset(self, queryset=None): if queryset is None: queryset = type(self).queryset From 64b947d455641c13454f33f18d347b622729a519 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 27 May 2020 17:14:38 +0300 Subject: [PATCH 121/251] Change invitations to also follow our list return type format. --- django_etesync/views.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index dfc0274..235d378 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -123,6 +123,17 @@ class BaseViewSet(viewsets.ModelViewSet): return queryset, new_stoken + # Change how our list works by default + def list(self, request, collection_uid=None): + queryset = self.get_queryset() + serializer = self.get_serializer(queryset, many=True) + + ret = { + 'data': serializer.data, + } + + return Response(ret) + class CollectionViewSet(BaseViewSet): allowed_methods = ['GET', 'POST', 'DELETE'] From 9347682997de3ff9d024b948d966620532568f20 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 13:29:03 +0300 Subject: [PATCH 122/251] Collection update: support limiting vs not limiting based on stoken. --- django_etesync/serializers.py | 6 +----- django_etesync/views.py | 7 +++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index c5734d2..3b50c36 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -203,7 +203,7 @@ class CollectionSerializer(serializers.ModelSerializer): with transaction.atomic(): if etag is not None: - raise serializers.ValidationError('Stoken is not None') + raise serializers.ValidationError('etag is not None') instance.save() main_item = models.CollectionItem.objects.create( @@ -222,13 +222,9 @@ class CollectionSerializer(serializers.ModelSerializer): def update(self, instance, validated_data): """Function that's called when this serializer is meant to update an item""" - etag = validated_data.pop('etag') revision_data = validated_data.pop('content') with transaction.atomic(): - if etag != instance.etag: - raise serializers.ValidationError('Wrong etag. Expected {} got {}'.format(instance.etag, etag)) - main_item = instance.main_item # We don't have to use select_for_update here because the unique constraint on current guards against # the race condition. But it's a good idea because it'll lock and wait rather than fail. diff --git a/django_etesync/views.py b/django_etesync/views.py index 235d378..dff1096 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -163,6 +163,13 @@ class CollectionViewSet(BaseViewSet): def update(self, request, *args, **kwargs): instance = self.get_object() + + stoken = request.GET.get('stoken', None) + + if stoken is not None and stoken != instance.stoken: + content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + serializer = self.get_serializer(instance, data=request.data) serializer.is_valid(raise_exception=True) self.perform_update(serializer) From ddc43c638acb2da8042ada6216a00c3f8a1778ca Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 14:56:14 +0300 Subject: [PATCH 123/251] Requirements: remove unused requirements. --- requirements.in/base.txt | 4 ---- requirements.txt | 16 ++++------------ 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/requirements.in/base.txt b/requirements.in/base.txt index e6d6379..1ab0e9a 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -1,11 +1,7 @@ django -django-allauth django-anymail -django-appconf django-cors-headers -django-debug-toolbar django-fullurl -django-ipware djangorestframework drf-nested-routers psycopg2-binary diff --git a/requirements.txt b/requirements.txt index dcc37cf..51b4d35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,26 +8,18 @@ asgiref==3.2.3 # via django certifi==2019.11.28 # via requests cffi==1.14.0 # via pynacl chardet==3.0.4 # via requests -defusedxml==0.6.0 # via python3-openid -django-allauth==0.41.0 # via -r requirements.in/base.txt django-anymail==7.0.0 # via -r requirements.in/base.txt -django-appconf==1.0.3 # via -r requirements.in/base.txt django-cors-headers==3.2.1 # via -r requirements.in/base.txt -django-debug-toolbar==2.2 # via -r requirements.in/base.txt django-fullurl==1.0 # via -r requirements.in/base.txt -django-ipware==2.1.0 # via -r requirements.in/base.txt -django==3.0.3 # via -r requirements.in/base.txt, django-allauth, django-anymail, django-appconf, django-cors-headers, django-debug-toolbar, django-fullurl, djangorestframework, drf-nested-routers +django==3.0.3 # via -r requirements.in/base.txt, django-anymail, django-cors-headers, django-fullurl, djangorestframework, drf-nested-routers djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers drf-nested-routers==0.91 # via -r requirements.in/base.txt idna==2.8 # via requests -oauthlib==3.1.0 # via requests-oauthlib psycopg2-binary==2.8.4 # via -r requirements.in/base.txt pycparser==2.20 # via cffi pynacl==1.3.0 # via -r requirements.in/base.txt -python3-openid==3.1.0 # via django-allauth pytz==2019.3 # via django -requests-oauthlib==1.3.0 # via django-allauth -requests==2.22.0 # via django-allauth, django-anymail, requests-oauthlib -six==1.14.0 # via django-anymail, django-appconf, pynacl -sqlparse==0.3.0 # via django, django-debug-toolbar +requests==2.22.0 # via django-anymail +six==1.14.0 # via django-anymail, pynacl +sqlparse==0.3.0 # via django urllib3==1.25.8 # via requests From 5b2040fda316f962bb39c82d57c0bd37ba85afc7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 16:05:19 +0300 Subject: [PATCH 124/251] Fix running with postgres: convert memoryview to bytes for nacl. --- django_etesync/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index dff1096..7864c6a 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -601,7 +601,7 @@ class AuthenticationViewSet(viewsets.ViewSet): if serializer.is_valid(): user = self.get_login_user(serializer) - salt = user.userinfo.salt + salt = bytes(user.userinfo.salt) enc_key = self.get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) @@ -637,7 +637,7 @@ class AuthenticationViewSet(viewsets.ViewSet): host = serializer.validated_data['host'] challenge = serializer.validated_data['challenge'] - salt = user.userinfo.salt + salt = bytes(user.userinfo.salt) enc_key = self.get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) @@ -654,7 +654,7 @@ class AuthenticationViewSet(viewsets.ViewSet): content = {'code': 'wrong_host', 'detail': detail} return Response(content, status=status.HTTP_400_BAD_REQUEST) - verify_key = nacl.signing.VerifyKey(user.userinfo.loginPubkey, encoder=nacl.encoding.RawEncoder) + verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) verify_key.verify(response_raw, signature) data = self.login_response_data(user) From 40db4e14b0f1663f2833839b71769811232960cb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 16:05:46 +0300 Subject: [PATCH 125/251] Signup: rename the UserQuerySerializer to Signup. --- django_etesync/serializers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 3b50c36..f117b31 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -339,12 +339,6 @@ class UserSerializer(serializers.ModelSerializer): fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, 'pubkey', 'encryptedSeckey') -class UserQuerySerializer(serializers.ModelSerializer): - class Meta: - model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) - - class UserInfoPubkeySerializer(serializers.ModelSerializer): pubkey = BinaryBase64Field() @@ -353,8 +347,14 @@ class UserInfoPubkeySerializer(serializers.ModelSerializer): fields = ('pubkey', ) +class UserSignupSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) + + class AuthenticationSignupSerializer(serializers.Serializer): - user = UserQuerySerializer(many=False) + user = UserSignupSerializer(many=False) salt = BinaryBase64Field() loginPubkey = BinaryBase64Field() pubkey = BinaryBase64Field() From 6051a5ae3a4250327b03ebc22cf562118b3fb218 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 16:06:59 +0300 Subject: [PATCH 126/251] Signup: use the recommended drf style for validation. --- django_etesync/views.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 7864c6a..52d3531 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -573,13 +573,11 @@ class AuthenticationViewSet(viewsets.ViewSet): @action_decorator(detail=False, methods=['POST']) def signup(self, request): serializer = AuthenticationSignupSerializer(data=request.data) - if serializer.is_valid(): - user = serializer.save() - - data = self.login_response_data(user) - return Response(data, status=status.HTTP_201_CREATED) + serializer.is_valid(raise_exception=True) + user = serializer.save() - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + data = self.login_response_data(user) + return Response(data, status=status.HTTP_201_CREATED) def get_login_user(self, serializer): username = serializer.validated_data.get('username') From 215a2607008b92d8f5fc082fc3a36cc4daf97910 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 16:13:43 +0300 Subject: [PATCH 127/251] Login: use only the username (not email) for login. We may add support for email in the future. --- django_etesync/serializers.py | 8 +------- django_etesync/views.py | 20 +++++++------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index f117b31..337f695 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -381,13 +381,7 @@ class AuthenticationSignupSerializer(serializers.Serializer): class AuthenticationLoginChallengeSerializer(serializers.Serializer): - username = serializers.CharField(required=False) - email = serializers.EmailField(required=False) - - def validate(self, data): - if not data.get('email') and not data.get('username'): - raise serializers.ValidationError('Either email or username must be set') - return data + username = serializers.CharField(required=True) def create(self, validated_data): raise NotImplementedError() diff --git a/django_etesync/views.py b/django_etesync/views.py index 52d3531..ac7a007 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -579,17 +579,9 @@ class AuthenticationViewSet(viewsets.ViewSet): data = self.login_response_data(user) return Response(data, status=status.HTTP_201_CREATED) - def get_login_user(self, serializer): - username = serializer.validated_data.get('username') - email = serializer.validated_data.get('email') - if username: - kwargs = {User.USERNAME_FIELD: username} - user = get_object_or_404(self.get_queryset(), **kwargs) - elif email: - kwargs = {User.EMAIL_FIELD: email} - user = get_object_or_404(self.get_queryset(), **kwargs) - - return user + def get_login_user(self, username): + kwargs = {User.USERNAME_FIELD: username} + return get_object_or_404(self.get_queryset(), **kwargs) @action_decorator(detail=False, methods=['POST']) def login_challenge(self, request): @@ -597,7 +589,8 @@ class AuthenticationViewSet(viewsets.ViewSet): serializer = AuthenticationLoginChallengeSerializer(data=request.data) if serializer.is_valid(): - user = self.get_login_user(serializer) + username = serializer.validated_data.get('username') + user = self.get_login_user(username) salt = bytes(user.userinfo.salt) enc_key = self.get_encryption_key(salt) @@ -631,7 +624,8 @@ class AuthenticationViewSet(viewsets.ViewSet): serializer = AuthenticationLoginInnerSerializer(data=response, context={'host': request.get_host()}) if serializer.is_valid(): - user = self.get_login_user(serializer) + username = serializer.validated_data.get('username') + user = self.get_login_user(username) host = serializer.validated_data['host'] challenge = serializer.validated_data['challenge'] From 15cd41db839833118dbbd6ec5ca5b91a8f61685d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 16:28:54 +0300 Subject: [PATCH 128/251] login: gracefully handle bad login attempts. --- django_etesync/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index ac7a007..e73d567 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -647,7 +647,11 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response(content, status=status.HTTP_400_BAD_REQUEST) verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) - verify_key.verify(response_raw, signature) + + try: + verify_key.verify(response_raw, signature) + except nacl.exceptions.BadSignatureError: + return Response({'code': 'login_bad_signature'}, status=status.HTTP_400_BAD_REQUEST) data = self.login_response_data(user) return Response(data, status=status.HTTP_200_OK) From c2337f244d6089d0b7f983c565b759d53f4a7556 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 16:53:33 +0300 Subject: [PATCH 129/251] Signup: fix signup for users without user info. --- django_etesync/serializers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 337f695..b3b83a0 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -15,6 +15,7 @@ import base64 from django.core.files.base import ContentFile +from django.core import exceptions as django_exceptions from django.contrib.auth import get_user_model from django.db import transaction from rest_framework import serializers @@ -351,6 +352,9 @@ class UserSignupSerializer(serializers.ModelSerializer): class Meta: model = User fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) + extra_kwargs = { + 'username': {'validators': []}, # We specifically validate in SignupSerializer + } class AuthenticationSignupSerializer(serializers.Serializer): @@ -370,6 +374,11 @@ class AuthenticationSignupSerializer(serializers.Serializer): raise serializers.ValidationError('User already exists') instance.set_unusable_password() + + try: + instance.clean_fields() + except django_exceptions.ValidationError as e: + raise serializers.ValidationError(e) # FIXME: send email verification models.UserInfo.objects.create(**validated_data, owner=instance) From 1bd4c5be52a4011adaa08b1befe93c430bf8df8d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 31 May 2020 18:26:21 +0300 Subject: [PATCH 130/251] Send the login signal on login. --- django_etesync/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index e73d567..ece894b 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -16,7 +16,7 @@ import json from functools import reduce from django.conf import settings -from django.contrib.auth import get_user_model +from django.contrib.auth import get_user_model, user_logged_in from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError from django.db.models import Max, Q @@ -654,6 +654,9 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response({'code': 'login_bad_signature'}, status=status.HTTP_400_BAD_REQUEST) data = self.login_response_data(user) + + user_logged_in.send(sender=user.__class__, request=request, user=user) + return Response(data, status=status.HTTP_200_OK) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) From 7842bd4d9cf06a0351f7a00a82b9e66eaced0b96 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 1 Jun 2020 12:45:06 +0300 Subject: [PATCH 131/251] CollectionItem list: don't return the main item. --- django_etesync/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_etesync/views.py b/django_etesync/views.py index ece894b..c41d0c7 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -227,6 +227,7 @@ class CollectionItemViewSet(BaseViewSet): raise Http404("Collection does not exist") # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = type(self).queryset.filter(collection__pk=collection.pk, + uid__isnull=False, revisions__current=True, revisions__deleted=False) From ad184f0ac3ae8d89bc70f00735ee86a5ff08443b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 2 Jun 2020 18:56:23 +0300 Subject: [PATCH 132/251] Rename encryptedSeckey to encryptedContent. --- .../migrations/0014_auto_20200602_1558.py | 18 ++++++++++++++++++ django_etesync/models.py | 2 +- django_etesync/serializers.py | 6 +++--- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 django_etesync/migrations/0014_auto_20200602_1558.py diff --git a/django_etesync/migrations/0014_auto_20200602_1558.py b/django_etesync/migrations/0014_auto_20200602_1558.py new file mode 100644 index 0000000..e226360 --- /dev/null +++ b/django_etesync/migrations/0014_auto_20200602_1558.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-06-02 15:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0013_collectionmemberremoved'), + ] + + operations = [ + migrations.RenameField( + model_name='userinfo', + old_name='encryptedSeckey', + new_name='encryptedContent', + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 7fbea61..18c14a9 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -214,7 +214,7 @@ class UserInfo(models.Model): version = models.PositiveSmallIntegerField(default=1) loginPubkey = models.BinaryField(editable=True, blank=False, null=False) pubkey = models.BinaryField(editable=True, blank=False, null=False) - encryptedSeckey = models.BinaryField(editable=True, blank=False, null=False) + encryptedContent = models.BinaryField(editable=True, blank=False, null=False) salt = models.BinaryField(editable=True, blank=False, null=False) def __str__(self): diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index b3b83a0..5925e46 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -333,11 +333,11 @@ class InvitationAcceptSerializer(serializers.Serializer): class UserSerializer(serializers.ModelSerializer): pubkey = BinaryBase64Field(source='userinfo.pubkey') - encryptedSeckey = BinaryBase64Field(source='userinfo.encryptedSeckey') + encryptedContent = BinaryBase64Field(source='userinfo.encryptedContent') class Meta: model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, 'pubkey', 'encryptedSeckey') + fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, 'pubkey', 'encryptedContent') class UserInfoPubkeySerializer(serializers.ModelSerializer): @@ -362,7 +362,7 @@ class AuthenticationSignupSerializer(serializers.Serializer): salt = BinaryBase64Field() loginPubkey = BinaryBase64Field() pubkey = BinaryBase64Field() - encryptedSeckey = BinaryBase64Field() + encryptedContent = BinaryBase64Field() def create(self, validated_data): """Function that's called when this serializer creates an item""" From 9cc68291df7d693c16458ab11318cbb7624e52c7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 3 Jun 2020 14:21:52 +0300 Subject: [PATCH 133/251] Authentication classes: add permissions to logout. --- django_etesync/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index c41d0c7..3233b7b 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -553,6 +553,7 @@ class InvitationIncomingViewSet(BaseViewSet): class AuthenticationViewSet(viewsets.ViewSet): allowed_methods = ['POST'] + authentication_classes = BaseViewSet.authentication_classes def get_encryption_key(self, salt): key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) @@ -662,7 +663,7 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - @action_decorator(detail=False, methods=['POST']) + @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) def logout(self, request): # FIXME: expire the token - we need better token handling - using knox? Something else? return Response({}, status=status.HTTP_200_OK) From cc23d516a0c8aa64c522fded50b32169976bcbb7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 3 Jun 2020 14:35:44 +0300 Subject: [PATCH 134/251] Add an endpoint to change password. --- django_etesync/serializers.py | 20 ++++++++++++++++++++ django_etesync/views.py | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 5925e46..178c914 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -419,3 +419,23 @@ class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer) def update(self, instance, validated_data): raise NotImplementedError() + + +class AuthenticationChangePasswordSerializer(serializers.ModelSerializer): + loginPubkey = BinaryBase64Field() + encryptedContent = BinaryBase64Field() + + class Meta: + model = models.UserInfo + fields = ('loginPubkey', 'encryptedContent') + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + with transaction.atomic(): + instance.loginPubkey = validated_data.pop('loginPubkey') + instance.encryptedContent = validated_data.pop('encryptedContent') + instance.save() + + return instance diff --git a/django_etesync/views.py b/django_etesync/views.py index 3233b7b..71ae93f 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -48,6 +48,7 @@ from .models import ( ) from .serializers import ( b64encode, + AuthenticationChangePasswordSerializer, AuthenticationSignupSerializer, AuthenticationLoginChallengeSerializer, AuthenticationLoginSerializer, @@ -668,6 +669,14 @@ class AuthenticationViewSet(viewsets.ViewSet): # FIXME: expire the token - we need better token handling - using knox? Something else? return Response({}, status=status.HTTP_200_OK) + @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) + def change_password(self, request): + serializer = AuthenticationChangePasswordSerializer(request.user.userinfo, data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save() + + return Response(status=status.HTTP_200_OK) + class TestAuthenticationViewSet(viewsets.ViewSet): authentication_classes = BaseViewSet.authentication_classes From c00c208199e336a30375c8d9896327375fa61f44 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 3 Jun 2020 15:49:38 +0300 Subject: [PATCH 135/251] Change to our own token authentication. --- django_etesync/token_auth/__init__.py | 0 django_etesync/token_auth/admin.py | 0 django_etesync/token_auth/apps.py | 5 ++ django_etesync/token_auth/authentication.py | 46 +++++++++++++++++++ .../token_auth/migrations/0001_initial.py | 28 +++++++++++ .../token_auth/migrations/__init__.py | 0 django_etesync/token_auth/models.py | 26 +++++++++++ django_etesync/views.py | 12 +++-- 8 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 django_etesync/token_auth/__init__.py create mode 100644 django_etesync/token_auth/admin.py create mode 100644 django_etesync/token_auth/apps.py create mode 100644 django_etesync/token_auth/authentication.py create mode 100644 django_etesync/token_auth/migrations/0001_initial.py create mode 100644 django_etesync/token_auth/migrations/__init__.py create mode 100644 django_etesync/token_auth/models.py diff --git a/django_etesync/token_auth/__init__.py b/django_etesync/token_auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etesync/token_auth/admin.py b/django_etesync/token_auth/admin.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etesync/token_auth/apps.py b/django_etesync/token_auth/apps.py new file mode 100644 index 0000000..dc793f2 --- /dev/null +++ b/django_etesync/token_auth/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class TokenAuthConfig(AppConfig): + name = 'django_etesync.token_auth' diff --git a/django_etesync/token_auth/authentication.py b/django_etesync/token_auth/authentication.py new file mode 100644 index 0000000..432c8cf --- /dev/null +++ b/django_etesync/token_auth/authentication.py @@ -0,0 +1,46 @@ +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from rest_framework import exceptions +from rest_framework.authentication import TokenAuthentication as DRFTokenAuthentication + +from .models import AuthToken, get_default_expiry + + +AUTO_REFRESH = True +MIN_REFRESH_INTERVAL = 60 + + +class TokenAuthentication(DRFTokenAuthentication): + keyword = 'Token' + model = AuthToken + + def authenticate_credentials(self, key): + msg = _('Invalid token.') + model = self.get_model() + try: + token = model.objects.select_related('user').get(key=key) + except model.DoesNotExist: + raise exceptions.AuthenticationFailed(msg) + + if not token.user.is_active: + raise exceptions.AuthenticationFailed(_('User inactive or deleted.')) + + if token.expiry is not None: + if token.expiry < timezone.now(): + token.delete() + raise exceptions.AuthenticationFailed(msg) + + if AUTO_REFRESH: + self.renew_token(token) + + return (token.user, token) + + def renew_token(self, auth_token): + current_expiry = auth_token.expiry + new_expiry = get_default_expiry() + # Throttle refreshing of token to avoid db writes + delta = (new_expiry - current_expiry).total_seconds() + if delta > MIN_REFRESH_INTERVAL: + auth_token.expiry = new_expiry + auth_token.save(update_fields=('expiry',)) diff --git a/django_etesync/token_auth/migrations/0001_initial.py b/django_etesync/token_auth/migrations/0001_initial.py new file mode 100644 index 0000000..f2024e3 --- /dev/null +++ b/django_etesync/token_auth/migrations/0001_initial.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-06-03 12:49 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +from django_etesync.token_auth import models as token_auth_models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='AuthToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(db_index=True, default=token_auth_models.generate_key, max_length=40, unique=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('expiry', models.DateTimeField(blank=True, default=token_auth_models.get_default_expiry, null=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='auth_token_set', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/django_etesync/token_auth/migrations/__init__.py b/django_etesync/token_auth/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etesync/token_auth/models.py b/django_etesync/token_auth/models.py new file mode 100644 index 0000000..9ac0955 --- /dev/null +++ b/django_etesync/token_auth/models.py @@ -0,0 +1,26 @@ +from django.contrib.auth import get_user_model +from django.db import models +from django.utils import timezone +from django.utils.crypto import get_random_string + +User = get_user_model() + + +def generate_key(): + return get_random_string(40) + + +def get_default_expiry(): + return timezone.now() + timezone.timedelta(days=14) + + +class AuthToken(models.Model): + + key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key) + user = models.ForeignKey(User, null=False, blank=False, + related_name='auth_token_set', on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + expiry = models.DateTimeField(null=True, blank=True, default=get_default_expiry) + + def __str__(self): + return '{}: {}'.format(self.key, self.user) diff --git a/django_etesync/views.py b/django_etesync/views.py index 71ae93f..88ae7c4 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -16,7 +16,7 @@ import json from functools import reduce from django.conf import settings -from django.contrib.auth import get_user_model, user_logged_in +from django.contrib.auth import get_user_model, user_logged_in, user_logged_out from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError from django.db.models import Max, Q @@ -28,13 +28,14 @@ from rest_framework import viewsets from rest_framework import parsers from rest_framework.decorators import action as action_decorator from rest_framework.response import Response -from rest_framework.authtoken.models import Token import nacl.encoding import nacl.signing import nacl.secret import nacl.hash +from .token_auth.models import AuthToken + from . import app_settings, permissions from .models import ( Collection, @@ -566,7 +567,7 @@ class AuthenticationViewSet(viewsets.ViewSet): def login_response_data(self, user): return { - 'token': Token.objects.get_or_create(user=user)[0].key, + 'token': AuthToken.objects.create(user=user).key, 'user': UserSerializer(user).data, } @@ -666,8 +667,9 @@ class AuthenticationViewSet(viewsets.ViewSet): @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) def logout(self, request): - # FIXME: expire the token - we need better token handling - using knox? Something else? - return Response({}, status=status.HTTP_200_OK) + request._auth.delete() + user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) + return Response(status=status.HTTP_204_NO_CONTENT) @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) def change_password(self, request): From 29145f22156f30e74e988b8b3efab89db4f510b7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 3 Jun 2020 16:19:07 +0300 Subject: [PATCH 136/251] Logout: don't use internal auth accessor. --- django_etesync/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 88ae7c4..245120b 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -667,7 +667,7 @@ class AuthenticationViewSet(viewsets.ViewSet): @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) def logout(self, request): - request._auth.delete() + request.auth.delete() user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) return Response(status=status.HTTP_204_NO_CONTENT) From 119479d22b6a63fc0b647b552b497016f106d06f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 3 Jun 2020 17:22:10 +0300 Subject: [PATCH 137/251] Test reset: allow anyone to reset test users and fully init accounts. --- django_etesync/views.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index 245120b..f23971d 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -681,8 +681,6 @@ class AuthenticationViewSet(viewsets.ViewSet): class TestAuthenticationViewSet(viewsets.ViewSet): - authentication_classes = BaseViewSet.authentication_classes - permission_classes = BaseViewSet.permission_classes allowed_methods = ['POST'] def list(self, request): @@ -694,13 +692,22 @@ class TestAuthenticationViewSet(viewsets.ViewSet): if not settings.DEBUG: return HttpResponseBadRequest("Only allowed in debug mode.") - # Only allow local users, for extra safety - if not getattr(request.user, User.EMAIL_FIELD).endswith('@localhost'): + user = get_object_or_404(User.objects.all(), username=request.data.get('user').get('username')) + + # Only allow test users for extra safety + if not getattr(user, User.USERNAME_FIELD).startswith('test_user'): return HttpResponseBadRequest("Endpoint not allowed for user.") + if hasattr(user, 'userinfo'): + user.userinfo.delete() + + serializer = AuthenticationSignupSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save() + # Delete all of the journal data for this user for a clear test env - request.user.collection_set.all().delete() - request.user.incoming_invitations.all().delete() + user.collection_set.all().delete() + user.incoming_invitations.all().delete() # FIXME: also delete chunk files!!! From e062fcd4298e6f0b995f6759cc1a32d6504691e1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 4 Jun 2020 15:23:10 +0300 Subject: [PATCH 138/251] Revision: add salt. --- .../0015_collectionitemrevision_salt.py | 18 ++++++++++++++++++ django_etesync/models.py | 1 + django_etesync/serializers.py | 3 ++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 django_etesync/migrations/0015_collectionitemrevision_salt.py diff --git a/django_etesync/migrations/0015_collectionitemrevision_salt.py b/django_etesync/migrations/0015_collectionitemrevision_salt.py new file mode 100644 index 0000000..f5553c9 --- /dev/null +++ b/django_etesync/migrations/0015_collectionitemrevision_salt.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-06-04 12:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0014_auto_20200602_1558'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemrevision', + name='salt', + field=models.BinaryField(default=b'', editable=True), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 18c14a9..53239e7 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -115,6 +115,7 @@ class CollectionItemRevision(models.Model): stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT) uid = models.CharField(db_index=True, unique=True, blank=False, null=False, max_length=43, validators=[Base64Url256BitlValidator]) + salt = models.BinaryField(editable=True, blank=False, null=False, default=b'') item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) meta = models.BinaryField(editable=True, blank=False, null=False) current = models.BooleanField(db_index=True, default=True, null=True) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 178c914..1f5d3c2 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -106,11 +106,12 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): queryset=models.RevisionChunkRelation.objects.all(), many=True ) + salt = BinaryBase64Field() meta = BinaryBase64Field() class Meta: model = models.CollectionItemRevision - fields = ('chunks', 'meta', 'uid', 'deleted') + fields = ('chunks', 'meta', 'uid', 'salt', 'deleted') class CollectionItemSerializer(serializers.ModelSerializer): From 653341115f1cb1e2e9515da1b8cea6e9e4e8e0d7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 4 Jun 2020 16:52:56 +0300 Subject: [PATCH 139/251] Chunks: add stricter validation. --- django_etesync/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index 1f5d3c2..c940d6e 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -91,6 +91,8 @@ class ChunksField(serializers.RelatedField): return (obj.uid, ) def to_internal_value(self, data): + if data[0] is None or data[1] is None: + raise serializers.ValidationError('null is not allowed') return (data[0], b64decode(data[1])) From 23b2bb3c0a0be5c24f3f5b1555fb733d8e161df0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 11 Jun 2020 11:17:01 +0300 Subject: [PATCH 140/251] Batch: refactor code and allow passing deps to check against. --- django_etesync/views.py | 93 +++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 59 deletions(-) diff --git a/django_etesync/views.py b/django_etesync/views.py index f23971d..1d29ae1 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -316,73 +316,48 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST']) def batch(self, request, collection_uid=None): - stoken = request.GET.get('stoken', None) - collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - - if stoken is not None and stoken != collection_object.stoken: - content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - items = request.data.get('items') - serializer = self.get_serializer(data=items, many=True) - - if serializer.is_valid(): - try: - with transaction.atomic(): - items = serializer.save(collection=collection_object) - except IntegrityError: - # FIXME: should return the items with a bad token (including deps) so we don't have to fetch them after - content = {'code': 'integrity_error'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - ret = { - } - return Response(ret, status=status.HTTP_200_OK) - - return Response( - { - "items": serializer.errors, - }, - status=status.HTTP_400_BAD_REQUEST) + return self.transaction(request, collection_uid, validate_etag=False) @action_decorator(detail=False, methods=['POST']) - def transaction(self, request, collection_uid=None): + def transaction(self, request, collection_uid=None, validate_etag=True): stoken = request.GET.get('stoken', None) - collection_object = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - - if stoken is not None and stoken != collection_object.stoken: - content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + with transaction.atomic(): # We need this for locking on the collection object + collection_object = get_object_or_404( + self.get_collection_queryset(Collection.objects).select_for_update(), # Lock writes on the collection + uid=collection_uid) - items = request.data.get('items') - deps = request.data.get('deps', None) - # FIXME: It should just be one serializer - context = self.get_serializer_context() - context.update({'validate_etag': True}) - serializer = self.get_serializer_class()(data=items, context=context, many=True) - deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) + if stoken is not None and stoken != collection_object.stoken: + content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) - ser_valid = serializer.is_valid() - deps_ser_valid = (deps is None or deps_serializer.is_valid()) - if ser_valid and deps_ser_valid: - try: - with transaction.atomic(): + items = request.data.get('items') + deps = request.data.get('deps', None) + # FIXME: It should just be one serializer + context = self.get_serializer_context() + context.update({'validate_etag': validate_etag}) + serializer = self.get_serializer_class()(data=items, context=context, many=True) + deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) + + ser_valid = serializer.is_valid() + deps_ser_valid = (deps is None or deps_serializer.is_valid()) + if ser_valid and deps_ser_valid: + try: items = serializer.save(collection=collection_object) - except IntegrityError: - # FIXME: should return the items with a bad token (including deps) so we don't have to fetch them after - content = {'code': 'integrity_error'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + except IntegrityError: + # FIXME: return the items with a bad token (including deps) so we don't have to fetch them after + content = {'code': 'integrity_error'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) - ret = { - } - return Response(ret, status=status.HTTP_200_OK) + ret = { + } + return Response(ret, status=status.HTTP_200_OK) - return Response( - { - "items": serializer.errors, - "deps": deps_serializer.errors if deps is not None else [], - }, - status=status.HTTP_400_BAD_REQUEST) + return Response( + { + "items": serializer.errors, + "deps": deps_serializer.errors if deps is not None else [], + }, + status=status.HTTP_400_BAD_REQUEST) class CollectionItemChunkViewSet(viewsets.ViewSet): From d1017aac761f06c2e2f7a36c2768da0bdd0edc7a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 16 Jun 2020 17:12:44 +0300 Subject: [PATCH 141/251] Rename django_etesync to django_etebase. --- {django_etesync => django_etebase}/__init__.py | 0 {django_etesync => django_etebase}/admin.py | 0 .../app_settings.py | 2 +- django_etebase/apps.py | 5 +++++ .../migrations/0001_initial.py | 16 ++++++++-------- .../migrations/0002_userinfo.py | 2 +- .../migrations/0003_collectioninvitation.py | 4 ++-- .../0004_collectioninvitation_version.py | 2 +- .../migrations/0005_auto_20200526_1021.py | 2 +- .../migrations/0006_auto_20200526_1040.py | 2 +- .../migrations/0007_auto_20200526_1336.py | 2 +- .../migrations/0008_auto_20200526_1535.py | 8 ++++---- .../migrations/0009_auto_20200526_1535.py | 6 +++--- .../migrations/0010_auto_20200526_1539.py | 4 ++-- .../migrations/0011_collectionmember_stoken.py | 4 ++-- .../migrations/0012_auto_20200527_0743.py | 6 +++--- .../migrations/0013_collectionmemberremoved.py | 6 +++--- .../migrations/0014_auto_20200602_1558.py | 2 +- .../0015_collectionitemrevision_salt.py | 2 +- .../migrations/__init__.py | 0 {django_etesync => django_etebase}/models.py | 0 .../permissions.py | 2 +- .../serializers.py | 0 {django_etesync => django_etebase}/tests.py | 0 .../token_auth/__init__.py | 0 .../token_auth/admin.py | 0 .../token_auth/apps.py | 2 +- .../token_auth/authentication.py | 0 .../token_auth/migrations/0001_initial.py | 2 +- .../token_auth/migrations/__init__.py | 0 .../token_auth/models.py | 0 {django_etesync => django_etebase}/views.py | 2 +- django_etesync/apps.py | 5 ----- 33 files changed, 44 insertions(+), 44 deletions(-) rename {django_etesync => django_etebase}/__init__.py (100%) rename {django_etesync => django_etebase}/admin.py (100%) rename {django_etesync => django_etebase}/app_settings.py (97%) create mode 100644 django_etebase/apps.py rename {django_etesync => django_etebase}/migrations/0001_initial.py (93%) rename {django_etesync => django_etebase}/migrations/0002_userinfo.py (94%) rename {django_etesync => django_etebase}/migrations/0003_collectioninvitation.py (92%) rename {django_etesync => django_etebase}/migrations/0004_collectioninvitation_version.py (86%) rename {django_etesync => django_etebase}/migrations/0005_auto_20200526_1021.py (83%) rename {django_etesync => django_etebase}/migrations/0006_auto_20200526_1040.py (91%) rename {django_etesync => django_etebase}/migrations/0007_auto_20200526_1336.py (96%) rename {django_etesync => django_etebase}/migrations/0008_auto_20200526_1535.py (80%) rename {django_etesync => django_etebase}/migrations/0009_auto_20200526_1535.py (69%) rename {django_etesync => django_etebase}/migrations/0010_auto_20200526_1539.py (79%) rename {django_etesync => django_etebase}/migrations/0011_collectionmember_stoken.py (77%) rename {django_etesync => django_etebase}/migrations/0012_auto_20200527_0743.py (68%) rename {django_etesync => django_etebase}/migrations/0013_collectionmemberremoved.py (86%) rename {django_etesync => django_etebase}/migrations/0014_auto_20200602_1558.py (84%) rename {django_etesync => django_etebase}/migrations/0015_collectionitemrevision_salt.py (86%) rename {django_etesync => django_etebase}/migrations/__init__.py (100%) rename {django_etesync => django_etebase}/models.py (100%) rename {django_etesync => django_etebase}/permissions.py (98%) rename {django_etesync => django_etebase}/serializers.py (100%) rename {django_etesync => django_etebase}/tests.py (100%) rename {django_etesync => django_etebase}/token_auth/__init__.py (100%) rename {django_etesync => django_etebase}/token_auth/admin.py (100%) rename {django_etesync => django_etebase}/token_auth/apps.py (64%) rename {django_etesync => django_etebase}/token_auth/authentication.py (100%) rename {django_etesync => django_etebase}/token_auth/migrations/0001_initial.py (94%) rename {django_etesync => django_etebase}/token_auth/migrations/__init__.py (100%) rename {django_etesync => django_etebase}/token_auth/models.py (100%) rename {django_etesync => django_etebase}/views.py (99%) delete mode 100644 django_etesync/apps.py diff --git a/django_etesync/__init__.py b/django_etebase/__init__.py similarity index 100% rename from django_etesync/__init__.py rename to django_etebase/__init__.py diff --git a/django_etesync/admin.py b/django_etebase/admin.py similarity index 100% rename from django_etesync/admin.py rename to django_etebase/admin.py diff --git a/django_etesync/app_settings.py b/django_etebase/app_settings.py similarity index 97% rename from django_etesync/app_settings.py rename to django_etebase/app_settings.py index 89b38f7..b1fb4c3 100644 --- a/django_etesync/app_settings.py +++ b/django_etebase/app_settings.py @@ -51,4 +51,4 @@ class AppSettings: return self._setting("CHALLENGE_VALID_SECONDS", 60) -app_settings = AppSettings('ETESYNC_') +app_settings = AppSettings('ETEBASE_') diff --git a/django_etebase/apps.py b/django_etebase/apps.py new file mode 100644 index 0000000..286a708 --- /dev/null +++ b/django_etebase/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class DjangoEtebaseConfig(AppConfig): + name = 'django_etebase' diff --git a/django_etesync/migrations/0001_initial.py b/django_etebase/migrations/0001_initial.py similarity index 93% rename from django_etesync/migrations/0001_initial.py rename to django_etebase/migrations/0001_initial.py index dc3a2ff..69a9a91 100644 --- a/django_etesync/migrations/0001_initial.py +++ b/django_etebase/migrations/0001_initial.py @@ -4,7 +4,7 @@ from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion -import django_etesync.models +import django_etebase.models class Migration(migrations.Migration): @@ -35,7 +35,7 @@ class Migration(migrations.Migration): ('uid', models.CharField(db_index=True, max_length=44, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')])), ('version', models.PositiveSmallIntegerField()), ('encryptionKey', models.BinaryField(editable=True, null=True)), - ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='django_etesync.Collection')), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='django_etebase.Collection')), ], options={ 'unique_together': {('uid', 'collection')}, @@ -46,8 +46,8 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), - ('chunkFile', models.FileField(max_length=150, unique=True, upload_to=django_etesync.models.chunk_directory_path)), - ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etesync.CollectionItem')), + ('chunkFile', models.FileField(max_length=150, unique=True, upload_to=django_etebase.models.chunk_directory_path)), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etebase.CollectionItem')), ], ), migrations.CreateModel( @@ -58,7 +58,7 @@ class Migration(migrations.Migration): ('meta', models.BinaryField(editable=True)), ('current', models.BooleanField(db_index=True, default=True, null=True)), ('deleted', models.BooleanField(default=False)), - ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='django_etesync.CollectionItem')), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='django_etebase.CollectionItem')), ], options={ 'unique_together': {('item', 'current')}, @@ -68,8 +68,8 @@ class Migration(migrations.Migration): name='RevisionChunkRelation', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('chunk', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions_relation', to='django_etesync.CollectionItemChunk')), - ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks_relation', to='django_etesync.CollectionItemRevision')), + ('chunk', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions_relation', to='django_etebase.CollectionItemChunk')), + ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks_relation', to='django_etebase.CollectionItemRevision')), ], options={ 'ordering': ('id',), @@ -81,7 +81,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('encryptionKey', models.BinaryField(editable=True)), ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), - ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='django_etesync.Collection')), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='django_etebase.Collection')), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], options={ diff --git a/django_etesync/migrations/0002_userinfo.py b/django_etebase/migrations/0002_userinfo.py similarity index 94% rename from django_etesync/migrations/0002_userinfo.py rename to django_etebase/migrations/0002_userinfo.py index ad7018a..6da0bb8 100644 --- a/django_etesync/migrations/0002_userinfo.py +++ b/django_etebase/migrations/0002_userinfo.py @@ -9,7 +9,7 @@ class Migration(migrations.Migration): dependencies = [ ('myauth', '0001_initial'), - ('django_etesync', '0001_initial'), + ('django_etebase', '0001_initial'), ] operations = [ diff --git a/django_etesync/migrations/0003_collectioninvitation.py b/django_etebase/migrations/0003_collectioninvitation.py similarity index 92% rename from django_etesync/migrations/0003_collectioninvitation.py rename to django_etebase/migrations/0003_collectioninvitation.py index 3880a63..8fd2066 100644 --- a/django_etesync/migrations/0003_collectioninvitation.py +++ b/django_etebase/migrations/0003_collectioninvitation.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('django_etesync', '0002_userinfo'), + ('django_etebase', '0002_userinfo'), ] operations = [ @@ -21,7 +21,7 @@ class Migration(migrations.Migration): ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), ('signedEncryptionKey', models.BinaryField()), ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), - ('fromMember', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etesync.CollectionMember')), + ('fromMember', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etebase.CollectionMember')), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='incoming_invitations', to=settings.AUTH_USER_MODEL)), ], options={ diff --git a/django_etesync/migrations/0004_collectioninvitation_version.py b/django_etebase/migrations/0004_collectioninvitation_version.py similarity index 86% rename from django_etesync/migrations/0004_collectioninvitation_version.py rename to django_etebase/migrations/0004_collectioninvitation_version.py index 3fbaed9..4052116 100644 --- a/django_etesync/migrations/0004_collectioninvitation_version.py +++ b/django_etebase/migrations/0004_collectioninvitation_version.py @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0003_collectioninvitation'), + ('django_etebase', '0003_collectioninvitation'), ] operations = [ diff --git a/django_etesync/migrations/0005_auto_20200526_1021.py b/django_etebase/migrations/0005_auto_20200526_1021.py similarity index 83% rename from django_etesync/migrations/0005_auto_20200526_1021.py rename to django_etebase/migrations/0005_auto_20200526_1021.py index 470556b..da0dc33 100644 --- a/django_etesync/migrations/0005_auto_20200526_1021.py +++ b/django_etebase/migrations/0005_auto_20200526_1021.py @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0004_collectioninvitation_version'), + ('django_etebase', '0004_collectioninvitation_version'), ] operations = [ diff --git a/django_etesync/migrations/0006_auto_20200526_1040.py b/django_etebase/migrations/0006_auto_20200526_1040.py similarity index 91% rename from django_etesync/migrations/0006_auto_20200526_1040.py rename to django_etebase/migrations/0006_auto_20200526_1040.py index 84e8fa3..b86a996 100644 --- a/django_etesync/migrations/0006_auto_20200526_1040.py +++ b/django_etebase/migrations/0006_auto_20200526_1040.py @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0005_auto_20200526_1021'), + ('django_etebase', '0005_auto_20200526_1021'), ] operations = [ diff --git a/django_etesync/migrations/0007_auto_20200526_1336.py b/django_etebase/migrations/0007_auto_20200526_1336.py similarity index 96% rename from django_etesync/migrations/0007_auto_20200526_1336.py rename to django_etebase/migrations/0007_auto_20200526_1336.py index 37e31ac..79978c7 100644 --- a/django_etesync/migrations/0007_auto_20200526_1336.py +++ b/django_etebase/migrations/0007_auto_20200526_1336.py @@ -7,7 +7,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0006_auto_20200526_1040'), + ('django_etebase', '0006_auto_20200526_1040'), ] operations = [ diff --git a/django_etesync/migrations/0008_auto_20200526_1535.py b/django_etebase/migrations/0008_auto_20200526_1535.py similarity index 80% rename from django_etesync/migrations/0008_auto_20200526_1535.py rename to django_etebase/migrations/0008_auto_20200526_1535.py index e544bdd..12656c0 100644 --- a/django_etesync/migrations/0008_auto_20200526_1535.py +++ b/django_etebase/migrations/0008_auto_20200526_1535.py @@ -3,13 +3,13 @@ import django.core.validators from django.db import migrations, models import django.db.models.deletion -import django_etesync.models +import django_etebase.models class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0007_auto_20200526_1336'), + ('django_etebase', '0007_auto_20200526_1336'), ] operations = [ @@ -17,12 +17,12 @@ class Migration(migrations.Migration): name='Stoken', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uid', models.CharField(db_index=True, default=django_etesync.models.generate_stoken_uid, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')])), + ('uid', models.CharField(db_index=True, default=django_etebase.models.generate_stoken_uid, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')])), ], ), migrations.AddField( model_name='collectionitemrevision', name='stoken', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken'), + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken'), ), ] diff --git a/django_etesync/migrations/0009_auto_20200526_1535.py b/django_etebase/migrations/0009_auto_20200526_1535.py similarity index 69% rename from django_etesync/migrations/0009_auto_20200526_1535.py rename to django_etebase/migrations/0009_auto_20200526_1535.py index 53100b3..a6ff498 100644 --- a/django_etesync/migrations/0009_auto_20200526_1535.py +++ b/django_etebase/migrations/0009_auto_20200526_1535.py @@ -4,8 +4,8 @@ from django.db import migrations def create_stokens(apps, schema_editor): - Stoken = apps.get_model('django_etesync', 'Stoken') - CollectionItemRevision = apps.get_model('django_etesync', 'CollectionItemRevision') + Stoken = apps.get_model('django_etebase', 'Stoken') + CollectionItemRevision = apps.get_model('django_etebase', 'CollectionItemRevision') for rev in CollectionItemRevision.objects.all(): rev.stoken = Stoken.objects.create() @@ -15,7 +15,7 @@ def create_stokens(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0008_auto_20200526_1535'), + ('django_etebase', '0008_auto_20200526_1535'), ] operations = [ diff --git a/django_etesync/migrations/0010_auto_20200526_1539.py b/django_etebase/migrations/0010_auto_20200526_1539.py similarity index 79% rename from django_etesync/migrations/0010_auto_20200526_1539.py rename to django_etebase/migrations/0010_auto_20200526_1539.py index c894fd2..7ef0eca 100644 --- a/django_etesync/migrations/0010_auto_20200526_1539.py +++ b/django_etebase/migrations/0010_auto_20200526_1539.py @@ -7,13 +7,13 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0009_auto_20200526_1535'), + ('django_etebase', '0009_auto_20200526_1535'), ] operations = [ migrations.AlterField( model_name='collectionitemrevision', name='stoken', - field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken'), + field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken'), ), ] diff --git a/django_etesync/migrations/0011_collectionmember_stoken.py b/django_etebase/migrations/0011_collectionmember_stoken.py similarity index 77% rename from django_etesync/migrations/0011_collectionmember_stoken.py rename to django_etebase/migrations/0011_collectionmember_stoken.py index 6b79bf6..bafaea7 100644 --- a/django_etesync/migrations/0011_collectionmember_stoken.py +++ b/django_etebase/migrations/0011_collectionmember_stoken.py @@ -7,13 +7,13 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0010_auto_20200526_1539'), + ('django_etebase', '0010_auto_20200526_1539'), ] operations = [ migrations.AddField( model_name='collectionmember', name='stoken', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken'), + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken'), ), ] diff --git a/django_etesync/migrations/0012_auto_20200527_0743.py b/django_etebase/migrations/0012_auto_20200527_0743.py similarity index 68% rename from django_etesync/migrations/0012_auto_20200527_0743.py rename to django_etebase/migrations/0012_auto_20200527_0743.py index 28b8745..ab6adbc 100644 --- a/django_etesync/migrations/0012_auto_20200527_0743.py +++ b/django_etebase/migrations/0012_auto_20200527_0743.py @@ -4,8 +4,8 @@ from django.db import migrations def create_stokens(apps, schema_editor): - Stoken = apps.get_model('django_etesync', 'Stoken') - CollectionMember = apps.get_model('django_etesync', 'CollectionMember') + Stoken = apps.get_model('django_etebase', 'Stoken') + CollectionMember = apps.get_model('django_etebase', 'CollectionMember') for member in CollectionMember.objects.all(): member.stoken = Stoken.objects.create() @@ -15,7 +15,7 @@ def create_stokens(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0011_collectionmember_stoken'), + ('django_etebase', '0011_collectionmember_stoken'), ] operations = [ diff --git a/django_etesync/migrations/0013_collectionmemberremoved.py b/django_etebase/migrations/0013_collectionmemberremoved.py similarity index 86% rename from django_etesync/migrations/0013_collectionmemberremoved.py rename to django_etebase/migrations/0013_collectionmemberremoved.py index e796232..2641c03 100644 --- a/django_etesync/migrations/0013_collectionmemberremoved.py +++ b/django_etebase/migrations/0013_collectionmemberremoved.py @@ -9,7 +9,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('django_etesync', '0012_auto_20200527_0743'), + ('django_etebase', '0012_auto_20200527_0743'), ] operations = [ @@ -17,8 +17,8 @@ class Migration(migrations.Migration): name='CollectionMemberRemoved', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='removed_members', to='django_etesync.Collection')), - ('stoken', models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etesync.Stoken')), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='removed_members', to='django_etebase.Collection')), + ('stoken', models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken')), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], options={ diff --git a/django_etesync/migrations/0014_auto_20200602_1558.py b/django_etebase/migrations/0014_auto_20200602_1558.py similarity index 84% rename from django_etesync/migrations/0014_auto_20200602_1558.py rename to django_etebase/migrations/0014_auto_20200602_1558.py index e226360..d1a555d 100644 --- a/django_etesync/migrations/0014_auto_20200602_1558.py +++ b/django_etebase/migrations/0014_auto_20200602_1558.py @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0013_collectionmemberremoved'), + ('django_etebase', '0013_collectionmemberremoved'), ] operations = [ diff --git a/django_etesync/migrations/0015_collectionitemrevision_salt.py b/django_etebase/migrations/0015_collectionitemrevision_salt.py similarity index 86% rename from django_etesync/migrations/0015_collectionitemrevision_salt.py rename to django_etebase/migrations/0015_collectionitemrevision_salt.py index f5553c9..7f3dd71 100644 --- a/django_etesync/migrations/0015_collectionitemrevision_salt.py +++ b/django_etebase/migrations/0015_collectionitemrevision_salt.py @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('django_etesync', '0014_auto_20200602_1558'), + ('django_etebase', '0014_auto_20200602_1558'), ] operations = [ diff --git a/django_etesync/migrations/__init__.py b/django_etebase/migrations/__init__.py similarity index 100% rename from django_etesync/migrations/__init__.py rename to django_etebase/migrations/__init__.py diff --git a/django_etesync/models.py b/django_etebase/models.py similarity index 100% rename from django_etesync/models.py rename to django_etebase/models.py diff --git a/django_etesync/permissions.py b/django_etebase/permissions.py similarity index 98% rename from django_etesync/permissions.py rename to django_etebase/permissions.py index 611977b..a4217a8 100644 --- a/django_etesync/permissions.py +++ b/django_etebase/permissions.py @@ -13,7 +13,7 @@ # along with this program. If not, see . from rest_framework import permissions -from django_etesync.models import Collection, AccessLevels +from django_etebase.models import Collection, AccessLevels def is_collection_admin(collection, user): diff --git a/django_etesync/serializers.py b/django_etebase/serializers.py similarity index 100% rename from django_etesync/serializers.py rename to django_etebase/serializers.py diff --git a/django_etesync/tests.py b/django_etebase/tests.py similarity index 100% rename from django_etesync/tests.py rename to django_etebase/tests.py diff --git a/django_etesync/token_auth/__init__.py b/django_etebase/token_auth/__init__.py similarity index 100% rename from django_etesync/token_auth/__init__.py rename to django_etebase/token_auth/__init__.py diff --git a/django_etesync/token_auth/admin.py b/django_etebase/token_auth/admin.py similarity index 100% rename from django_etesync/token_auth/admin.py rename to django_etebase/token_auth/admin.py diff --git a/django_etesync/token_auth/apps.py b/django_etebase/token_auth/apps.py similarity index 64% rename from django_etesync/token_auth/apps.py rename to django_etebase/token_auth/apps.py index dc793f2..118b872 100644 --- a/django_etesync/token_auth/apps.py +++ b/django_etebase/token_auth/apps.py @@ -2,4 +2,4 @@ from django.apps import AppConfig class TokenAuthConfig(AppConfig): - name = 'django_etesync.token_auth' + name = 'django_etebase.token_auth' diff --git a/django_etesync/token_auth/authentication.py b/django_etebase/token_auth/authentication.py similarity index 100% rename from django_etesync/token_auth/authentication.py rename to django_etebase/token_auth/authentication.py diff --git a/django_etesync/token_auth/migrations/0001_initial.py b/django_etebase/token_auth/migrations/0001_initial.py similarity index 94% rename from django_etesync/token_auth/migrations/0001_initial.py rename to django_etebase/token_auth/migrations/0001_initial.py index f2024e3..5a47366 100644 --- a/django_etesync/token_auth/migrations/0001_initial.py +++ b/django_etebase/token_auth/migrations/0001_initial.py @@ -3,7 +3,7 @@ from django.conf import settings from django.db import migrations, models import django.db.models.deletion -from django_etesync.token_auth import models as token_auth_models +from django_etebase.token_auth import models as token_auth_models class Migration(migrations.Migration): diff --git a/django_etesync/token_auth/migrations/__init__.py b/django_etebase/token_auth/migrations/__init__.py similarity index 100% rename from django_etesync/token_auth/migrations/__init__.py rename to django_etebase/token_auth/migrations/__init__.py diff --git a/django_etesync/token_auth/models.py b/django_etebase/token_auth/models.py similarity index 100% rename from django_etesync/token_auth/models.py rename to django_etebase/token_auth/models.py diff --git a/django_etesync/views.py b/django_etebase/views.py similarity index 99% rename from django_etesync/views.py rename to django_etebase/views.py index 1d29ae1..59fcaa2 100644 --- a/django_etesync/views.py +++ b/django_etebase/views.py @@ -534,7 +534,7 @@ class AuthenticationViewSet(viewsets.ViewSet): def get_encryption_key(self, salt): key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) - return nacl.hash.blake2b(b'', key=key, salt=salt[:nacl.hash.BLAKE2B_SALTBYTES], person=b'etesync-auth', + return nacl.hash.blake2b(b'', key=key, salt=salt[:nacl.hash.BLAKE2B_SALTBYTES], person=b'etebase-auth', encoder=nacl.encoding.RawEncoder) def get_queryset(self): diff --git a/django_etesync/apps.py b/django_etesync/apps.py deleted file mode 100644 index adb8f96..0000000 --- a/django_etesync/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class DjangoEtesyncConfig(AppConfig): - name = 'django_etesync' From 54268ac0273486e53829d812c6d30dccaac2e214 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 17 Jun 2020 14:08:08 +0300 Subject: [PATCH 142/251] Login: add an action indicator to know the user signed a login request. --- django_etebase/serializers.py | 1 + django_etebase/views.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index c940d6e..576431d 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -416,6 +416,7 @@ class AuthenticationLoginSerializer(serializers.Serializer): class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): challenge = BinaryBase64Field() host = serializers.CharField() + action = serializers.CharField() def create(self, validated_data): raise NotImplementedError() diff --git a/django_etebase/views.py b/django_etebase/views.py index 59fcaa2..2b0ec58 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -607,6 +607,7 @@ class AuthenticationViewSet(viewsets.ViewSet): user = self.get_login_user(username) host = serializer.validated_data['host'] challenge = serializer.validated_data['challenge'] + action = serializer.validated_data['action'] salt = bytes(user.userinfo.salt) enc_key = self.get_encryption_key(salt) @@ -614,7 +615,10 @@ class AuthenticationViewSet(viewsets.ViewSet): challenge_data = json.loads(box.decrypt(challenge).decode()) now = int(datetime.now().timestamp()) - if now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: + if action != "login": + content = {'code': 'wrong_action', 'detail': 'Expected "login" but got something else'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: content = {'code': 'challenge_expired', 'detail': 'Login challange has expired'} return Response(content, status=status.HTTP_400_BAD_REQUEST) elif challenge_data['userId'] != user.id: From ab0d85c84fa4ffb5ebd5e797283150a6a6881ab5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 17 Jun 2020 14:38:02 +0300 Subject: [PATCH 143/251] Change password: change to require a signed request, just like login. Without this, it would be sufficient to steal an auth token to render the account unusable because it would be possible to just reset the encrypted content of the account. With this change we require the user to actually know the account password in order to do it. --- django_etebase/serializers.py | 2 +- django_etebase/views.py | 117 +++++++++++++++++++++------------- 2 files changed, 72 insertions(+), 47 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 576431d..c401d91 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -425,7 +425,7 @@ class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer) raise NotImplementedError() -class AuthenticationChangePasswordSerializer(serializers.ModelSerializer): +class AuthenticationChangePasswordInnerSerializer(AuthenticationLoginInnerSerializer): loginPubkey = BinaryBase64Field() encryptedContent = BinaryBase64Field() diff --git a/django_etebase/views.py b/django_etebase/views.py index 2b0ec58..999dc65 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -49,7 +49,7 @@ from .models import ( ) from .serializers import ( b64encode, - AuthenticationChangePasswordSerializer, + AuthenticationChangePasswordInnerSerializer, AuthenticationSignupSerializer, AuthenticationLoginChallengeSerializer, AuthenticationLoginSerializer, @@ -562,6 +562,44 @@ class AuthenticationViewSet(viewsets.ViewSet): kwargs = {User.USERNAME_FIELD: username} return get_object_or_404(self.get_queryset(), **kwargs) + def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): + from datetime import datetime + + username = validated_data.get('username') + user = self.get_login_user(username) + host = validated_data['host'] + challenge = validated_data['challenge'] + action = validated_data['action'] + + salt = bytes(user.userinfo.salt) + enc_key = self.get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + + challenge_data = json.loads(box.decrypt(challenge).decode()) + now = int(datetime.now().timestamp()) + if action != expected_action: + content = {'code': 'wrong_action', 'detail': 'Expected "{}" but got something else'.format(expected_action)} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: + content = {'code': 'challenge_expired', 'detail': 'Login challange has expired'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif challenge_data['userId'] != user.id: + content = {'code': 'wrong_user', 'detail': 'This challenge is for the wrong user'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif not settings.DEBUG and host != request.get_host(): + detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) + content = {'code': 'wrong_host', 'detail': detail} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) + + try: + verify_key.verify(response_raw, signature) + except nacl.exceptions.BadSignatureError: + return Response({'code': 'login_bad_signature'}, status=status.HTTP_400_BAD_REQUEST) + + return None + @action_decorator(detail=False, methods=['POST']) def login_challenge(self, request): from datetime import datetime @@ -593,56 +631,29 @@ class AuthenticationViewSet(viewsets.ViewSet): @action_decorator(detail=False, methods=['POST']) def login(self, request): - from datetime import datetime - outer_serializer = AuthenticationLoginSerializer(data=request.data) - if outer_serializer.is_valid(): - response_raw = outer_serializer.validated_data['response'] - response = json.loads(response_raw.decode()) - signature = outer_serializer.validated_data['signature'] - - serializer = AuthenticationLoginInnerSerializer(data=response, context={'host': request.get_host()}) - if serializer.is_valid(): - username = serializer.validated_data.get('username') - user = self.get_login_user(username) - host = serializer.validated_data['host'] - challenge = serializer.validated_data['challenge'] - action = serializer.validated_data['action'] - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = json.loads(box.decrypt(challenge).decode()) - now = int(datetime.now().timestamp()) - if action != "login": - content = {'code': 'wrong_action', 'detail': 'Expected "login" but got something else'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: - content = {'code': 'challenge_expired', 'detail': 'Login challange has expired'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif challenge_data['userId'] != user.id: - content = {'code': 'wrong_user', 'detail': 'This challenge is for the wrong user'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif not settings.DEBUG and host != request.get_host(): - detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) - content = {'code': 'wrong_host', 'detail': detail} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + outer_serializer.is_valid(raise_exception=True) - verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) + response_raw = outer_serializer.validated_data['response'] + response = json.loads(response_raw.decode()) + signature = outer_serializer.validated_data['signature'] - try: - verify_key.verify(response_raw, signature) - except nacl.exceptions.BadSignatureError: - return Response({'code': 'login_bad_signature'}, status=status.HTTP_400_BAD_REQUEST) + serializer = AuthenticationLoginInnerSerializer(data=response, context={'host': request.get_host()}) + serializer.is_valid(raise_exception=True) - data = self.login_response_data(user) + bad_login_response = self.validate_login_request( + request, serializer.validated_data, response_raw, signature, "login") + if bad_login_response is not None: + return bad_login_response - user_logged_in.send(sender=user.__class__, request=request, user=user) + username = serializer.validated_data.get('username') + user = self.get_login_user(username) - return Response(data, status=status.HTTP_200_OK) + data = self.login_response_data(user) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + user_logged_in.send(sender=user.__class__, request=request, user=user) + + return Response(data, status=status.HTTP_200_OK) @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) def logout(self, request): @@ -652,11 +663,25 @@ class AuthenticationViewSet(viewsets.ViewSet): @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) def change_password(self, request): - serializer = AuthenticationChangePasswordSerializer(request.user.userinfo, data=request.data) + outer_serializer = AuthenticationLoginSerializer(data=request.data) + outer_serializer.is_valid(raise_exception=True) + + response_raw = outer_serializer.validated_data['response'] + response = json.loads(response_raw.decode()) + signature = outer_serializer.validated_data['signature'] + + serializer = AuthenticationChangePasswordInnerSerializer( + request.user.userinfo, data=response, context={'host': request.get_host()}) serializer.is_valid(raise_exception=True) + + bad_login_response = self.validate_login_request( + request, serializer.validated_data, response_raw, signature, "changePassword") + if bad_login_response is not None: + return bad_login_response + serializer.save() - return Response(status=status.HTTP_200_OK) + return Response({}, status=status.HTTP_200_OK) class TestAuthenticationViewSet(viewsets.ViewSet): From 2d7b90e8486faf6539d3a631890e7b84315c7837 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 18 Jun 2020 16:14:55 +0300 Subject: [PATCH 144/251] Collection items: also show deleted items. This was a mistake. We want deleted items to show because we want to know when things have been deleted when we ask for updates. --- django_etebase/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 999dc65..0fb4d5f 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -230,8 +230,7 @@ class CollectionItemViewSet(BaseViewSet): # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = type(self).queryset.filter(collection__pk=collection.pk, uid__isnull=False, - revisions__current=True, - revisions__deleted=False) + revisions__current=True) return queryset From 6117cac1111ae04722463c240b4eb03e7d237add Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 22 Jun 2020 13:03:58 +0300 Subject: [PATCH 145/251] List APIs: return a done field to indicate the fetch is done. --- django_etebase/views.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 0fb4d5f..2944fdb 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -122,8 +122,10 @@ class BaseViewSet(viewsets.ModelViewSet): queryset = queryset[:limit] queryset, new_stoken = self.get_queryset_stoken(queryset) new_stoken = new_stoken or stoken + # This is not the most efficient way of implementing this, but it's good enough + done = len(queryset) < limit - return queryset, new_stoken + return queryset, new_stoken, done # Change how our list works by default def list(self, request, collection_uid=None): @@ -193,13 +195,14 @@ class CollectionViewSet(BaseViewSet): def list(self, request): queryset = self.get_queryset() - queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) + queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) serializer = self.get_serializer(queryset, many=True) ret = { 'data': serializer.data, 'stoken': new_stoken, + 'done': done, } stoken_obj = self.get_stoken_obj(request) @@ -256,13 +259,14 @@ class CollectionItemViewSet(BaseViewSet): def list(self, request, collection_uid=None): queryset = self.get_queryset() - queryset, new_stoken = self.filter_by_stoken_and_limit(request, queryset) + queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) serializer = self.get_serializer(queryset, many=True) ret = { 'data': serializer.data, 'stoken': new_stoken, + 'done': done, } return Response(ret) @@ -275,6 +279,7 @@ class CollectionItemViewSet(BaseViewSet): serializer = CollectionItemRevisionSerializer(col_it.revisions.order_by('-id'), many=True) ret = { 'data': serializer.data, + 'done': True, # we always return all the items, so it's always done } return Response(ret) @@ -308,6 +313,7 @@ class CollectionItemViewSet(BaseViewSet): ret = { 'data': serializer.data, 'stoken': new_stoken, + 'done': True, # we always return all the items, so it's always done } return Response(ret) From fcb58f0f4ce353c2aa8bef3f68b7d70572ae63fa Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 22 Jun 2020 14:20:26 +0300 Subject: [PATCH 146/251] List APIs: fix the stoken calculation for collections. I'm not sure why it just wouldn't work with aggregate. I also couldn't get it to work with annotate then aggregate or any other alternative. --- django_etebase/views.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 2944fdb..955c49f 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -106,9 +106,15 @@ class BaseViewSet(viewsets.ModelViewSet): return queryset, stoken_rev def get_queryset_stoken(self, queryset): - aggr_fields = {x: Max(x) for x in self.stoken_id_fields} - aggr = queryset.aggregate(**aggr_fields) - maxid = max(map(lambda x: x or -1, aggr.values())) + aggr_field_names = ['max_{}'.format(i) for i, x in enumerate(self.stoken_id_fields)] + aggr_fields = {name: Max(field) for name, field in zip(aggr_field_names, self.stoken_id_fields)} + aggr = queryset.annotate(**aggr_fields).values(*aggr_field_names) + # FIXME: we are doing it in python instead of SQL because I just couldn't get aggregate to work over the + # annotated values. This should probably be fixed as it could be quite slow + maxid = -1 + for row in aggr: + rowmaxid = max(map(lambda x: x or -1, row.values())) + maxid = max(maxid, rowmaxid) new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid).uid return queryset, new_stoken From b4db35bca16d251bca2090876b7b903a4611492f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 22 Jun 2020 17:27:07 +0300 Subject: [PATCH 147/251] List APIs: add done to APIs that didn't have it. --- django_etebase/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_etebase/views.py b/django_etebase/views.py index 955c49f..0fb6395 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -140,6 +140,7 @@ class BaseViewSet(viewsets.ModelViewSet): ret = { 'data': serializer.data, + 'done': True, # we always return all the items, so it's always done } return Response(ret) @@ -449,6 +450,7 @@ class CollectionMemberViewSet(BaseViewSet): ret = { 'data': serializer.data, + 'done': True, # we always return all the items, so it's always done } return Response(ret) From d5300a76d8fd5c1e6d2154c74689b299a1563211 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 22 Jun 2020 17:51:56 +0300 Subject: [PATCH 148/251] Members: add support for iterators when listing members --- django_etebase/views.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 0fb6395..eb507f2 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -428,6 +428,7 @@ class CollectionMemberViewSet(BaseViewSet): serializer_class = CollectionMemberSerializer lookup_field = 'user__' + User.USERNAME_FIELD lookup_url_kwarg = 'username' + stoken_id_fields = ['stoken__id'] # FIXME: need to make sure that there's always an admin, and maybe also don't let an owner remove adm access # (if we want to transfer, we need to do that specifically) @@ -444,13 +445,24 @@ class CollectionMemberViewSet(BaseViewSet): return queryset.filter(collection=collection) + # We override this method because we expect the stoken to be called iterator + def get_stoken_obj(self, request): + stoken = request.GET.get('iterator', None) + + if stoken is not None: + return get_object_or_404(Stoken.objects.all(), uid=stoken) + + return None + def list(self, request, collection_uid=None): queryset = self.get_queryset() + queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) serializer = self.get_serializer(queryset, many=True) ret = { 'data': serializer.data, - 'done': True, # we always return all the items, so it's always done + 'iterator': new_stoken, # Here we call it an iterator, it's only stoken for collection/items + 'done': done, } return Response(ret) From 37bae63a466b4ad7bc8eee62817488acabc8327d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 22 Jun 2020 18:26:32 +0300 Subject: [PATCH 149/251] Invitations: add support for fetching using an iterator --- django_etebase/views.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index eb507f2..70e635b 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -486,14 +486,42 @@ class CollectionMemberViewSet(BaseViewSet): return Response({}) -class InvitationOutgoingViewSet(BaseViewSet): - allowed_methods = ['GET', 'POST', 'PUT', 'DELETE'] - permission_classes = BaseViewSet.permission_classes +class InvitationBaseViewSet(BaseViewSet): queryset = CollectionInvitation.objects.all() serializer_class = CollectionInvitationSerializer lookup_field = 'uid' lookup_url_kwarg = 'invitation_uid' + def list(self, request, collection_uid=None): + limit = int(request.GET.get('limit', 50)) + iterator = request.GET.get('iterator', None) + + queryset = self.get_queryset().order_by('id') + + if iterator is not None: + iterator = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__gt=iterator.id) + + queryset = queryset[:limit] + serializer = self.get_serializer(queryset, many=True) + + # This is not the most efficient way of implementing this, but it's good enough + done = len(queryset) < limit + + last_item = len(queryset) > 0 and serializer.data[-1] + + ret = { + 'data': serializer.data, + 'iterator': last_item and last_item['uid'], + 'done': done, + } + + return Response(ret) + + +class InvitationOutgoingViewSet(InvitationBaseViewSet): + allowed_methods = ['GET', 'POST', 'PUT', 'DELETE'] + def get_queryset(self, queryset=None): if queryset is None: queryset = type(self).queryset @@ -528,12 +556,8 @@ class InvitationOutgoingViewSet(BaseViewSet): return Response(serializer.data) -class InvitationIncomingViewSet(BaseViewSet): +class InvitationIncomingViewSet(InvitationBaseViewSet): allowed_methods = ['GET', 'DELETE'] - queryset = CollectionInvitation.objects.all() - serializer_class = CollectionInvitationSerializer - lookup_field = 'uid' - lookup_url_kwarg = 'invitation_uid' def get_queryset(self, queryset=None): if queryset is None: From 267d749c45a52378310705768c3b6685df8744f6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 23 Jun 2020 12:55:28 +0300 Subject: [PATCH 150/251] Collection: change collections to be an extension of items Each collection now has an item and the item's UID is the collections UID. This lets us manipulate collections just like items, and as part of transactions. This is significant because it lets us change them as part of transactions! --- .../migrations/0016_auto_20200623_0820.py | 31 +++++++++++++ django_etebase/models.py | 11 ++--- django_etebase/permissions.py | 6 +-- django_etebase/serializers.py | 43 ++++++++++--------- django_etebase/views.py | 37 ++++++---------- 5 files changed, 72 insertions(+), 56 deletions(-) create mode 100644 django_etebase/migrations/0016_auto_20200623_0820.py diff --git a/django_etebase/migrations/0016_auto_20200623_0820.py b/django_etebase/migrations/0016_auto_20200623_0820.py new file mode 100644 index 0000000..2c11157 --- /dev/null +++ b/django_etebase/migrations/0016_auto_20200623_0820.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.3 on 2020-06-23 08:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0015_collectionitemrevision_salt'), + ] + + operations = [ + migrations.AddField( + model_name='collection', + name='main_item', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='parent', to='django_etebase.CollectionItem'), + ), + migrations.AlterUniqueTogether( + name='collection', + unique_together=set(), + ), + migrations.RemoveField( + model_name='collection', + name='uid', + ), + migrations.RemoveField( + model_name='collection', + name='version', + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 53239e7..2702389 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -27,20 +27,15 @@ UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9]*$', message='Not a valid UID' class Collection(models.Model): - uid = models.CharField(db_index=True, blank=False, null=False, - max_length=43, validators=[UidValidator]) - version = models.PositiveSmallIntegerField() + main_item = models.ForeignKey('CollectionItem', related_name='parent', null=True, on_delete=models.SET_NULL) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - class Meta: - unique_together = ('uid', 'owner') - def __str__(self): return self.uid @cached_property - def main_item(self): - return self.items.get(uid=None) + def uid(self): + return self.main_item.uid @property def content(self): diff --git a/django_etebase/permissions.py b/django_etebase/permissions.py index a4217a8..6a36afb 100644 --- a/django_etebase/permissions.py +++ b/django_etebase/permissions.py @@ -31,7 +31,7 @@ class IsCollectionAdmin(permissions.BasePermission): def has_permission(self, request, view): collection_uid = view.kwargs['collection_uid'] try: - collection = view.get_collection_queryset().get(uid=collection_uid) + collection = view.get_collection_queryset().get(main_item__uid=collection_uid) return is_collection_admin(collection, request.user) except Collection.DoesNotExist: # If the collection does not exist, we want to 404 later, not permission denied. @@ -53,7 +53,7 @@ class IsCollectionAdminOrReadOnly(permissions.BasePermission): return True try: - collection = view.get_collection_queryset().get(uid=collection_uid) + collection = view.get_collection_queryset().get(main_item__uid=collection_uid) if request.method in permissions.SAFE_METHODS: return True @@ -73,7 +73,7 @@ class HasWriteAccessOrReadOnly(permissions.BasePermission): def has_permission(self, request, view): collection_uid = view.kwargs['collection_uid'] try: - collection = view.get_collection_queryset().get(uid=collection_uid) + collection = view.get_collection_queryset().get(main_item__uid=collection_uid) if request.method in permissions.SAFE_METHODS: return True else: diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index c401d91..c194fdd 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -182,15 +182,19 @@ class CollectionItemBulkGetSerializer(serializers.ModelSerializer): class CollectionSerializer(serializers.ModelSerializer): - encryptionKey = CollectionEncryptionKeyField() + collectionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') stoken = serializers.CharField(read_only=True) + + uid = serializers.CharField(source='main_item.uid') + encryptionKey = BinaryBase64Field(source='main_item.encryptionKey') etag = serializers.CharField(allow_null=True, write_only=True) - content = CollectionItemRevisionSerializer(many=False) + version = serializers.IntegerField(min_value=0, source='main_item.version') + content = CollectionItemRevisionSerializer(many=False, source='main_item.content') class Meta: model = models.Collection - fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'content', 'stoken', 'etag') + fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'collectionKey', 'content', 'stoken', 'etag') def get_access_level_from_context(self, obj): request = self.context.get('request', None) @@ -200,9 +204,16 @@ class CollectionSerializer(serializers.ModelSerializer): def create(self, validated_data): """Function that's called when this serializer creates an item""" + collection_key = validated_data.pop('collectionKey') + etag = validated_data.pop('etag') - revision_data = validated_data.pop('content') - encryption_key = validated_data.pop('encryptionKey') + + main_item_data = validated_data.pop('main_item') + uid = main_item_data.pop('uid') + version = main_item_data.pop('version') + revision_data = main_item_data.pop('content') + encryption_key = main_item_data.pop('encryptionKey') + instance = self.__class__.Meta.model(**validated_data) with transaction.atomic(): @@ -211,7 +222,10 @@ class CollectionSerializer(serializers.ModelSerializer): instance.save() main_item = models.CollectionItem.objects.create( - uid=None, encryptionKey=None, version=instance.version, collection=instance) + uid=uid, encryptionKey=encryption_key, version=version, collection=instance) + + instance.main_item = main_item + instance.save() process_revisions_for_item(main_item, revision_data) @@ -219,26 +233,13 @@ class CollectionSerializer(serializers.ModelSerializer): stoken=models.Stoken.objects.create(), user=validated_data.get('owner'), accessLevel=models.AccessLevels.ADMIN, - encryptionKey=encryption_key, + encryptionKey=collection_key, ).save() return instance def update(self, instance, validated_data): - """Function that's called when this serializer is meant to update an item""" - revision_data = validated_data.pop('content') - - with transaction.atomic(): - main_item = instance.main_item - # We don't have to use select_for_update here because the unique constraint on current guards against - # the race condition. But it's a good idea because it'll lock and wait rather than fail. - current_revision = main_item.revisions.filter(current=True).select_for_update().first() - current_revision.current = None - current_revision.save() - - process_revisions_for_item(main_item, revision_data) - - return instance + raise NotImplementedError() class CollectionMemberSerializer(serializers.ModelSerializer): diff --git a/django_etebase/views.py b/django_etebase/views.py index 70e635b..4f5d757 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -147,11 +147,12 @@ class BaseViewSet(viewsets.ModelViewSet): class CollectionViewSet(BaseViewSet): - allowed_methods = ['GET', 'POST', 'DELETE'] + allowed_methods = ['GET', 'POST'] permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdminOrReadOnly, ) queryset = Collection.objects.all() serializer_class = CollectionSerializer - lookup_field = 'uid' + lookup_field = 'main_item__uid' + lookup_url_kwarg = 'uid' stoken_id_fields = ['items__revisions__stoken__id', 'members__stoken__id'] def get_queryset(self, queryset=None): @@ -173,19 +174,7 @@ class CollectionViewSet(BaseViewSet): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def update(self, request, *args, **kwargs): - instance = self.get_object() - - stoken = request.GET.get('stoken', None) - - if stoken is not None and stoken != instance.stoken: - content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - serializer = self.get_serializer(instance, data=request.data) - serializer.is_valid(raise_exception=True) - self.perform_update(serializer) - - return Response({}) + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) @@ -216,7 +205,7 @@ class CollectionViewSet(BaseViewSet): if stoken_obj is not None: # FIXME: honour limit? (the limit should be combined for data and this because of stoken) remed = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) \ - .values_list('collection__uid', flat=True) + .values_list('collection__main_item__uid', flat=True) if len(remed) > 0: ret['removedMemberships'] = [{'uid': x} for x in remed] @@ -234,7 +223,7 @@ class CollectionItemViewSet(BaseViewSet): def get_queryset(self): collection_uid = self.kwargs['collection_uid'] try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) except Collection.DoesNotExist: raise Http404("Collection does not exist") # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') @@ -280,7 +269,7 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=True, methods=['GET']) def revision(self, request, collection_uid=None, uid=None): # FIXME: need pagination support - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) col_it = get_object_or_404(col.items, uid=uid) serializer = CollectionItemRevisionSerializer(col_it.revisions.order_by('-id'), many=True) @@ -336,7 +325,7 @@ class CollectionItemViewSet(BaseViewSet): with transaction.atomic(): # We need this for locking on the collection object collection_object = get_object_or_404( self.get_collection_queryset(Collection.objects).select_for_update(), # Lock writes on the collection - uid=collection_uid) + main_item__uid=collection_uid) if stoken is not None and stoken != collection_object.stoken: content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} @@ -388,7 +377,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return queryset.filter(members__user=user) def create(self, request, collection_uid=None, collection_item_uid=None): - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) col_it = get_object_or_404(col.items, uid=collection_item_uid) serializer = self.get_serializer_class()(data=request.data) @@ -408,7 +397,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): import os from django.views.static import serve - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) col_it = get_object_or_404(col.items, uid=collection_item_uid) chunk = get_object_or_404(col_it.chunks, uid=uid) @@ -436,7 +425,7 @@ class CollectionMemberViewSet(BaseViewSet): def get_queryset(self, queryset=None): collection_uid = self.kwargs['collection_uid'] try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) except Collection.DoesNotExist: raise Http404('Collection does not exist') @@ -478,7 +467,7 @@ class CollectionMemberViewSet(BaseViewSet): @action_decorator(detail=False, methods=['POST'], permission_classes=our_base_permission_classes) def leave(self, request, collection_uid=None): collection_uid = self.kwargs['collection_uid'] - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) + col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) member = col.members.get(user=request.user) self.perform_destroy(member) @@ -534,7 +523,7 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): collection_uid = serializer.validated_data.get('collection', {}).get('uid') try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) + collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) except Collection.DoesNotExist: raise Http404('Collection does not exist') From 291ebaa3f77d99d2a1ae8ef19d5950b52810117b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 23 Jun 2020 13:00:51 +0300 Subject: [PATCH 151/251] Items must have a uid now (not null). This is due to the previous change. --- .../migrations/0017_auto_20200623_0958.py | 25 +++++++++++++++++++ django_etebase/models.py | 2 +- django_etebase/views.py | 1 - 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 django_etebase/migrations/0017_auto_20200623_0958.py diff --git a/django_etebase/migrations/0017_auto_20200623_0958.py b/django_etebase/migrations/0017_auto_20200623_0958.py new file mode 100644 index 0000000..e244b13 --- /dev/null +++ b/django_etebase/migrations/0017_auto_20200623_0958.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-06-23 09:58 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0016_auto_20200623_0820'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='main_item', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='parent', to='django_etebase.CollectionItem'), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9]*$')]), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 2702389..87b0ed1 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -58,7 +58,7 @@ class Collection(models.Model): class CollectionItem(models.Model): - uid = models.CharField(db_index=True, blank=False, null=True, + uid = models.CharField(db_index=True, blank=False, max_length=43, validators=[UidValidator]) collection = models.ForeignKey(Collection, related_name='items', on_delete=models.CASCADE) version = models.PositiveSmallIntegerField() diff --git a/django_etebase/views.py b/django_etebase/views.py index 4f5d757..2e4bf3c 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -228,7 +228,6 @@ class CollectionItemViewSet(BaseViewSet): raise Http404("Collection does not exist") # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = type(self).queryset.filter(collection__pk=collection.pk, - uid__isnull=False, revisions__current=True) return queryset From 317c492688dab64bd2537ac231d3c9621af5aa1f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 23 Jun 2020 13:02:45 +0300 Subject: [PATCH 152/251] CollectionItem: add support for filtering collections' main items. This used to be the default, so it still is. It only affects the list endpoint, the rest all support withCollection anyway, because IDs are passed directly. --- django_etebase/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django_etebase/views.py b/django_etebase/views.py index 2e4bf3c..577d923 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -254,6 +254,10 @@ class CollectionItemViewSet(BaseViewSet): def list(self, request, collection_uid=None): queryset = self.get_queryset() + + if not self.request.query_params.get('withCollection', False): + queryset = queryset.filter(parent__isnull=True) + queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) serializer = self.get_serializer(queryset, many=True) From 786948c4568e615f5c6465fb9bac264fc68fe711 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 23 Jun 2020 18:04:49 +0300 Subject: [PATCH 153/251] Item revisions: never return the current revision, only old ones. --- django_etebase/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 577d923..f44ff5c 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -275,7 +275,8 @@ class CollectionItemViewSet(BaseViewSet): col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) col_it = get_object_or_404(col.items, uid=uid) - serializer = CollectionItemRevisionSerializer(col_it.revisions.order_by('-id'), many=True) + revisions = col_it.revisions.exclude(current=True).order_by('-id') + serializer = CollectionItemRevisionSerializer(revisions, many=True) ret = { 'data': serializer.data, 'done': True, # we always return all the items, so it's always done From 7183b975419096401910d53ba622888e0879bdf2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 23 Jun 2020 18:25:23 +0300 Subject: [PATCH 154/251] Collection revision: implement iteration. --- django_etebase/views.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index f44ff5c..0e153a4 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -271,15 +271,30 @@ class CollectionItemViewSet(BaseViewSet): @action_decorator(detail=True, methods=['GET']) def revision(self, request, collection_uid=None, uid=None): - # FIXME: need pagination support col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) - col_it = get_object_or_404(col.items, uid=uid) + item = get_object_or_404(col.items, uid=uid) + + limit = int(request.GET.get('limit', 50)) + iterator = request.GET.get('iterator', None) + + queryset = item.revisions.exclude(current=True).order_by('-id') + + if iterator is not None: + iterator = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__lt=iterator.id) + + queryset = queryset[:limit] + serializer = CollectionItemRevisionSerializer(queryset, many=True) + + # This is not the most efficient way of implementing this, but it's good enough + done = len(queryset) < limit + + last_item = len(queryset) > 0 and serializer.data[-1] - revisions = col_it.revisions.exclude(current=True).order_by('-id') - serializer = CollectionItemRevisionSerializer(revisions, many=True) ret = { 'data': serializer.data, - 'done': True, # we always return all the items, so it's always done + 'iterator': last_item and last_item['uid'], + 'done': done, } return Response(ret) From 68365f5d75c7c2cae14dad93c603430f4b2703eb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 23 Jun 2020 18:35:09 +0300 Subject: [PATCH 155/251] Collection revision: support the inline parameter. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 0e153a4..f692ce0 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -284,7 +284,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = queryset.filter(id__lt=iterator.id) queryset = queryset[:limit] - serializer = CollectionItemRevisionSerializer(queryset, many=True) + serializer = CollectionItemRevisionSerializer(queryset, context=self.get_serializer_context(), many=True) # This is not the most efficient way of implementing this, but it's good enough done = len(queryset) < limit From 2da49bb95e8b7b29fdfec04807a895d22b6f99ff Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 10:02:55 +0300 Subject: [PATCH 156/251] Item revisions: don't exclude current, let the client decide. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index f692ce0..bbd4e3e 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -277,7 +277,7 @@ class CollectionItemViewSet(BaseViewSet): limit = int(request.GET.get('limit', 50)) iterator = request.GET.get('iterator', None) - queryset = item.revisions.exclude(current=True).order_by('-id') + queryset = item.revisions.order_by('-id') if iterator is not None: iterator = get_object_or_404(queryset, uid=iterator) From 1bed39af9d3d4205d15d42947cbedc6a52cb1fc2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 10:48:47 +0300 Subject: [PATCH 157/251] Collection/item uid: allow base64-url not just base62. --- .../migrations/0018_auto_20200624_0748.py | 19 +++++++++++++++++++ django_etebase/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django_etebase/migrations/0018_auto_20200624_0748.py diff --git a/django_etebase/migrations/0018_auto_20200624_0748.py b/django_etebase/migrations/0018_auto_20200624_0748.py new file mode 100644 index 0000000..ec59e0c --- /dev/null +++ b/django_etebase/migrations/0018_auto_20200624_0748.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-06-24 07:48 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0017_auto_20200623_0958'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]*$')]), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 87b0ed1..5f7a6f2 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -23,7 +23,7 @@ from django.utils.crypto import get_random_string Base64Url256BitlValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{42,43}$', message='Expected a base64url.') -UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9]*$', message='Not a valid UID') +UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]*$', message='Not a valid UID') class Collection(models.Model): From f6ef514661a2aef45a92f368a1ad7296fe65814a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 10:58:27 +0300 Subject: [PATCH 158/251] Collection members: order by id so order is consistent. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index bbd4e3e..5f1cbff 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -463,7 +463,7 @@ class CollectionMemberViewSet(BaseViewSet): return None def list(self, request, collection_uid=None): - queryset = self.get_queryset() + queryset = self.get_queryset().order_by('id') queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) serializer = self.get_serializer(queryset, many=True) From 0a19cd7e2c84004a2f575f587737f0d90312e915 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 11:30:37 +0300 Subject: [PATCH 159/251] Stoken filtering: abstract getting the stoken id. --- django_etebase/views.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 5f1cbff..5aa4b26 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -88,8 +88,11 @@ class BaseViewSet(viewsets.ModelViewSet): user = self.request.user return queryset.filter(members__user=user) + def get_stoken_obj_id(self, request): + return request.GET.get('stoken', None) + def get_stoken_obj(self, request): - stoken = request.GET.get('stoken', None) + stoken = self.get_stoken_obj_id(request) if stoken is not None: return get_object_or_404(Stoken.objects.all(), uid=stoken) @@ -454,13 +457,8 @@ class CollectionMemberViewSet(BaseViewSet): return queryset.filter(collection=collection) # We override this method because we expect the stoken to be called iterator - def get_stoken_obj(self, request): - stoken = request.GET.get('iterator', None) - - if stoken is not None: - return get_object_or_404(Stoken.objects.all(), uid=stoken) - - return None + def get_stoken_obj_id(self, request): + return request.GET.get('iterator', None) def list(self, request, collection_uid=None): queryset = self.get_queryset().order_by('id') From caa84c2a96a1e2d456b65893292744e4feab24eb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 13:20:07 +0300 Subject: [PATCH 160/251] Stoken filtering: clean up stoken filtering and annotation. We are now querying the database less and simplified the queries. --- django_etebase/views.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 5aa4b26..4c528f1 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -19,7 +19,8 @@ from django.conf import settings from django.contrib.auth import get_user_model, user_logged_in, user_logged_out from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError -from django.db.models import Max, Q +from django.db.models import Max, Q, F, Value as V +from django.db.models.functions import Coalesce, Greatest from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -101,22 +102,20 @@ class BaseViewSet(viewsets.ModelViewSet): def filter_by_stoken(self, request, queryset): stoken_rev = self.get_stoken_obj(request) + + aggr_fields = [Coalesce(Max(field), V(0)) for field in self.stoken_id_fields] + max_stoken = Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] + queryset = queryset.annotate(max_stoken=max_stoken) + if stoken_rev is not None: - filter_by_map = map(lambda x: Q(**{x + '__gt': stoken_rev.id}), self.stoken_id_fields) - filter_by = reduce(lambda x, y: x | y, filter_by_map) - queryset = queryset.filter(filter_by).distinct() + queryset = queryset.filter(max_stoken__gt=stoken_rev.id) return queryset, stoken_rev def get_queryset_stoken(self, queryset): - aggr_field_names = ['max_{}'.format(i) for i, x in enumerate(self.stoken_id_fields)] - aggr_fields = {name: Max(field) for name, field in zip(aggr_field_names, self.stoken_id_fields)} - aggr = queryset.annotate(**aggr_fields).values(*aggr_field_names) - # FIXME: we are doing it in python instead of SQL because I just couldn't get aggregate to work over the - # annotated values. This should probably be fixed as it could be quite slow maxid = -1 - for row in aggr: - rowmaxid = max(map(lambda x: x or -1, row.values())) + for row in queryset: + rowmaxid = getattr(row, 'max_stoken') or -1 maxid = max(maxid, rowmaxid) new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid).uid From 61383b98965b788aa9a1fde3c18e1f2c1f02c537 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 13:35:23 +0300 Subject: [PATCH 161/251] Stoken filtering: order by max_stoken to make sure we have a reliable order. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 4c528f1..f1e3802 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -105,7 +105,7 @@ class BaseViewSet(viewsets.ModelViewSet): aggr_fields = [Coalesce(Max(field), V(0)) for field in self.stoken_id_fields] max_stoken = Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] - queryset = queryset.annotate(max_stoken=max_stoken) + queryset = queryset.annotate(max_stoken=max_stoken).order_by('max_stoken') if stoken_rev is not None: queryset = queryset.filter(max_stoken__gt=stoken_rev.id) From 0ce2e8d99685ba500dd34fb55d18ea293052e618 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 14:34:03 +0300 Subject: [PATCH 162/251] Filter by stoken: cleanup and fix the done implementation The done implementation wasn't great because it would indicate we are not done even when we are when the last chunk returned is exactly the size of limit. --- django_etebase/views.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index f1e3802..68353b4 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -119,7 +119,7 @@ class BaseViewSet(viewsets.ModelViewSet): maxid = max(maxid, rowmaxid) new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid).uid - return queryset, new_stoken + return new_stoken def filter_by_stoken_and_limit(self, request, queryset): limit = int(request.GET.get('limit', 50)) @@ -127,13 +127,16 @@ class BaseViewSet(viewsets.ModelViewSet): queryset, stoken_rev = self.filter_by_stoken(request, queryset) stoken = stoken_rev.uid if stoken_rev is not None else None - queryset = queryset[:limit] - queryset, new_stoken = self.get_queryset_stoken(queryset) - new_stoken = new_stoken or stoken - # This is not the most efficient way of implementing this, but it's good enough - done = len(queryset) < limit + result = list(queryset[:limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + new_stoken = self.get_queryset_stoken(result) or stoken - return queryset, new_stoken, done + return result, new_stoken, done # Change how our list works by default def list(self, request, collection_uid=None): @@ -193,9 +196,9 @@ class CollectionViewSet(BaseViewSet): def list(self, request): queryset = self.get_queryset() - queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) + result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) - serializer = self.get_serializer(queryset, many=True) + serializer = self.get_serializer(result, many=True) ret = { 'data': serializer.data, @@ -260,9 +263,9 @@ class CollectionItemViewSet(BaseViewSet): if not self.request.query_params.get('withCollection', False): queryset = queryset.filter(parent__isnull=True) - queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) + result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) - serializer = self.get_serializer(queryset, many=True) + serializer = self.get_serializer(result, many=True) ret = { 'data': serializer.data, @@ -321,7 +324,7 @@ class CollectionItemViewSet(BaseViewSet): revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) - queryset, new_stoken = self.get_queryset_stoken(queryset) + new_stoken = self.get_queryset_stoken(queryset) stoken = stoken_rev and stoken_rev.uid new_stoken = new_stoken or stoken @@ -461,8 +464,8 @@ class CollectionMemberViewSet(BaseViewSet): def list(self, request, collection_uid=None): queryset = self.get_queryset().order_by('id') - queryset, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) - serializer = self.get_serializer(queryset, many=True) + result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) + serializer = self.get_serializer(result, many=True) ret = { 'data': serializer.data, From c21c6af1d760be18b81bbe030b084462f13ce8df Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 14:38:29 +0300 Subject: [PATCH 163/251] Filter by stoken: fix the done implementation for more functions The done implementation wasn't great because it would indicate we are not done even when we are when the last chunk returned is exactly the size of limit. --- django_etebase/views.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 68353b4..0606e5b 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -288,13 +288,16 @@ class CollectionItemViewSet(BaseViewSet): iterator = get_object_or_404(queryset, uid=iterator) queryset = queryset.filter(id__lt=iterator.id) - queryset = queryset[:limit] - serializer = CollectionItemRevisionSerializer(queryset, context=self.get_serializer_context(), many=True) + result = list(queryset[:limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] - # This is not the most efficient way of implementing this, but it's good enough - done = len(queryset) < limit + serializer = CollectionItemRevisionSerializer(result, context=self.get_serializer_context(), many=True) - last_item = len(queryset) > 0 and serializer.data[-1] + last_item = len(result) > 0 and serializer.data[-1] ret = { 'data': serializer.data, @@ -510,13 +513,16 @@ class InvitationBaseViewSet(BaseViewSet): iterator = get_object_or_404(queryset, uid=iterator) queryset = queryset.filter(id__gt=iterator.id) - queryset = queryset[:limit] - serializer = self.get_serializer(queryset, many=True) + result = list(queryset[:limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] - # This is not the most efficient way of implementing this, but it's good enough - done = len(queryset) < limit + serializer = self.get_serializer(result, many=True) - last_item = len(queryset) > 0 and serializer.data[-1] + last_item = len(result) > 0 and serializer.data[-1] ret = { 'data': serializer.data, From cbb1d81850a514250650b1fcc4f0985c3b1adf71 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 24 Jun 2020 15:55:36 +0300 Subject: [PATCH 164/251] Rename inline to prefetch and have it on by default. --- django_etebase/serializers.py | 3 +-- django_etebase/views.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index c194fdd..9ca33b1 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -83,8 +83,7 @@ class CollectionContentField(BinaryBase64Field): class ChunksField(serializers.RelatedField): def to_representation(self, obj): obj = obj.chunk - inline = self.context.get('inline', False) - if inline: + if self.context.get('prefetch'): with open(obj.chunkFile.path, 'rb') as f: return (obj.uid, b64encode(f.read())) else: diff --git a/django_etebase/views.py b/django_etebase/views.py index 0606e5b..64acb18 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -167,8 +167,8 @@ class CollectionViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - inline = 'inline' in self.request.query_params - context.update({'request': self.request, 'inline': inline}) + prefetch = self.request.query_params.get('prefetch', True) + context.update({'request': self.request, 'prefetch': prefetch}) return context def destroy(self, request, uid=None): @@ -239,8 +239,8 @@ class CollectionItemViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - inline = 'inline' in self.request.query_params - context.update({'request': self.request, 'inline': inline}) + prefetch = self.request.query_params.get('prefetch', True) + context.update({'request': self.request, 'prefetch': prefetch}) return context def create(self, request, collection_uid=None): From 625df229895d6542485c93828cc87a299490541e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 26 Jun 2020 10:31:03 +0300 Subject: [PATCH 165/251] Make item encryption key optional for collections/items Collections still have a unique encryption key (their collection key), and items just have a unique key per item in a collection that's derived from the main key and if we ever want to share items across collections or do something fancy like that we can just add an encrypted key in there. --- django_etebase/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 9ca33b1..a76f388 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -116,7 +116,7 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): class CollectionItemSerializer(serializers.ModelSerializer): - encryptionKey = BinaryBase64Field() + encryptionKey = BinaryBase64Field(required=False, default=None) etag = serializers.CharField(allow_null=True, write_only=True) content = CollectionItemRevisionSerializer(many=False) @@ -186,7 +186,7 @@ class CollectionSerializer(serializers.ModelSerializer): stoken = serializers.CharField(read_only=True) uid = serializers.CharField(source='main_item.uid') - encryptionKey = BinaryBase64Field(source='main_item.encryptionKey') + encryptionKey = BinaryBase64Field(source='main_item.encryptionKey', required=False, default=None) etag = serializers.CharField(allow_null=True, write_only=True) version = serializers.IntegerField(min_value=0, source='main_item.version') content = CollectionItemRevisionSerializer(many=False, source='main_item.content') From 2b52eec41f5278428aa78735042258b6503bd408 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 26 Jun 2020 11:05:01 +0300 Subject: [PATCH 166/251] Allow chunk UIDs to be longer. --- .../migrations/0019_auto_20200626_0748.py | 19 +++++++++++++++++++ django_etebase/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django_etebase/migrations/0019_auto_20200626_0748.py diff --git a/django_etebase/migrations/0019_auto_20200626_0748.py b/django_etebase/migrations/0019_auto_20200626_0748.py new file mode 100644 index 0000000..991ca50 --- /dev/null +++ b/django_etebase/migrations/0019_auto_20200626_0748.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-06-26 07:48 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0018_auto_20200624_0748'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=60, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]*$')]), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 5f7a6f2..964012a 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -89,7 +89,7 @@ def chunk_directory_path(instance, filename): class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, - max_length=43, validators=[Base64Url256BitlValidator]) + max_length=60, validators=[UidValidator]) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) From c00cf501632692ada572f4a2e991717f81d78a74 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 26 Jun 2020 11:21:53 +0300 Subject: [PATCH 167/251] Revision: remove salt field. It's not really needed. More information in the respective change in the js client. --- .../0020_remove_collectionitemrevision_salt.py | 17 +++++++++++++++++ django_etebase/models.py | 1 - django_etebase/serializers.py | 3 +-- 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 django_etebase/migrations/0020_remove_collectionitemrevision_salt.py diff --git a/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py b/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py new file mode 100644 index 0000000..2df32bf --- /dev/null +++ b/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-06-26 08:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0019_auto_20200626_0748'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitemrevision', + name='salt', + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 964012a..c8ceaba 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -110,7 +110,6 @@ class CollectionItemRevision(models.Model): stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT) uid = models.CharField(db_index=True, unique=True, blank=False, null=False, max_length=43, validators=[Base64Url256BitlValidator]) - salt = models.BinaryField(editable=True, blank=False, null=False, default=b'') item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) meta = models.BinaryField(editable=True, blank=False, null=False) current = models.BooleanField(db_index=True, default=True, null=True) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index a76f388..2fe0a21 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -107,12 +107,11 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): queryset=models.RevisionChunkRelation.objects.all(), many=True ) - salt = BinaryBase64Field() meta = BinaryBase64Field() class Meta: model = models.CollectionItemRevision - fields = ('chunks', 'meta', 'uid', 'salt', 'deleted') + fields = ('chunks', 'meta', 'uid', 'deleted') class CollectionItemSerializer(serializers.ModelSerializer): From 785e4fae979b5462f73e7abe23644cf5103639e0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 26 Jun 2020 12:13:50 +0300 Subject: [PATCH 168/251] Merge the uidvalidator with the base64url validator and set a min length. --- .../migrations/0021_auto_20200626_0913.py | 40 +++++++++++++++++++ django_etebase/models.py | 9 ++--- 2 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 django_etebase/migrations/0021_auto_20200626_0913.py diff --git a/django_etebase/migrations/0021_auto_20200626_0913.py b/django_etebase/migrations/0021_auto_20200626_0913.py new file mode 100644 index 0000000..b890384 --- /dev/null +++ b/django_etebase/migrations/0021_auto_20200626_0913.py @@ -0,0 +1,40 @@ +# Generated by Django 3.0.3 on 2020-06-26 09:13 + +import django.core.validators +from django.db import migrations, models +import django_etebase.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0020_remove_collectionitemrevision_salt'), + ] + + operations = [ + migrations.AlterField( + model_name='collectioninvitation', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=60, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='collectionitemrevision', + name='uid', + field=models.CharField(db_index=True, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='stoken', + name='uid', + field=models.CharField(db_index=True, default=django_etebase.models.generate_stoken_uid, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index c8ceaba..b4b04fc 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -22,8 +22,7 @@ from django.utils.functional import cached_property from django.utils.crypto import get_random_string -Base64Url256BitlValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{42,43}$', message='Expected a base64url.') -UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]*$', message='Not a valid UID') +UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{20,}$', message='Not a valid UID') class Collection(models.Model): @@ -103,13 +102,13 @@ def generate_stoken_uid(): class Stoken(models.Model): uid = models.CharField(db_index=True, unique=True, blank=False, null=False, default=generate_stoken_uid, - max_length=43, validators=[Base64Url256BitlValidator]) + max_length=43, validators=[UidValidator]) class CollectionItemRevision(models.Model): stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT) uid = models.CharField(db_index=True, unique=True, blank=False, null=False, - max_length=43, validators=[Base64Url256BitlValidator]) + max_length=43, validators=[UidValidator]) item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) meta = models.BinaryField(editable=True, blank=False, null=False) current = models.BooleanField(db_index=True, default=True, null=True) @@ -179,7 +178,7 @@ class CollectionMemberRemoved(models.Model): class CollectionInvitation(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, - max_length=43, validators=[Base64Url256BitlValidator]) + max_length=43, validators=[UidValidator]) version = models.PositiveSmallIntegerField(default=1) fromMember = models.ForeignKey(CollectionMember, on_delete=models.CASCADE) # FIXME: make sure to delete all invitations for the same collection once one is accepted From 4948e91c65db0842d9c4d50929549df9353a2b65 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 28 Jun 2020 16:52:14 +0300 Subject: [PATCH 169/251] django_etebase: make migration generic and not depend on myauth. --- django_etebase/migrations/0002_userinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/migrations/0002_userinfo.py b/django_etebase/migrations/0002_userinfo.py index 6da0bb8..bfeb2cf 100644 --- a/django_etebase/migrations/0002_userinfo.py +++ b/django_etebase/migrations/0002_userinfo.py @@ -8,7 +8,7 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('myauth', '0001_initial'), + # XXX removed this to make this migration generic ('myauth', '0001_initial'), ('django_etebase', '0001_initial'), ] From 85de674ee28695474237b214de570682f0993c44 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 28 Jun 2020 17:11:20 +0300 Subject: [PATCH 170/251] Move the etebase urls configuration to django_etebase. --- django_etebase/urls.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 django_etebase/urls.py diff --git a/django_etebase/urls.py b/django_etebase/urls.py new file mode 100644 index 0000000..f6d982e --- /dev/null +++ b/django_etebase/urls.py @@ -0,0 +1,30 @@ +from django.conf import settings +from django.conf.urls import include +from django.urls import path + +from rest_framework_nested import routers + +from django_etebase import views + +router = routers.DefaultRouter() +router.register(r'collection', views.CollectionViewSet) +router.register(r'authentication', views.AuthenticationViewSet, basename='authentication') +router.register(r'invitation/incoming', views.InvitationIncomingViewSet, basename='invitation_incoming') +router.register(r'invitation/outgoing', views.InvitationOutgoingViewSet, basename='invitation_outgoing') + +collections_router = routers.NestedSimpleRouter(router, r'collection', lookup='collection') +collections_router.register(r'item', views.CollectionItemViewSet, basename='collection_item') +collections_router.register(r'member', views.CollectionMemberViewSet, basename='collection_member') + +item_router = routers.NestedSimpleRouter(collections_router, r'item', lookup='collection_item') +item_router.register(r'chunk', views.CollectionItemChunkViewSet, basename='collection_items_chunk') + +if settings.DEBUG: + router.register(r'test/authentication', views.TestAuthenticationViewSet, basename='test_authentication') + +app_name = 'django_etebase' +urlpatterns = [ + path('v1/', include(router.urls)), + path('v1/', include(collections_router.urls)), + path('v1/', include(item_router.urls)), +] From 453275eadf1e97b56d54bf3499693d3992af18f8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 29 Jun 2020 11:30:59 +0300 Subject: [PATCH 171/251] Authentication: move to msgpack for the encrypted parts. --- django_etebase/serializers.py | 11 ++++++++++- django_etebase/views.py | 26 +++++++++++++++++--------- requirements.in/base.txt | 1 + requirements.txt | 1 + 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 2fe0a21..e7c0d50 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -64,6 +64,15 @@ class BinaryBase64Field(serializers.Field): return b64decode(data) +# This field does nothing to the data. It's useful for raw binary data +class RawField(serializers.Field): + def to_representation(self, value): + return value + + def to_internal_value(self, data): + return data + + class CollectionEncryptionKeyField(BinaryBase64Field): def get_attribute(self, instance): request = self.context.get('request', None) @@ -413,7 +422,7 @@ class AuthenticationLoginSerializer(serializers.Serializer): class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): - challenge = BinaryBase64Field() + challenge = RawField() host = serializers.CharField() action = serializers.CharField() diff --git a/django_etebase/views.py b/django_etebase/views.py index 64acb18..c7fdab5 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -12,7 +12,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import json +import msgpack from functools import reduce from django.conf import settings @@ -72,6 +72,14 @@ from .serializers import ( User = get_user_model() +def msgpack_encode(content): + return msgpack.packb(content, use_bin_type=True) + + +def msgpack_decode(content): + return msgpack.unpackb(content, raw=False) + + class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) @@ -638,7 +646,7 @@ class AuthenticationViewSet(viewsets.ViewSet): enc_key = self.get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) - challenge_data = json.loads(box.decrypt(challenge).decode()) + challenge_data = msgpack_decode(box.decrypt(challenge)) now = int(datetime.now().timestamp()) if action != expected_action: content = {'code': 'wrong_action', 'detail': 'Expected "{}" but got something else'.format(expected_action)} @@ -680,8 +688,7 @@ class AuthenticationViewSet(viewsets.ViewSet): "timestamp": int(datetime.now().timestamp()), "userId": user.id, } - challenge = box.encrypt(json.dumps( - challenge_data, separators=(',', ':')).encode(), encoder=nacl.encoding.RawEncoder) + challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) ret = { "salt": b64encode(salt), @@ -698,10 +705,11 @@ class AuthenticationViewSet(viewsets.ViewSet): outer_serializer.is_valid(raise_exception=True) response_raw = outer_serializer.validated_data['response'] - response = json.loads(response_raw.decode()) + response = msgpack_decode(response_raw) signature = outer_serializer.validated_data['signature'] - serializer = AuthenticationLoginInnerSerializer(data=response, context={'host': request.get_host()}) + context = {'host': request.get_host(), 'supports_binary': True} + serializer = AuthenticationLoginInnerSerializer(data=response, context=context) serializer.is_valid(raise_exception=True) bad_login_response = self.validate_login_request( @@ -730,11 +738,11 @@ class AuthenticationViewSet(viewsets.ViewSet): outer_serializer.is_valid(raise_exception=True) response_raw = outer_serializer.validated_data['response'] - response = json.loads(response_raw.decode()) + response = msgpack_decode(response_raw) signature = outer_serializer.validated_data['signature'] - serializer = AuthenticationChangePasswordInnerSerializer( - request.user.userinfo, data=response, context={'host': request.get_host()}) + context = {'host': request.get_host(), 'supports_binary': True} + serializer = AuthenticationChangePasswordInnerSerializer(request.user.userinfo, data=response, context=context) serializer.is_valid(raise_exception=True) bad_login_response = self.validate_login_request( diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 1ab0e9a..d27e110 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -4,5 +4,6 @@ django-cors-headers django-fullurl djangorestframework drf-nested-routers +msgpack psycopg2-binary pynacl diff --git a/requirements.txt b/requirements.txt index 51b4d35..cd61ff1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ django==3.0.3 # via -r requirements.in/base.txt, django-anymail, dja djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers drf-nested-routers==0.91 # via -r requirements.in/base.txt idna==2.8 # via requests +msgpack==1.0.0 # via -r requirements.in/base.txt psycopg2-binary==2.8.4 # via -r requirements.in/base.txt pycparser==2.20 # via cffi pynacl==1.3.0 # via -r requirements.in/base.txt From fbf5552a62b770bdf5b2246da128b065ae75c9df Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 29 Jun 2020 13:20:23 +0300 Subject: [PATCH 172/251] Modify binary64 field to support binary renderers/parsers Fixes 39c1dfc53c30e65bcbff9e0ba0bb07bfc8bfc577 --- django_etebase/serializers.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index e7c0d50..1876572 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -58,19 +58,16 @@ def b64decode(data): class BinaryBase64Field(serializers.Field): def to_representation(self, value): - return b64encode(value) - - def to_internal_value(self, data): - return b64decode(data) - - -# This field does nothing to the data. It's useful for raw binary data -class RawField(serializers.Field): - def to_representation(self, value): - return value + if self.context.get('supports_binary', False): + return value + else: + return b64encode(value) def to_internal_value(self, data): - return data + if isinstance(data, bytes): + return data + else: + return b64decode(data) class CollectionEncryptionKeyField(BinaryBase64Field): @@ -422,7 +419,7 @@ class AuthenticationLoginSerializer(serializers.Serializer): class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): - challenge = RawField() + challenge = BinaryBase64Field() host = serializers.CharField() action = serializers.CharField() From 2880673e27e41126416d8c642fb6851880eaa8e2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 29 Jun 2020 13:01:40 +0300 Subject: [PATCH 173/251] drf_msgpack: add code to parse/serialise msgpack It's not actually used by clients but it's there and can be used. It works for receiving msgpack messages, but doesn't yet work for sending because some of the types will be converted to base64. --- django_etebase/drf_msgpack/__init__.py | 0 django_etebase/drf_msgpack/apps.py | 5 +++++ django_etebase/drf_msgpack/migrations/__init__.py | 0 django_etebase/drf_msgpack/parsers.py | 14 ++++++++++++++ django_etebase/drf_msgpack/renderers.py | 15 +++++++++++++++ django_etebase/drf_msgpack/views.py | 3 +++ django_etebase/views.py | 15 +++++++++++++-- 7 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 django_etebase/drf_msgpack/__init__.py create mode 100644 django_etebase/drf_msgpack/apps.py create mode 100644 django_etebase/drf_msgpack/migrations/__init__.py create mode 100644 django_etebase/drf_msgpack/parsers.py create mode 100644 django_etebase/drf_msgpack/renderers.py create mode 100644 django_etebase/drf_msgpack/views.py diff --git a/django_etebase/drf_msgpack/__init__.py b/django_etebase/drf_msgpack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/drf_msgpack/apps.py b/django_etebase/drf_msgpack/apps.py new file mode 100644 index 0000000..619e3e0 --- /dev/null +++ b/django_etebase/drf_msgpack/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class DrfMsgpackConfig(AppConfig): + name = 'drf_msgpack' diff --git a/django_etebase/drf_msgpack/migrations/__init__.py b/django_etebase/drf_msgpack/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/drf_msgpack/parsers.py b/django_etebase/drf_msgpack/parsers.py new file mode 100644 index 0000000..44cd33b --- /dev/null +++ b/django_etebase/drf_msgpack/parsers.py @@ -0,0 +1,14 @@ +import msgpack + +from rest_framework.parsers import BaseParser +from rest_framework.exceptions import ParseError + + +class MessagePackParser(BaseParser): + media_type = 'application/msgpack' + + def parse(self, stream, media_type=None, parser_context=None): + try: + return msgpack.unpackb(stream.read(), raw=False) + except Exception as exc: + raise ParseError('MessagePack parse error - %s' % str(exc)) diff --git a/django_etebase/drf_msgpack/renderers.py b/django_etebase/drf_msgpack/renderers.py new file mode 100644 index 0000000..9445231 --- /dev/null +++ b/django_etebase/drf_msgpack/renderers.py @@ -0,0 +1,15 @@ +import msgpack + +from rest_framework.renderers import BaseRenderer + + +class MessagePackRenderer(BaseRenderer): + media_type = 'application/msgpack' + format = 'msgpack' + render_style = 'binary' + charset = None + + def render(self, data, media_type=None, renderer_context=None): + if data is None: + return b'' + return msgpack.packb(data, use_bin_type=True) diff --git a/django_etebase/drf_msgpack/views.py b/django_etebase/drf_msgpack/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/django_etebase/drf_msgpack/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/django_etebase/views.py b/django_etebase/views.py index c7fdab5..93a7645 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -26,9 +26,10 @@ from django.shortcuts import get_object_or_404 from rest_framework import status from rest_framework import viewsets -from rest_framework import parsers from rest_framework.decorators import action as action_decorator from rest_framework.response import Response +from rest_framework.parsers import JSONParser, FormParser, MultiPartParser +from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer import nacl.encoding import nacl.signing @@ -37,6 +38,9 @@ import nacl.hash from .token_auth.models import AuthToken +from .drf_msgpack.parsers import MessagePackParser +from .drf_msgpack.renderers import MessagePackRenderer + from . import app_settings, permissions from .models import ( Collection, @@ -83,6 +87,8 @@ def msgpack_decode(content): class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) + renderer_classes = [JSONRenderer, MessagePackRenderer, BrowsableAPIRenderer] + parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] stoken_id_fields = None def get_serializer_class(self): @@ -398,9 +404,10 @@ class CollectionItemViewSet(BaseViewSet): class CollectionItemChunkViewSet(viewsets.ViewSet): allowed_methods = ['GET', 'POST'] - parser_classes = (parsers.MultiPartParser, ) authentication_classes = BaseViewSet.authentication_classes permission_classes = BaseViewSet.permission_classes + renderer_classes = BaseViewSet.renderer_classes + parser_classes = (MultiPartParser, ) serializer_class = CollectionItemChunkSerializer lookup_field = 'uid' @@ -602,6 +609,8 @@ class InvitationIncomingViewSet(InvitationBaseViewSet): class AuthenticationViewSet(viewsets.ViewSet): allowed_methods = ['POST'] authentication_classes = BaseViewSet.authentication_classes + renderer_classes = BaseViewSet.renderer_classes + parser_classes = BaseViewSet.parser_classes def get_encryption_key(self, salt): key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) @@ -757,6 +766,8 @@ class AuthenticationViewSet(viewsets.ViewSet): class TestAuthenticationViewSet(viewsets.ViewSet): allowed_methods = ['POST'] + renderer_classes = BaseViewSet.renderer_classes + parser_classes = BaseViewSet.parser_classes def list(self, request): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) From 3dfceb63b139efbf5d7196bf77e5a78d65761fac Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 29 Jun 2020 14:50:06 +0300 Subject: [PATCH 174/251] Views: move the base64 encoding to the renderers. Hard-coding the serialization encoding in the serializers is wrong. This fix now enables us to change to easily change to msgpack as the transport layer. --- django_etebase/renderers.py | 18 ++++++++++++++++++ django_etebase/serializers.py | 21 +++++++++++---------- django_etebase/views.py | 12 ++++++------ 3 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 django_etebase/renderers.py diff --git a/django_etebase/renderers.py b/django_etebase/renderers.py new file mode 100644 index 0000000..43c1a0d --- /dev/null +++ b/django_etebase/renderers.py @@ -0,0 +1,18 @@ +from rest_framework.utils.encoders import JSONEncoder as DRFJSONEncoder +from rest_framework.renderers import JSONRenderer as DRFJSONRenderer + +from .serializers import b64encode + + +class JSONEncoder(DRFJSONEncoder): + def default(self, obj): + if isinstance(obj, bytes) or isinstance(obj, memoryview): + return b64encode(obj) + return super().default(obj) + + +class JSONRenderer(DRFJSONRenderer): + """ + Renderer which serializes to JSON with support for our base64 + """ + encoder_class = JSONEncoder diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 1876572..f78cd6b 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -56,18 +56,19 @@ def b64decode(data): return base64.urlsafe_b64decode(data) +def b64decode_or_bytes(data): + if isinstance(data, bytes): + return data + else: + return b64decode(data) + + class BinaryBase64Field(serializers.Field): def to_representation(self, value): - if self.context.get('supports_binary', False): - return value - else: - return b64encode(value) + return value def to_internal_value(self, data): - if isinstance(data, bytes): - return data - else: - return b64decode(data) + return b64decode_or_bytes(data) class CollectionEncryptionKeyField(BinaryBase64Field): @@ -91,14 +92,14 @@ class ChunksField(serializers.RelatedField): obj = obj.chunk if self.context.get('prefetch'): with open(obj.chunkFile.path, 'rb') as f: - return (obj.uid, b64encode(f.read())) + return (obj.uid, f.read()) else: return (obj.uid, ) def to_internal_value(self, data): if data[0] is None or data[1] is None: raise serializers.ValidationError('null is not allowed') - return (data[0], b64decode(data[1])) + return (data[0], b64decode_or_bytes(data[1])) class CollectionItemChunkSerializer(serializers.ModelSerializer): diff --git a/django_etebase/views.py b/django_etebase/views.py index 93a7645..970ad8c 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -29,7 +29,7 @@ from rest_framework import viewsets from rest_framework.decorators import action as action_decorator from rest_framework.response import Response from rest_framework.parsers import JSONParser, FormParser, MultiPartParser -from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer +from rest_framework.renderers import BrowsableAPIRenderer import nacl.encoding import nacl.signing @@ -42,6 +42,7 @@ from .drf_msgpack.parsers import MessagePackParser from .drf_msgpack.renderers import MessagePackRenderer from . import app_settings, permissions +from .renderers import JSONRenderer from .models import ( Collection, CollectionItem, @@ -53,7 +54,6 @@ from .models import ( UserInfo, ) from .serializers import ( - b64encode, AuthenticationChangePasswordInnerSerializer, AuthenticationSignupSerializer, AuthenticationLoginChallengeSerializer, @@ -700,8 +700,8 @@ class AuthenticationViewSet(viewsets.ViewSet): challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) ret = { - "salt": b64encode(salt), - "challenge": b64encode(challenge), + "salt": salt, + "challenge": challenge, "version": user.userinfo.version, } return Response(ret, status=status.HTTP_200_OK) @@ -717,7 +717,7 @@ class AuthenticationViewSet(viewsets.ViewSet): response = msgpack_decode(response_raw) signature = outer_serializer.validated_data['signature'] - context = {'host': request.get_host(), 'supports_binary': True} + context = {'host': request.get_host()} serializer = AuthenticationLoginInnerSerializer(data=response, context=context) serializer.is_valid(raise_exception=True) @@ -750,7 +750,7 @@ class AuthenticationViewSet(viewsets.ViewSet): response = msgpack_decode(response_raw) signature = outer_serializer.validated_data['signature'] - context = {'host': request.get_host(), 'supports_binary': True} + context = {'host': request.get_host()} serializer = AuthenticationChangePasswordInnerSerializer(request.user.userinfo, data=response, context=context) serializer.is_valid(raise_exception=True) From f147f4ae58561b89cbac32d70e937e7e5d41ad83 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 29 Jun 2020 15:31:29 +0300 Subject: [PATCH 175/251] Serializers: allow encryptionKey to be null. --- django_etebase/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index f78cd6b..f2da771 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -122,7 +122,7 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): class CollectionItemSerializer(serializers.ModelSerializer): - encryptionKey = BinaryBase64Field(required=False, default=None) + encryptionKey = BinaryBase64Field(required=False, default=None, allow_null=True) etag = serializers.CharField(allow_null=True, write_only=True) content = CollectionItemRevisionSerializer(many=False) @@ -192,7 +192,7 @@ class CollectionSerializer(serializers.ModelSerializer): stoken = serializers.CharField(read_only=True) uid = serializers.CharField(source='main_item.uid') - encryptionKey = BinaryBase64Field(source='main_item.encryptionKey', required=False, default=None) + encryptionKey = BinaryBase64Field(source='main_item.encryptionKey', required=False, default=None, allow_null=True) etag = serializers.CharField(allow_null=True, write_only=True) version = serializers.IntegerField(min_value=0, source='main_item.version') content = CollectionItemRevisionSerializer(many=False, source='main_item.content') From f69c3a327cbb80a058d9fd60b02afe81f65bfa12 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 5 Jul 2020 13:15:42 +0300 Subject: [PATCH 176/251] Revert "django_etebase: make migration generic and not depend on myauth." This reverts commit 925dcac0fb99204e3373251e12f8496721879361. --- django_etebase/migrations/0002_userinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/migrations/0002_userinfo.py b/django_etebase/migrations/0002_userinfo.py index bfeb2cf..6da0bb8 100644 --- a/django_etebase/migrations/0002_userinfo.py +++ b/django_etebase/migrations/0002_userinfo.py @@ -8,7 +8,7 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - # XXX removed this to make this migration generic ('myauth', '0001_initial'), + ('myauth', '0001_initial'), ('django_etebase', '0001_initial'), ] From 4aa3daaa97d62fb6b826374a0b17a64383b743b1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 5 Jul 2020 14:57:38 +0300 Subject: [PATCH 177/251] Create a new django project. --- etebase_server/__init__.py | 0 etebase_server/asgi.py | 16 +++++ etebase_server/settings.py | 120 +++++++++++++++++++++++++++++++++++++ etebase_server/urls.py | 21 +++++++ etebase_server/wsgi.py | 16 +++++ manage.py | 21 +++++++ 6 files changed, 194 insertions(+) create mode 100644 etebase_server/__init__.py create mode 100644 etebase_server/asgi.py create mode 100644 etebase_server/settings.py create mode 100644 etebase_server/urls.py create mode 100644 etebase_server/wsgi.py create mode 100755 manage.py diff --git a/etebase_server/__init__.py b/etebase_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/etebase_server/asgi.py b/etebase_server/asgi.py new file mode 100644 index 0000000..44f1c53 --- /dev/null +++ b/etebase_server/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for etebase_server project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etebase_server.settings') + +application = get_asgi_application() diff --git a/etebase_server/settings.py b/etebase_server/settings.py new file mode 100644 index 0000000..4ec2216 --- /dev/null +++ b/etebase_server/settings.py @@ -0,0 +1,120 @@ +""" +Django settings for etebase_server project. + +Generated by 'django-admin startproject' using Django 3.0.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +# SECRET_KEY = '' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'etebase_server.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'etebase_server.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/etebase_server/urls.py b/etebase_server/urls.py new file mode 100644 index 0000000..4ac11e2 --- /dev/null +++ b/etebase_server/urls.py @@ -0,0 +1,21 @@ +"""etebase_server URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/etebase_server/wsgi.py b/etebase_server/wsgi.py new file mode 100644 index 0000000..cf449a1 --- /dev/null +++ b/etebase_server/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for etebase_server project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etebase_server.settings') + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..b793fd2 --- /dev/null +++ b/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etebase_server.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() From 08c4aa9d43b31c2004433538cbb719f1eb21e5bc Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 5 Jul 2020 15:09:46 +0300 Subject: [PATCH 178/251] Add .gitignore. --- .gitignore | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7f220af --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +/journal +/db.sqlite3* +Session.vim +/.venv +/assets +/logs +/.coverage +/tmp +/media + +__pycache__ +.*.swp + + +/etebase_server_settings.py From cc163d27af354052435047046e5107fd281f008c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 5 Jul 2020 15:04:24 +0300 Subject: [PATCH 179/251] Add settings and configuration to run the etebase app. --- etebase_server/settings.py | 37 ++++++++++++++++++++++++++++++++----- etebase_server/urls.py | 23 +++++++---------------- templates/success.html | 12 ++++++++++++ 3 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 templates/success.html diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 4ec2216..94ca4d2 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -15,15 +15,17 @@ import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +AUTH_USER_MODEL = 'myauth.User' + # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ -# SECURITY WARNING: keep the secret key used in production secret! -# SECRET_KEY = '' +# Should be set in the site specific settings +# SECRET_KEY = # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = [] @@ -37,11 +39,18 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'corsheaders', + 'rest_framework', + 'fullurl', + 'myauth.apps.MyauthConfig', + 'django_etebase.apps.DjangoEtebaseConfig', + 'django_etebase.token_auth.apps.TokenAuthConfig', ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', + 'corsheaders.middleware.CorsMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', @@ -54,7 +63,9 @@ ROOT_URLCONF = 'etebase_server.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], + 'DIRS': [ + os.path.join(BASE_DIR, 'templates') + ], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -76,7 +87,8 @@ WSGI_APPLICATION = 'etebase_server.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'NAME': os.environ.get('ETEBASE_DB_PATH', + os.path.join(BASE_DIR, 'db.sqlite3')), } } @@ -113,8 +125,23 @@ USE_L10N = True USE_TZ = True +# Cors +CORS_ORIGIN_ALLOW_ALL = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ +STATIC_ROOT = os.environ.get('DJANGO_STATIC_ROOT', os.path.join(BASE_DIR, 'assets')) STATIC_URL = '/static/' + +MEDIA_ROOT = os.environ.get('DJANGO_MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) +MEDIA_URL = '/user-media/' + +ETEBASE_API_PERMISSIONS = ('rest_framework.permissions.IsAuthenticated', ) +ETEBASE_API_AUTHENTICATORS = ('django_etebase.token_auth.authentication.TokenAuthentication', + 'rest_framework.authentication.SessionAuthentication') + +try: + from etebase_server_settings import * +except ImportError: + pass diff --git a/etebase_server/urls.py b/etebase_server/urls.py index 4ac11e2..0c114af 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,21 +1,12 @@ -"""etebase_server URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/3.0/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" +from django.conf.urls import include, url from django.contrib import admin from django.urls import path +from django.views.generic import TemplateView urlpatterns = [ - path('admin/', admin.site.urls), + url(r'^api/', include('django_etebase.urls')), + url(r'^admin/', admin.site.urls), + url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), + + path('', TemplateView.as_view(template_name='success.html')), ] diff --git a/templates/success.html b/templates/success.html new file mode 100644 index 0000000..c7cf494 --- /dev/null +++ b/templates/success.html @@ -0,0 +1,12 @@ + + + + It works! + + +

It works!

+

+ Please refer to the README to complete the final steps if you haven't done so already. +

+ + From ee23707fffee4347e0fea3c3e97ef3e601309ca0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 5 Jul 2020 15:43:37 +0300 Subject: [PATCH 180/251] Debug reset: put the whole request in a transaction. --- django_etebase/views.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 970ad8c..a8a6100 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -778,23 +778,24 @@ class TestAuthenticationViewSet(viewsets.ViewSet): if not settings.DEBUG: return HttpResponseBadRequest("Only allowed in debug mode.") - user = get_object_or_404(User.objects.all(), username=request.data.get('user').get('username')) + with transaction.atomic(): + user = get_object_or_404(User.objects.all(), username=request.data.get('user').get('username')) - # Only allow test users for extra safety - if not getattr(user, User.USERNAME_FIELD).startswith('test_user'): - return HttpResponseBadRequest("Endpoint not allowed for user.") + # Only allow test users for extra safety + if not getattr(user, User.USERNAME_FIELD).startswith('test_user'): + return HttpResponseBadRequest("Endpoint not allowed for user.") - if hasattr(user, 'userinfo'): - user.userinfo.delete() + if hasattr(user, 'userinfo'): + user.userinfo.delete() - serializer = AuthenticationSignupSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - serializer.save() + serializer = AuthenticationSignupSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save() - # Delete all of the journal data for this user for a clear test env - user.collection_set.all().delete() - user.incoming_invitations.all().delete() + # Delete all of the journal data for this user for a clear test env + user.collection_set.all().delete() + user.incoming_invitations.all().delete() - # FIXME: also delete chunk files!!! + # FIXME: also delete chunk files!!! return HttpResponse() From 2d4410ef36c5376de0b8e81b5e931157b3327e38 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 8 Jul 2020 17:45:44 +0300 Subject: [PATCH 181/251] Add license file. --- LICENSE | 661 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 661 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. From e8e859fa6acccbefb65a73a706e47e6bbb280e23 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 8 Jul 2020 17:45:50 +0300 Subject: [PATCH 182/251] Add README. --- README.md | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..781fd92 --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +

+ +

EteSync - Secure Data Sync

+

+ +A skeleton app for running your own [Etebase](https://www.etebase.com) server + +# Installation + +## From source + +Before installing the EteSync server make sure you install `virtualenv` (for **Python 3**): + +* Arch Linux: `pacman -S python-virtualenv` +* Debian/Ubuntu: `apt-get install python3-virtualenv` +* Mac/Windows/Other Linux: install virtualenv or just skip the instructions mentioning virtualenv. + +Then just clone the git repo and set up this app: + +``` +git clone https://github.com/etesync/server-skeleton.git + +cd server-skeleton + +# Set up the environment and deps +virtualenv -p python3 venv # If doesn't work, try: virtualenv3 venv +source venv/bin/activate + +pip install -r requirements.txt +``` + +# Configuration + +If you are familiar with Django you can just edit the [settings file](etesync_server/settings.py) +according to the [Django deployment checklist](https://docs.djangoproject.com/en/dev/howto/deployment/checklist/) +if you are not, we will soon provide a simple configuration file for easy deployment like we had with EteSync. + +Some particular settings that should be edited are: + * [`ALLOWED_HOSTS`](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-ALLOWED_HOSTS) + -- this is the list of host/domain names or addresses on which the app +will be served + * [`DEBUG`](https://docs.djangoproject.com/en/1.11/ref/settings/#debug) + -- handy for debugging, set to `False` for production + * [`SECRET_KEY`](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-SECRET_KEY) + -- an ephemeral secret used for various cryptographic signing and token +generation purposes. See below for how default configuration of +`SECRET_KEY` works for this project. + +Now you can initialise our django app + +``` +./manage.py migrate +``` + +And you are done! You can now run the debug server just to see everything works as expected by running: + +``` +./manage.py runserver 0.0.0.0:8000 +``` + +Using the debug server in production is not recommended, so please read the following section for a proper deployment. + +# Production deployment + +EteSync is based on Django so you should refer to one of the following + * The instructions of the Django project [here](https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/). + * Instructions from uwsgi [here](http://uwsgi-docs.readthedocs.io/en/latest/tutorials/Django_and_nginx.html). + +There are more details about a proper production setup using uWSGI and Nginx in the [wiki](https://github.com/etesync/server/wiki/Production-setup-using-uWSGI-and-Nginx). + +The webserver should also be configured to serve Etebase using TLS. +A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-EteSync) as well. + +# Usage + +Create yourself an admin user: + +``` +./manage.py createsuperuser +``` + +At this stage you can either just use the admin user, or better yet, go to: ```www.your-etesync-install.com/admin``` +and create a non-privileged user that you can use. + +That's it! + +Now all that's left is to open the EteSync app, add an account, and set your custom server address under the "advance" section. + +# `SECRET_KEY` and `secret.txt` + +The default configuration creates a file “`secret.txt`” in the project’s +base directory, which is used as the value of the Django `SECRET_KEY` +setting. You can revoke this key by deleting the `secret.txt` file and the +next time the app is run, a new one will be generated. Make sure you keep +the `secret.txt` file secret (don’t accidentally commit it to version +control, exclude it from your backups, etc.). If you want to change to a +more secure system for storing secrets, edit `etesync_server/settings.py` +and implement your own method for setting `SECRET_KEY` (remove the line +where it uses the `get_secret_from_file` function). Read the Django docs +for more information about the `SECRET_KEY` and its uses. + +# Updating + +First, run `git pull --rebase` to update this repository. +Then, inside the virtualenv: +1. Run `pip install -U -r requirements.txt` to update the dependencies. +2. Run `python manage.py migrate` to perform database migrations. + +You can now restart the server. + +# Supporting Etebase + +Please consider registering an account even if you self-host in order to support the development of Etebase, or help by spreading the word. From 86c5d711a6fef03a853d4e5c15e30cd38ffd71c4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 10 Jul 2020 09:09:11 +0300 Subject: [PATCH 183/251] Chunk upload: item.uid can never be None so use it directly. --- django_etebase/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index b4b04fc..397600e 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -82,8 +82,7 @@ def chunk_directory_path(instance, filename): item = instance.item col = item.collection user_id = col.owner.id - item_uid = item.uid or 'main' - return Path('user_{}'.format(user_id), col.uid, item_uid, instance.uid) + return Path('user_{}'.format(user_id), col.uid, item.uid, instance.uid) class CollectionItemChunk(models.Model): From fae15fe420e8cba63ecd4cbf04f95ef44e5cec5e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 10 Jul 2020 09:27:34 +0300 Subject: [PATCH 184/251] Views: clean up how we use serializers and remove integrity_errors catch-alls. The integrity errors were a bad relic from the EteSync sources and needed to be removed. --- django_etebase/views.py | 139 +++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 81 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index a8a6100..c8a537f 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -197,16 +197,10 @@ class CollectionViewSet(BaseViewSet): def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) - if serializer.is_valid(): - try: - serializer.save(owner=self.request.user) - except IntegrityError: - content = {'code': 'integrity_error'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - return Response({}, status=status.HTTP_201_CREATED) + serializer.is_valid(raise_exception=True) + serializer.save(owner=self.request.user) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + return Response({}, status=status.HTTP_201_CREATED) def list(self, request): queryset = self.get_queryset() @@ -326,35 +320,33 @@ class CollectionItemViewSet(BaseViewSet): queryset = self.get_queryset() serializer = CollectionItemBulkGetSerializer(data=request.data, many=True) - if serializer.is_valid(): - # FIXME: make configurable? - item_limit = 200 - - if len(serializer.validated_data) > item_limit: - content = {'code': 'too_many_items', - 'detail': 'Request has too many items. Limit: {}'. format(item_limit)} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + serializer.is_valid(raise_exception=True) + # FIXME: make configurable? + item_limit = 200 - queryset, stoken_rev = self.filter_by_stoken(request, queryset) + if len(serializer.validated_data) > item_limit: + content = {'code': 'too_many_items', + 'detail': 'Request has too many items. Limit: {}'. format(item_limit)} + return Response(content, status=status.HTTP_400_BAD_REQUEST) - uids, etags = zip(*[(item['uid'], item.get('etag')) for item in serializer.validated_data]) - revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) - queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) + queryset, stoken_rev = self.filter_by_stoken(request, queryset) - new_stoken = self.get_queryset_stoken(queryset) - stoken = stoken_rev and stoken_rev.uid - new_stoken = new_stoken or stoken + uids, etags = zip(*[(item['uid'], item.get('etag')) for item in serializer.validated_data]) + revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) + queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) - serializer = self.get_serializer(queryset, many=True) + new_stoken = self.get_queryset_stoken(queryset) + stoken = stoken_rev and stoken_rev.uid + new_stoken = new_stoken or stoken - ret = { - 'data': serializer.data, - 'stoken': new_stoken, - 'done': True, # we always return all the items, so it's always done - } - return Response(ret) + serializer = self.get_serializer(queryset, many=True) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + ret = { + 'data': serializer.data, + 'stoken': new_stoken, + 'done': True, # we always return all the items, so it's always done + } + return Response(ret) @action_decorator(detail=False, methods=['POST']) def batch(self, request, collection_uid=None): @@ -383,12 +375,7 @@ class CollectionItemViewSet(BaseViewSet): ser_valid = serializer.is_valid() deps_ser_valid = (deps is None or deps_serializer.is_valid()) if ser_valid and deps_ser_valid: - try: - items = serializer.save(collection=collection_object) - except IntegrityError: - # FIXME: return the items with a bad token (including deps) so we don't have to fetch them after - content = {'code': 'integrity_error'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + items = serializer.save(collection=collection_object) ret = { } @@ -423,16 +410,10 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): col_it = get_object_or_404(col.items, uid=collection_item_uid) serializer = self.get_serializer_class()(data=request.data) - if serializer.is_valid(): - try: - serializer.save(item=col_it) - except IntegrityError: - content = {'code': 'integrity_error'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - return Response({}, status=status.HTTP_201_CREATED) + serializer.is_valid(raise_exception=True) + serializer.save(item=col_it) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + return Response({}, status=status.HTTP_201_CREATED) @action_decorator(detail=True, methods=['GET']) def download(self, request, collection_uid=None, collection_item_uid=None, uid=None): @@ -559,22 +540,20 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) - if serializer.is_valid(): - collection_uid = serializer.validated_data.get('collection', {}).get('uid') - - try: - collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) - except Collection.DoesNotExist: - raise Http404('Collection does not exist') + serializer.is_valid(raise_exception=True) + collection_uid = serializer.validated_data.get('collection', {}).get('uid') - if not permissions.is_collection_admin(collection, request.user): - raise PermissionDenied('User is not an admin of this collection') + try: + collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) + except Collection.DoesNotExist: + raise Http404('Collection does not exist') - serializer.save(collection=collection) + if not permissions.is_collection_admin(collection, request.user): + raise PermissionDenied('User is not an admin of this collection') - return Response({}, status=status.HTTP_201_CREATED) + serializer.save(collection=collection) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + return Response({}, status=status.HTTP_201_CREATED) @action_decorator(detail=False, allowed_methods=['GET'], methods=['GET']) def fetch_user_profile(self, request): @@ -685,28 +664,26 @@ class AuthenticationViewSet(viewsets.ViewSet): from datetime import datetime serializer = AuthenticationLoginChallengeSerializer(data=request.data) - if serializer.is_valid(): - username = serializer.validated_data.get('username') - user = self.get_login_user(username) - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = { - "timestamp": int(datetime.now().timestamp()), - "userId": user.id, - } - challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) - - ret = { - "salt": salt, - "challenge": challenge, - "version": user.userinfo.version, - } - return Response(ret, status=status.HTTP_200_OK) - - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + serializer.is_valid(raise_exception=True) + username = serializer.validated_data.get('username') + user = self.get_login_user(username) + + salt = bytes(user.userinfo.salt) + enc_key = self.get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + + challenge_data = { + "timestamp": int(datetime.now().timestamp()), + "userId": user.id, + } + challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) + + ret = { + "salt": salt, + "challenge": challenge, + "version": user.userinfo.version, + } + return Response(ret, status=status.HTTP_200_OK) @action_decorator(detail=False, methods=['POST']) def login(self, request): From 9a518b3907ffd0ba95ead85724d2674988d46fcb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 10 Jul 2020 09:29:19 +0300 Subject: [PATCH 185/251] Chunks: add error handling for chunks having content or not existing. If the chunk already has a content and we try to upload it again, we assume the previous content was correct and this one is the same (chunks are immutable). We can't actually ensure they are the same due to the encryption, though they should be. If a chunk is being uploaded for the first time and doesn't have a content, throw a validation error rather than throwing an ugly error. --- django_etebase/serializers.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index f2da771..3f1b084 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -29,15 +29,19 @@ def process_revisions_for_item(item, revision_data): chunks = revision_data.pop('chunks_relation') for chunk in chunks: uid = chunk[0] + chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() if len(chunk) > 1: content = chunk[1] - chunk = models.CollectionItemChunk(uid=uid, item=item) - chunk.chunkFile.save('IGNORED', ContentFile(content)) - chunk.save() - chunks_objs.append(chunk) + # If the chunk already exists we assume it's fine. Otherwise, we upload it. + if chunk_obj is None: + chunk_obj = models.CollectionItemChunk(uid=uid, item=item) + chunk_obj.chunkFile.save('IGNORED', ContentFile(content)) + chunk_obj.save() else: - chunk = models.CollectionItemChunk.objects.get(uid=uid) - chunks_objs.append(chunk) + if chunk_obj is None: + raise serializers.ValidationError('Tried to create a new chunk without content') + + chunks_objs.append(chunk_obj) stoken = models.Stoken.objects.create() From 7ec45434ba4c29a6d8225dc659bab8be54797e8d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 12 Jul 2020 11:11:33 +0300 Subject: [PATCH 186/251] User: make username case insensitive (and save original styling). We want 'User' and 'UsEr' to mean the same user. Apparently that's not the default in django. This normalizes the user to ensure we enforce this. --- django_etebase/serializers.py | 7 ++++++- myauth/models.py | 13 ++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 3f1b084..908f126 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -382,7 +382,12 @@ class AuthenticationSignupSerializer(serializers.Serializer): user_data = validated_data.pop('user') with transaction.atomic(): - instance, _ = User.objects.get_or_create(**user_data) + try: + instance = User.objects.get_by_natural_key(user_data['username']) + except User.DoesNotExist: + # Create the user and save the casing the user chose as the first name + instance = User.objects.create_user(**user_data, first_name=user_data['username']) + if hasattr(instance, 'userinfo'): raise serializers.ValidationError('User already exists') diff --git a/myauth/models.py b/myauth/models.py index 4afc27c..4046b2f 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -1,4 +1,4 @@ -from django.contrib.auth.models import AbstractUser +from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager from django.core import validators from django.db import models from django.utils.deconstruct import deconstructible @@ -15,9 +15,16 @@ class UnicodeUsernameValidator(validators.RegexValidator): flags = 0 +class UserManager(DjangoUserManager): + def get_by_natural_key(self, username): + return self.get(**{self.model.USERNAME_FIELD + '__iexact': username}) + + class User(AbstractUser): username_validator = UnicodeUsernameValidator() + objects = UserManager() + username = models.CharField( _('username'), max_length=150, @@ -28,3 +35,7 @@ class User(AbstractUser): 'unique': _("A user with that username already exists."), }, ) + + @classmethod + def normalize_username(cls, username): + return super().normalize_username(username).lower() From 9f1bfceda7b76dcc829b9ddc639494cb65297259 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 12 Jul 2020 11:27:47 +0300 Subject: [PATCH 187/251] Increase token ttl to 30 days. --- django_etebase/token_auth/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/token_auth/models.py b/django_etebase/token_auth/models.py index 9ac0955..0fe4766 100644 --- a/django_etebase/token_auth/models.py +++ b/django_etebase/token_auth/models.py @@ -11,7 +11,7 @@ def generate_key(): def get_default_expiry(): - return timezone.now() + timezone.timedelta(days=14) + return timezone.now() + timezone.timedelta(days=30) class AuthToken(models.Model): From 41a03e9d3bbd2a7345a93b4ba6ded66af5a60d64 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 12 Jul 2020 13:23:45 +0300 Subject: [PATCH 188/251] Invitation: fix the checks making sure you can't invite yourself. --- django_etebase/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 908f126..7fe5050 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -294,7 +294,7 @@ class CollectionInvitationSerializer(serializers.ModelSerializer): def validate_user(self, value): request = self.context['request'] - if request.user == value.lower(): + if request.user.username == value.lower(): raise serializers.ValidationError('Inviting yourself is not allowed') return value From 9ea01d4d938ba90c99adfe4c492f9c8cdf4cd60d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 11:15:42 +0300 Subject: [PATCH 189/251] CollectionMemberSerializer: change the user field to be read only. --- django_etebase/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 7fe5050..13199b3 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -255,7 +255,7 @@ class CollectionMemberSerializer(serializers.ModelSerializer): username = serializers.SlugRelatedField( source='user', slug_field=User.USERNAME_FIELD, - queryset=User.objects + read_only=True, ) class Meta: From 3680bd53b1068d9ccb425d3b953dcdf08640af3f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 14:26:39 +0300 Subject: [PATCH 190/251] Views: change according to DRF best practices. --- django_etebase/views.py | 56 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index c8a537f..7e8bf98 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -153,7 +153,7 @@ class BaseViewSet(viewsets.ModelViewSet): return result, new_stoken, done # Change how our list works by default - def list(self, request, collection_uid=None): + def list(self, request, collection_uid=None, *args, **kwargs): queryset = self.get_queryset() serializer = self.get_serializer(queryset, many=True) @@ -185,11 +185,11 @@ class CollectionViewSet(BaseViewSet): context.update({'request': self.request, 'prefetch': prefetch}) return context - def destroy(self, request, uid=None): + def destroy(self, request, uid=None, *args, **kwargs): # FIXME: implement return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - def partial_update(self, request, uid=None): + def partial_update(self, request, uid=None, *args, **kwargs): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) def update(self, request, *args, **kwargs): @@ -202,7 +202,7 @@ class CollectionViewSet(BaseViewSet): return Response({}, status=status.HTTP_201_CREATED) - def list(self, request): + def list(self, request, *args, **kwargs): queryset = self.get_queryset() result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) @@ -251,21 +251,21 @@ class CollectionItemViewSet(BaseViewSet): context.update({'request': self.request, 'prefetch': prefetch}) return context - def create(self, request, collection_uid=None): + def create(self, request, collection_uid=None, *args, **kwargs): # We create using batch and transaction return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - def destroy(self, request, collection_uid=None, uid=None): + def destroy(self, request, collection_uid=None, uid=None, *args, **kwargs): # We can't have destroy because we need to get data from the user (in the body) such as hmac. return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - def update(self, request, collection_uid=None, uid=None): + def update(self, request, collection_uid=None, uid=None, *args, **kwargs): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - def partial_update(self, request, collection_uid=None, uid=None): + def partial_update(self, request, collection_uid=None, uid=None, *args, **kwargs): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - def list(self, request, collection_uid=None): + def list(self, request, collection_uid=None, *args, **kwargs): queryset = self.get_queryset() if not self.request.query_params.get('withCollection', False): @@ -283,7 +283,7 @@ class CollectionItemViewSet(BaseViewSet): return Response(ret) @action_decorator(detail=True, methods=['GET']) - def revision(self, request, collection_uid=None, uid=None): + def revision(self, request, collection_uid=None, uid=None, *args, **kwargs): col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) item = get_object_or_404(col.items, uid=uid) @@ -316,7 +316,7 @@ class CollectionItemViewSet(BaseViewSet): # FIXME: rename to something consistent with what the clients have - maybe list_updates? @action_decorator(detail=False, methods=['POST']) - def fetch_updates(self, request, collection_uid=None): + def fetch_updates(self, request, collection_uid=None, *args, **kwargs): queryset = self.get_queryset() serializer = CollectionItemBulkGetSerializer(data=request.data, many=True) @@ -349,11 +349,11 @@ class CollectionItemViewSet(BaseViewSet): return Response(ret) @action_decorator(detail=False, methods=['POST']) - def batch(self, request, collection_uid=None): + def batch(self, request, collection_uid=None, *args, **kwargs): return self.transaction(request, collection_uid, validate_etag=False) @action_decorator(detail=False, methods=['POST']) - def transaction(self, request, collection_uid=None, validate_etag=True): + def transaction(self, request, collection_uid=None, validate_etag=True, *args, **kwargs): stoken = request.GET.get('stoken', None) with transaction.atomic(): # We need this for locking on the collection object collection_object = get_object_or_404( @@ -405,7 +405,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): user = self.request.user return queryset.filter(members__user=user) - def create(self, request, collection_uid=None, collection_item_uid=None): + def create(self, request, collection_uid=None, collection_item_uid=None, *args, **kwargs): col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) col_it = get_object_or_404(col.items, uid=collection_item_uid) @@ -416,7 +416,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return Response({}, status=status.HTTP_201_CREATED) @action_decorator(detail=True, methods=['GET']) - def download(self, request, collection_uid=None, collection_item_uid=None, uid=None): + def download(self, request, collection_uid=None, collection_item_uid=None, uid=None, *args, **kwargs): import os from django.views.static import serve @@ -461,7 +461,7 @@ class CollectionMemberViewSet(BaseViewSet): def get_stoken_obj_id(self, request): return request.GET.get('iterator', None) - def list(self, request, collection_uid=None): + def list(self, request, collection_uid=None, *args, **kwargs): queryset = self.get_queryset().order_by('id') result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) serializer = self.get_serializer(result, many=True) @@ -474,7 +474,7 @@ class CollectionMemberViewSet(BaseViewSet): return Response(ret) - def create(self, request): + def create(self, request, *args, **kwargs): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) # FIXME: block leaving if we are the last admins - should be deleted / assigned in this case depending if there @@ -483,7 +483,7 @@ class CollectionMemberViewSet(BaseViewSet): instance.revoke() @action_decorator(detail=False, methods=['POST'], permission_classes=our_base_permission_classes) - def leave(self, request, collection_uid=None): + def leave(self, request, collection_uid=None, *args, **kwargs): collection_uid = self.kwargs['collection_uid'] col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) @@ -499,7 +499,7 @@ class InvitationBaseViewSet(BaseViewSet): lookup_field = 'uid' lookup_url_kwarg = 'invitation_uid' - def list(self, request, collection_uid=None): + def list(self, request, collection_uid=None, *args, **kwargs): limit = int(request.GET.get('limit', 50)) iterator = request.GET.get('iterator', None) @@ -556,7 +556,7 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): return Response({}, status=status.HTTP_201_CREATED) @action_decorator(detail=False, allowed_methods=['GET'], methods=['GET']) - def fetch_user_profile(self, request): + def fetch_user_profile(self, request, *args, **kwargs): username = request.GET.get('username') kwargs = {'owner__' + User.USERNAME_FIELD: username} user_info = get_object_or_404(UserInfo.objects.all(), **kwargs) @@ -574,7 +574,7 @@ class InvitationIncomingViewSet(InvitationBaseViewSet): return queryset.filter(user=self.request.user) @action_decorator(detail=True, allowed_methods=['POST'], methods=['POST']) - def accept(self, request, invitation_uid=None): + def accept(self, request, invitation_uid=None, *args, **kwargs): invitation = get_object_or_404(self.get_queryset(), uid=invitation_uid) context = self.get_serializer_context() context.update({'invitation': invitation}) @@ -605,11 +605,11 @@ class AuthenticationViewSet(viewsets.ViewSet): 'user': UserSerializer(user).data, } - def list(self, request): + def list(self, request, *args, **kwargs): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) @action_decorator(detail=False, methods=['POST']) - def signup(self, request): + def signup(self, request, *args, **kwargs): serializer = AuthenticationSignupSerializer(data=request.data) serializer.is_valid(raise_exception=True) user = serializer.save() @@ -660,7 +660,7 @@ class AuthenticationViewSet(viewsets.ViewSet): return None @action_decorator(detail=False, methods=['POST']) - def login_challenge(self, request): + def login_challenge(self, request, *args, **kwargs): from datetime import datetime serializer = AuthenticationLoginChallengeSerializer(data=request.data) @@ -686,7 +686,7 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response(ret, status=status.HTTP_200_OK) @action_decorator(detail=False, methods=['POST']) - def login(self, request): + def login(self, request, *args, **kwargs): outer_serializer = AuthenticationLoginSerializer(data=request.data) outer_serializer.is_valid(raise_exception=True) @@ -713,13 +713,13 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response(data, status=status.HTTP_200_OK) @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) - def logout(self, request): + def logout(self, request, *args, **kwargs): request.auth.delete() user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) return Response(status=status.HTTP_204_NO_CONTENT) @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) - def change_password(self, request): + def change_password(self, request, *args, **kwargs): outer_serializer = AuthenticationLoginSerializer(data=request.data) outer_serializer.is_valid(raise_exception=True) @@ -746,7 +746,7 @@ class TestAuthenticationViewSet(viewsets.ViewSet): renderer_classes = BaseViewSet.renderer_classes parser_classes = BaseViewSet.parser_classes - def list(self, request): + def list(self, request, *args, **kwargs): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) @action_decorator(detail=False, methods=['POST']) From f9add36f18e04288a4b5ea7768666d70ebc1abc0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 14:30:18 +0300 Subject: [PATCH 191/251] Add support for custom user filtering. --- django_etebase/app_settings.py | 7 +++++++ django_etebase/serializers.py | 16 ++++++++++++---- django_etebase/utils.py | 12 ++++++++++++ django_etebase/views.py | 11 +++++++---- 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 django_etebase/utils.py diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index b1fb4c3..7fe30b7 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -46,6 +46,13 @@ class AppSettings: ret.append(self.import_from_str(perm)) return ret + @property + def GET_USER_QUERYSET(self): # pylint: disable=invalid-name + get_user_queryset = self._setting("GET_USER_QUERYSET", None) + if get_user_queryset is not None: + return self.import_from_str(get_user_queryset) + return None + @property def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name return self._setting("CHALLENGE_VALID_SECONDS", 60) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 13199b3..0655775 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -20,6 +20,7 @@ from django.contrib.auth import get_user_model from django.db import transaction from rest_framework import serializers from . import models +from .utils import get_user_queryset User = get_user_model() @@ -91,6 +92,15 @@ class CollectionContentField(BinaryBase64Field): return None +class UserSlugRelatedField(serializers.SlugRelatedField): + def get_queryset(self): + view = self.context.get('view', None) + return get_user_queryset(super().get_queryset(), view) + + def __init__(self, **kwargs): + super().__init__(slug_field=User.USERNAME_FIELD, **kwargs) + + class ChunksField(serializers.RelatedField): def to_representation(self, obj): obj = obj.chunk @@ -252,9 +262,8 @@ class CollectionSerializer(serializers.ModelSerializer): class CollectionMemberSerializer(serializers.ModelSerializer): - username = serializers.SlugRelatedField( + username = UserSlugRelatedField( source='user', - slug_field=User.USERNAME_FIELD, read_only=True, ) @@ -278,9 +287,8 @@ class CollectionMemberSerializer(serializers.ModelSerializer): class CollectionInvitationSerializer(serializers.ModelSerializer): - username = serializers.SlugRelatedField( + username = UserSlugRelatedField( source='user', - slug_field=User.USERNAME_FIELD, queryset=User.objects ) collection = serializers.CharField(source='collection.uid') diff --git a/django_etebase/utils.py b/django_etebase/utils.py new file mode 100644 index 0000000..315b82f --- /dev/null +++ b/django_etebase/utils.py @@ -0,0 +1,12 @@ +from django.contrib.auth import get_user_model +from . import app_settings + + +User = get_user_model() + + +def get_user_queryset(queryset, view): + custom_func = app_settings.GET_USER_QUERYSET + if custom_func is not None: + return custom_func(queryset, view) + return queryset diff --git a/django_etebase/views.py b/django_etebase/views.py index 7e8bf98..480843e 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -71,6 +71,7 @@ from .serializers import ( UserInfoPubkeySerializer, UserSerializer, ) +from .utils import get_user_queryset User = get_user_model() @@ -558,8 +559,9 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): @action_decorator(detail=False, allowed_methods=['GET'], methods=['GET']) def fetch_user_profile(self, request, *args, **kwargs): username = request.GET.get('username') - kwargs = {'owner__' + User.USERNAME_FIELD: username} - user_info = get_object_or_404(UserInfo.objects.all(), **kwargs) + kwargs = {User.USERNAME_FIELD: username} + user = get_object_or_404(get_user_queryset(User.objects.all(), self), **kwargs) + user_info = get_object_or_404(UserInfo.objects.all(), owner=user) serializer = UserInfoPubkeySerializer(user_info) return Response(serializer.data) @@ -597,7 +599,7 @@ class AuthenticationViewSet(viewsets.ViewSet): encoder=nacl.encoding.RawEncoder) def get_queryset(self): - return User.objects.all() + return get_user_queryset(User.objects.all(), self) def login_response_data(self, user): return { @@ -756,7 +758,8 @@ class TestAuthenticationViewSet(viewsets.ViewSet): return HttpResponseBadRequest("Only allowed in debug mode.") with transaction.atomic(): - user = get_object_or_404(User.objects.all(), username=request.data.get('user').get('username')) + user_queryset = get_user_queryset(User.objects.all(), self) + user = get_object_or_404(user_queryset, username=request.data.get('user').get('username')) # Only allow test users for extra safety if not getattr(user, User.USERNAME_FIELD).startswith('test_user'): From 5c2f4d96ad65441ae18f2a0c2e8faa0763eb5e10 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 14:35:31 +0300 Subject: [PATCH 192/251] app settings: cache all the properties rather than recalc every time. They never change during runtime anyway. --- django_etebase/app_settings.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index 7fe30b7..2b9da4a 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -11,6 +11,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from django.utils.functional import cached_property class AppSettings: @@ -29,7 +30,7 @@ class AppSettings: from django.conf import settings return getattr(settings, self.prefix + name, dflt) - @property + @cached_property def API_PERMISSIONS(self): # pylint: disable=invalid-name perms = self._setting("API_PERMISSIONS", ('rest_framework.permissions.IsAuthenticated', )) ret = [] @@ -37,7 +38,7 @@ class AppSettings: ret.append(self.import_from_str(perm)) return ret - @property + @cached_property def API_AUTHENTICATORS(self): # pylint: disable=invalid-name perms = self._setting("API_AUTHENTICATORS", ('rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.SessionAuthentication')) @@ -46,14 +47,14 @@ class AppSettings: ret.append(self.import_from_str(perm)) return ret - @property + @cached_property def GET_USER_QUERYSET(self): # pylint: disable=invalid-name get_user_queryset = self._setting("GET_USER_QUERYSET", None) if get_user_queryset is not None: return self.import_from_str(get_user_queryset) return None - @property + @cached_property def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name return self._setting("CHALLENGE_VALID_SECONDS", 60) From a39617cf2e09e26b6209f203dd09715777556e26 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 15:26:05 +0300 Subject: [PATCH 193/251] Make sure usernames are case insensitive on lookup --- django_etebase/serializers.py | 3 +++ django_etebase/views.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 0655775..94ab3e7 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -100,6 +100,9 @@ class UserSlugRelatedField(serializers.SlugRelatedField): def __init__(self, **kwargs): super().__init__(slug_field=User.USERNAME_FIELD, **kwargs) + def to_internal_value(self, data): + return super().to_internal_value(data.lower()) + class ChunksField(serializers.RelatedField): def to_representation(self, obj): diff --git a/django_etebase/views.py b/django_etebase/views.py index 480843e..327bc08 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -439,7 +439,7 @@ class CollectionMemberViewSet(BaseViewSet): permission_classes = our_base_permission_classes + (permissions.IsCollectionAdmin, ) queryset = CollectionMember.objects.all() serializer_class = CollectionMemberSerializer - lookup_field = 'user__' + User.USERNAME_FIELD + lookup_field = f'user__{User.USERNAME_FIELD}__iexact' lookup_url_kwarg = 'username' stoken_id_fields = ['stoken__id'] @@ -559,7 +559,7 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): @action_decorator(detail=False, allowed_methods=['GET'], methods=['GET']) def fetch_user_profile(self, request, *args, **kwargs): username = request.GET.get('username') - kwargs = {User.USERNAME_FIELD: username} + kwargs = {User.USERNAME_FIELD: username.lower()} user = get_object_or_404(get_user_queryset(User.objects.all(), self), **kwargs) user_info = get_object_or_404(UserInfo.objects.all(), owner=user) serializer = UserInfoPubkeySerializer(user_info) @@ -620,7 +620,7 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response(data, status=status.HTTP_201_CREATED) def get_login_user(self, username): - kwargs = {User.USERNAME_FIELD: username} + kwargs = {User.USERNAME_FIELD: username.lower()} return get_object_or_404(self.get_queryset(), **kwargs) def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): From af86d877f2a26554748722b2dfa197f63020d893 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 15:40:14 +0300 Subject: [PATCH 194/251] Signup: use the shorthand version of setting an unusable password. It wasn't actually saving the unusable password before. --- django_etebase/serializers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 94ab3e7..a77c037 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -397,13 +397,11 @@ class AuthenticationSignupSerializer(serializers.Serializer): instance = User.objects.get_by_natural_key(user_data['username']) except User.DoesNotExist: # Create the user and save the casing the user chose as the first name - instance = User.objects.create_user(**user_data, first_name=user_data['username']) + instance = User.objects.create_user(**user_data, password=None, first_name=user_data['username']) if hasattr(instance, 'userinfo'): raise serializers.ValidationError('User already exists') - instance.set_unusable_password() - try: instance.clean_fields() except django_exceptions.ValidationError as e: From 46b4f08afa6b6fe96f3fdc3aeb98e96290f3ec05 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 16:03:34 +0300 Subject: [PATCH 195/251] Signup: use the get_user_queryset function when checking if user exists. --- django_etebase/serializers.py | 4 +++- django_etebase/views.py | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index a77c037..29e1d4f 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -394,7 +394,9 @@ class AuthenticationSignupSerializer(serializers.Serializer): with transaction.atomic(): try: - instance = User.objects.get_by_natural_key(user_data['username']) + view = self.context.get('view', None) + user_queryset = get_user_queryset(User.objects.all(), view) + instance = user_queryset.get(**{User.USERNAME_FIELD: user_data['username'].lower()}) except User.DoesNotExist: # Create the user and save the casing the user chose as the first name instance = User.objects.create_user(**user_data, password=None, first_name=user_data['username']) diff --git a/django_etebase/views.py b/django_etebase/views.py index 327bc08..8a6ff85 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -601,6 +601,13 @@ class AuthenticationViewSet(viewsets.ViewSet): def get_queryset(self): return get_user_queryset(User.objects.all(), self) + def get_serializer_context(self): + return { + 'request': self.request, + 'format': self.format_kwarg, + 'view': self + } + def login_response_data(self, user): return { 'token': AuthToken.objects.create(user=user).key, @@ -612,7 +619,7 @@ class AuthenticationViewSet(viewsets.ViewSet): @action_decorator(detail=False, methods=['POST']) def signup(self, request, *args, **kwargs): - serializer = AuthenticationSignupSerializer(data=request.data) + serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) serializer.is_valid(raise_exception=True) user = serializer.save() @@ -748,6 +755,13 @@ class TestAuthenticationViewSet(viewsets.ViewSet): renderer_classes = BaseViewSet.renderer_classes parser_classes = BaseViewSet.parser_classes + def get_serializer_context(self): + return { + 'request': self.request, + 'format': self.format_kwarg, + 'view': self + } + def list(self, request, *args, **kwargs): return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) @@ -768,7 +782,7 @@ class TestAuthenticationViewSet(viewsets.ViewSet): if hasattr(user, 'userinfo'): user.userinfo.delete() - serializer = AuthenticationSignupSerializer(data=request.data) + serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) serializer.is_valid(raise_exception=True) serializer.save() From e41f8455f2b57894dde22062e52da241020639e9 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 16:08:46 +0300 Subject: [PATCH 196/251] app settings: rename the get user queryset func setting name. --- django_etebase/app_settings.py | 4 ++-- django_etebase/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index 2b9da4a..b608717 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -48,8 +48,8 @@ class AppSettings: return ret @cached_property - def GET_USER_QUERYSET(self): # pylint: disable=invalid-name - get_user_queryset = self._setting("GET_USER_QUERYSET", None) + def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name + get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None) if get_user_queryset is not None: return self.import_from_str(get_user_queryset) return None diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 315b82f..bce2877 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -6,7 +6,7 @@ User = get_user_model() def get_user_queryset(queryset, view): - custom_func = app_settings.GET_USER_QUERYSET + custom_func = app_settings.GET_USER_QUERYSET_FUNC if custom_func is not None: return custom_func(queryset, view) return queryset From c9463cadbad200e5411627656b8ec3f61bc57caa Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 16:20:46 +0300 Subject: [PATCH 197/251] Add support for a custom user creation function. --- django_etebase/app_settings.py | 7 +++++++ django_etebase/serializers.py | 7 +++++-- django_etebase/utils.py | 8 ++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index b608717..3c659c8 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -54,6 +54,13 @@ class AppSettings: return self.import_from_str(get_user_queryset) return None + @cached_property + def CREATE_USER_FUNC(self): # pylint: disable=invalid-name + func = self._setting("CREATE_USER_FUNC", None) + if func is not None: + return self.import_from_str(func) + return None + @cached_property def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name return self._setting("CHALLENGE_VALID_SECONDS", 60) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 29e1d4f..b3f99e0 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -20,7 +20,7 @@ from django.contrib.auth import get_user_model from django.db import transaction from rest_framework import serializers from . import models -from .utils import get_user_queryset +from .utils import get_user_queryset, create_user User = get_user_model() @@ -399,7 +399,10 @@ class AuthenticationSignupSerializer(serializers.Serializer): instance = user_queryset.get(**{User.USERNAME_FIELD: user_data['username'].lower()}) except User.DoesNotExist: # Create the user and save the casing the user chose as the first name - instance = User.objects.create_user(**user_data, password=None, first_name=user_data['username']) + try: + instance = create_user(**user_data, password=None, first_name=user_data['username'], view=view) + except Exception as e: + raise serializers.ValidationError(e) if hasattr(instance, 'userinfo'): raise serializers.ValidationError('User already exists') diff --git a/django_etebase/utils.py b/django_etebase/utils.py index bce2877..08f81ae 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -10,3 +10,11 @@ def get_user_queryset(queryset, view): if custom_func is not None: return custom_func(queryset, view) return queryset + + +def create_user(*args, **kwargs): + custom_func = app_settings.CREATE_USER_FUNC + if custom_func is not None: + return custom_func(*args, **kwargs) + _ = kwargs.pop('view') + return User.objects.create_user(*args, **kwargs) From a7268443caf1da5ed7cf627904baa6dbfac99991 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 13 Jul 2020 17:08:36 +0300 Subject: [PATCH 198/251] Add support for a modifying the chunk storage location --- django_etebase/app_settings.py | 7 +++++++ django_etebase/models.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index 3c659c8..33dc65f 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -61,6 +61,13 @@ class AppSettings: return self.import_from_str(func) return None + @cached_property + def CHUNK_PATH_FUNC(self): # pylint: disable=invalid-name + func = self._setting("CHUNK_PATH_FUNC", None) + if func is not None: + return self.import_from_str(func) + return None + @cached_property def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name return self._setting("CHALLENGE_VALID_SECONDS", 60) diff --git a/django_etebase/models.py b/django_etebase/models.py index 397600e..f3704a3 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -21,6 +21,8 @@ from django.db.models import Q from django.utils.functional import cached_property from django.utils.crypto import get_random_string +from . import app_settings + UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{20,}$', message='Not a valid UID') @@ -79,6 +81,10 @@ class CollectionItem(models.Model): def chunk_directory_path(instance, filename): + custom_func = app_settings.CHUNK_PATH_FUNC + if custom_func is not None: + return custom_func(instance, filename) + item = instance.item col = item.collection user_id = col.owner.id From 3d6ba634ce6e4b291ae138345a81625e9759aaf6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 16 Jul 2020 10:40:30 +0300 Subject: [PATCH 199/251] Disallow + in usernames. --- myauth/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/myauth/models.py b/myauth/models.py index 4046b2f..611555b 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -7,10 +7,10 @@ from django.utils.translation import gettext_lazy as _ @deconstructible class UnicodeUsernameValidator(validators.RegexValidator): - regex = r'^[\w.+-]+\Z' + regex = r'^[\w.-]+\Z' message = _( 'Enter a valid username. This value may contain only letters, ' - 'numbers, and ./+/-/_ characters.' + 'numbers, and ./-/_ characters.' ) flags = 0 @@ -29,7 +29,7 @@ class User(AbstractUser): _('username'), max_length=150, unique=True, - help_text=_('Required. 150 characters or fewer. Letters, digits and ./+/-/_ only.'), + help_text=_('Required. 150 characters or fewer. Letters, digits and ./-/_ only.'), validators=[username_validator], error_messages={ 'unique': _("A user with that username already exists."), From 9c129e573187871130597b372ec045023ab3c034 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 22 Jul 2020 11:31:08 +0300 Subject: [PATCH 200/251] Collection erializer: make the item a child instead of trying to merge them. --- django_etebase/serializers.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index b3f99e0..bde8095 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -208,15 +208,11 @@ class CollectionSerializer(serializers.ModelSerializer): accessLevel = serializers.SerializerMethodField('get_access_level_from_context') stoken = serializers.CharField(read_only=True) - uid = serializers.CharField(source='main_item.uid') - encryptionKey = BinaryBase64Field(source='main_item.encryptionKey', required=False, default=None, allow_null=True) - etag = serializers.CharField(allow_null=True, write_only=True) - version = serializers.IntegerField(min_value=0, source='main_item.version') - content = CollectionItemRevisionSerializer(many=False, source='main_item.content') + item = CollectionItemSerializer(many=False, source='main_item') class Meta: model = models.Collection - fields = ('uid', 'version', 'accessLevel', 'encryptionKey', 'collectionKey', 'content', 'stoken', 'etag') + fields = ('item', 'accessLevel', 'collectionKey', 'stoken') def get_access_level_from_context(self, obj): request = self.context.get('request', None) @@ -228,13 +224,9 @@ class CollectionSerializer(serializers.ModelSerializer): """Function that's called when this serializer creates an item""" collection_key = validated_data.pop('collectionKey') - etag = validated_data.pop('etag') - main_item_data = validated_data.pop('main_item') - uid = main_item_data.pop('uid') - version = main_item_data.pop('version') + etag = main_item_data.pop('etag') revision_data = main_item_data.pop('content') - encryption_key = main_item_data.pop('encryptionKey') instance = self.__class__.Meta.model(**validated_data) @@ -243,8 +235,7 @@ class CollectionSerializer(serializers.ModelSerializer): raise serializers.ValidationError('etag is not None') instance.save() - main_item = models.CollectionItem.objects.create( - uid=uid, encryptionKey=encryption_key, version=version, collection=instance) + main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) instance.main_item = main_item instance.save() From 04231ebfe53c82a5715c1650530ca58d2bf7e9c3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sat, 25 Jul 2020 09:30:40 +0300 Subject: [PATCH 201/251] Views: fix issue with iterators sometimes returning the wrong type. --- django_etebase/views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 8a6ff85..9a71eee 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -306,11 +306,11 @@ class CollectionItemViewSet(BaseViewSet): serializer = CollectionItemRevisionSerializer(result, context=self.get_serializer_context(), many=True) - last_item = len(result) > 0 and serializer.data[-1] + iterator = serializer.data[-1]['uid'] if len(result) > 0 else None ret = { 'data': serializer.data, - 'iterator': last_item and last_item['uid'], + 'iterator': iterator, 'done': done, } return Response(ret) @@ -337,7 +337,7 @@ class CollectionItemViewSet(BaseViewSet): queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) new_stoken = self.get_queryset_stoken(queryset) - stoken = stoken_rev and stoken_rev.uid + stoken = getattr(stoken_rev, 'uid', None) if stoken_rev is not None else None new_stoken = new_stoken or stoken serializer = self.get_serializer(queryset, many=True) @@ -519,11 +519,11 @@ class InvitationBaseViewSet(BaseViewSet): serializer = self.get_serializer(result, many=True) - last_item = len(result) > 0 and serializer.data[-1] + iterator = serializer.data[-1]['uid'] if len(result) > 0 else None ret = { 'data': serializer.data, - 'iterator': last_item and last_item['uid'], + 'iterator': iterator, 'done': done, } From c0575cb64c64d19fd8336113bad0517912bfc81f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 30 Jul 2020 10:13:24 +0300 Subject: [PATCH 202/251] Exceptions: have correct code/status_code for every error. --- django_etebase/exceptions.py | 10 ++++++++++ django_etebase/serializers.py | 22 ++++++++++++---------- django_etebase/views.py | 15 +++++++++++---- 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 django_etebase/exceptions.py diff --git a/django_etebase/exceptions.py b/django_etebase/exceptions.py new file mode 100644 index 0000000..d05c4e5 --- /dev/null +++ b/django_etebase/exceptions.py @@ -0,0 +1,10 @@ +from rest_framework import serializers, status + + +class EtebaseValidationError(serializers.ValidationError): + def __init__(self, code, detail, status_code=status.HTTP_400_BAD_REQUEST): + super().__init__({ + 'code': code, + 'detail': detail, + }) + self.status_code = status_code diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index bde8095..c8e93cb 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -18,10 +18,12 @@ from django.core.files.base import ContentFile from django.core import exceptions as django_exceptions from django.contrib.auth import get_user_model from django.db import transaction -from rest_framework import serializers +from rest_framework import serializers, status from . import models from .utils import get_user_queryset, create_user +from .exceptions import EtebaseValidationError + User = get_user_model() @@ -40,7 +42,7 @@ def process_revisions_for_item(item, revision_data): chunk_obj.save() else: if chunk_obj is None: - raise serializers.ValidationError('Tried to create a new chunk without content') + raise EtebaseValidationError('chunk_no_content', 'Tried to create a new chunk without content') chunks_objs.append(chunk_obj) @@ -115,7 +117,7 @@ class ChunksField(serializers.RelatedField): def to_internal_value(self, data): if data[0] is None or data[1] is None: - raise serializers.ValidationError('null is not allowed') + raise EtebaseValidationError('null is not allowed') return (data[0], b64decode_or_bytes(data[1])) @@ -161,7 +163,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): cur_etag = instance.etag if not created else None if validate_etag and cur_etag != etag: - raise serializers.ValidationError('Wrong etag. Expected {} got {}'.format(cur_etag, etag)) + raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(cur_etag, etag), status_code=status.HTTP_409_CONFLICT) if not created: # We don't have to use select_for_update here because the unique constraint on current guards against @@ -190,7 +192,7 @@ class CollectionItemDepSerializer(serializers.ModelSerializer): item = self.__class__.Meta.model.objects.get(uid=data['uid']) etag = data['etag'] if item.etag != etag: - raise serializers.ValidationError('Wrong etag. Expected {} got {}'.format(item.etag, etag)) + raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(item.etag, etag), status_code=status.HTTP_409_CONFLICT) return data @@ -232,7 +234,7 @@ class CollectionSerializer(serializers.ModelSerializer): with transaction.atomic(): if etag is not None: - raise serializers.ValidationError('etag is not None') + raise EtebaseValidationError('bad_etag', 'etag is not null') instance.save() main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) @@ -297,7 +299,7 @@ class CollectionInvitationSerializer(serializers.ModelSerializer): request = self.context['request'] if request.user.username == value.lower(): - raise serializers.ValidationError('Inviting yourself is not allowed') + raise EtebaseValidationError('no_self_invite', 'Inviting yourself is not allowed') return value def create(self, validated_data): @@ -393,15 +395,15 @@ class AuthenticationSignupSerializer(serializers.Serializer): try: instance = create_user(**user_data, password=None, first_name=user_data['username'], view=view) except Exception as e: - raise serializers.ValidationError(e) + raise EtebaseValidationError('generic', str(e)) if hasattr(instance, 'userinfo'): - raise serializers.ValidationError('User already exists') + raise EtebaseValidationError('user_exists', 'User already exists', status_code=status.HTTP_409_CONFLICT) try: instance.clean_fields() except django_exceptions.ValidationError as e: - raise serializers.ValidationError(e) + raise EtebaseValidationError('generic', str(e)) # FIXME: send email verification models.UserInfo.objects.create(**validated_data, owner=instance) diff --git a/django_etebase/views.py b/django_etebase/views.py index 9a71eee..eb04e4b 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -72,7 +72,7 @@ from .serializers import ( UserSerializer, ) from .utils import get_user_queryset - +from .exceptions import EtebaseValidationError User = get_user_model() @@ -111,7 +111,14 @@ class BaseViewSet(viewsets.ModelViewSet): stoken = self.get_stoken_obj_id(request) if stoken is not None: - return get_object_or_404(Stoken.objects.all(), uid=stoken) + try: + return Stoken.objects.get(uid=stoken) + except Stoken.DoesNotExist: + raise EtebaseValidationError({ + 'code': 'bad_stoken', + 'detail': 'Invalid stoken.', + }, + status_code=status.HTTP_400_BAD_REQUEST) return None @@ -363,7 +370,7 @@ class CollectionItemViewSet(BaseViewSet): if stoken is not None and stoken != collection_object.stoken: content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} - return Response(content, status=status.HTTP_400_BAD_REQUEST) + return Response(content, status=status.HTTP_409_CONFLICT) items = request.data.get('items') deps = request.data.get('deps', None) @@ -387,7 +394,7 @@ class CollectionItemViewSet(BaseViewSet): "items": serializer.errors, "deps": deps_serializer.errors if deps is not None else [], }, - status=status.HTTP_400_BAD_REQUEST) + status=status.HTTP_409_CONFLICT) class CollectionItemChunkViewSet(viewsets.ViewSet): From f6af96ace6472804f52e66acafb411fa968da41b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 30 Jul 2020 10:17:26 +0300 Subject: [PATCH 203/251] Permissions: workaround DRF bug and expose exception code. --- django_etebase/permissions.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/django_etebase/permissions.py b/django_etebase/permissions.py index 6a36afb..c624404 100644 --- a/django_etebase/permissions.py +++ b/django_etebase/permissions.py @@ -25,8 +25,10 @@ class IsCollectionAdmin(permissions.BasePermission): """ Custom permission to only allow owners of a collection to view it """ - message = 'Only collection admins can perform this operation.' - code = 'admin_access_required' + message = { + 'detail': 'Only collection admins can perform this operation.', + 'code': 'admin_access_required', + } def has_permission(self, request, view): collection_uid = view.kwargs['collection_uid'] @@ -42,8 +44,10 @@ class IsCollectionAdminOrReadOnly(permissions.BasePermission): """ Custom permission to only allow owners of a collection to edit it """ - message = 'Only collection admins can edit collections.' - code = 'admin_access_required' + message = { + 'detail': 'Only collection admins can edit collections.', + 'code': 'admin_access_required', + } def has_permission(self, request, view): collection_uid = view.kwargs.get('collection_uid', None) @@ -67,8 +71,10 @@ class HasWriteAccessOrReadOnly(permissions.BasePermission): """ Custom permission to restrict write """ - message = 'You need write access to write to this collection' - code = 'no_write_access' + message = { + 'detail': 'You need write access to write to this collection', + 'code': 'no_write_access', + } def has_permission(self, request, view): collection_uid = view.kwargs['collection_uid'] From 11001ed62ce0f7ede28514f5a6efb609460fc9f6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 4 Aug 2020 13:17:48 +0300 Subject: [PATCH 204/251] Chunk serializer: fix bad error invocation. --- django_etebase/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index c8e93cb..0e44228 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -117,7 +117,7 @@ class ChunksField(serializers.RelatedField): def to_internal_value(self, data): if data[0] is None or data[1] is None: - raise EtebaseValidationError('null is not allowed') + raise EtebaseValidationError('no_null', 'null is not allowed') return (data[0], b64decode_or_bytes(data[1])) From 1d5baece1e1244558882835dacafc1d7c8def8ac Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 4 Aug 2020 13:42:28 +0300 Subject: [PATCH 205/251] Chunk uploading: implement properly using a custom Parser. --- .../migrations/0022_auto_20200804_1059.py | 17 ++++++++++++++ django_etebase/models.py | 3 +++ django_etebase/parsers.py | 15 +++++++++++++ django_etebase/views.py | 22 ++++++++++++++----- 4 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 django_etebase/migrations/0022_auto_20200804_1059.py create mode 100644 django_etebase/parsers.py diff --git a/django_etebase/migrations/0022_auto_20200804_1059.py b/django_etebase/migrations/0022_auto_20200804_1059.py new file mode 100644 index 0000000..c47e562 --- /dev/null +++ b/django_etebase/migrations/0022_auto_20200804_1059.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-08-04 10:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0021_auto_20200626_0913'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='collectionitemchunk', + unique_together={('item', 'uid')}, + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index f3704a3..0c33301 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -100,6 +100,9 @@ class CollectionItemChunk(models.Model): def __str__(self): return self.uid + class Meta: + unique_together = ('item', 'uid') + def generate_stoken_uid(): return get_random_string(32, allowed_chars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_') diff --git a/django_etebase/parsers.py b/django_etebase/parsers.py new file mode 100644 index 0000000..1ca1a70 --- /dev/null +++ b/django_etebase/parsers.py @@ -0,0 +1,15 @@ +from rest_framework.parsers import FileUploadParser + + +class ChunkUploadParser(FileUploadParser): + """ + Parser for chunk upload data. + """ + media_type = 'application/octet-stream' + + def get_filename(self, stream, media_type, parser_context): + """ + Detects the uploaded file name. + """ + view = parser_context['view'] + return parser_context['kwargs'][view.lookup_field] diff --git a/django_etebase/views.py b/django_etebase/views.py index eb04e4b..c299dc2 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -73,6 +73,7 @@ from .serializers import ( ) from .utils import get_user_queryset from .exceptions import EtebaseValidationError +from .parsers import ChunkUploadParser User = get_user_model() @@ -398,11 +399,11 @@ class CollectionItemViewSet(BaseViewSet): class CollectionItemChunkViewSet(viewsets.ViewSet): - allowed_methods = ['GET', 'POST'] + allowed_methods = ['GET', 'PUT'] authentication_classes = BaseViewSet.authentication_classes permission_classes = BaseViewSet.permission_classes renderer_classes = BaseViewSet.renderer_classes - parser_classes = (MultiPartParser, ) + parser_classes = (ChunkUploadParser, ) serializer_class = CollectionItemChunkSerializer lookup_field = 'uid' @@ -413,13 +414,24 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): user = self.request.user return queryset.filter(members__user=user) - def create(self, request, collection_uid=None, collection_item_uid=None, *args, **kwargs): + def update(self, request, *args, collection_uid=None, collection_item_uid=None, uid=None, **kwargs): col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) col_it = get_object_or_404(col.items, uid=collection_item_uid) - serializer = self.get_serializer_class()(data=request.data) + data = { + "uid": uid, + "chunkFile": request.data["file"], + } + + serializer = self.get_serializer_class()(data=data) serializer.is_valid(raise_exception=True) - serializer.save(item=col_it) + try: + serializer.save(item=col_it) + except IntegrityError: + return Response( + {"code": "chunk_exists", "detail": "Chunk already exists."}, + status=status.HTTP_409_CONFLICT + ) return Response({}, status=status.HTTP_201_CREATED) From 393b85d3ca23513a5ea7a61fdeab94f33c7eb490 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 4 Aug 2020 15:19:45 +0300 Subject: [PATCH 206/251] Chunks: move to reside under the collection. --- .../0023_collectionitemchunk_collection.py | 19 +++++++++++++ .../migrations/0024_auto_20200804_1209.py | 22 +++++++++++++++ .../migrations/0025_auto_20200804_1216.py | 27 +++++++++++++++++++ django_etebase/models.py | 9 +++---- django_etebase/serializers.py | 2 +- django_etebase/views.py | 8 +++--- 6 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 django_etebase/migrations/0023_collectionitemchunk_collection.py create mode 100644 django_etebase/migrations/0024_auto_20200804_1209.py create mode 100644 django_etebase/migrations/0025_auto_20200804_1216.py diff --git a/django_etebase/migrations/0023_collectionitemchunk_collection.py b/django_etebase/migrations/0023_collectionitemchunk_collection.py new file mode 100644 index 0000000..b5d6841 --- /dev/null +++ b/django_etebase/migrations/0023_collectionitemchunk_collection.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-08-04 12:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0022_auto_20200804_1059'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemchunk', + name='collection', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etebase.Collection'), + ), + ] diff --git a/django_etebase/migrations/0024_auto_20200804_1209.py b/django_etebase/migrations/0024_auto_20200804_1209.py new file mode 100644 index 0000000..54c80a3 --- /dev/null +++ b/django_etebase/migrations/0024_auto_20200804_1209.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.3 on 2020-08-04 12:09 + +from django.db import migrations + + +def change_chunk_to_collections(apps, schema_editor): + CollectionItemChunk = apps.get_model('django_etebase', 'CollectionItemChunk') + + for chunk in CollectionItemChunk.objects.all(): + chunk.collection = chunk.item.collection + chunk.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0023_collectionitemchunk_collection'), + ] + + operations = [ + migrations.RunPython(change_chunk_to_collections), + ] diff --git a/django_etebase/migrations/0025_auto_20200804_1216.py b/django_etebase/migrations/0025_auto_20200804_1216.py new file mode 100644 index 0000000..8849f53 --- /dev/null +++ b/django_etebase/migrations/0025_auto_20200804_1216.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.3 on 2020-08-04 12:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0024_auto_20200804_1209'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='collection', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etebase.Collection'), + ), + migrations.AlterUniqueTogether( + name='collectionitemchunk', + unique_together={('collection', 'uid')}, + ), + migrations.RemoveField( + model_name='collectionitemchunk', + name='item', + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 0c33301..403d2b7 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -85,23 +85,22 @@ def chunk_directory_path(instance, filename): if custom_func is not None: return custom_func(instance, filename) - item = instance.item - col = item.collection + col = instance.collection user_id = col.owner.id - return Path('user_{}'.format(user_id), col.uid, item.uid, instance.uid) + return Path('user_{}'.format(user_id), col.uid, instance.uid) class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=60, validators=[UidValidator]) - item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) + collection = models.ForeignKey(Collection, related_name='chunks', on_delete=models.CASCADE) chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) def __str__(self): return self.uid class Meta: - unique_together = ('item', 'uid') + unique_together = ('collection', 'uid') def generate_stoken_uid(): diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 0e44228..8cfddad 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -37,7 +37,7 @@ def process_revisions_for_item(item, revision_data): content = chunk[1] # If the chunk already exists we assume it's fine. Otherwise, we upload it. if chunk_obj is None: - chunk_obj = models.CollectionItemChunk(uid=uid, item=item) + chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) chunk_obj.chunkFile.save('IGNORED', ContentFile(content)) chunk_obj.save() else: diff --git a/django_etebase/views.py b/django_etebase/views.py index c299dc2..2678451 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -416,7 +416,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): def update(self, request, *args, collection_uid=None, collection_item_uid=None, uid=None, **kwargs): col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) - col_it = get_object_or_404(col.items, uid=collection_item_uid) + # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) data = { "uid": uid, @@ -426,7 +426,7 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): serializer = self.get_serializer_class()(data=data) serializer.is_valid(raise_exception=True) try: - serializer.save(item=col_it) + serializer.save(collection=col) except IntegrityError: return Response( {"code": "chunk_exists", "detail": "Chunk already exists."}, @@ -441,8 +441,8 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): from django.views.static import serve col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) - col_it = get_object_or_404(col.items, uid=collection_item_uid) - chunk = get_object_or_404(col_it.chunks, uid=uid) + # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) + chunk = get_object_or_404(col.chunks, uid=uid) filename = chunk.chunkFile.path dirname = os.path.dirname(filename) From e385aa8f20ef6f160e43a5461b9603c870b5716e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 4 Aug 2020 15:37:07 +0300 Subject: [PATCH 207/251] Chunks: use a prefix of the chunk for a subdirectory. Filesystems don't handle massive directories too well, so better to split. Using the prefix of the chunk gives us a maximum of 64 * 64 = 4096 entries in the main directory. --- django_etebase/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_etebase/models.py b/django_etebase/models.py index 403d2b7..7570bae 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -87,7 +87,9 @@ def chunk_directory_path(instance, filename): col = instance.collection user_id = col.owner.id - return Path('user_{}'.format(user_id), col.uid, instance.uid) + uid_prefix = instance.uid[:2] + uid_rest = instance.uid[2:] + return Path('user_{}'.format(user_id), col.uid, uid_prefix, uid_rest) class CollectionItemChunk(models.Model): From a613a326283e709a05c681f6c691c3994e91ab67 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 4 Aug 2020 15:59:31 +0300 Subject: [PATCH 208/251] prefetch: fix handling of the prefetch param. --- django_etebase/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 2678451..95031e3 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -190,7 +190,7 @@ class CollectionViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - prefetch = self.request.query_params.get('prefetch', True) + prefetch = self.request.query_params.get('prefetch', 'true') != 'false' context.update({'request': self.request, 'prefetch': prefetch}) return context @@ -256,7 +256,7 @@ class CollectionItemViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - prefetch = self.request.query_params.get('prefetch', True) + prefetch = self.request.query_params.get('prefetch', 'true') != 'false' context.update({'request': self.request, 'prefetch': prefetch}) return context From cf9b6f5904c22c7a03e719d914b308da22261371 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 4 Aug 2020 17:44:57 +0300 Subject: [PATCH 209/251] Prefetch: change the type of value prefetch accept. It's 'auto' by default, but can be changed to 'medium' and soon another value. --- django_etebase/serializers.py | 2 +- django_etebase/views.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 8cfddad..40f6068 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -109,7 +109,7 @@ class UserSlugRelatedField(serializers.SlugRelatedField): class ChunksField(serializers.RelatedField): def to_representation(self, obj): obj = obj.chunk - if self.context.get('prefetch'): + if self.context.get('prefetch') == 'auto': with open(obj.chunkFile.path, 'rb') as f: return (obj.uid, f.read()) else: diff --git a/django_etebase/views.py b/django_etebase/views.py index 95031e3..9d76d08 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -190,7 +190,7 @@ class CollectionViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - prefetch = self.request.query_params.get('prefetch', 'true') != 'false' + prefetch = self.request.query_params.get('prefetch', 'auto') context.update({'request': self.request, 'prefetch': prefetch}) return context @@ -256,7 +256,7 @@ class CollectionItemViewSet(BaseViewSet): def get_serializer_context(self): context = super().get_serializer_context() - prefetch = self.request.query_params.get('prefetch', 'true') != 'false' + prefetch = self.request.query_params.get('prefetch', 'auto') context.update({'request': self.request, 'prefetch': prefetch}) return context From 5af2aeda7e44e1b100b51df6da60e9ee7b2b2dcf Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 18 Aug 2020 12:02:56 +0300 Subject: [PATCH 210/251] Add an endpoint to know if a server is an etebase server or not. Very useful for when migrating people from legacy EteSync apps because we can automatically know if they are running a self-hosted etesync or etebase server. --- django_etebase/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django_etebase/views.py b/django_etebase/views.py index 9d76d08..1e9b8b5 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -687,6 +687,10 @@ class AuthenticationViewSet(viewsets.ViewSet): return None + @action_decorator(detail=False, methods=['GET']) + def is_etebase(self, request, *args, **kwargs): + return Response({}, status=status.HTTP_200_OK) + @action_decorator(detail=False, methods=['POST']) def login_challenge(self, request, *args, **kwargs): from datetime import datetime From 693a5ec778b9a4645dc52de65533f1e1591e752f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 18 Aug 2020 12:04:42 +0300 Subject: [PATCH 211/251] Login: return an UNAUTHORIZED (401) error on bad username/password, not 400. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 1e9b8b5..a39d443 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -683,7 +683,7 @@ class AuthenticationViewSet(viewsets.ViewSet): try: verify_key.verify(response_raw, signature) except nacl.exceptions.BadSignatureError: - return Response({'code': 'login_bad_signature'}, status=status.HTTP_400_BAD_REQUEST) + return Response({'code': 'login_bad_signature'}, status=status.HTTP_401_UNAUTHORIZED) return None From 8593ab13578dec500b5768c8e21f3d1afc649630 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 18 Aug 2020 12:24:30 +0300 Subject: [PATCH 212/251] Login: add a user visible error on password failure. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index a39d443..c45630b 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -683,7 +683,7 @@ class AuthenticationViewSet(viewsets.ViewSet): try: verify_key.verify(response_raw, signature) except nacl.exceptions.BadSignatureError: - return Response({'code': 'login_bad_signature'}, status=status.HTTP_401_UNAUTHORIZED) + return Response({'code': 'login_bad_signature', 'detail': 'Wrong password for user.'}, status=status.HTTP_401_UNAUTHORIZED) return None From 2327466113838e1594e25a787183fb0a2cea23eb Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 28 Aug 2020 13:55:15 +0300 Subject: [PATCH 213/251] Invitations: error when trying to invite oneself. --- django_etebase/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django_etebase/views.py b/django_etebase/views.py index c45630b..c8a98fc 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -568,6 +568,10 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): except Collection.DoesNotExist: raise Http404('Collection does not exist') + if request.user == serializer.validated_data.get('user'): + content = {'code': 'self_invite', 'detail': 'Inviting yourself is invalid'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + if not permissions.is_collection_admin(collection, request.user): raise PermissionDenied('User is not an admin of this collection') From bf22b1676f39ff05ca743fcd7f4789896997dc3f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 2 Sep 2020 11:07:43 +0300 Subject: [PATCH 214/251] Serializers: improve field serialization errors. --- django_etebase/serializers.py | 88 ++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 40f6068..356da82 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -121,13 +121,69 @@ class ChunksField(serializers.RelatedField): return (data[0], b64decode_or_bytes(data[1])) -class CollectionItemChunkSerializer(serializers.ModelSerializer): +class BetterErrorsMixin: + @property + def errors(self): + nice = [] + errors = super().errors + for error_type in errors: + if error_type == 'non_field_errors': + nice.extend( + self.flatten_errors(None, errors[error_type]) + ) + else: + nice.extend( + self.flatten_errors(error_type, errors[error_type]) + ) + if nice: + return {'code': 'field_errors', + 'message': 'Field validations failed.', + 'errors': nice} + return {} + + def flatten_errors(self, field_name, errors): + ret = [] + if isinstance(errors, dict): + for error_key in errors: + error = errors[error_key] + ret.extend(self.flatten_errors("{}.{}".format(field_name, error_key), error)) + else: + for error in errors: + if hasattr(error, 'detail'): + message = error.detail[0] + elif hasattr(error, 'message'): + message = error.message + else: + message = str(error) + ret.append({ + 'field': field_name, + 'code': error.code, + 'message': message, + }) + return ret + + def transform_validation_error(self, prefix, err): + if hasattr(err, 'error_dict'): + errors = self.flatten_errors(prefix, err.error_dict) + elif not hasattr(err, 'message'): + errors = self.flatten_errors(prefix, err.error_list) + else: + raise EtebaseValidationError(err.code, err.message) + + raise serializers.ValidationError({ + 'code': 'field_errors', + 'message': 'Field validations failed.', + 'errors': errors, + }) + + +class CollectionItemChunkSerializer(BetterErrorsMixin, serializers.ModelSerializer): class Meta: model = models.CollectionItemChunk fields = ('uid', 'chunkFile') -class CollectionItemRevisionSerializer(serializers.ModelSerializer): +class CollectionItemRevisionSerializer(BetterErrorsMixin, serializers.ModelSerializer): chunks = ChunksField( source='chunks_relation', queryset=models.RevisionChunkRelation.objects.all(), @@ -140,7 +196,7 @@ class CollectionItemRevisionSerializer(serializers.ModelSerializer): fields = ('chunks', 'meta', 'uid', 'deleted') -class CollectionItemSerializer(serializers.ModelSerializer): +class CollectionItemSerializer(BetterErrorsMixin, serializers.ModelSerializer): encryptionKey = BinaryBase64Field(required=False, default=None, allow_null=True) etag = serializers.CharField(allow_null=True, write_only=True) content = CollectionItemRevisionSerializer(many=False) @@ -181,7 +237,7 @@ class CollectionItemSerializer(serializers.ModelSerializer): raise NotImplementedError() -class CollectionItemDepSerializer(serializers.ModelSerializer): +class CollectionItemDepSerializer(BetterErrorsMixin, serializers.ModelSerializer): etag = serializers.CharField() class Meta: @@ -197,7 +253,7 @@ class CollectionItemDepSerializer(serializers.ModelSerializer): return data -class CollectionItemBulkGetSerializer(serializers.ModelSerializer): +class CollectionItemBulkGetSerializer(BetterErrorsMixin, serializers.ModelSerializer): etag = serializers.CharField(required=False) class Meta: @@ -205,7 +261,7 @@ class CollectionItemBulkGetSerializer(serializers.ModelSerializer): fields = ('uid', 'etag') -class CollectionSerializer(serializers.ModelSerializer): +class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): collectionKey = CollectionEncryptionKeyField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') stoken = serializers.CharField(read_only=True) @@ -257,7 +313,7 @@ class CollectionSerializer(serializers.ModelSerializer): raise NotImplementedError() -class CollectionMemberSerializer(serializers.ModelSerializer): +class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer): username = UserSlugRelatedField( source='user', read_only=True, @@ -282,7 +338,7 @@ class CollectionMemberSerializer(serializers.ModelSerializer): return instance -class CollectionInvitationSerializer(serializers.ModelSerializer): +class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSerializer): username = UserSlugRelatedField( source='user', queryset=User.objects @@ -320,7 +376,7 @@ class CollectionInvitationSerializer(serializers.ModelSerializer): return instance -class InvitationAcceptSerializer(serializers.Serializer): +class InvitationAcceptSerializer(BetterErrorsMixin, serializers.Serializer): encryptionKey = BinaryBase64Field() def create(self, validated_data): @@ -348,7 +404,7 @@ class InvitationAcceptSerializer(serializers.Serializer): raise NotImplementedError() -class UserSerializer(serializers.ModelSerializer): +class UserSerializer(BetterErrorsMixin, serializers.ModelSerializer): pubkey = BinaryBase64Field(source='userinfo.pubkey') encryptedContent = BinaryBase64Field(source='userinfo.encryptedContent') @@ -357,7 +413,7 @@ class UserSerializer(serializers.ModelSerializer): fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, 'pubkey', 'encryptedContent') -class UserInfoPubkeySerializer(serializers.ModelSerializer): +class UserInfoPubkeySerializer(BetterErrorsMixin, serializers.ModelSerializer): pubkey = BinaryBase64Field() class Meta: @@ -365,7 +421,7 @@ class UserInfoPubkeySerializer(serializers.ModelSerializer): fields = ('pubkey', ) -class UserSignupSerializer(serializers.ModelSerializer): +class UserSignupSerializer(BetterErrorsMixin, serializers.ModelSerializer): class Meta: model = User fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) @@ -374,7 +430,7 @@ class UserSignupSerializer(serializers.ModelSerializer): } -class AuthenticationSignupSerializer(serializers.Serializer): +class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): user = UserSignupSerializer(many=False) salt = BinaryBase64Field() loginPubkey = BinaryBase64Field() @@ -403,7 +459,7 @@ class AuthenticationSignupSerializer(serializers.Serializer): try: instance.clean_fields() except django_exceptions.ValidationError as e: - raise EtebaseValidationError('generic', str(e)) + self.transform_validation_error("user", e) # FIXME: send email verification models.UserInfo.objects.create(**validated_data, owner=instance) @@ -414,7 +470,7 @@ class AuthenticationSignupSerializer(serializers.Serializer): raise NotImplementedError() -class AuthenticationLoginChallengeSerializer(serializers.Serializer): +class AuthenticationLoginChallengeSerializer(BetterErrorsMixin, serializers.Serializer): username = serializers.CharField(required=True) def create(self, validated_data): @@ -424,7 +480,7 @@ class AuthenticationLoginChallengeSerializer(serializers.Serializer): raise NotImplementedError() -class AuthenticationLoginSerializer(serializers.Serializer): +class AuthenticationLoginSerializer(BetterErrorsMixin, serializers.Serializer): response = BinaryBase64Field() signature = BinaryBase64Field() From 7ab9513e055111aff68121ede0a27bb1c42c5d3c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 2 Sep 2020 11:11:17 +0300 Subject: [PATCH 215/251] Serializers: rename message to detail to conform with the rest of the API. This was a mistake in the previous commit. --- django_etebase/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 356da82..036da1e 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -137,7 +137,7 @@ class BetterErrorsMixin: ) if nice: return {'code': 'field_errors', - 'message': 'Field validations failed.', + 'detail': 'Field validations failed.', 'errors': nice} return {} @@ -158,7 +158,7 @@ class BetterErrorsMixin: ret.append({ 'field': field_name, 'code': error.code, - 'message': message, + 'detail': message, }) return ret @@ -172,7 +172,7 @@ class BetterErrorsMixin: raise serializers.ValidationError({ 'code': 'field_errors', - 'message': 'Field validations failed.', + 'detail': 'Field validations failed.', 'errors': errors, }) From 42a72ce5c7bda317a2fcf2218704540f89544c6f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 2 Sep 2020 12:50:47 +0300 Subject: [PATCH 216/251] Serializers user signup: correctly handle EtebaseValidationErrors. Don't coerce them to strings --- django_etebase/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 036da1e..babf7d0 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -450,6 +450,8 @@ class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): # Create the user and save the casing the user chose as the first name try: instance = create_user(**user_data, password=None, first_name=user_data['username'], view=view) + except EtebaseValidationError as e: + raise e except Exception as e: raise EtebaseValidationError('generic', str(e)) From 43569727f4dd0891636a01c851b8e3d5a92b0cfd Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 2 Sep 2020 12:54:27 +0300 Subject: [PATCH 217/251] Signup: send a signal on account signup. --- django_etebase/serializers.py | 1 - django_etebase/signals.py | 3 +++ django_etebase/views.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 django_etebase/signals.py diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index babf7d0..35b3562 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -462,7 +462,6 @@ class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): instance.clean_fields() except django_exceptions.ValidationError as e: self.transform_validation_error("user", e) - # FIXME: send email verification models.UserInfo.objects.create(**validated_data, owner=instance) diff --git a/django_etebase/signals.py b/django_etebase/signals.py new file mode 100644 index 0000000..03dbed5 --- /dev/null +++ b/django_etebase/signals.py @@ -0,0 +1,3 @@ +from django.dispatch import Signal + +user_signed_up = Signal(providing_args=['request', 'user']) diff --git a/django_etebase/views.py b/django_etebase/views.py index c8a98fc..97cb2f0 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -74,6 +74,7 @@ from .serializers import ( from .utils import get_user_queryset from .exceptions import EtebaseValidationError from .parsers import ChunkUploadParser +from .signals import user_signed_up User = get_user_model() @@ -646,6 +647,8 @@ class AuthenticationViewSet(viewsets.ViewSet): serializer.is_valid(raise_exception=True) user = serializer.save() + user_signed_up.send(sender=user.__class__, request=request, user=user) + data = self.login_response_data(user) return Response(data, status=status.HTTP_201_CREATED) From d90931fbe5b67112aba8c4976fa52c5310222f1e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 7 Sep 2020 11:02:40 +0300 Subject: [PATCH 218/251] Make access level an int instead of a string. We started with a string because we thought it could maybe provide more flexibility, though really, an int makes much more sense, especially on all the platforms etebase runs nowadays. --- .../migrations/0026_auto_20200907_0752.py | 23 +++++++++++ .../migrations/0027_auto_20200907_0752.py | 23 +++++++++++ .../migrations/0028_auto_20200907_0754.py | 39 +++++++++++++++++++ .../migrations/0029_auto_20200907_0801.py | 21 ++++++++++ django_etebase/models.py | 14 +++---- 5 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 django_etebase/migrations/0026_auto_20200907_0752.py create mode 100644 django_etebase/migrations/0027_auto_20200907_0752.py create mode 100644 django_etebase/migrations/0028_auto_20200907_0754.py create mode 100644 django_etebase/migrations/0029_auto_20200907_0801.py diff --git a/django_etebase/migrations/0026_auto_20200907_0752.py b/django_etebase/migrations/0026_auto_20200907_0752.py new file mode 100644 index 0000000..38c0b92 --- /dev/null +++ b/django_etebase/migrations/0026_auto_20200907_0752.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1 on 2020-09-07 07:52 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0025_auto_20200804_1216'), + ] + + operations = [ + migrations.RenameField( + model_name='collectioninvitation', + old_name='accessLevel', + new_name='accessLevelOld', + ), + migrations.RenameField( + model_name='collectionmember', + old_name='accessLevel', + new_name='accessLevelOld', + ), + ] diff --git a/django_etebase/migrations/0027_auto_20200907_0752.py b/django_etebase/migrations/0027_auto_20200907_0752.py new file mode 100644 index 0000000..d822d3d --- /dev/null +++ b/django_etebase/migrations/0027_auto_20200907_0752.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1 on 2020-09-07 07:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0026_auto_20200907_0752'), + ] + + operations = [ + migrations.AddField( + model_name='collectioninvitation', + name='accessLevel', + field=models.IntegerField(choices=[(0, 'Read Only'), (1, 'Admin'), (2, 'Read Write')], default=0), + ), + migrations.AddField( + model_name='collectionmember', + name='accessLevel', + field=models.IntegerField(choices=[(0, 'Read Only'), (1, 'Admin'), (2, 'Read Write')], default=0), + ), + ] diff --git a/django_etebase/migrations/0028_auto_20200907_0754.py b/django_etebase/migrations/0028_auto_20200907_0754.py new file mode 100644 index 0000000..cb62e63 --- /dev/null +++ b/django_etebase/migrations/0028_auto_20200907_0754.py @@ -0,0 +1,39 @@ +# Generated by Django 3.1 on 2020-09-07 07:54 + +from django.db import migrations + +from django_etebase.models import AccessLevels + + +def change_access_level_to_int(apps, schema_editor): + CollectionMember = apps.get_model('django_etebase', 'CollectionMember') + CollectionInvitation = apps.get_model('django_etebase', 'CollectionInvitation') + + for member in CollectionMember.objects.all(): + if member.accessLevelOld == 'adm': + member.accessLevel = AccessLevels.ADMIN + elif member.accessLevelOld == 'rw': + member.accessLevel = AccessLevels.READ_WRITE + elif member.accessLevelOld == 'ro': + member.accessLevel = AccessLevels.READ_ONLY + member.save() + + for invitation in CollectionInvitation.objects.all(): + if invitation.accessLevelOld == 'adm': + invitation.accessLevel = AccessLevels.ADMIN + elif invitation.accessLevelOld == 'rw': + invitation.accessLevel = AccessLevels.READ_WRITE + elif invitation.accessLevelOld == 'ro': + invitation.accessLevel = AccessLevels.READ_ONLY + invitation.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0027_auto_20200907_0752'), + ] + + operations = [ + migrations.RunPython(change_access_level_to_int), + ] diff --git a/django_etebase/migrations/0029_auto_20200907_0801.py b/django_etebase/migrations/0029_auto_20200907_0801.py new file mode 100644 index 0000000..7cd54d4 --- /dev/null +++ b/django_etebase/migrations/0029_auto_20200907_0801.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1 on 2020-09-07 08:01 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0028_auto_20200907_0754'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectioninvitation', + name='accessLevelOld', + ), + migrations.RemoveField( + model_name='collectionmember', + name='accessLevelOld', + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 7570bae..797aa0f 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -138,10 +138,10 @@ class RevisionChunkRelation(models.Model): ordering = ('id', ) -class AccessLevels(models.TextChoices): - ADMIN = 'adm' - READ_WRITE = 'rw' - READ_ONLY = 'ro' +class AccessLevels(models.IntegerChoices): + READ_ONLY = 0 + ADMIN = 1 + READ_WRITE = 2 class CollectionMember(models.Model): @@ -149,8 +149,7 @@ class CollectionMember(models.Model): collection = models.ForeignKey(Collection, related_name='members', on_delete=models.CASCADE) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) encryptionKey = models.BinaryField(editable=True, blank=False, null=False) - accessLevel = models.CharField( - max_length=3, + accessLevel = models.IntegerField( choices=AccessLevels.choices, default=AccessLevels.READ_ONLY, ) @@ -195,8 +194,7 @@ class CollectionInvitation(models.Model): user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='incoming_invitations', on_delete=models.CASCADE) signedEncryptionKey = models.BinaryField(editable=False, blank=False, null=False) - accessLevel = models.CharField( - max_length=3, + accessLevel = models.IntegerField( choices=AccessLevels.choices, default=AccessLevels.READ_ONLY, ) From a85e8168101441dfd535c3391bc893527f43f4ac Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 9 Sep 2020 17:07:32 +0300 Subject: [PATCH 219/251] User not found: return a 401 instead of a 404. --- django_etebase/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 97cb2f0..7a6ea7d 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -30,6 +30,7 @@ from rest_framework.decorators import action as action_decorator from rest_framework.response import Response from rest_framework.parsers import JSONParser, FormParser, MultiPartParser from rest_framework.renderers import BrowsableAPIRenderer +from rest_framework.exceptions import AuthenticationFailed import nacl.encoding import nacl.signing @@ -654,7 +655,11 @@ class AuthenticationViewSet(viewsets.ViewSet): def get_login_user(self, username): kwargs = {User.USERNAME_FIELD: username.lower()} - return get_object_or_404(self.get_queryset(), **kwargs) + try: + return self.get_queryset().get(**kwargs) + except User.DoesNotExist: + raise AuthenticationFailed({'code': 'user_not_found', 'detail': 'User not found'}) + def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): from datetime import datetime From 9c6a7e94282d29be9129eb85f99bebe4fa3512d2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 10 Sep 2020 13:31:54 +0300 Subject: [PATCH 220/251] Login: fix server error when trying to login to users without userinfo. --- django_etebase/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 7a6ea7d..636287e 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -656,7 +656,10 @@ class AuthenticationViewSet(viewsets.ViewSet): def get_login_user(self, username): kwargs = {User.USERNAME_FIELD: username.lower()} try: - return self.get_queryset().get(**kwargs) + user = self.get_queryset().get(**kwargs) + if not hasattr(user, 'userinfo'): + raise AuthenticationFailed({'code': 'user_not_init', 'detail': 'User not properly init'}) + return user except User.DoesNotExist: raise AuthenticationFailed({'code': 'user_not_found', 'detail': 'User not found'}) From 5785f803ac6bf676783903bc0478a99ceb3e80d4 Mon Sep 17 00:00:00 2001 From: Pierre-Alain TORET Date: Thu, 10 Sep 2020 18:51:25 +0300 Subject: [PATCH 221/251] Port over easyconfig from the etesync server code. Migrated by Tom, but kept the credit to daftaupe --- etebase-server.ini.example | 17 +++++++++++ etebase_server/settings.py | 62 +++++++++++++++++++++++++++++--------- etebase_server/utils.py | 25 +++++++++++++++ 3 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 etebase-server.ini.example create mode 100644 etebase_server/utils.py diff --git a/etebase-server.ini.example b/etebase-server.ini.example new file mode 100644 index 0000000..2b4682a --- /dev/null +++ b/etebase-server.ini.example @@ -0,0 +1,17 @@ +[global] +secret_file = secret.txt +debug = false +;Advanced options, only uncomment if you know what you're doing: +;static_root = /path/to/static +;static_url = /static/ +;media_root = /path/to/media +;media_url = /user-media/ +;language_code = en-us +;time_zone = UTC + +[allowed_hosts] +allowed_host1 = example.com + +[database] +engine = django.db.backends.sqlite3 +name = db.sqlite3 diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 94ca4d2..d5853a0 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -11,6 +11,8 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ import os +import configparser +from .utils import get_secret_from_file # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -21,14 +23,27 @@ AUTH_USER_MODEL = 'myauth.User' # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ -# Should be set in the site specific settings -# SECRET_KEY = +# SECURITY WARNING: keep the secret key used in production secret! +# See secret.py for how this is generated; uses a file 'secret.txt' in the root +# directory +SECRET_FILE = os.path.join(BASE_DIR, "secret.txt") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False ALLOWED_HOSTS = [] +# Database +# https://docs.djangoproject.com/en/2.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.environ.get('ETEBASE_DB_PATH', + os.path.join(BASE_DIR, 'db.sqlite3')), + } +} + # Application definition @@ -81,18 +96,6 @@ TEMPLATES = [ WSGI_APPLICATION = 'etebase_server.wsgi.application' -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.environ.get('ETEBASE_DB_PATH', - os.path.join(BASE_DIR, 'db.sqlite3')), - } -} - - # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -131,17 +134,46 @@ CORS_ORIGIN_ALLOW_ALL = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ -STATIC_ROOT = os.environ.get('DJANGO_STATIC_ROOT', os.path.join(BASE_DIR, 'assets')) STATIC_URL = '/static/' +STATIC_ROOT = os.environ.get('DJANGO_STATIC_ROOT', os.path.join(BASE_DIR, 'static')) MEDIA_ROOT = os.environ.get('DJANGO_MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) MEDIA_URL = '/user-media/' + +# Define where to find configuration files +config_locations = ['etebase-server.ini', '/etc/etebase-server/etebase-server.ini'] +# Use config file if present +if any(os.path.isfile(x) for x in config_locations): + config = configparser.ConfigParser() + config.read(config_locations) + + section = config['global'] + + SECRET_FILE = section.get('secret_file', SECRET_FILE) + STATIC_ROOT = section.get('static_root', STATIC_ROOT) + STATIC_URL = section.get('static_url', STATIC_URL) + MEDIA_ROOT = section.get('media_root', MEDIA_ROOT) + MEDIA_URL = section.get('media_url', MEDIA_URL) + LANGUAGE_CODE = section.get('language_code', LANGUAGE_CODE) + TIME_ZONE = section.get('time_zone', TIME_ZONE) + DEBUG = section.getboolean('debug', DEBUG) + + if 'allowed_hosts' in config: + ALLOWED_HOSTS = [y for x, y in config.items('allowed_hosts')] + + if 'database' in config: + DATABASES = { 'default': { x.upper(): y for x, y in config.items('database') } } + ETEBASE_API_PERMISSIONS = ('rest_framework.permissions.IsAuthenticated', ) ETEBASE_API_AUTHENTICATORS = ('django_etebase.token_auth.authentication.TokenAuthentication', 'rest_framework.authentication.SessionAuthentication') +# Make an `etebase_server_settings` module available to override settings. try: from etebase_server_settings import * except ImportError: pass + +if 'SECRET_KEY' not in locals(): + SECRET_KEY = get_secret_from_file(SECRET_FILE) diff --git a/etebase_server/utils.py b/etebase_server/utils.py new file mode 100644 index 0000000..21c99f2 --- /dev/null +++ b/etebase_server/utils.py @@ -0,0 +1,25 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from django.core.management import utils + +def get_secret_from_file(path): + try: + with open(path, "r") as f: + return f.read().strip() + except EnvironmentError: + with open(path, "w") as f: + secret_key = utils.get_random_secret_key() + f.write(secret_key) + return secret_key From 38e0700ac0557a8dfa14ebddf5be21da65bb1247 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 10 Sep 2020 18:54:18 +0300 Subject: [PATCH 222/251] Update django and remove unused deps. --- etebase_server/settings.py | 1 - requirements.in/base.txt | 2 -- requirements.txt | 13 +++---------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/etebase_server/settings.py b/etebase_server/settings.py index d5853a0..7af0c60 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -56,7 +56,6 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'corsheaders', 'rest_framework', - 'fullurl', 'myauth.apps.MyauthConfig', 'django_etebase.apps.DjangoEtebaseConfig', 'django_etebase.token_auth.apps.TokenAuthConfig', diff --git a/requirements.in/base.txt b/requirements.in/base.txt index d27e110..7d5bf7e 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -1,7 +1,5 @@ django -django-anymail django-cors-headers -django-fullurl djangorestframework drf-nested-routers msgpack diff --git a/requirements.txt b/requirements.txt index cd61ff1..f6c8ed4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,23 +4,16 @@ # # pip-compile --output-file=requirements.txt requirements.in/base.txt # -asgiref==3.2.3 # via django -certifi==2019.11.28 # via requests +asgiref==3.2.10 # via django cffi==1.14.0 # via pynacl -chardet==3.0.4 # via requests -django-anymail==7.0.0 # via -r requirements.in/base.txt django-cors-headers==3.2.1 # via -r requirements.in/base.txt -django-fullurl==1.0 # via -r requirements.in/base.txt -django==3.0.3 # via -r requirements.in/base.txt, django-anymail, django-cors-headers, django-fullurl, djangorestframework, drf-nested-routers +django==3.1.1 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers drf-nested-routers==0.91 # via -r requirements.in/base.txt -idna==2.8 # via requests msgpack==1.0.0 # via -r requirements.in/base.txt psycopg2-binary==2.8.4 # via -r requirements.in/base.txt pycparser==2.20 # via cffi pynacl==1.3.0 # via -r requirements.in/base.txt pytz==2019.3 # via django -requests==2.22.0 # via django-anymail -six==1.14.0 # via django-anymail, pynacl +six==1.14.0 # via pynacl sqlparse==0.3.0 # via django -urllib3==1.25.8 # via requests From b9f20d251a3be72b786844b6ba697aaa01b44158 Mon Sep 17 00:00:00 2001 From: "Prof. Jayanth R Varma" Date: Wed, 7 Nov 2018 02:18:46 +0530 Subject: [PATCH 223/251] Add example config for using using nginx with uwsgi --- example-configs/nginx-uwsgi/etesync.ini | 15 ++++++++++ .../nginx-uwsgi/my.server.name.conf | 30 +++++++++++++++++++ example-configs/nginx-uwsgi/readme.md | 20 +++++++++++++ example-configs/nginx-uwsgi/uwsgi.service | 15 ++++++++++ 4 files changed, 80 insertions(+) create mode 100644 example-configs/nginx-uwsgi/etesync.ini create mode 100644 example-configs/nginx-uwsgi/my.server.name.conf create mode 100644 example-configs/nginx-uwsgi/readme.md create mode 100644 example-configs/nginx-uwsgi/uwsgi.service diff --git a/example-configs/nginx-uwsgi/etesync.ini b/example-configs/nginx-uwsgi/etesync.ini new file mode 100644 index 0000000..e79eeee --- /dev/null +++ b/example-configs/nginx-uwsgi/etesync.ini @@ -0,0 +1,15 @@ +# uwsgi configuration file +# typical location of this file would be /etc/uwsgi/sites/etesync.ini + +[uwsgi] +socket = /path/to/etesync_server.sock +chown-socket = EtesyncUser:www-data +chmod-socket = 660 +vacuum = true + + +uid = EtesyncUser +chdir = /path/to/etesync +home = %(chdir)/.venv +module = etesync_server.wsgi +master = true diff --git a/example-configs/nginx-uwsgi/my.server.name.conf b/example-configs/nginx-uwsgi/my.server.name.conf new file mode 100644 index 0000000..b5b019d --- /dev/null +++ b/example-configs/nginx-uwsgi/my.server.name.conf @@ -0,0 +1,30 @@ +# nginx configuration for etesync server running on https://my.server.name +# typical location of this file would be /etc/nginx/sites-available/my.server.name.conf + +server { + server_name my.server.name; + + root /srv/http/etesync_server; + + client_max_body_size 5M; + + location /static { + expires 1y; + try_files $uri $uri/ =404; + } + + location / { + uwsgi_pass unix:/path/to/etesync_server.sock; + include uwsgi_params; + } + + # change 443 to say 9443 to run on a non standard port + listen 443 ssl; + listen [::]:443 ssl; + # Enable these two instead of the two above if your nginx supports http2 + # listen 443 ssl http2; + # listen [::]:443 ssl http2; + + ssl_certificate /path/to/certificate-file + ssl_certificate_key /path/to/certificate-key-file + # other ssl directives as needed diff --git a/example-configs/nginx-uwsgi/readme.md b/example-configs/nginx-uwsgi/readme.md new file mode 100644 index 0000000..dad98b6 --- /dev/null +++ b/example-configs/nginx-uwsgi/readme.md @@ -0,0 +1,20 @@ +# Running `etesync` under `nginx` and `uwsgi` + +This configuration assumes that etesync server has been installed in the home folder of a non privileged user +called `EtesyncUser` following the instructions in . Also that static +files have been collected at `/srv/http/etesync_server` by running the following commands: + + sudo mkdir -p /srv/http/etesync_server/static + sudo chown -R EtesyncUser /srv/http/etesync_server + sudo su EtesyncUser + cd /path/to/etesync + ln -s /srv/http/etesync_server/static static + ./manage.py collectstatic + +It is also assumed that `nginx` and `uwsgi` have been installed system wide by `root`, and that `nginx` is running as user/group `www-data`. + +In this setup, `uwsgi` running as a `systemd` service as `root` creates a unix socket with read-write access +to both `EtesyncUser` and `nginx`. It then drops its `root` privilege and runs `etesync` as `EtesyncUser`. + +`nginx` listens on the `https` port (or a non standard port `https` port if desired), delivers static pages directly +and for everything else, communicates with `etesync` over the unix socket. diff --git a/example-configs/nginx-uwsgi/uwsgi.service b/example-configs/nginx-uwsgi/uwsgi.service new file mode 100644 index 0000000..9941ec3 --- /dev/null +++ b/example-configs/nginx-uwsgi/uwsgi.service @@ -0,0 +1,15 @@ +# systemd unit for running uwsgi in emperor mode +# typical location of this file would be /etc/systemd/system/uwsgi.service + +[Unit] +Description=uWSGI Emperor service + +[Service] +ExecStart=/usr/local/bin/uwsgi --emperor /etc/uwsgi/sites +Restart=always +KillSignal=SIGQUIT +Type=notify +NotifyAccess=all + +[Install] +WantedBy=multi-user.target From 9efb8d4c4090be223d3a7416fe4a9aa83052fc95 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 10 Sep 2020 19:20:52 +0300 Subject: [PATCH 224/251] Update example-configs to etebase. --- example-configs/nginx-uwsgi/README.md | 22 +++++++++++++++++++ example-configs/nginx-uwsgi/etebase.ini | 15 +++++++++++++ example-configs/nginx-uwsgi/etesync.ini | 15 ------------- .../nginx-uwsgi/my.server.name.conf | 22 ++++++++++++------- example-configs/nginx-uwsgi/readme.md | 20 ----------------- 5 files changed, 51 insertions(+), 43 deletions(-) create mode 100644 example-configs/nginx-uwsgi/README.md create mode 100644 example-configs/nginx-uwsgi/etebase.ini delete mode 100644 example-configs/nginx-uwsgi/etesync.ini delete mode 100644 example-configs/nginx-uwsgi/readme.md diff --git a/example-configs/nginx-uwsgi/README.md b/example-configs/nginx-uwsgi/README.md new file mode 100644 index 0000000..55b5fa5 --- /dev/null +++ b/example-configs/nginx-uwsgi/README.md @@ -0,0 +1,22 @@ +# Running `etebase` under `nginx` and `uwsgi` + +This configuration assumes that etebase server has been installed in the home folder of a non privileged user +called `EtebaseUser` following the instructions in . Also that static +files have been collected at `/srv/http/etebase_server` by running the following commands: + +```shell +sudo mkdir -p /srv/http/etebase_server/static +sudo chown -R EtebaseUser /srv/http/etebase_server +sudo su EtebaseUser +cd /path/to/etebase +ln -s /srv/http/etebase_server/static static +./manage.py collectstatic +``` + +It is also assumed that `nginx` and `uwsgi` have been installed system wide by `root`, and that `nginx` is running as user/group `www-data`. + +In this setup, `uwsgi` running as a `systemd` service as `root` creates a unix socket with read-write access +to both `EtebaseUser` and `nginx`. It then drops its `root` privilege and runs `etebase` as `EtebaseUser`. + +`nginx` listens on the `https` port (or a non standard port `https` port if desired), delivers static pages directly +and for everything else, communicates with `etebase` over the unix socket. diff --git a/example-configs/nginx-uwsgi/etebase.ini b/example-configs/nginx-uwsgi/etebase.ini new file mode 100644 index 0000000..a2ebe97 --- /dev/null +++ b/example-configs/nginx-uwsgi/etebase.ini @@ -0,0 +1,15 @@ +# uwsgi configuration file +# typical location of this file would be /etc/uwsgi/sites/etebase.ini + +[uwsgi] +socket = /path/to/etebase_server.sock +chown-socket = EtebaseUser:www-data +chmod-socket = 660 +vacuum = true + + +uid = EtebaseUser +chdir = /path/to/etebase +home = %(chdir)/.venv +module = etebase_server.wsgi +master = true diff --git a/example-configs/nginx-uwsgi/etesync.ini b/example-configs/nginx-uwsgi/etesync.ini deleted file mode 100644 index e79eeee..0000000 --- a/example-configs/nginx-uwsgi/etesync.ini +++ /dev/null @@ -1,15 +0,0 @@ -# uwsgi configuration file -# typical location of this file would be /etc/uwsgi/sites/etesync.ini - -[uwsgi] -socket = /path/to/etesync_server.sock -chown-socket = EtesyncUser:www-data -chmod-socket = 660 -vacuum = true - - -uid = EtesyncUser -chdir = /path/to/etesync -home = %(chdir)/.venv -module = etesync_server.wsgi -master = true diff --git a/example-configs/nginx-uwsgi/my.server.name.conf b/example-configs/nginx-uwsgi/my.server.name.conf index b5b019d..6b5de6e 100644 --- a/example-configs/nginx-uwsgi/my.server.name.conf +++ b/example-configs/nginx-uwsgi/my.server.name.conf @@ -1,30 +1,36 @@ -# nginx configuration for etesync server running on https://my.server.name +# nginx configuration for etebase server running on https://my.server.name # typical location of this file would be /etc/nginx/sites-available/my.server.name.conf server { server_name my.server.name; - root /srv/http/etesync_server; + root /srv/http/etebase_server; + + client_max_body_size 20M; - client_max_body_size 5M; - location /static { expires 1y; try_files $uri $uri/ =404; } + location /media { + expires 1y; + try_files $uri $uri/ =404; + } + location / { - uwsgi_pass unix:/path/to/etesync_server.sock; + uwsgi_pass unix:/path/to/etebase_server.sock; include uwsgi_params; } # change 443 to say 9443 to run on a non standard port - listen 443 ssl; - listen [::]:443 ssl; + listen 443 ssl; + listen [::]:443 ssl; # Enable these two instead of the two above if your nginx supports http2 # listen 443 ssl http2; # listen [::]:443 ssl http2; - + ssl_certificate /path/to/certificate-file ssl_certificate_key /path/to/certificate-key-file # other ssl directives as needed +} diff --git a/example-configs/nginx-uwsgi/readme.md b/example-configs/nginx-uwsgi/readme.md deleted file mode 100644 index dad98b6..0000000 --- a/example-configs/nginx-uwsgi/readme.md +++ /dev/null @@ -1,20 +0,0 @@ -# Running `etesync` under `nginx` and `uwsgi` - -This configuration assumes that etesync server has been installed in the home folder of a non privileged user -called `EtesyncUser` following the instructions in . Also that static -files have been collected at `/srv/http/etesync_server` by running the following commands: - - sudo mkdir -p /srv/http/etesync_server/static - sudo chown -R EtesyncUser /srv/http/etesync_server - sudo su EtesyncUser - cd /path/to/etesync - ln -s /srv/http/etesync_server/static static - ./manage.py collectstatic - -It is also assumed that `nginx` and `uwsgi` have been installed system wide by `root`, and that `nginx` is running as user/group `www-data`. - -In this setup, `uwsgi` running as a `systemd` service as `root` creates a unix socket with read-write access -to both `EtesyncUser` and `nginx`. It then drops its `root` privilege and runs `etesync` as `EtesyncUser`. - -`nginx` listens on the `https` port (or a non standard port `https` port if desired), delivers static pages directly -and for everything else, communicates with `etesync` over the unix socket. From c04650f890005b9d1ba70cff1136afbb41531be8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 11 Sep 2020 16:02:47 +0300 Subject: [PATCH 225/251] README: update contribution information. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 781fd92..ee3447f 100644 --- a/README.md +++ b/README.md @@ -110,4 +110,4 @@ You can now restart the server. # Supporting Etebase -Please consider registering an account even if you self-host in order to support the development of Etebase, or help by spreading the word. +Please consider registering an account even if you self-host in order to support the development of Etebase, or visit the [contribution](https://www.etesync.com/contribute/) for more information on how to support the service. From 3de1d48b9ed43619c94e15d9f8b451e0d2e151b4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 13 Sep 2020 14:13:06 +0300 Subject: [PATCH 226/251] Browsable API: use input fields for relations. --- django_etebase/serializers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 35b3562..b683580 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -187,6 +187,7 @@ class CollectionItemRevisionSerializer(BetterErrorsMixin, serializers.ModelSeria chunks = ChunksField( source='chunks_relation', queryset=models.RevisionChunkRelation.objects.all(), + style={'base_template': 'input.html'}, many=True ) meta = BinaryBase64Field() @@ -317,6 +318,7 @@ class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer) username = UserSlugRelatedField( source='user', read_only=True, + style={'base_template': 'input.html'}, ) class Meta: @@ -341,7 +343,8 @@ class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer) class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSerializer): username = UserSlugRelatedField( source='user', - queryset=User.objects + queryset=User.objects, + style={'base_template': 'input.html'}, ) collection = serializers.CharField(source='collection.uid') fromPubkey = BinaryBase64Field(source='fromMember.user.userinfo.pubkey', read_only=True) From 00cf2d83a05440f6166918bab55316d3e95fab03 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 13 Sep 2020 14:17:25 +0300 Subject: [PATCH 227/251] Only enable browsable API when debugging is on. The reason for that is that the API may expose data that shouldn't be exposed, such as the list of users on the service. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 636287e..c5482d6 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -91,7 +91,7 @@ def msgpack_decode(content): class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) - renderer_classes = [JSONRenderer, MessagePackRenderer, BrowsableAPIRenderer] + renderer_classes = [JSONRenderer, MessagePackRenderer] + [BrowsableAPIRenderer] if settings.DEBUG else [] parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] stoken_id_fields = None From 374048f01329ad36cc42991c5ebcbeda35ce8308 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 13 Sep 2020 14:37:48 +0300 Subject: [PATCH 228/251] Fix disabling of browseable API when debug is off. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index c5482d6..971d0fe 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -91,7 +91,7 @@ def msgpack_decode(content): class BaseViewSet(viewsets.ModelViewSet): authentication_classes = tuple(app_settings.API_AUTHENTICATORS) permission_classes = tuple(app_settings.API_PERMISSIONS) - renderer_classes = [JSONRenderer, MessagePackRenderer] + [BrowsableAPIRenderer] if settings.DEBUG else [] + renderer_classes = [JSONRenderer, MessagePackRenderer] + ([BrowsableAPIRenderer] if settings.DEBUG else []) parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] stoken_id_fields = None From 4dbdb3d7cfeaaa07776a49caa1a80167c767708d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 20 Sep 2020 19:33:55 +0300 Subject: [PATCH 229/251] Invitations: gracefully error when trying to invite an already invited user. --- django_etebase/serializers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index b683580..3c32e10 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -17,7 +17,7 @@ import base64 from django.core.files.base import ContentFile from django.core import exceptions as django_exceptions from django.contrib.auth import get_user_model -from django.db import transaction +from django.db import IntegrityError, transaction from rest_framework import serializers, status from . import models from .utils import get_user_queryset, create_user @@ -368,7 +368,10 @@ class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSeriali member = collection.members.get(user=request.user) with transaction.atomic(): - return type(self).Meta.model.objects.create(**validated_data, fromMember=member) + try: + return type(self).Meta.model.objects.create(**validated_data, fromMember=member) + except IntegrityError: + raise EtebaseValidationError('invitation_exists', 'Invitation already exists') def update(self, instance, validated_data): with transaction.atomic(): From 7b8b0a568567b9046c1fbc32f73bedcae25039d5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 21 Sep 2020 12:09:19 +0300 Subject: [PATCH 230/251] Login: make case insensitive. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 971d0fe..0ba3b44 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -654,7 +654,7 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response(data, status=status.HTTP_201_CREATED) def get_login_user(self, username): - kwargs = {User.USERNAME_FIELD: username.lower()} + kwargs = {User.USERNAME_FIELD + '__iexact': username.lower()} try: user = self.get_queryset().get(**kwargs) if not hasattr(user, 'userinfo'): From 18b3f45b79c01915310b619038afdf09721669ea Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 22 Sep 2020 11:33:17 +0300 Subject: [PATCH 231/251] Collection main_item: make a OneToOneField intsead of just a foreign key. --- .../migrations/0030_auto_20200922_0832.py | 19 +++++++++++++++++++ django_etebase/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django_etebase/migrations/0030_auto_20200922_0832.py diff --git a/django_etebase/migrations/0030_auto_20200922_0832.py b/django_etebase/migrations/0030_auto_20200922_0832.py new file mode 100644 index 0000000..d5fa95d --- /dev/null +++ b/django_etebase/migrations/0030_auto_20200922_0832.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.1 on 2020-09-22 08:32 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0029_auto_20200907_0801'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='main_item', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='parent', to='django_etebase.collectionitem'), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 797aa0f..d3405f1 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -28,7 +28,7 @@ UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{20,}$', message='Not a val class Collection(models.Model): - main_item = models.ForeignKey('CollectionItem', related_name='parent', null=True, on_delete=models.SET_NULL) + main_item = models.OneToOneField('CollectionItem', related_name='parent', null=True, on_delete=models.SET_NULL) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) def __str__(self): From 5d9b47531ba99a0ee1803d6818814c9a3d1206b8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 22 Sep 2020 12:17:33 +0300 Subject: [PATCH 232/251] Collectin: make sure collections always have a unique UID. --- django_etebase/models.py | 13 +++++++++++++ django_etebase/serializers.py | 2 ++ 2 files changed, 15 insertions(+) diff --git a/django_etebase/models.py b/django_etebase/models.py index d3405f1..914b763 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -21,7 +21,10 @@ from django.db.models import Q from django.utils.functional import cached_property from django.utils.crypto import get_random_string +from rest_framework import status + from . import app_settings +from .exceptions import EtebaseValidationError UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{20,}$', message='Not a valid UID') @@ -57,6 +60,16 @@ class Collection(models.Model): return stoken.uid + def validate_unique(self, exclude=None): + super().validate_unique(exclude=exclude) + if exclude is None or 'main_item' in exclude: + return + + if self.__class__.objects.filter(owner=self.owner, main_item__uid=self.main_item.uid) \ + .exclude(id=self.id).exists(): + raise EtebaseValidationError('unique_uid', 'Collection with this uid already exists', + status_code=status.HTTP_409_CONFLICT) + class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 3c32e10..d371f13 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -297,6 +297,8 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) instance.main_item = main_item + + instance.full_clean() instance.save() process_revisions_for_item(main_item, revision_data) From 5c803d8a51e5d02e3a321cc5de1ff68245f78df8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 22 Sep 2020 18:00:28 +0300 Subject: [PATCH 233/251] Only expose drf's auth in debug mode. --- etebase_server/urls.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/etebase_server/urls.py b/etebase_server/urls.py index 0c114af..fddc32f 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.conf.urls import include, url from django.contrib import admin from django.urls import path @@ -6,7 +7,11 @@ from django.views.generic import TemplateView urlpatterns = [ url(r'^api/', include('django_etebase.urls')), url(r'^admin/', admin.site.urls), - url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), path('', TemplateView.as_view(template_name='success.html')), ] + +if settings.DEBUG: + urlpatterns += [ + url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), + ] From f5ced873ac659b1992bf0f3f9bd2e3b3b0b95099 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 23 Sep 2020 16:27:20 +0300 Subject: [PATCH 234/251] Lint: fix lint errors. --- django_etebase/serializers.py | 6 ++++-- django_etebase/views.py | 7 +++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index d371f13..4518f58 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -220,7 +220,8 @@ class CollectionItemSerializer(BetterErrorsMixin, serializers.ModelSerializer): cur_etag = instance.etag if not created else None if validate_etag and cur_etag != etag: - raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(cur_etag, etag), status_code=status.HTTP_409_CONFLICT) + raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(cur_etag, etag), + status_code=status.HTTP_409_CONFLICT) if not created: # We don't have to use select_for_update here because the unique constraint on current guards against @@ -249,7 +250,8 @@ class CollectionItemDepSerializer(BetterErrorsMixin, serializers.ModelSerializer item = self.__class__.Meta.model.objects.get(uid=data['uid']) etag = data['etag'] if item.etag != etag: - raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(item.etag, etag), status_code=status.HTTP_409_CONFLICT) + raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(item.etag, etag), + status_code=status.HTTP_409_CONFLICT) return data diff --git a/django_etebase/views.py b/django_etebase/views.py index 0ba3b44..ec3b373 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -13,13 +13,12 @@ # along with this program. If not, see . import msgpack -from functools import reduce from django.conf import settings from django.contrib.auth import get_user_model, user_logged_in, user_logged_out from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError -from django.db.models import Max, Q, F, Value as V +from django.db.models import Max, Value as V from django.db.models.functions import Coalesce, Greatest from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -663,7 +662,6 @@ class AuthenticationViewSet(viewsets.ViewSet): except User.DoesNotExist: raise AuthenticationFailed({'code': 'user_not_found', 'detail': 'User not found'}) - def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): from datetime import datetime @@ -698,7 +696,8 @@ class AuthenticationViewSet(viewsets.ViewSet): try: verify_key.verify(response_raw, signature) except nacl.exceptions.BadSignatureError: - return Response({'code': 'login_bad_signature', 'detail': 'Wrong password for user.'}, status=status.HTTP_401_UNAUTHORIZED) + return Response({'code': 'login_bad_signature', 'detail': 'Wrong password for user.'}, + status=status.HTTP_401_UNAUTHORIZED) return None From 8a557ff82cd4b9147915f7aec7a7e8ae51dbcbf0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Sep 2020 09:42:01 +0300 Subject: [PATCH 235/251] Disable signups by default. The next commit includes README instructions on how to create users and enable signups. --- django_etebase/utils.py | 6 ++++++ etebase_server/settings.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 08f81ae..1351f9b 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -1,4 +1,6 @@ from django.contrib.auth import get_user_model +from django.core.exceptions import PermissionDenied + from . import app_settings @@ -18,3 +20,7 @@ def create_user(*args, **kwargs): return custom_func(*args, **kwargs) _ = kwargs.pop('view') return User.objects.create_user(*args, **kwargs) + + +def create_user_blocked(*args, **kwargs): + raise PermissionDenied('Signup is disabled for this server. Please refer to the README for more information.') diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 7af0c60..f785cb7 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -29,7 +29,7 @@ AUTH_USER_MODEL = 'myauth.User' SECRET_FILE = os.path.join(BASE_DIR, "secret.txt") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = [] @@ -167,6 +167,7 @@ if any(os.path.isfile(x) for x in config_locations): ETEBASE_API_PERMISSIONS = ('rest_framework.permissions.IsAuthenticated', ) ETEBASE_API_AUTHENTICATORS = ('django_etebase.token_auth.authentication.TokenAuthentication', 'rest_framework.authentication.SessionAuthentication') +ETEBASE_CREATE_USER_FUNC = 'django_etebase.utils.create_user_blocked' # Make an `etebase_server_settings` module available to override settings. try: From 1e7e9eceacaae4a2c8c291c7d3c41227e1915925 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Sep 2020 09:45:31 +0300 Subject: [PATCH 236/251] README: update signup instructions to EteSync 2.0. Fixes #55. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ee3447f..80396b6 100644 --- a/README.md +++ b/README.md @@ -79,12 +79,12 @@ Create yourself an admin user: ./manage.py createsuperuser ``` -At this stage you can either just use the admin user, or better yet, go to: ```www.your-etesync-install.com/admin``` -and create a non-privileged user that you can use. +At this stage you need to create accounts to be used with the EteSync apps. To do that, please go to: +`www.your-etesync-install.com/admin` and create a new user to be used with the service. -That's it! - -Now all that's left is to open the EteSync app, add an account, and set your custom server address under the "advance" section. +After this user has been created, you can use any of the EteSync apps to signup (not login!) with the same username and +email in order to set up the account. Please make sure to click "advance" and set your customer server address when you +do. # `SECRET_KEY` and `secret.txt` From c9983fd79dc13a4a5b53ff31fe2533e00404312a Mon Sep 17 00:00:00 2001 From: Simon Vandevelde Date: Sun, 27 Sep 2020 16:48:52 +0200 Subject: [PATCH 237/251] Update README for Etebase with new wiki links (#56) --- README.md | 26 +++--- icon.svg | 241 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 11 deletions(-) create mode 100644 icon.svg diff --git a/README.md b/README.md index 80396b6..6e28a43 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

-

EteSync - Secure Data Sync

+

Etebase - Encrypt Everything

A skeleton app for running your own [Etebase](https://www.etebase.com) server @@ -9,7 +9,7 @@ A skeleton app for running your own [Etebase](https://www.etebase.com) server ## From source -Before installing the EteSync server make sure you install `virtualenv` (for **Python 3**): +Before installing the Etebase server make sure you install `virtualenv` (for **Python 3**): * Arch Linux: `pacman -S python-virtualenv` * Debian/Ubuntu: `apt-get install python3-virtualenv` @@ -18,9 +18,10 @@ Before installing the EteSync server make sure you install `virtualenv` (for **P Then just clone the git repo and set up this app: ``` -git clone https://github.com/etesync/server-skeleton.git +git clone https://github.com/etesync/server.git etebase -cd server-skeleton +cd etebase +git checkout etebase # Set up the environment and deps virtualenv -p python3 venv # If doesn't work, try: virtualenv3 venv @@ -32,8 +33,11 @@ pip install -r requirements.txt # Configuration If you are familiar with Django you can just edit the [settings file](etesync_server/settings.py) -according to the [Django deployment checklist](https://docs.djangoproject.com/en/dev/howto/deployment/checklist/) -if you are not, we will soon provide a simple configuration file for easy deployment like we had with EteSync. +according to the [Django deployment checklist](https://docs.djangoproject.com/en/dev/howto/deployment/checklist/). +If you are not, we also provide a simple [configuration file](https://github.com/etesync/server/blob/etebase/etebase-server.ini.example) for easy deployment which you can use. +To use the easy configuration file rename it to `etebase-server.ini` and place it either at the root of this repository or in `/etc/etebase-server`. + +There is also a [wikipage](https://github.com/etesync/server/wiki/Basic-Setup-Etebase-(EteSync-v2)) detailing this basic setup. Some particular settings that should be edited are: * [`ALLOWED_HOSTS`](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-ALLOWED_HOSTS) @@ -46,7 +50,7 @@ will be served generation purposes. See below for how default configuration of `SECRET_KEY` works for this project. -Now you can initialise our django app +Now you can initialise our django app. ``` ./manage.py migrate @@ -62,14 +66,14 @@ Using the debug server in production is not recommended, so please read the foll # Production deployment -EteSync is based on Django so you should refer to one of the following +There are more details about a proper production setup using Daphne and Nginx in the [wiki](https://github.com/etesync/server/wiki/Production-setup-using-Daphne-and-Nginx). + +Etebase is based on Django so you should refer to one of the following * The instructions of the Django project [here](https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/). * Instructions from uwsgi [here](http://uwsgi-docs.readthedocs.io/en/latest/tutorials/Django_and_nginx.html). -There are more details about a proper production setup using uWSGI and Nginx in the [wiki](https://github.com/etesync/server/wiki/Production-setup-using-uWSGI-and-Nginx). - The webserver should also be configured to serve Etebase using TLS. -A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-EteSync) as well. +A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-Etebase) as well. # Usage diff --git a/icon.svg b/icon.svg new file mode 100644 index 0000000..4827d1d --- /dev/null +++ b/icon.svg @@ -0,0 +1,241 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 62146881703a0c493f82367ac065395919e5b468 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Sep 2020 14:56:29 +0300 Subject: [PATCH 238/251] Invitations: share the username of the inviter. --- django_etebase/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 4518f58..4798f97 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -351,12 +351,14 @@ class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSeriali style={'base_template': 'input.html'}, ) collection = serializers.CharField(source='collection.uid') + fromUsername = BinaryBase64Field(source='fromMember.user.username', read_only=True) fromPubkey = BinaryBase64Field(source='fromMember.user.userinfo.pubkey', read_only=True) signedEncryptionKey = BinaryBase64Field() class Meta: model = models.CollectionInvitation - fields = ('username', 'uid', 'collection', 'signedEncryptionKey', 'accessLevel', 'fromPubkey', 'version') + fields = ('username', 'uid', 'collection', 'signedEncryptionKey', 'accessLevel', + 'fromUsername', 'fromPubkey', 'version') def validate_user(self, value): request = self.context['request'] From 06f2dd72a7458769355f48516f3f3feda4d53562 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 1 Oct 2020 16:45:47 +0300 Subject: [PATCH 239/251] Exception: fix detail/code for exception. --- django_etebase/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index ec3b373..31e914a 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -574,7 +574,8 @@ class InvitationOutgoingViewSet(InvitationBaseViewSet): return Response(content, status=status.HTTP_400_BAD_REQUEST) if not permissions.is_collection_admin(collection, request.user): - raise PermissionDenied('User is not an admin of this collection') + raise PermissionDenied({'code': 'admin_access_required', + 'detail': 'User is not an admin of this collection'}) serializer.save(collection=collection) From 9152e6f42d4afe8166b864170ea708f071441639 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 8 Oct 2020 21:01:45 +0300 Subject: [PATCH 240/251] Fix bad stoken error. We were calling the validation constructor wrong. --- django_etebase/views.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 31e914a..6003d4b 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -116,11 +116,7 @@ class BaseViewSet(viewsets.ModelViewSet): try: return Stoken.objects.get(uid=stoken) except Stoken.DoesNotExist: - raise EtebaseValidationError({ - 'code': 'bad_stoken', - 'detail': 'Invalid stoken.', - }, - status_code=status.HTTP_400_BAD_REQUEST) + raise EtebaseValidationError('bad_stoken', 'Invalid stoken.', status_code=status.HTTP_400_BAD_REQUEST) return None From 74f40abc65567674448bb5c4dfec4cc835d1a523 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 8 Oct 2020 21:03:54 +0300 Subject: [PATCH 241/251] Account: add a dashboard url endpoint. This lets servers share a dashboard url with clients so that they in turn can present clients with a settings dashboard. We currently use it on the main server, but self-hosted servers may also benefit from it for letting users manage some of their settings (e.g. 2FA). --- django_etebase/app_settings.py | 7 +++++++ django_etebase/views.py | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index 33dc65f..3c580b2 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -61,6 +61,13 @@ class AppSettings: return self.import_from_str(func) return None + @cached_property + def DASHBOARD_URL_FUNC(self): # pylint: disable=invalid-name + func = self._setting("DASHBOARD_URL_FUNC", None) + if func is not None: + return self.import_from_str(func) + return None + @cached_property def CHUNK_PATH_FUNC(self): # pylint: disable=invalid-name func = self._setting("CHUNK_PATH_FUNC", None) diff --git a/django_etebase/views.py b/django_etebase/views.py index 6003d4b..d421e43 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -783,6 +783,18 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response({}, status=status.HTTP_200_OK) + @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) + def dashboard_url(self, request, *args, **kwargs): + get_dashboard_url = app_settings.DASHBOARD_URL_FUNC + if get_dashboard_url is None: + raise EtebaseValidationError('not_supported', 'This server doesn\'t have a user dashboard.', + status_code=status.HTTP_400_BAD_REQUEST) + + ret = { + 'url': get_dashboard_url(request, *args, **kwargs), + } + return Response(ret) + class TestAuthenticationViewSet(viewsets.ViewSet): allowed_methods = ['POST'] From 9cad5d62e1518502ca149d77a8c254fba98eff89 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Fri, 9 Oct 2020 13:10:41 +0300 Subject: [PATCH 242/251] Account: change Dashboard URL endpoint's permissions. We only want to require that the account is authenticated, not the rest of the permissions. As we want to be able to get a dashboard url for accounts that aren't currently valid. --- django_etebase/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index d421e43..8c55366 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -30,6 +30,7 @@ from rest_framework.response import Response from rest_framework.parsers import JSONParser, FormParser, MultiPartParser from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.exceptions import AuthenticationFailed +from rest_framework.permissions import IsAuthenticated import nacl.encoding import nacl.signing @@ -783,7 +784,7 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response({}, status=status.HTTP_200_OK) - @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) + @action_decorator(detail=False, methods=['POST'], permission_classes=[IsAuthenticated]) def dashboard_url(self, request, *args, **kwargs): get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: From 24c161b0d84d59572ec342631416607ebc197e0e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 13 Oct 2020 11:09:22 +0300 Subject: [PATCH 243/251] Signup: don't try to clean fields for objects we haven't created. --- django_etebase/serializers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 4798f97..5cc24aa 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -462,19 +462,17 @@ class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): # Create the user and save the casing the user chose as the first name try: instance = create_user(**user_data, password=None, first_name=user_data['username'], view=view) + instance.clean_fields() except EtebaseValidationError as e: raise e + except django_exceptions.ValidationError as e: + self.transform_validation_error("user", e) except Exception as e: raise EtebaseValidationError('generic', str(e)) if hasattr(instance, 'userinfo'): raise EtebaseValidationError('user_exists', 'User already exists', status_code=status.HTTP_409_CONFLICT) - try: - instance.clean_fields() - except django_exceptions.ValidationError as e: - self.transform_validation_error("user", e) - models.UserInfo.objects.create(**validated_data, owner=instance) return instance From 47f3e088464bdc623a41139331085e3f2026e284 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 13 Oct 2020 11:10:55 +0300 Subject: [PATCH 244/251] Signup: improve docs. --- django_etebase/serializers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 5cc24aa..759536b 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -443,6 +443,9 @@ class UserSignupSerializer(BetterErrorsMixin, serializers.ModelSerializer): class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): + """Used both for creating new accounts and setting up existing ones for the first time. + When setting up existing ones the email is ignored." + """ user = UserSignupSerializer(many=False) salt = BinaryBase64Field() loginPubkey = BinaryBase64Field() From c7bd01b2d12ac437ddab95283fcd2eb8085ea5a4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 13 Oct 2020 12:09:29 +0300 Subject: [PATCH 245/251] Logout: allow any authenticated user (instead of normal permissions). We should always allow users to log out if they are authenticated. This doesn't need to use the global permissions. --- django_etebase/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 8c55366..2d9b76c 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -756,7 +756,7 @@ class AuthenticationViewSet(viewsets.ViewSet): return Response(data, status=status.HTTP_200_OK) - @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) + @action_decorator(detail=False, methods=['POST'], permission_classes=[IsAuthenticated]) def logout(self, request, *args, **kwargs): request.auth.delete() user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) From aa7b049b62e42316d0a342b3b6edbe066761f53b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 13 Oct 2020 13:37:05 +0300 Subject: [PATCH 246/251] Stoken: always return the stoken object, not the rev. --- django_etebase/views.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 2d9b76c..897bcb5 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -138,15 +138,14 @@ class BaseViewSet(viewsets.ModelViewSet): for row in queryset: rowmaxid = getattr(row, 'max_stoken') or -1 maxid = max(maxid, rowmaxid) - new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid).uid + new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) - return new_stoken + return new_stoken or None def filter_by_stoken_and_limit(self, request, queryset): limit = int(request.GET.get('limit', 50)) queryset, stoken_rev = self.filter_by_stoken(request, queryset) - stoken = stoken_rev.uid if stoken_rev is not None else None result = list(queryset[:limit + 1]) if len(result) < limit + 1: @@ -155,9 +154,9 @@ class BaseViewSet(viewsets.ModelViewSet): done = False result = result[:-1] - new_stoken = self.get_queryset_stoken(result) or stoken + new_stoken_obj = self.get_queryset_stoken(result) or stoken_rev - return result, new_stoken, done + return result, new_stoken_obj, done # Change how our list works by default def list(self, request, collection_uid=None, *args, **kwargs): @@ -211,7 +210,8 @@ class CollectionViewSet(BaseViewSet): def list(self, request, *args, **kwargs): queryset = self.get_queryset() - result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) + result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid serializer = self.get_serializer(result, many=True) @@ -278,7 +278,8 @@ class CollectionItemViewSet(BaseViewSet): if not self.request.query_params.get('withCollection', False): queryset = queryset.filter(parent__isnull=True) - result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) + result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid serializer = self.get_serializer(result, many=True) @@ -342,8 +343,9 @@ class CollectionItemViewSet(BaseViewSet): revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) - new_stoken = self.get_queryset_stoken(queryset) - stoken = getattr(stoken_rev, 'uid', None) if stoken_rev is not None else None + new_stoken_obj = self.get_queryset_stoken(queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + stoken = stoken_rev and getattr(stoken_rev, 'uid', None) new_stoken = new_stoken or stoken serializer = self.get_serializer(queryset, many=True) @@ -481,7 +483,8 @@ class CollectionMemberViewSet(BaseViewSet): def list(self, request, collection_uid=None, *args, **kwargs): queryset = self.get_queryset().order_by('id') - result, new_stoken, done = self.filter_by_stoken_and_limit(request, queryset) + result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid serializer = self.get_serializer(result, many=True) ret = { From 741b6d7c52dbe785dd3d5e94025a7c1dc23f6dff Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 13 Oct 2020 13:29:29 +0300 Subject: [PATCH 247/251] Collection removed memberships: only return removed memberships within our returned range. Before this change we were returning all of the removed memberships that happened after stoken. Though instead, we should just return the removed memberships that happened after stoken and before the new stoken we are returning. --- django_etebase/views.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/django_etebase/views.py b/django_etebase/views.py index 897bcb5..c7162dd 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -224,8 +224,13 @@ class CollectionViewSet(BaseViewSet): stoken_obj = self.get_stoken_obj(request) if stoken_obj is not None: # FIXME: honour limit? (the limit should be combined for data and this because of stoken) - remed = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) \ - .values_list('collection__main_item__uid', flat=True) + remed_qs = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) + if not ret['done']: + # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken + # can point to the most recent collection change rather than most recent removed membership. + remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) + + remed = remed_qs.values_list('collection__main_item__uid', flat=True) if len(remed) > 0: ret['removedMemberships'] = [{'uid': x} for x in remed] From acd22b9b47d7a899e676c921abfec81f86dd8533 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 13 Oct 2020 16:30:16 +0300 Subject: [PATCH 248/251] Serializers: remove unused field. --- django_etebase/serializers.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 759536b..fe9ae11 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -86,14 +86,6 @@ class CollectionEncryptionKeyField(BinaryBase64Field): return None -class CollectionContentField(BinaryBase64Field): - def get_attribute(self, instance): - request = self.context.get('request', None) - if request is not None: - return instance.members.get(user=request.user).encryptionKey - return None - - class UserSlugRelatedField(serializers.SlugRelatedField): def get_queryset(self): view = self.context.get('view', None) From 5d8a92f0001c6595826c2def2af25d509c69ef52 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 13 Oct 2020 17:13:07 +0300 Subject: [PATCH 249/251] Collections: add support for collection types. We also added the field for invitations, as it's needed for collections to work. --- .../migrations/0031_auto_20201013_1336.py | 29 ++++++++++++++++ .../migrations/0032_auto_20201013_1409.py | 18 ++++++++++ django_etebase/models.py | 6 ++++ django_etebase/serializers.py | 34 +++++++++++++++++-- django_etebase/views.py | 16 +++++++++ 5 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 django_etebase/migrations/0031_auto_20201013_1336.py create mode 100644 django_etebase/migrations/0032_auto_20201013_1409.py diff --git a/django_etebase/migrations/0031_auto_20201013_1336.py b/django_etebase/migrations/0031_auto_20201013_1336.py new file mode 100644 index 0000000..ca45dd4 --- /dev/null +++ b/django_etebase/migrations/0031_auto_20201013_1336.py @@ -0,0 +1,29 @@ +# Generated by Django 3.1.1 on 2020-10-13 13:36 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('django_etebase', '0030_auto_20200922_0832'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.BinaryField(db_index=True, editable=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.AddField( + model_name='collectionmember', + name='collectionType', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.collectiontype'), + ), + ] diff --git a/django_etebase/migrations/0032_auto_20201013_1409.py b/django_etebase/migrations/0032_auto_20201013_1409.py new file mode 100644 index 0000000..5594006 --- /dev/null +++ b/django_etebase/migrations/0032_auto_20201013_1409.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2020-10-13 14:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0031_auto_20201013_1336'), + ] + + operations = [ + migrations.AlterField( + model_name='collectiontype', + name='uid', + field=models.BinaryField(db_index=True, editable=True, unique=True), + ), + ] diff --git a/django_etebase/models.py b/django_etebase/models.py index 914b763..0036884 100644 --- a/django_etebase/models.py +++ b/django_etebase/models.py @@ -30,6 +30,11 @@ from .exceptions import EtebaseValidationError UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{20,}$', message='Not a valid UID') +class CollectionType(models.Model): + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + uid = models.BinaryField(editable=True, blank=False, null=False, db_index=True, unique=True) + + class Collection(models.Model): main_item = models.OneToOneField('CollectionItem', related_name='parent', null=True, on_delete=models.SET_NULL) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) @@ -162,6 +167,7 @@ class CollectionMember(models.Model): collection = models.ForeignKey(Collection, related_name='members', on_delete=models.CASCADE) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) encryptionKey = models.BinaryField(editable=True, blank=False, null=False) + collectionType = models.ForeignKey(CollectionType, on_delete=models.PROTECT, null=True) accessLevel = models.IntegerField( choices=AccessLevels.choices, default=AccessLevels.READ_ONLY, diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index fe9ae11..58ad10c 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -86,6 +86,15 @@ class CollectionEncryptionKeyField(BinaryBase64Field): return None +class CollectionTypeField(BinaryBase64Field): + def get_attribute(self, instance): + request = self.context.get('request', None) + if request is not None: + collection_type = instance.members.get(user=request.user).collectionType + return collection_type and collection_type.uid + return None + + class UserSlugRelatedField(serializers.SlugRelatedField): def get_queryset(self): view = self.context.get('view', None) @@ -256,8 +265,15 @@ class CollectionItemBulkGetSerializer(BetterErrorsMixin, serializers.ModelSerial fields = ('uid', 'etag') +class CollectionListMultiSerializer(BetterErrorsMixin, serializers.Serializer): + collectionTypes = serializers.ListField( + child=BinaryBase64Field() + ) + + class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): collectionKey = CollectionEncryptionKeyField() + collectionType = CollectionTypeField() accessLevel = serializers.SerializerMethodField('get_access_level_from_context') stoken = serializers.CharField(read_only=True) @@ -265,7 +281,7 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): class Meta: model = models.Collection - fields = ('item', 'accessLevel', 'collectionKey', 'stoken') + fields = ('item', 'accessLevel', 'collectionKey', 'collectionType', 'stoken') def get_access_level_from_context(self, obj): request = self.context.get('request', None) @@ -276,6 +292,7 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): def create(self, validated_data): """Function that's called when this serializer creates an item""" collection_key = validated_data.pop('collectionKey') + collection_type = validated_data.pop('collectionType') main_item_data = validated_data.pop('main_item') etag = main_item_data.pop('etag') @@ -297,11 +314,16 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): process_revisions_for_item(main_item, revision_data) + user = validated_data.get('owner') + + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) + models.CollectionMember(collection=instance, stoken=models.Stoken.objects.create(), - user=validated_data.get('owner'), + user=user, accessLevel=models.AccessLevels.ADMIN, encryptionKey=collection_key, + collectionType=collection_type_obj, ).save() return instance @@ -381,6 +403,7 @@ class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSeriali class InvitationAcceptSerializer(BetterErrorsMixin, serializers.Serializer): + collectionType = BinaryBase64Field() encryptionKey = BinaryBase64Field() def create(self, validated_data): @@ -388,13 +411,18 @@ class InvitationAcceptSerializer(BetterErrorsMixin, serializers.Serializer): with transaction.atomic(): invitation = self.context['invitation'] encryption_key = validated_data.get('encryptionKey') + collection_type = validated_data.pop('collectionType') + + user = invitation.user + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) member = models.CollectionMember.objects.create( collection=invitation.collection, stoken=models.Stoken.objects.create(), - user=invitation.user, + user=user, accessLevel=invitation.accessLevel, encryptionKey=encryption_key, + collectionType=collection_type_obj, ) models.CollectionMemberRemoved.objects.filter( diff --git a/django_etebase/views.py b/django_etebase/views.py index c7162dd..34f6254 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -66,6 +66,7 @@ from .serializers import ( CollectionItemDepSerializer, CollectionItemRevisionSerializer, CollectionItemChunkSerializer, + CollectionListMultiSerializer, CollectionMemberSerializer, CollectionInvitationSerializer, InvitationAcceptSerializer, @@ -210,6 +211,21 @@ class CollectionViewSet(BaseViewSet): def list(self, request, *args, **kwargs): queryset = self.get_queryset() + return self.list_common(request, queryset, *args, **kwargs) + + @action_decorator(detail=False, methods=['POST']) + def list_multi(self, request, *args, **kwargs): + serializer = CollectionListMultiSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + collection_types = serializer.validated_data['collectionTypes'] + + queryset = self.get_queryset() + queryset = queryset.filter(members__collectionType__uid__in=collection_types) + + return self.list_common(request, queryset, *args, **kwargs) + + def list_common(self, request, queryset, *args, **kwargs): result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) new_stoken = new_stoken_obj and new_stoken_obj.uid From 409248d419aa0fbbc23fad4ec163c95fb59ebad5 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 15 Oct 2020 10:50:07 +0300 Subject: [PATCH 250/251] CollectionTypes: add backward compatibility adjustments until 2.0 is out. --- django_etebase/serializers.py | 3 ++- django_etebase/views.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index 58ad10c..d753bf9 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -273,7 +273,8 @@ class CollectionListMultiSerializer(BetterErrorsMixin, serializers.Serializer): class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): collectionKey = CollectionEncryptionKeyField() - collectionType = CollectionTypeField() + # FIXME: make required once "collection-type-migration" is done + collectionType = CollectionTypeField(required=False) accessLevel = serializers.SerializerMethodField('get_access_level_from_context') stoken = serializers.CharField(read_only=True) diff --git a/django_etebase/views.py b/django_etebase/views.py index 34f6254..5328c84 100644 --- a/django_etebase/views.py +++ b/django_etebase/views.py @@ -18,7 +18,7 @@ from django.conf import settings from django.contrib.auth import get_user_model, user_logged_in, user_logged_out from django.core.exceptions import PermissionDenied from django.db import transaction, IntegrityError -from django.db.models import Max, Value as V +from django.db.models import Max, Value as V, Q from django.db.models.functions import Coalesce, Greatest from django.http import HttpResponseBadRequest, HttpResponse, Http404 from django.shortcuts import get_object_or_404 @@ -221,7 +221,9 @@ class CollectionViewSet(BaseViewSet): collection_types = serializer.validated_data['collectionTypes'] queryset = self.get_queryset() - queryset = queryset.filter(members__collectionType__uid__in=collection_types) + # FIXME: Remove the isnull part once "collection-type-migration" is done + queryset = queryset.filter( + Q(members__collectionType__uid__in=collection_types) | Q(members__collectionType__isnull=True)) return self.list_common(request, queryset, *args, **kwargs) From 5bce4d9932960e45b9ebefff94cfc1553e63761f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 15 Oct 2020 15:00:20 +0300 Subject: [PATCH 251/251] Collection Type: fix backwards compatibility for creating new collections. Continuation to 409248d419aa0fbbc23fad4ec163c95fb59ebad5. --- django_etebase/serializers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py index d753bf9..038e879 100644 --- a/django_etebase/serializers.py +++ b/django_etebase/serializers.py @@ -293,7 +293,8 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): def create(self, validated_data): """Function that's called when this serializer creates an item""" collection_key = validated_data.pop('collectionKey') - collection_type = validated_data.pop('collectionType') + # FIXME: remove the None fallback once "collection-type-migration" is done + collection_type = validated_data.pop('collectionType', None) main_item_data = validated_data.pop('main_item') etag = main_item_data.pop('etag') @@ -317,7 +318,11 @@ class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): user = validated_data.get('owner') - collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) + # FIXME: remove the if statement (and else branch) once "collection-type-migration" is done + if collection_type is not None: + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) + else: + collection_type_obj = None models.CollectionMember(collection=instance, stoken=models.Stoken.objects.create(),