More progress - support chunk uploading.

django_etesync/migrations/0008_collectionitemchunk_chunkfile.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.0.3 on 2020-02-20 12:16
+
+from django.db import migrations, models
+import django_etesync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_etesync', '0007_auto_20200220_1144'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='collectionitemchunk',
+            name='chunkFile',
+            field=models.FileField(default='', upload_to=django_etesync.models.chunk_directory_path),
+            preserve_default=False,
+        ),
+    ]

django_etesync/migrations/0009_auto_20200220_1220.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.3 on 2020-02-20 12:20
+
+from django.db import migrations, models
+import django_etesync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_etesync', '0008_collectionitemchunk_chunkfile'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collectionitemchunk',
+            name='chunkFile',
+            field=models.FileField(max_length=150, upload_to=django_etesync.models.chunk_directory_path),
+        ),
+    ]

django_etesync/models.py

@@ -36,12 +36,6 @@ class Collection(models.Model):
         return self.uid
 
 
-def chunk_directory_path(instance, filename):
-    col = instance.itemSnapshot.item.collection
-    user_id = col.owner.id
-    return Path('user_{}'.format(user_id), col.uid, instance.uid)
-
-
 
 class CollectionItem(models.Model):
     uid = models.CharField(db_index=True, blank=False, null=False,
@@ -59,12 +53,19 @@ class CollectionItem(models.Model):
         return self.snapshots.get(current=True)
 
 
+def chunk_directory_path(instance, filename):
+    item = instance.item
+    col = item.collection
+    user_id = col.owner.id
+    return Path('user_{}'.format(user_id), col.uid, item.uid, instance.uid)
+
+
 class CollectionItemChunk(models.Model):
     uid = models.CharField(db_index=True, blank=False, null=False,
                            max_length=44, validators=[UidValidator])
     item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE)
     order = models.CharField(max_length=100, blank=False, null=False)
-    # We probably just want to implement this manually because we can have more than one pointing to a file. chunkFile = models.FileField(upload_to=chunk_directory_path)
+    chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150)
 
     class Meta:
         unique_together = ('item', 'order')
@@ -87,4 +88,3 @@ class CollectionItemSnapshot(models.Model):
 
     def __str__(self):
         return '{} {} current={}'.format(self.item.uid, self.id, self.current)
-

django_etesync/serializers.py

@@ -61,10 +61,10 @@ class CollectionSerializer(serializers.ModelSerializer):
 class CollectionItemChunkSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.CollectionItemChunk
-        fields = ('uid', )
+        fields = ('uid', 'chunkFile')
 
 
-class CollectionItemSnapshotSerializer(serializers.ModelSerializer):
+class CollectionItemSnapshotBaseSerializer(serializers.ModelSerializer):
     encryptionKey = BinaryBase64Field()
     chunks = serializers.SlugRelatedField(
         slug_field='uid',
@@ -77,18 +77,36 @@ class CollectionItemSnapshotSerializer(serializers.ModelSerializer):
         fields = ('version', 'encryptionKey', 'chunks', 'hmac')
 
 
-class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotSerializer):
-    chunksData = serializers.SerializerMethodField('get_inline_chunks_from_context')
+class CollectionItemSnapshotSerializer(CollectionItemSnapshotBaseSerializer):
+    chunksUrls = serializers.SerializerMethodField('get_chunks_urls')
 
-    class Meta(CollectionItemSnapshotSerializer.Meta):
-        fields = CollectionItemSnapshotSerializer.Meta.fields + ('chunksData', )
+    class Meta(CollectionItemSnapshotBaseSerializer.Meta):
+        fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksUrls', )
 
-    def get_inline_chunks_from_context(self, obj):
-        request = self.context.get('request', None)
-        if request is not None:
-            return ['SomeInlineData', 'Somemoredata']
-        return 'readOnly'
+    # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still save it under the user.
+    # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot.
+    # E.g. chunkBaseUrl: "/media/bla/bla/" or chunkBaseUrl: "https://media.etesync.com/bla/bla"
+    def get_chunks_urls(self, obj):
+        ret = []
+        for chunk in obj.chunks.all():
+            ret.append(chunk.chunkFile.url)
+
+        return ret
+
+
+class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotBaseSerializer):
+    chunksData = serializers.SerializerMethodField('get_chunks_data')
+
+    class Meta(CollectionItemSnapshotBaseSerializer.Meta):
+        fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksData', )
+
+    def get_chunks_data(self, obj):
+        ret = []
+        for chunk in obj.chunks.all():
+            with open(chunk.chunkFile.path, 'rb') as f:
+                ret.append(base64.b64encode(f.read()).decode('ascii'))
 
+        return ret
 
 class CollectionItemSerializer(serializers.ModelSerializer):
     content = CollectionItemSnapshotSerializer(read_only=True, many=False)

django_etesync/views.py

@@ -148,20 +148,21 @@ class CollectionItemViewSet(BaseViewSet):
 
 class CollectionItemChunkViewSet(viewsets.ViewSet):
     allowed_methods = ['GET', 'POST']
+    parser_classes = (parsers.MultiPartParser, )
     authentication_classes = BaseViewSet.authentication_classes
     permission_classes = BaseViewSet.permission_classes
-    parser_classes = (parsers.MultiPartParser, )
+    serializer_class = CollectionItemChunkSerializer
     lookup_field = 'uid'
 
-    def create(self, request, collection_uid=None):
+    def create(self, request, collection_uid=None, collection_item_uid=None):
         # FIXME: we are potentially not getting the correct queryset
-        collection_object = Collection.objects.get(uid=collection_uid)
+        col = get_object_or_404(Collection.objects, uid=collection_uid)
+        col_it = get_object_or_404(col.items, uid=collection_item_uid)
 
-        many = isinstance(request.data, list)
-        serializer = self.serializer_class(data=request.data, many=many)
+        serializer = self.serializer_class(data=request.data)
         if serializer.is_valid():
             try:
-                serializer.save(collection=collection_object)
+                serializer.save(item=col_it, order='abc')
             except IntegrityError:
                 content = {'code': 'integrity_error'}
                 return Response(content, status=status.HTTP_400_BAD_REQUEST)
@@ -169,15 +170,3 @@ class CollectionItemChunkViewSet(viewsets.ViewSet):
             return Response({}, status=status.HTTP_201_CREATED)
 
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
-
-    def destroy(self, request, collection_uid=None, uid=None):
-        # FIXME: implement
-        return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
-
-    def update(self, request, collection_uid=None, uid=None):
-        # FIXME: implement, or should it be implemented elsewhere?
-        return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
-
-    def partial_update(self, request, collection_uid=None, uid=None):
-        # FIXME: implement, or should it be implemented elsewhere?
-        return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)