Sync manager: add etebase support (pushing changes)

app/src/main/java/com/etesync/syncadapter/EtebaseLocalCache.kt

@@ -76,9 +76,12 @@ class EtebaseLocalCache private constructor(context: Context, username: String)
         }
     }
 
-    fun collectionGet(colMgr: CollectionManager, colUid: String): CachedCollection {
+    fun collectionGet(colMgr: CollectionManager, colUid: String): CachedCollection? {
         val colDir = File(colsDir, colUid)
         val colFile = File(colDir, "col")
+        if (!colFile.exists()) {
+            return null
+        }
         val content = colFile.readBytes()
         return colMgr.cacheLoad(content).let {
             CachedCollection(it, it.meta)
@@ -110,6 +113,18 @@ class EtebaseLocalCache private constructor(context: Context, username: String)
         }
     }
 
+    fun itemGet(itemMgr: ItemManager, colUid: String, itemUid: String): CachedItem? {
+        val itemsDir = getCollectionItemsDir(colUid)
+        val itemFile = File(itemsDir, itemUid)
+        if (!itemFile.exists()) {
+            return null
+        }
+        val content = itemFile.readBytes()
+        return itemMgr.cacheLoad(content).let {
+            CachedItem(it, it.meta)
+        }
+    }
+
     fun itemSet(itemMgr: ItemManager, colUid: String, item: Item) {
         val itemsDir = getCollectionItemsDir(colUid)
         val itemFile = File(itemsDir, item.uid)

app/src/main/java/com/etesync/syncadapter/resource/LocalContact.kt

@@ -52,7 +52,7 @@ class LocalContact : AndroidContact, LocalAddress {
 
     override// The same now
     val uuid: String?
-        get() = fileName
+        get() = contact?.uid
 
     override val isLocalOnly: Boolean
         get() = TextUtils.isEmpty(eTag)
@@ -88,9 +88,11 @@ class LocalContact : AndroidContact, LocalAddress {
         addressBook.provider?.update(rawContactSyncURI(), values, null, null)
     }
 
-    override fun clearDirty(eTag: String) {
+    override fun clearDirty(eTag: String?) {
         val values = ContentValues(3)
-        values.put(AndroidContact.COLUMN_ETAG, eTag)
+        if (eTag != null) {
+            values.put(AndroidContact.COLUMN_ETAG, eTag)
+        }
         values.put(ContactsContract.RawContacts.DIRTY, 0)
 
         if (LocalContact.HASH_HACK) {
@@ -105,15 +107,16 @@ class LocalContact : AndroidContact, LocalAddress {
         this.eTag = eTag
     }
 
-    override fun prepareForUpload() {
+    override fun prepareForUpload(fileName_: String?) {
         val uid = UUID.randomUUID().toString()
 
         val values = ContentValues(2)
-        values.put(AndroidContact.COLUMN_FILENAME, uid)
+        val fileName = fileName_ ?: uid
+        values.put(AndroidContact.COLUMN_FILENAME, fileName)
         values.put(AndroidContact.COLUMN_UID, uid)
         addressBook.provider?.update(rawContactSyncURI(), values, null, null)
 
-        fileName = uid
+        this.fileName = fileName
     }
 
     override fun populateData(mimeType: String, row: ContentValues) {

app/src/main/java/com/etesync/syncadapter/resource/LocalEvent.kt

@@ -38,7 +38,7 @@ class LocalEvent : AndroidEvent, LocalResource<Event> {
 
     private var saveAsDirty = false // When true, the resource will be saved as dirty
 
-    private var fileName: String? = null
+    override var fileName: String? = null
     var eTag: String? = null
 
     var weAreOrganizer = true
@@ -58,7 +58,7 @@ class LocalEvent : AndroidEvent, LocalResource<Event> {
 
     override// Now the same
     val uuid: String?
-        get() = fileName
+        get() = event?.uid
 
     constructor(calendar: AndroidCalendar<*>, event: Event, fileName: String?, eTag: String?) : super(calendar, event) {
         this.fileName = fileName
@@ -133,7 +133,7 @@ class LocalEvent : AndroidEvent, LocalResource<Event> {
 
     /* custom queries */
 
-    override fun prepareForUpload() {
+    override fun prepareForUpload(fileName_: String?) {
         var uid: String? = null
         val c = calendar.provider.query(eventSyncURI(), arrayOf(COLUMN_UID), null, null, null)
         if (c.moveToNext())
@@ -142,14 +142,14 @@ class LocalEvent : AndroidEvent, LocalResource<Event> {
             uid = UUID.randomUUID().toString()
 
         c.close()
-        val newFileName = uid
 
+        val fileName = fileName_ ?: uid
         val values = ContentValues(2)
-        values.put(Events._SYNC_ID, newFileName)
+        values.put(Events._SYNC_ID, fileName)
         values.put(COLUMN_UID, uid)
         calendar.provider.update(eventSyncURI(), values, null, null)
 
-        fileName = newFileName
+        this.fileName = fileName
 
         val event = this.event
         if (event != null)
@@ -162,10 +162,12 @@ class LocalEvent : AndroidEvent, LocalResource<Event> {
         calendar.provider.update(eventSyncURI(), values, null, null)
     }
 
-    override fun clearDirty(eTag: String) {
+    override fun clearDirty(eTag: String?) {
         val values = ContentValues(2)
         values.put(CalendarContract.Events.DIRTY, 0)
-        values.put(COLUMN_ETAG, eTag)
+        if (eTag != null) {
+            values.put(COLUMN_ETAG, eTag)
+        }
         if (event != null)
             values.put(COLUMN_SEQUENCE, event?.sequence)
         calendar.provider.update(eventSyncURI(), values, null, null)

app/src/main/java/com/etesync/syncadapter/resource/LocalGroup.kt

@@ -120,13 +120,15 @@ class LocalGroup : AndroidGroup, LocalAddress {
         return values
     }
 
-    override fun clearDirty(eTag: String) {
+    override fun clearDirty(eTag: String?) {
         val id = requireNotNull(id)
 
         val values = ContentValues(2)
         values.put(Groups.DIRTY, 0)
         this.eTag = eTag
-        values.put(AndroidGroup.COLUMN_ETAG, eTag)
+        if (eTag != null) {
+            values.put(AndroidGroup.COLUMN_ETAG, eTag)
+        }
         update(values)
 
         // update cached group memberships
@@ -154,15 +156,16 @@ class LocalGroup : AndroidGroup, LocalAddress {
         batch.commit()
     }
 
-    override fun prepareForUpload() {
+    override fun prepareForUpload(fileName_: String?) {
         val uid = UUID.randomUUID().toString()
 
         val values = ContentValues(2)
-        values.put(AndroidGroup.COLUMN_FILENAME, uid)
+        val fileName = fileName_ ?: uid
+        values.put(AndroidGroup.COLUMN_FILENAME, fileName)
         values.put(AndroidGroup.COLUMN_UID, uid)
         update(values)
 
-        fileName = uid
+        this.fileName = fileName
     }
 
     override fun resetDeleted() {

app/src/main/java/com/etesync/syncadapter/resource/LocalResource.kt

@@ -10,6 +10,7 @@ package com.etesync.syncadapter.resource
 
 interface LocalResource<in TData: Any> {
     val uuid: String?
+    val fileName: String?
 
     /** True if doesn't exist on server yet, false otherwise.  */
     val isLocalOnly: Boolean
@@ -19,9 +20,10 @@ interface LocalResource<in TData: Any> {
 
     fun delete(): Int
 
-    fun prepareForUpload()
+    // FIXME: The null is for legacy
+    fun prepareForUpload(fileName: String?)
 
-    fun clearDirty(eTag: String)
+    fun clearDirty(eTag: String?)
 
     fun resetDeleted()
 }

app/src/main/java/com/etesync/syncadapter/resource/LocalTask.kt

@@ -31,7 +31,7 @@ class LocalTask : AndroidTask, LocalResource<Task> {
 
     private var saveAsDirty = false // When true, the resource will be saved as dirty
 
-    private var fileName: String? = null
+    override var fileName: String? = null
     var eTag: String? = null
 
     override val content: String
@@ -49,7 +49,7 @@ class LocalTask : AndroidTask, LocalResource<Task> {
 
     override// Now the same
     val uuid: String?
-        get() = fileName
+        get() = task?.uid
 
     constructor(taskList: AndroidTaskList<*>, task: Task, fileName: String?, eTag: String?)
             : super(taskList, task) {
@@ -96,7 +96,7 @@ class LocalTask : AndroidTask, LocalResource<Task> {
 
     /* custom queries */
 
-    override fun prepareForUpload() {
+    override fun prepareForUpload(fileName_: String?) {
         var uid: String? = null
         val c = taskList.provider.client.query(taskSyncURI(), arrayOf(COLUMN_UID), null, null, null)
         if (c.moveToNext())
@@ -106,12 +106,13 @@ class LocalTask : AndroidTask, LocalResource<Task> {
 
         c.close()
 
+        val fileName = fileName_ ?: uid
         val values = ContentValues(2)
-        values.put(TaskContract.Tasks._SYNC_ID, uid)
+        values.put(TaskContract.Tasks._SYNC_ID, fileName)
         values.put(COLUMN_UID, uid)
         taskList.provider.client.update(taskSyncURI(), values, null, null)
 
-        fileName = uid
+        this.fileName = fileName
         val task = this.task
         if (task != null)
             task.uid = uid
@@ -123,10 +124,12 @@ class LocalTask : AndroidTask, LocalResource<Task> {
         taskList.provider.client.update(taskSyncURI(), values, null, null)
     }
 
-    override fun clearDirty(eTag: String) {
+    override fun clearDirty(eTag: String?) {
         val values = ContentValues(2)
         values.put(TaskContract.Tasks._DIRTY, 0)
-        values.put(COLUMN_ETAG, eTag)
+        if (eTag != null) {
+            values.put(COLUMN_ETAG, eTag)
+        }
         if (task != null)
             values.put(COLUMN_SEQUENCE, task?.sequence)
         taskList.provider.client.update(taskSyncURI(), values, null, null)

app/src/main/java/com/etesync/syncadapter/syncadapter/CalendarSyncManager.kt

@@ -134,8 +134,8 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
     }
 
     @Throws(CalendarStorageException::class, ContactsStorageException::class, IOException::class)
-    override fun createLocalEntries() {
+    override fun prepareLocal() {
-        super.createLocalEntries()
+        super.prepareLocal()
 
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
             createInviteAttendeesNotification()

app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.kt

@@ -118,7 +118,7 @@ constructor(protected val context: Context, protected val account: Account, prot
             etebase = EtebaseLocalCache.getEtebase(context, httpClient.okHttpClient, settings)
             colMgr = etebase.collectionManager
             synchronized(etebaseLocalCache) {
-                cachedCollection = etebaseLocalCache.collectionGet(colMgr, journalUid)
+                cachedCollection = etebaseLocalCache.collectionGet(colMgr, journalUid)!!
             }
             itemMgr = colMgr.getItemManager(cachedCollection.col)
         }
@@ -199,6 +199,27 @@ constructor(protected val context: Context, protected val account: Account, prot
                     etebaseLocalCache.collectionLoadStoken(cachedCollection.col.uid)
                 }
                 // Push local changes
+                var chunkPushItems: List<Item>
+                do {
+                    if (Thread.interrupted())
+                        throw InterruptedException()
+                    syncPhase = R.string.sync_phase_prepare_local
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    prepareLocal()
+
+                    /* Create push items out of local changes. */
+                    if (Thread.interrupted())
+                        throw InterruptedException()
+                    syncPhase = R.string.sync_phase_create_local_entries
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    chunkPushItems = createPushItems()
+
+                    if (Thread.interrupted())
+                        throw InterruptedException()
+                    syncPhase = R.string.sync_phase_push_entries
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    pushItems(chunkPushItems)
+                } while (chunkPushItems.size == MAX_PUSH)
 
                 do {
                     if (Thread.interrupted())
@@ -239,6 +260,11 @@ constructor(protected val context: Context, protected val account: Account, prot
         } catch (e: SSLHandshakeException) {
             syncResult.stats.numIoExceptions++
 
+            notificationManager.setThrowable(e)
+            val detailsIntent = notificationManager.detailsIntent
+            detailsIntent.putExtra(KEY_ACCOUNT, account)
+            notificationManager.notify(syncErrorTitle, context.getString(syncPhase))
+        } catch (e: FileNotFoundException) {
             notificationManager.setThrowable(e)
             val detailsIntent = notificationManager.detailsIntent
             detailsIntent.putExtra(KEY_ACCOUNT, account)
@@ -395,7 +421,7 @@ constructor(protected val context: Context, protected val account: Account, prot
             Logger.log.info("Fetched items. Done=${ret.isDone}")
             return ret
         } else {
-            Logger.log.info("Skipping fetch because local lastUid == remoteLastUid (${remoteCTag})")
+            Logger.log.info("Skipping fetch because local stoken == lastStoken (${remoteCTag})")
             return null
         }
     }
@@ -527,8 +553,127 @@ constructor(protected val context: Context, protected val account: Account, prot
         }
     }
 
+    private fun pushItems(chunkPushItems_: List<Item>) {
+        var chunkPushItems = chunkPushItems_
+        // upload dirty contacts
+        var pushed = 0
+        try {
+            if (!chunkPushItems.isEmpty()) {
+                val items = chunkPushItems
+                itemMgr.batch(items.toTypedArray())
+
+                // Persist the items
+                synchronized(etebaseLocalCache) {
+                    val colUid = cachedCollection.col.uid
+
+                    for (item in items) {
+                        etebaseLocalCache.itemSet(itemMgr, colUid, item)
+                    }
+                }
+
+                pushed += items.size
+            }
+        } finally {
+            // FIXME: A bit fragile, we assume the order in createPushItems
+            var left = pushed
+            for (local in localDeleted!!) {
+                if (pushed-- <= 0) {
+                    break
+                }
+                local.delete()
+            }
+            if (left > 0) {
+                localDeleted = localDeleted?.drop(left)
+                chunkPushItems = chunkPushItems.drop(left - pushed)
+            }
+
+            left = pushed
+            var i = 0
+            for (local in localDirty) {
+                if (pushed-- <= 0) {
+                    break
+                }
+                Logger.log.info("Added/changed resource with filename: " + local.fileName)
+                local.clearDirty(chunkPushItems[i].etag)
+                i++
+            }
+            if (left > 0) {
+                localDirty = localDirty.drop(left)
+                chunkPushItems.drop(left)
+            }
+
+            if (pushed > 0) {
+                Logger.log.severe("Unprocessed localentries left, this should never happen!")
+            }
+        }
+    }
+
+    private fun itemUpdateMtime(item: Item) {
+        val meta = item.meta
+        meta.setMtime(System.currentTimeMillis())
+        item.meta = meta
+    }
+
+    private fun createPushItems(): List<Item> {
+        val ret = LinkedList<Item>()
+        val colUid = cachedCollection.col.uid
+
+        synchronized(etebaseLocalCache) {
+            for (local in localDeleted!!) {
+                val item = etebaseLocalCache.itemGet(itemMgr, colUid, local.fileName!!)!!.item
+                itemUpdateMtime(item)
+                item.delete()
+                ret.add(item)
+
+                if (ret.size == MAX_PUSH) {
+                    return ret
+                }
+            }
+        }
+
+        synchronized(etebaseLocalCache) {
+            for (local in localDirty) {
+                val cacheItem = if (local.fileName != null) etebaseLocalCache.itemGet(itemMgr, colUid, local.fileName!!)!! else null
+                val item: Item
+                if (cacheItem != null) {
+                    item = cacheItem.item
+                    itemUpdateMtime(item)
+                } else {
+                    val meta = ItemMetadata()
+                    meta.name = local.uuid
+                    meta.setMtime(System.currentTimeMillis())
+                    item = itemMgr.create(meta, "")
+
+                    local.prepareForUpload(item.uid)
+                }
+
+                try {
+                    item.setContent(local.content)
+                } catch (e: Exception) {
+                    Logger.log.warning("Failed creating local entry ${local.uuid}")
+                    if (local is LocalContact) {
+                        Logger.log.warning("Contact with title ${local.contact?.displayName}")
+                    } else if (local is LocalEvent) {
+                        Logger.log.warning("Event with title ${local.event?.summary}")
+                    } else if (local is LocalTask) {
+                        Logger.log.warning("Task with title ${local.task?.summary}")
+                    }
+                    throw e
+                }
+
+                ret.add(item)
+
+                if (ret.size == MAX_PUSH) {
+                    return ret
+                }
+            }
+        }
+
+        return ret
+    }
+
     @Throws(CalendarStorageException::class, ContactsStorageException::class, IOException::class)
-    protected open fun createLocalEntries() {
+    private fun createLocalEntries() {
         localEntries = LinkedList()
 
         // Not saving, just creating a fake one until we load it from a local db
@@ -581,7 +726,7 @@ constructor(protected val context: Context, protected val account: Account, prot
     /**
      */
     @Throws(CalendarStorageException::class, ContactsStorageException::class, FileNotFoundException::class)
-    private fun prepareLocal() {
+    protected open fun prepareLocal() {
         localDeleted = processLocallyDeleted()
         localDirty = localCollection!!.findDirty(MAX_PUSH)
         // This is done after fetching the local dirty so all the ones we are using will be prepared
@@ -598,7 +743,8 @@ constructor(protected val context: Context, protected val account: Account, prot
         val localList = localCollection!!.findDeleted()
         val ret = ArrayList<T>(localList.size)
 
-        if (journalEntity.isReadOnly) {
+        val readOnly = (isLegacy && journalEntity.isReadOnly) || (!isLegacy && (cachedCollection.col.accessLevel == "ro"))
+        if (readOnly) {
             for (local in localList) {
                 Logger.log.info("Restoring locally deleted resource on a read only collection: ${local.uuid}")
                 local.resetDeleted()
@@ -612,8 +758,11 @@ constructor(protected val context: Context, protected val account: Account, prot
                 if (local.uuid != null) {
                     Logger.log.info(local.uuid + " has been deleted locally -> deleting from server")
                 } else {
-                    Logger.log.fine("Entry deleted before ever syncing - genarting a UUID")
+                    if (isLegacy) {
-                    local.prepareForUpload()
+                        // It's done later for non-legacy
+                        Logger.log.fine("Entry deleted before ever syncing - genarting a UUID")
+                        local.prepareForUpload(null)
+                    }
                 }
 
                 ret.add(local)
@@ -627,20 +776,22 @@ constructor(protected val context: Context, protected val account: Account, prot
 
     @Throws(CalendarStorageException::class, ContactsStorageException::class)
     protected open fun prepareDirty() {
-        if (journalEntity.isReadOnly) {
+        val readOnly = (isLegacy && journalEntity.isReadOnly) || (!isLegacy && (cachedCollection.col.accessLevel == "ro"))
+        if (readOnly) {
             for (local in localDirty) {
                 Logger.log.info("Restoring locally modified resource on a read only collection: ${local.uuid}")
                 if (local.uuid == null) {
                     // If it was only local, delete.
                     local.delete()
                 } else {
-                    local.clearDirty(local.uuid!!)
+                    local.clearDirty(null)
                 }
                 numDiscarded++
             }
 
             localDirty = LinkedList()
-        } else {
+        } else if (isLegacy) {
+            // It's done later for non-legacy
             // assign file names and UIDs to new entries
             Logger.log.info("Looking for local entries without a uuid")
             for (local in localDirty) {
@@ -649,7 +800,7 @@ constructor(protected val context: Context, protected val account: Account, prot
                 }
 
                 Logger.log.fine("Found local record without file name; generating file name/UID if necessary")
-                local.prepareForUpload()
+                local.prepareForUpload(null)
             }
         }
     }