Sync manager: add etebase support (pulling changes)

app/src/main/java/com/etesync/syncadapter/EtebaseLocalCache.kt

@@ -11,12 +11,13 @@ import java.util.*
 File structure:
 cache_dir/
     user1/ <--- the name of the user
-        stoken
+        stoken <-- the stokens of the collection fetch
         cols/
             UID1/ - The uid of the first col
                 ...
             UID2/ - The uid of the second col
                 col <-- the col itself
+                stoken <-- the stoken of the items fetch
                 items/
                     item_uid1 <-- the item with uid 1
                     item_uid2
@@ -51,6 +52,19 @@ class EtebaseLocalCache private constructor(context: Context, username: String)
         return if (stokenFile.exists()) stokenFile.readText() else null
     }
 
+
+    fun collectionSaveStoken(colUid: String, stoken: String) {
+        val colDir = File(colsDir, colUid)
+        val stokenFile = File(colDir, "stoken")
+        stokenFile.writeText(stoken)
+    }
+
+    fun collectionLoadStoken(colUid: String): String? {
+        val colDir = File(colsDir, colUid)
+        val stokenFile = File(colDir, "stoken")
+        return if (stokenFile.exists()) stokenFile.readText() else null
+    }
+
     fun collectionList(colMgr: CollectionManager, withDeleted: Boolean = false): List<CachedCollection> {
         return colsDir.list().map {
             val colDir = File(colsDir, it)
@@ -62,6 +76,15 @@ class EtebaseLocalCache private constructor(context: Context, username: String)
         }
     }
 
+    fun collectionGet(colMgr: CollectionManager, colUid: String): CachedCollection {
+        val colDir = File(colsDir, colUid)
+        val colFile = File(colDir, "col")
+        val content = colFile.readBytes()
+        return colMgr.cacheLoad(content).let {
+            CachedCollection(it, it.meta)
+        }
+    }
+
     fun collectionSet(colMgr: CollectionManager, collection: Collection) {
         val colDir = File(colsDir, collection.uid)
         colDir.mkdir()

app/src/main/java/com/etesync/syncadapter/resource/LocalAddressBook.kt

@@ -344,7 +344,7 @@ class LocalAddressBook(
         return reallyDirty
     }
 
-    override fun findByUid(uid: String): LocalAddress? {
+    override fun findByFilename(uid: String): LocalAddress? {
         val found = findContactByUID(uid)
         if (found != null) {
             return found

app/src/main/java/com/etesync/syncadapter/resource/LocalCalendar.kt

@@ -165,7 +165,7 @@ class LocalCalendar private constructor(
     override fun findAll(): List<LocalEvent>
             = queryEvents(null, null)
 
-    override fun findByUid(uid: String): LocalEvent?
+    override fun findByFilename(uid: String): LocalEvent?
         = queryEvents(Events._SYNC_ID + " =? ", arrayOf(uid)).firstOrNull()
 
     fun processDirtyExceptions() {

app/src/main/java/com/etesync/syncadapter/resource/LocalCollection.kt

@@ -16,7 +16,7 @@ interface LocalCollection<out T: LocalResource<*>> {
     fun findWithoutFileName(): List<T>
     fun findAll(): List<T>
 
-    fun findByUid(uid: String): T?
+    fun findByFilename(uid: String): T?
 
 
     fun count(): Long

app/src/main/java/com/etesync/syncadapter/resource/LocalGroup.kt

@@ -63,7 +63,7 @@ class LocalGroup : AndroidGroup, LocalAddress {
                     // insert memberships
                     val membersIds = members.map {uid ->
                         Constants.log.fine("Assigning member: $uid")
-                        val contact = addressBook.findByUid(uid) as LocalContact?
+                        val contact = addressBook.findByFilename(uid) as LocalContact?
                         if (contact != null) contact.id else null
                     }.filterNotNull()
 

app/src/main/java/com/etesync/syncadapter/resource/LocalTaskList.kt

@@ -126,7 +126,7 @@ class LocalTaskList private constructor(
     override fun findWithoutFileName(): List<LocalTask>
         = queryTasks(Tasks._SYNC_ID + " IS NULL", null)
 
-    override fun findByUid(uid: String): LocalTask?
+    override fun findByFilename(uid: String): LocalTask?
         = queryTasks(Tasks._SYNC_ID + " =? ", arrayOf(uid)).firstOrNull()
 
     override fun count(): Long {

app/src/main/java/com/etesync/syncadapter/syncadapter/AddressBooksSyncAdapterService.kt

@@ -88,8 +88,6 @@ class AddressBooksSyncAdapterService : SyncAdapterService() {
 
             val local = LocalAddressBook.find(context, provider, account)
 
-            val updateColors = settings.manageCalendarColors
-
             // delete obsolete local calendar
             for (addressBook in local) {
                 val url = addressBook.url

app/src/main/java/com/etesync/syncadapter/syncadapter/CalendarSyncManager.kt

@@ -17,6 +17,7 @@ import at.bitfire.ical4android.CalendarStorageException
 import at.bitfire.ical4android.Event
 import at.bitfire.ical4android.InvalidCalendarException
 import at.bitfire.vcard4android.ContactsStorageException
+import com.etebase.client.Item
 import com.etesync.syncadapter.AccountSettings
 import com.etesync.syncadapter.Constants
 import com.etesync.syncadapter.R
@@ -59,7 +60,9 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
         if (!super.prepare())
             return false
 
-        journal = JournalEntryManager(httpClient.okHttpClient, remote, localCalendar().name!!)
+        if (isLegacy) {
+            journal = JournalEntryManager(httpClient.okHttpClient, remote, localCalendar().name!!)
+        }
         return true
     }
 
@@ -77,6 +80,32 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
         return localCollection as LocalCalendar
     }
 
+    override fun processItem(item: Item) {
+        val local = localCollection!!.findByFilename(item.uid)
+
+        if (!item.isDeleted) {
+            val inputReader = StringReader(String(item.content))
+
+            val events = Event.eventsFromReader(inputReader)
+            if (events.size == 0) {
+                Logger.log.warning("Received VCard without data, ignoring")
+                return
+            } else if (events.size > 1) {
+                Logger.log.warning("Received multiple VCALs, using first one")
+            }
+
+            val event = events[0]
+            processEvent(item, event, local)
+        } else {
+            if (local != null) {
+                Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server")
+                local.delete()
+            } else {
+                Logger.log.warning("Tried deleting a non-existent record: " + item.uid)
+            }
+        }
+    }
+
     @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, InvalidCalendarException::class)
     override fun processSyncEntryImpl(cEntry: SyncEntry) {
         val inputReader = StringReader(cEntry.content)
@@ -90,10 +119,10 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
         }
 
         val event = events[0]
-        val local = localCollection!!.findByUid(event.uid!!)
+        val local = localCollection!!.findByFilename(event.uid!!)
 
         if (cEntry.isAction(SyncEntry.Actions.ADD) || cEntry.isAction(SyncEntry.Actions.CHANGE)) {
-            processEvent(event, local)
+            legacyProcessEvent(event, local)
         } else {
             if (local != null) {
                 Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server")
@@ -138,8 +167,26 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
         }
     }
 
+    private fun processEvent(item: Item, newData: Event, _localEvent: LocalEvent?): LocalEvent {
+        var localEvent = _localEvent
+        // delete local event, if it exists
+        if (localEvent != null) {
+            Logger.log.info("Updating " + newData.uid + " in local calendar")
+            localEvent.eTag = item.etag
+            localEvent.update(newData)
+            syncResult.stats.numUpdates++
+        } else {
+            Logger.log.info("Adding " + newData.uid + " to local calendar")
+            localEvent = LocalEvent(localCalendar(), newData, item.uid, item.etag)
+            localEvent.add()
+            syncResult.stats.numInserts++
+        }
+
+        return localEvent
+    }
+
     @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class)
-    private fun processEvent(newData: Event, _localEvent: LocalEvent?): LocalEvent {
+    private fun legacyProcessEvent(newData: Event, _localEvent: LocalEvent?): LocalEvent {
         var localEvent = _localEvent
         // delete local event, if it exists
         if (localEvent != null) {

app/src/main/java/com/etesync/syncadapter/syncadapter/ContactsSyncManager.kt

@@ -13,12 +13,14 @@ import android.content.*
 import android.os.Bundle
 import android.provider.ContactsContract
 import at.bitfire.ical4android.CalendarStorageException
+import at.bitfire.ical4android.Event
 import at.bitfire.vcard4android.BatchOperation
 import at.bitfire.vcard4android.Contact
 import at.bitfire.vcard4android.ContactsStorageException
 import com.etesync.journalmanager.Exceptions
 import com.etesync.journalmanager.JournalEntryManager
 import com.etesync.journalmanager.model.SyncEntry
+import com.etebase.client.Item
 import com.etesync.syncadapter.AccountSettings
 import com.etesync.syncadapter.Constants
 import com.etesync.syncadapter.HttpClient
@@ -77,7 +79,9 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
             }
         }
 
-        journal = JournalEntryManager(httpClient.okHttpClient, remote, localAddressBook.url)
+        if (isLegacy) {
+            journal = JournalEntryManager(httpClient.okHttpClient, remote, localAddressBook.url)
+        }
 
         return true
     }
@@ -127,6 +131,34 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
         return localCollection as LocalAddressBook
     }
 
+    override fun processItem(item: Item) {
+        val uid = item.meta.name!!
+
+        val local = localCollection!!.findByFilename(uid)
+
+        if (!item.isDeleted) {
+            val inputReader = StringReader(String(item.content))
+
+            val contacts = Contact.fromReader(inputReader, resourceDownloader)
+            if (contacts.size == 0) {
+                Logger.log.warning("Received VCard without data, ignoring")
+                return
+            } else if (contacts.size > 1) {
+                Logger.log.warning("Received multiple VCALs, using first one")
+            }
+
+            val contact = contacts[0]
+            processContact(item, contact, local)
+        } else {
+            if (local != null) {
+                Logger.log.info("Removing local record which has been deleted on the server")
+                local.delete()
+            } else {
+                Logger.log.warning("Tried deleting a non-existent record: " + uid)
+            }
+        }
+    }
+
     @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class)
     override fun processSyncEntryImpl(cEntry: SyncEntry) {
         val inputReader = StringReader(cEntry.content)
@@ -139,10 +171,10 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
             Logger.log.warning("Received multiple VCards, using first one")
 
         val contact = contacts[0]
-        val local = localCollection!!.findByUid(contact.uid!!)
+        val local = localCollection!!.findByFilename(contact.uid!!)
 
         if (cEntry.isAction(SyncEntry.Actions.ADD) || cEntry.isAction(SyncEntry.Actions.CHANGE)) {
-            processContact(contact, local)
+            legacyProcessContact(contact, local)
         } else {
             if (local != null) {
                 Logger.log.info("Removing local record which has been deleted on the server")
@@ -153,8 +185,65 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra
         }
     }
 
+    private fun processContact(item: Item, newData: Contact, _local: LocalAddress?): LocalAddress {
+        var local = _local
+        val uuid = newData.uid
+        // update local contact, if it exists
+        if (local != null) {
+            Logger.log.log(Level.INFO, "Updating $uuid in local address book")
+
+            if (local is LocalGroup && newData.group) {
+                // update group
+                val group: LocalGroup = local
+                group.eTag = item.etag
+                group.update(newData)
+                syncResult.stats.numUpdates++
+
+            } else if (local is LocalContact && !newData.group) {
+                // update contact
+                val contact: LocalContact = local
+                contact.eTag = item.etag
+                contact.update(newData)
+                syncResult.stats.numUpdates++
+
+            } else {
+                // group has become an individual contact or vice versa
+                try {
+                    local.delete()
+                    local = null
+                } catch (e: CalendarStorageException) {
+                    // CalendarStorageException is not used by LocalGroup and LocalContact
+                }
+
+            }
+        }
+
+        if (local == null) {
+            if (newData.group) {
+                Logger.log.log(Level.INFO, "Creating local group", item.uid)
+                val group = LocalGroup(localAddressBook(), newData, item.uid, item.etag)
+                group.add()
+
+                local = group
+            } else {
+                Logger.log.log(Level.INFO, "Creating local contact", item.uid)
+                val contact = LocalContact(localAddressBook(), newData, item.uid, item.etag)
+                contact.add()
+
+                local = contact
+            }
+            syncResult.stats.numInserts++
+        }
+
+        if (LocalContact.HASH_HACK && local is LocalContact)
+        // workaround for Android 7 which sets DIRTY flag when only meta-data is changed
+            local.updateHashCode(null)
+
+        return local
+    }
+
     @Throws(IOException::class, ContactsStorageException::class)
-    private fun processContact(newData: Contact, _local: LocalAddress?): LocalAddress {
+    private fun legacyProcessContact(newData: Contact, _local: LocalAddress?): LocalAddress {
         var local = _local
         val uuid = newData.uid
         // update local contact, if it exists

app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.kt

@@ -16,6 +16,7 @@ import android.os.Bundle
 import at.bitfire.ical4android.CalendarStorageException
 import at.bitfire.ical4android.InvalidCalendarException
 import at.bitfire.vcard4android.ContactsStorageException
+import com.etebase.client.*
 import com.etesync.syncadapter.*
 import com.etesync.syncadapter.Constants.KEY_ACCOUNT
 import com.etesync.journalmanager.Crypto
@@ -25,6 +26,8 @@ import com.etesync.journalmanager.model.SyncEntry
 import com.etesync.syncadapter.log.Logger
 import com.etesync.syncadapter.model.*
 import com.etesync.journalmanager.model.SyncEntry.Actions.ADD
+import com.etesync.syncadapter.HttpClient
+import com.etesync.syncadapter.R
 import com.etesync.syncadapter.resource.*
 import com.etesync.syncadapter.ui.AccountsActivity
 import com.etesync.syncadapter.ui.DebugInfoActivity
@@ -41,21 +44,30 @@ import kotlin.concurrent.withLock
 
 abstract class SyncManager<T: LocalResource<*>> @Throws(Exceptions.IntegrityException::class, Exceptions.GenericCryptoException::class)
 constructor(protected val context: Context, protected val account: Account, protected val settings: AccountSettings, protected val extras: Bundle, protected val authority: String, protected val syncResult: SyncResult, journalUid: String, protected val serviceType: CollectionInfo.Type, accountName: String): Closeable {
+    // FIXME: remove all of the lateinit once we remove legacy (and make immutable)
+    // RemoteEntries and the likes are probably also just relevant for legacy
+    protected val isLegacy: Boolean = settings.isLegacy
 
     protected val notificationManager: SyncNotification
-    protected val info: CollectionInfo
+    protected lateinit var info: CollectionInfo
     protected var localCollection: LocalCollection<T>? = null
 
     protected var httpClient: HttpClient
 
+    protected lateinit var etebaseLocalCache: EtebaseLocalCache
+    protected lateinit var etebase: com.etebase.client.Account
+    protected lateinit var colMgr: CollectionManager
+    protected lateinit var itemMgr: ItemManager
+    protected lateinit var cachedCollection: CachedCollection
+
     protected var journal: JournalEntryManager? = null
     private var _journalEntity: JournalEntity? = null
 
     private var numDiscarded = 0
 
-    private val crypto: Crypto.CryptoManager
+    private lateinit var crypto: Crypto.CryptoManager
 
-    private val data: MyEntityDataStore
+    private lateinit var data: MyEntityDataStore
 
     /**
      * remote CTag (uuid of the last entry on the server). We update it when we fetch/push and save when everything works.
@@ -89,21 +101,31 @@ constructor(protected val context: Context, protected val account: Account, prot
         // create HttpClient with given logger
         httpClient = HttpClient.Builder(context, settings).setForeground(false).build()
 
-        data = (context.applicationContext as App).data
+        if (isLegacy) {
-        val serviceEntity = JournalModel.Service.fetchOrCreate(data, accountName, serviceType)
+            data = (context.applicationContext as App).data
-        info = JournalEntity.fetch(data, serviceEntity, journalUid)!!.info
+            val serviceEntity = JournalModel.Service.fetchOrCreate(data, accountName, serviceType)
+            info = JournalEntity.fetch(data, serviceEntity, journalUid)!!.info
+
+            Logger.log.info(String.format(Locale.getDefault(), "Syncing collection %s (version: %d)", journalUid, info.version))
+
+            if (journalEntity.encryptedKey != null) {
+                crypto = Crypto.CryptoManager(info.version, settings.keyPair!!, journalEntity.encryptedKey)
+            } else {
+                crypto = Crypto.CryptoManager(info.version, settings.password(), info.uid!!)
+            }
+        } else {
+            etebaseLocalCache = EtebaseLocalCache.getInstance(context, accountName)
+            etebase = EtebaseLocalCache.getEtebase(context, httpClient.okHttpClient, settings)
+            colMgr = etebase.collectionManager
+            synchronized(etebaseLocalCache) {
+                cachedCollection = etebaseLocalCache.collectionGet(colMgr, journalUid)
+            }
+            itemMgr = colMgr.getItemManager(cachedCollection.col)
+        }
 
         // dismiss previous error notifications
         notificationManager = SyncNotification(context, journalUid, notificationId())
         notificationManager.cancel()
-
-        Logger.log.info(String.format(Locale.getDefault(), "Syncing collection %s (version: %d)", journalUid, info.version))
-
-        if (journalEntity.encryptedKey != null) {
-            crypto = Crypto.CryptoManager(info.version, settings.keyPair!!, journalEntity.encryptedKey)
-        } else {
-            crypto = Crypto.CryptoManager(info.version, settings.password(), info.uid!!)
-        }
     }
 
     protected abstract fun notificationId(): Int
@@ -128,48 +150,78 @@ constructor(protected val context: Context, protected val account: Account, prot
             Logger.log.info("Sync phase: " + context.getString(syncPhase))
             prepareFetch()
 
-            do {
+            if (isLegacy) {
-                if (Thread.interrupted())
+                do {
-                    throw InterruptedException()
+                    if (Thread.interrupted())
-                syncPhase = R.string.sync_phase_fetch_entries
+                        throw InterruptedException()
-                Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    syncPhase = R.string.sync_phase_fetch_entries
-                fetchEntries()
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    fetchEntries()
 
-                if (Thread.interrupted())
+                    if (Thread.interrupted())
-                    throw InterruptedException()
+                        throw InterruptedException()
-                syncPhase = R.string.sync_phase_apply_remote_entries
+                    syncPhase = R.string.sync_phase_apply_remote_entries
-                Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
-                applyRemoteEntries()
+                    applyRemoteEntries()
-            } while (remoteEntries!!.size == MAX_FETCH)
+                } while (remoteEntries!!.size == MAX_FETCH)
 
-            do {
+                do {
-                if (Thread.interrupted())
+                    if (Thread.interrupted())
-                    throw InterruptedException()
+                        throw InterruptedException()
-                syncPhase = R.string.sync_phase_prepare_local
+                    syncPhase = R.string.sync_phase_prepare_local
-                Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
-                prepareLocal()
+                    prepareLocal()
 
-                /* Create journal entries out of local changes. */
+                    /* Create journal entries out of local changes. */
-                if (Thread.interrupted())
+                    if (Thread.interrupted())
-                    throw InterruptedException()
+                        throw InterruptedException()
-                syncPhase = R.string.sync_phase_create_local_entries
+                    syncPhase = R.string.sync_phase_create_local_entries
-                Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
-                createLocalEntries()
+                    createLocalEntries()
 
-                if (Thread.interrupted())
+                    if (Thread.interrupted())
-                    throw InterruptedException()
+                        throw InterruptedException()
-                syncPhase = R.string.sync_phase_apply_local_entries
+                    syncPhase = R.string.sync_phase_apply_local_entries
-                Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
-                /* FIXME: Skipping this now, because we already override with remote.
+                    /* FIXME: Skipping this now, because we already override with remote.
-                applyLocalEntries();
+                    applyLocalEntries();
-                */
+                    */
 
-                if (Thread.interrupted())
+                    if (Thread.interrupted())
-                    throw InterruptedException()
+                        throw InterruptedException()
-                syncPhase = R.string.sync_phase_push_entries
+                    syncPhase = R.string.sync_phase_push_entries
-                Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
-                pushEntries()
+                    pushEntries()
-            } while (localEntries!!.size == MAX_PUSH)
+                } while (localEntries!!.size == MAX_PUSH)
+            } else {
+                var itemList: ItemListResponse?
+                var stoken = synchronized(etebaseLocalCache) {
+                    etebaseLocalCache.collectionLoadStoken(cachedCollection.col.uid)
+                }
+                // Push local changes
+
+                do {
+                    if (Thread.interrupted())
+                        throw InterruptedException()
+                    syncPhase = R.string.sync_phase_fetch_entries
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    itemList = fetchItems(stoken)
+                    if (itemList == null) {
+                        break
+                    }
+
+                    if (Thread.interrupted())
+                        throw InterruptedException()
+                    syncPhase = R.string.sync_phase_apply_remote_entries
+                    Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                    applyRemoteItems(itemList)
+
+                    stoken = itemList.stoken
+                    synchronized(etebaseLocalCache) {
+                        etebaseLocalCache.collectionSaveStoken(cachedCollection.col.uid, stoken!!)
+                    }
+                } while (!itemList!!.isDone)
+            }
 
             /* Cleanup and finalize changes */
             if (Thread.interrupted())
@@ -241,7 +293,8 @@ constructor(protected val context: Context, protected val account: Account, prot
 
     private fun notifyUserOnSync() {
         val changeNotification = context.defaultSharedPreferences.getBoolean(App.CHANGE_NOTIFICATION, true)
-        if (remoteEntries!!.isEmpty() || !changeNotification) {
+        val remoteEntries = remoteEntries
+        if ((remoteEntries == null) || remoteEntries.isEmpty() || !changeNotification) {
             return
         }
         val notificationHelper = SyncNotification(context,
@@ -250,7 +303,7 @@ constructor(protected val context: Context, protected val account: Account, prot
         var deleted = 0
         var added = 0
         var changed = 0
-        for (entry in remoteEntries!!) {
+        for (entry in remoteEntries) {
             val cEntry = SyncEntry.fromJournalEntry(crypto, entry)
             val action = cEntry.action
             when (action) {
@@ -287,6 +340,14 @@ constructor(protected val context: Context, protected val account: Account, prot
         return true
     }
 
+    protected abstract fun processItem(item: Item)
+
+    private fun persistItem(item: Item) {
+        synchronized(etebaseLocalCache) {
+            etebaseLocalCache.itemSet(itemMgr, cachedCollection.col.uid, item)
+        }
+    }
+
     @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, InvalidCalendarException::class)
     protected abstract fun processSyncEntryImpl(cEntry: SyncEntry)
 
@@ -319,36 +380,46 @@ constructor(protected val context: Context, protected val account: Account, prot
         }
     }
 
-    @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, Exceptions.HttpException::class, InvalidCalendarException::class, InterruptedException::class)
+    @Throws(IOException::class, CalendarStorageException::class, ContactsStorageException::class)
-    protected fun applyLocalEntries() {
+    protected fun prepareFetch() {
-        // FIXME: Need a better strategy
+        if (isLegacy) {
-        // We re-apply local entries so our changes override whatever was written in the remote.
+            remoteCTag = journalEntity.getLastUid(data)
-        val strTotal = localEntries!!.size.toString()
+        } else {
+            remoteCTag = cachedCollection.col.stoken
+        }
+    }
+
+    private fun fetchItems(stoken: String?): ItemListResponse? {
+        if (remoteCTag != stoken) {
+            val ret = itemMgr.list(FetchOptions().stoken(stoken))
+            Logger.log.info("Fetched items. Done=${ret.isDone}")
+            return ret
+        } else {
+            Logger.log.info("Skipping fetch because local lastUid == remoteLastUid (${remoteCTag})")
+            return null
+        }
+    }
+
+    private fun applyRemoteItems(itemList: ItemListResponse) {
+        val items = itemList.data
+        // Process new vcards from server
+        val size = items.size
         var i = 0
 
-        for (entry in localEntries!!) {
+        for (item in items) {
             if (Thread.interrupted()) {
                 throw InterruptedException()
             }
             i++
-            Logger.log.info("Processing (" + i.toString() + "/" + strTotal + ") " + entry.toString())
+            Logger.log.info("Processing (${i}/${size}) UID=${item.uid} Etag=${item.etag}")
 
-            val cEntry = SyncEntry.fromJournalEntry(crypto, entry)
+            processItem(item)
-            if (cEntry.isAction(SyncEntry.Actions.DELETE)) {
+            persistItem(item)
-                continue
-            }
-            Logger.log.info("Processing resource for journal entry")
-            processSyncEntry(cEntry)
         }
     }
 
-    @Throws(IOException::class, CalendarStorageException::class, ContactsStorageException::class)
-    protected fun prepareFetch() {
-        remoteCTag = journalEntity.getLastUid(data)
-    }
-
     @Throws(Exceptions.HttpException::class, ContactsStorageException::class, CalendarStorageException::class, Exceptions.IntegrityException::class)
-    protected fun fetchEntries() {
+    private fun fetchEntries() {
         val count = data.count(EntryEntity::class.java).where(EntryEntity.JOURNAL.eq(journalEntity)).get().value()
         if (remoteCTag != null && count == 0) {
             // If we are updating an existing installation with no saved journal, we need to add
@@ -377,7 +448,7 @@ constructor(protected val context: Context, protected val account: Account, prot
     }
 
     @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, InvalidCalendarException::class, InterruptedException::class)
-    protected fun applyRemoteEntries() {
+    private fun applyRemoteEntries() {
         // Process new vcards from server
         val strTotal = remoteEntries!!.size.toString()
         var i = 0
@@ -406,7 +477,7 @@ constructor(protected val context: Context, protected val account: Account, prot
     }
 
     @Throws(Exceptions.HttpException::class, IOException::class, ContactsStorageException::class, CalendarStorageException::class)
-    protected fun pushEntries() {
+    private fun pushEntries() {
         // upload dirty contacts
         var pushed = 0
         // FIXME: Deal with failure (someone else uploaded before we go here)
@@ -510,7 +581,7 @@ constructor(protected val context: Context, protected val account: Account, prot
     /**
      */
     @Throws(CalendarStorageException::class, ContactsStorageException::class, FileNotFoundException::class)
-    protected fun prepareLocal() {
+    private fun prepareLocal() {
         localDeleted = processLocallyDeleted()
         localDirty = localCollection!!.findDirty(MAX_PUSH)
         // This is done after fetching the local dirty so all the ones we are using will be prepared

app/src/main/java/com/etesync/syncadapter/syncadapter/TasksSyncManager.kt

@@ -13,6 +13,7 @@ import android.content.Context
 import android.content.SyncResult
 import android.os.Bundle
 import at.bitfire.ical4android.Task
+import com.etebase.client.Item
 import com.etesync.syncadapter.AccountSettings
 import com.etesync.syncadapter.Constants
 import com.etesync.syncadapter.R
@@ -58,7 +59,9 @@ class TasksSyncManager(
         if (!super.prepare())
             return false
 
-        journal = JournalEntryManager(httpClient.okHttpClient, remote, localTaskList().url!!)
+        if (isLegacy) {
+            journal = JournalEntryManager(httpClient.okHttpClient, remote, localTaskList().url!!)
+        }
         return true
     }
 
@@ -68,6 +71,32 @@ class TasksSyncManager(
         return localCollection as LocalTaskList
     }
 
+    override fun processItem(item: Item) {
+        val local = localCollection!!.findByFilename(item.uid)
+
+        if (!item.isDeleted) {
+            val inputReader = StringReader(String(item.content))
+
+            val tasks = Task.tasksFromReader(inputReader)
+            if (tasks.size == 0) {
+                Logger.log.warning("Received VCard without data, ignoring")
+                return
+            } else if (tasks.size > 1) {
+                Logger.log.warning("Received multiple VCALs, using first one")
+            }
+
+            val task = tasks[0]
+            processTask(item, task, local)
+        } else {
+            if (local != null) {
+                Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server")
+                local.delete()
+            } else {
+                Logger.log.warning("Tried deleting a non-existent record: " + item.uid)
+            }
+        }
+    }
+
     override fun processSyncEntryImpl(cEntry: SyncEntry) {
         val inputReader = StringReader(cEntry.content)
 
@@ -80,10 +109,10 @@ class TasksSyncManager(
         }
 
         val event = tasks[0]
-        val local = localCollection!!.findByUid(event.uid!!)
+        val local = localCollection!!.findByFilename(event.uid!!)
 
         if (cEntry.isAction(SyncEntry.Actions.ADD) || cEntry.isAction(SyncEntry.Actions.CHANGE)) {
-            processTask(event, local)
+            legacyProcessTask(event, local)
         } else {
             if (local != null) {
                 Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server")
@@ -94,7 +123,25 @@ class TasksSyncManager(
         }
     }
 
-    private fun processTask(newData: Task, _localTask: LocalTask?): LocalTask {
+    private fun processTask(item: Item, newData: Task, _localTask: LocalTask?): LocalTask {
+        var localTask = _localTask
+        // delete local Task, if it exists
+        if (localTask != null) {
+            Logger.log.info("Updating " + item.uid + " in local calendar")
+            localTask.eTag = item.etag
+            localTask.update(newData)
+            syncResult.stats.numUpdates++
+        } else {
+            Logger.log.info("Adding " + item.uid + " to local calendar")
+            localTask = LocalTask(localTaskList(), newData, item.uid, item.etag)
+            localTask.add()
+            syncResult.stats.numInserts++
+        }
+
+        return localTask
+    }
+
+    private fun legacyProcessTask(newData: Task, _localTask: LocalTask?): LocalTask {
         var localTask = _localTask
         // delete local Task, if it exists
         if (localTask != null) {

app/src/main/java/com/etesync/syncadapter/ui/JournalItemActivity.kt

@@ -108,7 +108,7 @@ class JournalItemActivity : BaseActivity(), Refreshable {
                 val provider = contentResolver.acquireContentProviderClient(CalendarContract.CONTENT_URI)!!
                 val localCalendar = LocalCalendar.findByName(account, provider, LocalCalendar.Factory, info.uid!!)!!
                 val event = Event.eventsFromReader(StringReader(syncEntry.content))[0]
-                var localEvent = localCalendar.findByUid(event.uid!!)
+                var localEvent = localCalendar.findByFilename(event.uid!!)
                 if (localEvent != null) {
                     localEvent.updateAsDirty(event)
                 } else {
@@ -121,7 +121,7 @@ class JournalItemActivity : BaseActivity(), Refreshable {
                     val provider = TaskProvider.acquire(this, it)!!
                     val localTaskList = LocalTaskList.findByName(account, provider, LocalTaskList.Factory, info.uid!!)!!
                     val task = Task.tasksFromReader(StringReader(syncEntry.content))[0]
-                    var localTask = localTaskList.findByUid(task.uid!!)
+                    var localTask = localTaskList.findByFilename(task.uid!!)
                     if (localTask != null) {
                         localTask.updateAsDirty(task)
                     } else {
@@ -137,7 +137,7 @@ class JournalItemActivity : BaseActivity(), Refreshable {
                 if (contact.group) {
                     // FIXME: not currently supported
                 } else {
-                    var localContact = localAddressBook.findByUid(contact.uid!!) as LocalContact?
+                    var localContact = localAddressBook.findByFilename(contact.uid!!) as LocalContact?
                     if (localContact != null) {
                         localContact.updateAsDirty(contact)
                     } else {

app/src/main/java/com/etesync/syncadapter/ui/importlocal/ImportFragment.kt

@@ -15,7 +15,6 @@ import android.provider.CalendarContract
 import android.provider.ContactsContract
 import androidx.fragment.app.DialogFragment
 import at.bitfire.ical4android.*
-import at.bitfire.ical4android.TaskProvider.Companion.OPENTASK_PROVIDERS
 import at.bitfire.vcard4android.BatchOperation
 import at.bitfire.vcard4android.Contact
 import at.bitfire.vcard4android.ContactsStorageException
@@ -255,7 +254,7 @@ class ImportFragment : DialogFragment() {
 
                     for (event in events) {
                         try {
-                            var localEvent = localCalendar.findByUid(event.uid!!)
+                            var localEvent = localCalendar.findByFilename(event.uid!!)
                             if (localEvent != null) {
                                 localEvent.updateAsDirty(event)
                                 result.updated++
@@ -309,7 +308,7 @@ class ImportFragment : DialogFragment() {
 
                         for (task in tasks) {
                             try {
-                                var localTask = localTaskList.findByUid(task.uid!!)
+                                var localTask = localTaskList.findByFilename(task.uid!!)
                                 if (localTask != null) {
                                     localTask.updateAsDirty(task)
                                     result.updated++
@@ -353,7 +352,7 @@ class ImportFragment : DialogFragment() {
 
                     for (contact in contacts.filter { contact -> !contact.group }) {
                         try {
-                            var localContact = localAddressBook.findByUid(contact.uid!!) as LocalContact?
+                            var localContact = localAddressBook.findByFilename(contact.uid!!) as LocalContact?
 
                             if (localContact != null) {
                                 localContact.updateAsDirty(contact)
@@ -386,7 +385,7 @@ class ImportFragment : DialogFragment() {
                             }
 
                             val group = contact
-                            var localGroup: LocalGroup? = localAddressBook.findByUid(group.uid!!) as LocalGroup?
+                            var localGroup: LocalGroup? = localAddressBook.findByFilename(group.uid!!) as LocalGroup?
 
                             if (localGroup != null) {
                                 localGroup.updateAsDirty(group, memberIds)

app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalCalendarImportFragment.kt

@@ -222,7 +222,7 @@ class LocalCalendarImportFragment : ListFragment() {
                         var localEvent = if (event == null || event.uid == null)
                             null
                         else
-                            localCalendar.findByUid(event.uid!!)
+                            localCalendar.findByFilename(event.uid!!)
 
                         if (localEvent != null) {
                             localEvent.updateAsDirty(event!!)

app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalContactImportFragment.kt

@@ -158,7 +158,7 @@ class LocalContactImportFragment : Fragment() {
                         var localContact: LocalContact? = if (contact.uid == null)
                             null
                         else
-                            addressBook.findByUid(contact.uid!!) as LocalContact?
+                            addressBook.findByFilename(contact.uid!!) as LocalContact?
 
                         if (localContact != null) {
                             localContact.updateAsDirty(contact)
@@ -189,7 +189,7 @@ class LocalContactImportFragment : Fragment() {
                         var localGroup: LocalGroup? = if (group.uid == null)
                             null
                         else
-                            addressBook.findByUid(group.uid!!) as LocalGroup?
+                            addressBook.findByFilename(group.uid!!) as LocalGroup?
 
                         if (localGroup != null) {
                             localGroup.updateAsDirty(group, members)