From 6302ab42de93620009b2a362dd4e1c4a08027816 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 26 Aug 2020 08:44:33 +0300 Subject: [PATCH] Sync manager: add etebase support (pulling changes) --- .../etesync/syncadapter/EtebaseLocalCache.kt | 25 +- .../syncadapter/resource/LocalAddressBook.kt | 2 +- .../syncadapter/resource/LocalCalendar.kt | 2 +- .../syncadapter/resource/LocalCollection.kt | 2 +- .../syncadapter/resource/LocalGroup.kt | 2 +- .../syncadapter/resource/LocalTaskList.kt | 2 +- .../AddressBooksSyncAdapterService.kt | 2 - .../syncadapter/CalendarSyncManager.kt | 55 ++++- .../syncadapter/ContactsSyncManager.kt | 97 +++++++- .../syncadapter/syncadapter/SyncManager.kt | 231 ++++++++++++------ .../syncadapter/TasksSyncManager.kt | 55 ++++- .../syncadapter/ui/JournalItemActivity.kt | 6 +- .../ui/importlocal/ImportFragment.kt | 9 +- .../LocalCalendarImportFragment.kt | 2 +- .../importlocal/LocalContactImportFragment.kt | 4 +- 15 files changed, 385 insertions(+), 111 deletions(-) diff --git a/app/src/main/java/com/etesync/syncadapter/EtebaseLocalCache.kt b/app/src/main/java/com/etesync/syncadapter/EtebaseLocalCache.kt index d5068eec..5e94ff3c 100644 --- a/app/src/main/java/com/etesync/syncadapter/EtebaseLocalCache.kt +++ b/app/src/main/java/com/etesync/syncadapter/EtebaseLocalCache.kt @@ -11,12 +11,13 @@ import java.util.* File structure: cache_dir/ user1/ <--- the name of the user - stoken + stoken <-- the stokens of the collection fetch cols/ UID1/ - The uid of the first col ... UID2/ - The uid of the second col col <-- the col itself + stoken <-- the stoken of the items fetch items/ item_uid1 <-- the item with uid 1 item_uid2 @@ -51,6 +52,19 @@ class EtebaseLocalCache private constructor(context: Context, username: String) return if (stokenFile.exists()) stokenFile.readText() else null } + + fun collectionSaveStoken(colUid: String, stoken: String) { + val colDir = File(colsDir, colUid) + val stokenFile = File(colDir, "stoken") + stokenFile.writeText(stoken) + } + + fun collectionLoadStoken(colUid: String): String? { + val colDir = File(colsDir, colUid) + val stokenFile = File(colDir, "stoken") + return if (stokenFile.exists()) stokenFile.readText() else null + } + fun collectionList(colMgr: CollectionManager, withDeleted: Boolean = false): List { return colsDir.list().map { val colDir = File(colsDir, it) @@ -62,6 +76,15 @@ class EtebaseLocalCache private constructor(context: Context, username: String) } } + fun collectionGet(colMgr: CollectionManager, colUid: String): CachedCollection { + val colDir = File(colsDir, colUid) + val colFile = File(colDir, "col") + val content = colFile.readBytes() + return colMgr.cacheLoad(content).let { + CachedCollection(it, it.meta) + } + } + fun collectionSet(colMgr: CollectionManager, collection: Collection) { val colDir = File(colsDir, collection.uid) colDir.mkdir() diff --git a/app/src/main/java/com/etesync/syncadapter/resource/LocalAddressBook.kt b/app/src/main/java/com/etesync/syncadapter/resource/LocalAddressBook.kt index f288e4dd..b33919b6 100644 --- a/app/src/main/java/com/etesync/syncadapter/resource/LocalAddressBook.kt +++ b/app/src/main/java/com/etesync/syncadapter/resource/LocalAddressBook.kt @@ -344,7 +344,7 @@ class LocalAddressBook( return reallyDirty } - override fun findByUid(uid: String): LocalAddress? { + override fun findByFilename(uid: String): LocalAddress? { val found = findContactByUID(uid) if (found != null) { return found diff --git a/app/src/main/java/com/etesync/syncadapter/resource/LocalCalendar.kt b/app/src/main/java/com/etesync/syncadapter/resource/LocalCalendar.kt index e4d99a0a..8ce27a0d 100644 --- a/app/src/main/java/com/etesync/syncadapter/resource/LocalCalendar.kt +++ b/app/src/main/java/com/etesync/syncadapter/resource/LocalCalendar.kt @@ -165,7 +165,7 @@ class LocalCalendar private constructor( override fun findAll(): List = queryEvents(null, null) - override fun findByUid(uid: String): LocalEvent? + override fun findByFilename(uid: String): LocalEvent? = queryEvents(Events._SYNC_ID + " =? ", arrayOf(uid)).firstOrNull() fun processDirtyExceptions() { diff --git a/app/src/main/java/com/etesync/syncadapter/resource/LocalCollection.kt b/app/src/main/java/com/etesync/syncadapter/resource/LocalCollection.kt index 8e9d7a45..5a9b5340 100644 --- a/app/src/main/java/com/etesync/syncadapter/resource/LocalCollection.kt +++ b/app/src/main/java/com/etesync/syncadapter/resource/LocalCollection.kt @@ -16,7 +16,7 @@ interface LocalCollection> { fun findWithoutFileName(): List fun findAll(): List - fun findByUid(uid: String): T? + fun findByFilename(uid: String): T? fun count(): Long diff --git a/app/src/main/java/com/etesync/syncadapter/resource/LocalGroup.kt b/app/src/main/java/com/etesync/syncadapter/resource/LocalGroup.kt index b9ffaaa0..e0ff5cf6 100644 --- a/app/src/main/java/com/etesync/syncadapter/resource/LocalGroup.kt +++ b/app/src/main/java/com/etesync/syncadapter/resource/LocalGroup.kt @@ -63,7 +63,7 @@ class LocalGroup : AndroidGroup, LocalAddress { // insert memberships val membersIds = members.map {uid -> Constants.log.fine("Assigning member: $uid") - val contact = addressBook.findByUid(uid) as LocalContact? + val contact = addressBook.findByFilename(uid) as LocalContact? if (contact != null) contact.id else null }.filterNotNull() diff --git a/app/src/main/java/com/etesync/syncadapter/resource/LocalTaskList.kt b/app/src/main/java/com/etesync/syncadapter/resource/LocalTaskList.kt index 686da075..dd716adb 100644 --- a/app/src/main/java/com/etesync/syncadapter/resource/LocalTaskList.kt +++ b/app/src/main/java/com/etesync/syncadapter/resource/LocalTaskList.kt @@ -126,7 +126,7 @@ class LocalTaskList private constructor( override fun findWithoutFileName(): List = queryTasks(Tasks._SYNC_ID + " IS NULL", null) - override fun findByUid(uid: String): LocalTask? + override fun findByFilename(uid: String): LocalTask? = queryTasks(Tasks._SYNC_ID + " =? ", arrayOf(uid)).firstOrNull() override fun count(): Long { diff --git a/app/src/main/java/com/etesync/syncadapter/syncadapter/AddressBooksSyncAdapterService.kt b/app/src/main/java/com/etesync/syncadapter/syncadapter/AddressBooksSyncAdapterService.kt index 2508a621..db7bc054 100644 --- a/app/src/main/java/com/etesync/syncadapter/syncadapter/AddressBooksSyncAdapterService.kt +++ b/app/src/main/java/com/etesync/syncadapter/syncadapter/AddressBooksSyncAdapterService.kt @@ -88,8 +88,6 @@ class AddressBooksSyncAdapterService : SyncAdapterService() { val local = LocalAddressBook.find(context, provider, account) - val updateColors = settings.manageCalendarColors - // delete obsolete local calendar for (addressBook in local) { val url = addressBook.url diff --git a/app/src/main/java/com/etesync/syncadapter/syncadapter/CalendarSyncManager.kt b/app/src/main/java/com/etesync/syncadapter/syncadapter/CalendarSyncManager.kt index b65b942a..6d0a760b 100644 --- a/app/src/main/java/com/etesync/syncadapter/syncadapter/CalendarSyncManager.kt +++ b/app/src/main/java/com/etesync/syncadapter/syncadapter/CalendarSyncManager.kt @@ -17,6 +17,7 @@ import at.bitfire.ical4android.CalendarStorageException import at.bitfire.ical4android.Event import at.bitfire.ical4android.InvalidCalendarException import at.bitfire.vcard4android.ContactsStorageException +import com.etebase.client.Item import com.etesync.syncadapter.AccountSettings import com.etesync.syncadapter.Constants import com.etesync.syncadapter.R @@ -59,7 +60,9 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra if (!super.prepare()) return false - journal = JournalEntryManager(httpClient.okHttpClient, remote, localCalendar().name!!) + if (isLegacy) { + journal = JournalEntryManager(httpClient.okHttpClient, remote, localCalendar().name!!) + } return true } @@ -77,6 +80,32 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra return localCollection as LocalCalendar } + override fun processItem(item: Item) { + val local = localCollection!!.findByFilename(item.uid) + + if (!item.isDeleted) { + val inputReader = StringReader(String(item.content)) + + val events = Event.eventsFromReader(inputReader) + if (events.size == 0) { + Logger.log.warning("Received VCard without data, ignoring") + return + } else if (events.size > 1) { + Logger.log.warning("Received multiple VCALs, using first one") + } + + val event = events[0] + processEvent(item, event, local) + } else { + if (local != null) { + Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server") + local.delete() + } else { + Logger.log.warning("Tried deleting a non-existent record: " + item.uid) + } + } + } + @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, InvalidCalendarException::class) override fun processSyncEntryImpl(cEntry: SyncEntry) { val inputReader = StringReader(cEntry.content) @@ -90,10 +119,10 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra } val event = events[0] - val local = localCollection!!.findByUid(event.uid!!) + val local = localCollection!!.findByFilename(event.uid!!) if (cEntry.isAction(SyncEntry.Actions.ADD) || cEntry.isAction(SyncEntry.Actions.CHANGE)) { - processEvent(event, local) + legacyProcessEvent(event, local) } else { if (local != null) { Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server") @@ -138,8 +167,26 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra } } + private fun processEvent(item: Item, newData: Event, _localEvent: LocalEvent?): LocalEvent { + var localEvent = _localEvent + // delete local event, if it exists + if (localEvent != null) { + Logger.log.info("Updating " + newData.uid + " in local calendar") + localEvent.eTag = item.etag + localEvent.update(newData) + syncResult.stats.numUpdates++ + } else { + Logger.log.info("Adding " + newData.uid + " to local calendar") + localEvent = LocalEvent(localCalendar(), newData, item.uid, item.etag) + localEvent.add() + syncResult.stats.numInserts++ + } + + return localEvent + } + @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class) - private fun processEvent(newData: Event, _localEvent: LocalEvent?): LocalEvent { + private fun legacyProcessEvent(newData: Event, _localEvent: LocalEvent?): LocalEvent { var localEvent = _localEvent // delete local event, if it exists if (localEvent != null) { diff --git a/app/src/main/java/com/etesync/syncadapter/syncadapter/ContactsSyncManager.kt b/app/src/main/java/com/etesync/syncadapter/syncadapter/ContactsSyncManager.kt index 8bc2a875..ee17b13f 100644 --- a/app/src/main/java/com/etesync/syncadapter/syncadapter/ContactsSyncManager.kt +++ b/app/src/main/java/com/etesync/syncadapter/syncadapter/ContactsSyncManager.kt @@ -13,12 +13,14 @@ import android.content.* import android.os.Bundle import android.provider.ContactsContract import at.bitfire.ical4android.CalendarStorageException +import at.bitfire.ical4android.Event import at.bitfire.vcard4android.BatchOperation import at.bitfire.vcard4android.Contact import at.bitfire.vcard4android.ContactsStorageException import com.etesync.journalmanager.Exceptions import com.etesync.journalmanager.JournalEntryManager import com.etesync.journalmanager.model.SyncEntry +import com.etebase.client.Item import com.etesync.syncadapter.AccountSettings import com.etesync.syncadapter.Constants import com.etesync.syncadapter.HttpClient @@ -77,7 +79,9 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra } } - journal = JournalEntryManager(httpClient.okHttpClient, remote, localAddressBook.url) + if (isLegacy) { + journal = JournalEntryManager(httpClient.okHttpClient, remote, localAddressBook.url) + } return true } @@ -127,6 +131,34 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra return localCollection as LocalAddressBook } + override fun processItem(item: Item) { + val uid = item.meta.name!! + + val local = localCollection!!.findByFilename(uid) + + if (!item.isDeleted) { + val inputReader = StringReader(String(item.content)) + + val contacts = Contact.fromReader(inputReader, resourceDownloader) + if (contacts.size == 0) { + Logger.log.warning("Received VCard without data, ignoring") + return + } else if (contacts.size > 1) { + Logger.log.warning("Received multiple VCALs, using first one") + } + + val contact = contacts[0] + processContact(item, contact, local) + } else { + if (local != null) { + Logger.log.info("Removing local record which has been deleted on the server") + local.delete() + } else { + Logger.log.warning("Tried deleting a non-existent record: " + uid) + } + } + } + @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class) override fun processSyncEntryImpl(cEntry: SyncEntry) { val inputReader = StringReader(cEntry.content) @@ -139,10 +171,10 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra Logger.log.warning("Received multiple VCards, using first one") val contact = contacts[0] - val local = localCollection!!.findByUid(contact.uid!!) + val local = localCollection!!.findByFilename(contact.uid!!) if (cEntry.isAction(SyncEntry.Actions.ADD) || cEntry.isAction(SyncEntry.Actions.CHANGE)) { - processContact(contact, local) + legacyProcessContact(contact, local) } else { if (local != null) { Logger.log.info("Removing local record which has been deleted on the server") @@ -153,8 +185,65 @@ constructor(context: Context, account: Account, settings: AccountSettings, extra } } + private fun processContact(item: Item, newData: Contact, _local: LocalAddress?): LocalAddress { + var local = _local + val uuid = newData.uid + // update local contact, if it exists + if (local != null) { + Logger.log.log(Level.INFO, "Updating $uuid in local address book") + + if (local is LocalGroup && newData.group) { + // update group + val group: LocalGroup = local + group.eTag = item.etag + group.update(newData) + syncResult.stats.numUpdates++ + + } else if (local is LocalContact && !newData.group) { + // update contact + val contact: LocalContact = local + contact.eTag = item.etag + contact.update(newData) + syncResult.stats.numUpdates++ + + } else { + // group has become an individual contact or vice versa + try { + local.delete() + local = null + } catch (e: CalendarStorageException) { + // CalendarStorageException is not used by LocalGroup and LocalContact + } + + } + } + + if (local == null) { + if (newData.group) { + Logger.log.log(Level.INFO, "Creating local group", item.uid) + val group = LocalGroup(localAddressBook(), newData, item.uid, item.etag) + group.add() + + local = group + } else { + Logger.log.log(Level.INFO, "Creating local contact", item.uid) + val contact = LocalContact(localAddressBook(), newData, item.uid, item.etag) + contact.add() + + local = contact + } + syncResult.stats.numInserts++ + } + + if (LocalContact.HASH_HACK && local is LocalContact) + // workaround for Android 7 which sets DIRTY flag when only meta-data is changed + local.updateHashCode(null) + + return local + } + @Throws(IOException::class, ContactsStorageException::class) - private fun processContact(newData: Contact, _local: LocalAddress?): LocalAddress { + private fun legacyProcessContact(newData: Contact, _local: LocalAddress?): LocalAddress { var local = _local val uuid = newData.uid // update local contact, if it exists diff --git a/app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.kt b/app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.kt index b6f230e0..4879da4e 100644 --- a/app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.kt +++ b/app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.kt @@ -16,6 +16,7 @@ import android.os.Bundle import at.bitfire.ical4android.CalendarStorageException import at.bitfire.ical4android.InvalidCalendarException import at.bitfire.vcard4android.ContactsStorageException +import com.etebase.client.* import com.etesync.syncadapter.* import com.etesync.syncadapter.Constants.KEY_ACCOUNT import com.etesync.journalmanager.Crypto @@ -25,6 +26,8 @@ import com.etesync.journalmanager.model.SyncEntry import com.etesync.syncadapter.log.Logger import com.etesync.syncadapter.model.* import com.etesync.journalmanager.model.SyncEntry.Actions.ADD +import com.etesync.syncadapter.HttpClient +import com.etesync.syncadapter.R import com.etesync.syncadapter.resource.* import com.etesync.syncadapter.ui.AccountsActivity import com.etesync.syncadapter.ui.DebugInfoActivity @@ -41,21 +44,30 @@ import kotlin.concurrent.withLock abstract class SyncManager> @Throws(Exceptions.IntegrityException::class, Exceptions.GenericCryptoException::class) constructor(protected val context: Context, protected val account: Account, protected val settings: AccountSettings, protected val extras: Bundle, protected val authority: String, protected val syncResult: SyncResult, journalUid: String, protected val serviceType: CollectionInfo.Type, accountName: String): Closeable { + // FIXME: remove all of the lateinit once we remove legacy (and make immutable) + // RemoteEntries and the likes are probably also just relevant for legacy + protected val isLegacy: Boolean = settings.isLegacy protected val notificationManager: SyncNotification - protected val info: CollectionInfo + protected lateinit var info: CollectionInfo protected var localCollection: LocalCollection? = null protected var httpClient: HttpClient + protected lateinit var etebaseLocalCache: EtebaseLocalCache + protected lateinit var etebase: com.etebase.client.Account + protected lateinit var colMgr: CollectionManager + protected lateinit var itemMgr: ItemManager + protected lateinit var cachedCollection: CachedCollection + protected var journal: JournalEntryManager? = null private var _journalEntity: JournalEntity? = null private var numDiscarded = 0 - private val crypto: Crypto.CryptoManager + private lateinit var crypto: Crypto.CryptoManager - private val data: MyEntityDataStore + private lateinit var data: MyEntityDataStore /** * remote CTag (uuid of the last entry on the server). We update it when we fetch/push and save when everything works. @@ -89,21 +101,31 @@ constructor(protected val context: Context, protected val account: Account, prot // create HttpClient with given logger httpClient = HttpClient.Builder(context, settings).setForeground(false).build() - data = (context.applicationContext as App).data - val serviceEntity = JournalModel.Service.fetchOrCreate(data, accountName, serviceType) - info = JournalEntity.fetch(data, serviceEntity, journalUid)!!.info + if (isLegacy) { + data = (context.applicationContext as App).data + val serviceEntity = JournalModel.Service.fetchOrCreate(data, accountName, serviceType) + info = JournalEntity.fetch(data, serviceEntity, journalUid)!!.info - // dismiss previous error notifications - notificationManager = SyncNotification(context, journalUid, notificationId()) - notificationManager.cancel() + Logger.log.info(String.format(Locale.getDefault(), "Syncing collection %s (version: %d)", journalUid, info.version)) - Logger.log.info(String.format(Locale.getDefault(), "Syncing collection %s (version: %d)", journalUid, info.version)) - - if (journalEntity.encryptedKey != null) { - crypto = Crypto.CryptoManager(info.version, settings.keyPair!!, journalEntity.encryptedKey) + if (journalEntity.encryptedKey != null) { + crypto = Crypto.CryptoManager(info.version, settings.keyPair!!, journalEntity.encryptedKey) + } else { + crypto = Crypto.CryptoManager(info.version, settings.password(), info.uid!!) + } } else { - crypto = Crypto.CryptoManager(info.version, settings.password(), info.uid!!) + etebaseLocalCache = EtebaseLocalCache.getInstance(context, accountName) + etebase = EtebaseLocalCache.getEtebase(context, httpClient.okHttpClient, settings) + colMgr = etebase.collectionManager + synchronized(etebaseLocalCache) { + cachedCollection = etebaseLocalCache.collectionGet(colMgr, journalUid) + } + itemMgr = colMgr.getItemManager(cachedCollection.col) } + + // dismiss previous error notifications + notificationManager = SyncNotification(context, journalUid, notificationId()) + notificationManager.cancel() } protected abstract fun notificationId(): Int @@ -128,48 +150,78 @@ constructor(protected val context: Context, protected val account: Account, prot Logger.log.info("Sync phase: " + context.getString(syncPhase)) prepareFetch() - do { - if (Thread.interrupted()) - throw InterruptedException() - syncPhase = R.string.sync_phase_fetch_entries - Logger.log.info("Sync phase: " + context.getString(syncPhase)) - fetchEntries() - - if (Thread.interrupted()) - throw InterruptedException() - syncPhase = R.string.sync_phase_apply_remote_entries - Logger.log.info("Sync phase: " + context.getString(syncPhase)) - applyRemoteEntries() - } while (remoteEntries!!.size == MAX_FETCH) - - do { - if (Thread.interrupted()) - throw InterruptedException() - syncPhase = R.string.sync_phase_prepare_local - Logger.log.info("Sync phase: " + context.getString(syncPhase)) - prepareLocal() - - /* Create journal entries out of local changes. */ - if (Thread.interrupted()) - throw InterruptedException() - syncPhase = R.string.sync_phase_create_local_entries - Logger.log.info("Sync phase: " + context.getString(syncPhase)) - createLocalEntries() - - if (Thread.interrupted()) - throw InterruptedException() - syncPhase = R.string.sync_phase_apply_local_entries - Logger.log.info("Sync phase: " + context.getString(syncPhase)) - /* FIXME: Skipping this now, because we already override with remote. - applyLocalEntries(); - */ - - if (Thread.interrupted()) - throw InterruptedException() - syncPhase = R.string.sync_phase_push_entries - Logger.log.info("Sync phase: " + context.getString(syncPhase)) - pushEntries() - } while (localEntries!!.size == MAX_PUSH) + if (isLegacy) { + do { + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_fetch_entries + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + fetchEntries() + + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_apply_remote_entries + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + applyRemoteEntries() + } while (remoteEntries!!.size == MAX_FETCH) + + do { + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_prepare_local + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + prepareLocal() + + /* Create journal entries out of local changes. */ + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_create_local_entries + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + createLocalEntries() + + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_apply_local_entries + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + /* FIXME: Skipping this now, because we already override with remote. + applyLocalEntries(); + */ + + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_push_entries + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + pushEntries() + } while (localEntries!!.size == MAX_PUSH) + } else { + var itemList: ItemListResponse? + var stoken = synchronized(etebaseLocalCache) { + etebaseLocalCache.collectionLoadStoken(cachedCollection.col.uid) + } + // Push local changes + + do { + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_fetch_entries + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + itemList = fetchItems(stoken) + if (itemList == null) { + break + } + + if (Thread.interrupted()) + throw InterruptedException() + syncPhase = R.string.sync_phase_apply_remote_entries + Logger.log.info("Sync phase: " + context.getString(syncPhase)) + applyRemoteItems(itemList) + + stoken = itemList.stoken + synchronized(etebaseLocalCache) { + etebaseLocalCache.collectionSaveStoken(cachedCollection.col.uid, stoken!!) + } + } while (!itemList!!.isDone) + } /* Cleanup and finalize changes */ if (Thread.interrupted()) @@ -241,7 +293,8 @@ constructor(protected val context: Context, protected val account: Account, prot private fun notifyUserOnSync() { val changeNotification = context.defaultSharedPreferences.getBoolean(App.CHANGE_NOTIFICATION, true) - if (remoteEntries!!.isEmpty() || !changeNotification) { + val remoteEntries = remoteEntries + if ((remoteEntries == null) || remoteEntries.isEmpty() || !changeNotification) { return } val notificationHelper = SyncNotification(context, @@ -250,7 +303,7 @@ constructor(protected val context: Context, protected val account: Account, prot var deleted = 0 var added = 0 var changed = 0 - for (entry in remoteEntries!!) { + for (entry in remoteEntries) { val cEntry = SyncEntry.fromJournalEntry(crypto, entry) val action = cEntry.action when (action) { @@ -287,6 +340,14 @@ constructor(protected val context: Context, protected val account: Account, prot return true } + protected abstract fun processItem(item: Item) + + private fun persistItem(item: Item) { + synchronized(etebaseLocalCache) { + etebaseLocalCache.itemSet(itemMgr, cachedCollection.col.uid, item) + } + } + @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, InvalidCalendarException::class) protected abstract fun processSyncEntryImpl(cEntry: SyncEntry) @@ -319,36 +380,46 @@ constructor(protected val context: Context, protected val account: Account, prot } } - @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, Exceptions.HttpException::class, InvalidCalendarException::class, InterruptedException::class) - protected fun applyLocalEntries() { - // FIXME: Need a better strategy - // We re-apply local entries so our changes override whatever was written in the remote. - val strTotal = localEntries!!.size.toString() + @Throws(IOException::class, CalendarStorageException::class, ContactsStorageException::class) + protected fun prepareFetch() { + if (isLegacy) { + remoteCTag = journalEntity.getLastUid(data) + } else { + remoteCTag = cachedCollection.col.stoken + } + } + + private fun fetchItems(stoken: String?): ItemListResponse? { + if (remoteCTag != stoken) { + val ret = itemMgr.list(FetchOptions().stoken(stoken)) + Logger.log.info("Fetched items. Done=${ret.isDone}") + return ret + } else { + Logger.log.info("Skipping fetch because local lastUid == remoteLastUid (${remoteCTag})") + return null + } + } + + private fun applyRemoteItems(itemList: ItemListResponse) { + val items = itemList.data + // Process new vcards from server + val size = items.size var i = 0 - for (entry in localEntries!!) { + for (item in items) { if (Thread.interrupted()) { throw InterruptedException() } i++ - Logger.log.info("Processing (" + i.toString() + "/" + strTotal + ") " + entry.toString()) + Logger.log.info("Processing (${i}/${size}) UID=${item.uid} Etag=${item.etag}") - val cEntry = SyncEntry.fromJournalEntry(crypto, entry) - if (cEntry.isAction(SyncEntry.Actions.DELETE)) { - continue - } - Logger.log.info("Processing resource for journal entry") - processSyncEntry(cEntry) + processItem(item) + persistItem(item) } } - @Throws(IOException::class, CalendarStorageException::class, ContactsStorageException::class) - protected fun prepareFetch() { - remoteCTag = journalEntity.getLastUid(data) - } - @Throws(Exceptions.HttpException::class, ContactsStorageException::class, CalendarStorageException::class, Exceptions.IntegrityException::class) - protected fun fetchEntries() { + private fun fetchEntries() { val count = data.count(EntryEntity::class.java).where(EntryEntity.JOURNAL.eq(journalEntity)).get().value() if (remoteCTag != null && count == 0) { // If we are updating an existing installation with no saved journal, we need to add @@ -377,7 +448,7 @@ constructor(protected val context: Context, protected val account: Account, prot } @Throws(IOException::class, ContactsStorageException::class, CalendarStorageException::class, InvalidCalendarException::class, InterruptedException::class) - protected fun applyRemoteEntries() { + private fun applyRemoteEntries() { // Process new vcards from server val strTotal = remoteEntries!!.size.toString() var i = 0 @@ -406,7 +477,7 @@ constructor(protected val context: Context, protected val account: Account, prot } @Throws(Exceptions.HttpException::class, IOException::class, ContactsStorageException::class, CalendarStorageException::class) - protected fun pushEntries() { + private fun pushEntries() { // upload dirty contacts var pushed = 0 // FIXME: Deal with failure (someone else uploaded before we go here) @@ -510,7 +581,7 @@ constructor(protected val context: Context, protected val account: Account, prot /** */ @Throws(CalendarStorageException::class, ContactsStorageException::class, FileNotFoundException::class) - protected fun prepareLocal() { + private fun prepareLocal() { localDeleted = processLocallyDeleted() localDirty = localCollection!!.findDirty(MAX_PUSH) // This is done after fetching the local dirty so all the ones we are using will be prepared diff --git a/app/src/main/java/com/etesync/syncadapter/syncadapter/TasksSyncManager.kt b/app/src/main/java/com/etesync/syncadapter/syncadapter/TasksSyncManager.kt index 70de515e..f0b1a8de 100644 --- a/app/src/main/java/com/etesync/syncadapter/syncadapter/TasksSyncManager.kt +++ b/app/src/main/java/com/etesync/syncadapter/syncadapter/TasksSyncManager.kt @@ -13,6 +13,7 @@ import android.content.Context import android.content.SyncResult import android.os.Bundle import at.bitfire.ical4android.Task +import com.etebase.client.Item import com.etesync.syncadapter.AccountSettings import com.etesync.syncadapter.Constants import com.etesync.syncadapter.R @@ -58,7 +59,9 @@ class TasksSyncManager( if (!super.prepare()) return false - journal = JournalEntryManager(httpClient.okHttpClient, remote, localTaskList().url!!) + if (isLegacy) { + journal = JournalEntryManager(httpClient.okHttpClient, remote, localTaskList().url!!) + } return true } @@ -68,6 +71,32 @@ class TasksSyncManager( return localCollection as LocalTaskList } + override fun processItem(item: Item) { + val local = localCollection!!.findByFilename(item.uid) + + if (!item.isDeleted) { + val inputReader = StringReader(String(item.content)) + + val tasks = Task.tasksFromReader(inputReader) + if (tasks.size == 0) { + Logger.log.warning("Received VCard without data, ignoring") + return + } else if (tasks.size > 1) { + Logger.log.warning("Received multiple VCALs, using first one") + } + + val task = tasks[0] + processTask(item, task, local) + } else { + if (local != null) { + Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server") + local.delete() + } else { + Logger.log.warning("Tried deleting a non-existent record: " + item.uid) + } + } + } + override fun processSyncEntryImpl(cEntry: SyncEntry) { val inputReader = StringReader(cEntry.content) @@ -80,10 +109,10 @@ class TasksSyncManager( } val event = tasks[0] - val local = localCollection!!.findByUid(event.uid!!) + val local = localCollection!!.findByFilename(event.uid!!) if (cEntry.isAction(SyncEntry.Actions.ADD) || cEntry.isAction(SyncEntry.Actions.CHANGE)) { - processTask(event, local) + legacyProcessTask(event, local) } else { if (local != null) { Logger.log.info("Removing local record #" + local.id + " which has been deleted on the server") @@ -94,7 +123,25 @@ class TasksSyncManager( } } - private fun processTask(newData: Task, _localTask: LocalTask?): LocalTask { + private fun processTask(item: Item, newData: Task, _localTask: LocalTask?): LocalTask { + var localTask = _localTask + // delete local Task, if it exists + if (localTask != null) { + Logger.log.info("Updating " + item.uid + " in local calendar") + localTask.eTag = item.etag + localTask.update(newData) + syncResult.stats.numUpdates++ + } else { + Logger.log.info("Adding " + item.uid + " to local calendar") + localTask = LocalTask(localTaskList(), newData, item.uid, item.etag) + localTask.add() + syncResult.stats.numInserts++ + } + + return localTask + } + + private fun legacyProcessTask(newData: Task, _localTask: LocalTask?): LocalTask { var localTask = _localTask // delete local Task, if it exists if (localTask != null) { diff --git a/app/src/main/java/com/etesync/syncadapter/ui/JournalItemActivity.kt b/app/src/main/java/com/etesync/syncadapter/ui/JournalItemActivity.kt index 2f4a18bc..0a6fc2c6 100644 --- a/app/src/main/java/com/etesync/syncadapter/ui/JournalItemActivity.kt +++ b/app/src/main/java/com/etesync/syncadapter/ui/JournalItemActivity.kt @@ -108,7 +108,7 @@ class JournalItemActivity : BaseActivity(), Refreshable { val provider = contentResolver.acquireContentProviderClient(CalendarContract.CONTENT_URI)!! val localCalendar = LocalCalendar.findByName(account, provider, LocalCalendar.Factory, info.uid!!)!! val event = Event.eventsFromReader(StringReader(syncEntry.content))[0] - var localEvent = localCalendar.findByUid(event.uid!!) + var localEvent = localCalendar.findByFilename(event.uid!!) if (localEvent != null) { localEvent.updateAsDirty(event) } else { @@ -121,7 +121,7 @@ class JournalItemActivity : BaseActivity(), Refreshable { val provider = TaskProvider.acquire(this, it)!! val localTaskList = LocalTaskList.findByName(account, provider, LocalTaskList.Factory, info.uid!!)!! val task = Task.tasksFromReader(StringReader(syncEntry.content))[0] - var localTask = localTaskList.findByUid(task.uid!!) + var localTask = localTaskList.findByFilename(task.uid!!) if (localTask != null) { localTask.updateAsDirty(task) } else { @@ -137,7 +137,7 @@ class JournalItemActivity : BaseActivity(), Refreshable { if (contact.group) { // FIXME: not currently supported } else { - var localContact = localAddressBook.findByUid(contact.uid!!) as LocalContact? + var localContact = localAddressBook.findByFilename(contact.uid!!) as LocalContact? if (localContact != null) { localContact.updateAsDirty(contact) } else { diff --git a/app/src/main/java/com/etesync/syncadapter/ui/importlocal/ImportFragment.kt b/app/src/main/java/com/etesync/syncadapter/ui/importlocal/ImportFragment.kt index 94d12318..5decaa76 100644 --- a/app/src/main/java/com/etesync/syncadapter/ui/importlocal/ImportFragment.kt +++ b/app/src/main/java/com/etesync/syncadapter/ui/importlocal/ImportFragment.kt @@ -15,7 +15,6 @@ import android.provider.CalendarContract import android.provider.ContactsContract import androidx.fragment.app.DialogFragment import at.bitfire.ical4android.* -import at.bitfire.ical4android.TaskProvider.Companion.OPENTASK_PROVIDERS import at.bitfire.vcard4android.BatchOperation import at.bitfire.vcard4android.Contact import at.bitfire.vcard4android.ContactsStorageException @@ -255,7 +254,7 @@ class ImportFragment : DialogFragment() { for (event in events) { try { - var localEvent = localCalendar.findByUid(event.uid!!) + var localEvent = localCalendar.findByFilename(event.uid!!) if (localEvent != null) { localEvent.updateAsDirty(event) result.updated++ @@ -309,7 +308,7 @@ class ImportFragment : DialogFragment() { for (task in tasks) { try { - var localTask = localTaskList.findByUid(task.uid!!) + var localTask = localTaskList.findByFilename(task.uid!!) if (localTask != null) { localTask.updateAsDirty(task) result.updated++ @@ -353,7 +352,7 @@ class ImportFragment : DialogFragment() { for (contact in contacts.filter { contact -> !contact.group }) { try { - var localContact = localAddressBook.findByUid(contact.uid!!) as LocalContact? + var localContact = localAddressBook.findByFilename(contact.uid!!) as LocalContact? if (localContact != null) { localContact.updateAsDirty(contact) @@ -386,7 +385,7 @@ class ImportFragment : DialogFragment() { } val group = contact - var localGroup: LocalGroup? = localAddressBook.findByUid(group.uid!!) as LocalGroup? + var localGroup: LocalGroup? = localAddressBook.findByFilename(group.uid!!) as LocalGroup? if (localGroup != null) { localGroup.updateAsDirty(group, memberIds) diff --git a/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalCalendarImportFragment.kt b/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalCalendarImportFragment.kt index c740b8ea..6218dca8 100644 --- a/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalCalendarImportFragment.kt +++ b/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalCalendarImportFragment.kt @@ -222,7 +222,7 @@ class LocalCalendarImportFragment : ListFragment() { var localEvent = if (event == null || event.uid == null) null else - localCalendar.findByUid(event.uid!!) + localCalendar.findByFilename(event.uid!!) if (localEvent != null) { localEvent.updateAsDirty(event!!) diff --git a/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalContactImportFragment.kt b/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalContactImportFragment.kt index 1f0451be..079e0318 100644 --- a/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalContactImportFragment.kt +++ b/app/src/main/java/com/etesync/syncadapter/ui/importlocal/LocalContactImportFragment.kt @@ -158,7 +158,7 @@ class LocalContactImportFragment : Fragment() { var localContact: LocalContact? = if (contact.uid == null) null else - addressBook.findByUid(contact.uid!!) as LocalContact? + addressBook.findByFilename(contact.uid!!) as LocalContact? if (localContact != null) { localContact.updateAsDirty(contact) @@ -189,7 +189,7 @@ class LocalContactImportFragment : Fragment() { var localGroup: LocalGroup? = if (group.uid == null) null else - addressBook.findByUid(group.uid!!) as LocalGroup? + addressBook.findByFilename(group.uid!!) as LocalGroup? if (localGroup != null) { localGroup.updateAsDirty(group, members)