Sync: fix the sync to also do the initial preparation in chunks

This fixes issues with Android killing the sync when making massive
syncs such as importing 10,000 events.

Fixes #69

app/src/main/java/com/etesync/syncadapter/resource/LocalAddressBook.kt

@@ -207,14 +207,14 @@ class LocalAddressBook(
      * Returns an array of local contacts/groups which have been changed locally (DIRTY != 0).
      * @throws RemoteException on content provider errors
      */
-    override fun findDirty() =
+    override fun findDirty(limit: Int?) =
             if (includeGroups)
-                findDirtyContacts() + findDirtyGroups()
+                findDirtyContacts(limit) + findDirtyGroups(limit) // FIXME: Doesn't rspect limit correctly, but not a big deal for now
             else
-                findDirtyContacts()
+                findDirtyContacts(limit)
 
-    fun findDirtyContacts() = queryContacts("${RawContacts.DIRTY}!=0 AND ${RawContacts.DELETED}==0", null)
-    fun findDirtyGroups() = queryGroups("${Groups.DIRTY}!=0 AND ${Groups.DELETED}==0", null)
+    fun findDirtyContacts(limit: Int? = null) = queryContacts("${RawContacts.DIRTY}!=0 AND ${RawContacts.DELETED}==0", null, if (limit != null) "${RawContacts._ID} ASC LIMIT $limit" else null)
+    fun findDirtyGroups(limit: Int? = null) = queryGroups("${Groups.DIRTY}!=0 AND ${Groups.DELETED}==0", null, if (limit != null) "${Groups._ID} ASC LIMIT $limit" else null)
 
     /**
      * Returns an array of local contacts which don't have a file name yet.

app/src/main/java/com/etesync/syncadapter/resource/LocalCalendar.kt

@@ -97,11 +97,12 @@ class LocalCalendar private constructor(
     override fun findDeleted() =
             queryEvents("${Events.DELETED}!=0 AND ${Events.ORIGINAL_ID} IS NULL", null)
 
-    override fun findDirty(): List<LocalEvent> {
+    override fun findDirty(limit: Int?): List<LocalEvent> {
         val dirty = LinkedList<LocalEvent>()
+        val sortOrder = if (limit != null) "${Events._ID} ASC LIMIT $limit" else null
 
         // get dirty events which are required to have an increased SEQUENCE value
-        for (localEvent in queryEvents("${Events.DIRTY}!=0 AND ${Events.DELETED}==0 AND ${Events.ORIGINAL_ID} IS NULL", null)) {
+        for (localEvent in queryEvents("${Events.DIRTY}!=0 AND ${Events.DELETED}==0 AND ${Events.ORIGINAL_ID} IS NULL", null, sortOrder)) {
             val event = localEvent.event!!
             val sequence = event.sequence
             if (event.sequence == null)      // sequence has not been assigned yet (i.e. this event was just locally created)

app/src/main/java/com/etesync/syncadapter/resource/LocalCollection.kt

@@ -12,7 +12,7 @@ interface LocalCollection<out T: LocalResource<*>> {
     val url: String?
 
     fun findDeleted(): List<T>
-    fun findDirty(): List<T>
+    fun findDirty(limit: Int? = null): List<T>
     fun findWithoutFileName(): List<T>
     fun findAll(): List<T>
 

app/src/main/java/com/etesync/syncadapter/resource/LocalTaskList.kt

@@ -74,8 +74,9 @@ class LocalTaskList private constructor(
 
     override fun findDeleted() = queryTasks("${Tasks._DELETED}!=0", null)
 
-    override fun findDirty(): List<LocalTask> {
-        val tasks = queryTasks("${Tasks._DIRTY}!=0 AND ${Tasks._DELETED}==0", null)
+    override fun findDirty(limit: Int?): List<LocalTask> {
+        val sortOrder = if (limit != null) "${Tasks._ID} ASC LIMIT $limit" else null
+        val tasks = queryTasks("${Tasks._DIRTY}!=0 AND ${Tasks._DELETED}==0", null, sortOrder)
         for (localTask in tasks) {
             val task = requireNotNull(localTask.task)
             val sequence = task.sequence

app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.kt

@@ -128,13 +128,6 @@ constructor(protected val context: Context, protected val account: Account, prot
             Logger.log.info("Sync phase: " + context.getString(syncPhase))
             queryCapabilities()
 
-            if (Thread.interrupted())
-                throw InterruptedException()
-            syncPhase = R.string.sync_phase_prepare_local
-            Logger.log.info("Sync phase: " + context.getString(syncPhase))
-            prepareLocal()
-            Logger.log.info("Locally changed: (dirty=${localDirty.size}  deleted=${localDeleted?.size}")
-
             do {
                 if (Thread.interrupted())
                     throw InterruptedException()
@@ -150,6 +143,12 @@ constructor(protected val context: Context, protected val account: Account, prot
             } while (remoteEntries!!.size == MAX_FETCH)
 
             do {
+                if (Thread.interrupted())
+                    throw InterruptedException()
+                syncPhase = R.string.sync_phase_prepare_local
+                Logger.log.info("Sync phase: " + context.getString(syncPhase))
+                prepareLocal()
+
                 /* Create journal entries out of local changes. */
                 if (Thread.interrupted())
                     throw InterruptedException()
@@ -317,6 +316,8 @@ constructor(protected val context: Context, protected val account: Account, prot
 
     @Throws(IOException::class, CalendarStorageException::class, ContactsStorageException::class)
     protected fun queryCapabilities() {
+        // FIXME: Needs to rename this function
+        remoteCTag = journalEntity.getLastUid(data)
     }
 
     @Throws(Exceptions.HttpException::class, ContactsStorageException::class, CalendarStorageException::class, Exceptions.IntegrityException::class)
@@ -456,10 +457,8 @@ constructor(protected val context: Context, protected val account: Account, prot
      */
     @Throws(CalendarStorageException::class, ContactsStorageException::class, FileNotFoundException::class)
     protected fun prepareLocal() {
-        remoteCTag = journalEntity.getLastUid(data)
-
         localDeleted = processLocallyDeleted()
-        localDirty = localCollection!!.findDirty()
+        localDirty = localCollection!!.findDirty(MAX_PUSH)
         // This is done after fetching the local dirty so all the ones we are using will be prepared
         prepareDirty()
     }

ical4android

@@ -1 +1 @@
-Subproject commit 642ee9cbf19a508a6b5178e18c0f8f8c6b9ff279
+Subproject commit 2cbeae210e8733314638cb1bdb1be77dcd6ef66e

vcard4android

@@ -1 +1 @@
-Subproject commit 7e52b7d0d8089cbebfb31272cd776630110bcd76
+Subproject commit b3f443f2d87fd544ad3fb823ea754c93eb0a245b