New sync logic for ContactsSyncAdapter, using dav4android and vcard4android

app/src/main/java/at/bitfire/davdroid/HttpClient.java

@@ -40,7 +40,7 @@ public class HttpClient extends OkHttpClient {
         super();
         initialize();
 
-        // authentication
+        // authentication and User-Agent
         if (preemptive)
             networkInterceptors().add(new PreemptiveAuthenticationInterceptor(username, password));
         else
@@ -68,7 +68,7 @@ public class HttpClient extends OkHttpClient {
             }
         });
         logging.setLevel(HttpLoggingInterceptor.Level.BODY);
-        networkInterceptors().add(logging);
+        interceptors().add(logging);
     }
 
 

app/src/main/java/at/bitfire/davdroid/resource/LocalAddressBook.java

@@ -11,6 +11,7 @@ import android.accounts.Account;
 import android.content.ContentProviderClient;
 import android.provider.ContactsContract;
 
+import at.bitfire.davdroid.Constants;
 import at.bitfire.vcard4android.AndroidAddressBook;
 import at.bitfire.vcard4android.AndroidContact;
 import at.bitfire.vcard4android.AndroidContactFactory;
@@ -25,9 +26,30 @@ public class LocalAddressBook extends AndroidAddressBook {
         super(account, provider, AndroidGroupFactory.INSTANCE, LocalContact.Factory.INSTANCE);
     }
 
-    /*LocalContact[] queryAll() throws ContactsStorageException {
+    public LocalContact[] getAll() throws ContactsStorageException {
-        LocalContact contacts[] = (LocalContact[])queryContacts(ContactsContract.RawContacts.DELETED + "=0", null);
+        LocalContact contacts[] = (LocalContact[])queryContacts(null, null);
         return contacts;
-    }*/
+    }
+
+    /**
+     * Returns an array of local contacts which have been deleted locally. (DELETED != 0).
+     */
+    public LocalContact[] getDeleted() throws ContactsStorageException {
+        return (LocalContact[])queryContacts(ContactsContract.RawContacts.DELETED + " != 0", null);
+    }
+
+    /**
+     * Returns an array of local contacts which have been changed locally (DIRTY != 0).
+     */
+    public LocalContact[] getDirty() throws ContactsStorageException {
+        return (LocalContact[])queryContacts(ContactsContract.RawContacts.DIRTY + " != 0", null);
+    }
+
+    /**
+     * Returns an array of local contacts which don't have a file name yet.
+     */
+    public LocalContact[] getWithoutFileName() throws ContactsStorageException {
+        return (LocalContact[])queryContacts(AndroidContact.COLUMN_FILENAME + " IS NULL", null);
+    }
 
 }

app/src/main/java/at/bitfire/davdroid/resource/LocalContact.java

@@ -8,19 +8,34 @@
 
 package at.bitfire.davdroid.resource;
 
+import android.content.ContentValues;
+import android.os.RemoteException;
+import android.provider.ContactsContract;
+
 import at.bitfire.vcard4android.AndroidAddressBook;
 import at.bitfire.vcard4android.AndroidContact;
 import at.bitfire.vcard4android.AndroidContactFactory;
 import at.bitfire.vcard4android.Contact;
+import at.bitfire.vcard4android.ContactsStorageException;
 
 public class LocalContact extends AndroidContact {
 
-    protected LocalContact(AndroidAddressBook addressBook, long id) {
+    protected LocalContact(AndroidAddressBook addressBook, long id, String fileName, String eTag) {
-        super(addressBook, id);
+        super(addressBook, id, fileName, eTag);
     }
 
-    public LocalContact(AndroidAddressBook addressBook, Contact contact) {
+    public LocalContact(AndroidAddressBook addressBook, Contact contact, String fileName, String eTag) {
-        super(addressBook, contact);
+        super(addressBook, contact, fileName, eTag);
+    }
+
+    public void updateUID(String uid) throws ContactsStorageException {
+        try {
+            ContentValues values = new ContentValues(1);
+            values.put(COLUMN_UID, uid);
+            addressBook.provider.update(rawContactSyncURI(), values, null, null);
+        } catch (RemoteException e) {
+            throw new ContactsStorageException("Couldn't update UID", e);
+        }
     }
 
 
@@ -28,13 +43,13 @@ public class LocalContact extends AndroidContact {
         static final Factory INSTANCE = new Factory();
 
         @Override
-        public LocalContact newInstance(AndroidAddressBook addressBook, long id) {
+        public LocalContact newInstance(AndroidAddressBook addressBook, long id, String fileName, String eTag) {
-            return new LocalContact(addressBook, id);
+            return new LocalContact(addressBook, id, fileName, eTag);
         }
 
         @Override
-        public LocalContact newInstance(AndroidAddressBook addressBook, Contact contact) {
+        public LocalContact newInstance(AndroidAddressBook addressBook, Contact contact, String fileName, String eTag) {
-            return new LocalContact(addressBook, contact);
+            return new LocalContact(addressBook, contact, fileName, eTag);
         }
 
         public LocalContact[] newArray(int size) {

app/src/main/java/at/bitfire/davdroid/syncadapter/ContactsSyncAdapterService.java

@@ -16,22 +16,36 @@ import android.content.Intent;
 import android.content.SyncResult;
 import android.os.Bundle;
 import android.os.IBinder;
+import android.text.TextUtils;
 import android.util.Log;
 
 import com.squareup.okhttp.HttpUrl;
 import com.squareup.okhttp.MediaType;
+import com.squareup.okhttp.RequestBody;
 import com.squareup.okhttp.ResponseBody;
 
 import org.apache.commons.io.Charsets;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
 import at.bitfire.dav4android.DavAddressBook;
 import at.bitfire.dav4android.DavResource;
+import at.bitfire.dav4android.exception.DavException;
+import at.bitfire.dav4android.exception.HttpException;
+import at.bitfire.dav4android.property.GetETag;
 import at.bitfire.dav4android.property.SupportedAddressData;
 import at.bitfire.davdroid.Constants;
 import at.bitfire.davdroid.HttpClient;
 import at.bitfire.davdroid.resource.LocalAddressBook;
 import at.bitfire.davdroid.resource.LocalContact;
 import at.bitfire.vcard4android.Contact;
+import ezvcard.VCardVersion;
+import ezvcard.property.Uid;
 
 public class ContactsSyncAdapterService extends Service {
 	private static ContactsSyncAdapter syncAdapter;
@@ -66,8 +80,13 @@ public class ContactsSyncAdapterService extends Service {
             AccountSettings settings = new AccountSettings(getContext(), account);
             HttpClient httpClient = new HttpClient(settings.getUserName(), settings.getPassword(), settings.getPreemptiveAuth());
 
-            DavAddressBook dav = new DavAddressBook(httpClient, HttpUrl.parse(settings.getAddressBookURL()));
+            HttpUrl addressBookURL = HttpUrl.parse(settings.getAddressBookURL());
+            DavAddressBook dav = new DavAddressBook(httpClient, addressBookURL);
             try {
+                // prepare local address book
+                LocalAddressBook addressBook = new LocalAddressBook(account, provider);
+
+                // prepare remote address book
                 boolean hasVCard4 = false;
                 dav.propfind(0, SupportedAddressData.NAME);
                 SupportedAddressData supportedAddressData = (SupportedAddressData)dav.properties.get(SupportedAddressData.NAME);
@@ -77,22 +96,137 @@ public class ContactsSyncAdapterService extends Service {
                             hasVCard4 = true;
                 Constants.log.info("Server advertises VCard/4 support: " + hasVCard4);
 
-                LocalAddressBook addressBook = new LocalAddressBook(account, provider);
+                // Remove locally deleted contacts from server (if they have a name, i.e. if they were uploaded before),
-
+                // but only if they don't have changed on the server. Then finally remove them from the local address book.
-                dav.queryMemberETags();
+                LocalContact[] localList = addressBook.getDeleted();
-                for (DavResource vCard : dav.members) {
+                for (LocalContact local : localList) {
-                    Constants.log.info("Found remote VCard: " + vCard.location);
+                    final String fileName = local.getFileName();
-                    ResponseBody body = vCard.get("text/vcard;q=0.8, text/vcard;version=4.0");
+                    if (!TextUtils.isEmpty(fileName)) {
-
+                        Constants.log.info(fileName + " has been deleted locally -> deleting from server");
-                    Contact contacts[] = Contact.fromStream(body.byteStream(), body.contentType().charset(Charsets.UTF_8));
+                        try {
-                    if (contacts.length == 1) {
+                            new DavResource(httpClient, addressBookURL.newBuilder().addPathSegment(fileName).build())
-                        Contact contact = contacts[0];
+                                .delete(local.eTag);
-                        Constants.log.info(contact.toString());
+                        } catch(IOException|HttpException e) {
-
+                            Constants.log.warn("Couldn't delete " + fileName + " from server");
-                        LocalContact localContact = new LocalContact(addressBook, contact);
+                        }
-                        localContact.add();
                     } else
-                        Constants.log.error("Received VCard with not exactly one VCARD");
+                        Constants.log.info("Removing local contact #" + local.getId() + " which has been deleted locally and was never uploaded");
+                    local.delete();
+                }
+
+                // assign file names and UIDs to new contacts so that we can use the file name as an index
+                localList = addressBook.getWithoutFileName();
+                for (LocalContact local : localList) {
+                    String uuid = Uid.random().toString();
+                    Constants.log.info("Found local contact #" + local.getId() + " without file name; assigning name UID/name " + uuid + "[.vcf]");
+                    local.updateUID(uuid);
+                }
+
+                // upload dirty contacts
+                localList = addressBook.getDirty();
+                for (LocalContact local : localList) {
+                    final String fileName = local.getFileName();
+
+                    DavResource remote = new DavResource(httpClient, addressBookURL.newBuilder().addPathSegment(fileName).build());
+
+                    RequestBody vCard = RequestBody.create(
+                            hasVCard4 ? DavAddressBook.MIME_VCARD4 : DavAddressBook.MIME_VCARD3_UTF8,
+                            local.getContact().toStream(hasVCard4 ? VCardVersion.V4_0 : VCardVersion.V3_0).toByteArray()
+                    );
+
+                    if (local.eTag == null) {
+                        Constants.log.info("Uploading new contact " + fileName);
+                        remote.put(vCard, null, local.eTag);
+                    } else {
+                        Constants.log.info("Uploading locally modified contact " + fileName);
+                        remote.put(vCard, local.eTag, null);
+                    }
+
+                    // reset DIRTY
+                }
+
+                // check CTag (ignore on forced sync)
+                if (true) {
+                    // fetch list of local contacts and build hash table to index file name
+                    localList = addressBook.getAll();
+                    Map<String, LocalContact> localContacts = new HashMap<>(localList.length);
+                    for (LocalContact contact : localList) {
+                        Constants.log.debug("Found local contact: " + contact.getFileName());
+                        localContacts.put(contact.getFileName(), contact);
+                    }
+
+                    // fetch list of remote VCards and build hash table to index file name
+                    Constants.log.info("Listing remote VCards");
+                    dav.queryMemberETags();
+                    Map<String, DavResource> remoteContacts = new HashMap<>(dav.members.size());
+                    for (DavResource vCard : dav.members) {
+                        String fileName = vCard.fileName();
+                        Constants.log.debug("Found remote VCard: " + fileName);
+                        remoteContacts.put(fileName, vCard);
+                    }
+
+                    /* check which contacts
+                       1. are not present anymore remotely -> delete immediately on local side
+                       2. updated remotely -> add to downloadNames
+                       3. added remotely  -> add to downloadNames
+                     */
+                    Set<DavResource> toDownload = new HashSet<>();
+                    for (String localName : localContacts.keySet()) {
+                        DavResource remote = remoteContacts.get(localName);
+                        if (remote == null) {
+                            Constants.log.info(localName + " is not on server anymore, deleting");
+                            localContacts.get(localName).delete();
+                        } else {
+                            // contact is still on server, check whether it has been updated remotely
+                            GetETag getETag = (GetETag)remote.properties.get(GetETag.NAME);
+                            if (getETag == null || getETag.eTag == null)
+                                throw new DavException("Server didn't provide ETag");
+                            String  localETag = localContacts.get(localName).eTag,
+                                    remoteETag = getETag.eTag;
+                            if (!remoteETag.equals(localETag)) {
+                                Constants.log.info(localName + " has been changed on server (current ETag=" + remoteETag + ", last known ETag=" + localETag + ")");
+                                toDownload.add(remote);
+                            }
+
+                            // remote entry has been seen, remove from list
+                            remoteContacts.remove(localName);
+                        }
+                    }
+
+                    // add all unseen (= remotely added) remote contacts
+                    if (!remoteContacts.isEmpty()) {
+                        Constants.log.info("New VCards have been found on the server: " + TextUtils.join(", ", remoteContacts.keySet()));
+                        toDownload.addAll(remoteContacts.values());
+                    }
+
+                    // download new/updated VCards from server
+                    for (DavResource remoteContact : toDownload) {
+                        Constants.log.info("Downloading " + remoteContact.location);
+                        String fileName = remoteContact.fileName();
+
+                        ResponseBody body = remoteContact.get("text/vcard;q=0.5, text/vcard;charset=utf-8;q=0.8, text/vcard;version=4.0");
+                        String remoteETag = ((GetETag)remoteContact.properties.get(GetETag.NAME)).eTag;
+
+                        Contact contacts[] = Contact.fromStream(body.byteStream(), body.contentType().charset(Charsets.UTF_8));
+                        if (contacts.length == 1) {
+                            Contact newData = contacts[0];
+
+                            // delete local contact, if it exists
+                            LocalContact localContact = localContacts.get(fileName);
+                            if (localContact != null) {
+                                Constants.log.info("Updating " + fileName + " in local address book");
+                                localContact.eTag = remoteETag;
+                                localContact.update(newData);
+                            } else {
+                                Constants.log.info("Adding " + fileName + " to local address book");
+                                localContact = new LocalContact(addressBook, newData, fileName, remoteETag);
+                                localContact.add();
+                            }
+
+                            // add the new contact
+                        } else
+                            Constants.log.error("Received VCard with not exactly one VCARD, ignoring " + fileName);
+                    }
                 }
 
             } catch (Exception e) {

dav4android

@@ -1 +1 @@
-Subproject commit 57e1f34c45f070ca9b269f2bea109f6b1cdcb385
+Subproject commit 84a2cf0bbad257274e362851020da3822957449d

vcard4android

@@ -1 +1 @@
-Subproject commit 644ee03c74d35837974db771a7093f6f28623fbc
+Subproject commit 8053287e694d5a1b5c609fbbe3af97775e5a99af