Contact synchronization logic

* use VERSION_CODE and buildTime from BuildConfig
* new HTTP User-Agent, VCard PRODID values
* contact sync: store CTag in SyncState
* sync logic: upload contacts, check CTag, multiget

app/build.gradle

@@ -17,6 +17,11 @@ android {
         applicationId "at.bitfire.davdroid"
         minSdkVersion 14
         targetSdkVersion 23
+
+        versionCode 73
+        versionName "0.9-alpha1"
+
+        buildConfigField "java.util.Date", "buildTime", "new java.util.Date()"
     }
 
     buildTypes {

app/src/main/AndroidManifest.xml

@@ -9,7 +9,6 @@
 
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     package="at.bitfire.davdroid"
-    android:versionCode="72" android:versionName="0.8.4.1"
     android:installLocation="internalOnly">
 
     <uses-permission android:name="android.permission.INTERNET" />

app/src/main/java/at/bitfire/davdroid/Constants.java

@@ -14,13 +14,12 @@ import org.slf4j.LoggerFactory;
 
 public class Constants {
 	public static final String
-		APP_VERSION = "0.8.4.1",
 		ACCOUNT_TYPE = "bitfire.at.davdroid",
 		WEB_URL_MAIN = "https://davdroid.bitfire.at/?pk_campaign=davdroid-app",
 		WEB_URL_HELP = "https://davdroid.bitfire.at/configuration?pk_campaign=davdroid-app",
 		WEB_URL_VIEW_LOGS = "https://github.com/bitfireAT/davdroid/wiki/How-to-view-the-logs";
 
-	public static final ProdId ICAL_PRODID = new ProdId("-//bitfire web engineering//DAVdroid " + Constants.APP_VERSION + " (ical4j 2.0-beta1)//EN");
+	public static final ProdId ICAL_PRODID = new ProdId("-//bitfire web engineering//DAVdroid " + BuildConfig.VERSION_CODE + " (ical4j 2.0-beta1)//EN");
 
     public static final Logger log = LoggerFactory.getLogger("DAVdroid");
 }

app/src/main/java/at/bitfire/davdroid/HttpClient.java

@@ -8,6 +8,8 @@
 
 package at.bitfire.davdroid;
 
+import android.os.Build;
+
 import com.squareup.okhttp.Authenticator;
 import com.squareup.okhttp.Credentials;
 import com.squareup.okhttp.Interceptor;
@@ -18,6 +20,7 @@ import com.squareup.okhttp.logging.HttpLoggingInterceptor;
 
 import java.io.IOException;
 import java.net.Proxy;
+import java.text.SimpleDateFormat;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
@@ -26,8 +29,24 @@ import lombok.RequiredArgsConstructor;
 
 public class HttpClient extends OkHttpClient {
 
-    protected static final String
+    protected static final String HEADER_AUTHORIZATION = "Authorization";
-            HEADER_AUTHORIZATION = "Authorization";
+
+    final static UserAgentInterceptor userAgentInterceptor = new UserAgentInterceptor();
+    final static HttpLoggingInterceptor loggingInterceptor = new HttpLoggingInterceptor(new HttpLoggingInterceptor.Logger() {
+        @Override
+        public void log(String message) {
+            Constants.log.trace(message);
+        }
+    });
+    static {
+        loggingInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY);
+    }
+
+    static final String userAgent;
+    static {
+        String date = new SimpleDateFormat("yyyy/MM/dd").format(BuildConfig.buildTime);
+        userAgent = "DAVdroid/" + BuildConfig.VERSION_NAME + " (" + date + "; dav4android) Android/" + Build.VERSION.RELEASE;
+    }
 
 
     public HttpClient() {
@@ -40,13 +59,13 @@ public class HttpClient extends OkHttpClient {
         super();
         initialize();
 
-        // authentication and User-Agent
+        enableLogs();
+
+        // authentication
         if (preemptive)
             networkInterceptors().add(new PreemptiveAuthenticationInterceptor(username, password));
         else
             setAuthenticator(new DavAuthenticator(username, password));
-
-        enableLogs();
     }
 
 
@@ -54,21 +73,27 @@ public class HttpClient extends OkHttpClient {
         // don't follow redirects automatically because this may rewrite DAV methods to GET
         setFollowRedirects(false);
 
-        // timeouts
         setConnectTimeout(20, TimeUnit.SECONDS);
         setWriteTimeout(15, TimeUnit.SECONDS);
         setReadTimeout(45, TimeUnit.SECONDS);
+
+        // add User-Agent to every request
+        networkInterceptors().add(userAgentInterceptor);
     }
 
     protected void enableLogs() {
-        HttpLoggingInterceptor logging = new HttpLoggingInterceptor(new HttpLoggingInterceptor.Logger() {
+        interceptors().add(loggingInterceptor);
-            @Override
+    }
-            public void log(String message) {
+
-                at.bitfire.dav4android.Constants.log.trace(message);
+
+    static class UserAgentInterceptor implements Interceptor {
+        @Override
+        public Response intercept(Chain chain) throws IOException {
+            Request request = chain.request().newBuilder()
+                    .header("User-Agent", userAgent)
+                    .build();
+            return chain.proceed(request);
         }
-        });
-        logging.setLevel(HttpLoggingInterceptor.Level.BODY);
-        interceptors().add(logging);
     }
 
 
@@ -78,8 +103,7 @@ public class HttpClient extends OkHttpClient {
 
         @Override
         public Response intercept(Chain chain) throws IOException {
-            Request request = chain.request();
+            Request request = chain.request().newBuilder()
-            request = request.newBuilder()
                     .header("Authorization", Credentials.basic(username, password))
                     .build();
             return chain.proceed(request);

app/src/main/java/at/bitfire/davdroid/resource/LocalAddressBook.java

@@ -9,6 +9,8 @@ package at.bitfire.davdroid.resource;
 
 import android.accounts.Account;
 import android.content.ContentProviderClient;
+import android.os.Bundle;
+import android.os.Parcel;
 import android.provider.ContactsContract;
 
 import at.bitfire.davdroid.Constants;
@@ -18,14 +20,25 @@ import at.bitfire.vcard4android.AndroidContactFactory;
 import at.bitfire.vcard4android.AndroidGroupFactory;
 import at.bitfire.vcard4android.Contact;
 import at.bitfire.vcard4android.ContactsStorageException;
+import lombok.Cleanup;
+import lombok.Synchronized;
 
 
 public class LocalAddressBook extends AndroidAddressBook {
 
+    protected static final String SYNC_STATE_CTAG = "ctag";
+
+    private Bundle syncState = new Bundle();
+
+
     public LocalAddressBook(Account account, ContentProviderClient provider) {
         super(account, provider, AndroidGroupFactory.INSTANCE, LocalContact.Factory.INSTANCE);
     }
 
+
+    /**
+     * Returns an array of local contacts, excluding those which have been modified locally (and not uploaded yet).
+     */
     public LocalContact[] getAll() throws ContactsStorageException {
         LocalContact contacts[] = (LocalContact[])queryContacts(null, null);
         return contacts;
@@ -35,14 +48,14 @@ public class LocalAddressBook extends AndroidAddressBook {
      * Returns an array of local contacts which have been deleted locally. (DELETED != 0).
      */
     public LocalContact[] getDeleted() throws ContactsStorageException {
-        return (LocalContact[])queryContacts(ContactsContract.RawContacts.DELETED + " != 0", null);
+        return (LocalContact[])queryContacts(ContactsContract.RawContacts.DELETED + "!=0", null);
     }
 
     /**
      * Returns an array of local contacts which have been changed locally (DIRTY != 0).
      */
     public LocalContact[] getDirty() throws ContactsStorageException {
-        return (LocalContact[])queryContacts(ContactsContract.RawContacts.DIRTY + " != 0", null);
+        return (LocalContact[])queryContacts(ContactsContract.RawContacts.DIRTY + "!=0", null);
     }
 
     /**
@@ -52,4 +65,35 @@ public class LocalAddressBook extends AndroidAddressBook {
         return (LocalContact[])queryContacts(AndroidContact.COLUMN_FILENAME + " IS NULL", null);
     }
 
+
+    protected void readSyncState() throws ContactsStorageException {
+        @Cleanup("recycle") Parcel parcel = Parcel.obtain();
+        byte[] raw = getSyncState();
+        if (raw != null) {
+            parcel.unmarshall(raw, 0, raw.length);
+            parcel.setDataPosition(0);
+            syncState = parcel.readBundle();
+        } else
+            syncState.clear();
+    }
+
+    public String getCTag() throws ContactsStorageException {
+        synchronized (syncState) {
+            readSyncState();
+            return syncState.getString(SYNC_STATE_CTAG);
+        }
+    }
+
+    public void setCTag(String cTag) throws ContactsStorageException {
+        synchronized (syncState) {
+            readSyncState();
+            syncState.putString(SYNC_STATE_CTAG, cTag);
+
+            // write sync state bundle
+            @Cleanup("recycle") Parcel parcel = Parcel.obtain();
+            parcel.writeBundle(syncState);
+            setSyncState(parcel.marshall());
+        }
+    }
+
 }

app/src/main/java/at/bitfire/davdroid/resource/LocalContact.java

@@ -12,13 +12,18 @@ import android.content.ContentValues;
 import android.os.RemoteException;
 import android.provider.ContactsContract;
 
+import at.bitfire.davdroid.BuildConfig;
 import at.bitfire.vcard4android.AndroidAddressBook;
 import at.bitfire.vcard4android.AndroidContact;
 import at.bitfire.vcard4android.AndroidContactFactory;
 import at.bitfire.vcard4android.Contact;
 import at.bitfire.vcard4android.ContactsStorageException;
+import ezvcard.Ezvcard;
 
 public class LocalContact extends AndroidContact {
+    static {
+        Contact.productID = "+//IDN bitfire.at//DAVdroid/" + BuildConfig.VERSION_NAME + " ez-vcard/" + Ezvcard.VERSION;
+    }
 
     protected LocalContact(AndroidAddressBook addressBook, long id, String fileName, String eTag) {
         super(addressBook, id, fileName, eTag);
@@ -28,6 +33,17 @@ public class LocalContact extends AndroidContact {
         super(addressBook, contact, fileName, eTag);
     }
 
+    public void clearDirty(String eTag) throws ContactsStorageException {
+        try {
+            ContentValues values = new ContentValues(1);
+            values.put(COLUMN_ETAG, eTag);
+            values.put(ContactsContract.RawContacts.DIRTY, 0);
+            addressBook.provider.update(rawContactSyncURI(), values, null, null);
+        } catch (RemoteException e) {
+            throw new ContactsStorageException("Couldn't clear dirty flag", e);
+        }
+    }
+
     public void updateUID(String uid) throws ContactsStorageException {
         try {
             ContentValues values = new ContentValues(1);

app/src/main/java/at/bitfire/davdroid/syncadapter/ContactsSyncAdapterService.java

@@ -11,6 +11,7 @@ import android.accounts.Account;
 import android.app.Service;
 import android.content.AbstractThreadedSyncAdapter;
 import android.content.ContentProviderClient;
+import android.content.ContentResolver;
 import android.content.Context;
 import android.content.Intent;
 import android.content.SyncResult;
@@ -26,30 +27,43 @@ import com.squareup.okhttp.ResponseBody;
 
 import org.apache.commons.io.Charsets;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.UUID;
 
 import at.bitfire.dav4android.DavAddressBook;
 import at.bitfire.dav4android.DavResource;
 import at.bitfire.dav4android.exception.DavException;
 import at.bitfire.dav4android.exception.HttpException;
+import at.bitfire.dav4android.property.AddressData;
+import at.bitfire.dav4android.property.GetCTag;
+import at.bitfire.dav4android.property.GetContentType;
 import at.bitfire.dav4android.property.GetETag;
 import at.bitfire.dav4android.property.SupportedAddressData;
+import at.bitfire.davdroid.ArrayUtils;
 import at.bitfire.davdroid.Constants;
 import at.bitfire.davdroid.HttpClient;
 import at.bitfire.davdroid.resource.LocalAddressBook;
 import at.bitfire.davdroid.resource.LocalContact;
 import at.bitfire.vcard4android.Contact;
+import at.bitfire.vcard4android.ContactsStorageException;
 import ezvcard.VCardVersion;
 import ezvcard.property.Uid;
+import lombok.Cleanup;
 
 public class ContactsSyncAdapterService extends Service {
 	private static ContactsSyncAdapter syncAdapter;
 
+    protected static final int MAX_MULTIGET = 10;
+
+
 	@Override
 	public void onCreate() {
 		if (syncAdapter == null)
@@ -68,7 +82,6 @@ public class ContactsSyncAdapterService extends Service {
 	
 
 	private static class ContactsSyncAdapter extends AbstractThreadedSyncAdapter {
-
         public ContactsSyncAdapter(Context context) {
             super(context, false);
         }
@@ -88,7 +101,7 @@ public class ContactsSyncAdapterService extends Service {
 
                 // prepare remote address book
                 boolean hasVCard4 = false;
-                dav.propfind(0, SupportedAddressData.NAME);
+                dav.propfind(0, SupportedAddressData.NAME, GetCTag.NAME);
                 SupportedAddressData supportedAddressData = (SupportedAddressData)dav.properties.get(SupportedAddressData.NAME);
                 if (supportedAddressData != null)
                     for (MediaType type : supportedAddressData.types)
@@ -142,11 +155,24 @@ public class ContactsSyncAdapterService extends Service {
                         remote.put(vCard, local.eTag, null);
                     }
 
-                    // reset DIRTY
+                    GetETag newETag = (GetETag) remote.properties.get(GetETag.NAME);
+                    local.clearDirty(newETag != null ? newETag.eTag : null);
                 }
 
-                // check CTag (ignore on forced sync)
+                // check CTag (ignore on manual sync)
-                if (true) {
+                String currentCTag = null;
+                if (extras.containsKey(ContentResolver.SYNC_EXTRAS_MANUAL))
+                    Constants.log.info("Manual sync, ignoring CTag");
+                else {
+                    GetCTag getCTag = (GetCTag) dav.properties.get(GetCTag.NAME);
+                    if (getCTag != null)
+                        currentCTag = getCTag.cTag;
+                }
+
+                if (currentCTag != null && !(currentCTag.equals(addressBook.getCTag()))) {
+                    Constants.log.info("Remote address book didn't change (CTag=" + currentCTag + "), no need to list VCards");
+
+                } else {
                     // fetch list of local contacts and build hash table to index file name
                     localList = addressBook.getAll();
                     Map<String, LocalContact> localContacts = new HashMap<>(localList.length);
@@ -199,15 +225,73 @@ public class ContactsSyncAdapterService extends Service {
                         toDownload.addAll(remoteContacts.values());
                     }
 
+                    Constants.log.info("Downloading " + toDownload.size() + " contacts (" + MAX_MULTIGET + " at once)");
+
                     // download new/updated VCards from server
-                    for (DavResource remoteContact : toDownload) {
+                    for (DavResource[] bunch : ArrayUtils.partition(toDownload.toArray(new DavResource[toDownload.size()]), MAX_MULTIGET)) {
-                        Constants.log.info("Downloading " + remoteContact.location);
+                        Constants.log.info("Downloading " + TextUtils.join(" + ", bunch));
-                        String fileName = remoteContact.fileName();
+                        if (bunch.length == 1) {
+                            // only one contact, use GET
+                            DavResource remote = bunch[0];
+                            String fileName = remote.fileName();
 
-                        ResponseBody body = remoteContact.get("text/vcard;q=0.5, text/vcard;charset=utf-8;q=0.8, text/vcard;version=4.0");
+                            ResponseBody body = remote.get("text/vcard;q=0.5, text/vcard;charset=utf-8;q=0.8, text/vcard;version=4.0");
-                        String remoteETag = ((GetETag)remoteContact.properties.get(GetETag.NAME)).eTag;
+                            String eTag = ((GetETag)remote.properties.get(GetETag.NAME)).eTag;
 
-                        Contact contacts[] = Contact.fromStream(body.byteStream(), body.contentType().charset(Charsets.UTF_8));
+                            @Cleanup InputStream stream = body.byteStream();
+                            processVCard(addressBook, localContacts, remote.fileName(), eTag, stream, body.contentType().charset(Charsets.UTF_8));
+
+                        } else {
+                            // multiple contacts, use multi-get
+                            List<HttpUrl> urls = new LinkedList<>();
+                            for (DavResource remote : bunch)
+                                urls.add(remote.location);
+                            dav.multiget(urls.toArray(new HttpUrl[urls.size()]), hasVCard4);
+
+                            // process multiget results
+                            for (DavResource remote : dav.members) {
+                                String eTag = null;
+                                GetETag getETag = (GetETag)remote.properties.get(GetETag.NAME);
+                                if (getETag != null)
+                                    eTag = getETag.eTag;
+                                else
+                                    throw new DavException("Received multi-get response without ETag");
+
+                                Charset charset = Charsets.UTF_8;
+                                GetContentType getContentType = (GetContentType)remote.properties.get(GetContentType.NAME);
+                                if (getContentType != null && getContentType.type != null) {
+                                    MediaType type = MediaType.parse(getContentType.type);
+                                    if (type != null)
+                                        charset = type.charset(Charsets.UTF_8);
+                                }
+
+                                AddressData addressData = (AddressData)remote.properties.get(AddressData.NAME);
+                                if (addressData == null || addressData.vCard == null)
+                                    throw new DavException("Received multi-get response without address data");
+
+                                @Cleanup InputStream stream = new ByteArrayInputStream(addressData.vCard.getBytes());
+                                processVCard(addressBook, localContacts, remote.fileName(), eTag, stream, charset);
+                            }
+                        }
+                    }
+
+                    /* Save sync state (CTag). It doesn't matter if it has changed during the sync process
+                       (for instance, because another client has uploaded changes), because this will simply
+                       cause all remote entries to be listed at the next sync. */
+                    Constants.log.info("Saving sync state: CTag=" + currentCTag);
+                    addressBook.setCTag(currentCTag);
+                }
+
+            } catch (Exception e) {
+                Log.e("davdroid", "XXX", e);
+            }
+
+            Constants.log.info("Sync complete for authority " + authority);
+        }
+
+
+        private void processVCard(LocalAddressBook addressBook, Map<String, LocalContact>localContacts, String fileName, String eTag, InputStream stream, Charset charset) throws IOException, ContactsStorageException {
+            Contact contacts[] = Contact.fromStream(stream, charset);
             if (contacts.length == 1) {
                 Contact newData = contacts[0];
 
@@ -215,25 +299,18 @@ public class ContactsSyncAdapterService extends Service {
                 LocalContact localContact = localContacts.get(fileName);
                 if (localContact != null) {
                     Constants.log.info("Updating " + fileName + " in local address book");
-                                localContact.eTag = remoteETag;
+                    localContact.eTag = eTag;
                     localContact.update(newData);
                 } else {
                     Constants.log.info("Adding " + fileName + " to local address book");
-                                localContact = new LocalContact(addressBook, newData, fileName, remoteETag);
+                    localContact = new LocalContact(addressBook, newData, fileName, eTag);
                     localContact.add();
                 }
-
-                            // add the new contact
             } else
                 Constants.log.error("Received VCard with not exactly one VCARD, ignoring " + fileName);
         }
+
+
     }
 
-            } catch (Exception e) {
-                Log.e("davdroid", "querying member etags", e);
-            }
-
-            Constants.log.info("Sync complete for authority " + authority);
-        }
-    }
 }

app/src/main/java/at/bitfire/davdroid/ui/MainActivity.java

@@ -20,6 +20,7 @@ import android.view.MenuItem;
 import android.view.View;
 import android.widget.TextView;
 
+import at.bitfire.davdroid.BuildConfig;
 import at.bitfire.davdroid.Constants;
 import at.bitfire.davdroid.R;
 import at.bitfire.davdroid.ui.settings.SettingsActivity;
@@ -41,7 +42,7 @@ public class MainActivity extends Activity {
 		}
 		
 		TextView tvInfo = (TextView)findViewById(R.id.text_info);
-		tvInfo.setText(Html.fromHtml(getString(R.string.html_main_info, Constants.APP_VERSION)));
+		tvInfo.setText(Html.fromHtml(getString(R.string.html_main_info, BuildConfig.VERSION_NAME)));
 		tvInfo.setMovementMethod(LinkMovementMethod.getInstance());
 	}
 

app/src/main/java/at/bitfire/davdroid/webdav/DavHttpClient.java

@@ -20,6 +20,7 @@ import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 
+import at.bitfire.davdroid.BuildConfig;
 import at.bitfire.davdroid.Constants;
 
 
@@ -57,7 +58,7 @@ public class DavHttpClient {
 				.setDefaultRequestConfig(defaultRqConfig)
 				.setRetryHandler(DavHttpRequestRetryHandler.INSTANCE)
 				.setRedirectStrategy(DavRedirectStrategy.INSTANCE)
-				.setUserAgent("DAVdroid/" + Constants.APP_VERSION);
+				.setUserAgent("DAVdroid/" + BuildConfig.VERSION_NAME);
 		
 		if (Log.isLoggable("Wire", Log.DEBUG)) {
 			Log.i(TAG, "Wire logging active, disabling HTTP compression");

dav4android

@@ -1 +1 @@
-Subproject commit 84a2cf0bbad257274e362851020da3822957449d
+Subproject commit a6975918ed614eef93c222451fd0981c60ec3ad9