From d024cdb495bb960089a9c67eea14c7baa542a974 Mon Sep 17 00:00:00 2001 From: Ricki Hirner Date: Mon, 12 Oct 2015 01:56:28 +0200 Subject: [PATCH] Contact synchronization logic * use VERSION_CODE and buildTime from BuildConfig * new HTTP User-Agent, VCard PRODID values * contact sync: store CTag in SyncState * sync logic: upload contacts, check CTag, multiget --- app/build.gradle | 5 + app/src/main/AndroidManifest.xml | 1 - .../java/at/bitfire/davdroid/Constants.java | 3 +- .../java/at/bitfire/davdroid/HttpClient.java | 56 +++++--- .../davdroid/resource/LocalAddressBook.java | 48 ++++++- .../davdroid/resource/LocalContact.java | 16 +++ .../ContactsSyncAdapterService.java | 135 ++++++++++++++---- .../at/bitfire/davdroid/ui/MainActivity.java | 3 +- .../davdroid/webdav/DavHttpClient.java | 3 +- dav4android | 2 +- 10 files changed, 219 insertions(+), 53 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index a85e3418..f409a523 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -17,6 +17,11 @@ android { applicationId "at.bitfire.davdroid" minSdkVersion 14 targetSdkVersion 23 + + versionCode 73 + versionName "0.9-alpha1" + + buildConfigField "java.util.Date", "buildTime", "new java.util.Date()" } buildTypes { diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 078c8cb2..aa59af4c 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -9,7 +9,6 @@ diff --git a/app/src/main/java/at/bitfire/davdroid/Constants.java b/app/src/main/java/at/bitfire/davdroid/Constants.java index 4cfa5913..6d9c2c56 100644 --- a/app/src/main/java/at/bitfire/davdroid/Constants.java +++ b/app/src/main/java/at/bitfire/davdroid/Constants.java @@ -14,13 +14,12 @@ import org.slf4j.LoggerFactory; public class Constants { public static final String - APP_VERSION = "0.8.4.1", ACCOUNT_TYPE = "bitfire.at.davdroid", WEB_URL_MAIN = "https://davdroid.bitfire.at/?pk_campaign=davdroid-app", WEB_URL_HELP = "https://davdroid.bitfire.at/configuration?pk_campaign=davdroid-app", WEB_URL_VIEW_LOGS = "https://github.com/bitfireAT/davdroid/wiki/How-to-view-the-logs"; - public static final ProdId ICAL_PRODID = new ProdId("-//bitfire web engineering//DAVdroid " + Constants.APP_VERSION + " (ical4j 2.0-beta1)//EN"); + public static final ProdId ICAL_PRODID = new ProdId("-//bitfire web engineering//DAVdroid " + BuildConfig.VERSION_CODE + " (ical4j 2.0-beta1)//EN"); public static final Logger log = LoggerFactory.getLogger("DAVdroid"); } diff --git a/app/src/main/java/at/bitfire/davdroid/HttpClient.java b/app/src/main/java/at/bitfire/davdroid/HttpClient.java index 9f422be9..651600eb 100644 --- a/app/src/main/java/at/bitfire/davdroid/HttpClient.java +++ b/app/src/main/java/at/bitfire/davdroid/HttpClient.java @@ -8,6 +8,8 @@ package at.bitfire.davdroid; +import android.os.Build; + import com.squareup.okhttp.Authenticator; import com.squareup.okhttp.Credentials; import com.squareup.okhttp.Interceptor; @@ -18,6 +20,7 @@ import com.squareup.okhttp.logging.HttpLoggingInterceptor; import java.io.IOException; import java.net.Proxy; +import java.text.SimpleDateFormat; import java.util.List; import java.util.concurrent.TimeUnit; @@ -26,8 +29,24 @@ import lombok.RequiredArgsConstructor; public class HttpClient extends OkHttpClient { - protected static final String - HEADER_AUTHORIZATION = "Authorization"; + protected static final String HEADER_AUTHORIZATION = "Authorization"; + + final static UserAgentInterceptor userAgentInterceptor = new UserAgentInterceptor(); + final static HttpLoggingInterceptor loggingInterceptor = new HttpLoggingInterceptor(new HttpLoggingInterceptor.Logger() { + @Override + public void log(String message) { + Constants.log.trace(message); + } + }); + static { + loggingInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY); + } + + static final String userAgent; + static { + String date = new SimpleDateFormat("yyyy/MM/dd").format(BuildConfig.buildTime); + userAgent = "DAVdroid/" + BuildConfig.VERSION_NAME + " (" + date + "; dav4android) Android/" + Build.VERSION.RELEASE; + } public HttpClient() { @@ -40,13 +59,13 @@ public class HttpClient extends OkHttpClient { super(); initialize(); - // authentication and User-Agent + enableLogs(); + + // authentication if (preemptive) networkInterceptors().add(new PreemptiveAuthenticationInterceptor(username, password)); else setAuthenticator(new DavAuthenticator(username, password)); - - enableLogs(); } @@ -54,21 +73,27 @@ public class HttpClient extends OkHttpClient { // don't follow redirects automatically because this may rewrite DAV methods to GET setFollowRedirects(false); - // timeouts setConnectTimeout(20, TimeUnit.SECONDS); setWriteTimeout(15, TimeUnit.SECONDS); setReadTimeout(45, TimeUnit.SECONDS); + + // add User-Agent to every request + networkInterceptors().add(userAgentInterceptor); } protected void enableLogs() { - HttpLoggingInterceptor logging = new HttpLoggingInterceptor(new HttpLoggingInterceptor.Logger() { - @Override - public void log(String message) { - at.bitfire.dav4android.Constants.log.trace(message); - } - }); - logging.setLevel(HttpLoggingInterceptor.Level.BODY); - interceptors().add(logging); + interceptors().add(loggingInterceptor); + } + + + static class UserAgentInterceptor implements Interceptor { + @Override + public Response intercept(Chain chain) throws IOException { + Request request = chain.request().newBuilder() + .header("User-Agent", userAgent) + .build(); + return chain.proceed(request); + } } @@ -78,8 +103,7 @@ public class HttpClient extends OkHttpClient { @Override public Response intercept(Chain chain) throws IOException { - Request request = chain.request(); - request = request.newBuilder() + Request request = chain.request().newBuilder() .header("Authorization", Credentials.basic(username, password)) .build(); return chain.proceed(request); diff --git a/app/src/main/java/at/bitfire/davdroid/resource/LocalAddressBook.java b/app/src/main/java/at/bitfire/davdroid/resource/LocalAddressBook.java index b51d7947..0d80de83 100644 --- a/app/src/main/java/at/bitfire/davdroid/resource/LocalAddressBook.java +++ b/app/src/main/java/at/bitfire/davdroid/resource/LocalAddressBook.java @@ -9,6 +9,8 @@ package at.bitfire.davdroid.resource; import android.accounts.Account; import android.content.ContentProviderClient; +import android.os.Bundle; +import android.os.Parcel; import android.provider.ContactsContract; import at.bitfire.davdroid.Constants; @@ -18,14 +20,25 @@ import at.bitfire.vcard4android.AndroidContactFactory; import at.bitfire.vcard4android.AndroidGroupFactory; import at.bitfire.vcard4android.Contact; import at.bitfire.vcard4android.ContactsStorageException; +import lombok.Cleanup; +import lombok.Synchronized; public class LocalAddressBook extends AndroidAddressBook { + protected static final String SYNC_STATE_CTAG = "ctag"; + + private Bundle syncState = new Bundle(); + + public LocalAddressBook(Account account, ContentProviderClient provider) { super(account, provider, AndroidGroupFactory.INSTANCE, LocalContact.Factory.INSTANCE); } + + /** + * Returns an array of local contacts, excluding those which have been modified locally (and not uploaded yet). + */ public LocalContact[] getAll() throws ContactsStorageException { LocalContact contacts[] = (LocalContact[])queryContacts(null, null); return contacts; @@ -35,14 +48,14 @@ public class LocalAddressBook extends AndroidAddressBook { * Returns an array of local contacts which have been deleted locally. (DELETED != 0). */ public LocalContact[] getDeleted() throws ContactsStorageException { - return (LocalContact[])queryContacts(ContactsContract.RawContacts.DELETED + " != 0", null); + return (LocalContact[])queryContacts(ContactsContract.RawContacts.DELETED + "!=0", null); } /** * Returns an array of local contacts which have been changed locally (DIRTY != 0). */ public LocalContact[] getDirty() throws ContactsStorageException { - return (LocalContact[])queryContacts(ContactsContract.RawContacts.DIRTY + " != 0", null); + return (LocalContact[])queryContacts(ContactsContract.RawContacts.DIRTY + "!=0", null); } /** @@ -52,4 +65,35 @@ public class LocalAddressBook extends AndroidAddressBook { return (LocalContact[])queryContacts(AndroidContact.COLUMN_FILENAME + " IS NULL", null); } + + protected void readSyncState() throws ContactsStorageException { + @Cleanup("recycle") Parcel parcel = Parcel.obtain(); + byte[] raw = getSyncState(); + if (raw != null) { + parcel.unmarshall(raw, 0, raw.length); + parcel.setDataPosition(0); + syncState = parcel.readBundle(); + } else + syncState.clear(); + } + + public String getCTag() throws ContactsStorageException { + synchronized (syncState) { + readSyncState(); + return syncState.getString(SYNC_STATE_CTAG); + } + } + + public void setCTag(String cTag) throws ContactsStorageException { + synchronized (syncState) { + readSyncState(); + syncState.putString(SYNC_STATE_CTAG, cTag); + + // write sync state bundle + @Cleanup("recycle") Parcel parcel = Parcel.obtain(); + parcel.writeBundle(syncState); + setSyncState(parcel.marshall()); + } + } + } diff --git a/app/src/main/java/at/bitfire/davdroid/resource/LocalContact.java b/app/src/main/java/at/bitfire/davdroid/resource/LocalContact.java index 691ba75c..17a8fd89 100644 --- a/app/src/main/java/at/bitfire/davdroid/resource/LocalContact.java +++ b/app/src/main/java/at/bitfire/davdroid/resource/LocalContact.java @@ -12,13 +12,18 @@ import android.content.ContentValues; import android.os.RemoteException; import android.provider.ContactsContract; +import at.bitfire.davdroid.BuildConfig; import at.bitfire.vcard4android.AndroidAddressBook; import at.bitfire.vcard4android.AndroidContact; import at.bitfire.vcard4android.AndroidContactFactory; import at.bitfire.vcard4android.Contact; import at.bitfire.vcard4android.ContactsStorageException; +import ezvcard.Ezvcard; public class LocalContact extends AndroidContact { + static { + Contact.productID = "+//IDN bitfire.at//DAVdroid/" + BuildConfig.VERSION_NAME + " ez-vcard/" + Ezvcard.VERSION; + } protected LocalContact(AndroidAddressBook addressBook, long id, String fileName, String eTag) { super(addressBook, id, fileName, eTag); @@ -28,6 +33,17 @@ public class LocalContact extends AndroidContact { super(addressBook, contact, fileName, eTag); } + public void clearDirty(String eTag) throws ContactsStorageException { + try { + ContentValues values = new ContentValues(1); + values.put(COLUMN_ETAG, eTag); + values.put(ContactsContract.RawContacts.DIRTY, 0); + addressBook.provider.update(rawContactSyncURI(), values, null, null); + } catch (RemoteException e) { + throw new ContactsStorageException("Couldn't clear dirty flag", e); + } + } + public void updateUID(String uid) throws ContactsStorageException { try { ContentValues values = new ContentValues(1); diff --git a/app/src/main/java/at/bitfire/davdroid/syncadapter/ContactsSyncAdapterService.java b/app/src/main/java/at/bitfire/davdroid/syncadapter/ContactsSyncAdapterService.java index eabedc74..8ba4b2ac 100644 --- a/app/src/main/java/at/bitfire/davdroid/syncadapter/ContactsSyncAdapterService.java +++ b/app/src/main/java/at/bitfire/davdroid/syncadapter/ContactsSyncAdapterService.java @@ -11,6 +11,7 @@ import android.accounts.Account; import android.app.Service; import android.content.AbstractThreadedSyncAdapter; import android.content.ContentProviderClient; +import android.content.ContentResolver; import android.content.Context; import android.content.Intent; import android.content.SyncResult; @@ -26,30 +27,43 @@ import com.squareup.okhttp.ResponseBody; import org.apache.commons.io.Charsets; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import at.bitfire.dav4android.DavAddressBook; import at.bitfire.dav4android.DavResource; import at.bitfire.dav4android.exception.DavException; import at.bitfire.dav4android.exception.HttpException; +import at.bitfire.dav4android.property.AddressData; +import at.bitfire.dav4android.property.GetCTag; +import at.bitfire.dav4android.property.GetContentType; import at.bitfire.dav4android.property.GetETag; import at.bitfire.dav4android.property.SupportedAddressData; +import at.bitfire.davdroid.ArrayUtils; import at.bitfire.davdroid.Constants; import at.bitfire.davdroid.HttpClient; import at.bitfire.davdroid.resource.LocalAddressBook; import at.bitfire.davdroid.resource.LocalContact; import at.bitfire.vcard4android.Contact; +import at.bitfire.vcard4android.ContactsStorageException; import ezvcard.VCardVersion; import ezvcard.property.Uid; +import lombok.Cleanup; public class ContactsSyncAdapterService extends Service { private static ContactsSyncAdapter syncAdapter; + protected static final int MAX_MULTIGET = 10; + + @Override public void onCreate() { if (syncAdapter == null) @@ -68,7 +82,6 @@ public class ContactsSyncAdapterService extends Service { private static class ContactsSyncAdapter extends AbstractThreadedSyncAdapter { - public ContactsSyncAdapter(Context context) { super(context, false); } @@ -88,7 +101,7 @@ public class ContactsSyncAdapterService extends Service { // prepare remote address book boolean hasVCard4 = false; - dav.propfind(0, SupportedAddressData.NAME); + dav.propfind(0, SupportedAddressData.NAME, GetCTag.NAME); SupportedAddressData supportedAddressData = (SupportedAddressData)dav.properties.get(SupportedAddressData.NAME); if (supportedAddressData != null) for (MediaType type : supportedAddressData.types) @@ -142,11 +155,24 @@ public class ContactsSyncAdapterService extends Service { remote.put(vCard, local.eTag, null); } - // reset DIRTY + GetETag newETag = (GetETag) remote.properties.get(GetETag.NAME); + local.clearDirty(newETag != null ? newETag.eTag : null); } - // check CTag (ignore on forced sync) - if (true) { + // check CTag (ignore on manual sync) + String currentCTag = null; + if (extras.containsKey(ContentResolver.SYNC_EXTRAS_MANUAL)) + Constants.log.info("Manual sync, ignoring CTag"); + else { + GetCTag getCTag = (GetCTag) dav.properties.get(GetCTag.NAME); + if (getCTag != null) + currentCTag = getCTag.cTag; + } + + if (currentCTag != null && !(currentCTag.equals(addressBook.getCTag()))) { + Constants.log.info("Remote address book didn't change (CTag=" + currentCTag + "), no need to list VCards"); + + } else { // fetch list of local contacts and build hash table to index file name localList = addressBook.getAll(); Map localContacts = new HashMap<>(localList.length); @@ -199,41 +225,92 @@ public class ContactsSyncAdapterService extends Service { toDownload.addAll(remoteContacts.values()); } + Constants.log.info("Downloading " + toDownload.size() + " contacts (" + MAX_MULTIGET + " at once)"); + // download new/updated VCards from server - for (DavResource remoteContact : toDownload) { - Constants.log.info("Downloading " + remoteContact.location); - String fileName = remoteContact.fileName(); + for (DavResource[] bunch : ArrayUtils.partition(toDownload.toArray(new DavResource[toDownload.size()]), MAX_MULTIGET)) { + Constants.log.info("Downloading " + TextUtils.join(" + ", bunch)); + if (bunch.length == 1) { + // only one contact, use GET + DavResource remote = bunch[0]; + String fileName = remote.fileName(); - ResponseBody body = remoteContact.get("text/vcard;q=0.5, text/vcard;charset=utf-8;q=0.8, text/vcard;version=4.0"); - String remoteETag = ((GetETag)remoteContact.properties.get(GetETag.NAME)).eTag; + ResponseBody body = remote.get("text/vcard;q=0.5, text/vcard;charset=utf-8;q=0.8, text/vcard;version=4.0"); + String eTag = ((GetETag)remote.properties.get(GetETag.NAME)).eTag; - Contact contacts[] = Contact.fromStream(body.byteStream(), body.contentType().charset(Charsets.UTF_8)); - if (contacts.length == 1) { - Contact newData = contacts[0]; + @Cleanup InputStream stream = body.byteStream(); + processVCard(addressBook, localContacts, remote.fileName(), eTag, stream, body.contentType().charset(Charsets.UTF_8)); - // delete local contact, if it exists - LocalContact localContact = localContacts.get(fileName); - if (localContact != null) { - Constants.log.info("Updating " + fileName + " in local address book"); - localContact.eTag = remoteETag; - localContact.update(newData); - } else { - Constants.log.info("Adding " + fileName + " to local address book"); - localContact = new LocalContact(addressBook, newData, fileName, remoteETag); - localContact.add(); + } else { + // multiple contacts, use multi-get + List urls = new LinkedList<>(); + for (DavResource remote : bunch) + urls.add(remote.location); + dav.multiget(urls.toArray(new HttpUrl[urls.size()]), hasVCard4); + + // process multiget results + for (DavResource remote : dav.members) { + String eTag = null; + GetETag getETag = (GetETag)remote.properties.get(GetETag.NAME); + if (getETag != null) + eTag = getETag.eTag; + else + throw new DavException("Received multi-get response without ETag"); + + Charset charset = Charsets.UTF_8; + GetContentType getContentType = (GetContentType)remote.properties.get(GetContentType.NAME); + if (getContentType != null && getContentType.type != null) { + MediaType type = MediaType.parse(getContentType.type); + if (type != null) + charset = type.charset(Charsets.UTF_8); + } + + AddressData addressData = (AddressData)remote.properties.get(AddressData.NAME); + if (addressData == null || addressData.vCard == null) + throw new DavException("Received multi-get response without address data"); + + @Cleanup InputStream stream = new ByteArrayInputStream(addressData.vCard.getBytes()); + processVCard(addressBook, localContacts, remote.fileName(), eTag, stream, charset); } - - // add the new contact - } else - Constants.log.error("Received VCard with not exactly one VCARD, ignoring " + fileName); + } } + + /* Save sync state (CTag). It doesn't matter if it has changed during the sync process + (for instance, because another client has uploaded changes), because this will simply + cause all remote entries to be listed at the next sync. */ + Constants.log.info("Saving sync state: CTag=" + currentCTag); + addressBook.setCTag(currentCTag); } } catch (Exception e) { - Log.e("davdroid", "querying member etags", e); + Log.e("davdroid", "XXX", e); } Constants.log.info("Sync complete for authority " + authority); } + + + private void processVCard(LocalAddressBook addressBook, MaplocalContacts, String fileName, String eTag, InputStream stream, Charset charset) throws IOException, ContactsStorageException { + Contact contacts[] = Contact.fromStream(stream, charset); + if (contacts.length == 1) { + Contact newData = contacts[0]; + + // delete local contact, if it exists + LocalContact localContact = localContacts.get(fileName); + if (localContact != null) { + Constants.log.info("Updating " + fileName + " in local address book"); + localContact.eTag = eTag; + localContact.update(newData); + } else { + Constants.log.info("Adding " + fileName + " to local address book"); + localContact = new LocalContact(addressBook, newData, fileName, eTag); + localContact.add(); + } + } else + Constants.log.error("Received VCard with not exactly one VCARD, ignoring " + fileName); + } + + } + } diff --git a/app/src/main/java/at/bitfire/davdroid/ui/MainActivity.java b/app/src/main/java/at/bitfire/davdroid/ui/MainActivity.java index 165661fd..76c73c1e 100644 --- a/app/src/main/java/at/bitfire/davdroid/ui/MainActivity.java +++ b/app/src/main/java/at/bitfire/davdroid/ui/MainActivity.java @@ -20,6 +20,7 @@ import android.view.MenuItem; import android.view.View; import android.widget.TextView; +import at.bitfire.davdroid.BuildConfig; import at.bitfire.davdroid.Constants; import at.bitfire.davdroid.R; import at.bitfire.davdroid.ui.settings.SettingsActivity; @@ -41,7 +42,7 @@ public class MainActivity extends Activity { } TextView tvInfo = (TextView)findViewById(R.id.text_info); - tvInfo.setText(Html.fromHtml(getString(R.string.html_main_info, Constants.APP_VERSION))); + tvInfo.setText(Html.fromHtml(getString(R.string.html_main_info, BuildConfig.VERSION_NAME))); tvInfo.setMovementMethod(LinkMovementMethod.getInstance()); } diff --git a/app/src/main/java/at/bitfire/davdroid/webdav/DavHttpClient.java b/app/src/main/java/at/bitfire/davdroid/webdav/DavHttpClient.java index c102928d..26ab951c 100644 --- a/app/src/main/java/at/bitfire/davdroid/webdav/DavHttpClient.java +++ b/app/src/main/java/at/bitfire/davdroid/webdav/DavHttpClient.java @@ -20,6 +20,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import at.bitfire.davdroid.BuildConfig; import at.bitfire.davdroid.Constants; @@ -57,7 +58,7 @@ public class DavHttpClient { .setDefaultRequestConfig(defaultRqConfig) .setRetryHandler(DavHttpRequestRetryHandler.INSTANCE) .setRedirectStrategy(DavRedirectStrategy.INSTANCE) - .setUserAgent("DAVdroid/" + Constants.APP_VERSION); + .setUserAgent("DAVdroid/" + BuildConfig.VERSION_NAME); if (Log.isLoggable("Wire", Log.DEBUG)) { Log.i(TAG, "Wire logging active, disabling HTTP compression"); diff --git a/dav4android b/dav4android index 84a2cf0b..a6975918 160000 --- a/dav4android +++ b/dav4android @@ -1 +1 @@ -Subproject commit 84a2cf0bbad257274e362851020da3822957449d +Subproject commit a6975918ed614eef93c222451fd0981c60ec3ad9