Fetch resources for uploading one-by-one to save memory

src/at/bitfire/davdroid/ArrayUtils.java

@@ -0,0 +1,25 @@
+package at.bitfire.davdroid;
+
+import at.bitfire.davdroid.resource.Resource;
+
+public class ArrayUtils {
+
+	public static <T> Resource[][] partition(Resource[] bigArray, int max) {
+		int nItems = bigArray.length;
+		int nPartArrays = (nItems + max-1)/max;
+		
+		Resource[][] partArrays = new Resource[nPartArrays][];
+		
+		// nItems: number of remaining items
+		for (int i = 0; nItems > 0; i++) {
+			int n = (nItems < max) ? nItems : max;
+			partArrays[i] = new Resource[n];
+			System.arraycopy(bigArray, i*max, partArrays[i], 0, n);
+			
+			nItems -= n;
+		}
+		
+		return partArrays;
+	}
+	
+}

src/at/bitfire/davdroid/LoggingInputStream.java

@@ -8,10 +8,9 @@ import java.io.InputStream;
 import android.util.Log;
 
 public class LoggingInputStream extends FilterInputStream {
-	private static final int MAX_LENGTH = 10*1024;	// don't log more than 10 kB of data
+	private static final int MAX_LENGTH = 1000;	// don't log more than this amount of data
 
 	String tag;
-	InputStream proxy;
 
 	ByteArrayOutputStream log = new ByteArrayOutputStream(MAX_LENGTH);
 	int logSize = 0;
@@ -21,7 +20,6 @@ public class LoggingInputStream extends FilterInputStream {
 	public LoggingInputStream(String tag, InputStream proxy) {
 		super(proxy);
 		this.tag = tag;
-		this.proxy = proxy;
 	}
 	
 
@@ -59,7 +57,7 @@ public class LoggingInputStream extends FilterInputStream {
 
 	@Override
 	public void close() throws IOException {
-		Log.d(tag, "Content: " + log.toString() + (overflow ? "…" : ""));
+		Log.d(tag, log.toString() + (overflow ? "…" : ""));
 		super.close();
 	}
 

src/at/bitfire/davdroid/resource/LocalCalendar.java

@@ -562,7 +562,7 @@ public class LocalCalendar extends LocalCollection<Event> {
 		if (alarm.getTrigger() != null && (duration = alarm.getTrigger().getDuration()) != null)
 			minutes = duration.getDays() * 24*60 + duration.getHours()*60 + duration.getMinutes();
 		
-		Log.i(TAG, "Adding alarm " + minutes + " min before");
+		Log.d(TAG, "Adding alarm " + minutes + " min before");
 		
 		return builder
 				.withValue(Reminders.METHOD, Reminders.METHOD_ALERT)

src/at/bitfire/davdroid/resource/LocalCollection.java

@@ -10,12 +10,15 @@ package at.bitfire.davdroid.resource;
 import java.util.ArrayList;
 import java.util.LinkedList;
 
+import org.apache.commons.lang.ArrayUtils;
+
 import lombok.Cleanup;
 import android.accounts.Account;
 import android.content.ContentProviderClient;
 import android.content.ContentProviderOperation;
 import android.content.ContentProviderOperation.Builder;
 import android.content.ContentUris;
+import android.content.ContentValues;
 import android.content.OperationApplicationException;
 import android.database.Cursor;
 import android.net.Uri;
@@ -64,7 +67,7 @@ public abstract class LocalCollection<T extends Resource> {
 	
 	// content provider (= database) querying
 	
-	public Resource[] findDirty() throws LocalStorageException {
+	public long[] findDirty() throws LocalStorageException {
 		String where = entryColumnDirty() + "=1";
 		if (entryColumnParentID() != null)
 			where += " AND " + entryColumnParentID() + "=" + String.valueOf(getId());
@@ -72,23 +75,16 @@ public abstract class LocalCollection<T extends Resource> {
 			@Cleanup Cursor cursor = providerClient.query(entriesURI(),
 					new String[] { entryColumnID(), entryColumnRemoteName(), entryColumnETag() },
 					where, null, null);
-			LinkedList<T> dirty = new LinkedList<T>();
+			LinkedList<Long> dirty = new LinkedList<Long>();
-			while (cursor != null && cursor.moveToNext()) {
+			while (cursor != null && cursor.moveToNext())
-				long id = cursor.getLong(0);
+				dirty.add(cursor.getLong(0));
-				try {
+			return ArrayUtils.toPrimitive(dirty.toArray(new Long[0]), -1);
-					T resource = findById(id, true); 
-					dirty.add(resource);
-				} catch (RecordNotFoundException e) {
-					Log.w(TAG, "Couldn't load dirty resource: " + ContentUris.appendId(entriesURI().buildUpon(), id), e);
-				}
-			}
-			return dirty.toArray(new Resource[0]);
 		} catch(RemoteException ex) {
 			throw new LocalStorageException(ex);
 		}
 	}
 
-	public Resource[] findDeleted() throws LocalStorageException {
+	public long[] findDeleted() throws LocalStorageException {
 		String where = entryColumnDeleted() + "=1";
 		if (entryColumnParentID() != null)
 			where += " AND " + entryColumnParentID() + "=" + String.valueOf(getId());
@@ -96,23 +92,17 @@ public abstract class LocalCollection<T extends Resource> {
 			@Cleanup Cursor cursor = providerClient.query(entriesURI(),
 					new String[] { entryColumnID(), entryColumnRemoteName(), entryColumnETag() },
 					where, null, null);
-			LinkedList<T> deleted = new LinkedList<T>();
+			LinkedList<Long> deleted = new LinkedList<Long>();
-			while (cursor != null && cursor.moveToNext()) {
+			while (cursor != null && cursor.moveToNext())
-				long id = cursor.getLong(0);
+				deleted.add(cursor.getLong(0));
-				try {
+			return ArrayUtils.toPrimitive(deleted.toArray(new Long[0]), -1);
-					T resource = findById(id, false);
-					deleted.add(resource);
-				} catch (RecordNotFoundException e) {
-					Log.w(TAG, "Couldn't load resource marked for deletion: " + ContentUris.appendId(entriesURI().buildUpon(), id), e);
-				}
-			}
-			return deleted.toArray(new Resource[0]);
 		} catch(RemoteException ex) {
 			throw new LocalStorageException(ex);
 		}
 	}
 
-	public Resource[] findNew() throws LocalStorageException {
+	public long[] findNew() throws LocalStorageException {
+		// new records are 1) dirty, and 2) don't have a remote file name yet
 		String where = entryColumnDirty() + "=1 AND " + entryColumnRemoteName() + " IS NULL";
 		if (entryColumnParentID() != null)
 			where += " AND " + entryColumnParentID() + "=" + String.valueOf(getId());
@@ -120,26 +110,22 @@ public abstract class LocalCollection<T extends Resource> {
 			@Cleanup Cursor cursor = providerClient.query(entriesURI(),
 					new String[] { entryColumnID() },
 					where, null, null);
-			LinkedList<T> fresh = new LinkedList<T>();
+			LinkedList<Long> fresh = new LinkedList<Long>();
 			while (cursor != null && cursor.moveToNext()) {
 				long id = cursor.getLong(0);
-				try {
+				
-					T resource = findById(id, true);
+				// new record: generate UID + remote file name so that we can upload
-					resource.initialize();
+				T resource = findById(id, false);
-		
+				resource.initialize();
-					// new record: set generated UID + remote file name in database
+				// write generated UID + remote file name into database
-					pendingOperations.add(ContentProviderOperation
+				ContentValues values = new ContentValues(2);
-							.newUpdate(ContentUris.withAppendedId(entriesURI(), resource.getLocalID()))
+				values.put(entryColumnUID(), resource.getUid());
-							.withValue(entryColumnUID(), resource.getUid())
+				values.put(entryColumnRemoteName(), resource.getName());
-							.withValue(entryColumnRemoteName(), resource.getName())
+				providerClient.update(ContentUris.withAppendedId(entriesURI(), id), values, null, null);
-							.build());
+				
-					
+				fresh.add(id);
-					fresh.add(resource);
-				} catch (RecordNotFoundException e) {
-					Log.w(TAG, "Couldn't load fresh resource: " + ContentUris.appendId(entriesURI().buildUpon(), id), e);
-				}
 			}
-			return fresh.toArray(new Resource[0]);
+			return ArrayUtils.toPrimitive(fresh.toArray(new Long[0]), -1);
 		} catch(RemoteException ex) {
 			throw new LocalStorageException(ex);
 		}

src/at/bitfire/davdroid/resource/RemoteCollection.java

@@ -21,9 +21,7 @@ import lombok.Getter;
 import net.fortuna.ical4j.data.ParserException;
 import net.fortuna.ical4j.model.ValidationException;
 
-import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpException;
-import org.apache.http.protocol.HTTP;
 
 import android.util.Log;
 import at.bitfire.davdroid.webdav.DavException;
@@ -82,11 +80,11 @@ public abstract class RemoteCollection<T extends Resource> {
 			for (Resource resource : resources)
 				names.add(resource.getName());
 			
+			LinkedList<T> foundResources = new LinkedList<T>();
 			collection.multiGet(multiGetType(), names.toArray(new String[0]));
 			if (collection.getMembers() == null)
 				throw new DavNoContentException();
 			
-			LinkedList<T> foundResources = new LinkedList<T>();
 			for (WebDavResource member : collection.getMembers()) {
 				T resource = newResourceSkeleton(member.getName(), member.getETag());
 				try {
@@ -124,9 +122,6 @@ public abstract class RemoteCollection<T extends Resource> {
 		if (data == null)
 			throw new DavNoContentException();
 		
-		Log.i(TAG, "Received content:");
-		Log.i(TAG, IOUtils.toString(data, HTTP.UTF_8));
-		
 		@Cleanup InputStream is = new ByteArrayInputStream(data);
 		resource.parseEntity(is);
 		return resource;

src/at/bitfire/davdroid/syncadapter/SyncManager.java

@@ -95,20 +95,21 @@ public class SyncManager {
 	
 	private int pushDeleted(LocalCollection<? extends Resource> local, RemoteCollection<? extends Resource> remote) throws LocalStorageException, IOException, HttpException {
 		int count = 0;
-		Resource[] deletedResources = local.findDeleted();
+		long[] deletedIDs = local.findDeleted();
-		if (deletedResources != null) {
+		if (deletedIDs != null) {
-			Log.i(TAG, "Remotely removing " + deletedResources.length + " deleted resource(s) (if not changed)");
+			Log.i(TAG, "Remotely removing " + deletedIDs.length + " deleted resource(s) (if not changed)");
-			for (Resource res : deletedResources) {
+			for (long id : deletedIDs) {
 				try {
+					Resource res = local.findById(id, false);
 					if (res.getName() != null)	// is this resource even present remotely?
 						remote.delete(res);
+					local.delete(res);
+					count++;
 				} catch(NotFoundException e) {
 					Log.i(TAG, "Locally-deleted resource has already been removed from server");
 				} catch(PreconditionFailedException e) {
 					Log.i(TAG, "Locally-deleted resource has been changed on the server in the meanwhile");
 				}
-				local.delete(res);
-				count++;
 			}
 			local.commit();
 		}
@@ -117,18 +118,20 @@ public class SyncManager {
 	
 	private int pushNew(LocalCollection<? extends Resource> local, RemoteCollection<? extends Resource> remote) throws LocalStorageException, IOException, HttpException {
 		int count = 0;
-		Resource[] newResources = local.findNew();
+		long[] newIDs = local.findNew();
-		if (newResources != null) {
+		if (newIDs != null) {
-			Log.i(TAG, "Uploading " + newResources.length + " new resource(s) (if not existing)");
+			Log.i(TAG, "Uploading " + newIDs.length + " new resource(s) (if not existing)");
-			for (Resource res : newResources) {
+			for (long id : newIDs) {
 				try {
+					Resource res = local.findById(id, true);
+					Log.d(TAG, "Uploading " + res.toString());
 					remote.add(res);
+					local.clearDirty(res);
 				} catch(PreconditionFailedException e) {
 					Log.i(TAG, "Didn't overwrite existing resource with other content");
 				} catch (ValidationException e) {
 					Log.e(TAG, "Couldn't create entity for adding: " + e.toString());
 				}
-				local.clearDirty(res);
 				count++;
 			}
 			local.commit();
@@ -138,18 +141,19 @@ public class SyncManager {
 	
 	private int pushDirty(LocalCollection<? extends Resource> local, RemoteCollection<? extends Resource> remote) throws LocalStorageException, IOException, HttpException {
 		int count = 0;
-		Resource[] dirtyResources = local.findDirty();
+		long[] dirtyIDs = local.findDirty();
-		if (dirtyResources != null) {
+		if (dirtyIDs != null) {
-			Log.i(TAG, "Uploading " + dirtyResources.length + " modified resource(s) (if not changed)");
+			Log.i(TAG, "Uploading " + dirtyIDs.length + " modified resource(s) (if not changed)");
-			for (Resource res : dirtyResources) {
+			for (long id : dirtyIDs) {
 				try {
+					Resource res = local.findById(id, true);
 					remote.update(res);
+					local.clearDirty(res);
 				} catch(PreconditionFailedException e) {
 					Log.i(TAG, "Locally changed resource has been changed on the server in the meanwhile");
 				} catch (ValidationException e) {
 					Log.e(TAG, "Couldn't create entity for updating: " + e.toString());
 				}
-				local.clearDirty(res);
 				count++;
 			}
 			local.commit();

src/at/bitfire/davdroid/webdav/WebDavResource.java

@@ -240,8 +240,8 @@ public class WebDavResource {
 		if (entity == null)
 			throw new DavNoContentException();
 		
-		InputStream rawContent = entity.getContent();
+		@Cleanup InputStream rawContent = entity.getContent();
-		if (content == null)
+		if (rawContent == null)
 			throw new DavNoContentException();
 		@Cleanup LoggingInputStream content = new LoggingInputStream(TAG, rawContent);
 		
@@ -304,9 +304,15 @@ public class WebDavResource {
 		checkResponse(response);
 		
 		@Cleanup("consumeContent") HttpEntity entity = response.getEntity();
-		if (entity.getContent() == null)
+		if (entity == null)
 			throw new DavNoContentException();
-		content = IOUtils.toByteArray(entity.getContent());
+		
+		@Cleanup InputStream rawContent = entity.getContent();
+		if (rawContent == null)
+			throw new DavNoContentException();
+		@Cleanup LoggingInputStream content = new LoggingInputStream(TAG, rawContent);
+		
+		this.content = IOUtils.toByteArray(content);
 	}
 	
 	public void put(byte[] data, PutMode mode) throws IOException, HttpException {
@@ -328,7 +334,9 @@ public class WebDavResource {
 		if (getContentType() != null)
 			put.addHeader("Content-Type", getContentType());
 
-		checkResponse(client.execute(put));
+		HttpResponse response = client.execute(put);
+		@Cleanup("consumeContent") HttpEntity entity = response.getEntity();
+		checkResponse(response);
 	}
 	
 	public void delete() throws IOException, HttpException {
@@ -337,7 +345,9 @@ public class WebDavResource {
 		if (getETag() != null)
 			delete.addHeader("If-Match", getETag());
 		
-		checkResponse(client.execute(delete));
+		HttpResponse response = client.execute(delete);
+		@Cleanup("consumeContent") HttpEntity entity = response.getEntity();
+		checkResponse(response);
 	}