Process Content-Type character set information (fixes #594)

app/src/androidTest/assets/latin1.vcf

@@ -0,0 +1,5 @@
+BEGIN:VCARD
+VERSION:3.0
+N:Äuçek;Özkan
+FN:Özkan Äuçek
+END:VCARD

app/src/androidTest/assets/reference-vcard3.vcf

@@ -1,8 +1,8 @@
 BEGIN:VCARD
 VERSION:3.0
-N:Gump;Forrest;Mr.
+N:Gümp;Förrest;Mr.
-FN:Forrest Gump
+FN:Förrest Gümp
-ORG:Bubba Gump Shrimp Co.
+ORG:Bubba Gump Shrimpß Co.
 TITLE:Shrimp Man
 PHOTO;VALUE=URL;TYPE=PNG:http://192.168.0.11:3000/assets/davdroid-logo-192.png
 TEL;TYPE=WORK,VOICE:(111) 555-1212

app/src/androidTest/java/at/bitfire/davdroid/resource/ContactTest.java

@@ -9,15 +9,20 @@ package at.bitfire.davdroid.resource;
 
 import android.content.res.AssetManager;
 import android.test.InstrumentationTestCase;
+import android.util.Log;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.CharEncoding;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
+import at.bitfire.davdroid.ArrayUtils;
 import at.bitfire.davdroid.webdav.DavException;
 import at.bitfire.davdroid.webdav.HttpException;
 import ezvcard.VCardVersion;
@@ -36,21 +41,22 @@ public class ContactTest extends InstrumentationTestCase {
 		Contact c = new Contact("test.vcf", null);
 
 		// should generate VCard 3.0 by default
-		assertEquals("text/vcard;charset=UTF-8", c.getMimeType());
+		assertEquals("text/vcard; charset=utf-8", c.getContentType().toString().toLowerCase());
 		assertTrue(new String(c.toEntity().toByteArray()).contains("VERSION:3.0"));
 
 		// now let's generate VCard 4.0
 		c.setVCardVersion(VCardVersion.V4_0);
-		assertEquals("text/vcard;version=4.0", c.getMimeType());
+		assertEquals("text/vcard; version=4.0", c.getContentType().toString());
 		assertTrue(new String(c.toEntity().toByteArray()).contains("VERSION:4.0"));
 	}
 	
-	public void testReferenceVCard() throws IOException, InvalidResourceException {
+	public void testReferenceVCard3() throws IOException, InvalidResourceException {
-		Contact c = parseVCF("reference.vcf");
+		Contact c = parseVCF("reference-vcard3.vcf", Charset.forName(CharEncoding.UTF_8));
-		assertEquals("Gump", c.getFamilyName());
+
-		assertEquals("Forrest", c.getGivenName());
+		assertEquals("Gümp", c.getFamilyName());
-		assertEquals("Forrest Gump", c.getDisplayName());
+		assertEquals("Förrest", c.getGivenName());
-		assertEquals("Bubba Gump Shrimp Co.", c.getOrganization().getValues().get(0));
+		assertEquals("Förrest Gümp", c.getDisplayName());
+		assertEquals("Bubba Gump Shrimpß Co.", c.getOrganization().getValues().get(0));
 		assertEquals("Shrimp Man", c.getJobTitle());
 		
 		Telephone phone1 = c.getPhoneNumbers().get(0);
@@ -78,12 +84,20 @@ public class ContactTest extends InstrumentationTestCase {
 		assertEquals("VCard with invalid unknown properties", c.getDisplayName());
 		assertNull(c.getUnknownProperties());
 	}
+
+	public void testParseLatin1() throws IOException {
+		Contact c = parseVCF("latin1.vcf", Charset.forName(CharEncoding.ISO_8859_1));
+		assertEquals("Özkan Äuçek", c.getDisplayName());
+		assertEquals("Özkan", c.getGivenName());
+		assertEquals("Äuçek", c.getFamilyName());
+		assertNull(c.getUnknownProperties());
+	}
 	
 	
-	protected Contact parseVCF(String fname) throws IOException {
+	protected Contact parseVCF(String fname, Charset charset) throws IOException {
 		@Cleanup InputStream in = assetMgr.open(fname, AssetManager.ACCESS_STREAMING);
 		Contact c = new Contact(fname, null);
-		c.parseEntity(in, new Resource.AssetDownloader() {
+		c.parseEntity(in, charset, new Resource.AssetDownloader() {
             @Override
             public byte[] download(URI uri) throws URISyntaxException, IOException, HttpException, DavException {
                 return IOUtils.toByteArray(uri);
@@ -91,4 +105,8 @@ public class ContactTest extends InstrumentationTestCase {
         });
 		return c;
 	}
+
+	protected Contact parseVCF(String fname) throws IOException {
+		return parseVCF(fname, null);
+	}
 }

app/src/androidTest/java/at/bitfire/davdroid/resource/EventTest.java

@@ -111,7 +111,7 @@ public class EventTest extends InstrumentationTestCase {
 	protected Event parseCalendar(String fname) throws IOException, InvalidResourceException {
 		@Cleanup InputStream in = assetMgr.open(fname, AssetManager.ACCESS_STREAMING);
 		Event e = new Event(fname, null);
-		e.parseEntity(in, null);
+		e.parseEntity(in, null, null);
 		return e;
 	}
 }

app/src/androidTest/robohydra/plugins/headdav.js

@@ -4,12 +4,12 @@ var roboHydra = require("robohydra"),
 	roboHydraHeads = roboHydra.heads,
 	roboHydraHead = roboHydraHeads.RoboHydraHead;
 
-RoboHydraHeadDAV = roboHydraHeads.roboHydraHeadType({
+RoboHydraHeadDAV = roboHydraHeads.robohydraHeadType({
 	name: 'WebDAV Server',
 	mandatoryProperties: [ 'path' ],
     optionalProperties: [ 'handler' ],
 
-	parentPropBuilder: function() {
+	parentPropertyBuilder: function() {
 		var myHandler = this.handler;
 		return {
 			path: this.path,

app/src/androidTest/robohydra/plugins/simple.js

@@ -4,12 +4,12 @@ var roboHydra = require("robohydra"),
 	roboHydraHeads = roboHydra.heads,
 	roboHydraHead = roboHydraHeads.RoboHydraHead;
 
-SimpleResponseHead = roboHydraHeads.roboHydraHeadType({
+SimpleResponseHead = roboHydraHeads.robohydraHeadType({
 	name: 'Simple HTTP Response',
 	mandatoryProperties: [ 'path', 'status' ],
 	optionalProperties: [ 'headers', 'body' ],
 
-	parentPropBuilder: function() {
+	parentPropertyBuilder: function() {
         var head = this;
 		return {
 			path: this.path,

app/src/main/java/at/bitfire/davdroid/resource/Contact.java

@@ -9,12 +9,16 @@ package at.bitfire.davdroid.resource;
 
 import android.util.Log;
 
+import org.apache.commons.codec.CharEncoding;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.entity.ContentType;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.net.URI;
+import java.nio.charset.Charset;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.UUID;
@@ -52,6 +56,7 @@ import ezvcard.property.Telephone;
 import ezvcard.property.Title;
 import ezvcard.property.Uid;
 import ezvcard.property.Url;
+import lombok.Cleanup;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
@@ -67,6 +72,10 @@ public class Contact extends Resource {
 
 	@Getter @Setter protected VCardVersion vCardVersion = VCardVersion.V3_0;
 
+	public static final ContentType
+			MIME_VCARD3 = ContentType.create("text/vcard", CharEncoding.UTF_8),
+			MIME_VCARD4 = ContentType.parse("text/vcard; version=4.0");
+
 	public final static String
 		PROPERTY_STARRED = "X-DAVDROID-STARRED",
 		PROPERTY_PHONETIC_FIRST_NAME = "X-PHONETIC-FIRST-NAME",
@@ -89,7 +98,7 @@ public class Contact extends Resource {
 		RELATED_TYPE_MOTHER = RelatedType.get("mother"),
 		RELATED_TYPE_REFERRED_BY = RelatedType.get("referred-by"),
 		RELATED_TYPE_SISTER = RelatedType.get("sister");
-	
+
 	@Getter @Setter private String unknownProperties;
 	
 	@Getter @Setter private boolean starred;
@@ -140,8 +149,13 @@ public class Contact extends Resource {
 
 	@SuppressWarnings("LoopStatementThatDoesntLoop")
     @Override
-	public void parseEntity(InputStream is, AssetDownloader downloader) throws IOException {
+	public void parseEntity(InputStream is, Charset charset, AssetDownloader downloader) throws IOException {
-		VCard vcard = Ezvcard.parse(is).first();
+		final VCard vcard;
+		if (charset != null) {
+			@Cleanup InputStreamReader reader = new InputStreamReader(is, charset);
+			vcard = Ezvcard.parse(reader).first();
+		} else
+			vcard = Ezvcard.parse(is).first();
 		if (vcard == null)
 			return;
 		
@@ -318,11 +332,8 @@ public class Contact extends Resource {
 
 
 	@Override
-	public String getMimeType() {
+	public ContentType getContentType() {
-		if (vCardVersion == VCardVersion.V4_0)
+		return (vCardVersion == VCardVersion.V4_0) ? MIME_VCARD4 : MIME_VCARD3;
-			return "text/vcard;version=4.0";
-		else
-			return "text/vcard;charset=UTF-8";
 	}
 	
 	@Override

app/src/main/java/at/bitfire/davdroid/resource/Event.java

@@ -47,6 +47,8 @@ import net.fortuna.ical4j.util.TimeZones;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.Calendar;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -56,6 +58,7 @@ import java.util.TimeZone;
 
 import at.bitfire.davdroid.Constants;
 import at.bitfire.davdroid.DateUtils;
+import lombok.Cleanup;
 import lombok.Getter;
 import lombok.NonNull;
 import lombok.Setter;
@@ -99,11 +102,15 @@ public class Event extends iCalendar {
 
 	@Override
 	@SuppressWarnings("unchecked")
-	public void parseEntity(@NonNull InputStream entity, AssetDownloader downloader) throws IOException, InvalidResourceException {
+	public void parseEntity(@NonNull InputStream entity, Charset charset, AssetDownloader downloader) throws IOException, InvalidResourceException {
-		net.fortuna.ical4j.model.Calendar ical;
+		final net.fortuna.ical4j.model.Calendar ical;
 		try {
 			CalendarBuilder builder = new CalendarBuilder();
-			ical = builder.build(entity);
+			if (charset != null) {
+				@Cleanup InputStreamReader reader = new InputStreamReader(entity, charset);
+				ical = builder.build(reader);
+			} else
+				ical = builder.build(entity);
 
 			if (ical == null)
 				throw new InvalidResourceException("No iCalendar found");

app/src/main/java/at/bitfire/davdroid/resource/Resource.java

@@ -7,11 +7,14 @@
  */
 package at.bitfire.davdroid.resource;
 
+import org.apache.http.entity.ContentType;
+
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 
 import at.bitfire.davdroid.webdav.DavException;
 import at.bitfire.davdroid.webdav.HttpException;
@@ -46,11 +49,11 @@ public abstract class Resource {
 	 * @param entity        entity to parse
 	 * @param downloader    will be used to fetch additional resources like contact images
 	 **/
-	public abstract void parseEntity(InputStream entity, AssetDownloader downloader) throws IOException, InvalidResourceException;
+	public abstract void parseEntity(InputStream entity, Charset charset, AssetDownloader downloader) throws IOException, InvalidResourceException;
 
 
 	/* returns the MIME type that toEntity() will produce */
-	public abstract String getMimeType();
+	public abstract ContentType getContentType();
 
 	/** writes the resource data to an output stream (for instance, .vcf file for Contact) */
 	public abstract ByteArrayOutputStream toEntity() throws IOException;

app/src/main/java/at/bitfire/davdroid/resource/Task.java

@@ -40,12 +40,15 @@ import net.fortuna.ical4j.model.property.Version;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.util.HashSet;
 import java.util.Set;
 
 import at.bitfire.davdroid.Constants;
+import lombok.Cleanup;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -78,11 +81,15 @@ public class Task extends iCalendar {
 
 
 	@Override
-	public void parseEntity(InputStream entity, AssetDownloader downloader) throws IOException, InvalidResourceException {
+	public void parseEntity(InputStream entity, Charset charset, AssetDownloader downloader) throws IOException, InvalidResourceException {
-		net.fortuna.ical4j.model.Calendar ical;
+		final net.fortuna.ical4j.model.Calendar ical;
 		try {
 			CalendarBuilder builder = new CalendarBuilder();
-			ical = builder.build(entity);
+			if (charset != null) {
+				@Cleanup InputStreamReader reader = new InputStreamReader(entity, charset);
+				ical = builder.build(reader);
+			} else
+				ical = builder.build(entity);
 
 			if (ical == null)
 				throw new InvalidResourceException("No iCalendar found");

app/src/main/java/at/bitfire/davdroid/resource/WebDavCollection.java

@@ -10,6 +10,7 @@ package at.bitfire.davdroid.resource;
 import android.util.Log;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.http.entity.ContentType;
 import org.apache.http.impl.client.CloseableHttpClient;
 
 import java.io.ByteArrayInputStream;
@@ -18,6 +19,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -91,7 +93,6 @@ public abstract class WebDavCollection<T extends Resource> {
 				resources.add(newResourceSkeleton(member.getName(), member.getProperties().getETag()));
 
 		return resources.toArray(new Resource[resources.size()]);
-
 	}
 
 
@@ -117,7 +118,7 @@ public abstract class WebDavCollection<T extends Resource> {
 				try {
 					if (member.getContent() != null) {
 						@Cleanup InputStream is = new ByteArrayInputStream(member.getContent());
-						resource.parseEntity(is, getDownloader());
+						resource.parseEntity(is, null, getDownloader());
 						foundResources.add(resource);
 					} else
 						Log.e(TAG, "Ignoring entity without content");
@@ -142,13 +143,17 @@ public abstract class WebDavCollection<T extends Resource> {
 
 		member.get(memberAcceptedMimeTypes());
 
-		byte[] data = member.getContent();
+		final byte[] data = member.getContent();
 		if (data == null)
 			throw new DavNoContentException();
 
 		@Cleanup InputStream is = new ByteArrayInputStream(data);
 		try {
-			resource.parseEntity(is, getDownloader());
+			Charset charset = null;
+			ContentType mime = member.getProperties().getContentType();
+			if (mime != null)
+				charset = mime.getCharset();
+			resource.parseEntity(is, charset, getDownloader());
 		} catch (VCardParseException e) {
 			throw new InvalidResourceException(e);
 		}
@@ -158,12 +163,12 @@ public abstract class WebDavCollection<T extends Resource> {
 	// returns ETag of the created resource, if returned by server
 	public String add(Resource res) throws URISyntaxException, IOException, HttpException {
 		WebDavResource member = new WebDavResource(collection, res.getName(), res.getETag());
-		member.getProperties().setContentType(res.getMimeType());
+		member.getProperties().setContentType(res.getContentType());
 
 		@Cleanup ByteArrayOutputStream os = res.toEntity();
 		String eTag = member.put(os.toByteArray(), PutMode.ADD_DONT_OVERWRITE);
 
-		// after a successful upload, the collection has implicitely changed, too
+		// after a successful upload, the collection has implicitly changed, too
 		collection.getProperties().invalidateCTag();
 
 		return eTag;
@@ -179,7 +184,7 @@ public abstract class WebDavCollection<T extends Resource> {
 	// returns ETag of the updated resource, if returned by server
 	public String update(Resource res) throws URISyntaxException, IOException, HttpException {
 		WebDavResource member = new WebDavResource(collection, res.getName(), res.getETag());
-		member.getProperties().setContentType(res.getMimeType());
+		member.getProperties().setContentType(res.getContentType());
 
 		@Cleanup ByteArrayOutputStream os = res.toEntity();
 		String eTag = member.put(os.toByteArray(), PutMode.UPDATE_DONT_OVERWRITE);

app/src/main/java/at/bitfire/davdroid/resource/iCalendar.java

@@ -19,6 +19,9 @@ import net.fortuna.ical4j.util.CompatibilityHints;
 import net.fortuna.ical4j.util.SimpleHostInfo;
 import net.fortuna.ical4j.util.UidGenerator;
 
+import org.apache.commons.codec.CharEncoding;
+import org.apache.http.entity.ContentType;
+
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.TimeZone;
@@ -28,7 +31,7 @@ import at.bitfire.davdroid.syncadapter.DavSyncAdapter;
 import lombok.NonNull;
 
 public abstract class iCalendar extends Resource {
-	static private final String TAG = "DAVdroid.iCal";
+	private static final String TAG = "DAVdroid.iCal";
 
 	// static ical4j initialization
 	static {
@@ -37,6 +40,8 @@ public abstract class iCalendar extends Resource {
 		CompatibilityHints.setHintEnabled(CompatibilityHints.KEY_OUTLOOK_COMPATIBILITY, true);
 	}
 
+	public static final ContentType MIME_ICALENDAR = ContentType.create("text/calendar", CharEncoding.UTF_8);
+
 
 	public iCalendar(long localID, String name, String ETag) {
 		super(localID, name, ETag);
@@ -60,8 +65,8 @@ public abstract class iCalendar extends Resource {
 
 
 	@Override
-	public String getMimeType() {
+	public ContentType getContentType() {
-		return "text/calendar";
+		return MIME_ICALENDAR;
 	}
 
 

app/src/main/java/at/bitfire/davdroid/webdav/DavProp.java

@@ -24,7 +24,7 @@ import lombok.Setter;
 public class DavProp {
 	
 	/* RFC 4918 WebDAV */
-	
+
 	@Element(required=false)
 	ResourceType resourcetype;
 	
@@ -36,7 +36,7 @@ public class DavProp {
 	
 	@Element(required=false)
 	@Setter GetETag getetag;
-	
+
 	@Root(strict=false)
 	public static class ResourceType {
 		@Element(required=false)

app/src/main/java/at/bitfire/davdroid/webdav/WebDavResource.java

@@ -26,6 +26,7 @@ import org.apache.http.client.methods.HttpOptionsHC4;
 import org.apache.http.client.methods.HttpPutHC4;
 import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.entity.ByteArrayEntityHC4;
+import org.apache.http.entity.ContentType;
 import org.apache.http.impl.auth.BasicSchemeHC4;
 import org.apache.http.impl.client.BasicAuthCache;
 import org.apache.http.impl.client.BasicCredentialsProviderHC4;
@@ -278,6 +279,7 @@ public class WebDavResource {
 		if (entity == null)
 			throw new DavNoContentException();
 
+		properties.contentType = ContentType.get(entity);
 		content = EntityUtilsHC4.toByteArray(entity);
 	}
 
@@ -296,7 +298,7 @@ public class WebDavResource {
 		}
 
 		if (properties.contentType != null)
-			put.addHeader("Content-Type", properties.contentType);
+			put.addHeader("Content-Type", properties.contentType.toString());
 
 		@Cleanup CloseableHttpResponse response = httpClient.execute(put, context);
 		checkResponse(response);
@@ -364,6 +366,8 @@ public class WebDavResource {
 		HttpEntity entity = response.getEntity();
 		if (entity == null)
 			throw new DavNoContentException();
+
+		properties.contentType = ContentType.get(entity);
 		@Cleanup InputStream content = entity.getContent();
 
 		DavMultistatus multiStatus;
@@ -452,7 +456,7 @@ public class WebDavResource {
 				eTag,
 				cTag;
 
-		@Getter @Setter	protected String contentType;
+		@Getter @Setter	protected ContentType contentType;
 
 		@Getter protected boolean
 				readOnly,