Add encoding detection callback

src/main/java/spoon/compiler/Environment.java

@@ -34,6 +34,7 @@
 
 import java.io.File;
 import java.nio.charset.Charset;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 /**
@@ -378,11 +379,21 @@ void report(Processor<?> processor, Level level,
 	 */
 	Charset getEncoding();
 
+	/**
+	 * Get callback, which is used to detect encoding for each file separately
+	 */
+	Function<byte[], Charset> getEncodingDetectionCallback();
+
 	/**
 	 * Set the encoding to use for parsing source code
 	 */
 	void setEncoding(Charset encoding);
 
+	/**
+	 * Set callback, which is used to detect encoding for each file separately
+	 */
+	void setEncodingDetectionCallback(Function<byte[], Charset> callback);
+
 	/**
 	 * Set the output type used for processing files
 	 */

src/main/java/spoon/support/StandardEnvironment.java

@@ -55,6 +55,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 
@@ -100,6 +101,8 @@ public class StandardEnvironment implements Serializable, Environment {
 
 	private transient Charset encoding = Charset.defaultCharset();
 
+	private transient Function<byte[], Charset> encodingDetectionCallback;
+
 	private int complianceLevel = DEFAULT_CODE_COMPLIANCE_LEVEL;
 
 	private transient OutputDestinationHandler outputDestinationHandler = new DefaultOutputDestinationHandler(new File(Launcher.OUTPUTDIR), this);
@@ -582,11 +585,21 @@ public Charset getEncoding() {
 		return this.encoding;
 	}
 
+	@Override
+	public Function<byte[], Charset> getEncodingDetectionCallback() {
+		return encodingDetectionCallback;
+	}
+
 	@Override
 	public void setEncoding(Charset encoding) {
 		this.encoding = encoding;
 	}
 
+	@Override
+	public void setEncodingDetectionCallback(Function<byte[], Charset> encodingDetectionCallback) {
+		this.encodingDetectionCallback = encodingDetectionCallback;
+	}
+
 	@Override
 	public void setOutputType(OutputType outputType) {
 		this.outputType = outputType;

src/main/java/spoon/support/compiler/jdt/FileCompilerConfig.java

@@ -17,6 +17,7 @@
 package spoon.support.compiler.jdt;
 
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -64,8 +65,15 @@ public void initializeCompiler(JDTBatchCompiler compiler) {
 
 				String fName = f.isActualFile() ? f.getPath() : f.getName();
 				inputStream = f.getContent();
-				char[] content = IOUtils.toCharArray(inputStream, jdtCompiler.getEnvironment().getEncoding());
-				cuList.add(new CompilationUnit(content, fName, jdtCompiler.getEnvironment().getEncoding().displayName()));
+				if (jdtCompiler.getEnvironment().getEncodingDetectionCallback() == null) {
+					char[] content = IOUtils.toCharArray(inputStream, jdtCompiler.getEnvironment().getEncoding());
+					cuList.add(new CompilationUnit(content, fName, jdtCompiler.getEnvironment().getEncoding().displayName()));
+				} else {
+					byte[] bytes = IOUtils.toByteArray(inputStream);
+					Charset encoding = jdtCompiler.getEnvironment().getEncodingDetectionCallback().apply(bytes);
+					char[] content = new String(bytes, encoding).toCharArray();
+					cuList.add(new CompilationUnit(content, fName, encoding.displayName()));
+				}
 				IOUtils.closeQuietly(inputStream);
 			}
 		} catch (Exception e) {

src/test/java/spoon/test/compilationunit/TestCompilationUnit.java

@@ -19,6 +19,8 @@
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;
 import spoon.Launcher;
+import spoon.SpoonException;
+import spoon.reflect.CtModel;
 import spoon.reflect.cu.CompilationUnit;
 import spoon.reflect.cu.SourcePosition;
 import spoon.reflect.cu.position.BodyHolderSourcePosition;
@@ -42,12 +44,14 @@
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+
 /**
  * Created by urli on 18/08/2017.
  */
@@ -267,4 +271,40 @@ public void visitCtCompilationUnit(CtCompilationUnit compilationUnit) {
 			}
 		}.scan(type.getFactory().getModel().getRootPackage());
 	}
+
+	private Charset detectEncodingDummy(byte[] fileBytes) {
+		if (fileBytes.length == 76) {
+			return Charset.forName("Cp1251");
+		} else if (fileBytes.length == 86) {
+			return Charset.forName("UTF-8");
+		}
+		throw new SpoonException("unexpected length");
+	}
+
+	@Test
+	public void testDifferentEncodings() throws Exception {
+		//contract: both utf-8 and cp1251 files in the same project should be handled properly
+		final Launcher launcher = new Launcher();
+		launcher.addInputResource("./src/test/resources/encodings/Cp1251.java");
+		launcher.addInputResource("./src/test/resources/encodings/Utf8.java");
+		launcher.getEnvironment().setEncodingDetectionCallback(this::detectEncodingDummy);
+		CtModel model = launcher.buildModel();
+
+		CtType<?> utf8Type = model.getAllTypes()
+				.stream()
+				.filter(t -> "Utf8".equals(t.getSimpleName()))
+				.findFirst()
+				.get();
+
+		CtType<?> cp1251Type = model.getAllTypes()
+				.stream()
+				.filter(t -> "Cp1251".equals(t.getSimpleName()))
+				.findFirst()
+				.get();
+
+		assertEquals("\"Привет мир\"", utf8Type.getField("s1").getAssignment().toString());
+		assertEquals("\"Привет мир\"", cp1251Type.getField("s1").getAssignment().toString());
+		assertEquals(utf8Type.getField("s1"), cp1251Type.getField("s1"));
+		assertNotEquals(utf8Type.getField("s2"), cp1251Type.getField("s2"));
+	}
 }

src/test/resources/encodings/Cp1251.java

@@ -0,0 +1,4 @@
+public class Cp1251 {
+    String s1 = "������ ���";
+    String s2 = "���"
+}

src/test/resources/encodings/Utf8.java

@@ -0,0 +1,4 @@
+public class Utf8 {
+    String s1 = "Привет мир";
+    String s2 = "ГДЕ"
+}