Bug 292908: Implementation of AbstractCharArray for large files.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/InternalParserUtil.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2003, 2009 IBM Corporation and others.
+ * Copyright (c) 2003, 2010 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -20,7 +20,8 @@ import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.index.IIndexFileLocation;
 import org.eclipse.cdt.core.parser.FileContent;
 import org.eclipse.cdt.core.parser.ParserFactory;
-import org.eclipse.cdt.internal.core.parser.scanner.CharArray;
+import org.eclipse.cdt.internal.core.parser.scanner.AbstractCharArray;
+import org.eclipse.cdt.internal.core.parser.scanner.FileCharArray;
 import org.eclipse.cdt.internal.core.parser.scanner.InternalFileContent;
 import org.eclipse.cdt.internal.core.resources.PathCanonicalizationStrategy;
 import org.eclipse.core.resources.IFile;
@@ -197,9 +198,8 @@ public class InternalParserUtil extends ParserFactory {
 
 	private static InternalFileContent createFileContent(String path, String charset, InputStream in) {
 		try {
-			// replace with a better implementation 
-			org.eclipse.cdt.core.parser.CodeReader reader= new org.eclipse.cdt.core.parser.CodeReader(path, charset, in);
-			return new InternalFileContent(path, new CharArray(reader.buffer));
+			AbstractCharArray chars= FileCharArray.create(path, charset, in);
+			return new InternalFileContent(path, chars);
 		} catch (IOException e) {
 			CCorePlugin.log(e);
 		}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/FileCharArray.java

@@ -0,0 +1,156 @@
+/*******************************************************************************
+ * Copyright (c) 2010 Wind River Systems, Inc. and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Markus Schorn - initial API and implementation
+ *******************************************************************************/ 
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+
+/**
+ * Implementation of char array for a file referencing content via 
+ * soft references.
+ */
+public class FileCharArray extends LazyCharArray {
+	
+	public static AbstractCharArray create(String fileName, String charSet, InputStream in) throws IOException {
+		// no support for non-local files
+		if (!(in instanceof FileInputStream)) {
+			return null;
+		}
+		FileInputStream fis= (FileInputStream) in;
+		if (!Charset.isSupported(charSet)) {
+			charSet= System.getProperty("file.encoding"); //$NON-NLS-1$
+		}
+		FileChannel channel = fis.getChannel();
+		final long lsize = channel.size();
+		if (lsize < CHUNK_SIZE) {
+			return decodeSmallFile(channel, (int) lsize, charSet);
+		}
+
+		return new FileCharArray(fileName, charSet);
+	}
+	
+	private static AbstractCharArray decodeSmallFile(FileChannel channel, int lsize, String charSet) throws IOException {
+		ByteBuffer byteBuffer = ByteBuffer.allocate(lsize);
+		channel.read(byteBuffer);
+		byteBuffer.flip();
+
+		CharBuffer charBuffer = Charset.forName(charSet).decode(byteBuffer);
+		char[] buf= extractChars(charBuffer);
+		return new CharArray(buf);
+	}
+
+	private String fFileName;
+	private String fCharSet;
+	private FileChannel fChannel;
+
+	private FileCharArray(String fileName, String charSet) {
+		fFileName= fileName;
+		fCharSet= charSet;
+	}
+
+	
+	
+	@Override
+	protected Chunk createChunk(int chunkOffset) {
+		FileInputStream fis;
+		try {
+			fis = new FileInputStream(fFileName);
+		} catch (FileNotFoundException e1) {
+			// File has been deleted in the meantime
+			return null;
+		}
+		fChannel= fis.getChannel();
+		try {
+			return super.createChunk(chunkOffset);
+		} finally {
+			fChannel= null;
+			try {
+				fis.close();
+			} catch (IOException e) {
+			}
+		}
+	}
+	
+	@Override
+	protected long readChunkData(long fileOffset, CharBuffer dest) throws IOException {
+		assert fChannel != null;
+		final Charset charset = Charset.forName(fCharSet);
+		final CharsetDecoder decoder = charset.newDecoder().onMalformedInput(CodingErrorAction.REPLACE)
+				.onUnmappableCharacter(CodingErrorAction.REPLACE);
+
+		int needBytes = (int) (CHUNK_SIZE * (double) decoder.averageCharsPerByte()); // avoid rounding errors.
+		final ByteBuffer in = ByteBuffer.allocate(needBytes);
+
+		int total= 0;
+		boolean endOfInput= false;
+		while(total < CHUNK_SIZE && !endOfInput) {
+			fChannel.position(fileOffset);
+			in.clear();
+			int count= fChannel.read(in);
+			if (count == -1) {
+				return fileOffset;
+			}
+			
+			endOfInput= count < in.capacity();
+			total+= count;
+			in.flip();
+			decoder.decode(in, dest, endOfInput);
+			fileOffset+= in.position();
+		}
+		return fileOffset;
+	}
+	
+	
+	@Override
+	protected void rereadChunkData(long fileOffset, long fileEndOffset, CharBuffer dest) {
+		FileInputStream fis;
+		try {
+			fis = new FileInputStream(fFileName);
+		} catch (FileNotFoundException e1) {
+			// File has been deleted in the meantime
+			return;
+		}
+		try {
+			FileChannel channel = fis.getChannel();
+			decode(channel, fileOffset, fileEndOffset, dest);
+		} catch (IOException e) {
+			// File cannot be read
+		} finally {
+			try {
+				fis.close();
+			} catch (IOException e) {
+			}
+		}
+	}
+
+	private void decode(FileChannel channel, long fileOffset, long fileEndOffset, CharBuffer dest) throws IOException {
+		final Charset charset = Charset.forName(fCharSet);
+		final CharsetDecoder decoder = charset.newDecoder().onMalformedInput(CodingErrorAction.REPLACE)
+				.onUnmappableCharacter(CodingErrorAction.REPLACE);
+
+		int needBytes = (int) (fileEndOffset-fileOffset);
+		final ByteBuffer in = ByteBuffer.allocate(needBytes);
+
+		channel.position(fileOffset);
+		in.clear();
+		channel.read(in);
+		in.flip();
+		decoder.decode(in, dest, true);
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LazyCharArray.java

@@ -0,0 +1,198 @@
+/*******************************************************************************
+ * Copyright (c) 2010 Wind River Systems, Inc. and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Markus Schorn - initial API and implementation
+ *******************************************************************************/ 
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import java.lang.ref.SoftReference;
+import java.nio.CharBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Implementation of char array for a file referencing content via 
+ * soft references.
+ */
+public abstract class LazyCharArray extends AbstractCharArray {
+	private final static int CHUNK_BITS= 16;  // 2^16 == 64K
+	protected final static int CHUNK_SIZE= 1 << CHUNK_BITS;
+	
+	/**
+	 * Utility method to extract char[] out of CharBuffer.
+	 */
+	protected static char[] extractChars(CharBuffer charBuffer) {
+		if (charBuffer.hasArray() && charBuffer.arrayOffset() == 0) {
+			char[] buf = charBuffer.array();
+			if (buf.length == charBuffer.remaining())
+				return buf;
+		}
+		char[] buf = new char[charBuffer.remaining()];
+		charBuffer.get(buf);
+		return buf;
+	}
+
+	protected static class Chunk {
+		final int fDataLength;
+		final long fFileOffset;
+		final long fFileEndOffset;
+		private SoftReference<char[]> fData;
+
+		private Chunk(long fileOffset, long fileEndOffset, char[] data) {
+			fDataLength= data.length;
+			fFileOffset= fileOffset;
+			fFileEndOffset= fileEndOffset;
+			fData= new SoftReference<char[]>(data);
+		}
+	}
+	
+	private int fLength= -1;
+	private List<Chunk> fChunks= new ArrayList<Chunk>();
+
+	protected LazyCharArray() {
+	}
+
+	@Override
+	public final int tryGetLength() {
+		return fLength;
+	}
+
+	@Override
+	public final int getLength() {
+		readUpTo(Integer.MAX_VALUE);
+		return fLength;
+	}
+
+	@Override
+	public final boolean isValidOffset(int offset) {
+		if (offset < 0)
+			return false;
+
+		readUpTo(offset);
+		if (fLength >= 0)
+			return offset < fLength;
+		
+		return offset < fChunks.size() << CHUNK_BITS;
+	}
+
+	private void readUpTo(int offset) {
+		if (fLength >=0)
+			return;
+		
+		final int chunkOffset= offset >> CHUNK_BITS;
+		getChunkData(chunkOffset);
+	}
+
+	@Override
+	public final char get(int offset) {
+		int chunkOffset= offset >> CHUNK_BITS;
+		char[] data= getChunkData(chunkOffset);
+		return data[offset & (CHUNK_SIZE-1)];
+	}
+
+	@Override
+	public final void arraycopy(int offset, char[] destination, int destinationPos, int length) {
+		int chunkOffset= offset >> CHUNK_BITS;
+		int loffset= offset & (CHUNK_SIZE-1);
+		char[] data= getChunkData(chunkOffset);
+		final int canCopy = data.length-loffset;
+		if (length <= canCopy) {
+			System.arraycopy(data, loffset, destination, destinationPos, length);
+			return;
+		}
+		System.arraycopy(data, loffset, destination, destinationPos, canCopy);
+		arraycopy(offset+canCopy, destination, destinationPos+canCopy, length-canCopy);
+	}
+	
+	private char[] getChunkData(int chunkOffset) {
+		Chunk chunk= getChunk(chunkOffset);
+		if (chunk != null) {
+			char[] data= chunk.fData.get();
+			if (data != null)
+				return data;
+			
+			return loadChunkData(chunk);
+		}
+		return null;
+	}
+
+	private Chunk getChunk(int chunkOffset) {
+		final int chunkCount = fChunks.size();
+		if (chunkOffset < chunkCount)
+			return fChunks.get(chunkOffset);
+		
+		if (fLength >=0)
+			return null;
+		
+		return createChunk(chunkOffset);
+	}
+		
+	/**
+	 * Called when a chunk is requested for the first time. There is no
+	 * need to override this method.
+	 */
+	protected Chunk createChunk(int chunkOffset) {
+		final int chunkCount = fChunks.size();
+		long fileOffset= chunkCount == 0 ? 0 : fChunks.get(chunkCount-1).fFileEndOffset;
+		try {
+			CharBuffer dest= CharBuffer.allocate(CHUNK_SIZE);
+			for (int i = chunkCount; i <= chunkOffset; i++) {
+				dest.clear();
+				long fileEndOffset= readChunkData(fileOffset, dest);
+				dest.flip();
+				final int charCount= dest.remaining();
+				if (charCount == 0) {
+					fLength= fChunks.size() * CHUNK_SIZE;
+					break;
+				} 
+				// New chunk
+				Chunk chunk= new Chunk(fileOffset, fileEndOffset, extractChars(dest));
+				fChunks.add(chunk);
+				if (charCount < CHUNK_SIZE) {
+					fLength= (fChunks.size()-1) * CHUNK_SIZE + charCount;
+					break;
+				} 
+			}
+		} catch (Exception e) {
+			// File cannot be read
+			return null;
+		} 
+		
+		if (chunkOffset < fChunks.size())
+			return fChunks.get(chunkOffset);
+
+		return null;
+	}
+	
+	private char[] loadChunkData(Chunk chunk) {
+		CharBuffer dest= CharBuffer.allocate(chunk.fDataLength);
+		rereadChunkData(chunk.fFileOffset, chunk.fFileEndOffset, dest);
+		dest.flip();
+		char[] result= extractChars(dest);
+		if (result.length != chunk.fDataLength) {
+			// In case the file changed
+			char[] copy= new char[chunk.fDataLength];
+			System.arraycopy(result, 0, copy, 0, Math.min(result.length, copy.length));
+			result= copy;
+		}
+		chunk.fData= new SoftReference<char[]>(result);
+		return result;
+	}
+
+	/**
+	 * Read the chunk data at the given source offset and provide the end-offset in the
+	 * source.
+	 */
+	protected abstract long readChunkData(long sourceOffset, CharBuffer dest) throws Exception;
+		
+	/**
+	 * Read the chunk data at the given source range. In case the source range no longer (fully) exists,
+	 * read as much as possible.
+	 */
+	protected abstract void rereadChunkData(long fileOffset, long fileEndOffset, CharBuffer dest);
+}