diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/InternalParserUtil.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/InternalParserUtil.java index 1e827303879..6073ffccee8 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/InternalParserUtil.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/InternalParserUtil.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2003, 2009 IBM Corporation and others. + * Copyright (c) 2003, 2010 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -20,7 +20,8 @@ import org.eclipse.cdt.core.CCorePlugin; import org.eclipse.cdt.core.index.IIndexFileLocation; import org.eclipse.cdt.core.parser.FileContent; import org.eclipse.cdt.core.parser.ParserFactory; -import org.eclipse.cdt.internal.core.parser.scanner.CharArray; +import org.eclipse.cdt.internal.core.parser.scanner.AbstractCharArray; +import org.eclipse.cdt.internal.core.parser.scanner.FileCharArray; import org.eclipse.cdt.internal.core.parser.scanner.InternalFileContent; import org.eclipse.cdt.internal.core.resources.PathCanonicalizationStrategy; import org.eclipse.core.resources.IFile; @@ -197,9 +198,8 @@ public class InternalParserUtil extends ParserFactory { private static InternalFileContent createFileContent(String path, String charset, InputStream in) { try { - // replace with a better implementation - org.eclipse.cdt.core.parser.CodeReader reader= new org.eclipse.cdt.core.parser.CodeReader(path, charset, in); - return new InternalFileContent(path, new CharArray(reader.buffer)); + AbstractCharArray chars= FileCharArray.create(path, charset, in); + return new InternalFileContent(path, chars); } catch (IOException e) { CCorePlugin.log(e); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/FileCharArray.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/FileCharArray.java new file mode 100644 index 00000000000..190556efc44 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/FileCharArray.java @@ -0,0 +1,156 @@ +/******************************************************************************* + * Copyright (c) 2010 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; + +/** + * Implementation of char array for a file referencing content via + * soft references. + */ +public class FileCharArray extends LazyCharArray { + + public static AbstractCharArray create(String fileName, String charSet, InputStream in) throws IOException { + // no support for non-local files + if (!(in instanceof FileInputStream)) { + return null; + } + FileInputStream fis= (FileInputStream) in; + if (!Charset.isSupported(charSet)) { + charSet= System.getProperty("file.encoding"); //$NON-NLS-1$ + } + FileChannel channel = fis.getChannel(); + final long lsize = channel.size(); + if (lsize < CHUNK_SIZE) { + return decodeSmallFile(channel, (int) lsize, charSet); + } + + return new FileCharArray(fileName, charSet); + } + + private static AbstractCharArray decodeSmallFile(FileChannel channel, int lsize, String charSet) throws IOException { + ByteBuffer byteBuffer = ByteBuffer.allocate(lsize); + channel.read(byteBuffer); + byteBuffer.flip(); + + CharBuffer charBuffer = Charset.forName(charSet).decode(byteBuffer); + char[] buf= extractChars(charBuffer); + return new CharArray(buf); + } + + private String fFileName; + private String fCharSet; + private FileChannel fChannel; + + private FileCharArray(String fileName, String charSet) { + fFileName= fileName; + fCharSet= charSet; + } + + + + @Override + protected Chunk createChunk(int chunkOffset) { + FileInputStream fis; + try { + fis = new FileInputStream(fFileName); + } catch (FileNotFoundException e1) { + // File has been deleted in the meantime + return null; + } + fChannel= fis.getChannel(); + try { + return super.createChunk(chunkOffset); + } finally { + fChannel= null; + try { + fis.close(); + } catch (IOException e) { + } + } + } + + @Override + protected long readChunkData(long fileOffset, CharBuffer dest) throws IOException { + assert fChannel != null; + final Charset charset = Charset.forName(fCharSet); + final CharsetDecoder decoder = charset.newDecoder().onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + + int needBytes = (int) (CHUNK_SIZE * (double) decoder.averageCharsPerByte()); // avoid rounding errors. + final ByteBuffer in = ByteBuffer.allocate(needBytes); + + int total= 0; + boolean endOfInput= false; + while(total < CHUNK_SIZE && !endOfInput) { + fChannel.position(fileOffset); + in.clear(); + int count= fChannel.read(in); + if (count == -1) { + return fileOffset; + } + + endOfInput= count < in.capacity(); + total+= count; + in.flip(); + decoder.decode(in, dest, endOfInput); + fileOffset+= in.position(); + } + return fileOffset; + } + + + @Override + protected void rereadChunkData(long fileOffset, long fileEndOffset, CharBuffer dest) { + FileInputStream fis; + try { + fis = new FileInputStream(fFileName); + } catch (FileNotFoundException e1) { + // File has been deleted in the meantime + return; + } + try { + FileChannel channel = fis.getChannel(); + decode(channel, fileOffset, fileEndOffset, dest); + } catch (IOException e) { + // File cannot be read + } finally { + try { + fis.close(); + } catch (IOException e) { + } + } + } + + private void decode(FileChannel channel, long fileOffset, long fileEndOffset, CharBuffer dest) throws IOException { + final Charset charset = Charset.forName(fCharSet); + final CharsetDecoder decoder = charset.newDecoder().onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + + int needBytes = (int) (fileEndOffset-fileOffset); + final ByteBuffer in = ByteBuffer.allocate(needBytes); + + channel.position(fileOffset); + in.clear(); + channel.read(in); + in.flip(); + decoder.decode(in, dest, true); + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LazyCharArray.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LazyCharArray.java new file mode 100644 index 00000000000..30d73f6ef32 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LazyCharArray.java @@ -0,0 +1,198 @@ +/******************************************************************************* + * Copyright (c) 2010 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.lang.ref.SoftReference; +import java.nio.CharBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Implementation of char array for a file referencing content via + * soft references. + */ +public abstract class LazyCharArray extends AbstractCharArray { + private final static int CHUNK_BITS= 16; // 2^16 == 64K + protected final static int CHUNK_SIZE= 1 << CHUNK_BITS; + + /** + * Utility method to extract char[] out of CharBuffer. + */ + protected static char[] extractChars(CharBuffer charBuffer) { + if (charBuffer.hasArray() && charBuffer.arrayOffset() == 0) { + char[] buf = charBuffer.array(); + if (buf.length == charBuffer.remaining()) + return buf; + } + char[] buf = new char[charBuffer.remaining()]; + charBuffer.get(buf); + return buf; + } + + protected static class Chunk { + final int fDataLength; + final long fFileOffset; + final long fFileEndOffset; + private SoftReference fData; + + private Chunk(long fileOffset, long fileEndOffset, char[] data) { + fDataLength= data.length; + fFileOffset= fileOffset; + fFileEndOffset= fileEndOffset; + fData= new SoftReference(data); + } + } + + private int fLength= -1; + private List fChunks= new ArrayList(); + + protected LazyCharArray() { + } + + @Override + public final int tryGetLength() { + return fLength; + } + + @Override + public final int getLength() { + readUpTo(Integer.MAX_VALUE); + return fLength; + } + + @Override + public final boolean isValidOffset(int offset) { + if (offset < 0) + return false; + + readUpTo(offset); + if (fLength >= 0) + return offset < fLength; + + return offset < fChunks.size() << CHUNK_BITS; + } + + private void readUpTo(int offset) { + if (fLength >=0) + return; + + final int chunkOffset= offset >> CHUNK_BITS; + getChunkData(chunkOffset); + } + + @Override + public final char get(int offset) { + int chunkOffset= offset >> CHUNK_BITS; + char[] data= getChunkData(chunkOffset); + return data[offset & (CHUNK_SIZE-1)]; + } + + @Override + public final void arraycopy(int offset, char[] destination, int destinationPos, int length) { + int chunkOffset= offset >> CHUNK_BITS; + int loffset= offset & (CHUNK_SIZE-1); + char[] data= getChunkData(chunkOffset); + final int canCopy = data.length-loffset; + if (length <= canCopy) { + System.arraycopy(data, loffset, destination, destinationPos, length); + return; + } + System.arraycopy(data, loffset, destination, destinationPos, canCopy); + arraycopy(offset+canCopy, destination, destinationPos+canCopy, length-canCopy); + } + + private char[] getChunkData(int chunkOffset) { + Chunk chunk= getChunk(chunkOffset); + if (chunk != null) { + char[] data= chunk.fData.get(); + if (data != null) + return data; + + return loadChunkData(chunk); + } + return null; + } + + private Chunk getChunk(int chunkOffset) { + final int chunkCount = fChunks.size(); + if (chunkOffset < chunkCount) + return fChunks.get(chunkOffset); + + if (fLength >=0) + return null; + + return createChunk(chunkOffset); + } + + /** + * Called when a chunk is requested for the first time. There is no + * need to override this method. + */ + protected Chunk createChunk(int chunkOffset) { + final int chunkCount = fChunks.size(); + long fileOffset= chunkCount == 0 ? 0 : fChunks.get(chunkCount-1).fFileEndOffset; + try { + CharBuffer dest= CharBuffer.allocate(CHUNK_SIZE); + for (int i = chunkCount; i <= chunkOffset; i++) { + dest.clear(); + long fileEndOffset= readChunkData(fileOffset, dest); + dest.flip(); + final int charCount= dest.remaining(); + if (charCount == 0) { + fLength= fChunks.size() * CHUNK_SIZE; + break; + } + // New chunk + Chunk chunk= new Chunk(fileOffset, fileEndOffset, extractChars(dest)); + fChunks.add(chunk); + if (charCount < CHUNK_SIZE) { + fLength= (fChunks.size()-1) * CHUNK_SIZE + charCount; + break; + } + } + } catch (Exception e) { + // File cannot be read + return null; + } + + if (chunkOffset < fChunks.size()) + return fChunks.get(chunkOffset); + + return null; + } + + private char[] loadChunkData(Chunk chunk) { + CharBuffer dest= CharBuffer.allocate(chunk.fDataLength); + rereadChunkData(chunk.fFileOffset, chunk.fFileEndOffset, dest); + dest.flip(); + char[] result= extractChars(dest); + if (result.length != chunk.fDataLength) { + // In case the file changed + char[] copy= new char[chunk.fDataLength]; + System.arraycopy(result, 0, copy, 0, Math.min(result.length, copy.length)); + result= copy; + } + chunk.fData= new SoftReference(result); + return result; + } + + /** + * Read the chunk data at the given source offset and provide the end-offset in the + * source. + */ + protected abstract long readChunkData(long sourceOffset, CharBuffer dest) throws Exception; + + /** + * Read the chunk data at the given source range. In case the source range no longer (fully) exists, + * read as much as possible. + */ + protected abstract void rereadChunkData(long fileOffset, long fileEndOffset, CharBuffer dest); +}