mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-29 19:45:01 +02:00
Code Issues Wiki Activity

fix parsing international files

Browse source 
bug 70852, 70927
This commit is contained in:
Andrew Niefer 2004-08-13 19:52:32 +00:00
parent 4d63aafdaa
commit c2f1517d54
7 changed files with 54 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/org.eclipse.cdt.core/browser/org/eclipse/cdt/internal/core/browser/cache/TypeParser.java vendored
View file

@ -452,7 +452,7 @@ public class TypeParser implements ISourceElementRequestor {
try { try {
contents = file.getContents(); contents = file.getContents();
if (contents != null) if (contents != null)
reader = new CodeReader(resource.getLocation().toOSString(), contents); reader = new CodeReader(resource.getLocation().toOSString(), file.getCharset(), contents);
} catch (CoreException ex) { } catch (CoreException ex) {
ex.printStackTrace(); ex.printStackTrace();
} catch (IOException e) { } catch (IOException e) {

2
core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/SourceIndexer.java
View file

@ -103,7 +103,7 @@ public class SourceIndexer extends AbstractIndexer {
InputStream contents = null; InputStream contents = null;
try { try {
contents = resourceFile.getContents(); contents = resourceFile.getContents();
CodeReader reader = new CodeReader(resourceFile.getLocation().toOSString(), contents); CodeReader reader = new CodeReader(resourceFile.getLocation().toOSString(), resourceFile.getCharset(), contents);
parser = ParserFactory.createParser( parser = ParserFactory.createParser(
ParserFactory.createScanner(reader, scanInfo, ParserMode.COMPLETE_PARSE, language, requestor, ParserUtil.getScannerLogService(), null ), ParserFactory.createScanner(reader, scanInfo, ParserMode.COMPLETE_PARSE, language, requestor, ParserUtil.getScannerLogService(), null ),
requestor, ParserMode.COMPLETE_PARSE, language, ParserUtil.getParserLogService() ); requestor, ParserMode.COMPLETE_PARSE, language, ParserUtil.getParserLogService() );

38
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CodeReader.java
View file

@ -25,8 +25,8 @@ import org.eclipse.cdt.internal.core.parser.scanner2.CharArrayUtils;
* @author jcamelon * @author jcamelon
*/ */
public class CodeReader { public class CodeReader {
private static final String SYSTEM_DEFAULT_ENCODING = System.getProperty( "file.encoding" ); //$NON-NLS-1$
private static final String UTF_8 = "UTF-8"; //$NON-NLS-1$ //private static final String UTF_8 = "UTF-8"; //$NON-NLS-1$
private static final String NF = "<text>"; //$NON-NLS-1$ private static final String NF = "<text>"; //$NON-NLS-1$
private static final char [] NOFILE = NF.toCharArray(); //$NON-NLS-1$ private static final char [] NOFILE = NF.toCharArray(); //$NON-NLS-1$
@ -51,22 +51,31 @@ public class CodeReader {
FileInputStream stream = new FileInputStream(filename); FileInputStream stream = new FileInputStream(filename);
try { try {
buffer = load(stream); buffer = load(SYSTEM_DEFAULT_ENCODING, stream);
} finally { } finally {
stream.close(); stream.close();
} }
} }
public CodeReader(String filename, String charSet ) throws IOException
// If you have a handle on a stream to the file, e.g. IFile.getContents() {
public CodeReader(String filename, InputStream stream) throws IOException {
this.filename = filename.toCharArray(); this.filename = filename.toCharArray();
FileInputStream stream = new FileInputStream(filename);
try {
buffer = load(charSet, stream);
} finally {
stream.close();
}
}
public CodeReader( String fileName, String charSet, InputStream stream ) throws IOException {
filename = fileName.toCharArray();
FileInputStream fstream = FileInputStream fstream =
(stream instanceof FileInputStream) (stream instanceof FileInputStream)
? (FileInputStream)stream ? (FileInputStream)stream
: new FileInputStream(filename); : new FileInputStream(fileName);
try { try {
buffer = load(fstream); buffer = load(charSet, fstream);
} finally { } finally {
// If we create the FileInputStream we need close to it when done, // If we create the FileInputStream we need close to it when done,
// if not we figure the above layer will do it. // if not we figure the above layer will do it.
@ -76,21 +85,22 @@ public class CodeReader {
} }
} }
private char[] load(FileInputStream stream) throws IOException { private char[] load( String charSet, FileInputStream stream ) throws IOException {
FileChannel channel = stream.getChannel(); String encoding = Charset.isSupported( charSet ) ? charSet : SYSTEM_DEFAULT_ENCODING;
FileChannel channel = stream.getChannel();
ByteBuffer byteBuffer = ByteBuffer.allocateDirect((int)channel.size()); ByteBuffer byteBuffer = ByteBuffer.allocateDirect((int)channel.size());
channel.read(byteBuffer); channel.read(byteBuffer);
byteBuffer.rewind(); byteBuffer.rewind();
// TODO use the real encoding
CharBuffer charBuffer = Charset.forName(UTF_8).decode(byteBuffer); CharBuffer charBuffer = Charset.forName(encoding).decode(byteBuffer);
if (charBuffer.hasArray()) if (charBuffer.hasArray())
return charBuffer.array(); return charBuffer.array();
// Got to copy it out // Got to copy it out
char[] buff = new char[charBuffer.length()]; char[] buff = new char[charBuffer.length()];
charBuffer.get(buff); charBuffer.get(buff);
return buff; return buff;
} }
protected char[] xload(FileInputStream stream) throws IOException { protected char[] xload(FileInputStream stream) throws IOException {
@ -98,7 +108,7 @@ public class CodeReader {
MappedByteBuffer map = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()); MappedByteBuffer map = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
// TODO use the real encoding // TODO use the real encoding
CharBuffer charBuffer = Charset.forName(UTF_8).decode(map); CharBuffer charBuffer = Charset.forName(SYSTEM_DEFAULT_ENCODING).decode(map);
if (charBuffer.hasArray()) if (charBuffer.hasArray())
return charBuffer.array(); return charBuffer.array();

36
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java
View file

@ -732,6 +732,12 @@ public class Scanner2 implements IScanner, IScannerData {
return newToken(IToken.tCOMMA ); return newToken(IToken.tCOMMA );
default: default:
if( Character.isLetter( buffer[pos] ) ){
t = scanIdentifier();
if (t instanceof MacroExpansionToken)
continue;
return t;
}
// skip over anything we don't handle // skip over anything we don't handle
} }
} }
@ -773,7 +779,7 @@ public class Scanner2 implements IScanner, IScannerData {
while (++bufferPos[bufferStackPos] < limit) { while (++bufferPos[bufferStackPos] < limit) {
char c = buffer[bufferPos[bufferStackPos]]; char c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) { || c == '_' || (c >= '0' && c <= '9') || Character.isUnicodeIdentifierPart(c) ) {
++len; ++len;
continue; continue;
} }
@ -1346,7 +1352,7 @@ public class Scanner2 implements IScanner, IScannerData {
while (++bufferPos[bufferStackPos] < limit) { while (++bufferPos[bufferStackPos] < limit) {
c = buffer[bufferPos[bufferStackPos]]; c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) { || c == '_' || (c >= '0' && c <= '9') || Character.isUnicodeIdentifierPart(c)) {
++len; ++len;
continue; continue;
} }
@ -1481,7 +1487,7 @@ public class Scanner2 implements IScanner, IScannerData {
return; return;
char c = buffer[idstart]; char c = buffer[idstart];
if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_')) { if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Character.isUnicodeIdentifierPart(c))) {
handleProblem( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN, idstart, null ); handleProblem( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN, idstart, null );
skipToNewLine(); skipToNewLine();
return; return;
@ -1491,7 +1497,7 @@ public class Scanner2 implements IScanner, IScannerData {
while (++bufferPos[bufferStackPos] < limit) { while (++bufferPos[bufferStackPos] < limit) {
c = buffer[bufferPos[bufferStackPos]]; c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) { || c == '_' || (c >= '0' && c <= '9') || Character.isUnicodeIdentifierPart(c)) {
++idlen; ++idlen;
continue; continue;
} }
@ -1528,7 +1534,7 @@ public class Scanner2 implements IScanner, IScannerData {
bufferPos[bufferStackPos] += 2; bufferPos[bufferStackPos] += 2;
arglist[++currarg] = "...".toCharArray(); //$NON-NLS-1$ arglist[++currarg] = "...".toCharArray(); //$NON-NLS-1$
continue; continue;
} else if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { } else if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Character.isUnicodeIdentifierPart(c))) {
handleProblem( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN, idstart, name ); handleProblem( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN, idstart, name );
// yuck // yuck
skipToNewLine(); skipToNewLine();
@ -1655,7 +1661,7 @@ public class Scanner2 implements IScanner, IScannerData {
return; return;
char c = buffer[idstart]; char c = buffer[idstart];
if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_')) { if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Character.isUnicodeIdentifierPart(c))) {
skipToNewLine(); skipToNewLine();
return; return;
} }
@ -1664,7 +1670,7 @@ public class Scanner2 implements IScanner, IScannerData {
while (++bufferPos[bufferStackPos] < limit) { while (++bufferPos[bufferStackPos] < limit) {
c = buffer[bufferPos[bufferStackPos]]; c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) { || c == '_' || (c >= '0' && c <= '9' || Character.isUnicodeIdentifierPart(c))) {
++idlen; ++idlen;
continue; continue;
} }
@ -1700,7 +1706,7 @@ public class Scanner2 implements IScanner, IScannerData {
return; return;
char c = buffer[idstart]; char c = buffer[idstart];
if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_')) { if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Character.isUnicodeIdentifierPart(c))) {
skipToNewLine(); skipToNewLine();
return; return;
} }
@ -1709,7 +1715,7 @@ public class Scanner2 implements IScanner, IScannerData {
while (++bufferPos[bufferStackPos] < limit) { while (++bufferPos[bufferStackPos] < limit) {
c = buffer[bufferPos[bufferStackPos]]; c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) { || c == '_' || (c >= '0' && c <= '9' || Character.isUnicodeIdentifierPart(c))) {
++idlen; ++idlen;
continue; continue;
} }
@ -2048,7 +2054,7 @@ public class Scanner2 implements IScanner, IScannerData {
while (++bufferPos[bufferStackPos] < limit) { while (++bufferPos[bufferStackPos] < limit) {
char c = buffer[bufferPos[bufferStackPos]]; char c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) { || c == '_' || (c >= '0' && c <= '9') || Character.isUnicodeIdentifierPart(c)) {
continue; continue;
} }
break; break;
@ -2274,14 +2280,14 @@ public class Scanner2 implements IScanner, IScannerData {
while (++pos < limit) { while (++pos < limit) {
char c = expansion[pos]; char c = expansion[pos];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Character.isUnicodeIdentifierPart(c)) {
wsstart = -1; wsstart = -1;
int idstart = pos; int idstart = pos;
while (++pos < limit) { while (++pos < limit) {
c = expansion[pos]; c = expansion[pos];
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9') || c == '_')) { || (c >= '0' && c <= '9') || c == '_' || Character.isUnicodeIdentifierPart(c))) {
break; break;
} }
} }
@ -2482,11 +2488,11 @@ public class Scanner2 implements IScanner, IScannerData {
// grab the identifier // grab the identifier
c = expansion[pos]; c = expansion[pos];
int idstart = pos; int idstart = pos;
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'X') || c == '_') { if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'X') || c == '_' || Character.isUnicodeIdentifierPart(c)) {
while (++pos < limit) { while (++pos < limit) {
c = expansion[pos]; c = expansion[pos];
if( !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'X') if( !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'X')
|| (c >= '0' && c <= '9') || c == '_') ) || (c >= '0' && c <= '9') || c == '_' || Character.isUnicodeIdentifierPart(c)) )
break; break;
} }
} // else TODO something } // else TODO something
@ -3092,7 +3098,7 @@ public class Scanner2 implements IScanner, IScannerData {
{ {
char c = prefix[i]; char c = prefix[i];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) || c == '_' || (c >= '0' && c <= '9') || Character.isUnicodeIdentifierPart(c) )
continue; continue;
handleInvalidCompletion(); handleInvalidCompletion();
} }

2
core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/matching/MatchLocator.java
View file

@ -507,7 +507,7 @@ public class MatchLocator implements IMatchLocator{
if (currentResource.isAccessible() && currentResource instanceof IFile) { if (currentResource.isAccessible() && currentResource instanceof IFile) {
IFile file = (IFile) currentResource; IFile file = (IFile) currentResource;
contents = file.getContents(); contents = file.getContents();
reader = new CodeReader(currentResource.getLocation().toOSString(), contents); reader = new CodeReader(currentResource.getLocation().toOSString(), file.getCharset(), contents);
realPath = currentResource.getLocation(); realPath = currentResource.getLocation();
project = file.getProject(); project = file.getProject();
} else { } else {

2
core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/parser/ParserUtil.java
View file

@ -70,7 +70,7 @@ public class ParserUtil
try try
{ {
in = ((IFile)resultingResource).getContents(); in = ((IFile)resultingResource).getContents();
return new CodeReader(finalPath, in); return new CodeReader(finalPath, ((IFile)resultingResource).getCharset(), in);
} finally { } finally {
if (in != null) if (in != null)
{ {

7
core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/search/actions/SelectionParseAction.java
View file

@ -32,6 +32,7 @@ import org.eclipse.cdt.internal.ui.search.CSearchMessages;
import org.eclipse.cdt.ui.CUIPlugin; import org.eclipse.cdt.ui.CUIPlugin;
import org.eclipse.core.resources.IFile; import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject; import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.jface.action.Action; import org.eclipse.jface.action.Action;
import org.eclipse.jface.action.IStatusLineManager; import org.eclipse.jface.action.IStatusLineManager;
import org.eclipse.jface.text.BadLocationException; import org.eclipse.jface.text.BadLocationException;
@ -101,12 +102,14 @@ public class SelectionParseAction extends Action {
CodeReader reader = null; CodeReader reader = null;
try { try {
if( workingCopy == null ) if( workingCopy == null )
reader = new CodeReader(resourceFile.getLocation().toOSString()); reader = new CodeReader(resourceFile.getLocation().toOSString(), resourceFile.getCharset() );
else else
reader = new CodeReader(resourceFile.getLocation().toOSString(), workingCopy.getContents()); reader = new CodeReader(resourceFile.getLocation().toOSString(), workingCopy.getContents());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} } catch ( CoreException e ) {
e.printStackTrace();
}
try try
{ {