+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + *
+ * + * @since 4.0 + */ +public interface ILanguageMapper { + + /** + * Returns the language of a file. + * @param file - path of the file + * @return the ILanguage of the file + */ + ILanguage getLanguage(String file); +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexer.java new file mode 100644 index 00000000000..ea19e4c5b04 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexer.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2006, 2007 QNX Software Systems and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * QNX - Initial API and implementation + * Markus Schorn (Wind River Systems) + * IBM Corporation + *******************************************************************************/ + +package org.eclipse.cdt.internal.core.indexer; + +import java.io.File; +import java.util.List; +import java.util.Map; + +import org.eclipse.cdt.core.index.IIndexLocationConverter; +import org.eclipse.cdt.core.parser.IParserLogService; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.internal.core.index.IIndexFragment; +import org.eclipse.cdt.internal.core.index.IWritableIndexFragment; +import org.eclipse.cdt.internal.core.index.WritableCIndex; +import org.eclipse.cdt.internal.core.pdom.WritablePDOM; +import org.eclipse.core.runtime.CoreException; + +/** + * A standalone tool for populating an index. This indexer optimizes for + * speed at the expense of accuracy. + * + *+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + *
+ * + * @since 4.0 + */ +public class StandaloneFastIndexer extends StandaloneIndexer{ + + /** + * Construct a fast standalone indexer. + * @param writableIndexFile - the file where the PDOM index is stored + * @param converter - a converter used to convert between String locations and IIndexLocations + * @param linkageFactoryMappings - all of the available IPDOMLinkageFactories the index can use during indexing + * @param scanner - provides include paths and defined symbols + * @param mapper - a mapper used to determine ICLanguage for a particular file + * @param log - logger + * @throws CoreException + */ + public StandaloneFastIndexer(File writableIndexFile, IIndexLocationConverter converter, Map linkageFactoryMappings, + IScannerInfo scanner, ILanguageMapper mapper, IParserLogService log) throws CoreException { + WritablePDOM pdom = new WritablePDOM(writableIndexFile, converter, linkageFactoryMappings); + fIndex = new WritableCIndex( + new IWritableIndexFragment[] { pdom }, + new IIndexFragment[0]); + fIndexAllFiles = false; + fScanner = scanner; + fMapper = mapper; + fLog = log; + } + + /** + * Create a delegate standalone indexing task + */ + protected StandaloneIndexerTask createTask(List added, List changed, List removed) { + return new StandaloneFastIndexerTask(this, added, changed, removed); + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexerTask.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexerTask.java new file mode 100644 index 00000000000..248fe1047b1 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexerTask.java @@ -0,0 +1,209 @@ +/******************************************************************************* + * Copyright (c) 2006, 2007 QNX Software Systems and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * QNX - Initial API and implementation + * Markus Schorn (Wind River Systems) + * IBM Corporation + *******************************************************************************/ + +package org.eclipse.cdt.internal.core.indexer; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; +import org.eclipse.cdt.core.index.IIndexFile; +import org.eclipse.cdt.core.index.IIndexFileLocation; +import org.eclipse.cdt.core.model.AbstractLanguage; +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.internal.core.index.IWritableIndex; +import org.eclipse.cdt.internal.core.index.IndexFileLocation; +import org.eclipse.cdt.internal.core.index.IndexBasedCodeReaderFactory.CallbackHandler; +import org.eclipse.cdt.internal.core.index.IndexBasedCodeReaderFactory.IndexFileInfo; +import org.eclipse.core.filesystem.URIUtil; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IProgressMonitor; + +/** + * A task for index updates. + * + *+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + *
+ * + * @since 4.0 + */ +public class StandaloneFastIndexerTask extends StandaloneIndexerTask implements CallbackHandler { + private List fChanged = new ArrayList(); + private List fRemoved = new ArrayList(); + private IWritableIndex fIndex; + private StandaloneIndexBasedCodeReaderFactory fCodeReaderFactory; + private Map fIflCache; + private int fCurrentConfigHash= 0; + + public StandaloneFastIndexerTask(StandaloneFastIndexer indexer, List added, + List changed, List removed) { + super(indexer); + fChanged.addAll(added); + fChanged.addAll(changed); + fRemoved.addAll(removed); + updateInfo(0, 0, fChanged.size() + fRemoved.size()); + } + + public void run(IProgressMonitor monitor) throws IOException{ + long start = System.currentTimeMillis(); + try { + setupIndexAndReaderFactory(); + fIndex.acquireReadLock(); + try { + registerTUsInReaderFactory(fChanged); + + Iterator i= fRemoved.iterator(); + while (i.hasNext()) { + if (monitor.isCanceled()) + return; + String tu = (String) i.next(); + removeTU(fIndex, getIndexFileLocation(tu), 1); + if (isValidSourceUnitName(tu)) { + updateInfo(1, 0, 0); + } + else { + updateInfo(0, 1, -1); + } + } + + // separate headers + List headers= new ArrayList(); + List sources= fChanged; + for (Iterator iter = fChanged.iterator(); iter.hasNext();) { + String tu = (String) iter.next(); + if (!isValidSourceUnitName(tu)) { + headers.add(tu); + iter.remove(); + } + } + + parseTUs(fIndex, 1, sources, headers, monitor); + if (monitor.isCanceled()) { + return; + } + } + finally { + fIndex.releaseReadLock(); + } + } catch (CoreException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + } + traceEnd(start); + } + + private void setupIndexAndReaderFactory() throws CoreException { + fIndex= fIndexer.getIndex(); + fIndex.resetCacheCounters(); + fIflCache = new HashMap/*+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + *
+ * + * @since 4.0 + */ +public class StandaloneFullIndexer extends StandaloneIndexer{ + + private ICodeReaderFactory fCodeReaderFactory; + + /** + * Create a full indexer. + * @param writableIndexFile - the file where the PDOM index is stored + * @param converter - a converter used to convert between String locations and IIndexLocations + * @param linkageFactoryMappings - all of the available IPDOMLinkageFactories the index can use during indexing + * @param scanner - provides include paths and defined symbols + * @param mapper - a mapper used to determine ICLanguage for a particular file + * @param log - logger + * @param codeReaderFactory - factory that provides CodeReaders for files included + * by the source code being parsed. + * @throws CoreException + */ + public StandaloneFullIndexer(File writableIndexFile, IIndexLocationConverter converter, Map linkageFactoryMappings, + IScannerInfo scanner, ILanguageMapper mapper, IParserLogService log, ICodeReaderFactory codeReaderFactory) throws CoreException { + WritablePDOM pdom = new WritablePDOM(writableIndexFile, converter, linkageFactoryMappings); + fIndex = new WritableCIndex( + new IWritableIndexFragment[] { pdom }, + new IIndexFragment[0]); + fIndexAllFiles = false; + fScanner = scanner; + fMapper = mapper; + fCodeReaderFactory = codeReaderFactory; + fLog = log; + } + + /** + * Returns the factory that provides CodeReaders for files included + * by the source code being parsed. + * @return + */ + public ICodeReaderFactory getCodeReaderFactory() { + return fCodeReaderFactory; + } + + /** + * Creates a delegate standalone indexing task + */ + protected StandaloneIndexerTask createTask(List added, List changed, List removed) { + return new StandaloneFullIndexerTask(this, added, changed, removed); + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexerTask.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexerTask.java new file mode 100644 index 00000000000..7f34773363b --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexerTask.java @@ -0,0 +1,179 @@ +/******************************************************************************* + * Copyright (c) 2006, 2007 QNX Software Systems and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * QNX - Initial API and implementation + * Markus Schorn (Wind River Systems) + * IBM Corporation + *******************************************************************************/ + +package org.eclipse.cdt.internal.core.indexer; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; +import org.eclipse.cdt.core.index.IIndexFile; +import org.eclipse.cdt.core.index.IIndexFileLocation; +import org.eclipse.cdt.core.model.AbstractLanguage; +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.internal.core.index.IWritableIndex; +import org.eclipse.cdt.internal.core.index.IndexFileLocation; +import org.eclipse.core.filesystem.URIUtil; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IProgressMonitor; + +/** + * A task for index updates. + * + *+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + *
+ * + * @since 4.0 + */ +public class StandaloneFullIndexerTask extends StandaloneIndexerTask { + private final static Object REQUIRED= new Object(); + private final static Object MISSING = new Object(); + private final static Object SKIP= new Object(); + + private List fChanged = new ArrayList(); + private List fRemoved = new ArrayList(); + private IWritableIndex fIndex = null; + private Map filePathsToParse = new HashMap/*+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + *
+ * + * @since 4.0 + */ +public class StandaloneIndexBasedCodeReaderFactory extends IndexBasedCodeReaderFactory { + + public static class DefaultFallBackFactory implements ICodeReaderFactory { + + public CodeReader createCodeReaderForInclusion(IMacroCollector callback, String path) { + try { + if (!new File(path).exists()) + return null; + return new CodeReader(path); + } catch (IOException e) { + return null; + } + } + + public CodeReader createCodeReaderForTranslationUnit(String path) { + try { + if (!new File(path).exists()) + return null; + return new CodeReader(path); + } catch (IOException e) { + return null; + } + } + + public ICodeReaderCache getCodeReaderCache() { + return null; + } + + public int getUniqueIdentifier() { + return 0; + } + + } + + public StandaloneIndexBasedCodeReaderFactory(IIndex index) { + super(index); + } + + public StandaloneIndexBasedCodeReaderFactory(IIndex index, ICodeReaderFactory fallbackFactory) { + super(index, new HashMap/*+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + * + * This class is not thread safe. + *
+ * + * @since 4.0 + */ +public abstract class StandaloneIndexer { + + /** + * Parser should not skip any references. + */ + public static final int SKIP_NO_REFERENCES= 0; + + /** + * Parser to skip all references. + */ + public static final int SKIP_ALL_REFERENCES= 1; + + /** + * Parser to skp type references. + */ + public static final int SKIP_TYPE_REFERENCES= 2; + + /** + * Constant for indicating to update all translation units. + */ + public final static int UPDATE_ALL= 0x1; + + /** + * Constant for indicating to update translation units if their timestamp + * has changed. + */ + public final static int UPDATE_CHECK_TIMESTAMPS= 0x2; + + /** + * Empty list. + */ + protected static final List NO_TUS = new ArrayList(); + /** + * The IWritableIndex that stores all bindings and names. + */ + protected IWritableIndex fIndex; + + /** + * A flag that indiciates if all files (sources without config, headers not included) + * should be parsed. + */ + protected boolean fIndexAllFiles; + + /** + * Collection of valid file extensions for C/C++ source. + */ + protected Set fValidSourceUnitNames; + + /** + * The IScannerInfo that provides include paths and defined symbols. + */ + protected IScannerInfo fScanner; + + /** + * The ILanguageMapper that determines the ILanguage for a file. + */ + protected ILanguageMapper fMapper; + + /** + * The logger during parsing. + */ + protected IParserLogService fLog; + + /** + * A flag that indicates if all activities during indexing should be shown. + */ + protected boolean fShowActivity; + + /** + * A flag that indicates if any problems encountered during indexing. + * should be shown. + */ + protected boolean fShowProblems; + + /** + * A flag that indicates if statistics should be gathered during indexing. + */ + protected boolean fTraceStatistics; + + /** + * The type of references the parser should skip. + */ + protected int fSkipReferences = SKIP_NO_REFERENCES; + + /** + * The exclusion filter that skips over files that should not be indexed. + */ + protected FilenameFilter fExclusionFilter; + + /** + * Files to parse up front. + */ + protected String[] fFilesToParseUpFront = new String[0]; + + protected int fUpdateOptions = UPDATE_ALL; + + private IndexerProgress fProgress = null; + private volatile StandaloneIndexerTask fDelegate; + + private static FilenameFilter DEFAULT_FILTER = new FilenameFilter() { + public boolean accept(File dir, String name) { + return true; + } + }; + + /** + * Returns the index. + * @return the IWritable index the indexer is writing to + */ + public IWritableIndex getIndex() { + return fIndex; + } + + /** + * Returns true if all files (sources without config, headers not included) + * should be parsed. Otherwise, this method returns false. + */ + public boolean getIndexAllFiles() { + return fIndexAllFiles; + } + + /** + * Returns the collection of valid file extensions for C/C++ source. + * @return + */ + public Set getValidSourceUnitNames() { + return fValidSourceUnitNames; + } + + /** + * Sets the collection of valid file extensions for C/C++ source. + */ + public void setValidSourceUnitNames(Set validSourceUnitNames) { + fValidSourceUnitNames = validSourceUnitNames; + } + + /** + * Returns the IScannerInfo that provides include paths and defined symbols. + * @return + */ + public IScannerInfo getScannerInfo() { + return fScanner; + } + + /** + * Returns the ILanguageMapper that determines the ILanguage for a file. + * @return + */ + public ILanguageMapper getLanguageMapper() { + return fMapper; + } + + /** + * Returns the logger. + * @return + */ + public IParserLogService getParserLog() { + return fLog; + } + + /** + * Returns true if indexing activities should be shown. + * Otherwise, this method returns false. + * @return + */ + public boolean getShowActivity() { + return fShowActivity; + } + + /** + * Tells indexer if indexing activities should be shown. + */ + public void setShowActivity(boolean showActivity) { + fShowActivity = showActivity; + } + + /** + * Returns true if problems during indexing should be shown. + * Otherwise, this method returns false. + * @return + */ + public boolean getShowProblems() { + return fShowProblems; + } + + /** + * Tells indexer if problems during indexing should be shown. + */ + public void setShowProblems(boolean showProblems) { + fShowProblems = showProblems; + } + + /** + * Returns true if statistics should be gathered during indexing. + * Otherwise, this method returns false.. + * @return + */ + public boolean getTraceStatistics() { + return fTraceStatistics; + } + + /** + * Tells indexer if statistics should be gathered during indexing. + */ + public void setTraceStatistics(boolean traceStatistics) { + fTraceStatistics = traceStatistics; + } + + private IndexerProgress createProgress() { + IndexerProgress progress= new IndexerProgress(); + progress.fTimeEstimate= 1000; + return progress; + } + + private void clearIndex() throws CoreException, InterruptedException { + IWritableIndex index= getIndex(); + // First clear the pdom + index.acquireWriteLock(0); + try { + index.clear(); + } + finally { + index.releaseWriteLock(0); + } + } + + /** + * Returns the progress information. + * @return + */ + public synchronized IndexerProgress getProgressInformation() { + return fDelegate != null ? fDelegate.getProgressInformation() : fProgress; + } + + /** + * Returns the update options specified. + * @return + */ + public int getUpdateOptions() { + return fUpdateOptions; + } + + /** + * Specifies the update options, whether all translation units should be updated or only the ones + * with timestamp changes. + * @param options + */ + public void setUpdateOptions (int options) { + fUpdateOptions = options; + } + + /** + * Clears the index and rebuild + * @param tus - directories/files to be added to index + * @param monitor + * @throws IOException + */ + public void rebuild(List tus, IProgressMonitor monitor) throws IOException { + fProgress = createProgress(); + + try { + clearIndex(); + fDelegate= createTask(getFilesAdded(tus), NO_TUS, NO_TUS); + fDelegate.setUpdateFlags(fUpdateOptions); + + } catch (CoreException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + } + + if (fDelegate != null) { + fDelegate.run(monitor); + } + + } + + /** + * Updates the index with changes. + * @param added - directories/files to be added to the index + * @param changed - files that have been changed + * @param removed - files to be removed from the index + * @param monitor + * @throws IOException + */ + public void handleDelta(List added, List changed, List removed, IProgressMonitor monitor) throws IOException { + fProgress= new IndexerProgress(); + + fDelegate= createTask(getFilesAdded(added), changed, removed); + if (fDelegate instanceof StandaloneIndexerTask) { + fDelegate.setUpdateFlags(fUpdateOptions); + } + + if (fDelegate != null) { + fDelegate.run(monitor); + } + + } + + /** + * Returns files that are being added to the index, skipping over files that + * should not be excluded. + * @param tus + * @return + */ + private List getFilesAdded(List tus) { + List added = new ArrayList(); + + FilenameFilter filter = getExclusionFilter(); + if (filter == null) { + filter = DEFAULT_FILTER; + } + + Iterator iter = tus.iterator(); + while (iter.hasNext()) { + String path = (String) iter.next(); + File file = new File(path); + if (file.isDirectory()) { + String[] files = file.list(filter); + for (int i = 0; i < files.length; i++) { + added.add((String)files[i]); + } + } + else { + if (filter.accept(file.getParentFile(), file.getName())) { + added.add(path); + } + } + } + return added; + } + + /** + * Creates a delegate standalone indexing task + */ + protected abstract StandaloneIndexerTask createTask(List added, List changed, List removed); + + /** + * Return the type of references the parser should skip. + * @return + */ + public int getSkipReferences() { + return fSkipReferences; + } + + /** + * Sets the type of references the parser should skip. + * @param skipReferences + */ + public void setSkipReferences(int skipReferences) { + fSkipReferences = skipReferences; + } + + /** + * Returns an array of files that should be parsed up front. + * @return + */ + public String[] getFilesToParseUpFront() { + return fFilesToParseUpFront; + } + + /** + * Sets an array of files that should be parsed up front. + * @param filesToParseUpFront + */ + public void setFilesToParseUpFront(String[] filesToParseUpFront) { + fFilesToParseUpFront = filesToParseUpFront; + } + + /** + * Returns the exclusion filter for this indexer. + * @return + */ + public FilenameFilter getExclusionFilter() { + return fExclusionFilter; + } + + /** + * Sets the exclusion filter that tells the indexer to skip over + * files that should not be indexed. + * @param exclusionFilter + */ + public void setExclusionFilter(FilenameFilter exclusionFilter) { + fExclusionFilter = exclusionFilter; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexerTask.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexerTask.java new file mode 100644 index 00000000000..d5c92ff089c --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexerTask.java @@ -0,0 +1,499 @@ +/******************************************************************************* + * Copyright (c) 2006, 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + * IBM Corporation + *******************************************************************************/ + +package org.eclipse.cdt.internal.core.indexer; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.eclipse.cdt.core.CCorePlugin; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; +import org.eclipse.cdt.core.index.IIndex; +import org.eclipse.cdt.core.index.IIndexFile; +import org.eclipse.cdt.core.index.IIndexFileLocation; +import org.eclipse.cdt.core.index.IIndexInclude; +import org.eclipse.cdt.core.model.AbstractLanguage; +import org.eclipse.cdt.core.model.ILanguage; +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.IExtendedScannerInfo; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.internal.core.index.IIndexFragmentFile; +import org.eclipse.cdt.internal.core.index.IWritableIndex; +import org.eclipse.cdt.internal.core.index.IndexFileLocation; +import org.eclipse.cdt.internal.core.pdom.IndexerProgress; +import org.eclipse.cdt.internal.core.pdom.PDOMWriter; +import org.eclipse.core.filesystem.URIUtil; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IPath; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.Path; + +/** + * A task for index updates. + * + *+ * EXPERIMENTAL. This class or interface has been added as + * part of a work in progress. There is no guarantee that this API will work or + * that it will remain the same. Please do not use this API without consulting + * with the CDT team. + *
+ * + * @since 4.0 + */ +public abstract class StandaloneIndexerTask extends PDOMWriter{ + private static final Object NO_CONTEXT = new Object(); + + protected StandaloneIndexer fIndexer; + protected Map/*null.
+ * @see #parseTUs(IWritableIndex, int, Collection, Collection, IProgressMonitor)
+ * @since 4.0
+ */
+ abstract protected IASTTranslationUnit createAST(AbstractLanguage lang, CodeReader codeReader, IScannerInfo scanInfo, int options, IProgressMonitor pm) throws CoreException;
+
+ /**
+ * Convenience method for subclasses, parses the files calling out to the methods
+ * {@link #createAST(IIndexFileLocation, IProgressMonitor)},
+ * {@link #needToUpdate(IIndexFileLocation)},
+ * {@link #addSymbols(IASTTranslationUnit, IWritableIndex, int, IProgressMonitor)}
+ * {@link #postAddToIndex(IIndexFileLocation, IIndexFile)} and
+ * {@link #findLocation(String)}
+ * @since 4.0
+ */
+ protected void parseTUs(IWritableIndex index, int readlockCount, Collection sources, Collection headers, IProgressMonitor monitor) throws IOException, CoreException, InterruptedException {
+ int options= 0;
+ if (fIndexer.getSkipReferences() == StandaloneIndexer.SKIP_ALL_REFERENCES) {
+ options |= AbstractLanguage.OPTION_SKIP_FUNCTION_BODIES;
+ }
+ for (Iterator iter = fFilesUpFront.iterator(); iter.hasNext();) {
+ String upfront= (String) iter.next();
+ parseUpFront(upfront, options, index, readlockCount, monitor);
+ }
+
+ // sources first
+ for (Iterator iter = sources.iterator(); iter.hasNext();) {
+ if (monitor.isCanceled())
+ return;
+ String sourcePath = (String) iter.next();
+ String path = new File(sourcePath).getCanonicalPath();
+ final IIndexFileLocation ifl = getIndexFileLocation(path);
+
+ if (needToUpdate(ifl, 0)) {
+ parseTU(ifl, options, index, readlockCount, monitor);
+ }
+ }
+
+ // headers with context
+ for (Iterator iter = headers.iterator(); iter.hasNext();) {
+ if (monitor.isCanceled())
+ return;
+ String sourcePath = (String) iter.next();
+ String path = new File(sourcePath).getCanonicalPath();
+ final IIndexFileLocation location = getIndexFileLocation(path);
+
+ if (!needToUpdate(location, 0)) {
+ iter.remove();
+ }
+ else {
+ IIndexFileLocation context= findContext(index, location);
+ if (context != null) {
+ parseTU(context, options, index, readlockCount, monitor);
+ }
+ }
+ }
+
+ // headers without context
+ if (getIndexAllFiles()) {
+ for (Iterator iter = headers.iterator(); iter.hasNext();) {
+ if (monitor.isCanceled())
+ return;
+
+ String sourcePath = (String) iter.next();
+ String path = new File(sourcePath).getCanonicalPath();
+ final IIndexFileLocation ifl = getIndexFileLocation(path);
+
+ if (!needToUpdate(ifl, 0)) {
+ iter.remove();
+ }
+ else {
+ parseTU(ifl, options, index, readlockCount, monitor);
+ }
+ }
+ }
+ }
+
+ final protected boolean isOutdated(IIndexFileLocation ifl, IIndexFile indexFile) throws CoreException {
+ if (indexFile == null) {
+ return true;
+ }
+ File res = new File(ifl.getFullPath());
+ if (res != null) {
+ if (indexFile != null) {
+ if (res.lastModified() == indexFile.getTimestamp()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+
+ private void parseTU(IIndexFileLocation location, int options, IWritableIndex index, int readlockCount, IProgressMonitor pm) throws IOException, CoreException, InterruptedException {
+ String path = location.getFullPath();
+
+ try {
+ // skip if no scanner info
+ IScannerInfo scanner= fIndexer.getScannerInfo();
+ if (scanner == null) {
+ updateInfo(0, 0, -1);
+ }
+ else {
+ final int configHash = computeHashCode(scanner);
+ if (needToUpdate(location, configHash)) {
+ if (fShowActivity) {
+ System.out.println("Indexer: parsing " + path); //$NON-NLS-1$
+ }
+ long start= System.currentTimeMillis();
+ IASTTranslationUnit ast= createAST(location, scanner, options, pm);
+ fStatistics.fParsingTime += System.currentTimeMillis()-start;
+ if (ast != null) {
+ addSymbols(ast, index, readlockCount, configHash, pm);
+ }
+ }
+ }
+ }
+ catch (CoreException e) {
+ e.printStackTrace();
+ }
+ catch (RuntimeException e) {
+ e.printStackTrace();
+ }
+ catch (Error e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void parseUpFront(String file, int options, IWritableIndex index, int readlockCount, IProgressMonitor pm) throws CoreException, InterruptedException {
+ file= file.trim();
+ if (file.length() == 0) {
+ return;
+ }
+ try {
+ if (fShowActivity) {
+ System.out.println("Indexer: parsing " + file + " up front"); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+
+ long start= System.currentTimeMillis();
+
+ IASTTranslationUnit ast= null;
+ ILanguage l = fIndexer.getLanguageMapper().getLanguage(file);
+ if (l instanceof AbstractLanguage) {
+ AbstractLanguage lang= (AbstractLanguage) l;
+ IScannerInfo scanInfo = fIndexer.getScannerInfo();
+ String code= "#include \"" + file + "\"\n"; //$NON-NLS-1$ //$NON-NLS-2$
+ if (fDummyFileName == null) {
+ fDummyFileName= file + "___"; //$NON-NLS-1$
+ fDummyFileURI= findLocation(fDummyFileName).getURI();
+ }
+ CodeReader codeReader= new CodeReader(fDummyFileName, code.toCharArray());
+ ast= createAST(lang, codeReader, scanInfo, options, pm);
+ }
+ fStatistics.fParsingTime += System.currentTimeMillis()-start;
+ if (ast != null) {
+ addSymbols(ast, index, readlockCount, 0, pm);
+ updateInfo(-1, +1, 0);
+ }
+ }
+ catch (CoreException e) {
+ e.printStackTrace();
+ }
+ catch (RuntimeException e) {
+ e.printStackTrace();
+ }
+ catch (Error e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Overriders must call super.needToUpdate(). If false is returned
+ * this must be passed on to their caller:
+ *
+ * if (super.needToUpdate()) {
+ * // your code
+ * }
+ * return false;
+ */
+ protected boolean needToUpdate(IIndexFileLocation fileLoc, int configHash) throws CoreException {
+ return fDummyFileURI==null || !fDummyFileURI.equals(fileLoc.getURI());
+ }
+
+ private IIndexFileLocation findContext(IIndex index, IIndexFileLocation location) {
+ Object cachedContext= fContextMap.get(location);
+ if (cachedContext != null) {
+ return cachedContext == NO_CONTEXT ? null : (IIndexFileLocation) cachedContext;
+ }
+ IIndexFileLocation context= null;
+ fContextMap.put(location, NO_CONTEXT); // prevent recursion
+ IIndexFile pdomFile;
+ try {
+ pdomFile = index.getFile(location);
+ if (pdomFile != null) {
+ IIndexInclude[] includedBy = index.findIncludedBy(pdomFile, IIndex.DEPTH_ZERO);
+ ArrayList/**/ paths= new ArrayList/**/(includedBy.length);
+ for (int i = 0; i < includedBy.length; i++) {
+ IIndexInclude include = includedBy[i];
+ IIndexFileLocation incLocation = include.getIncludedByLocation();
+ if (isValidSourceUnitName(incLocation.getFullPath())) {
+ context = incLocation;
+ if (context != null) {
+ fContextMap.put(location, context);
+ return context;
+ }
+ }
+ paths.add(incLocation);
+ }
+ for (Iterator/**/ iter = paths.iterator(); iter.hasNext();) {
+ IIndexFileLocation nextLevel = (IIndexFileLocation) iter.next();
+ context = findContext(index, nextLevel);
+ if (context != null) {
+ fContextMap.put(location, context);
+ return context;
+ }
+ }
+ }
+ } catch (CoreException e) {
+ CCorePlugin.log(e);
+ }
+ return null;
+ }
+
+ /**
+ * Conveninence method for subclasses, removes a translation unit from the index.
+ * @since 4.0
+ */
+ protected void removeTU(IWritableIndex index, IIndexFileLocation ifl, int readlocks) throws CoreException, InterruptedException {
+ index.acquireWriteLock(readlocks);
+ try {
+ IIndexFragmentFile file = (IIndexFragmentFile) index.getFile(ifl);
+ if (file != null)
+ index.clearFile(file, null);
+ } finally {
+ index.releaseWriteLock(readlocks);
+ }
+ }
+
+ protected void traceEnd(long start) {
+ if (fIndexer.getTraceStatistics()) {
+ IndexerProgress info= getProgressInformation();
+ String name= getClass().getName();
+ name= name.substring(name.lastIndexOf('.')+1);
+
+ System.out.println(name + " " //$NON-NLS-1$
+ + " (" + info.fCompletedSources + " sources, " //$NON-NLS-1$ //$NON-NLS-2$
+ + info.fCompletedHeaders + " headers)"); //$NON-NLS-1$
+
+ boolean allFiles= getIndexAllFiles();
+ boolean skipRefs= fIndexer.getSkipReferences() == StandaloneIndexer.SKIP_ALL_REFERENCES;
+ boolean skipTypeRefs= skipRefs || fIndexer.getSkipReferences() == StandaloneIndexer.SKIP_TYPE_REFERENCES;
+ System.out.println(name + " Options: " //$NON-NLS-1$
+ + "parseAllFiles=" + allFiles //$NON-NLS-1$
+ + ",skipReferences=" + skipRefs //$NON-NLS-1$
+ + ", skipTypeReferences=" + skipTypeRefs //$NON-NLS-1$
+ + "."); //$NON-NLS-1$
+
+ System.out.println(name + " Timings: " //$NON-NLS-1$
+ + (System.currentTimeMillis() - start) + " total, " //$NON-NLS-1$
+ + fStatistics.fParsingTime + " parser, " //$NON-NLS-1$
+ + fStatistics.fResolutionTime + " resolution, " //$NON-NLS-1$
+ + fStatistics.fAddToIndexTime + " index update."); //$NON-NLS-1$
+ int sum= fStatistics.fDeclarationCount+fStatistics.fReferenceCount+fStatistics.fProblemBindingCount;
+ double problemPct= sum==0 ? 0.0 : (double) fStatistics.fProblemBindingCount / (double) sum;
+ NumberFormat nf= NumberFormat.getPercentInstance();
+ nf.setMaximumFractionDigits(2);
+ nf.setMinimumFractionDigits(2);
+ System.out.println(name + " Result: " //$NON-NLS-1$
+ + fStatistics.fDeclarationCount + " declarations, " //$NON-NLS-1$
+ + fStatistics.fReferenceCount + " references, " //$NON-NLS-1$
+ + fStatistics.fErrorCount + " errors, " //$NON-NLS-1$
+ + fStatistics.fProblemBindingCount + "(" + nf.format(problemPct) + ") problems."); //$NON-NLS-1$ //$NON-NLS-2$
+
+ IWritableIndex index = fIndexer.getIndex();
+ if (index != null) {
+ long misses= index.getCacheMisses();
+ long hits= index.getCacheHits();
+ long tries= misses+hits;
+ double missPct= tries==0 ? 0.0 : (double) misses / (double) tries;
+ System.out.println(name + " Cache: " //$NON-NLS-1$
+ + hits + " hits, " //$NON-NLS-1$
+ + misses + "(" + nf.format(missPct)+ ") misses."); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ }
+ }
+
+ public abstract void run(IProgressMonitor monitor) throws IOException;
+
+ protected IIndexFileLocation getIndexFileLocation(String path) {
+ String absolutePath = new File(path).getAbsolutePath();
+ //Standalone indexing stores the absolute paths of files being indexed
+ return new IndexFileLocation(URIUtil.toURI(absolutePath),absolutePath);
+ }
+
+ protected boolean isValidSourceUnitName(String filename) {
+ IPath path = new Path(filename);
+ if (fIndexer.getValidSourceUnitNames() == null || fIndexer.getValidSourceUnitNames().size() == 0)
+ return true;
+ return fIndexer.getValidSourceUnitNames().contains(path.getFileExtension());
+ }
+
+ protected long getLastModified(IIndexFileLocation location) {
+ return new File(location.getFullPath()).lastModified();
+ }
+
+ protected static int computeHashCode(IScannerInfo scannerInfo) {
+ int result= 0;
+ Map macros= scannerInfo.getDefinedSymbols();
+ if (macros != null) {
+ for (Iterator i = macros.entrySet().iterator(); i.hasNext();) {
+ Map.Entry entry = (Map.Entry) i.next();
+ String key = (String) entry.getKey();
+ String value = (String) entry.getValue();
+ result= addToHashcode(result, key);
+ if (value != null && value.length() > 0) {
+ result= addToHashcode(result, value);
+ }
+ }
+ }
+ String[] a= scannerInfo.getIncludePaths();
+ if (a != null) {
+ for (int i = 0; i < a.length; i++) {
+ result= addToHashcode(result, a[i]);
+
+ }
+ }
+ if (scannerInfo instanceof IExtendedScannerInfo) {
+ IExtendedScannerInfo esi= (IExtendedScannerInfo) scannerInfo;
+ a= esi.getIncludeFiles();
+ if (a != null) {
+ for (int i = 0; i < a.length; i++) {
+ result= addToHashcode(result, a[i]);
+
+ }
+ }
+ a= esi.getLocalIncludePath();
+ if (a != null) {
+ for (int i = 0; i < a.length; i++) {
+ result= addToHashcode(result, a[i]);
+
+ }
+ }
+ a= esi.getMacroFiles();
+ if (a != null) {
+ for (int i = 0; i < a.length; i++) {
+ result= addToHashcode(result, a[i]);
+
+ }
+ }
+ }
+ return result;
+ }
+
+ private static int addToHashcode(int result, String key) {
+ return result*31 + key.hashCode();
+ }
+}