/**
* Copyright (c) 2005-2013 by Appcelerator, Inc. All Rights Reserved.
* Licensed under the terms of the Eclipse Public License (EPL).
* Please see the license.txt included with this distribution for details.
* Any modifications to this file must keep this entire header intact.
*/
/*
* Created on 28/09/2005
*/
package com.python.pydev.analysis.additionalinfo;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.python.pydev.core.FastBufferedReader;
import org.python.pydev.core.IInterpreterManager;
import org.python.pydev.core.IModule;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.core.IToken;
import org.python.pydev.core.MisconfigurationException;
import org.python.pydev.core.ModulesKey;
import org.python.pydev.core.ObjectsInternPool;
import org.python.pydev.core.ObjectsInternPool.ObjectsPoolMap;
import org.python.pydev.core.cache.CompleteIndexKey;
import org.python.pydev.core.cache.DiskCache;
import org.python.pydev.core.log.Log;
import org.python.pydev.editor.codecompletion.revisited.PyPublicTreeMap;
import org.python.pydev.editor.codecompletion.revisited.PythonPathHelper;
import org.python.pydev.editor.codecompletion.revisited.javaintegration.AbstractJavaClassModule;
import org.python.pydev.logging.DebugSettings;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.Name;
import org.python.pydev.parser.jython.ast.stmtType;
import org.python.pydev.parser.jython.ast.factory.AdapterPrefs;
import org.python.pydev.parser.jython.ast.factory.PyAstFactory;
import org.python.pydev.shared_core.callbacks.CallbackWithListeners;
import org.python.pydev.shared_core.io.FileUtils;
import org.python.pydev.shared_core.string.FastStringBuffer;
import org.python.pydev.shared_core.string.StringUtils;
import org.python.pydev.shared_core.structure.Tuple;
import org.python.pydev.shared_core.structure.Tuple3;
import org.python.pydev.ui.pythonpathconf.InterpreterInfo;
/**
* Adds information on the modules being tracked.
*/
public abstract class AbstractAdditionalDependencyInfo extends AbstractAdditionalTokensInfo {
public static boolean TESTING = false;
public static final boolean DEBUG = false;
/**
* indexes all the names that are available
*
* Note that the key in the disk cache is the module name and each
* module points to a Set<Strings>
*
* So the key is the module name and the value is a Set of the strings it contains.
*/
public DiskCache completeIndex;
private volatile IReferenceSearches referenceSearches;
private final Object referenceSearchesLock = new Object();
public IReferenceSearches getReferenceSearches() {
if (referenceSearches == null) {
synchronized (referenceSearchesLock) {
if (referenceSearches == null) {
referenceSearches = new ReferenceSearchesLucene(this);
}
}
}
return referenceSearches;
}
public void dispose() {
if (this.referenceSearches != null) {
this.referenceSearches.dispose();
this.referenceSearches = null;
}
}
/**
* default constructor
* @throws MisconfigurationException
*/
public AbstractAdditionalDependencyInfo() throws MisconfigurationException {
init();
}
public AbstractAdditionalDependencyInfo(boolean callInit) throws MisconfigurationException {
if (callInit) {
init();
}
}
/**
* Initializes the internal DiskCache with the indexes.
* @throws MisconfigurationException
*/
protected void init() throws MisconfigurationException {
File persistingFolder = getCompleteIndexPersistingFolder();
completeIndex = new DiskCache(persistingFolder, ".v2_indexcache");
}
/**
* @return a folder where the index should be persisted
* @throws MisconfigurationException
*/
protected File getCompleteIndexPersistingFolder() throws MisconfigurationException {
File persistingFolder = getPersistingFolder();
persistingFolder = new File(persistingFolder, "v2_indexcache");
if (persistingFolder.exists()) {
if (!persistingFolder.isDirectory()) {
persistingFolder.delete();
}
}
if (!persistingFolder.exists()) {
persistingFolder.mkdirs();
}
return persistingFolder;
}
@Override
public void clearAllInfo() {
synchronized (lock) {
super.clearAllInfo();
try {
completeIndex.clear();
} catch (NullPointerException e) {
//that's ok... because it might be called before actually having any values
}
}
}
/**
* This is mostly for whitebox testing the updateKeysIfNeededAndSave. It'll called with a tuple containing
* the keys added and the keys removed.
*/
public static final CallbackWithListeners modulesAddedAndRemoved = new CallbackWithListeners(1);
public final Object updateKeysLock = new Object(); // Calls to updateKeysIfNeededAndSave should be synchronized.
/**
* If info == null we're dealing with project info (otherwise we're dealing with interpreter info).
*
* The main difference is that we don't index builtin modules for projects (but maybe we should?). Still,
* to index builtin modules we have to be a bit more careful, especially on changes (i.e.: when a builtin
* becomes a source module and vice-versa).
*/
public void updateKeysIfNeededAndSave(PyPublicTreeMap<ModulesKey, ModulesKey> keysFound, InterpreterInfo info,
IProgressMonitor monitor) {
Map<CompleteIndexKey, CompleteIndexKey> keys = this.completeIndex.keys();
ArrayList<ModulesKey> newKeys = new ArrayList<ModulesKey>();
ArrayList<ModulesKey> removedKeys = new ArrayList<ModulesKey>();
//temporary
CompleteIndexKey tempKey = new CompleteIndexKey((ModulesKey) null);
boolean isJython = info != null ? info.getInterpreterType() == IInterpreterManager.INTERPRETER_TYPE_JYTHON
: true;
Iterator<ModulesKey> it = keysFound.values().iterator();
while (it.hasNext()) {
ModulesKey next = it.next();
if (next.file != null) { //Can be a .pyd or a .py
long lastModified = FileUtils.lastModified(next.file);
if (lastModified != 0) {
tempKey.key = next;
CompleteIndexKey completeIndexKey = keys.get(tempKey);
if (completeIndexKey == null) {
newKeys.add(next);
} else {
if (completeIndexKey.lastModified != lastModified) {
//Just re-add it if the time changed!
newKeys.add(next);
}
}
}
} else { //at this point, it's always a compiled module (forced builtin), so, we can't check if it was modified (just re-add it).
tempKey.key = next;
CompleteIndexKey completeIndexKey = keys.get(tempKey);
if (completeIndexKey == null) {
newKeys.add(next); //Only add if it's not there already.
}
}
}
Iterator<CompleteIndexKey> it2 = keys.values().iterator();
while (it2.hasNext()) {
CompleteIndexKey next = it2.next();
if (!keysFound.containsKey(next.key)) {
removedKeys.add(next.key);
}
}
boolean hasNew = newKeys.size() != 0;
boolean hasRemoved = removedKeys.size() != 0;
modulesAddedAndRemoved.call(new Tuple(newKeys, removedKeys));
Set<File> ignoreFiles = new HashSet<File>();
// Remove first!
if (hasRemoved) {
for (ModulesKey removedKey : removedKeys) {
// Don't generate deltas (we'll save it in the end).
this.removeInfoFromModule(removedKey.name, false);
}
}
// Add last (a module could be removed/added).
if (hasNew) {
FastStringBuffer buffer = new FastStringBuffer();
int currI = 0;
int total = newKeys.size();
for (ModulesKey newKey : newKeys) {
currI += 1;
if (monitor.isCanceled()) {
return;
}
if (PythonPathHelper.canAddAstInfoForSourceModule(newKey)) {
buffer.clear().append("Indexing ").append(currI).append(" of ").append(total)
.append(" (source module): ").append(newKey.name).append(" (")
.append(currI).append(" of ").append(total).append(")");
try {
// Don't generate deltas (we'll save it in the end).
this.addAstInfo(newKey, false);
} catch (Exception e) {
Log.log(e);
}
} else {
if (info != null) {
if (isJython && ignoreFiles.contains(newKey.file)) {
continue;
}
buffer.clear().append("Indexing ").append(currI).append(" of ").append(total)
.append(" (builtin module): ").append(newKey.name);
monitor.setTaskName(buffer.toString());
IModule builtinModule = info.getModulesManager().getModule(newKey.name,
info.getModulesManager().getNature(), true);
if (builtinModule != null) {
if (builtinModule instanceof AbstractJavaClassModule) {
if (newKey.file != null) {
ignoreFiles.add(newKey.file);
} else {
Log.log("Not expecting null file for java class module: " + newKey);
}
continue;
}
boolean removeFirst = keys.containsKey(newKey);
addAstForCompiledModule(builtinModule, info, newKey, removeFirst);
}
}
}
}
}
if (hasNew || hasRemoved) {
if (DebugSettings.DEBUG_INTERPRETER_AUTO_UPDATE) {
Log.toLogFile(this,
StringUtils.format(
"Additional info modules. Added: %s Removed: %s", newKeys, removedKeys));
}
save();
}
}
private void addAstForCompiledModule(IModule module, InterpreterInfo info, ModulesKey newKey, boolean removeFirst) {
IToken[] globalTokens = module.getGlobalTokens();
PyAstFactory astFactory = new PyAstFactory(new AdapterPrefs("\n", info.getModulesManager().getNature()));
List<stmtType> body = new ArrayList<>(globalTokens.length);
for (IToken token : globalTokens) {
switch (token.getType()) {
case IToken.TYPE_CLASS:
body.add(astFactory.createClassDef(token.getRepresentation()));
break;
case IToken.TYPE_FUNCTION:
body.add(astFactory.createFunctionDef(token.getRepresentation()));
break;
default:
Name attr = astFactory.createName(token.getRepresentation());
body.add(astFactory.createAssign(attr, attr)); //assign to itself just for generation purposes.
break;
}
}
//System.out.println("Creating info for: " + module.getName());
if (removeFirst) {
removeInfoFromModule(newKey.name, false);
}
addAstInfo(astFactory.createModule(body), newKey, false);
}
static interface IBufferFiller {
void fillBuffer(FastStringBuffer buf);
}
protected abstract String getUIRepresentation();
protected abstract Set<String> getPythonPathFolders();
@Override
public List<IInfo> addAstInfo(SimpleNode node, ModulesKey key, boolean generateDelta) {
List<IInfo> addAstInfo = new ArrayList<IInfo>();
if (node == null || key == null || key.name == null) {
return addAstInfo;
}
try {
synchronized (lock) {
addAstInfo = super.addAstInfo(node, key, generateDelta);
CompleteIndexKey completeIndexKey = new CompleteIndexKey(key);
if (key.file != null) {
completeIndexKey.lastModified = FileUtils.lastModified(key.file);
}
completeIndex.add(completeIndexKey);
}
} catch (Exception e) {
Log.log(e);
}
return addAstInfo;
}
@Override
public void removeInfoFromModule(String moduleName, boolean generateDelta) {
synchronized (lock) {
if (moduleName == null) {
throw new AssertionError("The module name may not be null.");
}
completeIndex.remove(new CompleteIndexKey(moduleName));
super.removeInfoFromModule(moduleName, generateDelta);
}
}
@Override
protected void saveTo(OutputStreamWriter writer, FastStringBuffer tempBuf, File pathToSave) throws IOException {
synchronized (lock) {
completeIndex.writeTo(tempBuf);
writer.write(tempBuf.getInternalCharsArray(), 0, tempBuf.length());
tempBuf.clear();
super.saveTo(writer, tempBuf, pathToSave);
}
}
@Override
@SuppressWarnings("rawtypes")
protected void restoreSavedInfo(Object o) throws MisconfigurationException {
synchronized (lock) {
Tuple readFromFile = (Tuple) o;
if (!(readFromFile.o1 instanceof Tuple3)) {
throw new RuntimeException("Type Error: the info must be regenerated (changed across versions).");
}
completeIndex = (DiskCache) readFromFile.o2;
if (completeIndex == null) {
throw new RuntimeException(
"Type Error (index == null): the info must be regenerated (changed across versions).");
}
String shouldBeOn = FileUtils.getFileAbsolutePath(getCompleteIndexPersistingFolder());
if (!completeIndex.getFolderToPersist().equals(shouldBeOn)) {
//this can happen if the user moves its .metadata folder (so, we have to validate it).
completeIndex.setFolderToPersist(shouldBeOn);
}
super.restoreSavedInfo(readFromFile.o1);
}
}
/**
* actually does the load
* @return true if it was successfully loaded and false otherwise
*/
protected boolean load() {
Throwable errorFound = null;
synchronized (lock) {
File file;
try {
file = getPersistingLocation();
} catch (MisconfigurationException e) {
Log.log("Unable to restore previous info... (persisting location not available).", e);
return false;
}
if (file.exists() && file.isFile()) {
try {
return loadContentsFromFile(file, getNature()) != null;
} catch (Throwable e) {
errorFound = new RuntimeException("Unable to read: " + file, e);
}
}
}
try {
String msg = "Info: Rebuilding internal caches: " + this.getPersistingLocation();
if (errorFound == null) {
msg += " (Expected error to be provided and got no error!)";
Log.log(IStatus.ERROR, msg, errorFound);
} else {
Log.log(IStatus.INFO, msg, errorFound);
}
} catch (Exception e1) {
Log.log("Rebuilding internal caches (error getting persisting location).");
}
return false;
}
private Object loadContentsFromFile(File file, IPythonNature nature)
throws FileNotFoundException, IOException, MisconfigurationException {
FileInputStream fileInputStream = new FileInputStream(file);
try {
// Timer timer = new Timer();
String expected = "-- VERSION_" + AbstractAdditionalTokensInfo.version; //X is the version
InputStreamReader reader = new InputStreamReader(fileInputStream);
FastBufferedReader bufferedReader = new FastBufferedReader(reader);
FastStringBuffer string = bufferedReader.readLine();
ObjectsPoolMap objectsPoolMap = new ObjectsInternPool.ObjectsPoolMap();
if (string != null && string.startsWith("-- VERSION_")) {
Tuple<Tuple3<Object, Object, Object>, Object> tupWithResults = new Tuple<Tuple3<Object, Object, Object>, Object>(
new Tuple3<Object, Object, Object>(
null, null, null),
null);
Tuple3<Object, Object, Object> superTupWithResults = tupWithResults.o1;
//tupWithResults.o2 = DiskCache
if (string.toString().equals(expected)) {
//OK, proceed with new I/O format!
try {
try {
FastStringBuffer line;
Map<Integer, String> dictionary = null;
FastStringBuffer tempBuf = new FastStringBuffer(1024);
while ((line = bufferedReader.readLine()) != null) {
if (line.startsWith("-- ")) {
if (line.startsWith("-- START TREE 1")) {
superTupWithResults.o1 = TreeIO.loadTreeFrom(bufferedReader, dictionary,
tempBuf.clear(), objectsPoolMap, nature);
} else if (line.startsWith("-- START TREE 2")) {
superTupWithResults.o2 = TreeIO.loadTreeFrom(bufferedReader, dictionary,
tempBuf.clear(), objectsPoolMap, nature);
} else if (line.startsWith("-- START DICTIONARY")) {
dictionary = TreeIO.loadDictFrom(bufferedReader, tempBuf.clear(),
objectsPoolMap);
} else if (line.startsWith("-- START DISKCACHE")) {
if (!line.startsWith("-- START DISKCACHE_" + DiskCache.VERSION)) {
throw new RuntimeException("Disk cache version changed");
}
tupWithResults.o2 = DiskCache.loadFrom(bufferedReader, objectsPoolMap);
} else if (line.startsWith("-- VERSION_")) {
if (!line.endsWith(String.valueOf(AbstractAdditionalTokensInfo.version))) {
throw new RuntimeException("Expected the version to be: "
+ AbstractAdditionalTokensInfo.version + " Found: " + line);
}
} else if (line.startsWith("-- END TREE")) {
//just skip it in this situation.
} else {
throw new RuntimeException("Unexpected line: " + line);
}
}
}
} finally {
bufferedReader.close();
}
} finally {
reader.close();
}
restoreSavedInfo(tupWithResults);
// timer.printDiff("Time taken");
return tupWithResults;
} else {
throw new RuntimeException("Version does not match. Found: " + string + ". Expected: " + expected);
}
} else {
//Try the old way of loading it (backward compatibility).
fileInputStream.close();
// Timer timer2 = new Timer();
Object tupWithResults = IOUtils.readFromFile(file);
restoreSavedInfo(tupWithResults);
// timer2.printDiff("IOUtils time");
save(); //Save in new format!
return tupWithResults;
}
} finally {
try {
fileInputStream.close();
} catch (Exception e) {
//Ignore error closing.
}
}
}
protected void addInfoToModuleOnRestoreInsertCommand(Tuple<ModulesKey, List<IInfo>> data) {
CompleteIndexKey key = new CompleteIndexKey(data.o1);
if (data.o1.file != null) {
key.lastModified = FileUtils.lastModified(data.o1.file);
}
completeIndex.add(key);
//current way (saves a list of iinfo)
for (Iterator<IInfo> it = data.o2.iterator(); it.hasNext();) {
IInfo info = it.next();
if (info.getPath() == null || info.getPath().length() == 0) {
this.add(info, TOP_LEVEL);
} else {
this.add(info, INNER);
}
}
}
private CountDownLatch waitForIntegrity = null;
private static final Object waitForIntegrityLock = new Object();
public void setWaitForIntegrityCheck(boolean waitFor) {
synchronized (waitForIntegrityLock) {
if (waitFor) {
if (waitForIntegrity == null) {
waitForIntegrity = new CountDownLatch(1);
}
} else {
if (waitForIntegrity != null) {
waitForIntegrity.countDown();
waitForIntegrity = null;
}
}
}
}
public void waitForIntegrityCheck() {
CountDownLatch waitFor = null;
synchronized (waitForIntegrityLock) {
if (waitForIntegrity != null) {
waitFor = waitForIntegrity;
}
}
if (waitFor != null) {
try {
waitFor.await();
} catch (InterruptedException e) {
Log.log(e);
}
}
}
}