/**
* Copyright (c) 2005-2013 by Appcelerator, Inc. All Rights Reserved.
* Licensed under the terms of the Eclipse Public License (EPL).
* Please see the license.txt included with this distribution for details.
* Any modifications to this file must keep this entire header intact.
*/
/*
* Created on May 24, 2005
*
* @author Fabio Zadrozny
*/
package org.python.pydev.editor.codecompletion.revisited;
import java.io.File;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.Path;
import org.eclipse.jface.text.IDocument;
import org.python.pydev.core.FileUtilsFileBuffer;
import org.python.pydev.core.FullRepIterable;
import org.python.pydev.core.IGrammarVersionProvider;
import org.python.pydev.core.IModule;
import org.python.pydev.core.IModulesManager;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.core.MisconfigurationException;
import org.python.pydev.core.ModulesKey;
import org.python.pydev.core.ModulesKeyForZip;
import org.python.pydev.core.log.Log;
import org.python.pydev.editor.codecompletion.revisited.ModulesFoundStructure.ZipContents;
import org.python.pydev.editor.codecompletion.revisited.PyPublicTreeMap.Entry;
import org.python.pydev.editor.codecompletion.revisited.javaintegration.JythonModulesManagerUtils;
import org.python.pydev.editor.codecompletion.revisited.modules.AbstractModule;
import org.python.pydev.editor.codecompletion.revisited.modules.CompiledModule;
import org.python.pydev.editor.codecompletion.revisited.modules.EmptyModule;
import org.python.pydev.editor.codecompletion.revisited.modules.EmptyModuleForZip;
import org.python.pydev.editor.codecompletion.revisited.modules.SourceModule;
import org.python.pydev.parser.PyParser;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.Assign;
import org.python.pydev.parser.jython.ast.ClassDef;
import org.python.pydev.parser.jython.ast.Import;
import org.python.pydev.parser.jython.ast.Module;
import org.python.pydev.parser.jython.ast.Name;
import org.python.pydev.parser.jython.ast.NameTok;
import org.python.pydev.parser.jython.ast.aliasType;
import org.python.pydev.parser.jython.ast.exprType;
import org.python.pydev.parser.jython.ast.stmtType;
import org.python.pydev.parser.visitors.NodeUtils;
import org.python.pydev.plugin.PydevPlugin;
import org.python.pydev.shared_core.cache.LRUMap;
import org.python.pydev.shared_core.callbacks.ICallbackListener;
import org.python.pydev.shared_core.io.FileUtils;
import org.python.pydev.shared_core.out_of_memory.OnExpectedOutOfMemory;
import org.python.pydev.shared_core.parsing.BaseParser.ParseOutput;
import org.python.pydev.shared_core.string.FastStringBuffer;
import org.python.pydev.shared_core.string.StringUtils;
import org.python.pydev.shared_core.structure.Tuple;
import org.python.pydev.ui.filetypes.FileTypesPreferencesPage;
/**
* This class manages the modules that are available
*
* @author Fabio Zadrozny
*/
public abstract class ModulesManager implements IModulesManager {
/**
* Note: MODULES_MANAGER_V1 had a bug when writing/reading ModulesKeyForZip entries.
*/
private static final String MODULES_MANAGER_V2 = "MODULES_MANAGER_V2\n";
private final static boolean DEBUG_BUILD = false;
private final static boolean DEBUG_TEMPORARY_MODULES = false;
private final static boolean DEBUG_ZIP = false;
static {
OnExpectedOutOfMemory.clearCacheOnOutOfMemory.registerListener(new ICallbackListener<Object>() {
@Override
public Object call(Object obj) {
clearCache();
return null;
}
});
}
public ModulesManager() {
}
private final static double ONE_MINUTE_IN_MILLIS = 60.0 * 1000.0;
/**
* This class is a cache to help in getting the managers that are referenced or referred.
*
* It will not actually make any computations (the managers must be set from the outside)
*/
protected static class CompletionCache {
public IModulesManager[] referencedManagers;
public IModulesManager[] referredManagers;
private long creationTime;
private int calls = 0;
public IModulesManager[] getManagers(boolean referenced) {
calls += 1;
if (calls % 30 == 0) {
long diff = System.currentTimeMillis() - creationTime;
if (diff > ONE_MINUTE_IN_MILLIS) {
String msg = String.format(
"Warning: the cache related to project dependencies is the same for %.2f minutes.",
(diff / ONE_MINUTE_IN_MILLIS));
Log.logInfo(msg);
}
}
if (referenced) {
return this.referencedManagers;
} else {
return this.referredManagers;
}
}
public void setManagers(IModulesManager[] ret, boolean referenced) {
if (this.creationTime == 0) {
this.creationTime = System.currentTimeMillis();
}
if (referenced) {
this.referencedManagers = ret;
} else {
this.referredManagers = ret;
}
}
}
/**
* A stack for keeping the completion cache
*/
protected volatile CompletionCache completionCache = null;
protected final Object lockCompletionCache = new Object();
private volatile int completionCacheI = 0;
/**
* This method starts a new cache for this manager, so that needed info is kept while the request is happening
* (so, some info may not need to be re-asked over and over for requests)
*/
@Override
public boolean startCompletionCache() {
synchronized (lockCompletionCache) {
if (completionCache == null) {
completionCache = new CompletionCache();
}
completionCacheI += 1;
}
return true;
}
@Override
public void endCompletionCache() {
synchronized (lockCompletionCache) {
completionCacheI -= 1;
if (completionCacheI == 0) {
completionCache = null;
} else if (completionCacheI < 0) {
throw new RuntimeException("Completion cache negative (request unsynched)");
}
}
}
/**
* Modules that we have in memory. This is persisted when saved.
*
* Keys are ModulesKey with the name of the module. Values are AbstractModule objects.
*
* Implementation changed to contain a cache, so that it does not grow to much (some out of memo errors
* were thrown because of the may size when having too many modules).
*
* It is sorted so that we can get things in a 'subtree' faster
*/
protected final PyPublicTreeMap<ModulesKey, ModulesKey> modulesKeys = new PyPublicTreeMap<ModulesKey, ModulesKey>();
protected final Object modulesKeysLock = new Object();
protected static final ModulesManagerCache cache = new ModulesManagerCache();
/**
* Helper for using the pythonpath. Also persisted.
*/
protected final PythonPathHelper pythonPathHelper = new PythonPathHelper();
@Override
public PythonPathHelper getPythonPathHelper() {
return pythonPathHelper;
}
@Override
public void saveToFile(File workspaceMetadataFile) {
if (workspaceMetadataFile.exists() && !workspaceMetadataFile.isDirectory()) {
try {
FileUtils.deleteFile(workspaceMetadataFile);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (!workspaceMetadataFile.exists()) {
workspaceMetadataFile.mkdirs();
}
File modulesKeysFile = new File(workspaceMetadataFile, "modulesKeys");
File pythonpatHelperFile = new File(workspaceMetadataFile, "pythonpath");
FastStringBuffer buf;
HashMap<String, Integer> commonTokens = new HashMap<String, Integer>();
synchronized (modulesKeysLock) {
buf = new FastStringBuffer(this.modulesKeys.size() * 50);
buf.append(MODULES_MANAGER_V2);
for (Iterator<ModulesKey> iter = this.modulesKeys.keySet().iterator(); iter.hasNext();) {
ModulesKey next = iter.next();
buf.append(next.name);
if (next.file != null) {
buf.append("|");
if (next instanceof ModulesKeyForZip) {
ModulesKeyForZip modulesKeyForZip = (ModulesKeyForZip) next;
if (modulesKeyForZip.zipModulePath != null) {
String fileStr = next.file.toString();
Integer t = commonTokens.get(fileStr);
if (t == null) {
t = commonTokens.size();
commonTokens.put(fileStr, t);
}
buf.append(t);
buf.append("|");
buf.append(modulesKeyForZip.zipModulePath);
buf.append("|");
buf.append(modulesKeyForZip.isFile ? '1' : '0');
}
} else {
buf.append(next.file.toString());
}
}
buf.append('\n');
}
}
if (commonTokens.size() > 0) {
FastStringBuffer header = new FastStringBuffer(buf.length() + (commonTokens.size() * 50));
header.append(MODULES_MANAGER_V2);
header.append("--COMMON--\n");
for (Map.Entry<String, Integer> entries : commonTokens.entrySet()) {
header.append(entries.getValue());
header.append('=');
header.append(entries.getKey());
header.append('\n');
}
header.append("--END-COMMON--\n");
header.append(buf);
buf = header;
}
FileUtils.writeStrToFile(buf.toString(), modulesKeysFile);
this.pythonPathHelper.saveToFile(pythonpatHelperFile);
}
/**
* @param systemModulesManager
* @param workspaceMetadataFile
* @throws IOException
*/
public static void loadFromFile(ModulesManager modulesManager, File workspaceMetadataFile) throws IOException {
if (workspaceMetadataFile.exists() && !workspaceMetadataFile.isDirectory()) {
throw new IOException("Expecting: " + workspaceMetadataFile + " to be a directory.");
}
File modulesKeysFile = new File(workspaceMetadataFile, "modulesKeys");
File pythonpatHelperFile = new File(workspaceMetadataFile, "pythonpath");
if (!modulesKeysFile.isFile()) {
throw new IOException("Expecting: " + modulesKeysFile + " to exist (and be a file).");
}
if (!pythonpatHelperFile.isFile()) {
throw new IOException("Expecting: " + pythonpatHelperFile + " to exist (and be a file).");
}
String fileContents = FileUtils.getFileContents(modulesKeysFile);
if (!fileContents.startsWith(MODULES_MANAGER_V2)) {
throw new RuntimeException(
"Could not load modules manager from " + modulesKeysFile + " (version changed).");
}
HashMap<Integer, String> intToString = new HashMap<Integer, String>();
fileContents = fileContents.substring(MODULES_MANAGER_V2.length());
if (fileContents.startsWith("--COMMON--\n")) {
String header = fileContents.substring("--COMMON--\n".length());
header = header.substring(0, header.indexOf("--END-COMMON--\n"));
fileContents = fileContents.substring(fileContents.indexOf("--END-COMMON--\n")
+ "--END-COMMON--\n".length());
for (String line : StringUtils.iterLines(header)) {
line = line.trim();
List<String> split = StringUtils.split(line, '=');
if (split.size() == 2) {
try {
int i = Integer.parseInt(split.get(0));
intToString.put(i, split.get(1));
} catch (NumberFormatException e) {
Log.log(e);
}
}
}
if (fileContents.startsWith(MODULES_MANAGER_V2)) {
fileContents = fileContents.substring(MODULES_MANAGER_V2.length());
}
}
handleFileContents(modulesManager, fileContents, intToString);
if (modulesManager.pythonPathHelper == null) {
throw new IOException("Pythonpath helper not properly restored. " + modulesManager.getClass().getName()
+ " dir:" + workspaceMetadataFile);
}
modulesManager.pythonPathHelper.loadFromFile(pythonpatHelperFile);
if (modulesManager.pythonPathHelper.getPythonpath() == null) {
throw new IOException("Pythonpath helper pythonpath not properly restored. "
+ modulesManager.getClass().getName() + " dir:" + workspaceMetadataFile);
}
if (modulesManager.pythonPathHelper.getPythonpath().size() == 0) {
throw new IOException("Pythonpath helper pythonpath restored with no contents. "
+ modulesManager.getClass().getName() + " dir:" + workspaceMetadataFile);
}
if (modulesManager.modulesKeys.size() < 2) { //if we have few modules, that may indicate a problem...
//if the project is really small, modulesManager will be fast, otherwise, it'll fix the problem.
//Note: changed to a really low value because we now make a check after it's restored anyways.
throw new IOException("Only " + modulesManager.modulesKeys.size() + " modules restored in I/O. "
+ modulesManager.getClass().getName() + " dir:" + workspaceMetadataFile);
}
}
/**
* This method was simply:
*
* for(String line:StringUtils.iterLines(fileContents)){
* line = line.trim();
* List<String> split = StringUtils.split(line, '|');
* handleLineParts(modulesManager, intToString, split);
* }
*
* and was changed to be faster (as this was one of the slow things in startup).
*/
/*default*/@SuppressWarnings("rawtypes")
static void handleFileContents(ModulesManager modulesManager, String fileContents,
HashMap<Integer, String> intToString) {
String string = fileContents;
int len = string.length();
final ArrayList<ModulesKey> lst = new ArrayList<ModulesKey>();
char c;
int start = 0;
int i = 0;
String[] parts = new String[4];
int partsFound = 0;
for (; i < len; i++) {
c = string.charAt(i);
if (c == '\r') {
String trimmed = string.substring(start, i).trim();
if (trimmed.length() > 0) {
parts[partsFound] = trimmed;
partsFound++;
}
handleLineParts(modulesManager, intToString, parts, partsFound, lst);
partsFound = 0;
if (i < len - 1 && string.charAt(i + 1) == '\n') {
i++;
}
start = i + 1;
} else if (c == '\n') {
String trimmed = string.substring(start, i).trim();
if (trimmed.length() > 0) {
parts[partsFound] = trimmed;
partsFound++;
}
handleLineParts(modulesManager, intToString, parts, partsFound, lst);
partsFound = 0;
start = i + 1;
} else if (c == '|') {
String trimmed = string.substring(start, i).trim();
if (trimmed.length() > 0) {
parts[partsFound] = trimmed;
partsFound++;
}
start = i + 1;
}
}
if (start < len && start != i) {
String trimmed = string.substring(start, i).trim();
if (trimmed.length() > 0) {
parts[partsFound] = trimmed;
partsFound++;
}
handleLineParts(modulesManager, intToString, parts, partsFound, lst);
}
try {
final int size = lst.size();
//As we saved in sorted order, we can build in sorted order too (which is MUCH faster than adding items one
//by one).
modulesManager.modulesKeys.buildFromSorted(size, new Iterator() {
private int i = 0;
@Override
public boolean hasNext() {
return i < size;
}
@Override
public Object next() {
final ModulesKey next = lst.get(i);
i++;
return new Map.Entry() {
@Override
public Object getKey() {
return next;
}
@Override
public Object getValue() {
return next;
}
@Override
public Object setValue(Object value) {
throw new UnsupportedOperationException();
}
};
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
}, null, null);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static void handleLineParts(ModulesManager modulesManager, HashMap<Integer, String> intToString,
String[] split, int size, ArrayList<ModulesKey> lst) {
if (size > 0 && split[0].length() > 0) { //Just making sure we have something there.
ModulesKey key;
if (size == 1) {
key = new ModulesKey(split[0], null);
//restore with empty modules.
lst.add(key);
} else if (size == 2) {
key = new ModulesKey(split[0], new File(split[1]));
//restore with empty modules.
lst.add(key);
} else if (size == 4) {
try {
key = new ModulesKeyForZip(split[0], //module name
new File(intToString.get(Integer.parseInt(split[1]))), //zip file (usually repeated over and over again)
split[2], //path in zip
split[3].equals("1")); //is file (false = folder)
//restore with empty modules.
lst.add(key);
} catch (NumberFormatException e) {
Log.log(e);
}
}
}
}
/**
* @return Returns the modules.
*/
protected Map<ModulesKey, AbstractModule> getModules() {
throw new RuntimeException("Deprecated");
}
/**
* Change the pythonpath (used for both: system and project)
*
* @param project: may be null
* @param defaultSelectedInterpreter: may be null
*/
@Override
public void changePythonPath(String pythonpath, final IProject project, IProgressMonitor monitor) {
if (monitor == null) {
monitor = new NullProgressMonitor();
}
pythonPathHelper.setPythonPath(pythonpath);
ModulesFoundStructure modulesFound = pythonPathHelper.getModulesFoundStructure(project, monitor);
PyPublicTreeMap<ModulesKey, ModulesKey> keys = buildKeysFromModulesFound(monitor, modulesFound);
onChangePythonpath(keys);
synchronized (modulesKeysLock) {
cache.clear();
//assign to instance variable
this.modulesKeys.clear();
this.modulesKeys.putAll(keys);
}
}
/**
* @return a tuple with the new keys to be added to the modules manager (i.e.: found in keysFound but not in the
* modules manager) and the keys to be removed from the modules manager (i.e.: found in the modules manager but
* not in the keysFound)
*/
@Override
public Tuple<List<ModulesKey>, List<ModulesKey>> diffModules(AbstractMap<ModulesKey, ModulesKey> keysFound) {
ArrayList<ModulesKey> newKeys = new ArrayList<ModulesKey>();
ArrayList<ModulesKey> removedKeys = new ArrayList<ModulesKey>();
Iterator<ModulesKey> it = keysFound.keySet().iterator();
synchronized (modulesKeysLock) {
while (it.hasNext()) {
ModulesKey next = it.next();
ModulesKey modulesKey = modulesKeys.get(next);
if (modulesKey == null || modulesKey.getClass() != next.getClass()) {
//Check the class because ModulesKey and ModulesKeyForZip are equal considering only the name.
newKeys.add(next);
}
}
it = modulesKeys.keySet().iterator();
while (it.hasNext()) {
ModulesKey next = it.next();
ModulesKey modulesKey = keysFound.get(next);
if (modulesKey == null || modulesKey.getClass() != next.getClass()) {
removedKeys.add(next);
}
}
}
return new Tuple<List<ModulesKey>, List<ModulesKey>>(newKeys, removedKeys);
}
public static PyPublicTreeMap<ModulesKey, ModulesKey> buildKeysFromModulesFound(IProgressMonitor monitor,
ModulesFoundStructure modulesFound) {
//now, on to actually filling the module keys
PyPublicTreeMap<ModulesKey, ModulesKey> keys = new PyPublicTreeMap<ModulesKey, ModulesKey>();
buildKeysForRegularEntries(monitor, modulesFound, keys, false);
for (ZipContents zipContents : modulesFound.zipContents) {
if (monitor.isCanceled()) {
break;
}
buildKeysForZipContents(keys, zipContents);
}
return keys;
}
public static void buildKeysForRegularEntries(IProgressMonitor monitor, ModulesFoundStructure modulesFound,
PyPublicTreeMap<ModulesKey, ModulesKey> keys, boolean includeOnlySourceModules) {
String[] dottedValidSourceFiles = FileTypesPreferencesPage.getDottedValidSourceFiles();
int j = 0;
FastStringBuffer buffer = new FastStringBuffer();
//now, create in memory modules for all the loaded files (empty modules).
for (Iterator<Map.Entry<File, String>> iterator = modulesFound.regularModules.entrySet().iterator(); iterator
.hasNext() && monitor.isCanceled() == false; j++) {
Map.Entry<File, String> entry = iterator.next();
String m = entry.getValue();
if (m != null) {
if (j % 20 == 0) {
//no need to report all the time (that's pretty fast now)
buffer.clear();
monitor.setTaskName(buffer.append("Module resolved: ").append(m).toString());
monitor.worked(1);
}
//we don't load them at this time.
File f = entry.getKey();
if (includeOnlySourceModules) {
//check if we should include only source modules
if (!PythonPathHelper.isValidSourceFile(f.getName())) {
continue;
}
}
ModulesKey modulesKey = new ModulesKey(m, f);
//no conflict (easy)
if (!keys.containsKey(modulesKey)) {
keys.put(modulesKey, modulesKey);
} else {
//we have a conflict, so, let's resolve which one to keep (the old one or this one)
if (PythonPathHelper.isValidSourceFile(f.getName(), dottedValidSourceFiles)) {
//source files have priority over other modules (dlls) -- if both are source, there is no real way to resolve
//this priority, so, let's just add it over.
keys.put(modulesKey, modulesKey);
}
}
}
}
}
public static void buildKeysForZipContents(PyPublicTreeMap<ModulesKey, ModulesKey> keys, ZipContents zipContents) {
for (String filePathInZip : zipContents.foundFileZipPaths) {
String modName = StringUtils.stripExtension(filePathInZip).replace('/', '.');
if (DEBUG_ZIP) {
System.out.println("Found in zip:" + modName);
}
ModulesKey k = new ModulesKeyForZip(modName, zipContents.zipFile, filePathInZip, true);
keys.put(k, k);
if (zipContents.zipContentsType == ZipContents.ZIP_CONTENTS_TYPE_JAR) {
//folder modules are only created for jars (because for python files, the __init__.py is required).
for (String s : new FullRepIterable(FullRepIterable.getWithoutLastPart(modName))) { //the one without the last part was already added
k = new ModulesKeyForZip(s, zipContents.zipFile, s.replace('.', '/'), false);
keys.put(k, k);
}
}
}
}
/**
* Subclasses may do more things after the defaults were added to the cache (e.g.: the system modules manager may
* add builtins)
*/
protected void onChangePythonpath(SortedMap<ModulesKey, ModulesKey> keys) {
}
/**
* This is the only method that should remove a module.
* No other method should remove them directly.
*
* @param key this is the key that should be removed
*/
protected void doRemoveSingleModule(ModulesKey key) {
synchronized (modulesKeysLock) {
if (DEBUG_BUILD) {
System.out.println("Removing module:" + key + " - " + this.getClass());
}
this.modulesKeys.remove(key);
ModulesManager.cache.remove(key, this);
}
}
/**
* This method that actually removes some keys from the modules.
*
* @param toRem the modules to be removed
*/
protected void removeThem(Collection<ModulesKey> toRem) {
//really remove them here.
for (Iterator<ModulesKey> iter = toRem.iterator(); iter.hasNext();) {
doRemoveSingleModule(iter.next());
}
}
@Override
public void removeModules(Collection<ModulesKey> toRem) {
removeThem(toRem);
}
@Override
public IModule addModule(final ModulesKey key) {
AbstractModule ret = AbstractModule.createEmptyModule(key);
doAddSingleModule(key, ret);
return ret;
}
@Override
public boolean hasModule(ModulesKey key) {
synchronized (modulesKeysLock) {
return this.modulesKeys.containsKey(key);
}
}
/**
* This is the only method that should add / update a module.
* No other method should add it directly (unless it is loading or rebuilding it).
*
* @param key this is the key that should be added
* @param n
*/
public void doAddSingleModule(final ModulesKey key, AbstractModule n) {
if (DEBUG_BUILD) {
System.out.println("Adding module:" + key + " - " + this.getClass());
}
synchronized (modulesKeysLock) {
this.modulesKeys.put(key, key);
ModulesManager.cache.add(key, n, this);
}
}
/**
* @return a set of all module keys
*
* Note: addDependencies ignored at this point.
*/
@Override
public Set<String> getAllModuleNames(boolean addDependencies, String partStartingWithLowerCase) {
Set<String> s = new HashSet<String>();
synchronized (modulesKeysLock) {
for (ModulesKey key : this.modulesKeys.keySet()) {
if (key.hasPartStartingWith(partStartingWithLowerCase)) {
s.add(key.name);
}
}
}
return s;
}
@Override
public SortedMap<ModulesKey, ModulesKey> getAllDirectModulesStartingWith(String strStartingWith) {
if (strStartingWith.length() == 0) {
synchronized (modulesKeysLock) {
//we don't want it to be backed up by the same set (because it may be changed, so, we may get
//a java.util.ConcurrentModificationException on places that use it)
return new PyPublicTreeMap<ModulesKey, ModulesKey>(modulesKeys);
}
}
ModulesKey startingWith = new ModulesKey(strStartingWith, null);
ModulesKey endingWith = new ModulesKey(strStartingWith + "\uffff\uffff\uffff\uffff", null);
synchronized (modulesKeysLock) {
//we don't want it to be backed up by the same set (because it may be changed, so, we may get
//a java.util.ConcurrentModificationException on places that use it)
return new PyPublicTreeMap<ModulesKey, ModulesKey>(modulesKeys.subMap(startingWith, endingWith));
}
}
@Override
public SortedMap<ModulesKey, ModulesKey> getAllModulesStartingWith(String strStartingWith) {
return getAllDirectModulesStartingWith(strStartingWith);
}
@Override
public ModulesKey[] getOnlyDirectModules() {
synchronized (modulesKeysLock) {
return this.modulesKeys.keySet().toArray(new ModulesKey[0]);
}
}
/**
* Note: no dependencies at this point (so, just return the keys)
*/
@Override
public int getSize(boolean addDependenciesSize) {
synchronized (modulesKeysLock) {
return this.modulesKeys.size();
}
}
@Override
public IModule getModule(String name, IPythonNature nature, boolean dontSearchInit) {
return getModule(true, name, nature, dontSearchInit);
}
/**
* Note that the access must be synched.
*/
public final Map<String, SortedMap<Integer, IModule>> temporaryModules = new HashMap<String, SortedMap<Integer, IModule>>();
private final Object lockTemporaryModules = new Object();
private int nextHandle = 0;
/**
* Returns the handle to be used to remove the module added later on!
*/
@Override
public int pushTemporaryModule(String moduleName, IModule module) {
synchronized (lockTemporaryModules) {
SortedMap<Integer, IModule> map = temporaryModules.get(moduleName);
if (map == null) {
map = new TreeMap<Integer, IModule>(); //small initial size!
temporaryModules.put(moduleName, map);
}
if (module instanceof AbstractModule) {
module = decorateModule((AbstractModule) module, null);
}
nextHandle += 1; //Note: don't care about stack overflow!
map.put(nextHandle, module);
return nextHandle;
}
}
@Override
public void popTemporaryModule(String moduleName, int handle) {
synchronized (lockTemporaryModules) {
SortedMap<Integer, IModule> stack = temporaryModules.get(moduleName);
try {
if (stack != null) {
stack.remove(handle);
if (stack.size() == 0) {
temporaryModules.remove(moduleName);
}
}
} catch (Throwable e) {
Log.log(e);
}
}
}
/**
* This method returns the module that corresponds to the path passed as a parameter.
*
* @param name the name of the module we're looking for (e.g.: mod1.mod2)
* @param dontSearchInit is used in a negative form because initially it was isLookingForRelative, but
* it actually defines if we should look in __init__ modules too, so, the name matches the old signature.
*
* NOTE: isLookingForRelative description was: when looking for relative imports, we don't check for __init__
* @return the module represented by this name
*/
protected IModule getModule(boolean acceptCompiledModule, String name, IPythonNature nature,
boolean dontSearchInit) {
synchronized (lockTemporaryModules) {
SortedMap<Integer, IModule> map = temporaryModules.get(name);
if (map != null && map.size() > 0) {
if (DEBUG_TEMPORARY_MODULES) {
System.out.println("Returning temporary module: " + name);
}
return map.get(map.lastKey());
}
}
AbstractModule n = null;
ModulesKey keyForCacheAccess = new ModulesKey(null, null);
if (!dontSearchInit) {
if (n == null) {
keyForCacheAccess.name = (String) StringUtils.join(".", new String[] { name, "__init__" }, null);
n = cache.getObj(keyForCacheAccess, this);
if (n != null) {
name = keyForCacheAccess.name;
}
}
}
if (n == null) {
keyForCacheAccess.name = name;
n = cache.getObj(keyForCacheAccess, this);
}
if (n instanceof SourceModule) {
//ok, module exists, let's check if it is synched with the filesystem version...
SourceModule s = (SourceModule) n;
if (!s.isSynched()) {
//change it for an empty and proceed as usual.
n = (AbstractModule) addModule(createModulesKey(s.getName(), s.getFile()));
}
}
if (n instanceof EmptyModule) {
EmptyModule e = (EmptyModule) n;
if (e.f != null) {
if (!e.f.exists()) {
//if the file does not exist anymore, just remove it.
keyForCacheAccess.name = name;
keyForCacheAccess.file = e.f;
doRemoveSingleModule(keyForCacheAccess);
n = null;
} else {
//file exists
n = checkOverride(name, nature, n);
if (n instanceof EmptyModule) {
//ok, handle case where the file is actually from a zip file...
if (e instanceof EmptyModuleForZip) {
EmptyModuleForZip emptyModuleForZip = (EmptyModuleForZip) e;
if (emptyModuleForZip.pathInZip.endsWith(".class") || !emptyModuleForZip.isFile) {
//handle java class... (if it's a class or a folder in a jar)
try {
n = JythonModulesManagerUtils.createModuleFromJar(emptyModuleForZip, nature);
n = decorateModule(n, nature);
} catch (Throwable e1) {
Log.log("Unable to create module from jar (note: JDT is required for Jython development): "
+ emptyModuleForZip + " project: "
+ (nature != null ? nature.getProject() : "null"), e1);
n = null;
}
} else if (FileTypesPreferencesPage.isValidDll(emptyModuleForZip.pathInZip)) {
//.pyd
n = new CompiledModule(name, this, nature);
n = decorateModule(n, nature);
} else if (PythonPathHelper.isValidSourceFile(emptyModuleForZip.pathInZip)) {
//handle python file from zip... we have to create it getting the contents from the zip file
try {
IDocument doc = FileUtilsFileBuffer.getDocFromZip(emptyModuleForZip.f,
emptyModuleForZip.pathInZip);
//NOTE: The nature (and so the grammar to be used) must be defined by this modules
//manager (and not by the initial caller)!!
n = AbstractModule.createModuleFromDoc(name, emptyModuleForZip.f, doc,
this.getNature(), false);
SourceModule zipModule = (SourceModule) n;
zipModule.zipFilePath = emptyModuleForZip.pathInZip;
n = decorateModule(n, nature);
} catch (Exception exc1) {
Log.log(exc1);
n = null;
}
}
} else {
//regular case... just go on and create it.
try {
//NOTE: The nature (and so the grammar to be used) must be defined by this modules
//manager (and not by the initial caller)!!
n = AbstractModule.createModule(name, e.f, this.getNature(), true);
n = decorateModule(n, nature);
} catch (IOException exc) {
keyForCacheAccess.name = name;
keyForCacheAccess.file = e.f;
doRemoveSingleModule(keyForCacheAccess);
n = null;
} catch (MisconfigurationException exc) {
Log.log(exc);
n = null;
}
}
}
}
} else { //ok, it does not have a file associated, so, we treat it as a builtin (this can happen in java jars)
n = checkOverride(name, nature, n);
if (n instanceof EmptyModule) {
if (acceptCompiledModule) {
n = new CompiledModule(name, this, nature);
n = decorateModule(n, nature);
} else {
return null;
}
}
}
if (n != null) {
doAddSingleModule(createModulesKey(name, e.f), n);
} else {
Log.log(("The module " + name + " could not be found nor created!"));
}
}
if (n instanceof EmptyModule) {
throw new RuntimeException("Should not be an empty module anymore: " + n);
}
if (n instanceof SourceModule) {
SourceModule sourceModule = (SourceModule) n;
//now, here's a catch... it may be a bootstrap module...
if (sourceModule.isBootstrapModule()) {
//if it's a bootstrap module, we must replace it for the related compiled module.
n = new CompiledModule(name, this, nature);
n = decorateModule(n, nature);
}
}
return n;
}
/**
* Called after the creation of any module. Used as a workaround for filling tokens that are in no way
* available in the code-completion through the regular inspection.
*
* The django objects class is the reason why this happens... It's structure for the creation on a model class
* follows no real patterns for the creation of the 'objects' attribute in the class, and thus, we have no
* real generic way of discovering it (actually, even by looking at the class definition this is very obscure),
* so, the solution found is creating the objects by decorating the module with that info.
*/
private AbstractModule decorateModule(AbstractModule n, IPythonNature nature) {
if (n instanceof SourceModule) {
if ("django.db.models.base".equals(n.getName())) {
SourceModule sourceModule = (SourceModule) n;
SimpleNode ast = sourceModule.getAst();
boolean found = false;
Module module = (Module) ast;
stmtType[] body = module.body;
for (SimpleNode node : body) {
if (node instanceof ClassDef && "Model".equals(NodeUtils.getRepresentationString(node))) {
found = true;
Object[][] metaclassAttrs = new Object[][] {
{ "objects", NodeUtils.makeAttribute("django.db.models.manager.Manager()") },
{ "DoesNotExist", new Name("Exception", Name.Load, false) },
{ "MultipleObjectsReturned", new Name("Exception", Name.Load, false) }, };
ClassDef classDef = (ClassDef) node;
stmtType[] newBody = new stmtType[classDef.body.length + metaclassAttrs.length];
System.arraycopy(classDef.body, 0, newBody, metaclassAttrs.length, classDef.body.length);
int i = 0;
for (Object[] objAndType : metaclassAttrs) {
//Note that the line/col is important so that we correctly acknowledge it inside the "class Model" scope.
Name name = new Name((String) objAndType[0], Name.Store, false);
name.beginColumn = classDef.beginColumn + 4;
name.beginLine = classDef.beginLine + 1;
newBody[i] = new Assign(new exprType[] { name }, (exprType) objAndType[1], null);
newBody[i].beginColumn = classDef.beginColumn + 4;
newBody[i].beginLine = classDef.beginLine + 1;
i += 1;
}
classDef.body = newBody;
break;
}
}
if (found) {
stmtType[] newBody = new stmtType[body.length + 1];
System.arraycopy(body, 0, newBody, 1, body.length);
Import importNode = new Import(new aliasType[] { new aliasType(new NameTok(
"django.db.models.manager",
NameTok.ImportModule), null) });
newBody[0] = importNode;
module.body = newBody;
}
} else if ("django.db.models.manager".equals(n.getName())) {
SourceModule sourceModule = (SourceModule) n;
SimpleNode ast = sourceModule.getAst();
Module module = (Module) ast;
stmtType[] body = module.body;
for (SimpleNode node : body) {
if (node instanceof ClassDef && "Manager".equals(NodeUtils.getRepresentationString(node))) {
ClassDef classDef = (ClassDef) node;
stmtType[] newBody = null;
try {
File managerBody = PydevPlugin.getBundleInfo().getRelativePath(
new Path("pysrc/stubs/_django_manager_body.py"));
IDocument doc = FileUtilsFileBuffer.getDocFromFile(managerBody);
IGrammarVersionProvider provider = new IGrammarVersionProvider() {
@Override
public int getGrammarVersion() throws MisconfigurationException {
return IGrammarVersionProvider.LATEST_GRAMMAR_PY3_VERSION; // Always Python 3.0 here
}
@Override
public AdditionalGrammarVersionsToCheck getAdditionalGrammarVersions()
throws MisconfigurationException {
return null;
}
};
ParseOutput obj = PyParser.reparseDocument(new PyParser.ParserInfo(doc, provider,
"_django_manager_body", managerBody));
Module ast2 = (Module) obj.ast;
newBody = ast2.body;
for (stmtType b : newBody) {
//Note that the line/col is important so that we correctly acknowledge it inside the "class Manager" scope.
b.beginColumn = classDef.beginColumn + 4;
b.beginLine = classDef.beginLine + 1;
}
classDef.body = newBody;
break;
} catch (Exception e) {
Log.log(e);
}
}
}
}
}
return n;
}
/**
* Hook called to give clients a chance to override the module created (still experimenting, so, it's not public).
*/
private AbstractModule checkOverride(String name, IPythonNature nature, AbstractModule emptyModule) {
return emptyModule;
}
private ModulesKey createModulesKey(String name, File f) {
ModulesKey newEntry = new ModulesKey(name, f);
synchronized (modulesKeysLock) {
Entry<ModulesKey, ModulesKey> oldEntry = this.modulesKeys.getEntry(newEntry);
if (oldEntry != null) {
return oldEntry.getKey();
} else {
return newEntry;
}
}
}
/**
* Passes through all the compiled modules in memory and clears its tokens (so that
* we restore them when needed).
*/
public static void clearCache() {
ModulesManager.cache.clear();
}
/**
* @see org.python.pydev.core.IProjectModulesManager#isInPythonPath(org.eclipse.core.resources.IResource, org.eclipse.core.resources.IProject)
*/
@Override
public boolean isInPythonPath(IResource member, IProject container) {
return resolveModule(member, container) != null;
}
/**
* @see org.python.pydev.core.IProjectModulesManager#resolveModule(org.eclipse.core.resources.IResource, org.eclipse.core.resources.IProject)
*/
@Override
public String resolveModule(IResource member, IProject container) {
File inOs = member.getRawLocation().toFile();
return pythonPathHelper.resolveModule(FileUtils.getFileAbsolutePath(inOs), false, container);
}
protected String getResolveModuleErr(IResource member) {
return "Unable to find the path " + member + " in the project were it\n"
+ "is added as a source folder for pydev." + this.getClass();
}
/**
* @param full
* @return
*/
@Override
public String resolveModule(String full) {
return pythonPathHelper.resolveModule(full, false, null);
}
private final Object lockAccessCreateCompiledModuleLock = new Object();
private final Map<String, Object> createCompiledModuleLock = new LRUMap<String, Object>(50);
@Override
public Object getCompiledModuleCreationLock(String name) {
synchronized (lockAccessCreateCompiledModuleLock) {
Object lock = createCompiledModuleLock.get(name);
if (lock == null) {
lock = new Object();
createCompiledModuleLock.put(name, lock);
}
return lock;
}
}
}