/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Eclipse Public License, Version 1.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.eclipse.org/org/documents/epl-v10.php
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
* Gathers statistics about attribute usage in layout files. This is how the "topAttrs"
* attributes listed in ADT's extra-view-metadata.xml (which drives the common attributes
* listed in the top of the context menu) is determined by running this script on a body
* of sample layout code.
* <p>
* This program takes one or more directory paths, and then it searches all of them recursively
* for layout files that are not in folders containing the string "test", and computes and
* prints frequency statistics.
*/
public class Analyzer {
/** Number of attributes to print for each view */
public static final int ATTRIBUTE_COUNT = 6;
/** Separate out any attributes that constitute less than N percent of the total */
public static final int THRESHOLD = 10; // percent
private List<File> mDirectories;
private File mCurrentFile;
private boolean mListAdvanced;
/** Map from view id to map from attribute to frequency count */
private Map<String, Map<String, Usage>> mFrequencies =
new HashMap<String, Map<String, Usage>>(100);
private Map<String, Map<String, Usage>> mLayoutAttributeFrequencies =
new HashMap<String, Map<String, Usage>>(100);
private Map<String, String> mTopAttributes = new HashMap<String, String>(100);
private Map<String, String> mTopLayoutAttributes = new HashMap<String, String>(100);
private int mFileVisitCount;
private int mLayoutFileCount;
private File mXmlMetadataFile;
private Analyzer(List<File> directories, File xmlMetadataFile, boolean listAdvanced) {
mDirectories = directories;
mXmlMetadataFile = xmlMetadataFile;
mListAdvanced = listAdvanced;
}
public static void main(String[] args) {
if (args.length < 1) {
System.err.println("Usage: " + Analyzer.class.getSimpleName()
+ " <directory1> [directory2 [directory3 ...]]\n");
System.err.println("Recursively scans for layouts in the given directory and");
System.err.println("computes statistics about attribute frequencies.");
System.exit(-1);
}
File metadataFile = null;
List<File> directories = new ArrayList<File>();
boolean listAdvanced = false;
for (int i = 0, n = args.length; i < n; i++) {
String arg = args[i];
if (arg.equals("--list")) {
// List ALL encountered attributes
listAdvanced = true;
continue;
}
// The -metadata flag takes a pointer to an ADT extra-view-metadata.xml file
// and attempts to insert topAttrs attributes into it (and saves it as same
// file +.mod as an extension). This isn't listed on the usage flag because
// it's pretty brittle and requires some manual fixups to the file afterwards.
if (arg.equals("--metadata")) {
i++;
File file = new File(args[i]);
if (!file.exists()) {
System.err.println(file.getName() + " does not exist");
System.exit(-5);
}
if (!file.isFile() || !file.getName().endsWith(".xml")) {
System.err.println(file.getName() + " must be an XML file");
System.exit(-4);
}
metadataFile = file;
continue;
}
File directory = new File(arg);
if (!directory.exists()) {
System.err.println(directory.getName() + " does not exist");
System.exit(-2);
}
if (!directory.isDirectory()) {
System.err.println(directory.getName() + " is not a directory");
System.exit(-3);
}
directories.add(directory);
}
new Analyzer(directories, metadataFile, listAdvanced).analyze();
}
private void analyze() {
for (File directory : mDirectories) {
scanDirectory(directory);
}
if (mListAdvanced) {
listAdvanced();
}
printStatistics();
if (mXmlMetadataFile != null) {
printMergedMetadata();
}
}
private void scanDirectory(File directory) {
File[] files = directory.listFiles();
if (files == null) {
return;
}
for (File file : files) {
mFileVisitCount++;
if (mFileVisitCount % 50000 == 0) {
System.out.println("Analyzed " + mFileVisitCount + " files...");
}
if (file.isFile()) {
scanFile(file);
} else if (file.isDirectory()) {
// Skip stuff related to tests
if (file.getName().contains("test")) {
continue;
}
// Recurse over subdirectories
scanDirectory(file);
}
}
}
private void scanFile(File file) {
if (file.getName().endsWith(".xml")) {
File parent = file.getParentFile();
if (parent.getName().startsWith("layout")) {
analyzeLayout(file);
}
}
}
private void analyzeLayout(File file) {
mCurrentFile = file;
mLayoutFileCount++;
Document document = null;
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
InputSource is = new InputSource(new StringReader(readFile(file)));
try {
factory.setNamespaceAware(true);
factory.setValidating(false);
DocumentBuilder builder = factory.newDocumentBuilder();
document = builder.parse(is);
analyzeDocument(document);
} catch (ParserConfigurationException e) {
// pass -- ignore files we can't parse
} catch (SAXException e) {
// pass -- ignore files we can't parse
} catch (IOException e) {
// pass -- ignore files we can't parse
}
}
private void analyzeDocument(Document document) {
analyzeElement(document.getDocumentElement());
}
private void analyzeElement(Element element) {
if (element.getTagName().equals("item")) {
// Resource files shouldn't be in the layout/ folder but I came across
// some cases
System.out.println("Warning: found <item> tag in a layout file in "
+ mCurrentFile.getPath());
return;
}
countAttributes(element);
countLayoutAttributes(element);
// Recurse over children
NodeList childNodes = element.getChildNodes();
for (int i = 0, n = childNodes.getLength(); i < n; i++) {
Node child = childNodes.item(i);
if (child.getNodeType() == Node.ELEMENT_NODE) {
analyzeElement((Element) child);
}
}
}
private void countAttributes(Element element) {
String tag = element.getTagName();
Map<String, Usage> attributeMap = mFrequencies.get(tag);
if (attributeMap == null) {
attributeMap = new HashMap<String, Usage>(70);
mFrequencies.put(tag, attributeMap);
}
NamedNodeMap attributes = element.getAttributes();
for (int i = 0, n = attributes.getLength(); i < n; i++) {
Node attribute = attributes.item(i);
String name = attribute.getNodeName();
if (name.startsWith("android:layout_")) {
// Skip layout attributes; they are a function of the parent layout that this
// view is embedded within, not the view itself.
// TODO: Consider whether we should incorporate this info or make statistics
// about that as well?
continue;
}
if (name.equals("android:id")) {
// Skip ids: they are (mostly) unrelated to the view type and the tool
// already offers id editing prominently
continue;
}
if (name.startsWith("xmlns:")) {
// Unrelated to frequency counts
continue;
}
Usage usage = attributeMap.get(name);
if (usage == null) {
usage = new Usage(name);
} else {
usage.incrementCount();
}
attributeMap.put(name, usage);
}
}
private void countLayoutAttributes(Element element) {
String parentTag = element.getParentNode().getNodeName();
Map<String, Usage> attributeMap = mLayoutAttributeFrequencies.get(parentTag);
if (attributeMap == null) {
attributeMap = new HashMap<String, Usage>(70);
mLayoutAttributeFrequencies.put(parentTag, attributeMap);
}
NamedNodeMap attributes = element.getAttributes();
for (int i = 0, n = attributes.getLength(); i < n; i++) {
Node attribute = attributes.item(i);
String name = attribute.getNodeName();
if (!name.startsWith("android:layout_")) {
continue;
}
// Skip layout_width and layout_height; they are mandatory in all but GridLayout so not
// very interesting
if (name.equals("android:layout_width") || name.equals("android:layout_height")) {
continue;
}
Usage usage = attributeMap.get(name);
if (usage == null) {
usage = new Usage(name);
} else {
usage.incrementCount();
}
attributeMap.put(name, usage);
}
}
// Copied from AdtUtils
private static String readFile(File file) {
try {
return readFile(new FileReader(file));
} catch (FileNotFoundException e) {
e.printStackTrace();
}
return null;
}
private static String readFile(Reader inputStream) {
BufferedReader reader = null;
try {
reader = new BufferedReader(inputStream);
StringBuilder sb = new StringBuilder(2000);
while (true) {
int c = reader.read();
if (c == -1) {
return sb.toString();
} else {
sb.append((char)c);
}
}
} catch (IOException e) {
// pass -- ignore files we can't read
} finally {
try {
if (reader != null) {
reader.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
private void printStatistics() {
System.out.println("Analyzed " + mLayoutFileCount
+ " layouts (in a directory trees containing " + mFileVisitCount + " files)");
System.out.println("Top " + ATTRIBUTE_COUNT
+ " for each view (excluding layout_ attributes) :");
System.out.println("\n");
System.out.println(" Rank Count Share Attribute");
System.out.println("=========================================================");
List<String> views = new ArrayList<String>(mFrequencies.keySet());
Collections.sort(views);
for (String view : views) {
String top = processUageMap(view, mFrequencies.get(view));
if (top != null) {
mTopAttributes.put(view, top);
}
}
System.out.println("\n\n\nTop " + ATTRIBUTE_COUNT + " layout attributes (excluding "
+ "mandatory layout_width and layout_height):");
System.out.println("\n");
System.out.println(" Rank Count Share Attribute");
System.out.println("=========================================================");
views = new ArrayList<String>(mLayoutAttributeFrequencies.keySet());
Collections.sort(views);
for (String view : views) {
String top = processUageMap(view, mLayoutAttributeFrequencies.get(view));
if (top != null) {
mTopLayoutAttributes.put(view, top);
}
}
}
private static String processUageMap(String view, Map<String, Usage> map) {
if (map == null) {
return null;
}
if (view.indexOf('.') != -1 && !view.startsWith("android.")) {
// Skip custom views
return null;
}
List<Usage> values = new ArrayList<Usage>(map.values());
if (values.size() == 0) {
return null;
}
Collections.sort(values);
int totalCount = 0;
for (Usage usage : values) {
totalCount += usage.count;
}
System.out.println("\n<" + view + ">:");
if (view.equals("#document")) {
System.out.println("(Set on root tag, probably intended for included context)");
}
int place = 1;
int count = 0;
int prevCount = -1;
float prevPercentage = 0f;
StringBuilder sb = new StringBuilder();
for (Usage usage : values) {
if (count++ >= ATTRIBUTE_COUNT && usage.count < prevCount) {
break;
}
float percentage = 100 * usage.count/(float)totalCount;
if (percentage < THRESHOLD && prevPercentage >= THRESHOLD) {
System.out.println(" -----Less than 10%-------------------------------------");
}
System.out.printf(" %1d. %5d %5.1f%% %s\n", place, usage.count,
percentage, usage.attribute);
prevPercentage = percentage;
if (prevCount != usage.count) {
prevCount = usage.count;
place++;
}
if (percentage >= THRESHOLD /*&& usage.count > 1*/) { // 1:Ignore when not enough data?
if (sb.length() > 0) {
sb.append(',');
}
String name = usage.attribute;
if (name.startsWith("android:")) {
name = name.substring("android:".length());
}
sb.append(name);
}
}
return sb.length() > 0 ? sb.toString() : null;
}
private void printMergedMetadata() {
assert mXmlMetadataFile != null;
String metadata = readFile(mXmlMetadataFile);
if (metadata == null || metadata.length() == 0) {
System.err.println("Invalid metadata file");
System.exit(-6);
}
System.err.flush();
System.out.println("\n\nUpdating layout metadata file...");
System.out.flush();
StringBuilder sb = new StringBuilder((int) (2 * mXmlMetadataFile.length()));
String[] lines = metadata.split("\n");
for (int i = 0; i < lines.length; i++) {
String line = lines[i];
sb.append(line).append('\n');
int classIndex = line.indexOf("class=\"");
if (classIndex != -1) {
int start = classIndex + "class=\"".length();
int end = line.indexOf('"', start + 1);
if (end != -1) {
String view = line.substring(start, end);
if (view.startsWith("android.widget.")) {
view = view.substring("android.widget.".length());
} else if (view.startsWith("android.view.")) {
view = view.substring("android.view.".length());
} else if (view.startsWith("android.webkit.")) {
view = view.substring("android.webkit.".length());
}
String top = mTopAttributes.get(view);
if (top == null) {
System.err.println("Warning: No frequency data for view " + view);
} else {
sb.append(line.substring(0, classIndex)); // Indentation
sb.append("topAttrs=\"");
sb.append(top);
sb.append("\"\n");
}
top = mTopLayoutAttributes.get(view);
if (top != null) {
// It's a layout attribute
sb.append(line.substring(0, classIndex)); // Indentation
sb.append("topLayoutAttrs=\"");
sb.append(top);
sb.append("\"\n");
}
}
}
}
System.out.println("\nTop attributes:");
System.out.println("--------------------------");
List<String> views = new ArrayList<String>(mTopAttributes.keySet());
Collections.sort(views);
for (String view : views) {
String top = mTopAttributes.get(view);
System.out.println(view + ": " + top);
}
System.out.println("\nTop layout attributes:");
System.out.println("--------------------------");
views = new ArrayList<String>(mTopLayoutAttributes.keySet());
Collections.sort(views);
for (String view : views) {
String top = mTopLayoutAttributes.get(view);
System.out.println(view + ": " + top);
}
System.out.println("\nModified XML metadata file:\n");
String newContent = sb.toString();
File output = new File(mXmlMetadataFile.getParentFile(), mXmlMetadataFile.getName() + ".mod");
if (output.exists()) {
output.delete();
}
try {
BufferedWriter writer = new BufferedWriter(new FileWriter(output));
writer.write(newContent);
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Done - wrote " + output.getPath());
}
//private File mPublicFile = new File(location, "data/res/values/public.xml");
private File mPublicFile = new File("/Volumes/AndroidWork/git/frameworks/base/core/res/res/values/public.xml");
private void listAdvanced() {
Set<String> keys = new HashSet<String>(1000);
// Merged usages across view types
Map<String, Usage> mergedUsages = new HashMap<String, Usage>(100);
for (Entry<String,Map<String,Usage>> entry : mFrequencies.entrySet()) {
String view = entry.getKey();
if (view.indexOf('.') != -1 && !view.startsWith("android.")) {
// Skip custom views etc
continue;
}
Map<String, Usage> map = entry.getValue();
for (Usage usage : map.values()) {
// if (usage.count == 1) {
// System.out.println("Only found *one* usage of " + usage.attribute);
// }
// if (usage.count < 4) {
// System.out.println("Only found " + usage.count + " usage of " + usage.attribute);
// }
String attribute = usage.attribute;
int index = attribute.indexOf(':');
if (index == -1 || attribute.startsWith("android:")) {
Usage merged = mergedUsages.get(attribute);
if (merged == null) {
merged = new Usage(attribute);
merged.count = usage.count;
mergedUsages.put(attribute, merged);
} else {
merged.count += usage.count;
}
}
}
}
for (Usage usage : mergedUsages.values()) {
String attribute = usage.attribute;
if (usage.count < 4) {
System.out.println("Only found " + usage.count + " usage of " + usage.attribute);
continue;
}
int index = attribute.indexOf(':');
if (index != -1) {
attribute = attribute.substring(index + 1); // +1: skip ':'
}
keys.add(attribute);
}
List<String> sorted = new ArrayList<String>(keys);
Collections.sort(sorted);
System.out.println("\nEncountered Attributes");
System.out.println("-----------------------------");
for (String attribute : sorted) {
System.out.println(attribute);
}
System.out.println();
}
private static class Usage implements Comparable<Usage> {
public String attribute;
public int count;
public Usage(String attribute) {
super();
this.attribute = attribute;
count = 1;
}
public void incrementCount() {
count++;
}
@Override
public int compareTo(Usage o) {
// Sort by decreasing frequency, then sort alphabetically
int frequencyDelta = o.count - count;
if (frequencyDelta != 0) {
return frequencyDelta;
} else {
return attribute.compareTo(o.attribute);
}
}
@Override
public String toString() {
return attribute + ": " + count;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((attribute == null) ? 0 : attribute.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Usage other = (Usage) obj;
if (attribute == null) {
if (other.attribute != null)
return false;
} else if (!attribute.equals(other.attribute))
return false;
return true;
}
}
}