/*
* Copyright 2010 NCHOVY
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.krakenapps.malwaredomains.impl;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.felix.ipojo.annotations.Component;
import org.apache.felix.ipojo.annotations.Provides;
import org.apache.felix.ipojo.annotations.Validate;
import org.krakenapps.cron.HourlyJob;
import org.krakenapps.malwaredomains.MalwareDomain;
import org.krakenapps.malwaredomains.MalwareDomainService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@HourlyJob
@Component(name = "malware-domain-service")
@Provides
public class DefaultMalwareDomainService implements MalwareDomainService, Runnable {
private final Logger logger = LoggerFactory.getLogger(DefaultMalwareDomainService.class.getName());
private static final String malwareDomain = "http://www.malwaredomains.com";
private static final String updateUrl = malwareDomain + "/updates/";
private static final String totalFileExtension = ".mwd";
private static final String updateFileExtension = ".txt";
private File home;
private ConcurrentMap<String, MalwareDomain> domains;
public static void main(String[] args) throws IOException {
DefaultMalwareDomainService s = new DefaultMalwareDomainService();
s.reload();
}
public DefaultMalwareDomainService() {
setHomeDir(new File(System.getProperty("kraken.data.dir"), "kraken-maldomain/"));
domains = new ConcurrentHashMap<String, MalwareDomain>();
}
@Validate
public void start() {
reload();
}
@Override
public MalwareDomain match(URL url) {
return domains.get(url.getHost());
}
@Override
public void setHomeDir(File path) {
if (home != null && path != null && !path.equals(home))
logger.info("kraken maldomain: home directory is changed to {}", home.getAbsolutePath());
this.home = path;
this.home.mkdirs();
}
@Override
public void run() {
try {
update();
} catch (IOException e) {
// ignore
}
}
@Override
public void update() throws IOException {
logger.trace("kraken maldomain: start malware domains update");
try {
downloadWholeDomains();
downloadNewDomains();
reload();
} catch (IOException e) {
logger.error("kraken maldomain: cannot update malware domains", e);
throw e;
}
}
@Override
public void reload() {
// clear all
domains.clear();
// reload
File[] ruleFiles = getDataFiles(totalFileExtension);
for (File f : ruleFiles) {
try {
loadDomains(f);
} catch (Exception e) {
logger.warn("kraken maldomain: cannot open " + f.getAbsolutePath() + ", skipped", e);
}
}
}
@Override
public int getDomainCount() {
return domains.size();
}
private void downloadWholeDomains() throws IOException {
File[] baseDataFiles = getDataFiles(totalFileExtension);
if (baseDataFiles != null && baseDataFiles.length > 0)
return;
logger.trace("kraken maldomain: downloading malware domain base data");
downloadDomains("files", "domains.txt");
}
private void downloadNewDomains() throws IOException {
logger.trace("kraken maldomain: begin malware domain update");
List<String> targets = getUpdateUrls();
if (targets.isEmpty())
return;
targets = getNeededUpdateUrls(targets);
// download all targets
for (String updatename : targets) {
logger.info("kraken maldomain: downloading update '{}'", updatename);
downloadDomains("updates", updatename);
}
}
private List<String> getNeededUpdateUrls(List<String> targets) {
File[] files = getDataFiles(totalFileExtension);
if (files == null || files.length == 0)
return targets;
for (File f : files) {
String fileName = f.getName().split("\\.")[0] + updateFileExtension;
if (targets.contains(fileName))
targets.remove(fileName);
}
return targets;
}
private List<String> getUpdateUrls() throws MalformedURLException, IOException {
List<String> targets = new ArrayList<String>();
URL url = new URL(updateUrl);
Scanner scanner = null;
try {
scanner = openUrlScanner(url);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (line.contains(updateFileExtension)) {
line = (line.split("href=\"")[1]).split("\">")[0];
targets.add(line);
}
}
} finally {
if (scanner != null)
scanner.close();
}
return targets;
}
private void downloadDomains(String directoryName, String fileName) throws IOException {
URL url = new URL(malwareDomain + "/" + directoryName + "/" + fileName);
PrintWriter pw = null;
Scanner scanner = null;
try {
scanner = openUrlScanner(url);
if (fileName.equals("domains.txt")) {
// domains.txt contains unnecessary 2 lines
scanner.nextLine(); // ignore line 1
scanner.nextLine(); // ignore line 2
}
File f = new File(home, fileName.split("\\.")[0] + totalFileExtension);
pw = new PrintWriter(new FileWriter(f));
while (scanner.hasNextLine()) {
pw.write(scanner.nextLine());
pw.write("\n");
}
} finally {
if (pw != null)
pw.close();
if (scanner != null)
scanner.close();
}
}
private void loadDomains(File f) throws IOException {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd");
Scanner scanner = getFileScanner(f);
try {
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
String[] t = normalize(line.split("\t"));
try {
Date verified = t.length < 6 || t[5].isEmpty() ? null : dateFormat.parse(t[5]);
Date created = t.length < 7 || t[6].isEmpty() ? null : dateFormat.parse(t[6]);
Integer seq = t.length < 8 ? null : Integer.valueOf(t[7]);
MalwareDomain md = new MalwareDomain(t[2], t[3], t[4], verified, created, seq);
domains.put(md.getDomain(), md);
} catch (Exception e) {
logger.trace("kraken maldomain: cannot parse line [" + line + "]", e);
}
}
} finally {
if (scanner != null)
scanner.close();
}
}
private String[] normalize(String[] t) {
int count = 0;
for (String token : t)
if (!token.trim().isEmpty() || token.isEmpty())
count++;
String[] filtered = new String[count];
int i = 0;
for (String token : t)
if (!token.trim().isEmpty() || token.isEmpty())
filtered[i++] = token;
return filtered;
}
private Scanner getFileScanner(File file) throws IOException {
return new Scanner(new FileInputStream(file));
}
private Scanner openUrlScanner(URL url) throws IOException {
URLConnection conn = url.openConnection();
return new Scanner(conn.getInputStream());
}
private File[] getDataFiles(final String extension) {
return home.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(extension);
}
});
}
}