UrlExtractor.java

/**
 * Copyright (C) 2010 Peter Karich <jetwick_@_pannous_._info>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.jetwick.tw;

import de.jetwick.util.Helper;
import de.jetwick.data.UrlEntry;
import de.jetwick.snacktory.JResult;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

/**
 *
 * @author Peter Karich, peat_hal 'at' users 'dot' sourceforge 'dot' net
 */
public abstract class UrlExtractor extends Extractor {

    private List<UrlEntry> urlEntries = new ArrayList<UrlEntry>(1);

    @Override
    public String toLink(String url, String title) {
        return url;
    }

    @Override
    public UrlExtractor setText(String text) {
        super.setText(text);
        return this;
    }

    public Collection<UrlEntry> getUrlEntries() {
        return urlEntries;
    }

    @Override
    public UrlExtractor run() {
        int index = 0;
        urlEntries.clear();
        for (; (index = text.indexOf("http://", index)) >= 0; index++) {
            String subStr = text.substring(index);

            // url shorteners seems to have a "domain.de" shorter or equal to 11
            // the longest was tinyurl.com the shortest is t.co
            int index2 = subStr.indexOf("/", 7);
            if (index2 < 0)
                index2 = Math.max(0, subStr.indexOf(" ", 7));

            String domain = subStr.substring(0, index2);
            index2 = domain.lastIndexOf(".");
            StringBuilder tmpSb = new StringBuilder();
            int lastIndex = onNewRawUrl(index, tmpSb);

            if (lastIndex > 0) {
                String url = tmpSb.toString();
                JResult res = null;
                try {
                    res = getInfo(url, index2);
                } catch (Exception ex) {
//                    logger.info("Error while resolving:" + url, ex);
                    res = new JResult();
                }
                if (res.getUrl().isEmpty())
                    res.setUrl(url);

                UrlEntry entry = new UrlEntry(index, lastIndex, res.getUrl());
                if (res.getTitle().isEmpty())
                    entry.setResolvedTitle(url);
                else
                    entry.setResolvedTitle(res.getTitle());

                entry.setResolvedDomain(Helper.extractDomain(url));
                urlEntries.add(entry);
            }
        }

        return this;
    }

    public abstract JResult getInfo(String url, int timeout) throws Exception;
}