Java Examples for org.jsoup.Connection

public static String getJsonString(String url) {
    String ret = null;
    Connection conn = Jsoup.connect(url);
    Response resp = null;
    conn.ignoreContentType(true);
    try {
        resp = conn.execute();
        return resp.body();
    } catch (IOException e) {
        CLog.i("failed to get json result for %s, %s", url, e.getMessage());
    }
    return ret;
}

public String extract(String url) {
    String t = null;
    try {
        Connection connection = Jsoup.connect(url).userAgent(Constants.USER_AGENT).header("Accept", "text/html,application/xhtml+xml,application/xml").header("Accept-Encoding", "gzip,deflate,sdch").followRedirects(true).timeout(Constants.CONN_TIMEOUT);
        Connection.Response response = connection.execute();
        Document document = response.parse();
        t = extract(document);
    } catch (IOException e) {
        e.printStackTrace();
    }
    return t;
}

public String[] getHinduArticleContent() {
    Document doc;
    String[] result = new String[3];
    String url = mArticleURL;
    try {
        Connection connection = Jsoup.connect(url).userAgent("Mozilla").timeout(10 * 1000);
        Response response = connection.execute();
        if (response == null) {
            Crashlytics.log("Is response null ? " + (null == response));
            return null;
        } else if (response.statusCode() != 200) {
            Crashlytics.log("Received response - " + response.statusCode() + " -- " + response.statusMessage());
            Crashlytics.log("Received response - " + response.body());
            return null;
        }
        doc = connection.get();
        // get Body
        Element bodyElement = doc.body();
        // get Title
        String HinduTitleXPath = ConfigService.getInstance().getTheHinduHead();
        Elements titleElements = bodyElement.select(HinduTitleXPath);
        titleText = titleElements.first().text();
        // get HeaderImageUrl
        mImageURL = getImageURL(bodyElement);
        String HinduArticleXPath = ConfigService.getInstance().getTheHinduBody();
        Elements bodyArticleElements = bodyElement.select(HinduArticleXPath);
        for (Element textArticleElement : bodyArticleElements) {
            bodyText += textArticleElement.text() + "\n\n";
        }
        result[0] = titleText;
        result[1] = mImageURL;
        result[2] = bodyText;
    } catch (IOException e) {
        Crashlytics.logException(e);
        return null;
    } catch (NullPointerException npe) {
        bodyText = null;
        Crashlytics.logException(npe);
        return null;
    } catch (Exception e) {
        Crashlytics.logException(e);
        return null;
    }
    return result;
}

@Override
public void onRegister(CompileScope scope) {
    registerClass(scope, WrapJsoup.class);
    registerWrapperClass(scope, Connection.class, WrapConnection.class);
    registerWrapperClass(scope, Connection.Response.class, WrapConnectionResponse.class);
    registerWrapperClass(scope, Connection.Request.class, WrapConnectionRequest.class);
    registerWrapperClass(scope, Document.class, WrapDocument.class);
    registerWrapperClass(scope, Element.class, WrapElement.class);
    registerWrapperClass(scope, Elements.class, WrapElements.class);
    MemoryOperation.register(new UrlMemoryOperation());
//MemoryOperation.register(new BinaryMemoryOperation());
}

@Test
public void testConnectionOne() {
    String url = "http://www.google.com";
    Connection conn = Jsoup.connect(url).timeout(LONG_TIMEOUT);
    try {
        Document documentObject = conn.get();
        log.debug("Fetching title for: " + url);
        assertEquals("Google", documentObject.title());
    } catch (IOException e) {
        if (e instanceof java.net.SocketException) {
            log.warn("Handling network host exception\n" + e);
        } else {
            fail(e.getMessage());
        }
    }
}

@Override
public void call(Subscriber<? super SNFeed> subscriber) {
    try {
        Connection conn = mJsoupConnector.newJsoupConnection(url);
        Document doc = conn.get();
        SNFeedParser parser = new SNFeedParser();
        SNFeed feed = parser.parseDocument(doc);
        subscriber.onNext(feed);
        subscriber.onCompleted();
    } catch (Exception e) {
        subscriber.onError(e);
    }
}

private static Document getCommentsDocument(UserPrefs data, long storyId) throws IOException {
    Connection con;
    if (data.isLoggedIn()) {
        con = ConnectionManager.authConnect(ConnectionManager.ITEMS_URL + Long.toString(storyId), data.getUserCookie());
    } else {
        con = ConnectionManager.anonConnect(ConnectionManager.ITEMS_URL + Long.toString(storyId));
    }
    return con.get();
}

/* (non-Javadoc)
	 * @see io.devyse.scheduler.parse.jsoup.AbstractParser#parse(org.jsoup.nodes.Document)
	 */
@Override
protected void parse(Document document) throws IOException {
    //parse the form from the document
    FormElement form = parseForm(document);
    //prepare a connection from the form
    Connection connection = prepareForm(form);
    connection.timeout(this.getTimeout());
    //build the form parameters
    Collection<KeyVal> data = buildFormParameters(form, connection);
    //submit the form and store the resulting document
    submitForm(connection, data);
}

protected Document httpGet(Connection conn) {
    try {
        // TODO: execute network request in a separate thread pool
        return conn.get();
    } catch (IOException e) {
        if (e instanceof HttpStatusException) {
            HttpStatusException statusException = (HttpStatusException) e;
            if (statusException.getStatusCode() == 503) {
                try {
                    Thread.sleep(backoffTime);
                } catch (InterruptedException e1) {
                    throw new RuntimeException(e1);
                }
            }
        }
        throw new RuntimeException(e);
    }
}

@Override
@BotCommand("fed")
public String find(@Option(name = "키워드", help = "검색할 단어") String keyword) {
    String result = "";
    if (keyword.equals("")) {
        result = "키워드를 지정하지 않았습니다";
        return result;
    }
    try {
        String url = "http://rpmfind.net/linux/rpm2html/search.php?query=" + keyword + "&submit=Search";
        Connection conn = Jsoup.connect(url);
        conn.timeout(5000);
        Elements tables = conn.get().select("table");
        if (tables.size() < 2) {
            result = "[Fedora] 결과가 없습니다";
            return result;
        }
        System.out.println(tables.get(1).select("tbody>tr").toString());
        Iterator<Element> row = tables.get(1).select("tbody>tr").iterator();
        boolean firstrow = false;
        String pkgname = "";
        String description = "";
        String dist = "";
        String NamenVer = "";
        while (row.hasNext()) {
            if (!firstrow) {
                // dummy
                row.next();
                firstrow = true;
                continue;
            }
            Elements e = row.next().select("td");
            System.out.println(e);
            pkgname = e.get(0).select("a").get(0).text();
            description = e.get(1).text();
            dist = e.get(2).text();
            if (dist.split("\\s")[0].equals("Fedora")) {
                int pkg_ver;
                try {
                    pkg_ver = Integer.parseInt(dist.split("\\s")[1]);
                } catch (Exception ee) {
                    continue;
                }
                description += " in the Fedora " + pkg_ver;
                String[] spl = pkgname.split("\\.");
                int i = 0;
                // exclude .fcxx.html string
                for (; !spl[i].contains("fc"); i++) {
                    if (i != 0)
                        NamenVer += ".";
                    NamenVer += spl[i];
                }
                // split pkgname and version
                spl = NamenVer.split("\\-");
                String name = spl[0];
                String ver = "";
                i = 1;
                while (!(spl[i].charAt(0) >= '0' && spl[i].charAt(0) <= '9')) {
                    name += "-" + spl[i++];
                }
                int initial_veridx = i;
                while (i < spl.length) {
                    if (initial_veridx != i)
                        ver += "-";
                    ver += spl[i++];
                }
                NamenVer = name + "  " + ver;
                break;
            }
        }
        // need to give result into result.
        if (!dist.contains("Fedora")) {
            result = "[Fedora] 결과가 없습니다";
        } else {
            result = NamenVer + " : " + description;
        }
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        result = "오류: " + e.getMessage();
    }
    return result;
}

public void wallParse(int groupId) {
    String wallURL = "http://vk.com/wall-" + groupId;
    try {
        Document wallDome = Jsoup.connect(wallURL).userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36").get();
        Element date = wallDome.select("div.reply_link_wrap").get(0);
        String postLink = date.attr("id");
        String[] postLastIdString = postLink.split("_");
        int postIdCurrent = Integer.parseInt(postLastIdString[2]);
        String postIdCurrentString;
        for (int i = 1; i <= 100; i++) {
            postIdCurrentString = Integer.toString(postIdCurrent);
            Connection.Response response = Jsoup.connect("http://vk.com/wall-" + groupId + "_" + postIdCurrentString).userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36").timeout(10000).execute();
            if (response.statusCode() == 200) {
                String Result = postParse(groupId + "_" + postIdCurrentString);
                if (Result != null) {
                    System.out.println(Result);
                }
                --postIdCurrent;
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}

/**
	 * The workhorse for the class. Download the provided image uri. If there is
	 * a problem an exception is raised and the calling method is expected to
	 * handle it in an appropriate manner.
	 * <p>
	 * The uri is presumed to be an http page containing elements like the
	 * following: <code>
	 * <img 
	 *   src="http://somethingscrawlinginmyhair.com/wp-content/uploads/2012/10/Wasp.spider.and_.nearby.male_.jpg" 
	 *   alt="" title="Wasp.spider.and.nearby.male" 
	 *   width="600" 
	 *   height="334" 
	 *   class="alignnone size-full wp-image-7431" />
	 *   </code>
	 * 
	 * <p>
	 * The url is indicated by the
	 * <p>
	 * An allowance is made for a uri of a content provider, which have a shema
	 * of "content".
	 * 
	 * @param master
	 * 
	 * @param uri
	 *            the thing to download
	 */
protected String downloadImages(DownloadService master, Uri uri) throws FailedDownload, FileNotFoundException, IOException {
    logger.debug("download images:");
    master.clearImages(uri);
    final String mainUrl = uri.toString();
    final String scheme = uri.getScheme();
    if ("http".equals(scheme)) {
        try {
            final Connection channel = Jsoup.connect(mainUrl);
            final Document doc = channel.get();
            final String title = doc.title();
            final Elements imageUrlSet = doc.select("img[src]");
            int ordinal = 1;
            for (Element imageUrlElement : imageUrlSet) {
                final String imageUrlStr = imageUrlElement.attr("src");
                try {
                    final URL imageUrl = new URL(imageUrlStr);
                    final InputStream is = imageUrl.openStream();
                    final Bitmap bitmap = BitmapFactory.decodeStream(is);
                    logger.debug("bitmap size [{}:{}]", bitmap.getWidth(), bitmap.getHeight());
                    if (bitmap.getWidth() < MINIMUM_WIDTH) {
                        logger.warn("bitmap too narrow {} px", bitmap.getWidth());
                        continue;
                    }
                    if (bitmap.getHeight() < MINIMUM_HEIGHT) {
                        logger.warn("bitmap too short {} px", bitmap.getHeight());
                        continue;
                    }
                    master.storeBitmap(uri, ordinal, bitmap);
                    ordinal++;
                } catch (UnknownHostException ex) {
                    logger.warn("download failed bad host {}", imageUrlStr, ex);
                } catch (MalformedURLException ex) {
                    logger.warn("the request URL is not a HTTP or HTTPS URL, or is otherwise malformed {}", imageUrlStr, ex);
                } catch (HttpStatusException ex) {
                    logger.warn("the response is not OK and HTTP response errors are not ignored {}", imageUrlStr, ex);
                } catch (UnsupportedMimeTypeException ex) {
                    logger.warn("the response mime type is not supported and those errors are not ignored {}", imageUrlStr, ex);
                } catch (SocketTimeoutException ex) {
                    logger.warn("the connection times out {}", imageUrlStr, ex);
                } catch (IOException ex) {
                    logger.warn("download failed  {}", imageUrlStr, ex);
                }
            }
            logger.info("loaded page=<{}>", title);
            return mainUrl;
        } catch (UnknownHostException ex) {
            logger.warn("download failed bad host {}", uri, ex);
            throw new FailedDownload(uri, this.getResources().getText(R.string.error_downloading_url));
        } catch (MalformedURLException ex) {
            logger.warn("the request URL is not a HTTP or HTTPS URL, or is otherwise malformed {}", uri, ex);
            throw new FailedDownload(uri, this.getResources().getText(R.string.error_downloading_url));
        } catch (HttpStatusException ex) {
            logger.warn("the response is not OK and HTTP response errors are not ignored {}", uri, ex);
            throw new FailedDownload(uri, this.getResources().getText(R.string.error_downloading_url));
        } catch (UnsupportedMimeTypeException ex) {
            logger.warn("the response mime type is not supported and those errors are not ignored {}", uri, ex);
            throw new FailedDownload(uri, this.getResources().getText(R.string.error_downloading_url));
        } catch (SocketTimeoutException ex) {
            logger.warn("the connection times out {}", uri, ex);
            throw new FailedDownload(uri, this.getResources().getText(R.string.error_downloading_url));
        } catch (IOException ex) {
            logger.warn("download failed  {}", uri, ex);
            throw new FailedDownload(uri, this.getResources().getText(R.string.error_downloading_url));
        }
    } else {
    }
    return mainUrl;
}

/**
     * 下载课程
     * @param entranceURL 课程入�页�
     * @param type 课程类型
     * @param path �存到本地的路径
     * @param containEntrance 是�下载入�页�上的课程
     * @param justOriginalName 是å?¦ä½¿ç”¨åŽŸæ?¥çš„文件å??称ä¿?存文件
     */
public static void download(String entranceURL, String type, String path, boolean containEntrance, boolean justOriginalName) {
    int timeout = 300000;
    if (!entranceURL.endsWith("/")) {
        entranceURL += "/";
    }
    Set<String> urls = new HashSet<>();
    boolean ok = false;
    int limit = 0;
    while (!ok && (limit++) < 3) {
        try {
            System.out.println("�" + type + "】*** connect " + entranceURL);
            for (Element element : Jsoup.connect(entranceURL).timeout(timeout).get().select("a")) {
                String href = element.attr("href").trim();
                if (!href.startsWith("http")) {
                    if (href.startsWith("/")) {
                        href = "http://www.bbc.co.uk" + href;
                    } else {
                        href = entranceURL + href;
                    }
                }
                if (href.startsWith(entranceURL) && (!href.equals(entranceURL) || containEntrance)) {
                    urls.add(href);
                }
            }
            ok = true;
        } catch (Exception e) {
            System.out.println(e.getMessage() + " retry...");
        }
    }
    AtomicInteger i = new AtomicInteger(1);
    Set<String> resources = new HashSet<>();
    urls.stream().sorted().forEach( url -> {
        boolean success = false;
        int times = 0;
        while (!success && (times++) < 3) {
            try {
                System.out.println(i.get() + "�connect " + url);
                for (Element element : Jsoup.connect(url).timeout(timeout).get().select("a")) {
                    String href = element.attr("href").trim();
                    if (href.endsWith(".mp3") || href.endsWith(".wav") || href.endsWith(".mp4") || href.endsWith(".pdf")) {
                        if (!href.startsWith("http")) {
                            if (href.startsWith("/")) {
                                href = "http://www.bbc.co.uk" + href;
                            } else {
                                String[] attr = url.split("/");
                                href = url.substring(0, url.length() - attr[attr.length - 1].length()) + href;
                            }
                        }
                        resources.add(href);
                    }
                }
                i.incrementAndGet();
                success = true;
            } catch (Exception e) {
                System.out.println(e.getMessage() + " retry...");
            }
        }
    });
    AtomicInteger j = new AtomicInteger(1);
    count += resources.size();
    resources.stream().sorted().forEach( resource -> {
        boolean success = false;
        int times = 0;
        while (!success && (times++) < 3) {
            try {
                String[] attr = resource.split("/");
                String fileName = attr[attr.length - 2] + "_" + attr[attr.length - 1].replace(attr[attr.length - 2], "");
                if (attr[attr.length - 1].endsWith(attr[attr.length - 2])) {
                    fileName = attr[attr.length - 1];
                }
                fileName = fileName.replace("_download", "");
                if (justOriginalName) {
                    fileName = attr[attr.length - 1];
                }
                System.out.println(resources.size() + "/" + j.get() + "�find resource: " + resource);
                Path dir = Paths.get(path, type);
                if (!Files.exists(dir)) {
                    dir.toFile().mkdirs();
                }
                Path out = Paths.get(path, type, fileName);
                if (!Files.exists(out)) {
                    Connection.Response response = Jsoup.connect(resource).maxBodySize(0).ignoreContentType(true).timeout(timeout).execute();
                    Files.write(out, response.bodyAsBytes());
                    System.out.println(resources.size() + "/" + j.get() + "�save resource to: " + out);
                } else {
                    System.out.println(resources.size() + "/" + j.get() + "�resource exist, don't need to download");
                }
                j.incrementAndGet();
                success = true;
            } catch (Exception e) {
                System.out.println(e.getMessage() + " retry...");
            }
        }
    });
}

@Override
public void rip() throws IOException {
    String gid = getGID(this.url);
    String theurl = "http://newsfilter.org/gallery/" + gid;
    logger.info("Loading " + theurl);
    Connection.Response resp = Jsoup.connect(theurl).timeout(5000).referrer("").userAgent(USER_AGENT).method(Connection.Method.GET).execute();
    Document doc = resp.parse();
    Elements thumbnails = doc.select("#galleryImages .inner-block img");
    for (Element thumb : thumbnails) {
        String thumbUrl = thumb.attr("src");
        String picUrl = thumbUrl.replace("thumbs/", "");
        addURLToDownload(new URL(picUrl));
    }
    waitForThreads();
}

/**
     * 点赞
     *
     * @param statusId
     * @param like
     * @param cookie
     * @return
     * @throws TaskException
     */
public LikeResultBean doLike(String statusId, boolean like, String cookie) throws TaskException {
    try {
        String url = like ? "http://m.weibo.cn/attitudesDeal/add" : "http://m.weibo.cn/attitudesDeal/delete";
        Map<String, String> cookieMap = new HashMap<String, String>();
        String[] cookieValues = cookie.split(";");
        for (String cookieValue : cookieValues) {
            String key = cookieValue.split("=")[0];
            String value = cookieValue.split("=")[1];
            cookieMap.put(key, value);
        }
        //            Logger.d(WeiboClientActivity.TAG, cookieMap);
        Connection connection = Jsoup.connect(url);
        connection.userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:33.0) Gecko/20100101 Firefox/33.0").referrer("http://m.weibo.cn/").cookies(cookieMap).data("id", statusId).method(Connection.Method.POST);
        if (like)
            connection.data("attitude", "heart");
        String body = connection.execute().body();
        if (!TextUtils.isEmpty(body)) {
            Logger.d(ABizLogic.TAG, body);
            if (body.indexOf("http://passport.weibo.cn/sso/crossdomain") != -1)
                throw new TaskException("-100", "未登录");
            LikeResultBean likeBean = JSON.parseObject(body, LikeResultBean.class);
            if (likeBean.getOk() == 1) {
                return likeBean;
            } else if (likeBean.getOk() == -100) {
                throw new TaskException("-100", "未登录");
            } else {
                throw new TaskException("", likeBean.getMsg());
            }
        }
    } catch (Exception e) {
        if (e instanceof TaskException)
            throw (TaskException) e;
        e.printStackTrace();
    }
    throw new TaskException(TaskException.TaskError.timeout.toString());
}

public Connection data(String... keyvals) {
    Validate.notNull(keyvals, "Data key value pairs must not be null");
    Validate.isTrue(keyvals.length % 2 == 0, "Must supply an even number of key value pairs");
    for (int i = 0; i < keyvals.length; i += 2) {
        String key = keyvals[i];
        String value = keyvals[i + 1];
        Validate.notEmpty(key, "Data key must not be empty");
        Validate.notNull(value, "Data value must not be null");
        req.data(KeyVal.create(key, value));
    }
    return this;
}

@Override
protected String doInBackground(String... strings) {
    Connection.Response response = null;
    try {
        response = Jsoup.connect(strings[0]).execute();
    } catch (IOException e) {
        e.printStackTrace();
    }
    if (response != null) {
        int statusCode = response.statusCode();
        if (statusCode == 200) {
            Document doc = null;
            try {
                doc = response.parse();
            } catch (IOException e) {
                e.printStackTrace();
            }
            Elements metaElems = doc.select("#plato");
            for (Element metaElem : metaElems) {
                platosMenu = new ArrayList<>();
                Charset utf8charset = Charset.forName("UTF-8");
                Charset iso88591charset = Charset.forName("ISO-8859-1");
                String diaplato = metaElem.getElementById("diaplato").getElementById("fechaplato").text();
                diaplato = diaplato.replaceAll("\\<.*?>", "");
                diaplato = diaplato.replaceAll(" +", " ");
                Element menuDia = metaElem.getElementById("platos");
                Elements platos = menuDia.children();
                for (Element plato : platos) {
                    platosMenu.add(plato.html().replaceAll("\\<.*?>", "").replaceAll(" +", " "));
                }
                Dia d = new Dia(diaplato, platosMenu);
                semana.add(d);
            }
        } else {
            platosMenu = new ArrayList<>();
            platosMenu.add("No hay conexión");
            Dia d = new Dia("Algo falló", platosMenu);
            semana.add(d);
        }
    } else {
        platosMenu = new ArrayList<>();
        platosMenu.add("No hay conexión");
        Dia d = new Dia("Algo falló", platosMenu);
        semana.add(d);
    }
    return "";
}

public static String parseChatUrl(String stringToParse, User sender) {
    try {
        Connection conn = Jsoup.connect(stringToParse);
        conn.followRedirects(true).userAgent("FoxBot // http://foxbot.foxdev.co // Seeing this? It means your web address was posted on IRC and FoxBot is getting page info (title, size, content type) to send to the channel. Nothing to worry about.").timeout(3000).maxBodySize(100000).ignoreContentType(true);
        Connection.Response response = conn.execute();
        Document doc = response.parse();
        String size = response.header("Content-Length") == null ? "Unknown" : (Integer.parseInt(response.header("Content-Length")) / 1024) + "kb";
        String contentType = response.contentType().contains(";") ? response.contentType().split(";")[0] : response.contentType();
        if (response.statusCode() != 200 && response.statusCode() != 302 && response.statusCode() != 301) {
            return colourise(String.format("(%s's URL) &cError: &r%s %s ", munge(sender.getNick()), response.statusCode(), response.statusMessage()));
        }
        if (!contentType.contains("html")) {
            return colourise(String.format("(%s's URL) &2Content Type: &r%s &2Size: &r%s", munge(sender.getNick()), contentType, size));
        }
        String title = doc.title() == null || doc.title().isEmpty() ? "No title found" : doc.title();
        if (stringToParse.matches("^https?://(www\\.)?youtube\\.com/watch.*")) {
            title = doc.select("#eow-title").first().text();
            String views = doc.select("#watch7-views-info > div.watch-view-count").first().text();
            String likes = doc.select("span.likes-count").first().text();
            String dislikes = doc.select("span.dislikes-count").first().text();
            String uploader = doc.select("a.g-hovercard.yt-uix-sessionlink.yt-user-name").first().text();
            return colourise(String.format("(%s's URL) &2Title: &r%s &2Uploader: &r%s &2Views: &r%s &2Rating: &a%s&r/&c%s", munge(sender.getNick()), StringEscapeUtils.unescapeHtml4(title), uploader, views, likes, dislikes));
        }
        if (stringToParse.matches("^https?://(www\\.)?reddit\\.com/r/.*/comments/.*")) {
            String poster = doc.select("p.tagline").select("a.author").text().split(" ")[0];
            String comments = doc.select("a.comments").first().text().split(" ")[0];
            String score = doc.select("div.score").select("span.number").first().text().split(" ")[0];
            return colourise(String.format("(%s's URL) &2Title: &r%s &2Poster: &r%s &2Comments: &r%s &2Score: &6%s", munge(sender.getNick()), StringEscapeUtils.unescapeHtml4(title), poster, comments, score));
        }
        return colourise(String.format("(%s's URL) &2Title: &r%s &2Content Type: &r%s &2Size: &r%s", munge(sender.getNick()), StringEscapeUtils.unescapeHtml4(title), contentType, size));
    } catch (IllegalArgumentException ignored) {
    } catch (Exception ex) {
        foxbot.getLogger().error("Error occurred while parsing URL", ex);
    }
    return null;
}

public Connection data(String... keyvals) {
    Validate.notNull(keyvals, "Data key value pairs must not be null");
    Validate.isTrue(keyvals.length % 2 == 0, "Must supply an even number of key value pairs");
    for (int i = 0; i < keyvals.length; i += 2) {
        String key = keyvals[i];
        String value = keyvals[i + 1];
        Validate.notEmpty(key, "Data key must not be empty");
        Validate.notNull(value, "Data value must not be null");
        req.data(KeyVal.create(key, value));
    }
    return this;
}

@Override
public Optional<URL> findFullText(BibEntry entry) throws IOException {
    Objects.requireNonNull(entry);
    Optional<URL> pdfLink = Optional.empty();
    Optional<DOI> doi = entry.getField(FieldName.DOI).flatMap(DOI::parse);
    if (doi.isPresent()) {
        String sciLink = doi.get().getURIAsASCIIString();
        // follow all redirects and scan for a single pdf link
        if (!sciLink.isEmpty()) {
            try {
                Connection connection = Jsoup.connect(sciLink);
                // pretend to be a browser (agent & referrer)
                connection.userAgent(URLDownload.USER_AGENT);
                connection.referrer("http://www.google.com");
                connection.followRedirects(true);
                connection.ignoreHttpErrors(true);
                // some publishers are quite slow (default is 3s)
                connection.timeout(5000);
                Document html = connection.get();
                // scan for PDF
                Elements elements = html.body().select("a[href]");
                List<Optional<URL>> links = new ArrayList<>();
                for (Element element : elements) {
                    String href = element.attr("abs:href").toLowerCase(Locale.ENGLISH);
                    String hrefText = element.text().toLowerCase(Locale.ENGLISH);
                    // See https://github.com/lehner/LocalCopy for more scrape ideas
                    if ((href.contains("pdf") || hrefText.contains("pdf")) && new URLDownload(href).isPdf()) {
                        links.add(Optional.of(new URL(href)));
                    }
                }
                // return if only one link was found (high accuracy)
                if (links.size() == 1) {
                    LOGGER.info("Fulltext PDF found @ " + sciLink);
                    pdfLink = links.get(0);
                }
            } catch (IOException e) {
                LOGGER.warn("DoiResolution fetcher failed: ", e);
            }
        }
    }
    return pdfLink;
}

protected String getCSSSource(String htmlSource) throws Exception {
    org.jsoup.nodes.Document htmlDocument = Jsoup.parse(htmlSource);
    Elements elements = htmlDocument.select("link[type=text/css]");
    StringBuilder sb = new StringBuilder();
    for (Element element : elements) {
        String href = element.attr("href");
        if (!href.contains(PropsValues.PORTAL_URL)) {
            href = PropsValues.PORTAL_URL + href;
        }
        Connection connection = Jsoup.connect(href);
        org.jsoup.nodes.Document document = connection.get();
        sb.append(document.text());
        sb.append("\n");
    }
    return sb.toString();
}

@Override
protected Boolean doInBackground(Void... objects) {
    boolean isRefreshFromTop = (1 == mPage);
    try {
        ArrayList<Image> images = new ArrayList<Image>();
        Document doc;
        int pid;
        String time;
        String content;
        String author;
        String temp = "";
        if (isRefreshFromTop) {
            mtDataHelper.deleteAll();
            miDataHelper.deleteAll();
        }
        Connection.Response res = Request.execute(String.format(Api.POST_CONTENT, tid, next), "Mozilla", (Map<String, String>) Athority.getSharedPreference().getAll(), Connection.Method.GET);
        Athority.addCookies(res.cookies());
        doc = res.parse();
        Elements inputs = doc.getElementsByTag("input");
        for (Element input : inputs) {
            if (input.attr("name").equals("formhash")) {
                formhash = input.attr("value");
            }
        }
        Elements plhins = doc.getElementsByClass("plhin");
        posts.clear();
        for (Element plhin : plhins) {
            pid = Integer.valueOf(plhin.id().substring("pid".length()));
            Elements xw1s = plhin.getElementsByClass("xw1");
            author = xw1s.text();
            Elements t_fs = plhin.getElementsByClass("t_f");
            for (Element t_f : t_fs) {
                temp = t_f.getElementsByTag("blockquote").text();
                break;
            }
            content = t_fs.text();
            content = content.substring(temp.length());
            content = temp + "%%%%%" + content;
            time = plhin.getElementById("authorposton" + pid).text();
            Elements e_pics = plhin.getElementsByClass("zoom");
            images.clear();
            if (e_pics != null) {
                for (Element e_pic : e_pics) {
                    if (e_pic.attr("zoomfile").equals("")) {
                        if (e_pic.attr("file").equals("")) {
                            continue;
                        } else {
                            images.add(new Image(e_pic.attr("file"), pid));
                        }
                    } else {
                        images.add(new Image(e_pic.attr("zoomfile"), pid));
                    }
                }
            }
            if (images.size() == 0) {
                posts.add(new Post(fid, tid, pid, title, content, time, 0, 0, author, null));
            } else {
                posts.add(new PostWithPic(fid, tid, pid, title, content, time, 1, 0, author, null));
                miDataHelper.bulkInsert(images);
            }
        }
        if (!isloadmaxpage) {
            Element pgt = doc.getElementById("pgt");
            if (pgt == null) {
                maxPage = 1;
                mPage = 1;
                isloadmaxpage = true;
                return false;
            } else {
                if (pgt.getElementsByTag("strong").text().equals("")) {
                    maxPage = 1;
                    mPage = 1;
                } else {
                    maxPage = Integer.valueOf(pgt.getElementsByTag("strong").text());
                    mPage = maxPage;
                    Elements page_numbers = pgt.getElementsByTag("a");
                    int now_number = 1;
                    String str_now_number;
                    for (Element page_number : page_numbers) {
                        str_now_number = page_number.text();
                        if (str_now_number.startsWith("... ")) {
                            now_number = Integer.valueOf(str_now_number.substring("... ".length()));
                        } else if (str_now_number.equals("下一页")) {
                            continue;
                        } else if (str_now_number.equals("返回列表")) {
                            continue;
                        } else if (str_now_number.equals("")) {
                            continue;
                        } else {
                            now_number = Integer.valueOf(str_now_number);
                        }
                        if (now_number > maxPage) {
                            maxPage = now_number;
                        }
                    }
                    isloadmaxpage = true;
                }
            }
        }
        mtDataHelper.bulkInsert(posts);
    } catch (IOException e) {
        e.printStackTrace();
    }
    return isRefreshFromTop;
}

/**
     * Before using it please set the FOLDER variable with the path where you
     * want to create your extract html files.
     *
     * @param args the command line arguments
     */
public static void main(String[] args) {
    //      first boucle for is for the theme number
    for (int i = 1; i < MAX_THEME_NUMBER; i++) {
        // second boucle for is for the critere number
        for (int j = 1; j < MAX_CRITERE_NUMBER; j++) {
            // third boucle for is for the test number
            for (int k = 1; k < MAX_TEST_NUMBER; k++) {
                URL url = null;
                try {
                    Connection connection = Jsoup.connect(PREFIX_URL_TO_REFERENTIAL + i + "-" + j + "-" + k);
                    Connection.Response resp = connection.response();
                    if (resp.statusCode() != 404) {
                        url = new URL(PREFIX_URL_TO_REFERENTIAL + i + "-" + j + "-" + k);
                        Document doc = Jsoup.parse(url, 4000);
                        System.out.println(doc.title());
                        Elements summary = doc.select(".content.clear-block");
                        FileUtils.writeStringToFile(new File(FOLDER + "/RuleDesign/Rule-" + i + "-" + j + "-" + k + ".html"), summary.html());
                    }
                } catch (MalformedURLException ex) {
                    System.out.println("URL MAL FORMEE");
                } catch (IOException ex) {
                    if (url != null) {
                        System.out.println("URL 404 : " + url.toString());
                    } else {
                        System.out.println("EMPTY URL");
                    }
                }
            }
        }
    }
}

@Override
public void call(Subscriber<? super OperationResponse> subscriber) {
    if (voteUrl.equals(VoteUrlParser.EMPTY)) {
        subscriber.onNext(OperationResponse.FAILURE);
    }
    try {
        ConnectionProvider connectionProvider = Inject.connectionProvider();
        Connection.Response response = connectionProvider.voteConnection(voteUrl).execute();
        if (response.statusCode() == 200) {
            if (response.body() == null) {
                subscriber.onError(new Throwable(""));
            }
            Document doc = response.parse();
            String text = doc.text();
            if (text.equals(BAD_UPVOTE_RESPONSE)) {
                subscriber.onNext(OperationResponse.FAILURE);
            } else {
                subscriber.onNext(OperationResponse.SUCCESS);
            }
        } else {
            subscriber.onNext(OperationResponse.FAILURE);
        }
    } catch (IOException e) {
        subscriber.onError(e);
    }
}

@Override
public void widgetSelected(SelectionEvent se) {
    String link = browser.getUrl();
    if (link.startsWith(HOME_URL))
        return;
    FileDialog dlg = new FileDialog(shell, SWT.SAVE);
    dlg.setFilterExtensions(new String[] { "*.png" });
    dlg.setFilterNames(new String[] { "PNG (*.png)" });
    String filename = dlg.open();
    if (filename == null)
        return;
    Cursor cursor = shell.getCursor();
    shell.setCursor(new Cursor(shell.getDisplay(), SWT.CURSOR_WAIT));
    try {
        Thread.sleep(1000);
        String url = "http://api.snapito.com/?url=" + java.net.URLEncoder.encode(link, "UTF-8");
        org.jsoup.Connection conn = org.jsoup.Jsoup.connect(link);
        conn.timeout(60000);
        conn.ignoreContentType(true);
        conn.userAgent("Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1");
        conn.execute();
        url = "http://cache.snapito.com/api/image?_cache_redirect=true&url=" + java.net.URLEncoder.encode(link, "UTF-8") + "&type=png";
        conn.url(url);
        byte[] data = conn.execute().bodyAsBytes();
        OutputStream os = new FileOutputStream(filename);
        os.write(data);
        os.close();
    } catch (Exception e) {
        logger.error("", e);
        UITool.errorBox(shell, e);
    }
    shell.setCursor(cursor);
}

@Test
public void fetchBaidu() throws IOException {
    Connection.Response res = Jsoup.connect("http://www.baidu.com/").timeout(10 * 1000).execute();
    Document doc = res.parse();
    assertEquals("GBK", doc.outputSettings().charset().displayName());
    assertEquals("GBK", res.charset());
    assert (res.hasCookie("BAIDUID"));
    assertEquals("text/html;charset=gbk", res.contentType());
}

public static ArrayList<Lyrics> search(String query) {
    ArrayList<Lyrics> results = new ArrayList<>();
    query = Normalizer.normalize(query, Normalizer.Form.NFD).replaceAll("\\p{InCombiningDiacriticalMarks}+", "");
    JsonObject response = null;
    try {
        URL queryURL = new URL(String.format("http://api.genius.com/search?q=%s", URLEncoder.encode(query, "UTF-8")));
        Connection connection = Jsoup.connect(queryURL.toExternalForm()).header("Authorization", "Bearer " + Keys.GENIUS).timeout(0).ignoreContentType(true);
        Document document = connection.userAgent(Net.USER_AGENT).get();
        response = new JsonParser().parse(document.text()).getAsJsonObject();
    } catch (JsonSyntaxException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    if (response == null || response.getAsJsonObject("meta").get("status").getAsInt() != 200)
        return results;
    JsonArray hits = response.getAsJsonObject("response").getAsJsonArray("hits");
    int processed = 0;
    while (processed < hits.size()) {
        JsonObject song = hits.get(processed).getAsJsonObject().getAsJsonObject("result");
        String artist = song.getAsJsonObject("primary_artist").get("name").getAsString();
        String title = song.get("title").getAsString();
        String url = "http://genius.com/songs/" + song.get("id").getAsString();
        Lyrics l = new Lyrics(Lyrics.SEARCH_ITEM);
        l.setArtist(artist);
        l.setTitle(title);
        l.setURL(url);
        l.setSource("Genius");
        results.add(l);
        processed++;
    }
    return results;
}

public static MAHRequestResult requestPrograms(final Context context, String url) throws IOException {
    Document doc = Jsoup.connect(url.trim()).ignoreContentType(true).timeout(3000).header(// .header("Host", "85.132.44.28")
    "Connection", "keep-alive").header(// .header("Content-Length", "111")
    "Cache-Control", "max-age=0").header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8").header(// .header("Origin", "http://85.132.44.28")
    "User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36").header("Content-Type", "application/x-www-form-urlencoded").header("Referer", url.trim()).header(// This is Needed
    "Accept-Encoding", "gzip,deflate,sdch").header("Accept-Language", "en-US,en;q=0.8,ru;q=0.6").get();
    String jsonStr = doc.body().text();
    Log.i(Constants.LOG_TAG_MAH_ADS, "Programlist json = " + jsonStr);
    Utils.writeStringToCache(context, jsonStr);
    MAHRequestResult requestResult = jsonToProgramList(jsonStr);
    requestResult.setReadFromWeb(true);
    return requestResult;
}

public static ArrayList<Lyrics> search(String query) {
    ArrayList<Lyrics> results = new ArrayList<>();
    query = Normalizer.normalize(query, Normalizer.Form.NFD).replaceAll("\\p{InCombiningDiacriticalMarks}+", "");
    JsonObject response = null;
    try {
        URL queryURL = new URL(String.format("http://api.genius.com/search?q=%s", URLEncoder.encode(query, "UTF-8")));
        Connection connection = Jsoup.connect(queryURL.toExternalForm()).header("Authorization", "Bearer " + Config.GENIUS).timeout(0).ignoreContentType(true);
        Document document = connection.userAgent(Net.USER_AGENT).get();
        response = new JsonParser().parse(document.text()).getAsJsonObject();
    } catch (Exception e) {
        e.printStackTrace();
    }
    if (response == null || response.getAsJsonObject("meta").get("status").getAsInt() != 200)
        return results;
    JsonArray hits = response.getAsJsonObject("response").getAsJsonArray("hits");
    int processed = 0;
    while (processed < hits.size()) {
        JsonObject song = hits.get(processed).getAsJsonObject().getAsJsonObject("result");
        String artist = song.getAsJsonObject("primary_artist").get("name").getAsString();
        String title = song.get("title").getAsString();
        String url = "http://genius.com/songs/" + song.get("id").getAsString();
        Lyrics l = new Lyrics(Lyrics.SEARCH_ITEM);
        l.setArtist(artist);
        l.setTitle(title);
        l.setURL(url);
        l.setSource("Genius");
        results.add(l);
        processed++;
    }
    return results;
}

public static ArrayList<Lyrics> search(String query) {
    ArrayList<Lyrics> results = new ArrayList<>();
    query = Normalizer.normalize(query, Normalizer.Form.NFD).replaceAll("\\p{InCombiningDiacriticalMarks}+", "");
    JsonObject response = null;
    try {
        URL queryURL = new URL(String.format("http://api.genius.com/search?q=%s", URLEncoder.encode(query, "UTF-8")));
        Connection connection = Jsoup.connect(queryURL.toExternalForm()).header("Authorization", "Bearer " + Keys.GENIUS).timeout(0).ignoreContentType(true);
        Document document = connection.userAgent(Net.USER_AGENT).get();
        response = new JsonParser().parse(document.text()).getAsJsonObject();
    } catch (JsonSyntaxException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    if (response == null || response.getAsJsonObject("meta").get("status").getAsInt() != 200)
        return results;
    JsonArray hits = response.getAsJsonObject("response").getAsJsonArray("hits");
    int processed = 0;
    while (processed < hits.size()) {
        JsonObject song = hits.get(processed).getAsJsonObject().getAsJsonObject("result");
        String artist = song.getAsJsonObject("primary_artist").get("name").getAsString();
        String title = song.get("title").getAsString();
        String url = "http://genius.com/songs/" + song.get("id").getAsString();
        Lyrics l = new Lyrics(Lyrics.SEARCH_ITEM);
        l.setArtist(artist);
        l.setTitle(title);
        l.setURL(url);
        l.setSource("Genius");
        results.add(l);
        processed++;
    }
    return results;
}

//RxJsoup.connect(
//                  Jsoup.connect("www.gggggggg.fr")
//                      .userAgent(MY_USER_AGENT)
//                      .data("credential", email)
//                      .data("pwd", password)
//                      .cookies(loginForm.cookies())
//                      .method(Connection.Method.POST)
//
//                  ).subscibe( response -> {})
public static Observable<Connection.Response> connect(final Connection jsoupConnection) {
    return Observable.create(new ObservableOnSubscribe<Connection.Response>() {

        @Override
        public void subscribe(ObservableEmitter<Connection.Response> observableEmitter) throws Exception {
            try {
                final Connection.Response response = jsoupConnection.execute();
                observableEmitter.onNext(response);
                observableEmitter.onComplete();
            } catch (Exception e) {
                observableEmitter.onError(e);
            }
        }
    });
}

public static int parseSongsPage(String artist, String artistURL) {
    songMap = DataManager.getSongMap();
    Logger.LogToStatusBar("Attempting to parse song titles from " + artistURL);
    // Try to load page using Jsoup
    try {
        // Load page into Document
        Connection c = Jsoup.connect(artistURL);
        c.timeout(100000);
        Document doc = c.get();
        // Check for any errors
        Elements errors = doc.select("strong");
        if (errors.get(0).text().contains("This artist is currently not available.")) {
            Logger.LogToStatusBar("There is an error on this page, removing this artist");
            //artistMap.remove(artist);
            return 0;
        }
        // No errors, get the song list
        Elements songRows = doc.select("tbody#songslist tr td:eq(0)");
        extractSongs(artist, songRows);
    } catch (IOException e) {
        Logger.LogToStatusBar("Songs page for " + artist + " not found!");
    }
    return 1;
}

public TweetFieldsResponse fetchTweetFields(String tweetId) {
    // generate twitter URL (this one creates a redirect!)
    String twitterURL = TWITTER + DEFAULT_TWITTER_STATUS_ROOT + tweetId;
    // keep track of time
    long time0 = System.currentTimeMillis();
    Connection.Response response = null;
    try {
        // this is where the download takes place
        response = Jsoup.connect(twitterURL).execute();
    } catch (HttpStatusException e) {
        System.err.println(twitterURL);
        return createResponse(new TweetFields(tweetId, null, null, null), e.getStatusCode(), time0, false);
    } catch (IOException e) {
        e.printStackTrace();
        return createOtherErrorResponse(tweetId, time0);
    }
    // parsing the returned HTML code
    Document doc = null;
    try {
        doc = response.parse();
    } catch (IOException e) {
        e.printStackTrace();
        return createParseErrorResponse(tweetId, time0);
    }
    // extract parsed id
    String resolvedUrl = response.url().getPath();
    int lastSlashIdx = resolvedUrl.lastIndexOf('/');
    String originalId = resolvedUrl.substring(lastSlashIdx + 1);
    // extract username
    String username = doc.select(".user-actions").attr("data-screen-name");
    // extract text
    // this is not the most appropriate way to check
    Elements textEl = doc.select(".js-tweet-text");
    if (textEl == null || textEl.first() == null) {
        // is the user suspension the only reason for getting a null text?
        System.err.println(twitterURL + " (suspeneded)");
        return createResponse(new TweetFields(tweetId, null, null, null), response.statusCode(), time0, true);
    }
    String text = textEl.first().text();
    // should always be there, but not correct for response tweets
    String publicationTime = doc.select(".tweet-timestamp").attr("title");
    Elements mainEl = doc.select(".permalink-tweet");
    if (mainEl != null && mainEl.first() != null) {
        Elements inEl = mainEl.first().select(".js-tweet-text");
        if (inEl != null && inEl.first() != null) {
            text = inEl.first().text();
        }
        // get correct publication time
        publicationTime = mainEl.select(".metadata").first().text();
    }
    // get tweets to which this tweet replies (if available)
    Elements repEl = doc.select(".permalink-in-reply-tos");
    String[] responseTos = null;
    if (repEl != null && repEl.first() != null) {
        List<String> respIds = new ArrayList<String>();
        Elements inEl = repEl.first().select(".simple-tweet");
        if (inEl != null) {
            for (int i = 0; i < inEl.size(); i++) {
                respIds.add(inEl.get(i).attr("data-tweet-id"));
            }
        }
        responseTos = new String[respIds.size()];
        for (int i = 0; i < respIds.size(); i++) {
            responseTos[i] = respIds.get(i);
        }
    }
    // retweets (if available)
    Elements numRetweetsEl = doc.select(".stats .js-stat-retweets a strong");
    int numRetweets = 0;
    if (numRetweetsEl.text().length() > 0) {
        numRetweets = Integer.parseInt(numRetweetsEl.text().replaceAll(String.valueOf((char) 160), "").replaceAll(",", ""));
    }
    // favorites (if available)
    Elements numFavoritesEl = doc.select(".stats .js-stat-favorites a strong");
    int numFavorites = 0;
    if (numFavoritesEl.text().length() > 0) {
        numFavorites = Integer.parseInt(numFavoritesEl.text().replaceAll(String.valueOf((char) 160), "").replaceAll(",", ""));
    }
    TweetFields tweetFields = null;
    if (tweetId.equals(originalId)) {
        // original tweet
        tweetFields = new TweetFields(tweetId, username, text, publicationTime, numRetweets, numFavorites, null, responseTos);
    } else {
        tweetFields = new TweetFields(tweetId, username, text, publicationTime, numRetweets, numFavorites, originalId, null);
    }
    return createResponse(tweetFields, response.statusCode(), time0, false);
}

/**
   * Fetch the web page from the URL, parse the HTML to populate the metadata
   * required by WebPageSnapshotModel, and return the constructed
   * WebPageSnapshotModel.
   * 
   * @param url
   *          The URL to fetch the web page from
   * @return The WebPageSnapshotModel
   * @throws IOException
   *           Thrown if there's an issue fetching the web page.
   */
private WebPageSnapshotModel fetchWebPage(String url) throws IOException {
    long fetchTime = System.currentTimeMillis();
    Connection connection = Jsoup.connect(url);
    Response response = connection.execute();
    long postFetchTime = System.currentTimeMillis();
    int timeToFetch = (int) (postFetchTime - fetchTime);
    Document doc = response.parse();
    String destinationUrl = response.url().toString();
    String title = doc.title();
    String description = getDescriptionFromDocument(doc);
    List<String> keywords = getKeywordsFromDocument(doc);
    List<String> outlinks = getOutlinksFromDocument(doc);
    return WebPageSnapshotModel.newBuilder().setUrl(destinationUrl).setFetchedAtRevTs(Long.MAX_VALUE - fetchTime).setSize(doc.html().length()).setFetchedAt(fetchTime).setFetchTimeMs(timeToFetch).setTitle(title).setDescription(description).setKeywords(keywords).setOutlinks(outlinks).setContent(doc.html()).build();
}

/**
     * 点赞
     *
     * @param statusId
     * @param like
     * @param cookie
     * @return
     * @throws TaskException
     */
public LikeResultBean doLike(String statusId, boolean like, String cookie) throws TaskException {
    try {
        String url = like ? "http://m.weibo.cn/attitudesDeal/add" : "http://m.weibo.cn/attitudesDeal/delete";
        Map<String, String> cookieMap = new HashMap<String, String>();
        String[] cookieValues = cookie.split(";");
        for (String cookieValue : cookieValues) {
            String key = cookieValue.split("=")[0];
            String value = cookieValue.split("=")[1];
            cookieMap.put(key, value);
        }
        //            Logger.d(WeiboClientActivity.TAG, cookieMap);
        Connection connection = Jsoup.connect(url);
        connection.userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:33.0) Gecko/20100101 Firefox/33.0").referrer("http://m.weibo.cn/").cookies(cookieMap).data("id", statusId).method(Connection.Method.POST);
        if (like)
            connection.data("attitude", "heart");
        String body = connection.execute().body();
        if (!TextUtils.isEmpty(body)) {
            Logger.d(TAG, body);
            if (body.indexOf("http://passport.weibo.cn/sso/crossdomain") != -1)
                throw new TaskException("-100", "未登录");
            else if (body.indexOf("<html") != -1)
                throw new TaskException("-100", "未登录");
            LikeResultBean likeBean = JSON.parseObject(body, LikeResultBean.class);
            if (likeBean.getOk() == 1) {
                return likeBean;
            } else if (likeBean.getOk() == -100) {
                throw new TaskException("-100", "未登录");
            } else {
                throw new TaskException("", likeBean.getMsg());
            }
        }
    } catch (Exception e) {
        if (e instanceof TaskException)
            throw (TaskException) e;
        e.printStackTrace();
    }
    throw new TaskException(TaskException.TaskError.timeout.toString());
}

private static void scanBoilerLink(String url) throws IOException, ParseException {
    Connection com = Jsoup.connect(url);
    Document scan_homepage = com.ignoreContentType(true).get();
    Elements events = scan_homepage.select("item");
    for (Element event : events) {
        Event e = parseBoilerEvent(event);
        System.out.println(e.getSendFormat());
    // sendData(e);
    }
}

/**
     * getListing fetches a listing of items and parses them into a Item array.
     * Returns cache if in offline mode.
     */
public static Item[] getListing(Integer page, Context context, String path) throws IOException, ParseException {
    String cacheKey = "frontpage_" + page + "_" + path.replace("/", "");
    if (Utils.isOffline(context)) {
        Object cacheObj = API.getFromCache(context, cacheKey);
        //if no cached data present, return empty array
        if (cacheObj == null)
            return new Item[0];
        else {
            return (Item[]) cacheObj;
        }
    }
    Connection connection = Jsoup.connect("http://www.dumpert.nl" + path + ((page != 0) ? page : ""));
    if (PreferenceManager.getDefaultSharedPreferences(context).getBoolean("nsfw", false))
        connection.cookie("nsfw", "1");
    Document document = connection.get();
    Elements elements = document.select(".dump-cnt .dumpthumb");
    ArrayList<Item> itemArrayList = new ArrayList<Item>();
    for (Element element : elements) {
        Item item = new Item();
        item.url = element.attr("href");
        item.title = element.select("h1").first().text();
        Log.d(TAG, "Parsing '" + item.url + "'");
        item.description = element.select("p.description").first().html();
        item.thumbUrl = element.select("img").first().attr("src");
        String rawDate = element.select("date").first().text();
        Date date = new SimpleDateFormat("dd MMMM yyyy kk:ss", new Locale("nl", "NL")).parse(rawDate);
        item.date = new TimeAgo(context).timeAgo(date);
        item.stats = element.select("p.stats").first().text();
        item.photo = element.select(".foto").size() > 0;
        item.video = element.select(".video").size() > 0;
        item.audio = element.select(".audio").size() > 0;
        if (item.video)
            item.imageUrls = new String[] { item.thumbUrl.replace("sq_thumbs", "stills") };
        else if (item.photo) {
            //get the image itself from it's url.
            //sadly no other way to get full hq image :'(
            Log.d(TAG, "Got image, requesting " + item.url);
            Connection imageConn = Jsoup.connect(item.url);
            if (PreferenceManager.getDefaultSharedPreferences(context).getBoolean("nsfw", false))
                imageConn.cookie("nsfw", "1");
            Document imageDocument = imageConn.get();
            ArrayList<String> imgs = new ArrayList<String>();
            for (Element img : imageDocument.select("img.player")) {
                imgs.add(img.attr("src"));
            }
            item.imageUrls = new String[imgs.size()];
            imgs.toArray(item.imageUrls);
        }
        itemArrayList.add(item);
    }
    Item[] returnList = new Item[itemArrayList.size()];
    itemArrayList.toArray(returnList);
    saveToCache(context, cacheKey, returnList);
    return returnList;
}

public static String authWithWordpress(String usr, String pwd, String oauthUrl) throws IOException {
    //1. This will load the home page and get Auth link
    String authUrl = Jsoup.connect(oauthUrl).timeout(5000).get().getElementsByAttributeValueContaining("href", "wordpress.com").attr("href");
    //2. Auth
    Element loginform = Jsoup.connect(authUrl).referrer(oauthUrl).get().getElementById("loginform");
    if (loginform == null) {
        throw new IllegalStateException("Can't parse loginform");
    }
    //2.1 Login
    Connection.Response response = Jsoup.connect(loginform.attr("action")).data("log", usr).data("pwd", pwd).data("wp-submit", loginform.select("input[name=wp-submit]").attr("value")).data("action", loginform.select("input[name=action]").attr("value")).data("redirect_to", loginform.select("input[name=redirect_to]").attr("value")).method(Connection.Method.POST).timeout(5000).execute();
    // 2.2 Approve
    Element loginformApprove = response.parse().getElementById("loginform");
    if (loginformApprove == null) {
        throw new IllegalStateException("Can't parse loginform");
    }
    String redirectUrl = Jsoup.connect(loginformApprove.attr("action")).cookies(response.cookies()).data("client_id", loginformApprove.select("input[name=client_id]").attr("value")).data("response_type", loginformApprove.select("input[name=response_type]").attr("value")).data("redirect_uri", loginformApprove.select("input[name=redirect_uri]").attr("value")).data("state", loginformApprove.select("input[name=state]").attr("value")).data("action", loginformApprove.select("input[name=action]").attr("value")).data("blog_id", loginformApprove.select("input[name=blog_id]").attr("value")).data("_wpnonce", loginformApprove.select("input[name=_wpnonce]").attr("value")).data("wp-submit", loginformApprove.select("input[name=wp-submit]").attr("value")).data("redirect_to", loginformApprove.select("input[name=redirect_to]").attr("value")).method(Connection.Method.GET).followRedirects(false).timeout(5000).execute().header("Location");
    // 3  link with code
    return Jsoup.connect(redirectUrl).ignoreContentType(true).followRedirects(false).timeout(5000).method(Connection.Method.GET).execute().header("Location");
}

private void fetchUrlByJsoup(Row row, String url) throws Exception {
    SSLSocketFactory oldSocketFactory = null;
    HostnameVerifier oldHostnameVerifier = null;
    try {
        if (isHttps) {
            oldSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory();
            oldHostnameVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
            final SSLContext sslContext = SSLContext.getInstance("SSL");
            sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
            HttpsURLConnection.setDefaultHostnameVerifier(hostnameVerifier);
        }
        Connection conn = Jsoup.connect(url);
        if (authHeader != null)
            conn.header("Authorization", authHeader);
        conn.ignoreContentType(true);
        conn.timeout(timeout * 1000);
        Document doc = null;
        if (method.equals("get"))
            doc = conn.get();
        else if (method.equals("post"))
            doc = conn.post();
        if (doc != null) {
            if (selector != null) {
                Elements elements = doc.select(selector);
                ArrayList<Object> l = new ArrayList<Object>(elements.size());
                for (Element e : elements) {
                    Map<String, Object> m = new HashMap<String, Object>();
                    for (Attribute attr : e.attributes()) {
                        m.put(attr.getKey(), attr.getValue());
                    }
                    m.put("own_text", e.ownText());
                    m.put("text", e.text());
                    l.add(m);
                }
                row.put("elements", l);
            } else {
                row.put("html", doc.outerHtml());
            }
        }
    } finally {
        if (isHttps) {
            HttpsURLConnection.setDefaultSSLSocketFactory(oldSocketFactory);
            HttpsURLConnection.setDefaultHostnameVerifier(oldHostnameVerifier);
        }
    }
}

public static String getContent(String url) {
    LOGGER.debug("url:" + url);
    Connection conn = Jsoup.connect(url).header("Accept", ACCEPT).header("Accept-Encoding", ENCODING).header("Accept-Language", LANGUAGE).header("Connection", CONNECTION).header("Referer", REFERER).header("Host", HOST).header("User-Agent", USER_AGENT).ignoreContentType(true);
    String html = "";
    try {
        html = conn.post().html();
        html = html.replaceAll("[\n\r]", "");
    } catch (Exception e) {
        LOGGER.error("获�URL：" + url + "页�出错", e);
    }
    return html;
}

@Override
public JSONObject call() throws Exception {
    final int TIMEOUT = 2000;
    try {
        final JSONObject ret = new JSONObject();
        // Get meta info of the URL
        final Connection.Response res = Jsoup.connect(url).timeout(TIMEOUT).followRedirects(false).execute();
        if (HttpServletResponse.SC_OK != res.statusCode()) {
            return null;
        }
        String charset = res.charset();
        if (StringUtils.isBlank(charset)) {
            charset = "UTF-8";
        }
        final String html = new String(res.bodyAsBytes(), charset);
        String title = StringUtils.substringBetween(html, "<title>", "</title>");
        title = StringUtils.trim(title);
        if (!containsChinese(title)) {
            return null;
        }
        final String keywords = StringUtils.substringBetween(html, "eywords\" content=\"", "\"");
        ret.put(Link.LINK_ADDR, url);
        ret.put(Link.LINK_TITLE, title);
        ret.put(Link.LINK_T_KEYWORDS, keywords);
        ret.put(Link.LINK_T_HTML, html);
        final Document doc = Jsoup.parse(html);
        doc.select("pre").remove();
        ret.put(Link.LINK_T_TEXT, doc.text());
        // Evaluate the URL
        URL baiduURL = new URL("https://www.baidu.com/s?pn=0&wd=" + URLEncoder.encode(url, "UTF-8"));
        HttpURLConnection conn = (HttpURLConnection) baiduURL.openConnection();
        conn.setConnectTimeout(TIMEOUT);
        conn.setReadTimeout(TIMEOUT);
        conn.addRequestProperty(Common.USER_AGENT, Symphonys.USER_AGENT_BOT);
        InputStream inputStream = conn.getInputStream();
        String baiduRes = IOUtils.toString(inputStream, "UTF-8");
        IOUtils.closeQuietly(inputStream);
        conn.disconnect();
        int baiduRefCnt = StringUtils.countMatches(baiduRes, "<em>" + url + "</em>");
        if (1 > baiduRefCnt) {
            ret.put(Link.LINK_BAIDU_REF_CNT, baiduRefCnt);
            LOGGER.debug(ret.optString(Link.LINK_ADDR));
            return ret;
        } else {
            baiduURL = new URL("https://www.baidu.com/s?pn=10&wd=" + URLEncoder.encode(url, "UTF-8"));
            conn = (HttpURLConnection) baiduURL.openConnection();
            conn.setConnectTimeout(TIMEOUT);
            conn.setReadTimeout(TIMEOUT);
            conn.addRequestProperty(Common.USER_AGENT, Symphonys.USER_AGENT_BOT);
            inputStream = conn.getInputStream();
            baiduRes = IOUtils.toString(inputStream, "UTF-8");
            IOUtils.closeQuietly(inputStream);
            conn.disconnect();
            baiduRefCnt += StringUtils.countMatches(baiduRes, "<em>" + url + "</em>");
            ret.put(Link.LINK_BAIDU_REF_CNT, baiduRefCnt);
            LOGGER.debug(ret.optString(Link.LINK_ADDR));
            return ret;
        }
    } catch (final SocketTimeoutException e) {
        return null;
    } catch (final Exception e) {
        LOGGER.log(Level.WARN, "Parses URL [" + url + "] failed", e);
        return null;
    }
}

@Override
protected Integer doInBackground(Object... params) {
    try {
        mActivity = (MainActivity) params[0];
        String query = (String) params[1];
        Log.d(Utils.TAG, "Cerco le stazioni - " + query);
        publishProgress(loadStates.START);
        Connection connection = Jsoup.connect(CERCA_STAZIONE_URL);
        connection.timeout(30000);
        connection.data("q", query);
        Document document = connection.post();
        Log.d(Utils.TAG, "Analizzo le stazioni - " + query);
        Elements stazioniEl = document.getElementsByTag("stazione");
        risultatiRicerca.clear();
        for (Element cStazioneEl : stazioniEl) {
            Log.d(Utils.TAG, "---->  " + cStazioneEl.text());
            risultatiRicerca.add(cStazioneEl.text());
        }
        publishProgress(loadStates.FINISHED);
    } catch (Exception e) {
        Log.w(Utils.TAG, "---->  Errore!", e);
        e.printStackTrace();
        publishProgress(loadStates.UNKNOWN_PROBLEM);
        return -1;
    }
    return 0;
}

/**
     * Loads a GET request from a specified path and query info
     * @param path the path (beginning with a forward slash).
     * @param params parameter information to pass as query.
     * @return the body string, or null if error occurs.
     * @throws IOException
     */
public String request(String path, Map<String, String> params) throws IOException {
    if (checkExpiration() != 1)
        return null;
    Connection conn = Jsoup.connect(Session.GRADEBOOK_ROOT + path).timeout(60000).cookies(getCookies());
    if (params != null)
        conn.data(params);
    try {
        Connection.Response resp = conn.execute();
        return resp.body();
    } catch (HttpStatusException e) {
        e.printStackTrace();
        return null;
    }
}

HtmlPage extractArticleByJsoup(Anchor anchor) throws IOException {
    try {
        Connection conn = Jsoup.connect(anchor.getHref());
        conn.userAgent("Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)");
        conn.timeout(1000 * 6);
        Document doc = conn.get();
        logger.info("doc.baseUri:" + doc.baseUri());
        String title = HtmlPageTitleUtil.smartGetTitle(anchor, doc.title());
        String keywords = JsoupSelectorUtil.select(doc.head(), "[name=keywords]").attr("content");
        String description = JsoupSelectorUtil.select(doc.head(), "[name=description]").attr("content");
        String mainContentSelectorContent = JsoupSelectorUtil.select(doc.body(), mainContentSelector).text();
        Element smartMainContent = smartGetMainContent(doc);
        HtmlPage page = new HtmlPage();
        page.setAnchor(anchor);
        page.setContent(StringUtils.defaultIfBlank(mainContentSelectorContent, smartMainContent == null ? null : smartMainContent.text()));
        page.setDescription(description);
        page.setKeywords(keywords);
        page.setTitle(title);
        page.setSourceLang(sourceLang);
        page.setTags(tags);
        //TODO 增加anchor.text 与 page.title的比较或者是替�
        logger.info("------------------- url:" + page.getAnchor().getHref() + " ---------------------------");
        logger.info("smartMainContent.text:" + (smartMainContent == null ? "NOT_FOUND" : smartMainContent.text()));
        logger.info("title:" + page.getTitle());
        logger.info("keywords:" + page.getKeywords());
        logger.info("description:" + page.getDescription());
        logger.info("content,size:" + StringUtils.length(page.getContent()) + " " + page.getContent());
        logger.info("content.deepLevel:" + JsoupSelectorUtil.select(doc, mainContentSelector).parents().size());
        if (smartMainContent != null && StringUtils.isNotBlank(mainContentSelectorContent)) {
            if (!smartMainContent.text().equals(page.getContent())) {
                logger.warn("-------------------error: smart max length text != selector[" + StringUtils.join(mainContentSelector, ",") + "] text----------------------");
            }
        }
        if (StringUtils.length(page.getContent()) < minContentLength) {
            return null;
        }
        return page;
    } catch (Exception e) {
        throw new RuntimeException("error on extractArticleByJsoup anchor:" + anchor, e);
    }
}

private static void mainListPage(String url) throws IOException, InterruptedException {
    logger.info(ParserTest.class.getName() + ": mainListPage start");
    Connection conn = Jsoup.connect(url);
    conn.timeout(0);
    Document doc = conn.get();
    // get list of all categories
    Elements allCategories = doc.select("div[class=repast list_nav]");
    if (checkElementsSize(allCategories)) {
        Elements list = allCategories.get(0).getElementsByAttributeValue("class", "list");
        if (checkElementsSize(allCategories)) {
            Elements allList = list.first().children();
            logger.info(ParserTest.class.getName() + ": All categories url start");
            for (Element e : allList) {
                String menuUrl = e.children().first().attr("href");
                menuList.add(menuUrl);
                logger.info(menuUrl);
            }
            logger.info(ParserTest.class.getName() + ": All categories url end");
        }
    }
    Element e2 = allCategories.get(0).child(1);
    categoryPage(menuList.get(0), e2);
    logger.info(ParserTest.class.getName() + ": mainListPage end");
}

public String login(String num, String passwd) {
    try {
        Connection con = Jsoup.connect("http://202.115.47.141/loginAction.do").data("zjh", num).data("mm", passwd).timeout(10000).method(Method.POST);
        Response response = con.execute();
        Document doc = response.parse();
        if (doc.title().equals("学分制综�教务")) {
            Iterator<Entry<String, String>> ite = response.cookies().entrySet().iterator();
            while (ite.hasNext()) {
                Entry<String, String> entry = ite.next();
                String key = entry.getKey();
                String value = entry.getValue();
                if (key.equals("JSESSIONID"))
                    return value;
            }
            // TODO 错误代�100：未获�到SESSION
            return "100";
        }
        Element prompt = doc.getElementsByAttributeValue("class", "errorTop").first();
        String strong = prompt.select("strong>font").text().trim();
        if (strong.contains("�存在")) {
            // TODO 错误代�101：学��存在
            return "101";
        } else if (strong.contains("密��正确")) {
            // TODO 错误代�102：密��正确
            return "102";
        } else {
            // TODO 错误代�103：网页格�错误
            return "103";
        }
    } catch (Exception e) {
        return "104";
    }
}

public static Response get(AppContext appContext, String url, String referrer) throws IOException {
    Map<String, String> cookies = getCookies(appContext);
    String userAgent = getUserAgent(appContext);
    Connection connection = Jsoup.connect(url).cookies(cookies).referrer(referrer).userAgent(userAgent);
    Response response = connection.execute();
    cookies.putAll(response.cookies());
    mCookies = cookies;
    return response;
}

public synchronized String login(Context c, String user, String password) {
    if (user == null || password == null) {
        return null;
    }
    if (!isNetworkAvailable(c)) {
        return null;
    }
    try {
        if ((user.length() > 0) && (user.charAt(0) != 'k')) {
            user = "k" + user;
        }
        mCookies.getCookieStore().removeAll();
        Jsoup.connect(URL_KUSSS_INDEX).timeout(TIMEOUT_LOGIN).followRedirects(true).get();
        Connection.Response r = Jsoup.connect(URL_LOGIN).cookies(getCookieMap()).data("j_username", user).data("j_password", password).timeout(TIMEOUT_LOGIN).followRedirects(true).method(Connection.Method.POST).execute();
        if (r.url() != null) {
            r = Jsoup.connect(r.url().toString()).cookies(getCookieMap()).method(Connection.Method.GET).execute();
        }
        Document doc = r.parse();
        String sessionId = getSessionIDFromCookie();
        if (isLoggedIn(c, doc)) {
            return sessionId;
        }
        if (isLoggedIn(c, sessionId)) {
            return sessionId;
        }
        Log.w(TAG, "login failed: isLoggedIn=FALSE");
        return null;
    } catch (SocketTimeoutException e) {
        Log.w(TAG, "login failed: connection timeout", e);
        return null;
    } catch (Exception e) {
        Log.w(TAG, "login failed", e);
        Analytics.sendException(c, e, true);
        return null;
    }
}

/**
	 * 获�一个页�里的文章列表
	 * 
	 * @param set
	 * @param url
	 * @return
	 * @throws Exception
	 */
private HashSet<String> getArticleList(HashSet<String> set, String url) throws Exception {
    if (conf.getFetchArticleListProxyHost().trim().isEmpty()) {
        Connection conn = Jsoup.connect(url);
        Document doc = conn.get();
        Elements elements = doc.select(conf.getArticleLinkXpath());
        for (Element e : elements) {
            String u = e.attr("href");
            if (!u.startsWith("http")) {
                u = url + "/" + u;
            }
            System.out.println(u);
            set.add(u);
        }
    } else {
        String html = getHtmlWithProxy(url);
        Document doc = Jsoup.parse(html);
        Elements elements = doc.select(conf.getArticleLinkXpath());
        for (Element e : elements) {
            String u = e.attr("href");
            if (!u.startsWith("http")) {
                u = url + "/" + u;
            }
            System.out.println(u);
            set.add(u);
        }
    }
    return set;
}

@Override
public void call(Subscriber<? super String> subscriber) {
    try {
        Connection login = Jsoup.connect(HACKER_NEWS_BASE_URL + "login");
        login.header("Accept-Encoding", "gzip").data("go_to", "news").data("acct", username).data("pw", password).header("Origin", "https://news.ycombinator.com").followRedirects(true).referrer(HACKER_NEWS_BASE_URL + "login?go_to=news").method(Connection.Method.POST);
        Connection.Response response = login.execute();
        String cookie = response.cookie("user");
        if (cookie == null) {
            subscriber.onNext("");
        } else {
            subscriber.onNext(cookie);
        }
    } catch (Exception e) {
        subscriber.onError(e);
    }
}

@Override
public ReservationResult reservation(DetailedItem item, Account account, int useraction, String selection) throws IOException {
    if (selection == null) {
        // Which copy?
        List<Map<String, String>> options = new ArrayList<>();
        for (Copy copy : item.getCopies()) {
            if (copy.getResInfo() == null)
                continue;
            Map<String, String> option = new HashMap<>();
            option.put("key", copy.getResInfo());
            option.put("value", copy.getBarcode() + " - " + copy.getBranch() + " - " + copy.getReturnDate());
            options.add(option);
        }
        if (options.size() == 0) {
            return new ReservationResult(MultiStepResult.Status.ERROR, stringProvider.getString(StringProvider.NO_COPY_RESERVABLE));
        } else if (options.size() == 1) {
            return reservation(item, account, useraction, options.get(0).get("key"));
        } else {
            ReservationResult res = new ReservationResult(MultiStepResult.Status.SELECTION_NEEDED);
            res.setSelection(options);
            return res;
        }
    } else {
        // Reservation
        // the URL stored in selection might be absolute (WinBiap 4.3) or relative (4.2)
        String reservationUrl = new URL(new URL(opac_url), selection).toString();
        // the URL stored in selection contains "=" and other things inside params
        // and will be messed up by our cleanUrl function, therefore we use a direct HttpGet
        Document doc = Jsoup.parse(convertStreamToString(http_client.execute(new HttpGet(reservationUrl)).getEntity().getContent()));
        if (doc.select("[id$=LabelLoginMessage]").size() > 0) {
            doc.select("[id$=TextBoxLoginName]").val(account.getName());
            doc.select("[id$=TextBoxLoginPassword]").val(account.getPassword());
            FormElement form = (FormElement) doc.select("form").first();
            List<Connection.KeyVal> formData = form.formData();
            List<NameValuePair> params = new ArrayList<>();
            for (Connection.KeyVal kv : formData) {
                if (!kv.key().contains("Button") || kv.key().endsWith("ButtonLogin")) {
                    params.add(new BasicNameValuePair(kv.key(), kv.value()));
                }
            }
            doc = Jsoup.parse(httpPost(opac_url + "/user/" + form.attr("action"), new UrlEncodedFormEntity(params), getDefaultEncoding()));
        }
        FormElement confirmationForm = (FormElement) doc.select("form").first();
        List<Connection.KeyVal> formData = confirmationForm.formData();
        List<NameValuePair> params = new ArrayList<>();
        for (Connection.KeyVal kv : formData) {
            if (!kv.key().contains("Button") || kv.key().endsWith("ButtonVorbestOk")) {
                params.add(new BasicNameValuePair(kv.key(), kv.value()));
            }
        }
        httpPost(opac_url + "/user/" + confirmationForm.attr("action"), new UrlEncodedFormEntity(params), getDefaultEncoding());
        return new ReservationResult(MultiStepResult.Status.OK);
    }
}

public static boolean execute(Map<String, String> cookies, String action) {
    String url = "http://192.168.0.1/goform/SysStatusHandle";
    Map<String, String> map = new HashMap<>();
    map.put("action", action);
    map.put("CMD", "WAN_CON");
    map.put("GO", "system_status.asp");
    Connection conn = Jsoup.connect(url).header("Accept", ACCEPT).header("Accept-Encoding", ENCODING).header("Accept-Language", LANGUAGE).header("Connection", CONNECTION).header("Host", HOST).header("Referer", REFERER).header("User-Agent", USER_AGENT).ignoreContentType(true).timeout(30000);
    for (String cookie : cookies.keySet()) {
        conn.cookie(cookie, cookies.get(cookie));
    }
    String title = null;
    try {
        Connection.Response response = conn.method(Connection.Method.POST).data(map).execute();
        String html = response.body();
        Document doc = Jsoup.parse(html);
        title = doc.title();
        LOGGER.info("�作连接页�标题：" + title);
        Thread.sleep(10000);
    } catch (Exception e) {
        LOGGER.error(e.getMessage());
    }
    if ("LAN | LAN Settings".equals(title)) {
        if (("3".equals(action) && isConnected()) || ("4".equals(action) && !isConnected())) {
            return true;
        }
    }
    return false;
}

private String login(final String id, final String password) {
    Document doc;
    /**
         * 访问登录页�，获�需�的键值对
         */
    try {
        doc = Jsoup.connect(URL_LOGIN).timeout(10000).get();
    } catch (IOException e) {
        doc = null;
    }
    if (doc == null) {
        return "查询失败";
    }
    String param1 = "";
    String param2 = "btnLogin";
    String param3 = "";
    String param4 = doc.select("input#__VIEWSTATE").get(0).attr("value");
    String param5 = doc.select("input#__VIEWSTATEGENERATOR").get(0).attr("value");
    String param6 = doc.select("input#__EVENTVALIDATION").get(0).attr("value");
    Map<String, String> params = new HashMap<>();
    params.put("__LASTFOCUS", param1);
    params.put("__EVENTTARGET", param2);
    params.put("__EVENTARGUMENT", param3);
    params.put("__VIEWSTATE", param4);
    params.put("__VIEWSTATEGENERATOR", param5);
    params.put("__EVENTVALIDATION", param6);
    params.put("txtUserName", id);
    params.put("txtPassword", password);
    params.put("hfIsManager", "0");
    /**
         * 登录
         */
    try {
        Connection conn = Jsoup.connect(URL_LOGIN).data(params).timeout(10000);
        doc = conn.post();
        mCookies = conn.response().cookies();
    } catch (IOException e) {
        doc = null;
    }
    if (doc == null) {
        return "查询失败";
    }
    // 账��密�是�有误
    if (doc.select("span#lblError").size() > 0) {
        return doc.select("span#lblError").get(0).text();
    }
    return null;
}

@NotNull
@Override
public BilingualQueryResult executeBilingualQuery(@NotNull String queryInput, @NotNull Language inputLanguage, @NotNull Language outputLanguage, boolean allowBothWay) throws MetadictTechnicalException {
    Connection targetConnection = buildTargetConnection(queryInput, inputLanguage, outputLanguage);
    Document doc;
    try {
        doc = targetConnection.get();
    } catch (IOException e) {
        LOGGER.error("Fetching response from backend failed", e);
        throw new MetadictTechnicalException(e);
    }
    BilingualQueryResultBuilder builder = processDocument(doc);
    return builder.build();
}

public Connection data(String... keyvals) {
    Validate.notNull(keyvals, "Data key value pairs must not be null");
    Validate.isTrue(keyvals.length % 2 == 0, "Must supply an even number of key value pairs");
    for (int i = 0; i < keyvals.length; i += 2) {
        String key = keyvals[i];
        String value = keyvals[i + 1];
        Validate.notEmpty(key, "Data key must not be empty");
        Validate.notNull(value, "Data value must not be null");
        req.data(KeyVal.create(key, value));
    }
    return this;
}

public Connection data(String... keyvals) {
    Validate.notNull(keyvals, "Data key value pairs must not be null");
    Validate.isTrue(keyvals.length % 2 == 0, "Must supply an even number of key value pairs");
    for (int i = 0; i < keyvals.length; i += 2) {
        String key = keyvals[i];
        String value = keyvals[i + 1];
        Validate.notEmpty(key, "Data key must not be empty");
        Validate.notNull(value, "Data value must not be null");
        req.data(KeyVal.create(key, value));
    }
    return this;
}

/**
     * Method to check if we can connect with JSOUP to a specific url
     * @param link_html the url to connect
     * @return true/false
     */
public boolean checkconn(String link_html) {
    try {
        Connection.Response response = Jsoup.connect(link_html).timeout(10 * 1000).execute();
        return response.statusCode() == 200;
    } catch (Exception ex) {
        Logger.getLogger(WebParser.class.getName()).log(Level.SEVERE, null, ex);
        System.out.print("can not connect to:" + link_html);
        return false;
    }
}

private void getSession() {
    try {
        String key = LJTypes.createKey(ljUser.authInfo.hash, ljUser.accountadded);
        String p = SimpleCrypto.decrypt(key, ljUser.authInfo.pcrypt);
        Connection login = Jsoup.connect("https://www.livejournal.com/login.bml?ret=1&nojs=1");
        login.data("user", ljUser.journalname);
        login.data("password", p);
        login.data("remember_me", "1");
        login.data("action:login", "Log in...");
        Document response = login.post();
        Connection.Response result2 = login.response();
        String expires = result2.header("expires");
        //
        DateFormat formatter = new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss z");
        Date expiration = formatter.parse(expires);
        long etime = expiration.getTime();
        //{ljuniq=imgRxcAgrmdJEdF:1295597273:pgstats0:m0, ljmastersession=v1:u13976821:s438:a0e45S5kYuh//Thanks%20for%20signing%20in%20%2F%20LiveJournal%20loves%20you%20a%20lot%20%2F%20Here%20have%20a%20cookie, ljloggedin=u13976821:s438, BMLschemepref=, langpref=, ljsession=v1:u13976821:s438:t1295596800:g6306e64c5247483e98f2a30a4b471b2d68dbed53//Thanks%20for%20signing%20in%20%2F%20LiveJournal%20loves%20you%20a%20lot%20%2F%20Here%20have%20a%20cookie}
        Map<String, String> cookies2 = result2.cookies();
        String ljmastersession = cookies2.get("ljmastersession");
        String ljloggedin = cookies2.get("ljloggedin");
        String ljsession = cookies2.get("ljsession");
        ContentValues session = new ContentValues();
        session.put("ljmastersession", ljmastersession);
        session.put("ljsession", ljsession);
        session.put("ljloggedin", ljloggedin);
        session.put("expiration", etime);
        LJDBAdapter.updateAccountSession(ljUser.journalname, session);
        ljUser.ljmastersession = ljmastersession;
        ljUser.ljsession = ljsession;
        ljUser.ljloggedin = ljloggedin;
        ljUser.expiration = etime;
    } catch (Exception e) {
        Log.e(TAG, e.getMessage(), e);
        handleError("getsession");
    }
}

public static Document getDocument(String url, boolean loginCoolApk) throws IOException {
    if (!url.startsWith("https://") || !url.startsWith("http://"))
        url = "http://" + url;
    Connection connection = Jsoup.connect(url);
    if (loginCoolApk) {
        connection.cookies(new UserSave().buildWebRequestCookie());
    }
    return connection.get();
}

private Connection getConnect(String url) {
    return Jsoup.connect(url).header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36").timeout(100000);
}

@Override
public void asyncFetchImage() {
    Connection connection = Jsoup.connect(url.toString());
    try {
        imageReceiver.onSuccess(new Image(selectImage(connection.get())));
    } catch (IOException e) {
        throwableReceiver.onError(e);
    }
}

private Response getResponse(String url) throws IOException {
    Logger.debug("getResponse: " + url);
    Connection connection = Jsoup.connect(url);
    connection.request().method(Method.GET);
    connection.ignoreContentType(true);
    connection.execute();
    return connection.response();
}

/**
     * Creates a JSoup connection to Projectile with method POST and given timeout
     */
private Connection jsoupConnection() {
    return Jsoup.connect(settings.getProjectileUrl()).timeout(settings.getTimeout()).method(Method.POST);
}

protected Connection getConnection(String URL) {
    return Jsoup.connect(URL).ignoreContentType(true).userAgent(this.userAgent).referrer(this.referrer).timeout(this.timeout).followRedirects(this.followRedirects);
}

public void setMethod(String method) {
    if (method != null && method.equalsIgnoreCase("post"))
        this.method = Connection.Method.POST;
}

// endregion
// region network
private void checkInternetConnection() throws BakaReaderException {
    if (!LNReaderApplication.getInstance().isOnline())
        throw new BakaReaderException("OFFLINE (No Internet Connection)", BakaReaderException.OFFLINE);
}