TweetDetectorTest.java

/**
 * Copyright (C) 2010 Peter Karich <jetwick_@_pannous_._info>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package de.jetwick.tw;

import java.io.IOException;
import de.jetwick.data.JTweet;
import de.jetwick.data.JUser;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.junit.Test;
import static org.junit.Assert.*;

/**
 *
 * @author Peter Karich, peat_hal 'at' users 'dot' sourceforge 'dot' net
 */
public class TweetDetectorTest {

    public TweetDetectorTest() {
    }

    @Test
    public void testRun() {
        List<JTweet> tweets = new ArrayList<JTweet>();
        tweets.add(createTweet(1, "term5 term2 term3 term4!"));
        tweets.add(createTweet(1, "term o"));
        tweets.add(createTweet(1, "TERM term6 Gehts?"));
        TweetDetector extractor = new TweetDetector(tweets);
        extractor.setTermMaxCount(10);
        List<Entry<String, Integer>> mostFrequentTerms = extractor.run().getSortedTerms();

        assertEquals(7, mostFrequentTerms.size());
        assertEquals("term", mostFrequentTerms.get(0).getKey());
        assertEquals(2, (int) mostFrequentTerms.get(0).getValue());
    }

    @Test
    public void testSkipUser() {
        List<JTweet> tweets = new ArrayList<JTweet>();
        tweets.add(createTweet(1L, "@userA term @userB term4!"));
        tweets.add(createTweet(2L, "@userA term o"));
        TweetDetector extractor = new TweetDetector(tweets);
        extractor.setTermMaxCount(10);
        List<Entry<String, Integer>> mostFrequentTerms = extractor.run().getSortedTerms();

        // ignore o which is too short
        // ignore @usera which starts with @
        assertEquals(2, mostFrequentTerms.size());
        assertEquals("term", mostFrequentTerms.get(0).getKey());
        assertEquals(2, (int) mostFrequentTerms.get(0).getValue());
    }

    @Test
    public void testUrlsInTerms() {
        List<JTweet> tweets = new ArrayList<JTweet>();
        tweets.add(createTweet(1, "the god http://www.jetwick.com/hihiho/test.html <b>http</b>://<b>bit</b>.ly/9FZv5E"));
        List<Entry<String, Integer>> mostFrequentTerms = createExtractor(tweets).run().getSortedTerms();
//        System.out.println(mostFrequentTerms);
        assertEquals(1, mostFrequentTerms.size());
    }

    @Test
    public void testTermsWithRemove() {
        List<JTweet> tweets = new ArrayList<JTweet>();
        tweets.add(createTweet(1, "the god"));
        tweets.add(createTweet(2, "the thing"));
        tweets.add(createTweet(3, "it's now"));
        List<Entry<String, Integer>> mostFrequentTerms = createExtractor(tweets).run().getSortedTerms();

        // remove "the", "it's" and "now"
        assertEquals(2, mostFrequentTerms.size());

        tweets.add(createTweet(4, "p d"));
        mostFrequentTerms = createExtractor(tweets).run().getSortedTerms();
        assertEquals(2, mostFrequentTerms.size());

        tweets.add(createTweet(5, "we're --"));
        mostFrequentTerms = createExtractor(tweets).run().getSortedTerms();
        assertEquals(2, mostFrequentTerms.size());

        tweets.clear();
        tweets.add(createTweet(6, "c++"));
        tweets.add(createTweet(7, "c#"));
        tweets.add(createTweet(8, "DivaDOD:"));

        mostFrequentTerms = createExtractor(tweets).run().getSortedTerms();
        assertEquals(3, mostFrequentTerms.size());
        assertEquals("c++", mostFrequentTerms.get(0).getKey());
        assertEquals("c#", mostFrequentTerms.get(1).getKey());
        assertEquals("divadod", mostFrequentTerms.get(2).getKey());
    }

    @Test
    public void testStripNoiseFromWords() {
        assertEquals("@hi@", TweetDetector.stripNoiseFromWord("@hi@"));
        assertEquals("pet ", TweetDetector.stripNoiseFromWord("pet."));
        assertEquals("@peter ", TweetDetector.stripNoiseFromWord("@peter."));
        assertEquals("@pet er ", TweetDetector.stripNoiseFromWord("@pet,er!"));
        assertEquals("@pet er ", TweetDetector.stripNoiseFromWord("@pet,er!"));
        assertEquals("@peter_mueller", TweetDetector.stripNoiseFromWord("@<b>peter</b>_mueller"));
        assertEquals("  peter  ", TweetDetector.stripNoiseFromWord(">>peter<<"));
        assertEquals(" peter ", TweetDetector.stripNoiseFromWord("\"peter\""));
        assertEquals("don't", TweetDetector.stripNoiseFromWord("don't"));
        assertEquals("hi how are you ", TweetDetector.stripNoiseFromWord("hi\nhow\tare you?"));

        assertEquals("all things after urls should remain!",
                "  hi", TweetDetector.stripNoiseFromWord("http://blibla.de hi"));
        assertEquals("test_t   test", TweetDetector.stripNoiseFromWord("test_t https://www.stupid.de test"));
        assertEquals(" ", TweetDetector.stripNoiseFromWord("http://blibla.de"));
        assertEquals(" ", TweetDetector.stripNoiseFromWord(" "));
        assertEquals("  test", TweetDetector.stripNoiseFromWord("http:// test"));
        assertEquals("http test", TweetDetector.stripNoiseFromWord("http&test"));
        assertEquals("http test ", TweetDetector.stripNoiseFromWord("http;test;"));
    }
    
    @Test
    public void testLanguageDetection() {
        // skip the noise words and last terms for language detection:
        List<JTweet> tweets = new ArrayList<JTweet>();
        tweets.add(createTweet(1, "das geht ja ab!"));
        Map<String, Integer> langs = createExtractor(tweets).run().getLanguages();
        assertEquals(3, langs.get(TweetDetector.DE).intValue());
    }

    @Test
    public void testTerms() {
        JUser user = new JUser("Peter");
        user.addOwnTweet(new JTweet(1, "test pest alpha", user));
        user.addOwnTweet(new JTweet(2, "alpha", user));

        assertEquals(3, (int) createExtractor(user.getOwnTweets()).run().getSortedTerms().size());
        assertEquals(2, (int) createExtractor(user.getOwnTweets()).run().getSortedTerms().get(0).getValue());
    }

    @Test
    public void testChinAnalyzing() throws IOException {
         JUser user = new JUser("Peter");
        user.addOwnTweet(new JTweet(1, "飼い主さん!!ペットを迎えに行ってください!!", user));
        user.addOwnTweet(new JTweet(2, "希望を捨てないでくださいペット", user));
        
//        assertEquals(1, (int) createExtractor(user.getOwnTweets()).run().getSortedTerms().size());
//        assertEquals(2, (int) createExtractor(user.getOwnTweets()).run().getSortedTerms().get(0).getValue());
    }
    
    TweetDetector createExtractor(Collection<JTweet> tweets) {
        return new TweetDetector(tweets);
    }

    JTweet createTweet(long id, String twText) {
        return new JTweet(id, twText, new JUser("tmp")).setCreatedAt(new Date(id));
    }
}