including small junit test case for WordTokenizer

1 files changed, 33 insertions, 0 deletions

diff --git a/test/net/yacy/document/WordTokenizerTest.java b/test/net/yacy/document/WordTokenizerTest.java
new file mode 100644
index 000000000..c32e71ead
--- /dev/null
+++ b/test/net/yacy/document/WordTokenizerTest.java
@@ -0,0 +1,33 @@
+package net.yacy.document;

+

+import org.junit.Test;

+import static org.junit.Assert.*;

+

+public class WordTokenizerTest {

+

+    /**

+     * Test of nextElement method, of class WordTokenizer.

+     */

+    @Test

+    public void testNextElement() {

+        // test sentences containing 10x the word "word"

+        String[] testTxtArr = new String[]{

+            "  word word..... (word) [word] . 'word word' \"word word\" word ?  word! ",

+            "word-word word . word.word@word.word ....word... word,word "

+        };

+        

+        for (String testTxt : testTxtArr) {

+            SentenceReader sr = new SentenceReader(testTxt);

+            WordTokenizer wt = new WordTokenizer(sr, null);

+            int cnt = 0;

+            while (wt.hasMoreElements()) {

+                StringBuilder sb = wt.nextElement();

+                assertEquals("word", sb.toString());

+                cnt++;

+            }

+            wt.close();

+            assertEquals(10, cnt);

+        }

+    }

+

+}