View Javadoc
1   package org.bitbucket.jrsofty.parser.logging.util;
2   
3   import org.bitbucket.jrsofty.parser.logging.util.Tokenizer;
4   import org.junit.Assert;
5   import org.junit.Before;
6   import org.junit.Test;
7   
8   @SuppressWarnings("PMD.AvoidDuplicateLiterals")
9   public class TestTokenizer {
10  
11    private static final String[] WHITE_SPACE_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
12        "jumps", "over", "the", "lazy", "dog." };
13    private static final String[] WHITE_SPACE_TOKENS2 = new String[] { "The", "quick", "\"brown",
14        "fox\"", "jumps", "over", "the", "lazy", "dog." };
15  
16    private static final String[] WHITE_SPACE_TOKENS3 = new String[] { "The", "[quick", "brown",
17        "fox]", "jumps", "over", "the", "\"lazy", "dog\"." };
18  
19    private static final String[] QUOTED_SPACE_TOKENS1 = new String[] { "The", "quick", "brown",
20        "fox", "jumps", "over", "the", "lazy", "dog." };
21    private static final String[] QUOTED_SPACE_TOKENS2 = new String[] { "The", "quick", "brown fox",
22        "jumps", "over", "the", "lazy", "dog." };
23    private static final String[] QUOTED_SPACE_TOKENS3 = new String[] { "The", "[quick", "brown",
24        "fox]", "jumps", "over", "the", "lazy dog", "." };
25  
26    private static final String[] PATTERN1_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
27        "jumps", "over", "the", "lazy", "dog." };
28    private static final String[] PATTERN1_TOKENS2 = new String[] { "The", "quick", "\"brown",
29        "fox\"", "jumps", "over", "the", "lazy", "dog." };
30    private static final String[] PATTERN1_TOKENS3 = new String[] { "The", "[quick", "brown", "fox]",
31        "jumps", "over", "the", "\"lazy", "dog\"." };
32  
33    private static final String[] PATTERN2_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
34        "jumps", "over", "the", "lazy", "dog." };
35    private static final String[] PATTERN2_TOKENS2 = new String[] { "The", "quick", "brown fox",
36        "jumps", "over", "the", "lazy", "dog." };
37    private static final String[] PATTERN2_TOKENS3 = new String[] { "The", "[quick", "brown", "fox]",
38        "jumps", "over", "the", "lazy dog", "." };
39  
40    private static final String[] PATTERN3_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
41        "jumps", "over", "the", "lazy", "dog." };
42    private static final String[] PATTERN3_TOKENS2 = new String[] { "The", "quick", "brown fox",
43        "jumps", "over", "the", "lazy", "dog." };
44    private static final String[] PATTERN3_TOKENS3 = new String[] { "The", "quick brown fox", "jumps",
45        "over", "the", "lazy dog", "." };
46    private static final String[] TOKEN_WITH_NO_WHITESPACE = new String[] {
47        "TheQuickBrownFoxJumpsOverTheLazyDog" };
48    private static final String[] EMPTY_ARRAY = new String[] {};
49  
50    private static final String TEST_VALUE1 = "The quick brown fox jumps over the lazy dog.";
51    private static final String TEST_VALUE2 = "The quick \"brown fox\" jumps over the lazy dog.";
52    private static final String TEST_VALUE3 = "The [quick brown fox] jumps over the \"lazy dog\".";
53    private static final String TEST_VALUE4 = "TheQuickBrownFoxJumpsOverTheLazyDog";
54  
55    private static final String PATTERN1 = "(\\S+)";
56    private static final String PATTERN2 = "\"([^\"]*)\"|(\\S+)";
57    private static final String PATTERN3 = "\\[([^\\]]*)\\]|\"([^\"]*)\"|(\\S+)";
58  
59    public Tokenizer tokenizer;
60  
61    /**
62     * Creation of tokenizer test.
63     */
64    @Before
65    public void start() {
66      if (this.tokenizer != null) {
67        this.tokenizer = null;
68      }
69      this.tokenizer = new Tokenizer();
70    }
71  
72    @Test
73    public void testSimpleWhiteSpaceTokenizerWithNull() {
74      final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(null);
75      Assert.assertArrayEquals(TestTokenizer.EMPTY_ARRAY, tokens);
76    }
77  
78    @Test
79    public void testSimpleWhiteSpaceTokenizerWithoutWhiteSpace() {
80      final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE4);
81      Assert.assertArrayEquals(TestTokenizer.TOKEN_WITH_NO_WHITESPACE, tokens);
82    }
83  
84    @Test
85    public void testSimpleWhiteSpaceTokenizer_1() {
86      final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE1);
87      Assert.assertEquals(9, tokens.length);
88    }
89  
90    @Test
91    public void testSimpleWhiteSpaceTokenizer_2() {
92      final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE2);
93      Assert.assertEquals(9, tokens.length);
94    }
95  
96    @Test
97    public void testSimpleWhiteSpaceTokenizer_3() {
98      final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE3);
99      Assert.assertEquals(9, tokens.length);
100   }
101 
102   @Test
103   public void testSimpleWhiteSpaceTokenizer_4() {
104     final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE1);
105     Assert.assertArrayEquals(TestTokenizer.WHITE_SPACE_TOKENS1, tokens);
106   }
107 
108   @Test
109   public void testSimpleWhiteSpaceTokenizer_5() {
110     final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE2);
111     Assert.assertArrayEquals(TestTokenizer.WHITE_SPACE_TOKENS2, tokens);
112   }
113 
114   @Test
115   public void testSimpleWhiteSpaceTokenizer_6() {
116     final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE3);
117     Assert.assertArrayEquals(TestTokenizer.WHITE_SPACE_TOKENS3, tokens);
118   }
119 
120   @Test
121   public void testQuotedWhiteSpaceTokenizer_1() {
122     final String[] tokens = this.tokenizer.quotedWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE1);
123     Assert.assertArrayEquals(TestTokenizer.QUOTED_SPACE_TOKENS1, tokens);
124   }
125 
126   @Test
127   public void testQuotedWhiteSpaceTokenizer_2() {
128     final String[] tokens = this.tokenizer.quotedWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE2);
129     Assert.assertArrayEquals(TestTokenizer.QUOTED_SPACE_TOKENS2, tokens);
130   }
131 
132   @Test
133   public void testQuotedWhiteSpaceTokenizer_3() {
134     final String[] tokens = this.tokenizer.quotedWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE3);
135     Assert.assertArrayEquals(TestTokenizer.QUOTED_SPACE_TOKENS3, tokens);
136   }
137 
138   @Test
139   public void testTokenizeWithPattern_1_1() {
140     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN1,
141         TestTokenizer.TEST_VALUE1);
142     Assert.assertArrayEquals(TestTokenizer.PATTERN1_TOKENS1, tokens);
143   }
144 
145   @Test
146   public void testTokenizeWithPattern_1_2() {
147     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN1,
148         TestTokenizer.TEST_VALUE2);
149     Assert.assertArrayEquals(TestTokenizer.PATTERN1_TOKENS2, tokens);
150   }
151 
152   @Test
153   public void testTokenizeWithPattern_1_3() {
154     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN1,
155         TestTokenizer.TEST_VALUE3);
156     Assert.assertArrayEquals(TestTokenizer.PATTERN1_TOKENS3, tokens);
157   }
158 
159   @Test
160   public void testTokenizeWithPattern_2_1() {
161     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN2,
162         TestTokenizer.TEST_VALUE1);
163     Assert.assertArrayEquals(TestTokenizer.PATTERN2_TOKENS1, tokens);
164   }
165 
166   @Test
167   public void testTokenizeWithPattern_2_2() {
168     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN2,
169         TestTokenizer.TEST_VALUE2);
170     Assert.assertArrayEquals(TestTokenizer.PATTERN2_TOKENS2, tokens);
171   }
172 
173   @Test
174   public void testTokenizeWithPattern_2_3() {
175     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN2,
176         TestTokenizer.TEST_VALUE3);
177     Assert.assertArrayEquals(TestTokenizer.PATTERN2_TOKENS3, tokens);
178   }
179 
180   @Test
181   public void testTokenizeWithPattern_3_1() {
182     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN3,
183         TestTokenizer.TEST_VALUE1);
184     Assert.assertArrayEquals(TestTokenizer.PATTERN3_TOKENS1, tokens);
185   }
186 
187   @Test
188   public void testTokenizeWithPattern_3_2() {
189     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN3,
190         TestTokenizer.TEST_VALUE2);
191     Assert.assertArrayEquals(TestTokenizer.PATTERN3_TOKENS2, tokens);
192   }
193 
194   @Test
195   public void testTokenizeWithPattern_3_3() {
196     final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN3,
197         TestTokenizer.TEST_VALUE3);
198     Assert.assertArrayEquals(TestTokenizer.PATTERN3_TOKENS3, tokens);
199   }
200 }