1 package org.bitbucket.jrsofty.parser.logging.util;
2
3 import org.bitbucket.jrsofty.parser.logging.util.Tokenizer;
4 import org.junit.Assert;
5 import org.junit.Before;
6 import org.junit.Test;
7
8 @SuppressWarnings("PMD.AvoidDuplicateLiterals")
9 public class TestTokenizer {
10
11 private static final String[] WHITE_SPACE_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
12 "jumps", "over", "the", "lazy", "dog." };
13 private static final String[] WHITE_SPACE_TOKENS2 = new String[] { "The", "quick", "\"brown",
14 "fox\"", "jumps", "over", "the", "lazy", "dog." };
15
16 private static final String[] WHITE_SPACE_TOKENS3 = new String[] { "The", "[quick", "brown",
17 "fox]", "jumps", "over", "the", "\"lazy", "dog\"." };
18
19 private static final String[] QUOTED_SPACE_TOKENS1 = new String[] { "The", "quick", "brown",
20 "fox", "jumps", "over", "the", "lazy", "dog." };
21 private static final String[] QUOTED_SPACE_TOKENS2 = new String[] { "The", "quick", "brown fox",
22 "jumps", "over", "the", "lazy", "dog." };
23 private static final String[] QUOTED_SPACE_TOKENS3 = new String[] { "The", "[quick", "brown",
24 "fox]", "jumps", "over", "the", "lazy dog", "." };
25
26 private static final String[] PATTERN1_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
27 "jumps", "over", "the", "lazy", "dog." };
28 private static final String[] PATTERN1_TOKENS2 = new String[] { "The", "quick", "\"brown",
29 "fox\"", "jumps", "over", "the", "lazy", "dog." };
30 private static final String[] PATTERN1_TOKENS3 = new String[] { "The", "[quick", "brown", "fox]",
31 "jumps", "over", "the", "\"lazy", "dog\"." };
32
33 private static final String[] PATTERN2_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
34 "jumps", "over", "the", "lazy", "dog." };
35 private static final String[] PATTERN2_TOKENS2 = new String[] { "The", "quick", "brown fox",
36 "jumps", "over", "the", "lazy", "dog." };
37 private static final String[] PATTERN2_TOKENS3 = new String[] { "The", "[quick", "brown", "fox]",
38 "jumps", "over", "the", "lazy dog", "." };
39
40 private static final String[] PATTERN3_TOKENS1 = new String[] { "The", "quick", "brown", "fox",
41 "jumps", "over", "the", "lazy", "dog." };
42 private static final String[] PATTERN3_TOKENS2 = new String[] { "The", "quick", "brown fox",
43 "jumps", "over", "the", "lazy", "dog." };
44 private static final String[] PATTERN3_TOKENS3 = new String[] { "The", "quick brown fox", "jumps",
45 "over", "the", "lazy dog", "." };
46 private static final String[] TOKEN_WITH_NO_WHITESPACE = new String[] {
47 "TheQuickBrownFoxJumpsOverTheLazyDog" };
48 private static final String[] EMPTY_ARRAY = new String[] {};
49
50 private static final String TEST_VALUE1 = "The quick brown fox jumps over the lazy dog.";
51 private static final String TEST_VALUE2 = "The quick \"brown fox\" jumps over the lazy dog.";
52 private static final String TEST_VALUE3 = "The [quick brown fox] jumps over the \"lazy dog\".";
53 private static final String TEST_VALUE4 = "TheQuickBrownFoxJumpsOverTheLazyDog";
54
55 private static final String PATTERN1 = "(\\S+)";
56 private static final String PATTERN2 = "\"([^\"]*)\"|(\\S+)";
57 private static final String PATTERN3 = "\\[([^\\]]*)\\]|\"([^\"]*)\"|(\\S+)";
58
59 public Tokenizer tokenizer;
60
61
62
63
64 @Before
65 public void start() {
66 if (this.tokenizer != null) {
67 this.tokenizer = null;
68 }
69 this.tokenizer = new Tokenizer();
70 }
71
72 @Test
73 public void testSimpleWhiteSpaceTokenizerWithNull() {
74 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(null);
75 Assert.assertArrayEquals(TestTokenizer.EMPTY_ARRAY, tokens);
76 }
77
78 @Test
79 public void testSimpleWhiteSpaceTokenizerWithoutWhiteSpace() {
80 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE4);
81 Assert.assertArrayEquals(TestTokenizer.TOKEN_WITH_NO_WHITESPACE, tokens);
82 }
83
84 @Test
85 public void testSimpleWhiteSpaceTokenizer_1() {
86 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE1);
87 Assert.assertEquals(9, tokens.length);
88 }
89
90 @Test
91 public void testSimpleWhiteSpaceTokenizer_2() {
92 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE2);
93 Assert.assertEquals(9, tokens.length);
94 }
95
96 @Test
97 public void testSimpleWhiteSpaceTokenizer_3() {
98 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE3);
99 Assert.assertEquals(9, tokens.length);
100 }
101
102 @Test
103 public void testSimpleWhiteSpaceTokenizer_4() {
104 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE1);
105 Assert.assertArrayEquals(TestTokenizer.WHITE_SPACE_TOKENS1, tokens);
106 }
107
108 @Test
109 public void testSimpleWhiteSpaceTokenizer_5() {
110 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE2);
111 Assert.assertArrayEquals(TestTokenizer.WHITE_SPACE_TOKENS2, tokens);
112 }
113
114 @Test
115 public void testSimpleWhiteSpaceTokenizer_6() {
116 final String[] tokens = this.tokenizer.simpleWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE3);
117 Assert.assertArrayEquals(TestTokenizer.WHITE_SPACE_TOKENS3, tokens);
118 }
119
120 @Test
121 public void testQuotedWhiteSpaceTokenizer_1() {
122 final String[] tokens = this.tokenizer.quotedWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE1);
123 Assert.assertArrayEquals(TestTokenizer.QUOTED_SPACE_TOKENS1, tokens);
124 }
125
126 @Test
127 public void testQuotedWhiteSpaceTokenizer_2() {
128 final String[] tokens = this.tokenizer.quotedWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE2);
129 Assert.assertArrayEquals(TestTokenizer.QUOTED_SPACE_TOKENS2, tokens);
130 }
131
132 @Test
133 public void testQuotedWhiteSpaceTokenizer_3() {
134 final String[] tokens = this.tokenizer.quotedWhiteSpaceTokenizer(TestTokenizer.TEST_VALUE3);
135 Assert.assertArrayEquals(TestTokenizer.QUOTED_SPACE_TOKENS3, tokens);
136 }
137
138 @Test
139 public void testTokenizeWithPattern_1_1() {
140 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN1,
141 TestTokenizer.TEST_VALUE1);
142 Assert.assertArrayEquals(TestTokenizer.PATTERN1_TOKENS1, tokens);
143 }
144
145 @Test
146 public void testTokenizeWithPattern_1_2() {
147 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN1,
148 TestTokenizer.TEST_VALUE2);
149 Assert.assertArrayEquals(TestTokenizer.PATTERN1_TOKENS2, tokens);
150 }
151
152 @Test
153 public void testTokenizeWithPattern_1_3() {
154 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN1,
155 TestTokenizer.TEST_VALUE3);
156 Assert.assertArrayEquals(TestTokenizer.PATTERN1_TOKENS3, tokens);
157 }
158
159 @Test
160 public void testTokenizeWithPattern_2_1() {
161 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN2,
162 TestTokenizer.TEST_VALUE1);
163 Assert.assertArrayEquals(TestTokenizer.PATTERN2_TOKENS1, tokens);
164 }
165
166 @Test
167 public void testTokenizeWithPattern_2_2() {
168 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN2,
169 TestTokenizer.TEST_VALUE2);
170 Assert.assertArrayEquals(TestTokenizer.PATTERN2_TOKENS2, tokens);
171 }
172
173 @Test
174 public void testTokenizeWithPattern_2_3() {
175 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN2,
176 TestTokenizer.TEST_VALUE3);
177 Assert.assertArrayEquals(TestTokenizer.PATTERN2_TOKENS3, tokens);
178 }
179
180 @Test
181 public void testTokenizeWithPattern_3_1() {
182 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN3,
183 TestTokenizer.TEST_VALUE1);
184 Assert.assertArrayEquals(TestTokenizer.PATTERN3_TOKENS1, tokens);
185 }
186
187 @Test
188 public void testTokenizeWithPattern_3_2() {
189 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN3,
190 TestTokenizer.TEST_VALUE2);
191 Assert.assertArrayEquals(TestTokenizer.PATTERN3_TOKENS2, tokens);
192 }
193
194 @Test
195 public void testTokenizeWithPattern_3_3() {
196 final String[] tokens = this.tokenizer.tokenizeWithPattern(TestTokenizer.PATTERN3,
197 TestTokenizer.TEST_VALUE3);
198 Assert.assertArrayEquals(TestTokenizer.PATTERN3_TOKENS3, tokens);
199 }
200 }