1 /*
2 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /*
25 * @test
26 * @bug 6380723
27 * @summary Decode many byte sequences in many ways
28 * @run main/timeout=1800 FindDecoderBugs
29 * @author Martin Buchholz
30 * @key randomness
31 */
32
33 import java.util.*;
34 import java.util.regex.*;
35 import java.nio.*;
36 import java.nio.charset.*;
37
38 public class FindDecoderBugs {
39
40 static boolean isBroken(String csn) {
41 if (csn.equals("x-COMPOUND_TEXT")) return true;
42 return false;
43 }
44
45 static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) {
46 List<T> list = new ArrayList<T>(c);
47 Collections.sort(list);
48 return list;
49 }
50
51 static class TooManyFailures extends RuntimeException {
52 private static final long serialVersionUID = 0L;
53 }
54
55 static String string(byte[] a) {
56 final StringBuilder sb = new StringBuilder();
57 for (byte b : a) {
58 if (sb.length() != 0) sb.append(' ');
59 sb.append(String.format("%02x", b & 0xff));
60 }
61 return sb.toString();
62 }
63
64 static String string(char[] a) {
65 final StringBuilder sb = new StringBuilder();
66 for (char c : a) {
67 if (sb.length() != 0) sb.append(' ');
68 sb.append(String.format("\\u%04x", (int) c));
69 }
70 return sb.toString();
71 }
72
73 static class Reporter {
74 // Some machinery to make sure only a small number of errors
75 // that are "too similar" are reported.
76 static class Counts extends HashMap<String, Long> {
77 private static final long serialVersionUID = -1;
78 long inc(String signature) {
79 Long count = get(signature);
80 if (count == null) count = 0L;
81 put(signature, count+1);
82 return count+1;
83 }
84 }
85
86 final Counts failureCounts = new Counts();
87 final static long maxFailures = 2;
88
89 final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\"");
90 final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}");
91
92 boolean bug(String format, Object... args) {
93 String signature = String.format(format, args);
94 signature = hideBytes.matcher(signature).replaceAll("\"??\"");
95 signature = hideChars.matcher(signature).replaceAll("\\u????");
96 failed++;
97 if (failureCounts.inc(signature) <= maxFailures) {
98 System.out.printf(format, args);
99 System.out.println();
100 return true;
101 }
102 return false;
103 }
104
105 void summarize() {
106 for (String key : sort(failureCounts.keySet()))
107 System.out.printf("-----%n%s%nfailures=%d%n",
108 key, failureCounts.get(key));
109 }
110 }
111
112 static final Reporter reporter = new Reporter();
113
114 static class Result {
115 final int limit;
116 final int ipos;
117 final boolean direct;
118 final byte[] ia;
119 final char[] oa;
120 final CoderResult cr;
121
122 Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) {
123 ipos = ib.position();
124 ia = toArray(ib);
125 oa = toArray(ob);
126 direct = ib.isDirect();
127 limit = ob.limit();
128 this.cr = cr;
129 }
130
131 static byte[] toArray(ByteBuffer b) {
132 int pos = b.position();
133 byte[] a = new byte[b.limit()];
134 b.position(0);
135 b.get(a);
136 b.position(pos);
137 return a;
138 }
139
140 static char[] toArray(CharBuffer b) {
141 char[] a = new char[b.position()];
142 b.position(0);
143 b.get(a);
144 return a;
145 }
146
147 static boolean eq(Result x, Result y) {
148 return x == y ||
149 (x != null && y != null &&
150 (Arrays.equals(x.oa, y.oa) &&
151 x.ipos == y.ipos &&
152 x.cr == y.cr));
153 }
154
155 public String toString() {
156 return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s",
157 string(ia), ipos, ia.length,
158 cr, string(oa), oa.length, limit,
159 (direct ? " (direct)" : ""));
160 }
161 }
162
163 // legend: r=regular d=direct In=Input Ou=Output
164 static final int maxBufSize = 20;
165 static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize];
166 static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize];
167
168 static final CharBuffer[] robs = new CharBuffer[maxBufSize];
169 static final CharBuffer[] dobs = new CharBuffer[maxBufSize];
170 static {
171 for (int i = 0; i < maxBufSize; i++) {
172 ribs[i] = ByteBuffer.allocate(i);
173 dibs[i] = ByteBuffer.allocateDirect(i);
174 robs[i] = CharBuffer.allocate(i);
175 dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer();
176 }
177 }
178
179 static class CharsetTester {
180 private final Charset cs;
181 private static final long maxFailures = 5;
182 private long failures = 0;
183 // private static final long maxCharsetFailures = Long.MAX_VALUE;
184 private static final long maxCharsetFailures = 10000L;
185 private final long failed0 = failed;
186
187 CharsetTester(Charset cs) {
188 this.cs = cs;
189 }
190
191 static boolean bug(String format, Object... args) {
192 return reporter.bug(format, args);
193 }
194
195 Result recode(ByteBuffer ib, CharBuffer ob) {
196 try {
197 char canary = '\u4242';
198 ib.clear(); // Prepare to read
199 ob.clear(); // Prepare to write
200 for (int i = 0; i < ob.limit(); i++)
201 ob.put(i, canary);
202 CharsetDecoder coder = cs.newDecoder();
203 CoderResult cr = coder.decode(ib, ob, false);
204 equal(ib.limit(), ib.capacity());
205 equal(ob.limit(), ob.capacity());
206 Result r = new Result(ib, ob, cr);
207 if (cr.isError())
208 check(cr.length() > 0);
209 if (cr.isOverflow() && ob.remaining() > 10)
210 bug("OVERFLOW, but there's lots of room: %s %s",
211 cs, r);
212 // if (cr.isOverflow() && ib.remaining() == 0)
213 // bug("OVERFLOW, yet remaining() == 0: %s %s",
214 // cs, r);
215 if (cr.isError() && ib.remaining() < cr.length())
216 bug("remaining() < CoderResult.length(): %s %s",
217 cs, r);
218 // if (ib.position() == 0 && ob.position() > 0)
219 // reporter. bug("output only if input consumed: %s %s",
220 // cs, r);
221 // Should we warn if cr.isUnmappable() ??
222 CoderResult cr2 = coder.decode(ib, ob, false);
223 if (ib.position() != r.ipos ||
224 ob.position() != r.oa.length ||
225 cr != cr2)
226 bug("Coding operation not idempotent: %s%n %s%n %s",
227 cs, r, new Result(ib, ob, cr2));
228 if (ob.position() < ob.limit() &&
229 ob.get(ob.position()) != canary)
230 bug("Buffer overrun: %s %s %s",
231 cs, r, ob.get(ob.position()));
232 return r;
233 } catch (Throwable t) {
234 if (bug("Unexpected exception: %s %s %s",
235 cs, t.getClass().getSimpleName(),
236 new Result(ib, ob, null)))
237 t.printStackTrace();
238 return null;
239 }
240 }
241
242 Result recode2(byte[] ia, int n) {
243 int len = ia.length;
244 ByteBuffer rib = ByteBuffer.wrap(ia);
245 ByteBuffer dib = dibs[len];
246 dib.clear(); dib.put(ia); dib.clear();
247 CharBuffer rob = robs[n];
248 CharBuffer dob = dobs[n];
249 equal(rob.limit(), n);
250 equal(dob.limit(), n);
251 check(dib.isDirect());
252 check(dob.isDirect());
253 Result r1 = recode(rib, rob);
254 Result r2 = recode(dib, dob);
255 if (r1 != null && r2 != null && ! Result.eq(r1, r2))
256 bug("Results differ for direct buffers: %s%n %s%n %s",
257 cs, r1, r2);
258 return r1;
259 }
260
261 Result test(byte[] ia) {
262 if (failed - failed0 >= maxCharsetFailures)
263 throw new TooManyFailures();
264
265 Result roomy = recode2(ia, maxBufSize - 1);
266 if (roomy == null) return roomy;
267 int olen = roomy.oa.length;
268 if (olen > 0) {
269 if (roomy.ipos == roomy.ia.length) {
270 Result perfectFit = recode2(ia, olen);
271 if (! Result.eq(roomy, perfectFit))
272 bug("Results differ: %s%n %s%n %s",
273 cs, roomy, perfectFit);
274 }
275 for (int i = 0; i < olen; i++) {
276 Result claustrophobic = recode2(ia, i);
277 if (claustrophobic == null) return roomy;
278 if (roomy.cr.isUnderflow() &&
279 ! claustrophobic.cr.isOverflow())
280 bug("Expected OVERFLOW: %s%n %s%n %s",
281 cs, roomy, claustrophobic);
282 }
283 }
284 return roomy;
285 }
286
287 void testExhaustively(byte[] prefix, int n) {
288 int len = prefix.length;
289 byte[] ia = Arrays.copyOf(prefix, len + 1);
290 for (int i = 0; i < 0x100; i++) {
291 ia[len] = (byte) i;
292 if (n == 1)
293 test(ia);
294 else
295 testExhaustively(ia, n - 1);
296 }
297 }
298
299 void testRandomly(byte[] prefix, int n) {
300 int len = prefix.length;
301 byte[] ia = Arrays.copyOf(prefix, len + n);
302 for (int i = 0; i < 5000; i++) {
303 for (int j = 0; j < n; j++)
304 ia[len + j] = randomByte();
305 test(ia);
306 }
307 }
308
309 void testPrefix(byte[] prefix) {
310 if (prefix.length > 0)
311 System.out.printf("Testing prefix %s%n", string(prefix));
312
313 test(prefix);
314
315 testExhaustively(prefix, 1);
316 testExhaustively(prefix, 2);
317 // Can you spare a week of CPU time?
318 // testExhaustively(cs, tester, prefix, 3);
319
320 testRandomly(prefix, 3);
321 testRandomly(prefix, 4);
322 }
323 }
324
325 private final static Random rnd = new Random();
326 private static byte randomByte() {
327 return (byte) rnd.nextInt(0x100);
328 }
329 private static byte[] randomBytes(int len) {
330 byte[] a = new byte[len];
331 for (int i = 0; i < len; i++)
332 a[i] = randomByte();
333 return a;
334 }
335
336 private static final byte SS2 = (byte) 0x8e;
337 private static final byte SS3 = (byte) 0x8f;
338 private static final byte ESC = (byte) 0x1b;
339 private static final byte SO = (byte) 0x0e;
340 private static final byte SI = (byte) 0x0f;
341
342 private final static byte[][] stateChangers = {
343 {SS2}, {SS3}, {SO}, {SI}
344 };
345
346 private final static byte[][]escapeSequences = {
347 {ESC, '(', 'B'},
348 {ESC, '(', 'I'},
349 {ESC, '(', 'J'},
350 {ESC, '$', '@'},
351 {ESC, '$', 'A'},
352 {ESC, '$', ')', 'A'},
353 {ESC, '$', ')', 'C'},
354 {ESC, '$', ')', 'G'},
355 {ESC, '$', '*', 'H'},
356 {ESC, '$', '+', 'I'},
357 {ESC, '$', 'B'},
358 {ESC, 'N'},
359 {ESC, 'O'},
360 {ESC, '$', '(', 'D'},
361 };
362
363 private static boolean isStateChanger(Charset cs, byte[] ia) {
364 Result r = new CharsetTester(cs).recode2(ia, 9);
365 return r == null ? false :
366 (r.cr.isUnderflow() &&
367 r.ipos == ia.length &&
368 r.oa.length == 0);
369 }
370
371 private final static byte[][] incompletePrefixes = {
372 {ESC},
373 {ESC, '('},
374 {ESC, '$'},
375 {ESC, '$', '(',},
376 };
377
378 private static boolean isIncompletePrefix(Charset cs, byte[] ia) {
379 Result r = new CharsetTester(cs).recode2(ia, 9);
380 return r == null ? false :
381 (r.cr.isUnderflow() &&
382 r.ipos == 0 &&
383 r.oa.length == 0);
384 }
385
386 private static void testCharset(Charset cs) throws Throwable {
387 final String csn = cs.name();
388
389 if (isBroken(csn)) {
390 System.out.printf("Skipping possibly broken charset %s%n", csn);
391 return;
392 }
393 System.out.println(csn);
394 CharsetTester tester = new CharsetTester(cs);
395
396 tester.testPrefix(new byte[0]);
397
398 if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) {
399 for (byte[] prefix : stateChangers)
400 if (isStateChanger(cs, prefix))
401 tester.testPrefix(prefix);
402
403 for (byte[] prefix : incompletePrefixes)
404 if (isIncompletePrefix(cs, prefix))
405 tester.testPrefix(prefix);
406
407 if (isIncompletePrefix(cs, new byte[] {ESC}))
408 for (byte[] prefix : escapeSequences)
409 if (isStateChanger(cs, prefix))
410 tester.testPrefix(prefix);
411 }
412 }
413
414 private static void realMain(String[] args) {
415 for (Charset cs : sort(Charset.availableCharsets().values())) {
416 try {
417 testCharset(cs);
418 } catch (TooManyFailures e) {
419 System.out.printf("Too many failures for %s%n", cs);
420 } catch (Throwable t) {
421 unexpected(t);
422 }
423 }
424 reporter.summarize();
425 }
426
427 //--------------------- Infrastructure ---------------------------
428 static volatile long passed = 0, failed = 0;
429 static void pass() {passed++;}
430 static void fail() {failed++; Thread.dumpStack();}
431 static void fail(String format, Object... args) {
432 System.out.println(String.format(format, args)); failed++;}
433 static void fail(String msg) {System.out.println(msg); fail();}
434 static void unexpected(Throwable t) {failed++; t.printStackTrace();}
435 static void check(boolean cond) {if (cond) pass(); else fail();}
436 static void equal(Object x, Object y) {
437 if (x == null ? y == null : x.equals(y)) pass();
438 else fail(x + " not equal to " + y);}
439 static void equal(int x, int y) {
440 if (x == y) pass();
441 else fail(x + " not equal to " + y);}
442 public static void main(String[] args) throws Throwable {
443 try {realMain(args);} catch (Throwable t) {unexpected(t);}
444 System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed);
445 if (failed > 0) throw new AssertionError("Some tests failed");}
446 }