1 /*
   2  * Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package java.lang;
  27 
  28 import java.io.UnsupportedEncodingException;
  29 import java.lang.ref.SoftReference;
  30 import java.nio.ByteBuffer;
  31 import java.nio.CharBuffer;
  32 import java.nio.charset.Charset;
  33 import java.nio.charset.CharsetDecoder;
  34 import java.nio.charset.CharsetEncoder;
  35 import java.nio.charset.CharacterCodingException;
  36 import java.nio.charset.CoderResult;
  37 import java.nio.charset.CodingErrorAction;
  38 import java.nio.charset.IllegalCharsetNameException;
  39 import java.nio.charset.UnsupportedCharsetException;
  40 import java.util.Arrays;
  41 import jdk.internal.HotSpotIntrinsicCandidate;
  42 import sun.nio.cs.HistoricallyNamedCharset;
  43 import sun.nio.cs.ArrayDecoder;
  44 import sun.nio.cs.ArrayEncoder;
  45 
  46 import static java.lang.String.LATIN1;
  47 import static java.lang.String.UTF16;
  48 import static java.lang.String.COMPACT_STRINGS;
  49 import static java.lang.Character.isSurrogate;
  50 import static java.lang.Character.highSurrogate;
  51 import static java.lang.Character.lowSurrogate;
  52 import static java.lang.Character.isSupplementaryCodePoint;
  53 import static java.lang.StringUTF16.putChar;
  54 
  55 /**
  56  * Utility class for string encoding and decoding.
  57  */
  58 
  59 class StringCoding {
  60 
  61     private StringCoding() { }
  62 
  63     /** The cached coders for each thread */
  64     private static final ThreadLocal<SoftReference<StringDecoder>> decoder =
  65         new ThreadLocal<>();
  66     private static final ThreadLocal<SoftReference<StringEncoder>> encoder =
  67         new ThreadLocal<>();
  68 
  69     private static final Charset ISO_8859_1 = sun.nio.cs.ISO_8859_1.INSTANCE;
  70     private static final Charset US_ASCII = sun.nio.cs.US_ASCII.INSTANCE;
  71     private static final Charset UTF_8 = sun.nio.cs.UTF_8.INSTANCE;
  72 
  73     private static <T> T deref(ThreadLocal<SoftReference<T>> tl) {
  74         SoftReference<T> sr = tl.get();
  75         if (sr == null)
  76             return null;
  77         return sr.get();
  78     }
  79 
  80     private static <T> void set(ThreadLocal<SoftReference<T>> tl, T ob) {
  81         tl.set(new SoftReference<>(ob));
  82     }
  83 
  84     // Trim the given byte array to the given length
  85     private static byte[] safeTrim(byte[] ba, int len, boolean isTrusted) {
  86         if (len == ba.length && (isTrusted || System.getSecurityManager() == null))
  87             return ba;
  88         else
  89             return Arrays.copyOf(ba, len);
  90     }
  91 
  92     private static int scale(int len, float expansionFactor) {
  93         // We need to perform double, not float, arithmetic; otherwise
  94         // we lose low order bits when len is larger than 2**24.
  95         return (int)(len * (double)expansionFactor);
  96     }
  97 
  98     private static Charset lookupCharset(String csn) {
  99         if (Charset.isSupported(csn)) {
 100             try {
 101                 return Charset.forName(csn);
 102             } catch (UnsupportedCharsetException x) {
 103                 throw new Error(x);
 104             }
 105         }
 106         return null;
 107     }
 108 
 109     static class Result {
 110         byte[] value;
 111         byte coder;
 112 
 113         Result with() {
 114             coder = COMPACT_STRINGS ? LATIN1 : UTF16;
 115             value = new byte[0];
 116             return this;
 117         }
 118 
 119         Result with(char[] val, int off, int len) {
 120             if (String.COMPACT_STRINGS) {
 121                 byte[] bs = StringUTF16.compress(val, off, len);
 122                 if (bs != null) {
 123                     value = bs;
 124                     coder = LATIN1;
 125                     return this;
 126                 }
 127             }
 128             coder = UTF16;
 129             value = StringUTF16.toBytes(val, off, len);
 130             return this;
 131         }
 132 
 133         Result with(byte[] val, byte coder) {
 134             this.coder = coder;
 135             value = val;
 136             return this;
 137         }
 138     }
 139 
 140     @HotSpotIntrinsicCandidate
 141     public static boolean hasNegatives(byte[] ba, int off, int len) {
 142         for (int i = off; i < off + len; i++) {
 143             if (ba[i] < 0) {
 144                 return true;
 145             }
 146         }
 147         return false;
 148     }
 149 
 150     // -- Decoding --
 151     static class StringDecoder {
 152         private final String requestedCharsetName;
 153         private final Charset cs;
 154         private final boolean isASCIICompatible;
 155         private final CharsetDecoder cd;
 156         protected final Result result;
 157 
 158         StringDecoder(Charset cs, String rcn) {
 159             this.requestedCharsetName = rcn;
 160             this.cs = cs;
 161             this.cd = cs.newDecoder()
 162                 .onMalformedInput(CodingErrorAction.REPLACE)
 163                 .onUnmappableCharacter(CodingErrorAction.REPLACE);
 164             this.result = new Result();
 165             this.isASCIICompatible = (cd instanceof ArrayDecoder) &&
 166                     ((ArrayDecoder)cd).isASCIICompatible();
 167         }
 168 
 169         String charsetName() {
 170             if (cs instanceof HistoricallyNamedCharset)
 171                 return ((HistoricallyNamedCharset)cs).historicalName();
 172             return cs.name();
 173         }
 174 
 175         final String requestedCharsetName() {
 176             return requestedCharsetName;
 177         }
 178 
 179         Result decode(byte[] ba, int off, int len) {
 180             if (len == 0) {
 181                 return result.with();
 182             }
 183             // fastpath for ascii compatible
 184             if (isASCIICompatible && !hasNegatives(ba, off, len)) {
 185                 if (COMPACT_STRINGS) {
 186                     return result.with(Arrays.copyOfRange(ba, off, off + len),
 187                                       LATIN1);
 188                 } else {
 189                     return result.with(StringLatin1.inflate(ba, off, len), UTF16);
 190                 }
 191             }
 192             int en = scale(len, cd.maxCharsPerByte());
 193             char[] ca = new char[en];
 194             if (cd instanceof ArrayDecoder) {
 195                 int clen = ((ArrayDecoder)cd).decode(ba, off, len, ca);
 196                 return result.with(ca, 0, clen);
 197             }
 198             cd.reset();
 199             ByteBuffer bb = ByteBuffer.wrap(ba, off, len);
 200             CharBuffer cb = CharBuffer.wrap(ca);
 201             try {
 202                 CoderResult cr = cd.decode(bb, cb, true);
 203                 if (!cr.isUnderflow())
 204                     cr.throwException();
 205                 cr = cd.flush(cb);
 206                 if (!cr.isUnderflow())
 207                     cr.throwException();
 208             } catch (CharacterCodingException x) {
 209                 // Substitution is always enabled,
 210                 // so this shouldn't happen
 211                 throw new Error(x);
 212             }
 213             return result.with(ca, 0, cb.position());
 214         }
 215     }
 216 
 217     static Result decode(String charsetName, byte[] ba, int off, int len)
 218         throws UnsupportedEncodingException
 219     {
 220         StringDecoder sd = deref(decoder);
 221         String csn = (charsetName == null) ? "ISO-8859-1" : charsetName;
 222         if ((sd == null) || !(csn.equals(sd.requestedCharsetName())
 223                               || csn.equals(sd.charsetName()))) {
 224             sd = null;
 225             try {
 226                 Charset cs = lookupCharset(csn);
 227                 if (cs != null) {
 228                     if (cs == UTF_8) {
 229                         return decodeUTF8(ba, off, len, true);
 230                     }
 231                     if (cs == ISO_8859_1) {
 232                         return decodeLatin1(ba, off, len);
 233                     }
 234                     if (cs == US_ASCII) {
 235                         return decodeASCII(ba, off, len);
 236                     }
 237                     sd = new StringDecoder(cs, csn);
 238                 }
 239             } catch (IllegalCharsetNameException x) {}
 240             if (sd == null)
 241                 throw new UnsupportedEncodingException(csn);
 242             set(decoder, sd);
 243         }
 244         return sd.decode(ba, off, len);
 245     }
 246 
 247     static Result decode(Charset cs, byte[] ba, int off, int len) {
 248         if (cs == UTF_8) {
 249             return decodeUTF8(ba, off, len, true);
 250         }
 251         if (cs == ISO_8859_1) {
 252             return decodeLatin1(ba, off, len);
 253         }
 254         if (cs == US_ASCII) {
 255             return decodeASCII(ba, off, len);
 256         }
 257 
 258         // (1)We never cache the "external" cs, the only benefit of creating
 259         // an additional StringDe/Encoder object to wrap it is to share the
 260         // de/encode() method. These SD/E objects are short-lived, the young-gen
 261         // gc should be able to take care of them well. But the best approach
 262         // is still not to generate them if not really necessary.
 263         // (2)The defensive copy of the input byte/char[] has a big performance
 264         // impact, as well as the outgoing result byte/char[]. Need to do the
 265         // optimization check of (sm==null && classLoader0==null) for both.
 266         // (3)There might be a timing gap in isTrusted setting. getClassLoader0()
 267         // is only checked (and then isTrusted gets set) when (SM==null). It is
 268         // possible that the SM==null for now but then SM is NOT null later
 269         // when safeTrim() is invoked...the "safe" way to do is to redundant
 270         // check (... && (isTrusted || SM == null || getClassLoader0())) in trim
 271         // but it then can be argued that the SM is null when the operation
 272         // is started...
 273         CharsetDecoder cd = cs.newDecoder();
 274         // ascii fastpath
 275         if ((cd instanceof ArrayDecoder) &&
 276             ((ArrayDecoder)cd).isASCIICompatible() && !hasNegatives(ba, off, len)) {
 277             return decodeLatin1(ba, off, len);
 278         }
 279         int en = scale(len, cd.maxCharsPerByte());
 280         if (len == 0) {
 281             return new Result().with();
 282         }
 283         cd.onMalformedInput(CodingErrorAction.REPLACE)
 284           .onUnmappableCharacter(CodingErrorAction.REPLACE)
 285           .reset();
 286         char[] ca = new char[en];
 287         if (cd instanceof ArrayDecoder) {
 288             int clen = ((ArrayDecoder)cd).decode(ba, off, len, ca);
 289             return new Result().with(ca, 0, clen);
 290         }
 291         if (cs.getClass().getClassLoader0() != null &&
 292             System.getSecurityManager() != null) {
 293             ba = Arrays.copyOfRange(ba, off, off + len);
 294             off = 0;
 295         }
 296         ByteBuffer bb = ByteBuffer.wrap(ba, off, len);
 297         CharBuffer cb = CharBuffer.wrap(ca);
 298         try {
 299             CoderResult cr = cd.decode(bb, cb, true);
 300             if (!cr.isUnderflow())
 301                 cr.throwException();
 302             cr = cd.flush(cb);
 303             if (!cr.isUnderflow())
 304                 cr.throwException();
 305         } catch (CharacterCodingException x) {
 306             // Substitution is always enabled,
 307             // so this shouldn't happen
 308             throw new Error(x);
 309         }
 310         return new Result().with(ca, 0, cb.position());
 311     }
 312 
 313     static Result decode(byte[] ba, int off, int len) {
 314         Charset cs = Charset.defaultCharset();
 315         if (cs == UTF_8) {
 316             return decodeUTF8(ba, off, len, true);
 317         }
 318         if (cs == ISO_8859_1) {
 319             return decodeLatin1(ba, off, len);
 320         }
 321         if (cs == US_ASCII) {
 322             return decodeASCII(ba, off, len);
 323         }
 324         StringDecoder sd = deref(decoder);
 325         if (sd == null || !cs.name().equals(sd.cs.name())) {
 326             sd = new StringDecoder(cs, cs.name());
 327             set(decoder, sd);
 328         }
 329         return sd.decode(ba, off, len);
 330     }
 331 
 332     // -- Encoding --
 333     private static class StringEncoder {
 334         private Charset cs;
 335         private CharsetEncoder ce;
 336         private final boolean isASCIICompatible;
 337         private final String requestedCharsetName;
 338         private final boolean isTrusted;
 339 
 340         private StringEncoder(Charset cs, String rcn) {
 341             this.requestedCharsetName = rcn;
 342             this.cs = cs;
 343             this.ce = cs.newEncoder()
 344                 .onMalformedInput(CodingErrorAction.REPLACE)
 345                 .onUnmappableCharacter(CodingErrorAction.REPLACE);
 346             this.isTrusted = (cs.getClass().getClassLoader0() == null);
 347             this.isASCIICompatible = (ce instanceof ArrayEncoder) &&
 348                     ((ArrayEncoder)ce).isASCIICompatible();
 349         }
 350 
 351         String charsetName() {
 352             if (cs instanceof HistoricallyNamedCharset)
 353                 return ((HistoricallyNamedCharset)cs).historicalName();
 354             return cs.name();
 355         }
 356 
 357         final String requestedCharsetName() {
 358             return requestedCharsetName;
 359         }
 360 
 361         byte[] encode(byte coder, byte[] val) {
 362             // fastpath for ascii compatible
 363             if (coder == LATIN1 && isASCIICompatible &&
 364                 !hasNegatives(val, 0, val.length)) {
 365                 return Arrays.copyOf(val, val.length);
 366             }
 367             int len = val.length >> coder;  // assume LATIN1=0/UTF16=1;
 368             int en = scale(len, ce.maxBytesPerChar());
 369             byte[] ba = new byte[en];
 370             if (len == 0) {
 371                 return ba;
 372             }
 373             if (ce instanceof ArrayEncoder) {
 374                 int blen = (coder == LATIN1 ) ? ((ArrayEncoder)ce).encodeFromLatin1(val, 0, len, ba)
 375                                               : ((ArrayEncoder)ce).encodeFromUTF16(val, 0, len, ba);
 376                 if (blen != -1) {
 377                     return safeTrim(ba, blen, isTrusted);
 378                 }
 379             }
 380             char[] ca = (coder == LATIN1 ) ? StringLatin1.toChars(val)
 381                                            : StringUTF16.toChars(val);
 382             ce.reset();
 383             ByteBuffer bb = ByteBuffer.wrap(ba);
 384             CharBuffer cb = CharBuffer.wrap(ca, 0, len);
 385             try {
 386                 CoderResult cr = ce.encode(cb, bb, true);
 387                 if (!cr.isUnderflow())
 388                     cr.throwException();
 389                 cr = ce.flush(bb);
 390                 if (!cr.isUnderflow())
 391                     cr.throwException();
 392             } catch (CharacterCodingException x) {
 393                 // Substitution is always enabled,
 394                 // so this shouldn't happen
 395                 throw new Error(x);
 396             }
 397             return safeTrim(ba, bb.position(), isTrusted);
 398         }
 399     }
 400 
 401     static byte[] encode(String charsetName, byte coder, byte[] val)
 402         throws UnsupportedEncodingException
 403     {
 404         StringEncoder se = deref(encoder);
 405         String csn = (charsetName == null) ? "ISO-8859-1" : charsetName;
 406         if ((se == null) || !(csn.equals(se.requestedCharsetName())
 407                               || csn.equals(se.charsetName()))) {
 408             se = null;
 409             try {
 410                 Charset cs = lookupCharset(csn);
 411                 if (cs != null) {
 412                     if (cs == UTF_8) {
 413                         return encodeUTF8(coder, val, true);
 414                     }
 415                     if (cs == ISO_8859_1) {
 416                         return encode8859_1(coder, val);
 417                     }
 418                     if (cs == US_ASCII) {
 419                         return encodeASCII(coder, val);
 420                     }
 421                     se = new StringEncoder(cs, csn);
 422                 }
 423             } catch (IllegalCharsetNameException x) {}
 424             if (se == null) {
 425                 throw new UnsupportedEncodingException (csn);
 426             }
 427             set(encoder, se);
 428         }
 429         return se.encode(coder, val);
 430     }
 431 
 432     static byte[] encode(Charset cs, byte coder, byte[] val) {
 433         if (cs == UTF_8) {
 434             return encodeUTF8(coder, val, true);
 435         }
 436         if (cs == ISO_8859_1) {
 437             return encode8859_1(coder, val);
 438         }
 439         if (cs == US_ASCII) {
 440             return encodeASCII(coder, val);
 441         }
 442         CharsetEncoder ce = cs.newEncoder();
 443         // fastpath for ascii compatible
 444         if (coder == LATIN1 && (((ce instanceof ArrayEncoder) &&
 445                                  ((ArrayEncoder)ce).isASCIICompatible() &&
 446                                  !hasNegatives(val, 0, val.length)))) {
 447             return Arrays.copyOf(val, val.length);
 448         }
 449         int len = val.length >> coder;  // assume LATIN1=0/UTF16=1;
 450         int en = scale(len, ce.maxBytesPerChar());
 451         byte[] ba = new byte[en];
 452         if (len == 0) {
 453             return ba;
 454         }
 455         ce.onMalformedInput(CodingErrorAction.REPLACE)
 456           .onUnmappableCharacter(CodingErrorAction.REPLACE)
 457           .reset();
 458         if (ce instanceof ArrayEncoder) {
 459             int blen = (coder == LATIN1 ) ? ((ArrayEncoder)ce).encodeFromLatin1(val, 0, len, ba)
 460                                           : ((ArrayEncoder)ce).encodeFromUTF16(val, 0, len, ba);
 461             if (blen != -1) {
 462                 return safeTrim(ba, blen, true);
 463             }
 464         }
 465         boolean isTrusted = cs.getClass().getClassLoader0() == null ||
 466                             System.getSecurityManager() == null;
 467         char[] ca = (coder == LATIN1 ) ? StringLatin1.toChars(val)
 468                                        : StringUTF16.toChars(val);
 469         ByteBuffer bb = ByteBuffer.wrap(ba);
 470         CharBuffer cb = CharBuffer.wrap(ca, 0, len);
 471         try {
 472             CoderResult cr = ce.encode(cb, bb, true);
 473             if (!cr.isUnderflow())
 474                 cr.throwException();
 475             cr = ce.flush(bb);
 476             if (!cr.isUnderflow())
 477                 cr.throwException();
 478         } catch (CharacterCodingException x) {
 479             throw new Error(x);
 480         }
 481         return safeTrim(ba, bb.position(), isTrusted);
 482     }
 483 
 484     static byte[] encode(byte coder, byte[] val) {
 485         Charset cs = Charset.defaultCharset();
 486         if (cs == UTF_8) {
 487             return encodeUTF8(coder, val, true);
 488         }
 489         if (cs == ISO_8859_1) {
 490             return encode8859_1(coder, val);
 491         }
 492         if (cs == US_ASCII) {
 493             return encodeASCII(coder, val);
 494         }
 495         StringEncoder se = deref(encoder);
 496         if (se == null || !cs.name().equals(se.cs.name())) {
 497             se = new StringEncoder(cs, cs.name());
 498             set(encoder, se);
 499         }
 500         return se.encode(coder, val);
 501     }
 502 
 503     /**
 504      *  Print a message directly to stderr, bypassing all character conversion
 505      *  methods.
 506      *  @param msg  message to print
 507      */
 508     private static native void err(String msg);
 509 
 510      /* The cached Result for each thread */
 511     private static final ThreadLocal<StringCoding.Result>
 512         resultCached = new ThreadLocal<>() {
 513             protected StringCoding.Result initialValue() {
 514                 return new StringCoding.Result();
 515             }};
 516 
 517     ////////////////////////// ascii //////////////////////////////
 518 
 519     private static Result decodeASCII(byte[] ba, int off, int len) {
 520         Result result = resultCached.get();
 521         if (COMPACT_STRINGS && !hasNegatives(ba, off, len)) {
 522             return result.with(Arrays.copyOfRange(ba, off, off + len),
 523                                LATIN1);
 524         }
 525         byte[] dst = new byte[len<<1];
 526         int dp = 0;
 527         while (dp < len) {
 528             int b = ba[off++];
 529             putChar(dst, dp++, (b >= 0) ? (char)b : repl);
 530         }
 531         return result.with(dst, UTF16);
 532     }
 533 
 534     private static byte[] encodeASCII(byte coder, byte[] val) {
 535         if (coder == LATIN1) {
 536             byte[] dst = new byte[val.length];
 537             for (int i = 0; i < val.length; i++) {
 538                 if (val[i] < 0) {
 539                     dst[i] = '?';
 540                 } else {
 541                     dst[i] = val[i];
 542                 }
 543             }
 544             return dst;
 545         }
 546         int len = val.length >> 1;
 547         byte[] dst = new byte[len];
 548         int dp = 0;
 549         for (int i = 0; i < len; i++) {
 550             char c = StringUTF16.getChar(val, i);
 551             if (c < 0x80) {
 552                 dst[dp++] = (byte)c;
 553                 continue;
 554             }
 555             if (Character.isHighSurrogate(c) && i + 1 < len &&
 556                 Character.isLowSurrogate(StringUTF16.getChar(val, i + 1))) {
 557                 i++;
 558             }
 559             dst[dp++] = '?';
 560         }
 561         if (len == dp) {
 562             return dst;
 563         }
 564         return Arrays.copyOf(dst, dp);
 565     }
 566 
 567     ////////////////////////// latin1/8859_1 ///////////////////////////
 568 
 569     private static Result decodeLatin1(byte[] ba, int off, int len) {
 570        Result result = resultCached.get();
 571        if (COMPACT_STRINGS) {
 572            return result.with(Arrays.copyOfRange(ba, off, off + len), LATIN1);
 573        } else {
 574            return result.with(StringLatin1.inflate(ba, off, len), UTF16);
 575        }
 576     }
 577 
 578     @HotSpotIntrinsicCandidate
 579     private static int implEncodeISOArray(byte[] sa, int sp,
 580                                           byte[] da, int dp, int len) {
 581         int i = 0;
 582         for (; i < len; i++) {
 583             char c = StringUTF16.getChar(sa, sp++);
 584             if (c > '\u00FF')
 585                 break;
 586             da[dp++] = (byte)c;
 587         }
 588         return i;
 589     }
 590 
 591     private static byte[] encode8859_1(byte coder, byte[] val) {
 592         if (coder == LATIN1) {
 593             return Arrays.copyOf(val, val.length);
 594         }
 595         int len = val.length >> 1;
 596         byte[] dst = new byte[len];
 597         int dp = 0;
 598         int sp = 0;
 599         int sl = len;
 600         while (sp < sl) {
 601             int ret = implEncodeISOArray(val, sp, dst, dp, len);
 602             sp = sp + ret;
 603             dp = dp + ret;
 604             if (ret != len) {
 605                 char c = StringUTF16.getChar(val, sp++);
 606                 if (Character.isHighSurrogate(c) && sp < sl &&
 607                     Character.isLowSurrogate(StringUTF16.getChar(val, sp))) {
 608                     sp++;
 609                 }
 610                 dst[dp++] = '?';
 611                 len = sl - sp;
 612             }
 613         }
 614         if (dp == dst.length) {
 615             return dst;
 616         }
 617         return Arrays.copyOf(dst, dp);
 618     }
 619 
 620     //////////////////////////////// utf8 ////////////////////////////////////
 621 
 622     private static boolean isNotContinuation(int b) {
 623         return (b & 0xc0) != 0x80;
 624     }
 625 
 626     private static boolean isMalformed3(int b1, int b2, int b3) {
 627         return (b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) ||
 628                (b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80;
 629     }
 630 
 631     private static boolean isMalformed3_2(int b1, int b2) {
 632         return (b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) ||
 633                (b2 & 0xc0) != 0x80;
 634     }
 635 
 636     private static boolean isMalformed4(int b2, int b3, int b4) {
 637         return (b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80 ||
 638                (b4 & 0xc0) != 0x80;
 639     }
 640 
 641     private static boolean isMalformed4_2(int b1, int b2) {
 642         return (b1 == 0xf0 && (b2  < 0x90 || b2 > 0xbf)) ||
 643                (b1 == 0xf4 && (b2 & 0xf0) != 0x80) ||
 644                (b2 & 0xc0) != 0x80;
 645     }
 646 
 647     private static boolean isMalformed4_3(int b3) {
 648         return (b3 & 0xc0) != 0x80;
 649     }
 650 
 651     // for nb == 3/4
 652     private static int malformedN(byte[] src, int sp, int nb) {
 653         if (nb == 3) {
 654             int b1 = src[sp++];
 655             int b2 = src[sp++];    // no need to lookup b3
 656             return ((b1 == (byte)0xe0 && (b2 & 0xe0) == 0x80) ||
 657                     isNotContinuation(b2)) ? 1 : 2;
 658         } else if (nb == 4) { // we don't care the speed here
 659             int b1 = src[sp++] & 0xff;
 660             int b2 = src[sp++] & 0xff;
 661             if (b1 > 0xf4 ||
 662                 (b1 == 0xf0 && (b2 < 0x90 || b2 > 0xbf)) ||
 663                 (b1 == 0xf4 && (b2 & 0xf0) != 0x80) ||
 664                 isNotContinuation(b2))
 665                 return 1;
 666             if (isNotContinuation(src[sp++]))
 667                 return 2;
 668             return 3;
 669         }
 670         assert false;
 671         return -1;
 672     }
 673 
 674     private static void throwMalformed(int off, int nb) {
 675         throw new IllegalArgumentException("malformed input off : " + off +
 676                                            ", length : " + nb);
 677     }
 678 
 679     private static char repl = '\ufffd';
 680 
 681     private static Result decodeUTF8(byte[] src, int sp, int len, boolean doReplace) {
 682         // ascii-bais, which has a relative impact to the non-ascii-only bytes
 683         if (COMPACT_STRINGS && !hasNegatives(src, sp, len))
 684             return resultCached.get().with(Arrays.copyOfRange(src, sp, sp + len),
 685                                            LATIN1);
 686         return decodeUTF8_0(src, sp, len, doReplace);
 687     }
 688 
 689     private static Result decodeUTF8_0(byte[] src, int sp, int len, boolean doReplace) {
 690         Result ret = resultCached.get();
 691 
 692         int sl = sp + len;
 693         int dp = 0;
 694         byte[] dst = new byte[len];
 695 
 696         if (COMPACT_STRINGS) {
 697             while (sp < sl) {
 698                 int b1 = src[sp];
 699                 if (b1 >= 0) {
 700                     dst[dp++] = (byte)b1;
 701                     sp++;
 702                     continue;
 703                 }
 704                 if ((b1 == (byte)0xc2 || b1 == (byte)0xc3) &&
 705                     sp + 1 < sl) {
 706                     int b2 = src[sp + 1];
 707                     if (!isNotContinuation(b2)) {
 708                         dst[dp++] = (byte)(((b1 << 6) ^ b2)^
 709                                            (((byte) 0xC0 << 6) ^
 710                                            ((byte) 0x80 << 0)));
 711                         sp += 2;
 712                         continue;
 713                     }
 714                 }
 715                 // anything not a latin1, including the repl
 716                 // we have to go with the utf16
 717                 break;
 718             }
 719             if (sp == sl) {
 720                 if (dp != dst.length) {
 721                     dst = Arrays.copyOf(dst, dp);
 722                 }
 723                 return ret.with(dst, LATIN1);
 724             }
 725         }
 726         if (dp == 0) {
 727             dst = new byte[len << 1];
 728         } else {
 729             byte[] buf = new byte[len << 1];
 730             StringLatin1.inflate(dst, 0, buf, 0, dp);
 731             dst = buf;
 732         }
 733         while (sp < sl) {
 734             int b1 = src[sp++];
 735             if (b1 >= 0) {
 736                 putChar(dst, dp++, (char) b1);
 737             } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) {
 738                 if (sp < sl) {
 739                     int b2 = src[sp++];
 740                     if (isNotContinuation(b2)) {
 741                         if (!doReplace) {
 742                             throwMalformed(sp - 1, 1);
 743                         }
 744                         putChar(dst, dp++, repl);
 745                         sp--;
 746                     } else {
 747                         putChar(dst, dp++, (char)(((b1 << 6) ^ b2)^
 748                                                   (((byte) 0xC0 << 6) ^
 749                                                   ((byte) 0x80 << 0))));
 750                     }
 751                     continue;
 752                 }
 753                 if (!doReplace) {
 754                     throwMalformed(sp, 1);  // underflow()
 755                 }
 756                 putChar(dst, dp++, repl);
 757                 break;
 758             } else if ((b1 >> 4) == -2) {
 759                 if (sp + 1 < sl) {
 760                     int b2 = src[sp++];
 761                     int b3 = src[sp++];
 762                     if (isMalformed3(b1, b2, b3)) {
 763                         if (!doReplace) {
 764                             throwMalformed(sp - 3, 3);
 765                         }
 766                         putChar(dst, dp++, repl);
 767                         sp -= 3;
 768                         sp += malformedN(src, sp, 3);
 769                     } else {
 770                         char c = (char)((b1 << 12) ^
 771                                         (b2 <<  6) ^
 772                                         (b3 ^
 773                                          (((byte) 0xE0 << 12) ^
 774                                          ((byte) 0x80 <<  6) ^
 775                                          ((byte) 0x80 <<  0))));
 776                         if (isSurrogate(c)) {
 777                             if (!doReplace) {
 778                                 throwMalformed(sp - 3, 3);
 779                             }
 780                             putChar(dst, dp++, repl);
 781                         } else {
 782                             putChar(dst, dp++, c);
 783                         }
 784                     }
 785                     continue;
 786                 }
 787                 if (sp  < sl && isMalformed3_2(b1, src[sp])) {
 788                     if (!doReplace) {
 789                         throwMalformed(sp - 1, 2);
 790                     }
 791                     putChar(dst, dp++, repl);
 792                     continue;
 793                 }
 794                 if (!doReplace){
 795                     throwMalformed(sp, 1);
 796                 }
 797                 putChar(dst, dp++, repl);
 798                 break;
 799             } else if ((b1 >> 3) == -2) {
 800                 if (sp + 2 < sl) {
 801                     int b2 = src[sp++];
 802                     int b3 = src[sp++];
 803                     int b4 = src[sp++];
 804                     int uc = ((b1 << 18) ^
 805                               (b2 << 12) ^
 806                               (b3 <<  6) ^
 807                               (b4 ^
 808                                (((byte) 0xF0 << 18) ^
 809                                ((byte) 0x80 << 12) ^
 810                                ((byte) 0x80 <<  6) ^
 811                                ((byte) 0x80 <<  0))));
 812                     if (isMalformed4(b2, b3, b4) ||
 813                         !isSupplementaryCodePoint(uc)) { // shortest form check
 814                         if (!doReplace) {
 815                             throwMalformed(sp - 4, 4);
 816                         }
 817                         putChar(dst, dp++, repl);
 818                         sp -= 4;
 819                         sp += malformedN(src, sp, 4);
 820                     } else {
 821                         putChar(dst, dp++, highSurrogate(uc));
 822                         putChar(dst, dp++, lowSurrogate(uc));
 823                     }
 824                     continue;
 825                 }
 826                 b1 &= 0xff;
 827                 if (b1 > 0xf4 ||
 828                     sp  < sl && isMalformed4_2(b1, src[sp] & 0xff)) {
 829                     if (!doReplace) {
 830                         throwMalformed(sp - 1, 1);  // or 2
 831                     }
 832                     putChar(dst, dp++, repl);
 833                     continue;
 834                 }
 835                 if (!doReplace) {
 836                     throwMalformed(sp - 1, 1);
 837                 }
 838                 sp++;
 839                 putChar(dst, dp++, repl);
 840                 if (sp  < sl && isMalformed4_3(src[sp])) {
 841                     continue;
 842                 }
 843                 break;
 844             } else {
 845                 if (!doReplace) {
 846                     throwMalformed(sp - 1, 1);
 847                 }
 848                 putChar(dst, dp++, repl);
 849             }
 850         }
 851         if (dp != len) {
 852             dst = Arrays.copyOf(dst, dp << 1);
 853         }
 854         return ret.with(dst, UTF16);
 855     }
 856 
 857     private static byte[] encodeUTF8(byte coder, byte[] val, boolean doReplace) {
 858         if (coder == UTF16)
 859             return encodeUTF8_UTF16(val, doReplace);
 860 
 861         if (!hasNegatives(val, 0, val.length))
 862             return Arrays.copyOf(val, val.length);
 863 
 864         int dp = 0;
 865         byte[] dst = new byte[val.length << 1];
 866         for (int sp = 0; sp < val.length; sp++) {
 867             byte c = val[sp];
 868             if (c < 0) {
 869                 dst[dp++] = (byte)(0xc0 | ((c & 0xff) >> 6));
 870                 dst[dp++] = (byte)(0x80 | (c & 0x3f));
 871             } else {
 872                 dst[dp++] = c;
 873             }
 874         }
 875         if (dp == dst.length)
 876             return dst;
 877         return Arrays.copyOf(dst, dp);
 878     }
 879 
 880     private static byte[] encodeUTF8_UTF16(byte[] val, boolean doReplace) {
 881         int dp = 0;
 882         int sp = 0;
 883         int sl = val.length >> 1;
 884         byte[] dst = new byte[sl * 3];
 885         char c;
 886         while (sp < sl && (c = StringUTF16.getChar(val, sp)) < '\u0080') {
 887             // ascii fast loop;
 888             dst[dp++] = (byte)c;
 889             sp++;
 890         }
 891         while (sp < sl) {
 892             c = StringUTF16.getChar(val, sp++);
 893             if (c < 0x80) {
 894                 dst[dp++] = (byte)c;
 895             } else if (c < 0x800) {
 896                 dst[dp++] = (byte)(0xc0 | (c >> 6));
 897                 dst[dp++] = (byte)(0x80 | (c & 0x3f));
 898             } else if (Character.isSurrogate(c)) {
 899                 int uc = -1;
 900                 char c2;
 901                 if (Character.isHighSurrogate(c) && sp < sl &&
 902                     Character.isLowSurrogate(c2 = StringUTF16.getChar(val, sp))) {
 903                     uc = Character.toCodePoint(c, c2);
 904                 }
 905                 if (uc < 0) {
 906                     if (doReplace) {
 907                         dst[dp++] = '?';
 908                     } else {
 909                         throwMalformed(sp - 1, 1); // or 2, does not matter here
 910                     }
 911                 } else {
 912                     dst[dp++] = (byte)(0xf0 | ((uc >> 18)));
 913                     dst[dp++] = (byte)(0x80 | ((uc >> 12) & 0x3f));
 914                     dst[dp++] = (byte)(0x80 | ((uc >>  6) & 0x3f));
 915                     dst[dp++] = (byte)(0x80 | (uc & 0x3f));
 916                     sp++;  // 2 chars
 917                 }
 918             } else {
 919                 // 3 bytes, 16 bits
 920                 dst[dp++] = (byte)(0xe0 | ((c >> 12)));
 921                 dst[dp++] = (byte)(0x80 | ((c >>  6) & 0x3f));
 922                 dst[dp++] = (byte)(0x80 | (c & 0x3f));
 923             }
 924         }
 925         if (dp == dst.length) {
 926             return dst;
 927         }
 928         return Arrays.copyOf(dst, dp);
 929     }
 930 
 931     ////////////////////// for j.u.z.ZipCoder //////////////////////////
 932 
 933     /*
 934      * Throws iae, instead of replacing, if malformed or unmappble.
 935      */
 936     static String newStringUTF8NoRepl(byte[] src, int off, int len) {
 937         if (COMPACT_STRINGS && !hasNegatives(src, off, len))
 938             return new String(Arrays.copyOfRange(src, off, off + len), LATIN1);
 939         Result ret = decodeUTF8_0(src, off, len, false);
 940         return new String(ret.value, ret.coder);
 941     }
 942 
 943     /*
 944      * Throws iae, instead of replacing, if unmappble.
 945      */
 946     static byte[] getBytesUTF8NoRepl(String s) {
 947         return encodeUTF8(s.coder(), s.value(), false);
 948     }
 949 }