rev 16767 : 8175561: Memory churn in jimage code affects startup after resource encapsulation changes
Reviewed-by: jlaskey
rev 16768 : imported patch imgstr_oneup

   1 /*
   2  * Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.internal.jimage;
  26 
  27 import java.io.ByteArrayInputStream;
  28 import java.io.IOException;
  29 import java.io.InputStream;
  30 import java.lang.reflect.InvocationTargetException;
  31 import java.lang.reflect.Method;
  32 import java.nio.ByteBuffer;
  33 import java.nio.ByteOrder;
  34 import java.nio.IntBuffer;
  35 import java.nio.channels.FileChannel;
  36 import java.nio.file.Path;
  37 import java.nio.file.StandardOpenOption;
  38 import java.security.AccessController;
  39 import java.security.PrivilegedAction;
  40 import java.util.Objects;
  41 import java.util.stream.IntStream;
  42 import jdk.internal.jimage.decompressor.Decompressor;
  43 
  44 /**
  45  * @implNote This class needs to maintain JDK 8 source compatibility.
  46  *
  47  * It is used internally in the JDK to implement jimage/jrtfs access,
  48  * but also compiled and delivered as part of the jrtfs.jar to support access
  49  * to the jimage file provided by the shipped JDK by tools running on JDK 8.
  50  */
  51 public class BasicImageReader implements AutoCloseable {
  52     private static boolean isSystemProperty(String key, String value, String def) {
  53         // No lambdas during bootstrap
  54         return AccessController.doPrivileged(
  55             new PrivilegedAction<Boolean>() {
  56                 @Override
  57                 public Boolean run() {
  58                     return value.equals(System.getProperty(key, def));
  59                 }
  60             });
  61     }
  62 
  63     static private final boolean IS_64_BIT =
  64             isSystemProperty("sun.arch.data.model", "64", "32");
  65     static private final boolean USE_JVM_MAP =
  66             isSystemProperty("jdk.image.use.jvm.map", "true", "true");
  67     static private final boolean MAP_ALL =
  68             isSystemProperty("jdk.image.map.all", "true", IS_64_BIT ? "true" : "false");
  69 
  70     private final Path imagePath;
  71     private final ByteOrder byteOrder;
  72     private final String name;
  73     private final ByteBuffer memoryMap;
  74     private final FileChannel channel;
  75     private final ImageHeader header;
  76     private final long indexSize;
  77     private final IntBuffer redirect;
  78     private final IntBuffer offsets;
  79     private final ByteBuffer locations;
  80     private final ByteBuffer strings;
  81     private final ImageStringsReader stringsReader;
  82     private final Decompressor decompressor;
  83 
  84     protected BasicImageReader(Path path, ByteOrder byteOrder)
  85             throws IOException {
  86         this.imagePath = Objects.requireNonNull(path);
  87         this.byteOrder = Objects.requireNonNull(byteOrder);
  88         this.name = this.imagePath.toString();
  89 
  90         ByteBuffer map;
  91 
  92         if (USE_JVM_MAP && BasicImageReader.class.getClassLoader() == null) {
  93             // Check to see if the jvm has opened the file using libjimage
  94             // native entry when loading the image for this runtime
  95             map = NativeImageBuffer.getNativeMap(name);
  96          } else {
  97             map = null;
  98         }
  99 
 100         // Open the file only if no memory map yet or is 32 bit jvm
 101         if (map != null && MAP_ALL) {
 102             channel = null;
 103         } else {
 104             channel = FileChannel.open(imagePath, StandardOpenOption.READ);
 105             // No lambdas during bootstrap
 106             AccessController.doPrivileged(new PrivilegedAction<Void>() {
 107                 @Override
 108                 public Void run() {
 109                     if (BasicImageReader.class.getClassLoader() == null) {
 110                         try {
 111                             Class<?> fileChannelImpl =
 112                                 Class.forName("sun.nio.ch.FileChannelImpl");
 113                             Method setUninterruptible =
 114                                     fileChannelImpl.getMethod("setUninterruptible");
 115                             setUninterruptible.invoke(channel);
 116                         } catch (ClassNotFoundException |
 117                                  NoSuchMethodException |
 118                                  IllegalAccessException |
 119                                  InvocationTargetException ex) {
 120                             // fall thru - will only happen on JDK-8 systems where this code
 121                             // is only used by tools using jrt-fs (non-critical.)
 122                         }
 123                     }
 124 
 125                     return null;
 126                 }
 127             });
 128         }
 129 
 130         // If no memory map yet and 64 bit jvm then memory map entire file
 131         if (MAP_ALL && map == null) {
 132             map = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
 133         }
 134 
 135         // Assume we have a memory map to read image file header
 136         ByteBuffer headerBuffer = map;
 137         int headerSize = ImageHeader.getHeaderSize();
 138 
 139         // If no memory map then read header from image file
 140         if (headerBuffer == null) {
 141             headerBuffer = ByteBuffer.allocateDirect(headerSize);
 142             if (channel.read(headerBuffer, 0L) == headerSize) {
 143                 headerBuffer.rewind();
 144             } else {
 145                 throw new IOException("\"" + name + "\" is not an image file");
 146             }
 147         } else if (headerBuffer.capacity() < headerSize) {
 148             throw new IOException("\"" + name + "\" is not an image file");
 149         }
 150 
 151         // Interpret the image file header
 152         header = readHeader(intBuffer(headerBuffer, 0, headerSize));
 153         indexSize = header.getIndexSize();
 154 
 155         // If no memory map yet then must be 32 bit jvm not previously mapped
 156         if (map == null) {
 157             // Just map the image index
 158             map = channel.map(FileChannel.MapMode.READ_ONLY, 0, indexSize);
 159         }
 160 
 161         memoryMap = map.asReadOnlyBuffer();
 162 
 163         // Interpret the image index
 164         if (memoryMap.capacity() < indexSize) {
 165             throw new IOException("The image file \"" + name + "\" is corrupted");
 166         }
 167         redirect = intBuffer(memoryMap, header.getRedirectOffset(), header.getRedirectSize());
 168         offsets = intBuffer(memoryMap, header.getOffsetsOffset(), header.getOffsetsSize());
 169         locations = slice(memoryMap, header.getLocationsOffset(), header.getLocationsSize());
 170         strings = slice(memoryMap, header.getStringsOffset(), header.getStringsSize());
 171 
 172         stringsReader = new ImageStringsReader(this);
 173         decompressor = new Decompressor();
 174     }
 175 
 176     protected BasicImageReader(Path imagePath) throws IOException {
 177         this(imagePath, ByteOrder.nativeOrder());
 178     }
 179 
 180     public static BasicImageReader open(Path imagePath) throws IOException {
 181         return new BasicImageReader(imagePath, ByteOrder.nativeOrder());
 182     }
 183 
 184     public ImageHeader getHeader() {
 185         return header;
 186     }
 187 
 188     private ImageHeader readHeader(IntBuffer buffer) throws IOException {
 189         ImageHeader result = ImageHeader.readFrom(buffer);
 190 
 191         if (result.getMagic() != ImageHeader.MAGIC) {
 192             throw new IOException("\"" + name + "\" is not an image file");
 193         }
 194 
 195         if (result.getMajorVersion() != ImageHeader.MAJOR_VERSION ||
 196             result.getMinorVersion() != ImageHeader.MINOR_VERSION) {
 197             throw new IOException("The image file \"" + name + "\" is not " +
 198                 "the correct version. Major: " + result.getMajorVersion() +
 199                 ". Minor: " + result.getMinorVersion());
 200         }
 201 
 202         return result;
 203     }
 204 
 205     private static ByteBuffer slice(ByteBuffer buffer, int position, int capacity) {
 206         // Note that this is the only limit and position manipulation of
 207         // BasicImageReader private ByteBuffers.  The synchronize could be avoided
 208         // by cloning the buffer to make a local copy, but at the cost of creating
 209         // a new object.
 210         synchronized(buffer) {
 211             buffer.limit(position + capacity);
 212             buffer.position(position);
 213             return buffer.slice();
 214         }
 215     }
 216 
 217     private IntBuffer intBuffer(ByteBuffer buffer, int offset, int size) {
 218         return slice(buffer, offset, size).order(byteOrder).asIntBuffer();
 219     }
 220 
 221     public static void releaseByteBuffer(ByteBuffer buffer) {
 222         Objects.requireNonNull(buffer);
 223 
 224         if (!MAP_ALL) {
 225             ImageBufferCache.releaseBuffer(buffer);
 226         }
 227     }
 228 
 229     public String getName() {
 230         return name;
 231     }
 232 
 233     public ByteOrder getByteOrder() {
 234         return byteOrder;
 235     }
 236 
 237     public Path getImagePath() {
 238         return imagePath;
 239     }
 240 
 241     @Override
 242     public void close() throws IOException {
 243         if (channel != null) {
 244             channel.close();
 245         }
 246     }
 247 
 248     public ImageStringsReader getStrings() {
 249         return stringsReader;
 250     }
 251 
 252     public synchronized ImageLocation findLocation(String module, String name) {
 253         Objects.requireNonNull(module);
 254         Objects.requireNonNull(name);
 255         // Details of the algorithm used here can be found in
 256         // jdk.tools.jlink.internal.PerfectHashBuilder.
 257         int count = header.getTableLength();
 258         int index = redirect.get(ImageStringsReader.hashCode(module, name) % count);
 259 
 260         if (index < 0) {
 261             // index is twos complement of location attributes index.
 262             index = -index - 1;
 263         } else if (index > 0) {
 264             // index is hash seed needed to compute location attributes index.
 265             index = ImageStringsReader.hashCode(module, name, index) % count;
 266         } else {
 267             // No entry.
 268             return null;
 269         }
 270 
 271         long[] attributes = getAttributes(offsets.get(index));
 272 
 273         if (!ImageLocation.verify(module, name, attributes, stringsReader)) {
 274             return null;
 275         }
 276         return new ImageLocation(attributes, stringsReader);
 277     }
 278 
 279     public synchronized ImageLocation findLocation(String name) {
 280         Objects.requireNonNull(name);
 281         // Details of the algorithm used here can be found in
 282         // jdk.tools.jlink.internal.PerfectHashBuilder.
 283         int count = header.getTableLength();
 284         int index = redirect.get(ImageStringsReader.hashCode(name) % count);
 285 
 286         if (index < 0) {
 287             // index is twos complement of location attributes index.
 288             index = -index - 1;
 289         } else if (index > 0) {
 290             // index is hash seed needed to compute location attributes index.
 291             index = ImageStringsReader.hashCode(name, index) % count;
 292         } else {
 293             // No entry.
 294             return null;
 295         }
 296 
 297         long[] attributes = getAttributes(offsets.get(index));
 298 
 299         if (!ImageLocation.verify(name, attributes, stringsReader)) {
 300             return null;
 301         }
 302         return new ImageLocation(attributes, stringsReader);
 303     }
 304 
 305     public String[] getEntryNames() {
 306         int[] attributeOffsets = new int[offsets.capacity()];
 307         offsets.get(attributeOffsets);
 308         return IntStream.of(attributeOffsets)
 309                         .filter(o -> o != 0)
 310                         .mapToObj(o -> ImageLocation.readFrom(this, o).getFullName())
 311                         .sorted()
 312                         .toArray(String[]::new);
 313     }
 314 
 315     ImageLocation getLocation(int offset) {
 316         return ImageLocation.readFrom(this, offset);
 317     }
 318 
 319     public long[] getAttributes(int offset) {
 320         if (offset < 0 || offset >= locations.limit()) {
 321             throw new IndexOutOfBoundsException("offset");
 322         }
 323 
 324         ByteBuffer buffer = slice(locations, offset, locations.limit() - offset);
 325         return ImageLocation.decompress(buffer);
 326     }
 327 
 328     public String getString(int offset) {
 329         if (offset < 0 || offset >= strings.limit()) {
 330             throw new IndexOutOfBoundsException("offset");
 331         }
 332 
 333         ByteBuffer buffer = slice(strings, offset, strings.limit() - offset);
 334         return ImageStringsReader.stringFromByteBuffer(buffer);
 335     }
 336 
 337     private byte[] getBufferBytes(ByteBuffer buffer) {
 338         Objects.requireNonNull(buffer);
 339         byte[] bytes = new byte[buffer.limit()];
 340         buffer.get(bytes);
 341 
 342         return bytes;
 343     }
 344 
 345     private ByteBuffer readBuffer(long offset, long size) {
 346         if (offset < 0 || Integer.MAX_VALUE <= offset) {
 347             throw new IndexOutOfBoundsException("Bad offset: " + offset);
 348         }
 349 
 350         if (size < 0 || Integer.MAX_VALUE <= size) {
 351             throw new IndexOutOfBoundsException("Bad size: " + size);
 352         }
 353 
 354         if (MAP_ALL) {
 355             ByteBuffer buffer = slice(memoryMap, (int)offset, (int)size);
 356             buffer.order(ByteOrder.BIG_ENDIAN);
 357 
 358             return buffer;
 359         } else {
 360             if (channel == null) {
 361                 throw new InternalError("Image file channel not open");
 362             }
 363 
 364             ByteBuffer buffer = ImageBufferCache.getBuffer(size);
 365             int read;
 366             try {
 367                 read = channel.read(buffer, offset);
 368                 buffer.rewind();
 369             } catch (IOException ex) {
 370                 ImageBufferCache.releaseBuffer(buffer);
 371                 throw new RuntimeException(ex);
 372             }
 373 
 374             if (read != size) {
 375                 ImageBufferCache.releaseBuffer(buffer);
 376                 throw new RuntimeException("Short read: " + read +
 377                                            " instead of " + size + " bytes");
 378             }
 379 
 380             return buffer;
 381         }
 382     }
 383 
 384     public byte[] getResource(String name) {
 385         Objects.requireNonNull(name);
 386         ImageLocation location = findLocation(name);
 387 
 388         return location != null ? getResource(location) : null;
 389     }
 390 
 391     public byte[] getResource(ImageLocation loc) {
 392         ByteBuffer buffer = getResourceBuffer(loc);
 393 
 394         if (buffer != null) {
 395             byte[] bytes = getBufferBytes(buffer);
 396             ImageBufferCache.releaseBuffer(buffer);
 397 
 398             return bytes;
 399         }
 400 
 401         return null;
 402     }
 403 
 404     public ByteBuffer getResourceBuffer(ImageLocation loc) {
 405         Objects.requireNonNull(loc);
 406         long offset = loc.getContentOffset() + indexSize;
 407         long compressedSize = loc.getCompressedSize();
 408         long uncompressedSize = loc.getUncompressedSize();
 409 
 410         if (compressedSize < 0 || Integer.MAX_VALUE < compressedSize) {
 411             throw new IndexOutOfBoundsException(
 412                 "Bad compressed size: " + compressedSize);
 413         }
 414 
 415         if (uncompressedSize < 0 || Integer.MAX_VALUE < uncompressedSize) {
 416             throw new IndexOutOfBoundsException(
 417                 "Bad uncompressed size: " + uncompressedSize);
 418         }
 419 
 420         if (compressedSize == 0) {
 421             return readBuffer(offset, uncompressedSize);
 422         } else {
 423             ByteBuffer buffer = readBuffer(offset, compressedSize);
 424 
 425             if (buffer != null) {
 426                 byte[] bytesIn = getBufferBytes(buffer);
 427                 ImageBufferCache.releaseBuffer(buffer);
 428                 byte[] bytesOut;
 429 
 430                 try {
 431                     bytesOut = decompressor.decompressResource(byteOrder,
 432                             (int strOffset) -> getString(strOffset), bytesIn);
 433                 } catch (IOException ex) {
 434                     throw new RuntimeException(ex);
 435                 }
 436 
 437                 return ByteBuffer.wrap(bytesOut);
 438             }
 439         }
 440 
 441         return null;
 442     }
 443 
 444     public InputStream getResourceStream(ImageLocation loc) {
 445         Objects.requireNonNull(loc);
 446         byte[] bytes = getResource(loc);
 447 
 448         return new ByteArrayInputStream(bytes);
 449     }
 450 }
--- EOF ---