1 /*
2 * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zArray.inline.hpp"
26 #include "gc/z/zBackingFile_linux_x86.hpp"
27 #include "gc/z/zBackingPath_linux_x86.hpp"
28 #include "gc/z/zErrno.hpp"
29 #include "gc/z/zLargePages.inline.hpp"
30 #include "logging/log.hpp"
31 #include "runtime/os.hpp"
32 #include "utilities/align.hpp"
33 #include "utilities/debug.hpp"
34
35 #include <fcntl.h>
36 #include <sys/mman.h>
37 #include <sys/stat.h>
38 #include <sys/statfs.h>
39 #include <sys/types.h>
40 #include <unistd.h>
41
42 // Filesystem names
43 #define ZFILESYSTEM_TMPFS "tmpfs"
44 #define ZFILESYSTEM_HUGETLBFS "hugetlbfs"
45
46 // Sysfs file for transparent huge page on tmpfs
47 #define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled"
48
49 // Default mount points
50 #define ZMOUNTPOINT_TMPFS "/dev/shm"
51 #define ZMOUNTPOINT_HUGETLBFS "/hugepages"
52
53 // Java heap filename
54 #define ZFILENAME_HEAP "java_heap"
55
56 // Support for building on older Linux systems
57 #ifndef __NR_memfd_create
58 #define __NR_memfd_create 319
59 #endif
60 #ifndef MFD_CLOEXEC
61 #define MFD_CLOEXEC 0x0001U
62 #endif
63 #ifndef MFD_HUGETLB
64 #define MFD_HUGETLB 0x0004U
65 #endif
66 #ifndef O_CLOEXEC
67 #define O_CLOEXEC 02000000
68 #endif
69 #ifndef O_TMPFILE
70 #define O_TMPFILE (020000000 | O_DIRECTORY)
71 #endif
72
73 // Filesystem types, see statfs(2)
74 #ifndef TMPFS_MAGIC
75 #define TMPFS_MAGIC 0x01021994
76 #endif
77 #ifndef HUGETLBFS_MAGIC
78 #define HUGETLBFS_MAGIC 0x958458f6
79 #endif
80
81 static int z_memfd_create(const char *name, unsigned int flags) {
82 return syscall(__NR_memfd_create, name, flags);
83 }
84
85 bool ZBackingFile::_hugetlbfs_mmap_retry = true;
86
87 ZBackingFile::ZBackingFile() :
88 _fd(-1),
89 _filesystem(0),
90 _available(0),
91 _initialized(false) {
92
93 // Create backing file
94 _fd = create_fd(ZFILENAME_HEAP);
95 if (_fd == -1) {
96 return;
97 }
98
99 // Get filesystem statistics
100 struct statfs statfs_buf;
101 if (fstatfs(_fd, &statfs_buf) == -1) {
102 ZErrno err;
103 log_error(gc, init)("Failed to determine filesystem type for backing file (%s)",
104 err.to_string());
105 return;
106 }
107
108 _filesystem = statfs_buf.f_type;
109 _available = statfs_buf.f_bavail * statfs_buf.f_bsize;
110
111 // Make sure we're on a supported filesystem
112 if (!is_tmpfs() && !is_hugetlbfs()) {
113 log_error(gc, init)("Backing file must be located on a %s or a %s filesystem",
114 ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS);
115 return;
116 }
117
118 // Make sure the filesystem type matches requested large page type
119 if (ZLargePages::is_transparent() && !is_tmpfs()) {
120 log_error(gc, init)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem",
121 ZFILESYSTEM_TMPFS);
122 return;
123 }
124
125 if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) {
126 log_error(gc, init)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel",
127 ZFILESYSTEM_TMPFS);
128 return;
129 }
130
131 if (ZLargePages::is_explicit() && !is_hugetlbfs()) {
132 log_error(gc, init)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled when using a %s filesystem",
133 ZFILESYSTEM_HUGETLBFS);
134 return;
135 }
136
137 if (!ZLargePages::is_explicit() && is_hugetlbfs()) {
138 log_error(gc, init)("-XX:+UseLargePages must be enabled when using a %s filesystem",
139 ZFILESYSTEM_HUGETLBFS);
140 return;
141 }
142
143 // Successfully initialized
144 _initialized = true;
145 }
146
147 int ZBackingFile::create_mem_fd(const char* name) const {
148 // Create file name
149 char filename[PATH_MAX];
150 snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : "");
151
152 // Create file
153 const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0;
154 const int fd = z_memfd_create(filename, MFD_CLOEXEC | extra_flags);
155 if (fd == -1) {
156 ZErrno err;
157 log_debug(gc, init)("Failed to create memfd file (%s)",
158 ((UseLargePages && err == EINVAL) ? "Hugepages not supported" : err.to_string()));
159 return -1;
160 }
161
162 log_info(gc, init)("Heap backed by file: /memfd:%s", filename);
163
164 return fd;
165 }
166
167 int ZBackingFile::create_file_fd(const char* name) const {
168 const char* const filesystem = ZLargePages::is_explicit() ? ZFILESYSTEM_HUGETLBFS : ZFILESYSTEM_TMPFS;
169 const char* const mountpoint = ZLargePages::is_explicit() ? ZMOUNTPOINT_HUGETLBFS : ZMOUNTPOINT_TMPFS;
170
171 // Find mountpoint
172 ZBackingPath path(filesystem, mountpoint);
173 if (path.get() == NULL) {
174 log_error(gc, init)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem);
175 return -1;
176 }
177
178 // Try to create an anonymous file using the O_TMPFILE flag. Note that this
179 // flag requires kernel >= 3.11. If this fails we fall back to open/unlink.
180 const int fd_anon = open(path.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
181 if (fd_anon == -1) {
182 ZErrno err;
183 log_debug(gc, init)("Failed to create anonymous file in %s (%s)", path.get(),
184 (err == EINVAL ? "Not supported" : err.to_string()));
185 } else {
186 // Get inode number for anonymous file
187 struct stat stat_buf;
188 if (fstat(fd_anon, &stat_buf) == -1) {
189 ZErrno err;
190 log_error(gc, init)("Failed to determine inode number for anonymous file (%s)", err.to_string());
191 return -1;
192 }
193
194 log_info(gc, init)("Heap backed by file: %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino);
195
196 return fd_anon;
197 }
198
199 log_debug(gc, init)("Falling back to open/unlink");
200
201 // Create file name
202 char filename[PATH_MAX];
203 snprintf(filename, sizeof(filename), "%s/%s.%d", path.get(), name, os::current_process_id());
204
205 // Create file
206 const int fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
207 if (fd == -1) {
208 ZErrno err;
209 log_error(gc, init)("Failed to create file %s (%s)", filename, err.to_string());
210 return -1;
211 }
212
213 // Unlink file
214 if (unlink(filename) == -1) {
215 ZErrno err;
216 log_error(gc, init)("Failed to unlink file %s (%s)", filename, err.to_string());
217 return -1;
218 }
219
220 log_info(gc, init)("Heap backed by file: %s", filename);
221
222 return fd;
223 }
224
225 int ZBackingFile::create_fd(const char* name) const {
226 if (ZPath == NULL) {
227 // If the path is not explicitly specified, then we first try to create a memfd file
228 // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might
229 // not be supported at all (requires kernel >= 3.17), or it might not support large
230 // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a
231 // file on an accessible tmpfs or hugetlbfs mount point.
232 const int fd = create_mem_fd(name);
233 if (fd != -1) {
234 return fd;
235 }
236
237 log_debug(gc, init)("Falling back to searching for an accessible mount point");
238 }
239
240 return create_file_fd(name);
241 }
242
243 bool ZBackingFile::is_initialized() const {
244 return _initialized;
245 }
246
247 int ZBackingFile::fd() const {
248 return _fd;
249 }
250
251 size_t ZBackingFile::available() const {
252 return _available;
253 }
254
255 bool ZBackingFile::is_tmpfs() const {
256 return _filesystem == TMPFS_MAGIC;
257 }
258
259 bool ZBackingFile::is_hugetlbfs() const {
260 return _filesystem == HUGETLBFS_MAGIC;
261 }
262
263 bool ZBackingFile::tmpfs_supports_transparent_huge_pages() const {
264 // If the shmem_enabled file exists and is readable then we
265 // know the kernel supports transparent huge pages for tmpfs.
266 return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0;
267 }
268
269 bool ZBackingFile::try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const {
270 // Try first smaller part.
271 const size_t offset0 = offset;
272 const size_t length0 = align_up(length / 2, alignment);
273 if (!try_expand_tmpfs(offset0, length0, alignment)) {
274 return false;
275 }
276
277 // Try second smaller part.
278 const size_t offset1 = offset0 + length0;
279 const size_t length1 = length - length0;
280 if (!try_expand_tmpfs(offset1, length1, alignment)) {
281 return false;
282 }
283
284 return true;
285 }
286
287 bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const {
288 assert(length > 0, "Invalid length");
289 assert(is_aligned(length, alignment), "Invalid length");
290
291 ZErrno err = posix_fallocate(_fd, offset, length);
292
293 if (err == EINTR && length > alignment) {
294 // Calling posix_fallocate() with a large length can take a long
295 // time to complete. When running profilers, such as VTune, this
296 // syscall will be constantly interrupted by signals. Expanding
297 // the file in smaller steps avoids this problem.
298 return try_split_and_expand_tmpfs(offset, length, alignment);
299 }
300
301 if (err) {
302 log_error(gc)("Failed to allocate backing file (%s)", err.to_string());
303 return false;
304 }
305
306 return true;
307 }
308
309 bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length) const {
310 assert(is_tmpfs(), "Wrong filesystem");
311 return try_expand_tmpfs(offset, length, os::vm_page_size());
312 }
313
314 bool ZBackingFile::try_expand_hugetlbfs(size_t offset, size_t length) const {
315 assert(is_hugetlbfs(), "Wrong filesystem");
316
317 // Prior to kernel 4.3, hugetlbfs did not support posix_fallocate().
318 // Instead of posix_fallocate() we can use a well-known workaround,
319 // which involves truncating the file to requested size and then try
320 // to map it to verify that there are enough huge pages available to
321 // back it.
322 while (ftruncate(_fd, offset + length) == -1) {
323 ZErrno err;
324 if (err != EINTR) {
325 log_error(gc)("Failed to truncate backing file (%s)", err.to_string());
326 return false;
327 }
328 }
329
330 // If we fail mapping during initialization, i.e. when we are pre-mapping
331 // the heap, then we wait and retry a few times before giving up. Otherwise
332 // there is a risk that running JVMs back-to-back will fail, since there
333 // is a delay between process termination and the huge pages owned by that
334 // process being returned to the huge page pool and made available for new
335 // allocations.
336 void* addr = MAP_FAILED;
337 const int max_attempts = 5;
338 for (int attempt = 1; attempt <= max_attempts; attempt++) {
339 addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
340 if (addr != MAP_FAILED || !_hugetlbfs_mmap_retry) {
341 // Mapping was successful or mmap retry is disabled
342 break;
343 }
344
345 ZErrno err;
346 log_debug(gc)("Failed to map backing file (%s), attempt %d of %d",
347 err.to_string(), attempt, max_attempts);
348
349 // Wait and retry in one second, in the hope that
350 // huge pages will be available by then.
351 sleep(1);
352 }
353
354 // Disable mmap retry from now on
355 if (_hugetlbfs_mmap_retry) {
356 _hugetlbfs_mmap_retry = false;
357 }
358
359 if (addr == MAP_FAILED) {
360 // Not enough huge pages left
361 ZErrno err;
362 log_error(gc)("Failed to map backing file (%s)", err.to_string());
363 return false;
364 }
365
366 // Successful mapping, unmap again. From now on the pages we mapped
367 // will be reserved for this file.
368 if (munmap(addr, length) == -1) {
369 ZErrno err;
370 log_error(gc)("Failed to unmap backing file (%s)", err.to_string());
371 return false;
372 }
373
374 return true;
375 }
376
377 bool ZBackingFile::try_expand_tmpfs_or_hugetlbfs(size_t offset, size_t length, size_t alignment) const {
378 assert(is_aligned(offset, alignment), "Invalid offset");
379 assert(is_aligned(length, alignment), "Invalid length");
380
381 log_debug(gc)("Expanding heap from " SIZE_FORMAT "M to " SIZE_FORMAT "M", offset / M, (offset + length) / M);
382
383 return is_hugetlbfs() ? try_expand_hugetlbfs(offset, length) : try_expand_tmpfs(offset, length);
384 }
385
386 size_t ZBackingFile::try_expand(size_t offset, size_t length, size_t alignment) const {
387 size_t start = offset;
388 size_t end = offset + length;
389
390 // Try to expand
391 if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) {
392 // Success
393 return end;
394 }
395
396 // Failed, try to expand as much as possible
397 for (;;) {
398 length = align_down((end - start) / 2, alignment);
399 if (length < alignment) {
400 // Done, don't expand more
401 return start;
402 }
403
404 if (try_expand_tmpfs_or_hugetlbfs(start, length, alignment)) {
405 // Success, try expand more
406 start += length;
407 } else {
408 // Failed, try expand less
409 end -= length;
410 }
411 }
412 }