1
2 /*
3 * Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc/parallel/mutableNUMASpace.hpp"
28 #include "gc/shared/collectedHeap.hpp"
29 #include "gc/shared/spaceDecorator.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/atomic.inline.hpp"
32 #include "runtime/thread.inline.hpp"
33
34 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
35 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
36 _page_size = os::vm_page_size();
37 _adaptation_cycles = 0;
38 _samples_count = 0;
39 update_layout(true);
40 }
41
42 MutableNUMASpace::~MutableNUMASpace() {
43 for (int i = 0; i < lgrp_spaces()->length(); i++) {
44 delete lgrp_spaces()->at(i);
45 }
46 delete lgrp_spaces();
47 }
48
49 #ifndef PRODUCT
50 void MutableNUMASpace::mangle_unused_area() {
51 // This method should do nothing.
52 // It can be called on a numa space during a full compaction.
53 }
54 void MutableNUMASpace::mangle_unused_area_complete() {
55 // This method should do nothing.
56 // It can be called on a numa space during a full compaction.
57 }
58 void MutableNUMASpace::mangle_region(MemRegion mr) {
59 // This method should do nothing because numa spaces are not mangled.
60 }
61 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
62 assert(false, "Do not mangle MutableNUMASpace's");
63 }
64 void MutableNUMASpace::set_top_for_allocations() {
65 // This method should do nothing.
66 }
67 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
68 // This method should do nothing.
69 }
70 void MutableNUMASpace::check_mangled_unused_area_complete() {
71 // This method should do nothing.
72 }
73 #endif // NOT_PRODUCT
74
75 // There may be unallocated holes in the middle chunks
76 // that should be filled with dead objects to ensure parsability.
77 void MutableNUMASpace::ensure_parsability() {
78 for (int i = 0; i < lgrp_spaces()->length(); i++) {
79 LGRPSpace *ls = lgrp_spaces()->at(i);
80 MutableSpace *s = ls->space();
81 if (s->top() < top()) { // For all spaces preceding the one containing top()
82 if (s->free_in_words() > 0) {
83 intptr_t cur_top = (intptr_t)s->top();
84 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
85 while (words_left_to_fill > 0) {
86 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
87 assert(words_to_fill >= CollectedHeap::min_fill_size(),
88 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
89 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
90 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
91 if (!os::numa_has_static_binding()) {
92 size_t touched_words = words_to_fill;
93 #ifndef ASSERT
94 if (!ZapUnusedHeapArea) {
95 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
96 touched_words);
97 }
98 #endif
99 MemRegion invalid;
100 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
101 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
102 if (crossing_start != crossing_end) {
103 // If object header crossed a small page boundary we mark the area
104 // as invalid rounding it to a page_size().
105 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
106 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
107 invalid = MemRegion(start, end);
108 }
109
110 ls->add_invalid_region(invalid);
111 }
112 cur_top = cur_top + (words_to_fill * HeapWordSize);
113 words_left_to_fill -= words_to_fill;
114 }
115 }
116 } else {
117 if (!os::numa_has_static_binding()) {
118 #ifdef ASSERT
119 MemRegion invalid(s->top(), s->end());
120 ls->add_invalid_region(invalid);
121 #else
122 if (ZapUnusedHeapArea) {
123 MemRegion invalid(s->top(), s->end());
124 ls->add_invalid_region(invalid);
125 } else {
126 return;
127 }
128 #endif
129 } else {
130 return;
131 }
132 }
133 }
134 }
135
136 size_t MutableNUMASpace::used_in_words() const {
137 size_t s = 0;
138 for (int i = 0; i < lgrp_spaces()->length(); i++) {
139 s += lgrp_spaces()->at(i)->space()->used_in_words();
140 }
141 return s;
142 }
143
144 size_t MutableNUMASpace::free_in_words() const {
145 size_t s = 0;
146 for (int i = 0; i < lgrp_spaces()->length(); i++) {
147 s += lgrp_spaces()->at(i)->space()->free_in_words();
148 }
149 return s;
150 }
151
152
153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
154 guarantee(thr != NULL, "No thread");
155 int lgrp_id = thr->lgrp_id();
156 if (lgrp_id == -1) {
157 // This case can occur after the topology of the system has
158 // changed. Thread can change their location, the new home
159 // group will be determined during the first allocation
160 // attempt. For now we can safely assume that all spaces
161 // have equal size because the whole space will be reinitialized.
162 if (lgrp_spaces()->length() > 0) {
163 return capacity_in_bytes() / lgrp_spaces()->length();
164 } else {
165 assert(false, "There should be at least one locality group");
166 return 0;
167 }
168 }
169 // That's the normal case, where we know the locality group of the thread.
170 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
171 if (i == -1) {
172 return 0;
173 }
174 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
175 }
176
177 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
178 // Please see the comments for tlab_capacity().
179 guarantee(thr != NULL, "No thread");
180 int lgrp_id = thr->lgrp_id();
181 if (lgrp_id == -1) {
182 if (lgrp_spaces()->length() > 0) {
183 return (used_in_bytes()) / lgrp_spaces()->length();
184 } else {
185 assert(false, "There should be at least one locality group");
186 return 0;
187 }
188 }
189 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
190 if (i == -1) {
191 return 0;
192 }
193 return lgrp_spaces()->at(i)->space()->used_in_bytes();
194 }
195
196
197 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
198 // Please see the comments for tlab_capacity().
199 guarantee(thr != NULL, "No thread");
200 int lgrp_id = thr->lgrp_id();
201 if (lgrp_id == -1) {
202 if (lgrp_spaces()->length() > 0) {
203 return free_in_bytes() / lgrp_spaces()->length();
204 } else {
205 assert(false, "There should be at least one locality group");
206 return 0;
207 }
208 }
209 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
210 if (i == -1) {
211 return 0;
212 }
213 return lgrp_spaces()->at(i)->space()->free_in_bytes();
214 }
215
216
217 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
218 guarantee(thr != NULL, "No thread");
219 int lgrp_id = thr->lgrp_id();
220 if (lgrp_id == -1) {
221 if (lgrp_spaces()->length() > 0) {
222 return capacity_in_words() / lgrp_spaces()->length();
223 } else {
224 assert(false, "There should be at least one locality group");
225 return 0;
226 }
227 }
228 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
229 if (i == -1) {
230 return 0;
231 }
232 return lgrp_spaces()->at(i)->space()->capacity_in_words();
233 }
234
235 // Check if the NUMA topology has changed. Add and remove spaces if needed.
236 // The update can be forced by setting the force parameter equal to true.
237 bool MutableNUMASpace::update_layout(bool force) {
238 // Check if the topology had changed.
239 bool changed = os::numa_topology_changed();
240 if (force || changed) {
241 // Compute lgrp intersection. Add/remove spaces.
242 int lgrp_limit = (int)os::numa_get_groups_num();
243 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
244 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
245 assert(lgrp_num > 0, "There should be at least one locality group");
246 // Add new spaces for the new nodes
247 for (int i = 0; i < lgrp_num; i++) {
248 bool found = false;
249 for (int j = 0; j < lgrp_spaces()->length(); j++) {
250 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
251 found = true;
252 break;
253 }
254 }
255 if (!found) {
256 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
257 }
258 }
259
260 // Remove spaces for the removed nodes.
261 for (int i = 0; i < lgrp_spaces()->length();) {
262 bool found = false;
263 for (int j = 0; j < lgrp_num; j++) {
264 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
265 found = true;
266 break;
267 }
268 }
269 if (!found) {
270 delete lgrp_spaces()->at(i);
271 lgrp_spaces()->remove_at(i);
272 } else {
273 i++;
274 }
275 }
276
277 FREE_C_HEAP_ARRAY(int, lgrp_ids);
278
279 if (changed) {
280 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
281 thread->set_lgrp_id(-1);
282 }
283 }
284 return true;
285 }
286 return false;
287 }
288
289 // Bias region towards the first-touching lgrp. Set the right page sizes.
290 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
291 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
292 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
293 if (end > start) {
294 MemRegion aligned_region(start, end);
295 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
296 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
297 assert(region().contains(aligned_region), "Sanity");
298 // First we tell the OS which page size we want in the given range. The underlying
299 // large page can be broken down if we require small pages.
300 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
301 // Then we uncommit the pages in the range.
302 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
303 // And make them local/first-touch biased.
304 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
305 }
306 }
307
308 // Free all pages in the region.
309 void MutableNUMASpace::free_region(MemRegion mr) {
310 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
311 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
312 if (end > start) {
313 MemRegion aligned_region(start, end);
314 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
315 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
316 assert(region().contains(aligned_region), "Sanity");
317 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
318 }
319 }
320
321 // Update space layout. Perform adaptation.
322 void MutableNUMASpace::update() {
323 if (update_layout(false)) {
324 // If the topology has changed, make all chunks zero-sized.
325 // And clear the alloc-rate statistics.
326 // In future we may want to handle this more gracefully in order
327 // to avoid the reallocation of the pages as much as possible.
328 for (int i = 0; i < lgrp_spaces()->length(); i++) {
329 LGRPSpace *ls = lgrp_spaces()->at(i);
330 MutableSpace *s = ls->space();
331 s->set_end(s->bottom());
332 s->set_top(s->bottom());
333 ls->clear_alloc_rate();
334 }
335 // A NUMA space is never mangled
336 initialize(region(),
337 SpaceDecorator::Clear,
338 SpaceDecorator::DontMangle);
339 } else {
340 bool should_initialize = false;
341 if (!os::numa_has_static_binding()) {
342 for (int i = 0; i < lgrp_spaces()->length(); i++) {
343 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
344 should_initialize = true;
345 break;
346 }
347 }
348 }
349
350 if (should_initialize ||
351 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
352 // A NUMA space is never mangled
353 initialize(region(),
354 SpaceDecorator::Clear,
355 SpaceDecorator::DontMangle);
356 }
357 }
358
359 if (NUMAStats) {
360 for (int i = 0; i < lgrp_spaces()->length(); i++) {
361 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
362 }
363 }
364
365 scan_pages(NUMAPageScanRate);
366 }
367
368 // Scan pages. Free pages that have smaller size or wrong placement.
369 void MutableNUMASpace::scan_pages(size_t page_count)
370 {
371 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
372 if (pages_per_chunk > 0) {
373 for (int i = 0; i < lgrp_spaces()->length(); i++) {
374 LGRPSpace *ls = lgrp_spaces()->at(i);
375 ls->scan_pages(page_size(), pages_per_chunk);
376 }
377 }
378 }
379
380 // Accumulate statistics about the allocation rate of each lgrp.
381 void MutableNUMASpace::accumulate_statistics() {
382 if (UseAdaptiveNUMAChunkSizing) {
383 for (int i = 0; i < lgrp_spaces()->length(); i++) {
384 lgrp_spaces()->at(i)->sample();
385 }
386 increment_samples_count();
387 }
388
389 if (NUMAStats) {
390 for (int i = 0; i < lgrp_spaces()->length(); i++) {
391 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
392 }
393 }
394 }
395
396 // Get the current size of a chunk.
397 // This function computes the size of the chunk based on the
398 // difference between chunk ends. This allows it to work correctly in
399 // case the whole space is resized and during the process of adaptive
400 // chunk resizing.
401 size_t MutableNUMASpace::current_chunk_size(int i) {
402 HeapWord *cur_end, *prev_end;
403 if (i == 0) {
404 prev_end = bottom();
405 } else {
406 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
407 }
408 if (i == lgrp_spaces()->length() - 1) {
409 cur_end = end();
410 } else {
411 cur_end = lgrp_spaces()->at(i)->space()->end();
412 }
413 if (cur_end > prev_end) {
414 return pointer_delta(cur_end, prev_end, sizeof(char));
415 }
416 return 0;
417 }
418
419 // Return the default chunk size by equally diving the space.
420 // page_size() aligned.
421 size_t MutableNUMASpace::default_chunk_size() {
422 return base_space_size() / lgrp_spaces()->length() * page_size();
423 }
424
425 // Produce a new chunk size. page_size() aligned.
426 // This function is expected to be called on sequence of i's from 0 to
427 // lgrp_spaces()->length().
428 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
429 size_t pages_available = base_space_size();
430 for (int j = 0; j < i; j++) {
431 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
432 }
433 pages_available -= lgrp_spaces()->length() - i - 1;
434 assert(pages_available > 0, "No pages left");
435 float alloc_rate = 0;
436 for (int j = i; j < lgrp_spaces()->length(); j++) {
437 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
438 }
439 size_t chunk_size = 0;
440 if (alloc_rate > 0) {
441 LGRPSpace *ls = lgrp_spaces()->at(i);
442 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
443 }
444 chunk_size = MAX2(chunk_size, page_size());
445
446 if (limit > 0) {
447 limit = round_down(limit, page_size());
448 if (chunk_size > current_chunk_size(i)) {
449 size_t upper_bound = pages_available * page_size();
450 if (upper_bound > limit &&
451 current_chunk_size(i) < upper_bound - limit) {
452 // The resulting upper bound should not exceed the available
453 // amount of memory (pages_available * page_size()).
454 upper_bound = current_chunk_size(i) + limit;
455 }
456 chunk_size = MIN2(chunk_size, upper_bound);
457 } else {
458 size_t lower_bound = page_size();
459 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
460 lower_bound = current_chunk_size(i) - limit;
461 }
462 chunk_size = MAX2(chunk_size, lower_bound);
463 }
464 }
465 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
466 return chunk_size;
467 }
468
469
470 // Return the bottom_region and the top_region. Align them to page_size() boundary.
471 // |------------------new_region---------------------------------|
472 // |----bottom_region--|---intersection---|------top_region------|
473 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
474 MemRegion* bottom_region, MemRegion *top_region) {
475 // Is there bottom?
476 if (new_region.start() < intersection.start()) { // Yes
477 // Try to coalesce small pages into a large one.
478 if (UseLargePages && page_size() >= alignment()) {
479 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
480 if (new_region.contains(p)
481 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
482 if (intersection.contains(p)) {
483 intersection = MemRegion(p, intersection.end());
484 } else {
485 intersection = MemRegion(p, p);
486 }
487 }
488 }
489 *bottom_region = MemRegion(new_region.start(), intersection.start());
490 } else {
491 *bottom_region = MemRegion();
492 }
493
494 // Is there top?
495 if (intersection.end() < new_region.end()) { // Yes
496 // Try to coalesce small pages into a large one.
497 if (UseLargePages && page_size() >= alignment()) {
498 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
499 if (new_region.contains(p)
500 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
501 if (intersection.contains(p)) {
502 intersection = MemRegion(intersection.start(), p);
503 } else {
504 intersection = MemRegion(p, p);
505 }
506 }
507 }
508 *top_region = MemRegion(intersection.end(), new_region.end());
509 } else {
510 *top_region = MemRegion();
511 }
512 }
513
514 // Try to merge the invalid region with the bottom or top region by decreasing
515 // the intersection area. Return the invalid_region aligned to the page_size()
516 // boundary if it's inside the intersection. Return non-empty invalid_region
517 // if it lies inside the intersection (also page-aligned).
518 // |------------------new_region---------------------------------|
519 // |----------------|-------invalid---|--------------------------|
520 // |----bottom_region--|---intersection---|------top_region------|
521 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
522 MemRegion *invalid_region) {
523 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
524 *intersection = MemRegion(invalid_region->end(), intersection->end());
525 *invalid_region = MemRegion();
526 } else
527 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
528 *intersection = MemRegion(intersection->start(), invalid_region->start());
529 *invalid_region = MemRegion();
530 } else
531 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
532 *intersection = MemRegion(new_region.start(), new_region.start());
533 *invalid_region = MemRegion();
534 } else
535 if (intersection->contains(invalid_region)) {
536 // That's the only case we have to make an additional bias_region() call.
537 HeapWord* start = invalid_region->start();
538 HeapWord* end = invalid_region->end();
539 if (UseLargePages && page_size() >= alignment()) {
540 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
541 if (new_region.contains(p)) {
542 start = p;
543 }
544 p = (HeapWord*)round_to((intptr_t) end, alignment());
545 if (new_region.contains(end)) {
546 end = p;
547 }
548 }
549 if (intersection->start() > start) {
550 *intersection = MemRegion(start, intersection->end());
551 }
552 if (intersection->end() < end) {
553 *intersection = MemRegion(intersection->start(), end);
554 }
555 *invalid_region = MemRegion(start, end);
556 }
557 }
558
559 void MutableNUMASpace::initialize(MemRegion mr,
560 bool clear_space,
561 bool mangle_space,
562 bool setup_pages) {
563 assert(clear_space, "Reallocation will destroy data!");
564 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
565
566 MemRegion old_region = region(), new_region;
567 set_bottom(mr.start());
568 set_end(mr.end());
569 // Must always clear the space
570 clear(SpaceDecorator::DontMangle);
571
572 // Compute chunk sizes
573 size_t prev_page_size = page_size();
574 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
575 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
576 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
577 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
578
579 // Try small pages if the chunk size is too small
580 if (base_space_size_pages / lgrp_spaces()->length() == 0
581 && page_size() > (size_t)os::vm_page_size()) {
582 #ifdef LINUX
583 // Changing the page size below can lead to freeing of memory. When using large pages
584 // and the memory has been both reserved and committed, Linux does not support
585 // freeing parts of it. So we fail initialization.
586 if (UseLargePages && !os::can_commit_large_page_memory()) {
587 vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
588 }
589 #endif // LINUX
590 set_page_size(os::vm_page_size());
591 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
592 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
593 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
594 }
595 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
596 set_base_space_size(base_space_size_pages);
597
598 // Handle space resize
599 MemRegion top_region, bottom_region;
600 if (!old_region.equals(region())) {
601 new_region = MemRegion(rounded_bottom, rounded_end);
602 MemRegion intersection = new_region.intersection(old_region);
603 if (intersection.start() == NULL ||
604 intersection.end() == NULL ||
605 prev_page_size > page_size()) { // If the page size got smaller we have to change
606 // the page size preference for the whole space.
607 intersection = MemRegion(new_region.start(), new_region.start());
608 }
609 select_tails(new_region, intersection, &bottom_region, &top_region);
610 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
611 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
612 }
613
614 // Check if the space layout has changed significantly?
615 // This happens when the space has been resized so that either head or tail
616 // chunk became less than a page.
617 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
618 current_chunk_size(0) > page_size() &&
619 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
620
621
622 for (int i = 0; i < lgrp_spaces()->length(); i++) {
623 LGRPSpace *ls = lgrp_spaces()->at(i);
624 MutableSpace *s = ls->space();
625 old_region = s->region();
626
627 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
628 if (i < lgrp_spaces()->length() - 1) {
629 if (!UseAdaptiveNUMAChunkSizing ||
630 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
631 samples_count() < AdaptiveSizePolicyReadyThreshold) {
632 // No adaptation. Divide the space equally.
633 chunk_byte_size = default_chunk_size();
634 } else
635 if (!layout_valid || NUMASpaceResizeRate == 0) {
636 // Fast adaptation. If no space resize rate is set, resize
637 // the chunks instantly.
638 chunk_byte_size = adaptive_chunk_size(i, 0);
639 } else {
640 // Slow adaptation. Resize the chunks moving no more than
641 // NUMASpaceResizeRate bytes per collection.
642 size_t limit = NUMASpaceResizeRate /
643 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
644 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
645 }
646
647 assert(chunk_byte_size >= page_size(), "Chunk size too small");
648 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
649 }
650
651 if (i == 0) { // Bottom chunk
652 if (i != lgrp_spaces()->length() - 1) {
653 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
654 } else {
655 new_region = MemRegion(bottom(), end());
656 }
657 } else
658 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
659 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
660 new_region = MemRegion(ps->end(),
661 ps->end() + (chunk_byte_size >> LogHeapWordSize));
662 } else { // Top chunk
663 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
664 new_region = MemRegion(ps->end(), end());
665 }
666 guarantee(region().contains(new_region), "Region invariant");
667
668
669 // The general case:
670 // |---------------------|--invalid---|--------------------------|
671 // |------------------new_region---------------------------------|
672 // |----bottom_region--|---intersection---|------top_region------|
673 // |----old_region----|
674 // The intersection part has all pages in place we don't need to migrate them.
675 // Pages for the top and bottom part should be freed and then reallocated.
676
677 MemRegion intersection = old_region.intersection(new_region);
678
679 if (intersection.start() == NULL || intersection.end() == NULL) {
680 intersection = MemRegion(new_region.start(), new_region.start());
681 }
682
683 if (!os::numa_has_static_binding()) {
684 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
685 // Invalid region is a range of memory that could've possibly
686 // been allocated on the other node. That's relevant only on Solaris where
687 // there is no static memory binding.
688 if (!invalid_region.is_empty()) {
689 merge_regions(new_region, &intersection, &invalid_region);
690 free_region(invalid_region);
691 ls->set_invalid_region(MemRegion());
692 }
693 }
694
695 select_tails(new_region, intersection, &bottom_region, &top_region);
696
697 if (!os::numa_has_static_binding()) {
698 // If that's a system with the first-touch policy then it's enough
699 // to free the pages.
700 free_region(bottom_region);
701 free_region(top_region);
702 } else {
703 // In a system with static binding we have to change the bias whenever
704 // we reshape the heap.
705 bias_region(bottom_region, ls->lgrp_id());
706 bias_region(top_region, ls->lgrp_id());
707 }
708
709 // Clear space (set top = bottom) but never mangle.
710 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
711
712 set_adaptation_cycles(samples_count());
713 }
714 }
715
716 // Set the top of the whole space.
717 // Mark the the holes in chunks below the top() as invalid.
718 void MutableNUMASpace::set_top(HeapWord* value) {
719 bool found_top = false;
720 for (int i = 0; i < lgrp_spaces()->length();) {
721 LGRPSpace *ls = lgrp_spaces()->at(i);
722 MutableSpace *s = ls->space();
723 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
724
725 if (s->contains(value)) {
726 // Check if setting the chunk's top to a given value would create a hole less than
727 // a minimal object; assuming that's not the last chunk in which case we don't care.
728 if (i < lgrp_spaces()->length() - 1) {
729 size_t remainder = pointer_delta(s->end(), value);
730 const size_t min_fill_size = CollectedHeap::min_fill_size();
731 if (remainder < min_fill_size && remainder > 0) {
732 // Add a minimum size filler object; it will cross the chunk boundary.
733 CollectedHeap::fill_with_object(value, min_fill_size);
734 value += min_fill_size;
735 assert(!s->contains(value), "Should be in the next chunk");
736 // Restart the loop from the same chunk, since the value has moved
737 // to the next one.
738 continue;
739 }
740 }
741
742 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
743 ls->add_invalid_region(MemRegion(top, value));
744 }
745 s->set_top(value);
746 found_top = true;
747 } else {
748 if (found_top) {
749 s->set_top(s->bottom());
750 } else {
751 if (!os::numa_has_static_binding() && top < s->end()) {
752 ls->add_invalid_region(MemRegion(top, s->end()));
753 }
754 s->set_top(s->end());
755 }
756 }
757 i++;
758 }
759 MutableSpace::set_top(value);
760 }
761
762 void MutableNUMASpace::clear(bool mangle_space) {
763 MutableSpace::set_top(bottom());
764 for (int i = 0; i < lgrp_spaces()->length(); i++) {
765 // Never mangle NUMA spaces because the mangling will
766 // bind the memory to a possibly unwanted lgroup.
767 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
768 }
769 }
770
771 /*
772 Linux supports static memory binding, therefore the most part of the
773 logic dealing with the possible invalid page allocation is effectively
774 disabled. Besides there is no notion of the home node in Linux. A
775 thread is allowed to migrate freely. Although the scheduler is rather
776 reluctant to move threads between the nodes. We check for the current
777 node every allocation. And with a high probability a thread stays on
778 the same node for some time allowing local access to recently allocated
779 objects.
780 */
781
782 HeapWord* MutableNUMASpace::allocate(size_t size) {
783 Thread* thr = Thread::current();
784 int lgrp_id = thr->lgrp_id();
785 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
786 lgrp_id = os::numa_get_group_id();
787 thr->set_lgrp_id(lgrp_id);
788 }
789
790 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
791
792 // It is possible that a new CPU has been hotplugged and
793 // we haven't reshaped the space accordingly.
794 if (i == -1) {
795 i = os::random() % lgrp_spaces()->length();
796 }
797
798 LGRPSpace* ls = lgrp_spaces()->at(i);
799 MutableSpace *s = ls->space();
800 HeapWord *p = s->allocate(size);
801
802 if (p != NULL) {
803 size_t remainder = s->free_in_words();
804 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
805 s->set_top(s->top() - size);
806 p = NULL;
807 }
808 }
809 if (p != NULL) {
810 if (top() < s->top()) { // Keep _top updated.
811 MutableSpace::set_top(s->top());
812 }
813 }
814 // Make the page allocation happen here if there is no static binding..
815 if (p != NULL && !os::numa_has_static_binding()) {
816 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
817 *(int*)i = 0;
818 }
819 }
820 if (p == NULL) {
821 ls->set_allocation_failed();
822 }
823 return p;
824 }
825
826 // This version is lock-free.
827 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
828 Thread* thr = Thread::current();
829 int lgrp_id = thr->lgrp_id();
830 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
831 lgrp_id = os::numa_get_group_id();
832 thr->set_lgrp_id(lgrp_id);
833 }
834
835 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
836 // It is possible that a new CPU has been hotplugged and
837 // we haven't reshaped the space accordingly.
838 if (i == -1) {
839 i = os::random() % lgrp_spaces()->length();
840 }
841 LGRPSpace *ls = lgrp_spaces()->at(i);
842 MutableSpace *s = ls->space();
843 HeapWord *p = s->cas_allocate(size);
844 if (p != NULL) {
845 size_t remainder = pointer_delta(s->end(), p + size);
846 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
847 if (s->cas_deallocate(p, size)) {
848 // We were the last to allocate and created a fragment less than
849 // a minimal object.
850 p = NULL;
851 } else {
852 guarantee(false, "Deallocation should always succeed");
853 }
854 }
855 }
856 if (p != NULL) {
857 HeapWord* cur_top, *cur_chunk_top = p + size;
858 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
859 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
860 break;
861 }
862 }
863 }
864
865 // Make the page allocation happen here if there is no static binding.
866 if (p != NULL && !os::numa_has_static_binding() ) {
867 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
868 *(int*)i = 0;
869 }
870 }
871 if (p == NULL) {
872 ls->set_allocation_failed();
873 }
874 return p;
875 }
876
877 void MutableNUMASpace::print_short_on(outputStream* st) const {
878 MutableSpace::print_short_on(st);
879 st->print(" (");
880 for (int i = 0; i < lgrp_spaces()->length(); i++) {
881 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
882 lgrp_spaces()->at(i)->space()->print_short_on(st);
883 if (i < lgrp_spaces()->length() - 1) {
884 st->print(", ");
885 }
886 }
887 st->print(")");
888 }
889
890 void MutableNUMASpace::print_on(outputStream* st) const {
891 MutableSpace::print_on(st);
892 for (int i = 0; i < lgrp_spaces()->length(); i++) {
893 LGRPSpace *ls = lgrp_spaces()->at(i);
894 st->print(" lgrp %d", ls->lgrp_id());
895 ls->space()->print_on(st);
896 if (NUMAStats) {
897 for (int i = 0; i < lgrp_spaces()->length(); i++) {
898 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
899 }
900 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
901 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
902 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
903 ls->space_stats()->_local_space / K,
904 ls->space_stats()->_remote_space / K,
905 ls->space_stats()->_unbiased_space / K,
906 ls->space_stats()->_uncommited_space / K,
907 ls->space_stats()->_large_pages,
908 ls->space_stats()->_small_pages);
909 }
910 }
911 }
912
913 void MutableNUMASpace::verify() {
914 // This can be called after setting an arbitrary value to the space's top,
915 // so an object can cross the chunk boundary. We ensure the parsability
916 // of the space and just walk the objects in linear fashion.
917 ensure_parsability();
918 MutableSpace::verify();
919 }
920
921 // Scan pages and gather stats about page placement and size.
922 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
923 clear_space_stats();
924 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
925 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
926 if (start < end) {
927 for (char *p = start; p < end;) {
928 os::page_info info;
929 if (os::get_page_info(p, &info)) {
930 if (info.size > 0) {
931 if (info.size > (size_t)os::vm_page_size()) {
932 space_stats()->_large_pages++;
933 } else {
934 space_stats()->_small_pages++;
935 }
936 if (info.lgrp_id == lgrp_id()) {
937 space_stats()->_local_space += info.size;
938 } else {
939 space_stats()->_remote_space += info.size;
940 }
941 p += info.size;
942 } else {
943 p += os::vm_page_size();
944 space_stats()->_uncommited_space += os::vm_page_size();
945 }
946 } else {
947 return;
948 }
949 }
950 }
951 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
952 pointer_delta(space()->end(), end, sizeof(char));
953
954 }
955
956 // Scan page_count pages and verify if they have the right size and right placement.
957 // If invalid pages are found they are freed in hope that subsequent reallocation
958 // will be more successful.
959 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
960 {
961 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
962 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
963
964 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
965 set_last_page_scanned(range_start);
966 }
967
968 char *scan_start = last_page_scanned();
969 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
970
971 os::page_info page_expected, page_found;
972 page_expected.size = page_size;
973 page_expected.lgrp_id = lgrp_id();
974
975 char *s = scan_start;
976 while (s < scan_end) {
977 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
978 if (e == NULL) {
979 break;
980 }
981 if (e != scan_end) {
982 assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
983
984 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
985 && page_expected.size != 0) {
986 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
987 }
988 page_expected = page_found;
989 }
990 s = e;
991 }
992
993 set_last_page_scanned(scan_end);
994 }