1
2 /*
3 * Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc/parallel/mutableNUMASpace.hpp"
28 #include "gc/shared/collectedHeap.hpp"
29 #include "gc/shared/spaceDecorator.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/atomic.inline.hpp"
32 #include "runtime/thread.inline.hpp"
33
34 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
35 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
36 _page_size = os::vm_page_size();
37 _adaptation_cycles = 0;
38 _samples_count = 0;
39 update_layout(true);
40 }
41
42 MutableNUMASpace::~MutableNUMASpace() {
43 for (int i = 0; i < lgrp_spaces()->length(); i++) {
44 delete lgrp_spaces()->at(i);
45 }
46 delete lgrp_spaces();
47 }
48
49 #ifndef PRODUCT
50 void MutableNUMASpace::mangle_unused_area() {
51 // This method should do nothing.
52 // It can be called on a numa space during a full compaction.
53 }
54 void MutableNUMASpace::mangle_unused_area_complete() {
55 // This method should do nothing.
56 // It can be called on a numa space during a full compaction.
57 }
58 void MutableNUMASpace::mangle_region(MemRegion mr) {
59 // This method should do nothing because numa spaces are not mangled.
60 }
61 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
62 assert(false, "Do not mangle MutableNUMASpace's");
63 }
64 void MutableNUMASpace::set_top_for_allocations() {
65 // This method should do nothing.
66 }
67 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
68 // This method should do nothing.
69 }
70 void MutableNUMASpace::check_mangled_unused_area_complete() {
71 // This method should do nothing.
72 }
73 #endif // NOT_PRODUCT
74
75 // There may be unallocated holes in the middle chunks
76 // that should be filled with dead objects to ensure parsability.
77 void MutableNUMASpace::ensure_parsability() {
78 for (int i = 0; i < lgrp_spaces()->length(); i++) {
79 LGRPSpace *ls = lgrp_spaces()->at(i);
80 MutableSpace *s = ls->space();
81 if (s->top() < top()) { // For all spaces preceding the one containing top()
82 if (s->free_in_words() > 0) {
83 intptr_t cur_top = (intptr_t)s->top();
84 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
85 while (words_left_to_fill > 0) {
86 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
87 assert(words_to_fill >= CollectedHeap::min_fill_size(),
88 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
89 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
90 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
91 if (!os::numa_has_static_binding()) {
92 size_t touched_words = words_to_fill;
93 #ifndef ASSERT
94 if (!ZapUnusedHeapArea) {
95 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
96 touched_words);
97 }
98 #endif
99 MemRegion invalid;
100 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
101 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
102 if (crossing_start != crossing_end) {
103 // If object header crossed a small page boundary we mark the area
104 // as invalid rounding it to a page_size().
105 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
106 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
107 invalid = MemRegion(start, end);
108 }
109
110 ls->add_invalid_region(invalid);
111 }
112 cur_top = cur_top + (words_to_fill * HeapWordSize);
113 words_left_to_fill -= words_to_fill;
114 }
115 }
116 } else {
117 if (!os::numa_has_static_binding()) {
118 #ifdef ASSERT
119 MemRegion invalid(s->top(), s->end());
120 ls->add_invalid_region(invalid);
121 #else
122 if (ZapUnusedHeapArea) {
123 MemRegion invalid(s->top(), s->end());
124 ls->add_invalid_region(invalid);
125 } else {
126 return;
127 }
128 #endif
129 } else {
130 return;
131 }
132 }
133 }
134 }
135
136 size_t MutableNUMASpace::used_in_words() const {
137 size_t s = 0;
138 for (int i = 0; i < lgrp_spaces()->length(); i++) {
139 s += lgrp_spaces()->at(i)->space()->used_in_words();
140 }
141 return s;
142 }
143
144 size_t MutableNUMASpace::free_in_words() const {
145 size_t s = 0;
146 for (int i = 0; i < lgrp_spaces()->length(); i++) {
147 s += lgrp_spaces()->at(i)->space()->free_in_words();
148 }
149 return s;
150 }
151
152
153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
154 guarantee(thr != NULL, "No thread");
155 int lgrp_id = thr->lgrp_id();
156 if (lgrp_id == -1) {
157 // This case can occur after the topology of the system has
158 // changed. Thread can change their location, the new home
159 // group will be determined during the first allocation
160 // attempt. For now we can safely assume that all spaces
161 // have equal size because the whole space will be reinitialized.
162 if (lgrp_spaces()->length() > 0) {
163 return capacity_in_bytes() / lgrp_spaces()->length();
164 } else {
165 assert(false, "There should be at least one locality group");
166 return 0;
167 }
168 }
169 // That's the normal case, where we know the locality group of the thread.
170 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
171 if (i == -1) {
172 return 0;
173 }
174 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
175 }
176
177 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
178 // Please see the comments for tlab_capacity().
179 guarantee(thr != NULL, "No thread");
180 int lgrp_id = thr->lgrp_id();
181 if (lgrp_id == -1) {
182 if (lgrp_spaces()->length() > 0) {
183 return (used_in_bytes()) / lgrp_spaces()->length();
184 } else {
185 assert(false, "There should be at least one locality group");
186 return 0;
187 }
188 }
189 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
190 if (i == -1) {
191 return 0;
192 }
193 return lgrp_spaces()->at(i)->space()->used_in_bytes();
194 }
195
196
197 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
198 // Please see the comments for tlab_capacity().
199 guarantee(thr != NULL, "No thread");
200 int lgrp_id = thr->lgrp_id();
201 if (lgrp_id == -1) {
202 if (lgrp_spaces()->length() > 0) {
203 return free_in_bytes() / lgrp_spaces()->length();
204 } else {
205 assert(false, "There should be at least one locality group");
206 return 0;
207 }
208 }
209 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
210 if (i == -1) {
211 return 0;
212 }
213 return lgrp_spaces()->at(i)->space()->free_in_bytes();
214 }
215
216
217 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
218 guarantee(thr != NULL, "No thread");
219 int lgrp_id = thr->lgrp_id();
220 if (lgrp_id == -1) {
221 if (lgrp_spaces()->length() > 0) {
222 return capacity_in_words() / lgrp_spaces()->length();
223 } else {
224 assert(false, "There should be at least one locality group");
225 return 0;
226 }
227 }
228 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
229 if (i == -1) {
230 return 0;
231 }
232 return lgrp_spaces()->at(i)->space()->capacity_in_words();
233 }
234
235 // Check if the NUMA topology has changed. Add and remove spaces if needed.
236 // The update can be forced by setting the force parameter equal to true.
237 bool MutableNUMASpace::update_layout(bool force) {
238 // Check if the topology had changed.
239 bool changed = os::numa_topology_changed();
240 if (force || changed) {
241 // Compute lgrp intersection. Add/remove spaces.
242 int lgrp_limit = (int)os::numa_get_groups_num();
243 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
244 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
245 assert(lgrp_num > 0, "There should be at least one locality group");
246 // Add new spaces for the new nodes
247 for (int i = 0; i < lgrp_num; i++) {
248 bool found = false;
249 for (int j = 0; j < lgrp_spaces()->length(); j++) {
250 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
251 found = true;
252 break;
253 }
254 }
255 if (!found) {
256 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
257 }
258 }
259
260 // Remove spaces for the removed nodes.
261 for (int i = 0; i < lgrp_spaces()->length();) {
262 bool found = false;
263 for (int j = 0; j < lgrp_num; j++) {
264 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
265 found = true;
266 break;
267 }
268 }
269 if (!found) {
270 delete lgrp_spaces()->at(i);
271 lgrp_spaces()->remove_at(i);
272 } else {
273 i++;
274 }
275 }
276
277 FREE_C_HEAP_ARRAY(int, lgrp_ids);
278
279 if (changed) {
280 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
281 thread->set_lgrp_id(-1);
282 }
283 }
284 return true;
285 }
286 return false;
287 }
288
289 // Bias region towards the first-touching lgrp. Set the right page sizes.
290 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
291 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
292 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
293 if (end > start) {
294 MemRegion aligned_region(start, end);
295 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
296 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
297 assert(region().contains(aligned_region), "Sanity");
298 // First we tell the OS which page size we want in the given range. The underlying
299 // large page can be broken down if we require small pages.
300 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
301 // Then we uncommit the pages in the range.
302 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
303 // And make them local/first-touch biased.
304 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
305 }
306 }
307
308 // Free all pages in the region.
309 void MutableNUMASpace::free_region(MemRegion mr) {
310 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
311 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
312 if (end > start) {
313 MemRegion aligned_region(start, end);
314 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
315 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
316 assert(region().contains(aligned_region), "Sanity");
317 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
318 }
319 }
320
321 // Update space layout. Perform adaptation.
322 void MutableNUMASpace::update() {
323 if (update_layout(false)) {
324 // If the topology has changed, make all chunks zero-sized.
325 // And clear the alloc-rate statistics.
326 // In future we may want to handle this more gracefully in order
327 // to avoid the reallocation of the pages as much as possible.
328 for (int i = 0; i < lgrp_spaces()->length(); i++) {
329 LGRPSpace *ls = lgrp_spaces()->at(i);
330 MutableSpace *s = ls->space();
331 s->set_end(s->bottom());
332 s->set_top(s->bottom());
333 ls->clear_alloc_rate();
334 }
335 // A NUMA space is never mangled
336 initialize(region(),
337 SpaceDecorator::Clear,
338 SpaceDecorator::DontMangle);
339 } else {
340 bool should_initialize = false;
341 if (!os::numa_has_static_binding()) {
342 for (int i = 0; i < lgrp_spaces()->length(); i++) {
343 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
344 should_initialize = true;
345 break;
346 }
347 }
348 }
349
350 if (should_initialize ||
351 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
352 // A NUMA space is never mangled
353 initialize(region(),
354 SpaceDecorator::Clear,
355 SpaceDecorator::DontMangle);
356 }
357 }
358
359 if (NUMAStats) {
360 for (int i = 0; i < lgrp_spaces()->length(); i++) {
361 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
362 }
363 }
364
365 scan_pages(NUMAPageScanRate);
366 }
367
368 // Scan pages. Free pages that have smaller size or wrong placement.
369 void MutableNUMASpace::scan_pages(size_t page_count)
370 {
371 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
372 if (pages_per_chunk > 0) {
373 for (int i = 0; i < lgrp_spaces()->length(); i++) {
374 LGRPSpace *ls = lgrp_spaces()->at(i);
375 ls->scan_pages(page_size(), pages_per_chunk);
376 }
377 }
378 }
379
380 // Accumulate statistics about the allocation rate of each lgrp.
381 void MutableNUMASpace::accumulate_statistics() {
382 if (UseAdaptiveNUMAChunkSizing) {
383 for (int i = 0; i < lgrp_spaces()->length(); i++) {
384 lgrp_spaces()->at(i)->sample();
385 }
386 increment_samples_count();
387 }
388
389 if (NUMAStats) {
390 for (int i = 0; i < lgrp_spaces()->length(); i++) {
391 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
392 }
393 }
394 }
395
396 // Get the current size of a chunk.
397 // This function computes the size of the chunk based on the
398 // difference between chunk ends. This allows it to work correctly in
399 // case the whole space is resized and during the process of adaptive
400 // chunk resizing.
401 size_t MutableNUMASpace::current_chunk_size(int i) {
402 HeapWord *cur_end, *prev_end;
403 if (i == 0) {
404 prev_end = bottom();
405 } else {
406 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
407 }
408 if (i == lgrp_spaces()->length() - 1) {
409 cur_end = end();
410 } else {
411 cur_end = lgrp_spaces()->at(i)->space()->end();
412 }
413 if (cur_end > prev_end) {
414 return pointer_delta(cur_end, prev_end, sizeof(char));
415 }
416 return 0;
417 }
418
419 // Return the default chunk size by equally diving the space.
420 // page_size() aligned.
421 size_t MutableNUMASpace::default_chunk_size() {
422 return base_space_size() / lgrp_spaces()->length() * page_size();
423 }
424
425 // Produce a new chunk size. page_size() aligned.
426 // This function is expected to be called on sequence of i's from 0 to
427 // lgrp_spaces()->length().
428 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
429 size_t pages_available = base_space_size();
430 for (int j = 0; j < i; j++) {
431 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
432 }
433 pages_available -= lgrp_spaces()->length() - i - 1;
434 assert(pages_available > 0, "No pages left");
435 float alloc_rate = 0;
436 for (int j = i; j < lgrp_spaces()->length(); j++) {
437 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
438 }
439 size_t chunk_size = 0;
440 if (alloc_rate > 0) {
441 LGRPSpace *ls = lgrp_spaces()->at(i);
442 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
443 }
444 chunk_size = MAX2(chunk_size, page_size());
445
446 if (limit > 0) {
447 limit = round_down(limit, page_size());
448 if (chunk_size > current_chunk_size(i)) {
449 size_t upper_bound = pages_available * page_size();
450 if (upper_bound > limit &&
451 current_chunk_size(i) < upper_bound - limit) {
452 // The resulting upper bound should not exceed the available
453 // amount of memory (pages_available * page_size()).
454 upper_bound = current_chunk_size(i) + limit;
455 }
456 chunk_size = MIN2(chunk_size, upper_bound);
457 } else {
458 size_t lower_bound = page_size();
459 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
460 lower_bound = current_chunk_size(i) - limit;
461 }
462 chunk_size = MAX2(chunk_size, lower_bound);
463 }
464 }
465 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
466 return chunk_size;
467 }
468
469
470 // Return the bottom_region and the top_region. Align them to page_size() boundary.
471 // |------------------new_region---------------------------------|
472 // |----bottom_region--|---intersection---|------top_region------|
473 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
474 MemRegion* bottom_region, MemRegion *top_region) {
475 // Is there bottom?
476 if (new_region.start() < intersection.start()) { // Yes
477 // Try to coalesce small pages into a large one.
478 if (UseLargePages && page_size() >= alignment()) {
479 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
480 if (new_region.contains(p)
481 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
482 if (intersection.contains(p)) {
483 intersection = MemRegion(p, intersection.end());
484 } else {
485 intersection = MemRegion(p, p);
486 }
487 }
488 }
489 *bottom_region = MemRegion(new_region.start(), intersection.start());
490 } else {
491 *bottom_region = MemRegion();
492 }
493
494 // Is there top?
495 if (intersection.end() < new_region.end()) { // Yes
496 // Try to coalesce small pages into a large one.
497 if (UseLargePages && page_size() >= alignment()) {
498 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
499 if (new_region.contains(p)
500 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
501 if (intersection.contains(p)) {
502 intersection = MemRegion(intersection.start(), p);
503 } else {
504 intersection = MemRegion(p, p);
505 }
506 }
507 }
508 *top_region = MemRegion(intersection.end(), new_region.end());
509 } else {
510 *top_region = MemRegion();
511 }
512 }
513
514 // Try to merge the invalid region with the bottom or top region by decreasing
515 // the intersection area. Return the invalid_region aligned to the page_size()
516 // boundary if it's inside the intersection. Return non-empty invalid_region
517 // if it lies inside the intersection (also page-aligned).
518 // |------------------new_region---------------------------------|
519 // |----------------|-------invalid---|--------------------------|
520 // |----bottom_region--|---intersection---|------top_region------|
521 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
522 MemRegion *invalid_region) {
523 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
524 *intersection = MemRegion(invalid_region->end(), intersection->end());
525 *invalid_region = MemRegion();
526 } else
527 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
528 *intersection = MemRegion(intersection->start(), invalid_region->start());
529 *invalid_region = MemRegion();
530 } else
531 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
532 *intersection = MemRegion(new_region.start(), new_region.start());
533 *invalid_region = MemRegion();
534 } else
535 if (intersection->contains(invalid_region)) {
536 // That's the only case we have to make an additional bias_region() call.
537 HeapWord* start = invalid_region->start();
538 HeapWord* end = invalid_region->end();
539 if (UseLargePages && page_size() >= alignment()) {
540 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
541 if (new_region.contains(p)) {
542 start = p;
543 }
544 p = (HeapWord*)round_to((intptr_t) end, alignment());
545 if (new_region.contains(end)) {
546 end = p;
547 }
548 }
549 if (intersection->start() > start) {
550 *intersection = MemRegion(start, intersection->end());
551 }
552 if (intersection->end() < end) {
553 *intersection = MemRegion(intersection->start(), end);
554 }
555 *invalid_region = MemRegion(start, end);
556 }
557 }
558
559 void MutableNUMASpace::initialize(MemRegion mr,
560 bool clear_space,
561 bool mangle_space,
562 bool setup_pages) {
563 assert(clear_space, "Reallocation will destroy data!");
564 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
565
566 MemRegion old_region = region(), new_region;
567 set_bottom(mr.start());
568 set_end(mr.end());
569 // Must always clear the space
570 clear(SpaceDecorator::DontMangle);
571
572 // Compute chunk sizes
573 size_t prev_page_size = page_size();
574 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
575 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
576 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
577 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
578
579 // Try small pages if the chunk size is too small
580 if (base_space_size_pages / lgrp_spaces()->length() == 0
581 && page_size() > (size_t)os::vm_page_size()) {
582 #ifdef LINUX
583 // If we are using pin region, we cannot change the page size to default size
584 // as we could free memory which is not expected for pin region in Linux.
585 if (UseLargePages && !os::can_commit_large_page_memory()) {
586 vm_exit_during_initialization("Failed initializing NUMA. Too small heap size");
587 }
588 #endif // LINUX
589 set_page_size(os::vm_page_size());
590 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
591 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
592 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
593 }
594 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
595 set_base_space_size(base_space_size_pages);
596
597 // Handle space resize
598 MemRegion top_region, bottom_region;
599 if (!old_region.equals(region())) {
600 new_region = MemRegion(rounded_bottom, rounded_end);
601 MemRegion intersection = new_region.intersection(old_region);
602 if (intersection.start() == NULL ||
603 intersection.end() == NULL ||
604 prev_page_size > page_size()) { // If the page size got smaller we have to change
605 // the page size preference for the whole space.
606 intersection = MemRegion(new_region.start(), new_region.start());
607 }
608 select_tails(new_region, intersection, &bottom_region, &top_region);
609 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
610 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
611 }
612
613 // Check if the space layout has changed significantly?
614 // This happens when the space has been resized so that either head or tail
615 // chunk became less than a page.
616 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
617 current_chunk_size(0) > page_size() &&
618 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
619
620
621 for (int i = 0; i < lgrp_spaces()->length(); i++) {
622 LGRPSpace *ls = lgrp_spaces()->at(i);
623 MutableSpace *s = ls->space();
624 old_region = s->region();
625
626 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
627 if (i < lgrp_spaces()->length() - 1) {
628 if (!UseAdaptiveNUMAChunkSizing ||
629 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
630 samples_count() < AdaptiveSizePolicyReadyThreshold) {
631 // No adaptation. Divide the space equally.
632 chunk_byte_size = default_chunk_size();
633 } else
634 if (!layout_valid || NUMASpaceResizeRate == 0) {
635 // Fast adaptation. If no space resize rate is set, resize
636 // the chunks instantly.
637 chunk_byte_size = adaptive_chunk_size(i, 0);
638 } else {
639 // Slow adaptation. Resize the chunks moving no more than
640 // NUMASpaceResizeRate bytes per collection.
641 size_t limit = NUMASpaceResizeRate /
642 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
643 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
644 }
645
646 assert(chunk_byte_size >= page_size(), "Chunk size too small");
647 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
648 }
649
650 if (i == 0) { // Bottom chunk
651 if (i != lgrp_spaces()->length() - 1) {
652 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
653 } else {
654 new_region = MemRegion(bottom(), end());
655 }
656 } else
657 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
658 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
659 new_region = MemRegion(ps->end(),
660 ps->end() + (chunk_byte_size >> LogHeapWordSize));
661 } else { // Top chunk
662 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
663 new_region = MemRegion(ps->end(), end());
664 }
665 guarantee(region().contains(new_region), "Region invariant");
666
667
668 // The general case:
669 // |---------------------|--invalid---|--------------------------|
670 // |------------------new_region---------------------------------|
671 // |----bottom_region--|---intersection---|------top_region------|
672 // |----old_region----|
673 // The intersection part has all pages in place we don't need to migrate them.
674 // Pages for the top and bottom part should be freed and then reallocated.
675
676 MemRegion intersection = old_region.intersection(new_region);
677
678 if (intersection.start() == NULL || intersection.end() == NULL) {
679 intersection = MemRegion(new_region.start(), new_region.start());
680 }
681
682 if (!os::numa_has_static_binding()) {
683 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
684 // Invalid region is a range of memory that could've possibly
685 // been allocated on the other node. That's relevant only on Solaris where
686 // there is no static memory binding.
687 if (!invalid_region.is_empty()) {
688 merge_regions(new_region, &intersection, &invalid_region);
689 free_region(invalid_region);
690 ls->set_invalid_region(MemRegion());
691 }
692 }
693
694 select_tails(new_region, intersection, &bottom_region, &top_region);
695
696 if (!os::numa_has_static_binding()) {
697 // If that's a system with the first-touch policy then it's enough
698 // to free the pages.
699 free_region(bottom_region);
700 free_region(top_region);
701 } else {
702 // In a system with static binding we have to change the bias whenever
703 // we reshape the heap.
704 bias_region(bottom_region, ls->lgrp_id());
705 bias_region(top_region, ls->lgrp_id());
706 }
707
708 // Clear space (set top = bottom) but never mangle.
709 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
710
711 set_adaptation_cycles(samples_count());
712 }
713 }
714
715 // Set the top of the whole space.
716 // Mark the the holes in chunks below the top() as invalid.
717 void MutableNUMASpace::set_top(HeapWord* value) {
718 bool found_top = false;
719 for (int i = 0; i < lgrp_spaces()->length();) {
720 LGRPSpace *ls = lgrp_spaces()->at(i);
721 MutableSpace *s = ls->space();
722 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
723
724 if (s->contains(value)) {
725 // Check if setting the chunk's top to a given value would create a hole less than
726 // a minimal object; assuming that's not the last chunk in which case we don't care.
727 if (i < lgrp_spaces()->length() - 1) {
728 size_t remainder = pointer_delta(s->end(), value);
729 const size_t min_fill_size = CollectedHeap::min_fill_size();
730 if (remainder < min_fill_size && remainder > 0) {
731 // Add a minimum size filler object; it will cross the chunk boundary.
732 CollectedHeap::fill_with_object(value, min_fill_size);
733 value += min_fill_size;
734 assert(!s->contains(value), "Should be in the next chunk");
735 // Restart the loop from the same chunk, since the value has moved
736 // to the next one.
737 continue;
738 }
739 }
740
741 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
742 ls->add_invalid_region(MemRegion(top, value));
743 }
744 s->set_top(value);
745 found_top = true;
746 } else {
747 if (found_top) {
748 s->set_top(s->bottom());
749 } else {
750 if (!os::numa_has_static_binding() && top < s->end()) {
751 ls->add_invalid_region(MemRegion(top, s->end()));
752 }
753 s->set_top(s->end());
754 }
755 }
756 i++;
757 }
758 MutableSpace::set_top(value);
759 }
760
761 void MutableNUMASpace::clear(bool mangle_space) {
762 MutableSpace::set_top(bottom());
763 for (int i = 0; i < lgrp_spaces()->length(); i++) {
764 // Never mangle NUMA spaces because the mangling will
765 // bind the memory to a possibly unwanted lgroup.
766 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
767 }
768 }
769
770 /*
771 Linux supports static memory binding, therefore the most part of the
772 logic dealing with the possible invalid page allocation is effectively
773 disabled. Besides there is no notion of the home node in Linux. A
774 thread is allowed to migrate freely. Although the scheduler is rather
775 reluctant to move threads between the nodes. We check for the current
776 node every allocation. And with a high probability a thread stays on
777 the same node for some time allowing local access to recently allocated
778 objects.
779 */
780
781 HeapWord* MutableNUMASpace::allocate(size_t size) {
782 Thread* thr = Thread::current();
783 int lgrp_id = thr->lgrp_id();
784 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
785 lgrp_id = os::numa_get_group_id();
786 thr->set_lgrp_id(lgrp_id);
787 }
788
789 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
790
791 // It is possible that a new CPU has been hotplugged and
792 // we haven't reshaped the space accordingly.
793 if (i == -1) {
794 i = os::random() % lgrp_spaces()->length();
795 }
796
797 LGRPSpace* ls = lgrp_spaces()->at(i);
798 MutableSpace *s = ls->space();
799 HeapWord *p = s->allocate(size);
800
801 if (p != NULL) {
802 size_t remainder = s->free_in_words();
803 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
804 s->set_top(s->top() - size);
805 p = NULL;
806 }
807 }
808 if (p != NULL) {
809 if (top() < s->top()) { // Keep _top updated.
810 MutableSpace::set_top(s->top());
811 }
812 }
813 // Make the page allocation happen here if there is no static binding..
814 if (p != NULL && !os::numa_has_static_binding()) {
815 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
816 *(int*)i = 0;
817 }
818 }
819 if (p == NULL) {
820 ls->set_allocation_failed();
821 }
822 return p;
823 }
824
825 // This version is lock-free.
826 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
827 Thread* thr = Thread::current();
828 int lgrp_id = thr->lgrp_id();
829 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
830 lgrp_id = os::numa_get_group_id();
831 thr->set_lgrp_id(lgrp_id);
832 }
833
834 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
835 // It is possible that a new CPU has been hotplugged and
836 // we haven't reshaped the space accordingly.
837 if (i == -1) {
838 i = os::random() % lgrp_spaces()->length();
839 }
840 LGRPSpace *ls = lgrp_spaces()->at(i);
841 MutableSpace *s = ls->space();
842 HeapWord *p = s->cas_allocate(size);
843 if (p != NULL) {
844 size_t remainder = pointer_delta(s->end(), p + size);
845 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
846 if (s->cas_deallocate(p, size)) {
847 // We were the last to allocate and created a fragment less than
848 // a minimal object.
849 p = NULL;
850 } else {
851 guarantee(false, "Deallocation should always succeed");
852 }
853 }
854 }
855 if (p != NULL) {
856 HeapWord* cur_top, *cur_chunk_top = p + size;
857 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
858 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
859 break;
860 }
861 }
862 }
863
864 // Make the page allocation happen here if there is no static binding.
865 if (p != NULL && !os::numa_has_static_binding() ) {
866 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
867 *(int*)i = 0;
868 }
869 }
870 if (p == NULL) {
871 ls->set_allocation_failed();
872 }
873 return p;
874 }
875
876 void MutableNUMASpace::print_short_on(outputStream* st) const {
877 MutableSpace::print_short_on(st);
878 st->print(" (");
879 for (int i = 0; i < lgrp_spaces()->length(); i++) {
880 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
881 lgrp_spaces()->at(i)->space()->print_short_on(st);
882 if (i < lgrp_spaces()->length() - 1) {
883 st->print(", ");
884 }
885 }
886 st->print(")");
887 }
888
889 void MutableNUMASpace::print_on(outputStream* st) const {
890 MutableSpace::print_on(st);
891 for (int i = 0; i < lgrp_spaces()->length(); i++) {
892 LGRPSpace *ls = lgrp_spaces()->at(i);
893 st->print(" lgrp %d", ls->lgrp_id());
894 ls->space()->print_on(st);
895 if (NUMAStats) {
896 for (int i = 0; i < lgrp_spaces()->length(); i++) {
897 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
898 }
899 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
900 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
901 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
902 ls->space_stats()->_local_space / K,
903 ls->space_stats()->_remote_space / K,
904 ls->space_stats()->_unbiased_space / K,
905 ls->space_stats()->_uncommited_space / K,
906 ls->space_stats()->_large_pages,
907 ls->space_stats()->_small_pages);
908 }
909 }
910 }
911
912 void MutableNUMASpace::verify() {
913 // This can be called after setting an arbitrary value to the space's top,
914 // so an object can cross the chunk boundary. We ensure the parsability
915 // of the space and just walk the objects in linear fashion.
916 ensure_parsability();
917 MutableSpace::verify();
918 }
919
920 // Scan pages and gather stats about page placement and size.
921 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
922 clear_space_stats();
923 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
924 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
925 if (start < end) {
926 for (char *p = start; p < end;) {
927 os::page_info info;
928 if (os::get_page_info(p, &info)) {
929 if (info.size > 0) {
930 if (info.size > (size_t)os::vm_page_size()) {
931 space_stats()->_large_pages++;
932 } else {
933 space_stats()->_small_pages++;
934 }
935 if (info.lgrp_id == lgrp_id()) {
936 space_stats()->_local_space += info.size;
937 } else {
938 space_stats()->_remote_space += info.size;
939 }
940 p += info.size;
941 } else {
942 p += os::vm_page_size();
943 space_stats()->_uncommited_space += os::vm_page_size();
944 }
945 } else {
946 return;
947 }
948 }
949 }
950 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
951 pointer_delta(space()->end(), end, sizeof(char));
952
953 }
954
955 // Scan page_count pages and verify if they have the right size and right placement.
956 // If invalid pages are found they are freed in hope that subsequent reallocation
957 // will be more successful.
958 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
959 {
960 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
961 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
962
963 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
964 set_last_page_scanned(range_start);
965 }
966
967 char *scan_start = last_page_scanned();
968 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
969
970 os::page_info page_expected, page_found;
971 page_expected.size = page_size;
972 page_expected.lgrp_id = lgrp_id();
973
974 char *s = scan_start;
975 while (s < scan_end) {
976 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
977 if (e == NULL) {
978 break;
979 }
980 if (e != scan_end) {
981 assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
982
983 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
984 && page_expected.size != 0) {
985 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
986 }
987 page_expected = page_found;
988 }
989 s = e;
990 }
991
992 set_last_page_scanned(scan_end);
993 }