1
2 /*
3 * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc/parallel/mutableNUMASpace.hpp"
28 #include "gc/shared/collectedHeap.hpp"
29 #include "gc/shared/spaceDecorator.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/atomic.inline.hpp"
32 #include "runtime/thread.inline.hpp"
33
34 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
35 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
36 _page_size = os::vm_page_size();
37 _adaptation_cycles = 0;
38 _samples_count = 0;
39 update_layout(true);
40 }
41
42 MutableNUMASpace::~MutableNUMASpace() {
43 for (int i = 0; i < lgrp_spaces()->length(); i++) {
44 delete lgrp_spaces()->at(i);
45 }
46 delete lgrp_spaces();
47 }
48
49 #ifndef PRODUCT
50 void MutableNUMASpace::mangle_unused_area() {
51 // This method should do nothing.
52 // It can be called on a numa space during a full compaction.
53 }
54 void MutableNUMASpace::mangle_unused_area_complete() {
55 // This method should do nothing.
56 // It can be called on a numa space during a full compaction.
57 }
58 void MutableNUMASpace::mangle_region(MemRegion mr) {
59 // This method should do nothing because numa spaces are not mangled.
60 }
61 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
62 assert(false, "Do not mangle MutableNUMASpace's");
63 }
64 void MutableNUMASpace::set_top_for_allocations() {
65 // This method should do nothing.
66 }
67 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
68 // This method should do nothing.
69 }
70 void MutableNUMASpace::check_mangled_unused_area_complete() {
71 // This method should do nothing.
72 }
73 #endif // NOT_PRODUCT
74
75 // There may be unallocated holes in the middle chunks
76 // that should be filled with dead objects to ensure parsability.
77 void MutableNUMASpace::ensure_parsability() {
78 for (int i = 0; i < lgrp_spaces()->length(); i++) {
79 LGRPSpace *ls = lgrp_spaces()->at(i);
80 MutableSpace *s = ls->space();
81 if (s->top() < top()) { // For all spaces preceding the one containing top()
82 if (s->free_in_words() > 0) {
83 intptr_t cur_top = (intptr_t)s->top();
84 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
85 while (words_left_to_fill > 0) {
86 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
87 assert(words_to_fill >= CollectedHeap::min_fill_size(),
88 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
89 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
90 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
91 if (!os::numa_has_static_binding()) {
92 size_t touched_words = words_to_fill;
93 #ifndef ASSERT
94 if (!ZapUnusedHeapArea) {
95 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
96 touched_words);
97 }
98 #endif
99 MemRegion invalid;
100 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
101 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
102 if (crossing_start != crossing_end) {
103 // If object header crossed a small page boundary we mark the area
104 // as invalid rounding it to a page_size().
105 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
106 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
107 invalid = MemRegion(start, end);
108 }
109
110 ls->add_invalid_region(invalid);
111 }
112 cur_top = cur_top + (words_to_fill * HeapWordSize);
113 words_left_to_fill -= words_to_fill;
114 }
115 }
116 } else {
117 if (!os::numa_has_static_binding()) {
118 #ifdef ASSERT
119 MemRegion invalid(s->top(), s->end());
120 ls->add_invalid_region(invalid);
121 #else
122 if (ZapUnusedHeapArea) {
123 MemRegion invalid(s->top(), s->end());
124 ls->add_invalid_region(invalid);
125 } else {
126 return;
127 }
128 #endif
129 } else {
130 return;
131 }
132 }
133 }
134 }
135
136 size_t MutableNUMASpace::used_in_words() const {
137 size_t s = 0;
138 for (int i = 0; i < lgrp_spaces()->length(); i++) {
139 s += lgrp_spaces()->at(i)->space()->used_in_words();
140 }
141 return s;
142 }
143
144 size_t MutableNUMASpace::free_in_words() const {
145 size_t s = 0;
146 for (int i = 0; i < lgrp_spaces()->length(); i++) {
147 s += lgrp_spaces()->at(i)->space()->free_in_words();
148 }
149 return s;
150 }
151
152
153 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
154 guarantee(thr != NULL, "No thread");
155 int lgrp_id = thr->lgrp_id();
156 if (lgrp_id == -1) {
157 // This case can occur after the topology of the system has
158 // changed. Thread can change their location, the new home
159 // group will be determined during the first allocation
160 // attempt. For now we can safely assume that all spaces
161 // have equal size because the whole space will be reinitialized.
162 if (lgrp_spaces()->length() > 0) {
163 return capacity_in_bytes() / lgrp_spaces()->length();
164 } else {
165 assert(false, "There should be at least one locality group");
166 return 0;
167 }
168 }
169 // That's the normal case, where we know the locality group of the thread.
170 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
171 if (i == -1) {
172 return 0;
173 }
174 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
175 }
176
177 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
178 // Please see the comments for tlab_capacity().
179 guarantee(thr != NULL, "No thread");
180 int lgrp_id = thr->lgrp_id();
181 if (lgrp_id == -1) {
182 if (lgrp_spaces()->length() > 0) {
183 return (used_in_bytes()) / lgrp_spaces()->length();
184 } else {
185 assert(false, "There should be at least one locality group");
186 return 0;
187 }
188 }
189 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
190 if (i == -1) {
191 return 0;
192 }
193 return lgrp_spaces()->at(i)->space()->used_in_bytes();
194 }
195
196
197 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
198 // Please see the comments for tlab_capacity().
199 guarantee(thr != NULL, "No thread");
200 int lgrp_id = thr->lgrp_id();
201 if (lgrp_id == -1) {
202 if (lgrp_spaces()->length() > 0) {
203 return free_in_bytes() / lgrp_spaces()->length();
204 } else {
205 assert(false, "There should be at least one locality group");
206 return 0;
207 }
208 }
209 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
210 if (i == -1) {
211 return 0;
212 }
213 return lgrp_spaces()->at(i)->space()->free_in_bytes();
214 }
215
216
217 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
218 guarantee(thr != NULL, "No thread");
219 int lgrp_id = thr->lgrp_id();
220 if (lgrp_id == -1) {
221 if (lgrp_spaces()->length() > 0) {
222 return capacity_in_words() / lgrp_spaces()->length();
223 } else {
224 assert(false, "There should be at least one locality group");
225 return 0;
226 }
227 }
228 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
229 if (i == -1) {
230 return 0;
231 }
232 return lgrp_spaces()->at(i)->space()->capacity_in_words();
233 }
234
235 // Check if the NUMA topology has changed. Add and remove spaces if needed.
236 // The update can be forced by setting the force parameter equal to true.
237 bool MutableNUMASpace::update_layout(bool force) {
238 // Check if the topology had changed.
239 bool changed = os::numa_topology_changed();
240 if (force || changed) {
241 // Compute lgrp intersection. Add/remove spaces.
242 int lgrp_limit = (int)os::numa_get_groups_num();
243 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
244 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
245 assert(lgrp_num > 0, "There should be at least one locality group");
246 // Add new spaces for the new nodes
247 for (int i = 0; i < lgrp_num; i++) {
248 bool found = false;
249 for (int j = 0; j < lgrp_spaces()->length(); j++) {
250 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
251 found = true;
252 break;
253 }
254 }
255 if (!found) {
256 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
257 }
258 }
259
260 // Remove spaces for the removed nodes.
261 for (int i = 0; i < lgrp_spaces()->length();) {
262 bool found = false;
263 for (int j = 0; j < lgrp_num; j++) {
264 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
265 found = true;
266 break;
267 }
268 }
269 if (!found) {
270 delete lgrp_spaces()->at(i);
271 lgrp_spaces()->remove_at(i);
272 } else {
273 i++;
274 }
275 }
276
277 FREE_C_HEAP_ARRAY(int, lgrp_ids);
278
279 if (changed) {
280 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
281 thread->set_lgrp_id(-1);
282 }
283 }
284 return true;
285 }
286 return false;
287 }
288
289 // Bias region towards the first-touching lgrp. Set the right page sizes.
290 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
291 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
292 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
293 if (end > start) {
294 MemRegion aligned_region(start, end);
295 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
296 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
297 assert(region().contains(aligned_region), "Sanity");
298 // First we tell the OS which page size we want in the given range. The underlying
299 // large page can be broken down if we require small pages.
300 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
301 // Then we uncommit the pages in the range.
302 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
303 // And make them local/first-touch biased.
304 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
305 }
306 }
307
308 // Free all pages in the region.
309 void MutableNUMASpace::free_region(MemRegion mr) {
310 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
311 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
312 if (end > start) {
313 MemRegion aligned_region(start, end);
314 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
315 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
316 assert(region().contains(aligned_region), "Sanity");
317 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
318 }
319 }
320
321 // Update space layout. Perform adaptation.
322 void MutableNUMASpace::update() {
323 if (update_layout(false)) {
324 // If the topology has changed, make all chunks zero-sized.
325 // And clear the alloc-rate statistics.
326 // In future we may want to handle this more gracefully in order
327 // to avoid the reallocation of the pages as much as possible.
328 for (int i = 0; i < lgrp_spaces()->length(); i++) {
329 LGRPSpace *ls = lgrp_spaces()->at(i);
330 MutableSpace *s = ls->space();
331 s->set_end(s->bottom());
332 s->set_top(s->bottom());
333 ls->clear_alloc_rate();
334 }
335 // A NUMA space is never mangled
336 initialize(region(),
337 SpaceDecorator::Clear,
338 SpaceDecorator::DontMangle);
339 } else {
340 bool should_initialize = false;
341 if (!os::numa_has_static_binding()) {
342 for (int i = 0; i < lgrp_spaces()->length(); i++) {
343 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
344 should_initialize = true;
345 break;
346 }
347 }
348 }
349
350 if (should_initialize ||
351 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
352 // A NUMA space is never mangled
353 initialize(region(),
354 SpaceDecorator::Clear,
355 SpaceDecorator::DontMangle);
356 }
357 }
358
359 if (NUMAStats) {
360 for (int i = 0; i < lgrp_spaces()->length(); i++) {
361 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
362 }
363 }
364
365 scan_pages(NUMAPageScanRate);
366 }
367
368 // Scan pages. Free pages that have smaller size or wrong placement.
369 void MutableNUMASpace::scan_pages(size_t page_count)
370 {
371 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
372 if (pages_per_chunk > 0) {
373 for (int i = 0; i < lgrp_spaces()->length(); i++) {
374 LGRPSpace *ls = lgrp_spaces()->at(i);
375 ls->scan_pages(page_size(), pages_per_chunk);
376 }
377 }
378 }
379
380 // Accumulate statistics about the allocation rate of each lgrp.
381 void MutableNUMASpace::accumulate_statistics() {
382 if (UseAdaptiveNUMAChunkSizing) {
383 for (int i = 0; i < lgrp_spaces()->length(); i++) {
384 lgrp_spaces()->at(i)->sample();
385 }
386 increment_samples_count();
387 }
388
389 if (NUMAStats) {
390 for (int i = 0; i < lgrp_spaces()->length(); i++) {
391 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
392 }
393 }
394 }
395
396 // Get the current size of a chunk.
397 // This function computes the size of the chunk based on the
398 // difference between chunk ends. This allows it to work correctly in
399 // case the whole space is resized and during the process of adaptive
400 // chunk resizing.
401 size_t MutableNUMASpace::current_chunk_size(int i) {
402 HeapWord *cur_end, *prev_end;
403 if (i == 0) {
404 prev_end = bottom();
405 } else {
406 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
407 }
408 if (i == lgrp_spaces()->length() - 1) {
409 cur_end = end();
410 } else {
411 cur_end = lgrp_spaces()->at(i)->space()->end();
412 }
413 if (cur_end > prev_end) {
414 return pointer_delta(cur_end, prev_end, sizeof(char));
415 }
416 return 0;
417 }
418
419 // Return the default chunk size by equally diving the space.
420 // page_size() aligned.
421 size_t MutableNUMASpace::default_chunk_size() {
422 return base_space_size() / lgrp_spaces()->length() * page_size();
423 }
424
425 // Produce a new chunk size. page_size() aligned.
426 // This function is expected to be called on sequence of i's from 0 to
427 // lgrp_spaces()->length().
428 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
429 size_t pages_available = base_space_size();
430 for (int j = 0; j < i; j++) {
431 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
432 }
433 pages_available -= lgrp_spaces()->length() - i - 1;
434 assert(pages_available > 0, "No pages left");
435 float alloc_rate = 0;
436 for (int j = i; j < lgrp_spaces()->length(); j++) {
437 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
438 }
439 size_t chunk_size = 0;
440 if (alloc_rate > 0) {
441 LGRPSpace *ls = lgrp_spaces()->at(i);
442 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
443 }
444 chunk_size = MAX2(chunk_size, page_size());
445
446 if (limit > 0) {
447 limit = round_down(limit, page_size());
448 if (chunk_size > current_chunk_size(i)) {
449 size_t upper_bound = pages_available * page_size();
450 if (upper_bound > limit &&
451 current_chunk_size(i) < upper_bound - limit) {
452 // The resulting upper bound should not exceed the available
453 // amount of memory (pages_available * page_size()).
454 upper_bound = current_chunk_size(i) + limit;
455 }
456 chunk_size = MIN2(chunk_size, upper_bound);
457 } else {
458 size_t lower_bound = page_size();
459 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
460 lower_bound = current_chunk_size(i) - limit;
461 }
462 chunk_size = MAX2(chunk_size, lower_bound);
463 }
464 }
465 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
466 return chunk_size;
467 }
468
469
470 // Return the bottom_region and the top_region. Align them to page_size() boundary.
471 // |------------------new_region---------------------------------|
472 // |----bottom_region--|---intersection---|------top_region------|
473 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
474 MemRegion* bottom_region, MemRegion *top_region) {
475 // Is there bottom?
476 if (new_region.start() < intersection.start()) { // Yes
477 // Try to coalesce small pages into a large one.
478 if (UseLargePages && page_size() >= alignment()) {
479 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
480 if (new_region.contains(p)
481 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
482 if (intersection.contains(p)) {
483 intersection = MemRegion(p, intersection.end());
484 } else {
485 intersection = MemRegion(p, p);
486 }
487 }
488 }
489 *bottom_region = MemRegion(new_region.start(), intersection.start());
490 } else {
491 *bottom_region = MemRegion();
492 }
493
494 // Is there top?
495 if (intersection.end() < new_region.end()) { // Yes
496 // Try to coalesce small pages into a large one.
497 if (UseLargePages && page_size() >= alignment()) {
498 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
499 if (new_region.contains(p)
500 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
501 if (intersection.contains(p)) {
502 intersection = MemRegion(intersection.start(), p);
503 } else {
504 intersection = MemRegion(p, p);
505 }
506 }
507 }
508 *top_region = MemRegion(intersection.end(), new_region.end());
509 } else {
510 *top_region = MemRegion();
511 }
512 }
513
514 // Try to merge the invalid region with the bottom or top region by decreasing
515 // the intersection area. Return the invalid_region aligned to the page_size()
516 // boundary if it's inside the intersection. Return non-empty invalid_region
517 // if it lies inside the intersection (also page-aligned).
518 // |------------------new_region---------------------------------|
519 // |----------------|-------invalid---|--------------------------|
520 // |----bottom_region--|---intersection---|------top_region------|
521 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
522 MemRegion *invalid_region) {
523 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
524 *intersection = MemRegion(invalid_region->end(), intersection->end());
525 *invalid_region = MemRegion();
526 } else
527 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
528 *intersection = MemRegion(intersection->start(), invalid_region->start());
529 *invalid_region = MemRegion();
530 } else
531 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
532 *intersection = MemRegion(new_region.start(), new_region.start());
533 *invalid_region = MemRegion();
534 } else
535 if (intersection->contains(invalid_region)) {
536 // That's the only case we have to make an additional bias_region() call.
537 HeapWord* start = invalid_region->start();
538 HeapWord* end = invalid_region->end();
539 if (UseLargePages && page_size() >= alignment()) {
540 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
541 if (new_region.contains(p)) {
542 start = p;
543 }
544 p = (HeapWord*)round_to((intptr_t) end, alignment());
545 if (new_region.contains(end)) {
546 end = p;
547 }
548 }
549 if (intersection->start() > start) {
550 *intersection = MemRegion(start, intersection->end());
551 }
552 if (intersection->end() < end) {
553 *intersection = MemRegion(intersection->start(), end);
554 }
555 *invalid_region = MemRegion(start, end);
556 }
557 }
558
559 void MutableNUMASpace::initialize(MemRegion mr,
560 bool clear_space,
561 bool mangle_space,
562 bool setup_pages) {
563 assert(clear_space, "Reallocation will destroy data!");
564 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
565
566 MemRegion old_region = region(), new_region;
567 set_bottom(mr.start());
568 set_end(mr.end());
569 // Must always clear the space
570 clear(SpaceDecorator::DontMangle);
571
572 // Compute chunk sizes
573 size_t prev_page_size = page_size();
574 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
575 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
576 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
577 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
578
579 // Try small pages if the chunk size is too small
580 if (base_space_size_pages / lgrp_spaces()->length() == 0
581 && page_size() > (size_t)os::vm_page_size()) {
582 set_page_size(os::vm_page_size());
583 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
584 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
585 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
586 }
587 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
588 set_base_space_size(base_space_size_pages);
589
590 // Handle space resize
591 MemRegion top_region, bottom_region;
592 if (!old_region.equals(region())) {
593 new_region = MemRegion(rounded_bottom, rounded_end);
594 MemRegion intersection = new_region.intersection(old_region);
595 if (intersection.start() == NULL ||
596 intersection.end() == NULL ||
597 prev_page_size > page_size()) { // If the page size got smaller we have to change
598 // the page size preference for the whole space.
599 intersection = MemRegion(new_region.start(), new_region.start());
600 }
601 select_tails(new_region, intersection, &bottom_region, &top_region);
602 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
603 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
604 }
605
606 // Check if the space layout has changed significantly?
607 // This happens when the space has been resized so that either head or tail
608 // chunk became less than a page.
609 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
610 current_chunk_size(0) > page_size() &&
611 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
612
613
614 for (int i = 0; i < lgrp_spaces()->length(); i++) {
615 LGRPSpace *ls = lgrp_spaces()->at(i);
616 MutableSpace *s = ls->space();
617 old_region = s->region();
618
619 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
620 if (i < lgrp_spaces()->length() - 1) {
621 if (!UseAdaptiveNUMAChunkSizing ||
622 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
623 samples_count() < AdaptiveSizePolicyReadyThreshold) {
624 // No adaptation. Divide the space equally.
625 chunk_byte_size = default_chunk_size();
626 } else
627 if (!layout_valid || NUMASpaceResizeRate == 0) {
628 // Fast adaptation. If no space resize rate is set, resize
629 // the chunks instantly.
630 chunk_byte_size = adaptive_chunk_size(i, 0);
631 } else {
632 // Slow adaptation. Resize the chunks moving no more than
633 // NUMASpaceResizeRate bytes per collection.
634 size_t limit = NUMASpaceResizeRate /
635 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
636 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
637 }
638
639 assert(chunk_byte_size >= page_size(), "Chunk size too small");
640 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
641 }
642
643 if (i == 0) { // Bottom chunk
644 if (i != lgrp_spaces()->length() - 1) {
645 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
646 } else {
647 new_region = MemRegion(bottom(), end());
648 }
649 } else
650 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
651 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
652 new_region = MemRegion(ps->end(),
653 ps->end() + (chunk_byte_size >> LogHeapWordSize));
654 } else { // Top chunk
655 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
656 new_region = MemRegion(ps->end(), end());
657 }
658 guarantee(region().contains(new_region), "Region invariant");
659
660
661 // The general case:
662 // |---------------------|--invalid---|--------------------------|
663 // |------------------new_region---------------------------------|
664 // |----bottom_region--|---intersection---|------top_region------|
665 // |----old_region----|
666 // The intersection part has all pages in place we don't need to migrate them.
667 // Pages for the top and bottom part should be freed and then reallocated.
668
669 MemRegion intersection = old_region.intersection(new_region);
670
671 if (intersection.start() == NULL || intersection.end() == NULL) {
672 intersection = MemRegion(new_region.start(), new_region.start());
673 }
674
675 if (!os::numa_has_static_binding()) {
676 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
677 // Invalid region is a range of memory that could've possibly
678 // been allocated on the other node. That's relevant only on Solaris where
679 // there is no static memory binding.
680 if (!invalid_region.is_empty()) {
681 merge_regions(new_region, &intersection, &invalid_region);
682 free_region(invalid_region);
683 ls->set_invalid_region(MemRegion());
684 }
685 }
686
687 select_tails(new_region, intersection, &bottom_region, &top_region);
688
689 if (!os::numa_has_static_binding()) {
690 // If that's a system with the first-touch policy then it's enough
691 // to free the pages.
692 free_region(bottom_region);
693 free_region(top_region);
694 } else {
695 // In a system with static binding we have to change the bias whenever
696 // we reshape the heap.
697 bias_region(bottom_region, ls->lgrp_id());
698 bias_region(top_region, ls->lgrp_id());
699 }
700
701 // Clear space (set top = bottom) but never mangle.
702 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
703
704 set_adaptation_cycles(samples_count());
705 }
706 }
707
708 // Set the top of the whole space.
709 // Mark the the holes in chunks below the top() as invalid.
710 void MutableNUMASpace::set_top(HeapWord* value) {
711 bool found_top = false;
712 for (int i = 0; i < lgrp_spaces()->length();) {
713 LGRPSpace *ls = lgrp_spaces()->at(i);
714 MutableSpace *s = ls->space();
715 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
716
717 if (s->contains(value)) {
718 // Check if setting the chunk's top to a given value would create a hole less than
719 // a minimal object; assuming that's not the last chunk in which case we don't care.
720 if (i < lgrp_spaces()->length() - 1) {
721 size_t remainder = pointer_delta(s->end(), value);
722 const size_t min_fill_size = CollectedHeap::min_fill_size();
723 if (remainder < min_fill_size && remainder > 0) {
724 // Add a minimum size filler object; it will cross the chunk boundary.
725 CollectedHeap::fill_with_object(value, min_fill_size);
726 value += min_fill_size;
727 assert(!s->contains(value), "Should be in the next chunk");
728 // Restart the loop from the same chunk, since the value has moved
729 // to the next one.
730 continue;
731 }
732 }
733
734 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
735 ls->add_invalid_region(MemRegion(top, value));
736 }
737 s->set_top(value);
738 found_top = true;
739 } else {
740 if (found_top) {
741 s->set_top(s->bottom());
742 } else {
743 if (!os::numa_has_static_binding() && top < s->end()) {
744 ls->add_invalid_region(MemRegion(top, s->end()));
745 }
746 s->set_top(s->end());
747 }
748 }
749 i++;
750 }
751 MutableSpace::set_top(value);
752 }
753
754 void MutableNUMASpace::clear(bool mangle_space) {
755 MutableSpace::set_top(bottom());
756 for (int i = 0; i < lgrp_spaces()->length(); i++) {
757 // Never mangle NUMA spaces because the mangling will
758 // bind the memory to a possibly unwanted lgroup.
759 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
760 }
761 }
762
763 /*
764 Linux supports static memory binding, therefore the most part of the
765 logic dealing with the possible invalid page allocation is effectively
766 disabled. Besides there is no notion of the home node in Linux. A
767 thread is allowed to migrate freely. Although the scheduler is rather
768 reluctant to move threads between the nodes. We check for the current
769 node every allocation. And with a high probability a thread stays on
770 the same node for some time allowing local access to recently allocated
771 objects.
772 */
773
774 HeapWord* MutableNUMASpace::allocate(size_t size) {
775 Thread* thr = Thread::current();
776 int lgrp_id = thr->lgrp_id();
777 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
778 lgrp_id = os::numa_get_group_id();
779 thr->set_lgrp_id(lgrp_id);
780 }
781
782 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
783
784 // It is possible that a new CPU has been hotplugged and
785 // we haven't reshaped the space accordingly.
786 if (i == -1) {
787 i = os::random() % lgrp_spaces()->length();
788 }
789
790 LGRPSpace* ls = lgrp_spaces()->at(i);
791 MutableSpace *s = ls->space();
792 HeapWord *p = s->allocate(size);
793
794 if (p != NULL) {
795 size_t remainder = s->free_in_words();
796 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
797 s->set_top(s->top() - size);
798 p = NULL;
799 }
800 }
801 if (p != NULL) {
802 if (top() < s->top()) { // Keep _top updated.
803 MutableSpace::set_top(s->top());
804 }
805 }
806 // Make the page allocation happen here if there is no static binding..
807 if (p != NULL && !os::numa_has_static_binding()) {
808 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
809 *(int*)i = 0;
810 }
811 }
812 if (p == NULL) {
813 ls->set_allocation_failed();
814 }
815 return p;
816 }
817
818 // This version is lock-free.
819 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
820 Thread* thr = Thread::current();
821 int lgrp_id = thr->lgrp_id();
822 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
823 lgrp_id = os::numa_get_group_id();
824 thr->set_lgrp_id(lgrp_id);
825 }
826
827 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
828 // It is possible that a new CPU has been hotplugged and
829 // we haven't reshaped the space accordingly.
830 if (i == -1) {
831 i = os::random() % lgrp_spaces()->length();
832 }
833 LGRPSpace *ls = lgrp_spaces()->at(i);
834 MutableSpace *s = ls->space();
835 HeapWord *p = s->cas_allocate(size);
836 if (p != NULL) {
837 size_t remainder = pointer_delta(s->end(), p + size);
838 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
839 if (s->cas_deallocate(p, size)) {
840 // We were the last to allocate and created a fragment less than
841 // a minimal object.
842 p = NULL;
843 } else {
844 guarantee(false, "Deallocation should always succeed");
845 }
846 }
847 }
848 if (p != NULL) {
849 HeapWord* cur_top, *cur_chunk_top = p + size;
850 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
851 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
852 break;
853 }
854 }
855 }
856
857 // Make the page allocation happen here if there is no static binding.
858 if (p != NULL && !os::numa_has_static_binding() ) {
859 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
860 *(int*)i = 0;
861 }
862 }
863 if (p == NULL) {
864 ls->set_allocation_failed();
865 }
866 return p;
867 }
868
869 void MutableNUMASpace::print_short_on(outputStream* st) const {
870 MutableSpace::print_short_on(st);
871 st->print(" (");
872 for (int i = 0; i < lgrp_spaces()->length(); i++) {
873 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
874 lgrp_spaces()->at(i)->space()->print_short_on(st);
875 if (i < lgrp_spaces()->length() - 1) {
876 st->print(", ");
877 }
878 }
879 st->print(")");
880 }
881
882 void MutableNUMASpace::print_on(outputStream* st) const {
883 MutableSpace::print_on(st);
884 for (int i = 0; i < lgrp_spaces()->length(); i++) {
885 LGRPSpace *ls = lgrp_spaces()->at(i);
886 st->print(" lgrp %d", ls->lgrp_id());
887 ls->space()->print_on(st);
888 if (NUMAStats) {
889 for (int i = 0; i < lgrp_spaces()->length(); i++) {
890 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
891 }
892 st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
893 SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
894 "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
895 ls->space_stats()->_local_space / K,
896 ls->space_stats()->_remote_space / K,
897 ls->space_stats()->_unbiased_space / K,
898 ls->space_stats()->_uncommited_space / K,
899 ls->space_stats()->_large_pages,
900 ls->space_stats()->_small_pages);
901 }
902 }
903 }
904
905 void MutableNUMASpace::verify() {
906 // This can be called after setting an arbitrary value to the space's top,
907 // so an object can cross the chunk boundary. We ensure the parsability
908 // of the space and just walk the objects in linear fashion.
909 ensure_parsability();
910 MutableSpace::verify();
911 }
912
913 // Scan pages and gather stats about page placement and size.
914 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
915 clear_space_stats();
916 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
917 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
918 if (start < end) {
919 for (char *p = start; p < end;) {
920 os::page_info info;
921 if (os::get_page_info(p, &info)) {
922 if (info.size > 0) {
923 if (info.size > (size_t)os::vm_page_size()) {
924 space_stats()->_large_pages++;
925 } else {
926 space_stats()->_small_pages++;
927 }
928 if (info.lgrp_id == lgrp_id()) {
929 space_stats()->_local_space += info.size;
930 } else {
931 space_stats()->_remote_space += info.size;
932 }
933 p += info.size;
934 } else {
935 p += os::vm_page_size();
936 space_stats()->_uncommited_space += os::vm_page_size();
937 }
938 } else {
939 return;
940 }
941 }
942 }
943 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
944 pointer_delta(space()->end(), end, sizeof(char));
945
946 }
947
948 // Scan page_count pages and verify if they have the right size and right placement.
949 // If invalid pages are found they are freed in hope that subsequent reallocation
950 // will be more successful.
951 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
952 {
953 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
954 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
955
956 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
957 set_last_page_scanned(range_start);
958 }
959
960 char *scan_start = last_page_scanned();
961 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
962
963 os::page_info page_expected, page_found;
964 page_expected.size = page_size;
965 page_expected.lgrp_id = lgrp_id();
966
967 char *s = scan_start;
968 while (s < scan_end) {
969 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
970 if (e == NULL) {
971 break;
972 }
973 if (e != scan_end) {
974 assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
975
976 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
977 && page_expected.size != 0) {
978 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
979 }
980 page_expected = page_found;
981 }
982 s = e;
983 }
984
985 set_last_page_scanned(scan_end);
986 }
--- EOF ---