331 public:
332 ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
333 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
334 _preserved_marks(preserved_marks),
335 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
336 }
337
338 static bool is_candidate_region(ShenandoahHeapRegion* r) {
339 // Empty region: get it into the slice to defragment the slice itself.
340 // We could have skipped this without violating correctness, but we really
341 // want to compact all live regions to the start of the heap, which sometimes
342 // means moving them into the fully empty regions.
343 if (r->is_empty()) return true;
344
345 // Can move the region, and this is not the humongous region. Humongous
346 // moves are special cased here, because their moves are handled separately.
347 return r->is_stw_move_allowed() && !r->is_humongous();
348 }
349
350 void work(uint worker_id) {
351 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
352 ShenandoahHeapRegionSetIterator it(slice);
353 ShenandoahHeapRegion* from_region = it.next();
354 // No work?
355 if (from_region == NULL) {
356 return;
357 }
358
359 // Sliding compaction. Walk all regions in the slice, and compact them.
360 // Remember empty regions and reuse them as needed.
361 ResourceMark rm;
362
363 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
364
365 ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
366
367 while (from_region != NULL) {
368 assert(is_candidate_region(from_region), "Sanity");
369
370 cl.set_from_region(from_region);
710 _heap(ShenandoahHeap::heap()) {
711 }
712 void do_object(oop p) {
713 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
714 p->oop_iterate(&_cl);
715 }
716 };
717
718 class ShenandoahAdjustPointersTask : public AbstractGangTask {
719 private:
720 ShenandoahHeap* const _heap;
721 ShenandoahRegionIterator _regions;
722
723 public:
724 ShenandoahAdjustPointersTask() :
725 AbstractGangTask("Shenandoah Adjust Pointers Task"),
726 _heap(ShenandoahHeap::heap()) {
727 }
728
729 void work(uint worker_id) {
730 ShenandoahAdjustPointersObjectClosure obj_cl;
731 ShenandoahHeapRegion* r = _regions.next();
732 while (r != NULL) {
733 if (!r->is_humongous_continuation() && r->has_live()) {
734 _heap->marked_object_iterate(r, &obj_cl);
735 }
736 r = _regions.next();
737 }
738 }
739 };
740
741 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
742 private:
743 ShenandoahRootAdjuster* _rp;
744 PreservedMarksSet* _preserved_marks;
745 public:
746 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
747 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
748 _rp(rp),
749 _preserved_marks(preserved_marks) {}
750
751 void work(uint worker_id) {
752 ShenandoahAdjustPointersClosure cl;
753 _rp->roots_do(worker_id, &cl);
754 _preserved_marks->get(worker_id)->adjust_during_full_gc();
755 }
756 };
757
758 void ShenandoahMarkCompact::phase3_update_references() {
759 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
760 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
761
762 ShenandoahHeap* heap = ShenandoahHeap::heap();
763
764 WorkGang* workers = heap->workers();
765 uint nworkers = workers->active_workers();
766 {
767 #if COMPILER2_OR_JVMCI
768 DerivedPointerTable::clear();
769 #endif
770 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_adjust_roots);
771 ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
797 Copy::aligned_conjoint_words(compact_from, compact_to, size);
798 oop new_obj = oop(compact_to);
799 new_obj->init_mark_raw();
800 }
801 }
802 };
803
804 class ShenandoahCompactObjectsTask : public AbstractGangTask {
805 private:
806 ShenandoahHeap* const _heap;
807 ShenandoahHeapRegionSet** const _worker_slices;
808
809 public:
810 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
811 AbstractGangTask("Shenandoah Compact Objects Task"),
812 _heap(ShenandoahHeap::heap()),
813 _worker_slices(worker_slices) {
814 }
815
816 void work(uint worker_id) {
817 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
818
819 ShenandoahCompactObjectsClosure cl(worker_id);
820 ShenandoahHeapRegion* r = slice.next();
821 while (r != NULL) {
822 assert(!r->is_humongous(), "must not get humongous regions here");
823 if (r->has_live()) {
824 _heap->marked_object_iterate(r, &cl);
825 }
826 r->set_top(r->new_top());
827 r = slice.next();
828 }
829 }
830 };
831
832 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
833 private:
834 ShenandoahHeap* const _heap;
835 size_t _live;
836
943 }
944 }
945
946 // This is slightly different to ShHeap::reset_next_mark_bitmap:
947 // we need to remain able to walk pinned regions.
948 // Since pinned region do not move and don't get compacted, we will get holes with
949 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
950 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
951 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
952 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
953 class ShenandoahMCResetCompleteBitmapTask : public AbstractGangTask {
954 private:
955 ShenandoahRegionIterator _regions;
956
957 public:
958 ShenandoahMCResetCompleteBitmapTask() :
959 AbstractGangTask("Parallel Reset Bitmap Task") {
960 }
961
962 void work(uint worker_id) {
963 ShenandoahHeapRegion* region = _regions.next();
964 ShenandoahHeap* heap = ShenandoahHeap::heap();
965 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
966 while (region != NULL) {
967 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
968 ctx->clear_bitmap(region);
969 }
970 region = _regions.next();
971 }
972 }
973 };
974
975 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
976 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
977 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
978
979 ShenandoahHeap* heap = ShenandoahHeap::heap();
980
981 // Compact regular objects first
982 {
|
331 public:
332 ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
333 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
334 _preserved_marks(preserved_marks),
335 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
336 }
337
338 static bool is_candidate_region(ShenandoahHeapRegion* r) {
339 // Empty region: get it into the slice to defragment the slice itself.
340 // We could have skipped this without violating correctness, but we really
341 // want to compact all live regions to the start of the heap, which sometimes
342 // means moving them into the fully empty regions.
343 if (r->is_empty()) return true;
344
345 // Can move the region, and this is not the humongous region. Humongous
346 // moves are special cased here, because their moves are handled separately.
347 return r->is_stw_move_allowed() && !r->is_humongous();
348 }
349
350 void work(uint worker_id) {
351 ShenandoahParallelWorkerSession worker_session(worker_id);
352 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
353 ShenandoahHeapRegionSetIterator it(slice);
354 ShenandoahHeapRegion* from_region = it.next();
355 // No work?
356 if (from_region == NULL) {
357 return;
358 }
359
360 // Sliding compaction. Walk all regions in the slice, and compact them.
361 // Remember empty regions and reuse them as needed.
362 ResourceMark rm;
363
364 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
365
366 ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
367
368 while (from_region != NULL) {
369 assert(is_candidate_region(from_region), "Sanity");
370
371 cl.set_from_region(from_region);
711 _heap(ShenandoahHeap::heap()) {
712 }
713 void do_object(oop p) {
714 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
715 p->oop_iterate(&_cl);
716 }
717 };
718
719 class ShenandoahAdjustPointersTask : public AbstractGangTask {
720 private:
721 ShenandoahHeap* const _heap;
722 ShenandoahRegionIterator _regions;
723
724 public:
725 ShenandoahAdjustPointersTask() :
726 AbstractGangTask("Shenandoah Adjust Pointers Task"),
727 _heap(ShenandoahHeap::heap()) {
728 }
729
730 void work(uint worker_id) {
731 ShenandoahParallelWorkerSession worker_session(worker_id);
732 ShenandoahAdjustPointersObjectClosure obj_cl;
733 ShenandoahHeapRegion* r = _regions.next();
734 while (r != NULL) {
735 if (!r->is_humongous_continuation() && r->has_live()) {
736 _heap->marked_object_iterate(r, &obj_cl);
737 }
738 r = _regions.next();
739 }
740 }
741 };
742
743 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
744 private:
745 ShenandoahRootAdjuster* _rp;
746 PreservedMarksSet* _preserved_marks;
747 public:
748 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
749 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
750 _rp(rp),
751 _preserved_marks(preserved_marks) {}
752
753 void work(uint worker_id) {
754 ShenandoahParallelWorkerSession worker_session(worker_id);
755 ShenandoahAdjustPointersClosure cl;
756 _rp->roots_do(worker_id, &cl);
757 _preserved_marks->get(worker_id)->adjust_during_full_gc();
758 }
759 };
760
761 void ShenandoahMarkCompact::phase3_update_references() {
762 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
763 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
764
765 ShenandoahHeap* heap = ShenandoahHeap::heap();
766
767 WorkGang* workers = heap->workers();
768 uint nworkers = workers->active_workers();
769 {
770 #if COMPILER2_OR_JVMCI
771 DerivedPointerTable::clear();
772 #endif
773 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_adjust_roots);
774 ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
800 Copy::aligned_conjoint_words(compact_from, compact_to, size);
801 oop new_obj = oop(compact_to);
802 new_obj->init_mark_raw();
803 }
804 }
805 };
806
807 class ShenandoahCompactObjectsTask : public AbstractGangTask {
808 private:
809 ShenandoahHeap* const _heap;
810 ShenandoahHeapRegionSet** const _worker_slices;
811
812 public:
813 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
814 AbstractGangTask("Shenandoah Compact Objects Task"),
815 _heap(ShenandoahHeap::heap()),
816 _worker_slices(worker_slices) {
817 }
818
819 void work(uint worker_id) {
820 ShenandoahParallelWorkerSession worker_session(worker_id);
821 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
822
823 ShenandoahCompactObjectsClosure cl(worker_id);
824 ShenandoahHeapRegion* r = slice.next();
825 while (r != NULL) {
826 assert(!r->is_humongous(), "must not get humongous regions here");
827 if (r->has_live()) {
828 _heap->marked_object_iterate(r, &cl);
829 }
830 r->set_top(r->new_top());
831 r = slice.next();
832 }
833 }
834 };
835
836 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
837 private:
838 ShenandoahHeap* const _heap;
839 size_t _live;
840
947 }
948 }
949
950 // This is slightly different to ShHeap::reset_next_mark_bitmap:
951 // we need to remain able to walk pinned regions.
952 // Since pinned region do not move and don't get compacted, we will get holes with
953 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
954 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
955 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
956 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
957 class ShenandoahMCResetCompleteBitmapTask : public AbstractGangTask {
958 private:
959 ShenandoahRegionIterator _regions;
960
961 public:
962 ShenandoahMCResetCompleteBitmapTask() :
963 AbstractGangTask("Parallel Reset Bitmap Task") {
964 }
965
966 void work(uint worker_id) {
967 ShenandoahParallelWorkerSession worker_session(worker_id);
968 ShenandoahHeapRegion* region = _regions.next();
969 ShenandoahHeap* heap = ShenandoahHeap::heap();
970 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
971 while (region != NULL) {
972 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
973 ctx->clear_bitmap(region);
974 }
975 region = _regions.next();
976 }
977 }
978 };
979
980 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
981 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
982 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
983
984 ShenandoahHeap* heap = ShenandoahHeap::heap();
985
986 // Compact regular objects first
987 {
|