--- old/src/share/vm/gc_implementation/g1/concurrentMark.hpp 2011-12-20 11:11:08.275878533 -0800 +++ new/src/share/vm/gc_implementation/g1/concurrentMark.hpp 2011-12-20 11:11:08.021102893 -0800 @@ -430,7 +430,6 @@ WorkGangBarrierSync _first_overflow_barrier_sync; WorkGangBarrierSync _second_overflow_barrier_sync; - // this is set by any task, when an overflow on the global data // structures is detected. volatile bool _has_overflown; @@ -582,6 +581,27 @@ } } + // Live Data Counting data structures... + // These data structures are initialized at the start of + // marking. They are written to while marking is active. + // They are aggregated during remark; the aggregated values + // are then used to populate the _region_bm, _card_bm, and + // the total live bytes, which are then subsequently updated + // during cleanup. + + // An array of bitmaps (one bit map per task). Each bitmap + // is used to record the cards spanned by the live objects + // marked by that task/worker. + BitMap* _count_card_bitmaps; + + // Used to record the number of marked live bytes + // (for each region, by worker thread). + size_t** _count_marked_bytes; + + // Card index of the bottom of the G1 heap. Used for biasing indices into + // the card bitmaps. + intptr_t _heap_bottom_card_num; + public: // Manipulation of the global mark stack. // Notice that the first mark_stack_push is CAS-based, whereas the @@ -703,10 +723,10 @@ // It grays an object by first marking it. Then, if it's behind the // global finger, it also pushes it on the global stack. - void deal_with_reference(oop obj); + void deal_with_reference(oop obj, int worker_i); ConcurrentMark(ReservedSpace rs, int max_regions); - ~ConcurrentMark(); + ConcurrentMarkThread* cmThread() { return _cmThread; } CMBitMapRO* prevMarkBitMap() const { return _prevMarkBitMap; } @@ -727,7 +747,8 @@ // grayed and it's MT-safe. Currently, we just mark it. But, in the // future, we can experiment with pushing it on the stack and we can // do this without changing G1CollectedHeap. - void grayRoot(oop p); + void grayRoot(oop p, int worker_i); + // It's used during evacuation pauses to gray a region, if // necessary, and it's MT-safe. It assumes that the caller has // marked any objects on that region. If _should_gray_objects is @@ -735,13 +756,14 @@ // pushed on the region stack, if it is located below the global // finger, otherwise we do nothing. void grayRegionIfNecessary(MemRegion mr); + // It's used during evacuation pauses to mark and, if necessary, // gray a single object and it's MT-safe. It assumes the caller did // not mark the object. If _should_gray_objects is true and we're // still doing concurrent marking, the objects is pushed on the // global stack, if it is located below the global finger, otherwise // we do nothing. - void markAndGrayObjectIfNecessary(oop p); + void markAndGrayObjectIfNecessary(oop p, int worker_i); // It iterates over the heap and for each object it comes across it // will dump the contents of its reference fields, as well as @@ -785,14 +807,16 @@ void checkpointRootsFinal(bool clear_all_soft_refs); void checkpointRootsFinalWork(); - void calcDesiredRegions(); void cleanup(); void completeCleanup(); // Mark in the previous bitmap. NB: this is usually read-only, so use // this carefully! void markPrev(oop p); - void clear(oop p); + + // Clears the mark in the next bitmap for the given object. + void clear_mark(oop p); + // Clears marks for all objects in the given range, for both prev and // next bitmaps. NB: the previous bitmap is usually read-only, so use // this carefully! @@ -901,6 +925,98 @@ bool verbose_high() { return _MARKING_VERBOSE_ && _verbose_level >= high_verbose; } + + // Counting data structure accessors + + // Returns the card number of the bottom of the G1 heap. + // Used in biasing indices into accounting card bitmaps. + intptr_t heap_bottom_card_num() const { + return _heap_bottom_card_num; + } + + // Returns the card bitmap for a given task or worker id. + BitMap* count_card_bitmap_for(int worker_i) { + assert(0 <= worker_i && (size_t) worker_i < _max_task_num, "oob"); + assert(_count_card_bitmaps != NULL, "uninitialized"); + BitMap* task_card_bm = &_count_card_bitmaps[worker_i]; + assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); + return task_card_bm; + } + + // Returns the array containing the marked bytes for each region, + // for the given worker or task id. + size_t* count_marked_bytes_array_for(int worker_i) { + assert(0 <= worker_i && (size_t) worker_i < _max_task_num, "oob"); + assert(_count_marked_bytes != NULL, "uninitialized"); + size_t* marked_bytes_array = _count_marked_bytes[worker_i]; + assert(marked_bytes_array != NULL, "uninitialized"); + return marked_bytes_array; + } + + // Counts the size of the given memory region in the the given + // marked_bytes array slot for the given HeapRegion. + // Sets the bits in the given card bitmap that are associated with the + // cards that are spanned by the memory region. + inline void count_region(MemRegion mr, HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm); + + // Counts the given memory region in the ask/worker counting + // data structures for the given worker id. + inline void count_region(MemRegion mr, int worker_i); + + // Counts the given object in the given task/worker counting + // data structures. + inline void count_object(oop obj, HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm); + + // Counts the given object in the task/worker counting data + // structures for the given worker id. + inline void count_object(oop obj, HeapRegion* hr, int worker_i); + + // Attempts to mark the given object and, if successful, counts + // the object in the given task/worker counting structures. + inline bool par_mark_and_count(oop obj, HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm); + + // Attempts to mark the given object and, if successful, counts + // the object in the task/worker counting structures for the + // given worker id. + inline bool par_mark_and_count(oop obj, HeapRegion* hr, int worker_i); + + // Similar to the above routine but we don't know the heap region that + // contains the object to be marked/counted, which this routine looks up. + inline bool par_mark_and_count(oop obj, int worker_i); + + // Unconditionally mark the given object, and unconditinally count + // the object in the counting structures for worker id 0. + // Should *not* be called from parallel code. + inline bool mark_and_count(oop obj, HeapRegion* hr); + + // Similar to the above routine but we don't know the heap region that + // contains the object to be marked/counted, which this routine looks up. + // Should *not* be called from parallel code. + inline bool mark_and_count(oop obj); + + // Clears the count data for the given region from _all_ of + // the per-task counting data structures. + void clear_count_data_for_heap_region(HeapRegion* hr); + +protected: + // Clear all the per-task bitmaps and arrays used to store the + // counting data. + void clear_all_count_data(); + + // Aggregates the counting data for each worker/task + // that was constructed while marking. Also sets + // the amount of marked bytes for each region and + // the top at concurrent mark count. + void aggregate_and_clear_count_data(); + + // Verification routine + void verify_count_data(); }; // A class representing a marking task. @@ -1019,6 +1135,12 @@ TruncatedSeq _marking_step_diffs_ms; + // Counting data structures. Embedding the task's marked_bytes_array + // and card bitmap into the actual task saves having to go through + // the ConcurrentMark object. + size_t* _marked_bytes_array; + BitMap* _card_bm; + // LOTS of statistics related with this task #if _MARKING_STATS_ NumberSeq _all_clock_intervals_ms; @@ -1183,6 +1305,7 @@ } CMTask(int task_num, ConcurrentMark *cm, + size_t* marked_bytes, BitMap* card_bm, CMTaskQueue* task_queue, CMTaskQueueSet* task_queues); // it prints statistics associated with this task